Views
No views yet
@article{cheong2023py,
title={Py-feat: Python facial expression analysis toolbox},
author={Cheong, Jin Hyun and Jolly, Eshin and Xie, Tiankang and Byrne, Sophie and Kenney, Matthew and Chang, Luke J},
journal={Affective Science},
volume={4},
number={4},
pages={781--796},
year={2023},
publisher={Springer}
}1import numpy as np
2from skops.io import dump, load, get_untrusted_types
3from huggingface_hub import hf_hub_download
4
5class XGBClassifier:
6 def __init__(self) -> None:
7
8 self.au_keys = [
9 "AU1", "AU2", "AU4", "AU5", "AU6", "AU7", "AU9", "AU10", "AU11", "AU12",
10 "AU14", "AU15", "AU17", "AU20", "AU23", "AU24", "AU25", "AU26", "AU28", "AU43"
11 ]
12 self.weights_loaded = False
13
14 def load_weights(self, scaler_upper=None, pca_model_upper=None, scaler_lower=None, pca_model_lower=None, scaler_full=None, pca_model_full=None, classifiers=None):
15 self.scaler_upper = scaler_upper
16 self.pca_model_upper = pca_model_upper
17 self.scaler_lower = scaler_lower
18 self.pca_model_lower = pca_model_lower
19 self.scaler_full = scaler_full
20 self.pca_model_full = pca_model_full
21 self.classifiers = classifiers
22 self.weights_loaded = True
23
24 def pca_transform(self, frame, scaler, pca_model, landmarks):
25 if not self.weights_loaded:
26 raise ValueError('Need to load weights before running pca_transform')
27 else:
28 transformed_frame = pca_model.transform(scaler.transform(frame))
29 return np.concatenate((transformed_frame, landmarks), axis=1)
30
31 def detect_au(self, frame, landmarks):
32 if not self.weights_loaded:
33 raise ValueError('Need to load weights before running detect_au')
34 else:
35 landmarks = np.concatenate(landmarks)
36 landmarks = landmarks.reshape(-1, landmarks.shape[1] * landmarks.shape[2])
37
38 pca_transformed_upper = self.pca_transform(frame, self.scaler_upper, self.pca_model_upper, landmarks)
39 pca_transformed_lower = self.pca_transform(frame, self.scaler_lower, self.pca_model_lower, landmarks)
40 pca_transformed_full = self.pca_transform(frame, self.scaler_full, self.pca_model_full, landmarks)
41
42 pred_aus = []
43 for key in self.au_keys:
44 classifier = self.classifiers[key]
45
46 if key in ["AU1", "AU2", "AU7"]:
47 au_pred = classifier.predict_proba(pca_transformed_upper)[:, 1]
48 elif key in ["AU11", "AU14", "AU17", "AU23", "AU24", "AU26"]:
49 au_pred = classifier.predict_proba(pca_transformed_lower)[:, 1]
50 else:
51 au_pred = classifier.predict_proba(pca_transformed_full)[:, 1]
52
53 pred_aus.append(au_pred)
54
55 return np.array(pred_aus).T def __init__(self) -> None:
56 self.weights_loaded = False
57
58 def load_weights(self, scaler_upper=None, pca_model_upper=None, scaler_lower=None, pca_model_lower=None, scaler_full=None, pca_model_full=None, classifiers=None):
59 self.scaler_upper = scaler_upper
60 self.pca_model_upper = pca_model_upper
61 self.scaler_lower = scaler_lower
62 self.pca_model_lower = pca_model_lower
63 self.scaler_full = scaler_full
64 self.pca_model_full = pca_model_full
65 self.classifiers = classifiers
66 self.weights_loaded = True
67
68 def pca_transform(self, frame, scaler, pca_model, landmarks):
69 if not self.weights_loaded:
70 raise ValueError('Need to load weights before running pca_transform')
71 else:
72 transformed_frame = pca_model.transform(scaler.transform(frame))
73 return np.concatenate((transformed_frame, landmarks), axis=1)
74
75 def detect_au(self, frame, landmarks):
76 """
77 Note that here frame is represented by hogs
78 """
79 if not self.weights_loaded:
80 raise ValueError('Need to load weights before running detect_au')
81 else:
82 landmarks = np.concatenate(landmarks)
83 landmarks = landmarks.reshape(-1, landmarks.shape[1] * landmarks.shape[2])
84
85 pca_transformed_upper = self.pca_transform(frame, self.scaler_upper, self.pca_model_upper, landmarks)
86 pca_transformed_lower = self.pca_transform(frame, self.scaler_lower, self.pca_model_lower, landmarks)
87 pca_transformed_full = self.pca_transform(frame, self.scaler_full, self.pca_model_full, landmarks)
88
89 aus_list = sorted(self.classifiers.keys(), key=lambda x: int(x[2::]))
90
91 pred_aus = []
92 for keys in aus_list:
93 if keys in ["AU1", "AU4", "AU6"]:
94 au_pred = self.classifiers[keys].predict(pca_transformed_upper)
95 elif keys in ["AU11", "AU12", "AU17"]:
96 au_pred = self.classifiers[keys].predict(pca_transformed_lower)
97 elif keys in [
98 "AU2",
99 "AU5",
100 "AU7",
101 "AU9",
102 "AU10",
103 "AU14",
104 "AU15",
105 "AU20",
106 "AU23",
107 "AU24",
108 "AU25",
109 "AU26",
110 "AU28",
111 "AU43",
112 ]:
113 au_pred = self.classifiers[keys].predict(pca_transformed_full)
114 else:
115 raise ValueError("unknown AU detected")
116
117 pred_aus.append(au_pred)
118 pred_aus = np.array(pred_aus).T
119 return pred_aus
120
121# Load model and weights
122au_model = XGBClassifier()
123model_path = hf_hub_download(repo_id="py-feat/xgb_au", filename="xgb_au_classifier.skops")
124unknown_types = get_untrusted_types(file=model_path)
125loaded_model = load(model_path, trusted=unknown_types)
126au_model.load_weights(scaler_upper = loaded_model.scaler_upper,
127 pca_model_upper = loaded_model.pca_model_upper,
128 scaler_lower = loaded_model.scaler_lower,
129 pca_model_lower = loaded_model.scaler_full,
130 pca_model_full=loaded_model.pca_model_full,
131 classifiers=loaded_model.classifiers)
132
133# Test model
134frame = "path/to/your/test_image.jpg" # Replace with your loaded image
135landmarks = np.array([...]) # Replace with your landmarks data
136pred = au_model.detect_au(frame, landmarks)
137print(pred)
138