diff --git a/models/emotion.joblib b/models/emotion.joblib new file mode 100644 index 0000000..9e22980 Binary files /dev/null and b/models/emotion.joblib differ diff --git a/models/shape_predictor_68_face_landmarks.dat b/models/shape_predictor_68_face_landmarks.dat new file mode 100644 index 0000000..e0ec20d Binary files /dev/null and b/models/shape_predictor_68_face_landmarks.dat differ diff --git a/src/face_recognition.py b/src/face_recognition.py new file mode 100644 index 0000000..cc1059c --- /dev/null +++ b/src/face_recognition.py @@ -0,0 +1,64 @@ +# In order to get the key points we need to find the face in the image +# for that we can use opencv's built in har cascade or the pretrained model in dlib +# after that we need to use a facial landmarks detector for that we can also use dlib +# get more info here https://pyimagesearch.com/2017/04/03/facial-landmarks-dlib-opencv-python/ +# how to download dlib https://pyimagesearch.com/2017/03/27/how-to-install-dlib/ and https://github.com/davisking/dlib +# the facial landmark model used can be find on https://dlib.net/files/ +# I also found another way to do it using MediaPipe Face Mesh +# the missing methods can be found here https://github.com/PyImageSearch/imutils +# emotion model repo https://github.com/niebardzo/Emotions + +import numpy as np +import dlib +import cv2 +import pyzed.sl as sl +from camera import Camera +from joblib import load +from utils.image_processing import Face + +def convert_dlib_BB_to_openCV_BB(rect): + x = rect.left() + y = rect.top() + w = rect.right() - rect.left() + h = rect.bottom() - rect.top() + return (x, y, w, h) + +if (__name__ == "__main__"): + #load the pretrained face detector from dlib + detector = dlib.get_frontal_face_detector() + #load the facial landmark predictor (97 358 KB) + predictor = dlib.shape_predictor("../models/shape_predictor_68_face_landmarks.dat") + #load the emotion model + emotion_model = load('../models/emotion.joblib') + + myCamera = Camera() + image = None + # grab a frame + with myCamera: + if (myCamera.grab() == sl.ERROR_CODE.SUCCESS): + image = myCamera.get_frame() + else: + print("brooo the camera doesn't work :(") + + # resizing the image can positvely impact the computing time + #convert the image to grayscale + grayScale_image = cv2.cvtColor(image, cv2.COLOR_BGRA2RGB) + # upscale the image and get BB of the face can also work with RGB image + face_BB = detector(grayScale_image, 1) + + for BB in face_BB: + face = Face(grayScale_image, BB, predictor) + prediction = emotion_model.predict([face.extract_features()]) + #get the facial landmarks coordinates (x,y) + #convert to openCv BB + (x, y, w, h) = convert_dlib_BB_to_openCV_BB(BB) + #draw the BB + cv2.rectangle(image, (x,y), (x+w, y+h), (0,255,0),2) + cv2.putText(image, "###{}".format(emotion_model.le.inverse_transform(prediction)[0]), (x - 10, y - 10),cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2) + #show the result + cv2.imshow("Frame", image) + + while 1: + if cv2.waitKey(1) & 0xFF == ord('q'): + break + cv2.destroyAllWindows() \ No newline at end of file diff --git a/src/utils/__init__.py b/src/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/utils/image_processing.py b/src/utils/image_processing.py new file mode 100644 index 0000000..7ad659d --- /dev/null +++ b/src/utils/image_processing.py @@ -0,0 +1,168 @@ +from scipy.spatial import distance as dist +from imutils import face_utils +import numpy as np +import imutils +import cv2 +import math + + +def eye_aspect_ratio(eye): + """Method returns Eye Aspect Ratio.""" + A = dist.euclidean(eye[1], eye[5]) + B = dist.euclidean(eye[2], eye[4]) + C = dist.euclidean(eye[0],eye[3]) + ear = (A+B)/(2.0*C) + return ear + + +def mouth_aspect_ratio(mouth): + """Method returns Mouth Aspect Ratio.""" + A = dist.euclidean(mouth[13], mouth[19]) + B = dist.euclidean(mouth[14], mouth[18]) + C = dist.euclidean(mouth[15], mouth[17]) + F = dist.euclidean(mouth[12], mouth[16]) + mar = (A+B+C)/(3.0*F) + return mar + + +class Image(object): + """ + A class used to represent the Image provided. + + Attributes: + ----------- + image: object. + Opencv object representing the loaded image. + + gray: object. + Opencv object representing the loaded image in gray scale. + + detector: object + Dlib frontal face detector object. + + """ + + def __init__(self, image, detector): + """Consutructor of the class that handles images.""" + self.image = imutils.resize(image, width=562) + self.gray = self.generate_gray() + self.detector = detector + + def detect_faces(self): + """Method that returns faces detected on the image itself.""" + rects = self.detector(self.gray, 1) + return rects + + def generate_gray(self): + """Image that generate the grayscale object opencv.""" + return cv2.cvtColor(self.image, cv2.COLOR_BGR2GRAY) + + + +class Face(object): + """ + A class used to represent the Face. + + Attributes: + ----------- + predictor: object + A landmark predictor used to get the face landmark. + shape: array + Array represents face landmark. + face_parts: arrays + Subarrays of shape describing face parts. + gravity_point: array + X,Y array represents gravity point of the face. + normalizer: float + Value to normalize features. + features: array + Array with all face features. + + """ + + def __init__(self, gray, rect, predictor): + """Constructor of the class describing face. Setting up all necassary attributes.""" + self.predictor = predictor + self.shape = self.get_landmark(gray, rect) + self.left_eye = self.extract_part("left_eye") + self.right_eye = self.extract_part("right_eye") + self.left_eyebrow = self.extract_part("left_eyebrow") + self.right_eyebrow = self.extract_part("right_eyebrow") + self.mouth = self.extract_part("mouth") + + self.gravity_point = self.calculate_face_gravity_center() + self.normalizer = self.calculate_normalizer() + + self.features = [] + + def get_landmark(self, gray, rect): + """Method returns face landmark.""" + shape = self.predictor(gray, rect) + shape = face_utils.shape_to_np(shape) + return shape + + + def calculate_face_gravity_center(self): + """Method returns the face gravity center.""" + return self.shape.mean(axis=0).astype("int") + + def extract_part(self, part): + """Method returns the subarray of face landmark.""" + (Start, End) = face_utils.FACIAL_LANDMARKS_IDXS[part] + return self.shape[Start:End] + + def calculate_normalizer(self): + """Method returns face normalizer.""" + left_eye_center = self.left_eye.mean(axis=0).astype("int") + right_eye_center = self.right_eye.mean(axis=0).astype("int") + A = dist.euclidean(left_eye_center, self.gravity_point) + B = dist.euclidean(right_eye_center, self.gravity_point) + return (A+B)/2.0 + + def get_eyes_features(self): + """Method that appends to the features attribute all eyes features.""" + left_eye_center = self.left_eye.mean(axis=0).astype("int") + + left_1 = dist.euclidean(self.left_eyebrow[0], left_eye_center)/self.normalizer + left_2 = dist.euclidean(self.left_eyebrow[2], left_eye_center)/self.normalizer + left_3 = dist.euclidean(self.left_eyebrow[4], left_eye_center)/self.normalizer + + self.features.append(eye_aspect_ratio(self.left_eye)) + self.features.append(left_1) + self.features.append(left_2) + self.features.append(left_3) + + right_eye_center = self.right_eye.mean(axis=0).astype("int") + + right_3 = dist.euclidean(self.right_eyebrow[0], right_eye_center)/self.normalizer + right_2 = dist.euclidean(self.right_eyebrow[2], right_eye_center)/self.normalizer + right_1 = dist.euclidean(self.right_eyebrow[4], right_eye_center)/self.normalizer + + self.features.append(eye_aspect_ratio(self.right_eye)) + self.features.append(right_1) + self.features.append(right_2) + self.features.append(right_3) + + def get_mouth_features(self): + """Method that appends to the features attributes all mouth features.""" + self.features.append(mouth_aspect_ratio(self.mouth)) + + mouth_1 = dist.euclidean(self.mouth[3], self.gravity_point)/self.normalizer + mouth_2 = dist.euclidean(self.mouth[9], self.gravity_point)/self.normalizer + mouth_3 = dist.euclidean(self.mouth[6], self.gravity_point)/self.normalizer + mouth_4 = dist.euclidean(self.mouth[0], self.gravity_point)/self.normalizer + + self.features.append(mouth_1) + self.features.append(mouth_2) + self.features.append(mouth_3) + self.features.append(mouth_4) + + + def extract_features(self): + """Method returns the features of the face.""" + self.get_eyes_features() + self.get_mouth_features() + return np.array(self.features) + + + diff --git a/src/utils/modelmanager.py b/src/utils/modelmanager.py new file mode 100644 index 0000000..932823d --- /dev/null +++ b/src/utils/modelmanager.py @@ -0,0 +1,123 @@ +from sklearn.neighbors import KNeighborsClassifier +from sklearn.naive_bayes import GaussianNB +from sklearn.svm import SVC +from sklearn.tree import DecisionTreeClassifier +from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier +from sklearn.ensemble import VotingClassifier, GradientBoostingClassifier + +from sklearn.neural_network import MLPClassifier + +from sklearn.preprocessing import LabelEncoder +from sklearn.model_selection import train_test_split +from sklearn.metrics import classification_report +from sklearn.pipeline import Pipeline +from sklearn.feature_selection import SelectKBest, SelectPercentile, SelectFpr +from sklearn.feature_selection import chi2, f_classif, mutual_info_classif +from sklearn.feature_selection import RFE + + + +class Model(object): + """ + Class that representats the model. + + Attributes: + le: object + Object of Label Encoder + model: object + Object of sklearn model of choice. + training_data: array + Array of data. + training_labels: array + Encoded labels for training data. + test_data: array + Array of test data. Initially empty. + test_labels: array + Array of test labels. Initially empty. + + """ + + def __init__(self, model, data=None, labels=None): + """Constructor for model class.""" + if data is None or labels is None: + raise AttributeError("No Data in a constructor provided.") + + + self.models = { + "knn": KNeighborsClassifier(n_neighbors=9, algorithm="brute", weights="distance"), + "naive_bayes": GaussianNB(), + "svm": SVC(C=15.6, gamma="scale", kernel="rbf"), + "decision_tree": DecisionTreeClassifier(criterion="entropy", max_depth=55, splitter="best"), + "random_forest": RandomForestClassifier(n_estimators=50, criterion="entropy"), + "extra_tree": ExtraTreesClassifier(n_estimators=122, criterion="entropy"), + "gradient_boost": GradientBoostingClassifier(n_estimators=33, learning_rate=0.14), + "mlp": MLPClassifier(solver="lbfgs", hidden_layer_sizes=(13, 12), alpha=5E-06) + + } + + self.le = LabelEncoder() + self.model = self.models[model] + + self.training_data = data + self.training_labels = self.le.fit_transform(labels) + self.feature_names = ['EARL','L1','L2','L3', 'EARR', 'R1', 'R2', 'R3', 'MAR', 'M1', 'M2', 'M3', 'M4'] + self.feature_mask = [True,True,True,True,True,True,True,True,True,True,True,True,True] + + + def use_voting_classifier(self): + """Method for changing to VotingClassifier.""" + self.model = VotingClassifier(estimators=[('nb', self.models["naive_bayes"]), ('et', self.models["extra_tree"]), ('gb', self.models["gradient_boost"])], voting='hard', weights=[2,3,1.5]) + + def split_dataset(self, test_size=0.20): + """Method for spliting dataset to the training and test.""" + (self.training_data, self.test_data, self.training_labels, self.test_labels) = train_test_split(self.training_data, self.training_labels, test_size=test_size) + + def train(self): + """Method for training a model with the training dataset.""" + self.model.fit(self.training_data, self.training_labels) + + def test(self): + """Method returns the classification report.""" + return classification_report(self.test_labels, self.predict(self.test_data), target_names=self.le.classes_) + + def predict(self, to_predict): + """Method returns the prefiction for new data.""" + return self.model.predict(to_predict) + + def univariate_feature_selection(self, method, scoring, number): + """Method that creates the pipeline for only important feature extraction with univariate_feature_selection.""" + self.scoring_functions = { + "f_classif": f_classif, + "mutual_info_classif": mutual_info_classif, + "chi2": chi2 + } + + self.selection_methods = { + "select_k_best": SelectKBest(self.scoring_functions[scoring], k=number), + "select_percentile": SelectPercentile(self.scoring_functions[scoring], percentile=number) + } + + + self.model = Pipeline([ + ('feature_selection', self.selection_methods[method]), + ('classification', self.model) + ]) + + + + def recursive_feature_elimination(self): + """Method that creates the pipeline for only important feature extraction with RFE method.""" + svc = SVC(kernel="linear") + self.model = Pipeline([ + ('feature_selection', RFE(estimator=svc, n_features_to_select=8, step=10)), + ('classification', self.model) + ]) + + def get_feature_labels(self): + """Method for retriving feature lables from the model.""" + feature_labels = [] + for feature, i in zip(self.feature_names,self.feature_mask): + if i == True: + feature_labels.append(feature) + return feature_labels + \ No newline at end of file