Feature/face recognition

src/face_recognition.py

@@ -0,0 +1,64 @@
+# In order to get the key points we need to find the face in the image
+# for that we can use opencv's built in har cascade or the pretrained model in dlib
+# after that we need to use a facial landmarks detector for that we can also use dlib
+# get more info here https://pyimagesearch.com/2017/04/03/facial-landmarks-dlib-opencv-python/
+# how to download dlib https://pyimagesearch.com/2017/03/27/how-to-install-dlib/ and https://github.com/davisking/dlib
+# the facial landmark model used can be find on https://dlib.net/files/
+# I also found another way to do it using MediaPipe Face Mesh
+# the missing methods can be found here https://github.com/PyImageSearch/imutils
+# emotion model repo https://github.com/niebardzo/Emotions
+
+import numpy as np
+import dlib
+import cv2
+import pyzed.sl as sl
+from camera import Camera
+from joblib import load
+from utils.image_processing import Face
+
+def convert_dlib_BB_to_openCV_BB(rect):
+    x = rect.left()
+    y = rect.top()
+    w = rect.right() - rect.left()
+    h = rect.bottom() - rect.top()
+    return (x, y, w, h)
+
+if (__name__ == "__main__"):
+    #load the pretrained face detector from dlib
+    detector = dlib.get_frontal_face_detector()
+    #load the facial landmark predictor (97 358 KB)
+    predictor = dlib.shape_predictor("../models/shape_predictor_68_face_landmarks.dat")
+    #load the emotion model
+    emotion_model = load('../models/emotion.joblib')
+
+    myCamera = Camera()
+    image = None
+    # grab a frame
+    with myCamera:
+        if (myCamera.grab() == sl.ERROR_CODE.SUCCESS):
+            image = myCamera.get_frame()
+        else:
+            print("brooo the camera doesn't work :(")
+
+    # resizing the image can positvely impact the computing time
+    #convert the image to grayscale
+    grayScale_image = cv2.cvtColor(image, cv2.COLOR_BGRA2RGB)
+    # upscale the image and get BB of the face can also work with RGB image
+    face_BB = detector(grayScale_image, 1)
+
+    for BB in face_BB:
+        face = Face(grayScale_image, BB, predictor)
+        prediction = emotion_model.predict([face.extract_features()])
+        #get the facial landmarks coordinates (x,y)
+        #convert to openCv BB
+        (x, y, w, h) = convert_dlib_BB_to_openCV_BB(BB)
+        #draw the BB
+        cv2.rectangle(image, (x,y), (x+w, y+h), (0,255,0),2)
+        cv2.putText(image, "###{}".format(emotion_model.le.inverse_transform(prediction)[0]), (x - 10, y - 10),cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
+    #show the result
+    cv2.imshow("Frame", image)
+
+    while 1:
+        if cv2.waitKey(1) & 0xFF == ord('q'):
+            break
+    cv2.destroyAllWindows()

src/utils/image_processing.py

@@ -0,0 +1,168 @@
+from scipy.spatial import distance as dist
+from imutils import face_utils
+import numpy as np
+import imutils
+import cv2
+import math
+
+
+def eye_aspect_ratio(eye):
+	"""Method returns Eye Aspect Ratio."""
+	A = dist.euclidean(eye[1], eye[5])
+	B = dist.euclidean(eye[2], eye[4])
+	C = dist.euclidean(eye[0],eye[3])
+	ear = (A+B)/(2.0*C)
+	return ear
+
+
+def mouth_aspect_ratio(mouth):
+	"""Method returns Mouth Aspect Ratio."""
+	A = dist.euclidean(mouth[13], mouth[19])
+	B = dist.euclidean(mouth[14], mouth[18])
+	C = dist.euclidean(mouth[15], mouth[17])
+	F = dist.euclidean(mouth[12], mouth[16])
+	mar = (A+B+C)/(3.0*F)
+	return mar
+
+
+class Image(object):
+	"""
+	A class used to represent the Image provided.
+
+	Attributes:
+	-----------
+	image: object.
+		Opencv object representing the loaded image.
+
+	gray: object.
+		Opencv object representing the loaded image in gray scale.
+
+	detector: object
+		Dlib frontal face detector object.
+
+	"""
+
+	def __init__(self, image, detector):
+		"""Consutructor of the class that handles images."""
+		self.image = imutils.resize(image, width=562)
+		self.gray = self.generate_gray()
+		self.detector = detector
+
+	def detect_faces(self):
+		"""Method that returns faces detected on the image itself."""
+		rects = self.detector(self.gray, 1)
+		return rects
+
+	def generate_gray(self):
+		"""Image that generate the grayscale object opencv."""
+		return cv2.cvtColor(self.image, cv2.COLOR_BGR2GRAY)
+
+
+
+class Face(object):
+	"""
+	A class used to represent the Face.
+
+	Attributes:
+	-----------
+	predictor: object
+		A landmark predictor used to get the face landmark.
+	shape: array
+		Array represents face landmark.
+	face_parts: arrays
+		Subarrays of shape describing face parts.
+	gravity_point: array
+		X,Y array represents gravity point of the face.
+	normalizer: float
+		Value to normalize features.
+	features: array
+		Array with all face features.
+
+	"""
+
+	def __init__(self, gray, rect, predictor):
+		"""Constructor of the class describing face. Setting up all necassary attributes."""
+		self.predictor = predictor
+		self.shape = self.get_landmark(gray, rect)
+		self.left_eye = self.extract_part("left_eye")
+		self.right_eye = self.extract_part("right_eye")
+		self.left_eyebrow = self.extract_part("left_eyebrow")
+		self.right_eyebrow = self.extract_part("right_eyebrow")
+		self.mouth = self.extract_part("mouth")
+
+		self.gravity_point = self.calculate_face_gravity_center()
+		self.normalizer = self.calculate_normalizer()
+
+		self.features = []
+
+	def get_landmark(self, gray, rect):
+		"""Method returns face landmark."""
+		shape = self.predictor(gray, rect)
+		shape = face_utils.shape_to_np(shape)
+		return shape
+
+
+	def calculate_face_gravity_center(self):
+		"""Method returns the face gravity center."""
+		return self.shape.mean(axis=0).astype("int")
+
+	def extract_part(self, part):
+		"""Method returns the subarray of face landmark."""
+		(Start, End) = face_utils.FACIAL_LANDMARKS_IDXS[part]
+		return self.shape[Start:End]
+
+	def calculate_normalizer(self):
+		"""Method returns face normalizer."""
+		left_eye_center = self.left_eye.mean(axis=0).astype("int")
+		right_eye_center = self.right_eye.mean(axis=0).astype("int")
+		A = dist.euclidean(left_eye_center, self.gravity_point)
+		B = dist.euclidean(right_eye_center, self.gravity_point)
+		return (A+B)/2.0
+
+	def get_eyes_features(self):
+		"""Method that appends to the features attribute all eyes features."""
+		left_eye_center = self.left_eye.mean(axis=0).astype("int")
+
+		left_1 = dist.euclidean(self.left_eyebrow[0], left_eye_center)/self.normalizer
+		left_2 = dist.euclidean(self.left_eyebrow[2], left_eye_center)/self.normalizer
+		left_3 = dist.euclidean(self.left_eyebrow[4], left_eye_center)/self.normalizer
+
+		self.features.append(eye_aspect_ratio(self.left_eye))
+		self.features.append(left_1)
+		self.features.append(left_2)
+		self.features.append(left_3)
+
+		right_eye_center = self.right_eye.mean(axis=0).astype("int")
+
+		right_3 = dist.euclidean(self.right_eyebrow[0], right_eye_center)/self.normalizer
+		right_2 = dist.euclidean(self.right_eyebrow[2], right_eye_center)/self.normalizer
+		right_1 = dist.euclidean(self.right_eyebrow[4], right_eye_center)/self.normalizer
+
+		self.features.append(eye_aspect_ratio(self.right_eye))
+		self.features.append(right_1)
+		self.features.append(right_2)
+		self.features.append(right_3)
+
+	def get_mouth_features(self):
+		"""Method that appends to the features attributes all mouth features."""
+		self.features.append(mouth_aspect_ratio(self.mouth))
+
+		mouth_1 = dist.euclidean(self.mouth[3], self.gravity_point)/self.normalizer
+		mouth_2 = dist.euclidean(self.mouth[9], self.gravity_point)/self.normalizer
+		mouth_3 = dist.euclidean(self.mouth[6], self.gravity_point)/self.normalizer
+		mouth_4 = dist.euclidean(self.mouth[0], self.gravity_point)/self.normalizer
+
+		self.features.append(mouth_1)
+		self.features.append(mouth_2)
+		self.features.append(mouth_3)
+		self.features.append(mouth_4)
+
+
+	def extract_features(self):
+		"""Method returns the features of the face."""
+		self.get_eyes_features()
+		self.get_mouth_features()
+		return np.array(self.features)
+
+
+

src/utils/modelmanager.py

@@ -0,0 +1,123 @@
+from sklearn.neighbors import KNeighborsClassifier
+from sklearn.naive_bayes import GaussianNB
+from sklearn.svm import SVC
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier
+from sklearn.ensemble import VotingClassifier, GradientBoostingClassifier
+
+from sklearn.neural_network import MLPClassifier
+
+from sklearn.preprocessing import LabelEncoder
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import classification_report
+from sklearn.pipeline import Pipeline
+from sklearn.feature_selection import SelectKBest, SelectPercentile, SelectFpr
+from sklearn.feature_selection import chi2, f_classif, mutual_info_classif
+from sklearn.feature_selection import RFE
+
+
+
+class Model(object):
+	"""
+	Class that representats the model.
+
+	Attributes:
+	le: object
+		Object of Label Encoder
+	model: object
+		Object of sklearn model of choice.
+	training_data: array
+		Array of data.
+	training_labels: array
+		Encoded labels for training data.
+	test_data: array
+		Array of test data. Initially empty.
+	test_labels: array
+		Array of test labels. Initially empty.
+
+	"""
+
+	def __init__(self, model, data=None, labels=None):
+		"""Constructor for model class."""
+		if data is None or labels is None:
+			raise AttributeError("No Data in a constructor provided.")
+
+
+		self.models = {
+			"knn": KNeighborsClassifier(n_neighbors=9, algorithm="brute", weights="distance"),
+			"naive_bayes": GaussianNB(),
+			"svm": SVC(C=15.6, gamma="scale", kernel="rbf"),
+			"decision_tree": DecisionTreeClassifier(criterion="entropy", max_depth=55, splitter="best"),
+			"random_forest": RandomForestClassifier(n_estimators=50, criterion="entropy"),
+			"extra_tree": ExtraTreesClassifier(n_estimators=122, criterion="entropy"),
+			"gradient_boost": GradientBoostingClassifier(n_estimators=33, learning_rate=0.14),
+			"mlp":  MLPClassifier(solver="lbfgs", hidden_layer_sizes=(13, 12), alpha=5E-06)
+
+		}
+
+		self.le = LabelEncoder()
+		self.model = self.models[model]
+
+		self.training_data = data
+		self.training_labels = self.le.fit_transform(labels)
+		self.feature_names = ['EARL','L1','L2','L3', 'EARR', 'R1', 'R2', 'R3', 'MAR', 'M1', 'M2', 'M3', 'M4']
+		self.feature_mask = [True,True,True,True,True,True,True,True,True,True,True,True,True]
+
+
+	def use_voting_classifier(self):
+		"""Method for changing to VotingClassifier."""
+		self.model = VotingClassifier(estimators=[('nb', self.models["naive_bayes"]), ('et', self.models["extra_tree"]), ('gb', self.models["gradient_boost"])], voting='hard', weights=[2,3,1.5])
+
+	def split_dataset(self, test_size=0.20):
+		"""Method for spliting dataset to the training and test."""
+		(self.training_data, self.test_data, self.training_labels, self.test_labels) = train_test_split(self.training_data, self.training_labels, test_size=test_size)
+
+	def train(self):
+		"""Method for training a model with the training dataset."""
+		self.model.fit(self.training_data, self.training_labels)
+
+	def test(self):
+		"""Method returns the classification report."""
+		return classification_report(self.test_labels, self.predict(self.test_data), target_names=self.le.classes_)
+
+	def predict(self, to_predict):
+		"""Method returns the prefiction for new data."""
+		return self.model.predict(to_predict)
+
+	def univariate_feature_selection(self, method, scoring, number):
+		"""Method that creates the pipeline for only important feature extraction with univariate_feature_selection."""
+		self.scoring_functions = {
+			"f_classif": f_classif,
+			"mutual_info_classif": mutual_info_classif,
+			"chi2": chi2
+		}
+
+		self.selection_methods = {
+			"select_k_best": SelectKBest(self.scoring_functions[scoring], k=number),
+			"select_percentile": SelectPercentile(self.scoring_functions[scoring], percentile=number)
+		}
+
+
+		self.model = Pipeline([
+			('feature_selection', self.selection_methods[method]),
+			('classification', self.model)
+			])
+
+
+
+	def recursive_feature_elimination(self):
+		"""Method that creates the pipeline for only important feature extraction with RFE method."""
+		svc = SVC(kernel="linear")
+		self.model = Pipeline([
+			('feature_selection', RFE(estimator=svc, n_features_to_select=8, step=10)),
+			('classification', self.model)
+			])
+
+	def get_feature_labels(self):
+		"""Method for retriving feature lables from the model."""
+		feature_labels = []
+		for feature, i in zip(self.feature_names,self.feature_mask):
+			if i == True:
+				feature_labels.append(feature)
+		return feature_labels
+