init

data_classification.py

+import cv2
+import json
+import numpy as np
+import tensorflow as tf
+import os
+import time
+from tensorflow.keras.preprocessing.image import img_to_array
+from tensorflow.keras.preprocessing import image
+from PIL import ImageOps
+
+class FramePupilPositions:
+    def __init__(self, left_pupil_position, right_pupil_position):
+        self.left_pupil_position = left_pupil_position
+        self.right_pupil_position = right_pupil_position
+
+class FrameEyeFrames:
+    def __init__(self, left_eye_frame, right_eye_frame):
+        self.left_eye_frame = left_eye_frame
+        self.right_eye_frame = right_eye_frame
+
+class Point:
+    def __init__(self, x, y):
+        self.x = x
+        self.y = y
+
+class EyeFrame:
+    def __init__(self, x, y, width, height):
+        self.x = x
+        self.y = y
+        self.width = width
+        self.height = height
+
+
+def parse_json_eye_landmarks(json_landmarks):
+    landmarks = []
+    for json_point in json_landmarks:
+        landmarks.append(str(int(json_point["x"])) + "-" + str(int(json_point["y"])))
+    return landmarks
+
+def read_json_file(path):
+    with open(path, 'r') as f:
+        data = json.load(f)
+        eye_frames = []
+        pupil_positions = []
+        for entry in data:
+            # currently no pupil positions provided, TODO
+            pupil_positions.append([]) # FramePupilPositions(Point(entry["left_Iris_Position"]["x"],entry["left_Iris_Position"]["y"]), Point(entry["right_Iris_Position"]["x"],entry["right_Iris_Position"]["y"]))
+            left_eye_frame = EyeFrame(entry["left_MBR_x"], entry["left_MBR_y"], entry["left_MBR_width"], entry["left_MBR_height"])
+            right_eye_frame = EyeFrame(entry["right_MBR_x"], entry["right_MBR_y"], entry["right_MBR_width"], entry["right_MBR_height"])
+            #left_landmarks = parse_json_eye_landmarks(entry["left_landmarks"])
+            #right_landmarks = parse_json_eye_landmarks(entry["right_landmarks"])
+            eye_frames.append(FrameEyeFrames(left_eye_frame, right_eye_frame))
+    return eye_frames, pupil_positions#, left_landmarks, right_landmarks
+
+def get_biggest_contour(contours):
+    if len(contours) >= 2:
+        c = sorted(contours, key=lambda x: cv2.contourArea(x))
+        c.reverse()
+        x,y,w,h = cv2.boundingRect(c[0])
+        x2,y2,w2,h2 = cv2.boundingRect(c[1])
+
+    return {"1": {"x": x, "y": y, "width": w, "height": h}, "2": {"x": x2, "y": y2, "width": w2, "height": h2}}
+
+def downscale_if_necessary(eye_img):
+    shape = eye_img.shape
+    if (shape[0] > 128 or shape[1] > 256):
+        return cv2.resize(eye_img, dsize=(256, 128), interpolation=cv2.INTER_CUBIC)
+    return eye_img
+
+cap = cv2.VideoCapture('VertigoVideosIII/data/sophia_reinhardt/Test_selfie_leeraufnahme_sophia_2021-09-28-13-12-39.mp4')
+eye_frames, pupil_positions = read_json_file('VertigoVideosIII/data/sophia_reinhardt/Test_selfie_leeraufnahme_sophia_2021-09-28-13-12-39_fixed.json')
+
+if (cap.isOpened()== False):
+  print("Error opening video stream or file")
+
+model_path = os.path.abspath('best_models_BO_32_fixed_11_0')
+loaded_model = tf.keras.models.load_model(model_path, compile=True)
+
+current_frame = 0
+json_data = []
+while(cap.isOpened()):
+  ret, frame = cap.read()
+  if ret == True:
+    current_eye_frame = eye_frames[current_frame]
+    left_eye_frame = current_eye_frame.left_eye_frame
+    right_eye_frame = current_eye_frame.right_eye_frame
+    #frame = cv2.rectangle(frame, (left_eye_frame.x, left_eye_frame.y), (left_eye_frame.x+left_eye_frame.width, left_eye_frame.y+left_eye_frame.height), (0, 255, 0), 2)
+    #frame = cv2.rectangle(frame, (right_eye_frame.x, right_eye_frame.y), (right_eye_frame.x+right_eye_frame.width, right_eye_frame.y+right_eye_frame.height), (255, 0, 0), 2)
+    # fix
+    #frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+    #ret, frame_thresh = cv2.threshold(frame_gray, 10, 255, cv2.THRESH_BINARY)
+    #contours, hierarchy = cv2.findContours(image=frame_thresh, mode=cv2.RETR_TREE, method=cv2.CHAIN_APPROX_NONE)
+    #biggest_contour = get_biggest_contour(contours)
+    #frame = cv2.rectangle(frame, (biggest_contour["1"]["x"], biggest_contour["1"]["y"]), (biggest_contour["1"]["x"]+biggest_contour["1"]["width"], biggest_contour["1"]["y"]+biggest_contour["1"]["height"]), (255, 0, 0), 2)
+    #frame = cv2.rectangle(frame, (biggest_contour["2"]["x"], biggest_contour["2"]["y"]), (biggest_contour["2"]["x"]+biggest_contour["2"]["width"], biggest_contour["2"]["y"]+biggest_contour["2"]["height"]), (255, 0, 0), 2)
+    #if (biggest_contour["1"]["x"] < biggest_contour["2"]["x"]):
+    #    left_contour = biggest_contour["1"]
+    #    right_contour = biggest_contour["2"]
+    #else:
+    #    left_contour = biggest_contour["2"]
+    #    right_contour = biggest_contour["1"]
+    #json_data.append({"frame": current_frame, "left_MBR_x": left_contour["x"], "left_MBR_y": left_contour["y"], "left_MBR_width": left_contour["width"], "left_MBR_height": left_contour["height"], "right_MBR_x": right_contour["x"], "right_MBR_y": right_contour["y"], "right_MBR_width": right_contour["width"], "right_MBR_height": right_contour["height"]})
+
+    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+    offset = 2
+    left_eye = gray[left_eye_frame.y+offset:left_eye_frame.y+left_eye_frame.height-offset*2,left_eye_frame.x+offset:left_eye_frame.x+left_eye_frame.width-offset*2]
+    left_eye = downscale_if_necessary(left_eye)
+    #cv2.imshow('Frame', frame)
+    #cv2.imshow('left eye', left_eye)
+    right_eye = gray[right_eye_frame.y+offset:right_eye_frame.y+right_eye_frame.height-offset*2,right_eye_frame.x+offset:right_eye_frame.x+right_eye_frame.width-offset*2]
+    right_eye = downscale_if_necessary(right_eye)
+    #cv2.imshow('right eye', right_eye)
+    #cv2.waitKey(0)
+    left_eye_bordered = cv2.copyMakeBorder(left_eye, 0, 128-left_eye.shape[0], 0, 256-left_eye.shape[1], cv2.BORDER_CONSTANT, None, value=0)
+    left_eye_bordered = left_eye_bordered.astype('float64')
+    left_eye_bordered *= 255.0/left_eye_bordered.max()
+    left_eye_bordered = img_to_array(left_eye_bordered)
+    left_eye_bordered = np.expand_dims(left_eye_bordered, axis=0)
+    right_eye_bordered = cv2.copyMakeBorder(right_eye, 0, 128-right_eye.shape[0], 0, 256-right_eye.shape[1], cv2.BORDER_CONSTANT, None, value=0)
+    right_eye_bordered = right_eye_bordered.astype('float64')
+    right_eye_bordered *= 255.0/right_eye_bordered.max()
+    right_eye_bordered = img_to_array(right_eye_bordered)
+    right_eye_bordered = np.expand_dims(right_eye_bordered, axis=0)
+
+    #left_start = time.time()
+    left_prediction = loaded_model.predict(left_eye_bordered)
+    left_prediction = left_prediction[0]
+    left_prediction_x = left_eye_frame.x + offset + left_prediction[0]*256
+    left_prediction_y = left_eye_frame.y + offset + left_prediction[1]*128
+    print(left_prediction_x)
+    print(left_prediction_y)
+    frame = cv2.circle(frame, (int(left_prediction_x),int(left_prediction_y)), 3, (255,0,0), 1)
+    #left_end = time.time()
+    #right_start = time.time()
+    right_prediction = loaded_model.predict(right_eye_bordered)
+    right_prediction = right_prediction[0]
+    right_prediction_x = right_eye_frame.x + offset + right_prediction[0]*256
+    right_prediction_y = right_eye_frame.y + offset + right_prediction[1]*128
+    frame = cv2.circle(frame, (int(right_prediction_x),int(right_prediction_y)), 3, (255,0,0), 1)
+    #right_end = time.time()
+    #print(left_prediction)
+    #print(right_prediction)
+    #print("Left time: ")
+    #print(left_end-left_start)
+    #print("\n")
+    #print("Right time: ")
+    #print(right_end-right_start)
+    #print("\n")
+    cv2.imshow('Frame', frame)
+    cv2.waitKey(0)
+
+    current_frame += 1
+    if cv2.waitKey(25) & 0xFF == ord('q'):
+      break
+  else:
+    break
+#with open('vertigo_data/VertigoVideosIII/data/sophia_reinhardt/Test_selfie_leeraufnahme_sophia_2021-09-28-13-12-39_fixed.json', 'w') as outfile:
+#    json.dump(json_data, outfile)
+cap.release()
+cv2.destroyAllWindows()