Skip to content
Snippets Groups Projects
Select Git revision
  • 5e18f99443d070135b77e16a255471f4e9ddd74d
  • master default protected
  • exec_auto_adjust_trace
  • let_variables
  • v1.4.1
  • v1.4.0
  • v1.3.0
  • v1.2.0
  • v1.1.0
  • v1.0.0
10 results

Parameter.java

Blame
  • Code owners
    Assign users and groups as approvers for specific file changes. Learn more.
    data_classification.py 7.76 KiB
    import cv2
    import json
    import numpy as np
    import tensorflow as tf
    import os
    import time
    from tensorflow.keras.preprocessing.image import img_to_array
    from tensorflow.keras.preprocessing import image
    from PIL import ImageOps
    
    class FramePupilPositions:
        def __init__(self, left_pupil_position, right_pupil_position):
            self.left_pupil_position = left_pupil_position
            self.right_pupil_position = right_pupil_position
    
    class FrameEyeFrames:
        def __init__(self, left_eye_frame, right_eye_frame):
            self.left_eye_frame = left_eye_frame
            self.right_eye_frame = right_eye_frame
    
    class Point:
        def __init__(self, x, y):
            self.x = x
            self.y = y
    
    class EyeFrame:
        def __init__(self, x, y, width, height):
            self.x = x
            self.y = y
            self.width = width
            self.height = height
    
    
    def parse_json_eye_landmarks(json_landmarks):
        landmarks = []
        for json_point in json_landmarks:
            landmarks.append(str(int(json_point["x"])) + "-" + str(int(json_point["y"])))
        return landmarks
    
    def read_json_file(path):
        with open(path, 'r') as f:
            data = json.load(f)
            eye_frames = []
            pupil_positions = []
            for entry in data:
                # currently no pupil positions provided, TODO
                pupil_positions.append([]) # FramePupilPositions(Point(entry["left_Iris_Position"]["x"],entry["left_Iris_Position"]["y"]), Point(entry["right_Iris_Position"]["x"],entry["right_Iris_Position"]["y"]))
                left_eye_frame = EyeFrame(entry["left_MBR_x"], entry["left_MBR_y"], entry["left_MBR_width"], entry["left_MBR_height"])
                right_eye_frame = EyeFrame(entry["right_MBR_x"], entry["right_MBR_y"], entry["right_MBR_width"], entry["right_MBR_height"])
                #left_landmarks = parse_json_eye_landmarks(entry["left_landmarks"])
                #right_landmarks = parse_json_eye_landmarks(entry["right_landmarks"])
                eye_frames.append(FrameEyeFrames(left_eye_frame, right_eye_frame))
        return eye_frames, pupil_positions#, left_landmarks, right_landmarks
    
    def get_biggest_contour(contours):
        if len(contours) >= 2:
            c = sorted(contours, key=lambda x: cv2.contourArea(x))
            c.reverse()
            x,y,w,h = cv2.boundingRect(c[0])
            x2,y2,w2,h2 = cv2.boundingRect(c[1])
    
        return {"1": {"x": x, "y": y, "width": w, "height": h}, "2": {"x": x2, "y": y2, "width": w2, "height": h2}}
    
    def downscale_if_necessary(eye_img):
        shape = eye_img.shape
        if (shape[0] > 128 or shape[1] > 256):
            return cv2.resize(eye_img, dsize=(256, 128), interpolation=cv2.INTER_CUBIC)
        return eye_img
    
    cap = cv2.VideoCapture('VertigoVideosIII/data/sophia_reinhardt/Test_selfie_leeraufnahme_sophia_2021-09-28-13-12-39.mp4')
    eye_frames, pupil_positions = read_json_file('VertigoVideosIII/data/sophia_reinhardt/Test_selfie_leeraufnahme_sophia_2021-09-28-13-12-39_fixed.json')
    
    if (cap.isOpened()== False):
      print("Error opening video stream or file")
    
    model_path = os.path.abspath('best_models_BO_32_fixed_11_0')
    loaded_model = tf.keras.models.load_model(model_path, compile=True)
    
    current_frame = 0
    json_data = []
    while(cap.isOpened()):
      ret, frame = cap.read()
      if ret == True:
        current_eye_frame = eye_frames[current_frame]
        left_eye_frame = current_eye_frame.left_eye_frame
        right_eye_frame = current_eye_frame.right_eye_frame
        #frame = cv2.rectangle(frame, (left_eye_frame.x, left_eye_frame.y), (left_eye_frame.x+left_eye_frame.width, left_eye_frame.y+left_eye_frame.height), (0, 255, 0), 2)
        #frame = cv2.rectangle(frame, (right_eye_frame.x, right_eye_frame.y), (right_eye_frame.x+right_eye_frame.width, right_eye_frame.y+right_eye_frame.height), (255, 0, 0), 2)
        # fix
        #frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        #ret, frame_thresh = cv2.threshold(frame_gray, 10, 255, cv2.THRESH_BINARY)
        #contours, hierarchy = cv2.findContours(image=frame_thresh, mode=cv2.RETR_TREE, method=cv2.CHAIN_APPROX_NONE)
        #biggest_contour = get_biggest_contour(contours)
        #frame = cv2.rectangle(frame, (biggest_contour["1"]["x"], biggest_contour["1"]["y"]), (biggest_contour["1"]["x"]+biggest_contour["1"]["width"], biggest_contour["1"]["y"]+biggest_contour["1"]["height"]), (255, 0, 0), 2)
        #frame = cv2.rectangle(frame, (biggest_contour["2"]["x"], biggest_contour["2"]["y"]), (biggest_contour["2"]["x"]+biggest_contour["2"]["width"], biggest_contour["2"]["y"]+biggest_contour["2"]["height"]), (255, 0, 0), 2)
        #if (biggest_contour["1"]["x"] < biggest_contour["2"]["x"]):
        #    left_contour = biggest_contour["1"]
        #    right_contour = biggest_contour["2"]
        #else:
        #    left_contour = biggest_contour["2"]
        #    right_contour = biggest_contour["1"]
        #json_data.append({"frame": current_frame, "left_MBR_x": left_contour["x"], "left_MBR_y": left_contour["y"], "left_MBR_width": left_contour["width"], "left_MBR_height": left_contour["height"], "right_MBR_x": right_contour["x"], "right_MBR_y": right_contour["y"], "right_MBR_width": right_contour["width"], "right_MBR_height": right_contour["height"]})
    
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        offset = 2
        left_eye = gray[left_eye_frame.y+offset:left_eye_frame.y+left_eye_frame.height-offset*2,left_eye_frame.x+offset:left_eye_frame.x+left_eye_frame.width-offset*2]
        left_eye = downscale_if_necessary(left_eye)
        #cv2.imshow('Frame', frame)
        #cv2.imshow('left eye', left_eye)
        right_eye = gray[right_eye_frame.y+offset:right_eye_frame.y+right_eye_frame.height-offset*2,right_eye_frame.x+offset:right_eye_frame.x+right_eye_frame.width-offset*2]
        right_eye = downscale_if_necessary(right_eye)
        #cv2.imshow('right eye', right_eye)
        #cv2.waitKey(0)
        left_eye_bordered = cv2.copyMakeBorder(left_eye, 0, 128-left_eye.shape[0], 0, 256-left_eye.shape[1], cv2.BORDER_CONSTANT, None, value=0)
        left_eye_bordered = left_eye_bordered.astype('float64')
        left_eye_bordered *= 255.0/left_eye_bordered.max()
        left_eye_bordered = img_to_array(left_eye_bordered)
        left_eye_bordered = np.expand_dims(left_eye_bordered, axis=0)
        right_eye_bordered = cv2.copyMakeBorder(right_eye, 0, 128-right_eye.shape[0], 0, 256-right_eye.shape[1], cv2.BORDER_CONSTANT, None, value=0)
        right_eye_bordered = right_eye_bordered.astype('float64')
        right_eye_bordered *= 255.0/right_eye_bordered.max()
        right_eye_bordered = img_to_array(right_eye_bordered)
        right_eye_bordered = np.expand_dims(right_eye_bordered, axis=0)
    
        #left_start = time.time()
        left_prediction = loaded_model.predict(left_eye_bordered)
        left_prediction = left_prediction[0]
        left_prediction_x = left_eye_frame.x + offset + left_prediction[0]*256
        left_prediction_y = left_eye_frame.y + offset + left_prediction[1]*128
        print(left_prediction_x)
        print(left_prediction_y)
        frame = cv2.circle(frame, (int(left_prediction_x),int(left_prediction_y)), 3, (255,0,0), 1)
        #left_end = time.time()
        #right_start = time.time()
        right_prediction = loaded_model.predict(right_eye_bordered)
        right_prediction = right_prediction[0]
        right_prediction_x = right_eye_frame.x + offset + right_prediction[0]*256
        right_prediction_y = right_eye_frame.y + offset + right_prediction[1]*128
        frame = cv2.circle(frame, (int(right_prediction_x),int(right_prediction_y)), 3, (255,0,0), 1)
        #right_end = time.time()
        #print(left_prediction)
        #print(right_prediction)
        #print("Left time: ")
        #print(left_end-left_start)
        #print("\n")
        #print("Right time: ")
        #print(right_end-right_start)
        #print("\n")
        cv2.imshow('Frame', frame)
        cv2.waitKey(0)
    
        current_frame += 1
        if cv2.waitKey(25) & 0xFF == ord('q'):
          break
      else:
        break
    #with open('vertigo_data/VertigoVideosIII/data/sophia_reinhardt/Test_selfie_leeraufnahme_sophia_2021-09-28-13-12-39_fixed.json', 'w') as outfile:
    #    json.dump(json_data, outfile)
    cap.release()
    cv2.destroyAllWindows()