Select Git revision
Parameter.java
Code owners
Assign users and groups as approvers for specific file changes. Learn more.
data_classification.py 7.76 KiB
import cv2
import json
import numpy as np
import tensorflow as tf
import os
import time
from tensorflow.keras.preprocessing.image import img_to_array
from tensorflow.keras.preprocessing import image
from PIL import ImageOps
class FramePupilPositions:
def __init__(self, left_pupil_position, right_pupil_position):
self.left_pupil_position = left_pupil_position
self.right_pupil_position = right_pupil_position
class FrameEyeFrames:
def __init__(self, left_eye_frame, right_eye_frame):
self.left_eye_frame = left_eye_frame
self.right_eye_frame = right_eye_frame
class Point:
def __init__(self, x, y):
self.x = x
self.y = y
class EyeFrame:
def __init__(self, x, y, width, height):
self.x = x
self.y = y
self.width = width
self.height = height
def parse_json_eye_landmarks(json_landmarks):
landmarks = []
for json_point in json_landmarks:
landmarks.append(str(int(json_point["x"])) + "-" + str(int(json_point["y"])))
return landmarks
def read_json_file(path):
with open(path, 'r') as f:
data = json.load(f)
eye_frames = []
pupil_positions = []
for entry in data:
# currently no pupil positions provided, TODO
pupil_positions.append([]) # FramePupilPositions(Point(entry["left_Iris_Position"]["x"],entry["left_Iris_Position"]["y"]), Point(entry["right_Iris_Position"]["x"],entry["right_Iris_Position"]["y"]))
left_eye_frame = EyeFrame(entry["left_MBR_x"], entry["left_MBR_y"], entry["left_MBR_width"], entry["left_MBR_height"])
right_eye_frame = EyeFrame(entry["right_MBR_x"], entry["right_MBR_y"], entry["right_MBR_width"], entry["right_MBR_height"])
#left_landmarks = parse_json_eye_landmarks(entry["left_landmarks"])
#right_landmarks = parse_json_eye_landmarks(entry["right_landmarks"])
eye_frames.append(FrameEyeFrames(left_eye_frame, right_eye_frame))
return eye_frames, pupil_positions#, left_landmarks, right_landmarks
def get_biggest_contour(contours):
if len(contours) >= 2:
c = sorted(contours, key=lambda x: cv2.contourArea(x))
c.reverse()
x,y,w,h = cv2.boundingRect(c[0])
x2,y2,w2,h2 = cv2.boundingRect(c[1])
return {"1": {"x": x, "y": y, "width": w, "height": h}, "2": {"x": x2, "y": y2, "width": w2, "height": h2}}
def downscale_if_necessary(eye_img):
shape = eye_img.shape
if (shape[0] > 128 or shape[1] > 256):
return cv2.resize(eye_img, dsize=(256, 128), interpolation=cv2.INTER_CUBIC)
return eye_img
cap = cv2.VideoCapture('VertigoVideosIII/data/sophia_reinhardt/Test_selfie_leeraufnahme_sophia_2021-09-28-13-12-39.mp4')
eye_frames, pupil_positions = read_json_file('VertigoVideosIII/data/sophia_reinhardt/Test_selfie_leeraufnahme_sophia_2021-09-28-13-12-39_fixed.json')
if (cap.isOpened()== False):
print("Error opening video stream or file")
model_path = os.path.abspath('best_models_BO_32_fixed_11_0')
loaded_model = tf.keras.models.load_model(model_path, compile=True)
current_frame = 0
json_data = []
while(cap.isOpened()):
ret, frame = cap.read()
if ret == True:
current_eye_frame = eye_frames[current_frame]
left_eye_frame = current_eye_frame.left_eye_frame
right_eye_frame = current_eye_frame.right_eye_frame
#frame = cv2.rectangle(frame, (left_eye_frame.x, left_eye_frame.y), (left_eye_frame.x+left_eye_frame.width, left_eye_frame.y+left_eye_frame.height), (0, 255, 0), 2)
#frame = cv2.rectangle(frame, (right_eye_frame.x, right_eye_frame.y), (right_eye_frame.x+right_eye_frame.width, right_eye_frame.y+right_eye_frame.height), (255, 0, 0), 2)
# fix
#frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
#ret, frame_thresh = cv2.threshold(frame_gray, 10, 255, cv2.THRESH_BINARY)
#contours, hierarchy = cv2.findContours(image=frame_thresh, mode=cv2.RETR_TREE, method=cv2.CHAIN_APPROX_NONE)
#biggest_contour = get_biggest_contour(contours)
#frame = cv2.rectangle(frame, (biggest_contour["1"]["x"], biggest_contour["1"]["y"]), (biggest_contour["1"]["x"]+biggest_contour["1"]["width"], biggest_contour["1"]["y"]+biggest_contour["1"]["height"]), (255, 0, 0), 2)
#frame = cv2.rectangle(frame, (biggest_contour["2"]["x"], biggest_contour["2"]["y"]), (biggest_contour["2"]["x"]+biggest_contour["2"]["width"], biggest_contour["2"]["y"]+biggest_contour["2"]["height"]), (255, 0, 0), 2)
#if (biggest_contour["1"]["x"] < biggest_contour["2"]["x"]):
# left_contour = biggest_contour["1"]
# right_contour = biggest_contour["2"]
#else:
# left_contour = biggest_contour["2"]
# right_contour = biggest_contour["1"]
#json_data.append({"frame": current_frame, "left_MBR_x": left_contour["x"], "left_MBR_y": left_contour["y"], "left_MBR_width": left_contour["width"], "left_MBR_height": left_contour["height"], "right_MBR_x": right_contour["x"], "right_MBR_y": right_contour["y"], "right_MBR_width": right_contour["width"], "right_MBR_height": right_contour["height"]})
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
offset = 2
left_eye = gray[left_eye_frame.y+offset:left_eye_frame.y+left_eye_frame.height-offset*2,left_eye_frame.x+offset:left_eye_frame.x+left_eye_frame.width-offset*2]
left_eye = downscale_if_necessary(left_eye)
#cv2.imshow('Frame', frame)
#cv2.imshow('left eye', left_eye)
right_eye = gray[right_eye_frame.y+offset:right_eye_frame.y+right_eye_frame.height-offset*2,right_eye_frame.x+offset:right_eye_frame.x+right_eye_frame.width-offset*2]
right_eye = downscale_if_necessary(right_eye)
#cv2.imshow('right eye', right_eye)
#cv2.waitKey(0)
left_eye_bordered = cv2.copyMakeBorder(left_eye, 0, 128-left_eye.shape[0], 0, 256-left_eye.shape[1], cv2.BORDER_CONSTANT, None, value=0)
left_eye_bordered = left_eye_bordered.astype('float64')
left_eye_bordered *= 255.0/left_eye_bordered.max()
left_eye_bordered = img_to_array(left_eye_bordered)
left_eye_bordered = np.expand_dims(left_eye_bordered, axis=0)
right_eye_bordered = cv2.copyMakeBorder(right_eye, 0, 128-right_eye.shape[0], 0, 256-right_eye.shape[1], cv2.BORDER_CONSTANT, None, value=0)
right_eye_bordered = right_eye_bordered.astype('float64')
right_eye_bordered *= 255.0/right_eye_bordered.max()
right_eye_bordered = img_to_array(right_eye_bordered)
right_eye_bordered = np.expand_dims(right_eye_bordered, axis=0)
#left_start = time.time()
left_prediction = loaded_model.predict(left_eye_bordered)
left_prediction = left_prediction[0]
left_prediction_x = left_eye_frame.x + offset + left_prediction[0]*256
left_prediction_y = left_eye_frame.y + offset + left_prediction[1]*128
print(left_prediction_x)
print(left_prediction_y)
frame = cv2.circle(frame, (int(left_prediction_x),int(left_prediction_y)), 3, (255,0,0), 1)
#left_end = time.time()
#right_start = time.time()
right_prediction = loaded_model.predict(right_eye_bordered)
right_prediction = right_prediction[0]
right_prediction_x = right_eye_frame.x + offset + right_prediction[0]*256
right_prediction_y = right_eye_frame.y + offset + right_prediction[1]*128
frame = cv2.circle(frame, (int(right_prediction_x),int(right_prediction_y)), 3, (255,0,0), 1)
#right_end = time.time()
#print(left_prediction)
#print(right_prediction)
#print("Left time: ")
#print(left_end-left_start)
#print("\n")
#print("Right time: ")
#print(right_end-right_start)
#print("\n")
cv2.imshow('Frame', frame)
cv2.waitKey(0)
current_frame += 1
if cv2.waitKey(25) & 0xFF == ord('q'):
break
else:
break
#with open('vertigo_data/VertigoVideosIII/data/sophia_reinhardt/Test_selfie_leeraufnahme_sophia_2021-09-28-13-12-39_fixed.json', 'w') as outfile:
# json.dump(json_data, outfile)
cap.release()
cv2.destroyAllWindows()