Commit 8f57d1da authored by Joshua Schmidt's avatar Joshua Schmidt
Browse files

init

parent bbf34766
import cv2
import json
import numpy as np
import tensorflow as tf
import os
import time
from tensorflow.keras.preprocessing.image import img_to_array
from tensorflow.keras.preprocessing import image
from PIL import ImageOps
class FramePupilPositions:
def __init__(self, left_pupil_position, right_pupil_position):
self.left_pupil_position = left_pupil_position
self.right_pupil_position = right_pupil_position
class FrameEyeFrames:
def __init__(self, left_eye_frame, right_eye_frame):
self.left_eye_frame = left_eye_frame
self.right_eye_frame = right_eye_frame
class Point:
def __init__(self, x, y):
self.x = x
self.y = y
class EyeFrame:
def __init__(self, x, y, width, height):
self.x = x
self.y = y
self.width = width
self.height = height
def parse_json_eye_landmarks(json_landmarks):
landmarks = []
for json_point in json_landmarks:
landmarks.append(str(int(json_point["x"])) + "-" + str(int(json_point["y"])))
return landmarks
def read_json_file(path):
with open(path, 'r') as f:
data = json.load(f)
eye_frames = []
pupil_positions = []
for entry in data:
# currently no pupil positions provided, TODO
pupil_positions.append([]) # FramePupilPositions(Point(entry["left_Iris_Position"]["x"],entry["left_Iris_Position"]["y"]), Point(entry["right_Iris_Position"]["x"],entry["right_Iris_Position"]["y"]))
left_eye_frame = EyeFrame(entry["left_MBR_x"], entry["left_MBR_y"], entry["left_MBR_width"], entry["left_MBR_height"])
right_eye_frame = EyeFrame(entry["right_MBR_x"], entry["right_MBR_y"], entry["right_MBR_width"], entry["right_MBR_height"])
#left_landmarks = parse_json_eye_landmarks(entry["left_landmarks"])
#right_landmarks = parse_json_eye_landmarks(entry["right_landmarks"])
eye_frames.append(FrameEyeFrames(left_eye_frame, right_eye_frame))
return eye_frames, pupil_positions#, left_landmarks, right_landmarks
def get_biggest_contour(contours):
if len(contours) >= 2:
c = sorted(contours, key=lambda x: cv2.contourArea(x))
c.reverse()
x,y,w,h = cv2.boundingRect(c[0])
x2,y2,w2,h2 = cv2.boundingRect(c[1])
return {"1": {"x": x, "y": y, "width": w, "height": h}, "2": {"x": x2, "y": y2, "width": w2, "height": h2}}
def downscale_if_necessary(eye_img):
shape = eye_img.shape
if (shape[0] > 128 or shape[1] > 256):
return cv2.resize(eye_img, dsize=(256, 128), interpolation=cv2.INTER_CUBIC)
return eye_img
cap = cv2.VideoCapture('VertigoVideosIII/data/sophia_reinhardt/Test_selfie_leeraufnahme_sophia_2021-09-28-13-12-39.mp4')
eye_frames, pupil_positions = read_json_file('VertigoVideosIII/data/sophia_reinhardt/Test_selfie_leeraufnahme_sophia_2021-09-28-13-12-39_fixed.json')
if (cap.isOpened()== False):
print("Error opening video stream or file")
model_path = os.path.abspath('best_models_BO_32_fixed_11_0')
loaded_model = tf.keras.models.load_model(model_path, compile=True)
current_frame = 0
json_data = []
while(cap.isOpened()):
ret, frame = cap.read()
if ret == True:
current_eye_frame = eye_frames[current_frame]
left_eye_frame = current_eye_frame.left_eye_frame
right_eye_frame = current_eye_frame.right_eye_frame
#frame = cv2.rectangle(frame, (left_eye_frame.x, left_eye_frame.y), (left_eye_frame.x+left_eye_frame.width, left_eye_frame.y+left_eye_frame.height), (0, 255, 0), 2)
#frame = cv2.rectangle(frame, (right_eye_frame.x, right_eye_frame.y), (right_eye_frame.x+right_eye_frame.width, right_eye_frame.y+right_eye_frame.height), (255, 0, 0), 2)
# fix
#frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
#ret, frame_thresh = cv2.threshold(frame_gray, 10, 255, cv2.THRESH_BINARY)
#contours, hierarchy = cv2.findContours(image=frame_thresh, mode=cv2.RETR_TREE, method=cv2.CHAIN_APPROX_NONE)
#biggest_contour = get_biggest_contour(contours)
#frame = cv2.rectangle(frame, (biggest_contour["1"]["x"], biggest_contour["1"]["y"]), (biggest_contour["1"]["x"]+biggest_contour["1"]["width"], biggest_contour["1"]["y"]+biggest_contour["1"]["height"]), (255, 0, 0), 2)
#frame = cv2.rectangle(frame, (biggest_contour["2"]["x"], biggest_contour["2"]["y"]), (biggest_contour["2"]["x"]+biggest_contour["2"]["width"], biggest_contour["2"]["y"]+biggest_contour["2"]["height"]), (255, 0, 0), 2)
#if (biggest_contour["1"]["x"] < biggest_contour["2"]["x"]):
# left_contour = biggest_contour["1"]
# right_contour = biggest_contour["2"]
#else:
# left_contour = biggest_contour["2"]
# right_contour = biggest_contour["1"]
#json_data.append({"frame": current_frame, "left_MBR_x": left_contour["x"], "left_MBR_y": left_contour["y"], "left_MBR_width": left_contour["width"], "left_MBR_height": left_contour["height"], "right_MBR_x": right_contour["x"], "right_MBR_y": right_contour["y"], "right_MBR_width": right_contour["width"], "right_MBR_height": right_contour["height"]})
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
offset = 2
left_eye = gray[left_eye_frame.y+offset:left_eye_frame.y+left_eye_frame.height-offset*2,left_eye_frame.x+offset:left_eye_frame.x+left_eye_frame.width-offset*2]
left_eye = downscale_if_necessary(left_eye)
#cv2.imshow('Frame', frame)
#cv2.imshow('left eye', left_eye)
right_eye = gray[right_eye_frame.y+offset:right_eye_frame.y+right_eye_frame.height-offset*2,right_eye_frame.x+offset:right_eye_frame.x+right_eye_frame.width-offset*2]
right_eye = downscale_if_necessary(right_eye)
#cv2.imshow('right eye', right_eye)
#cv2.waitKey(0)
left_eye_bordered = cv2.copyMakeBorder(left_eye, 0, 128-left_eye.shape[0], 0, 256-left_eye.shape[1], cv2.BORDER_CONSTANT, None, value=0)
left_eye_bordered = left_eye_bordered.astype('float64')
left_eye_bordered *= 255.0/left_eye_bordered.max()
left_eye_bordered = img_to_array(left_eye_bordered)
left_eye_bordered = np.expand_dims(left_eye_bordered, axis=0)
right_eye_bordered = cv2.copyMakeBorder(right_eye, 0, 128-right_eye.shape[0], 0, 256-right_eye.shape[1], cv2.BORDER_CONSTANT, None, value=0)
right_eye_bordered = right_eye_bordered.astype('float64')
right_eye_bordered *= 255.0/right_eye_bordered.max()
right_eye_bordered = img_to_array(right_eye_bordered)
right_eye_bordered = np.expand_dims(right_eye_bordered, axis=0)
#left_start = time.time()
left_prediction = loaded_model.predict(left_eye_bordered)
left_prediction = left_prediction[0]
left_prediction_x = left_eye_frame.x + offset + left_prediction[0]*256
left_prediction_y = left_eye_frame.y + offset + left_prediction[1]*128
print(left_prediction_x)
print(left_prediction_y)
frame = cv2.circle(frame, (int(left_prediction_x),int(left_prediction_y)), 3, (255,0,0), 1)
#left_end = time.time()
#right_start = time.time()
right_prediction = loaded_model.predict(right_eye_bordered)
right_prediction = right_prediction[0]
right_prediction_x = right_eye_frame.x + offset + right_prediction[0]*256
right_prediction_y = right_eye_frame.y + offset + right_prediction[1]*128
frame = cv2.circle(frame, (int(right_prediction_x),int(right_prediction_y)), 3, (255,0,0), 1)
#right_end = time.time()
#print(left_prediction)
#print(right_prediction)
#print("Left time: ")
#print(left_end-left_start)
#print("\n")
#print("Right time: ")
#print(right_end-right_start)
#print("\n")
cv2.imshow('Frame', frame)
cv2.waitKey(0)
current_frame += 1
if cv2.waitKey(25) & 0xFF == ord('q'):
break
else:
break
#with open('vertigo_data/VertigoVideosIII/data/sophia_reinhardt/Test_selfie_leeraufnahme_sophia_2021-09-28-13-12-39_fixed.json', 'w') as outfile:
# json.dump(json_data, outfile)
cap.release()
cv2.destroyAllWindows()
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment