Skip to content
Snippets Groups Projects
Commit d1c05339 authored by shreyasb's avatar shreyasb
Browse files

resolved conflicts

parents fd73ecb3 9cfedc8c
Branches
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
config.py
+ 2
− 0
View file @ d1c05339
# The intrinsic camera matrix of the webcam added onto Tobii glasses
K1 = [[1146.8119767102853, 0, 995.8472226069734, 0],
[0, 1150.968615728085, 560.9054428070731, 0],
[0, 0, 1, 0]]
......@@ -6,6 +7,7 @@ K2 = [[1478.0057732719538, 0, 964.3897381426681, 0],
[0, 1478.2826857743617, 559.6902650031882, 0],
[0, 0, 1, 0]]
# The rigid body transformation between the tobii coordinate system and the webcam coordinate system
Rigid_body = [[0.9998443871765447, 0.016930978475685435, 0.004954129531717574, 0.0014797681252487246],
[-0.017536129544264166, 0.9844701379081924, 0.1746740728546538, 0.017710310592413483],
[-0.0019197896155425946, -0.1747337675862396, 0.9846138455622852, -0.017384735431967623],
......
countBlinks.py deleted 100644 → 0
+ 0
− 270
View file @ fd73ecb3
from pupil_apriltags import Detector
import cv2
import sys
import numpy as np
from tobiiglassesctrl import TobiiGlassesController
import time
import os
def get_hom(result, fix):
src_pts = np.empty((0, 2))
dst_pts = np.empty((0, 2))
H = None
for detection in result:
if detection.tag_id in fix:
center = detection.center
corners = detection.corners
point_list = list(corners) + [center]
src_pts = np.append(src_pts, point_list, axis=0)
dst_pts = np.append(dst_pts, np.array(fix[detection.tag_id]), axis=0)
if src_pts.shape[0] > 0:
H, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0)
return H
def get_dict_from_image(img, detector):
fix = {}
detections = detector.detect(img)
for detection in detections:
id_ = detection.tag_id
center = detection.center
corners = detection.corners
point_list = list(corners) + [center]
fix[id_] = point_list
return fix
def get_center(img, detector, tag_id):
detections = detector.detect(img)
found = False
for detection in detections:
if detection.tag_id == tag_id:
found = True
return detection.center
if not found:
print("could not find tag with id:", tag_id)
sys.exit(1)
ip = '192.168.71.50'
tobiiglasses = TobiiGlassesController(ip, video_scene=True)
project_id = tobiiglasses.create_project("Test live_scene_and_gaze.py")
participant_id = tobiiglasses.create_participant(project_id, "participant_test")
calibration_id = tobiiglasses.create_calibration(project_id, participant_id)
input("Put the calibration marker in front of the user, then press enter to calibrate")
tobiiglasses.start_calibration(calibration_id)
res = tobiiglasses.wait_until_calibration_is_done(calibration_id)
if res is False:
print("Calibration failed!")
exit(1)
tobiiglasses.start_streaming()
eye_camera_id = "usb-046d_HD_Webcam_C615_29CD8CC0-video-index0"
K1 = [[1146.8119767102853, 0, 995.8472226069734, 0],
[0, 1150.968615728085, 560.9054428070731, 0],
[0, 0, 1, 0]]
K2 = [[1478.0057732719538, 0, 964.3897381426681, 0],
[0, 1478.2826857743617, 559.6902650031882, 0],
[0, 0, 1, 0]]
Rigid_body = [[0.9998443871765447, 0.016930978475685435, 0.004954129531717574, 0.0014797681252487246],
[-0.017536129544264166, 0.9844701379081924, 0.1746740728546538, 0.017710310592413483],
[-0.0019197896155425946, -0.1747337675862396, 0.9846138455622852, -0.017384735431967623],
[0., 0., 0., 1.]]
num_apriltags = 12
# screen1_ids = [32, 31, 3, 5, 10, 7]
# screen2_ids = [33, 30, 17, 6]
# screen3_ids = [9, 2]
print("before webcam")
cap = cv2.VideoCapture(1)
if cap.isOpened():
print("Succesfully connected to webcam...")
else:
print("Could not connect to camera...")
tobiiglasses.stop_streaming()
sys.exit(1)
background = cv2.imread('background.jpg')
background = cv2.resize(background, (640,480))
gray_back = cv2.cvtColor(background, cv2.COLOR_RGB2GRAY)
detector = Detector(families='tag25h9', nthreads=1, quad_decimate=1.0)
back_detections = detector.detect(gray_back)
# print(back_detections)
if len(back_detections) < num_apriltags:
print("Could not find all tags in background image, only found:", len(back_detections))
tobiiglasses.stop_streaming()
sys.exit(1)
screen1 = cv2.imread('screen1.jpg')
screen1 = cv2.resize(screen1, (640,480))
screen2 = cv2.imread('screen2.jpg')
screen2 = cv2.resize(screen2, (640,480))
screen3 = cv2.imread('screen3.jpg')
screen3 = cv2.resize(screen3, (640,480))
s1_grey = cv2.cvtColor(screen1, cv2.COLOR_RGB2GRAY)
s2_grey = cv2.cvtColor(screen2, cv2.COLOR_RGB2GRAY)
s3_grey = cv2.cvtColor(screen3, cv2.COLOR_RGB2GRAY)
fix_s1 = get_dict_from_image(s1_grey, detector)
fix_s2 = get_dict_from_image(s2_grey, detector)
fix_s3 = get_dict_from_image(s3_grey, detector)
err_tag_id = 8
detector_2 = Detector(families="tag36h11", nthreads=1, quad_decimate=1.0)
err_center = get_center(s1_grey, detector_2, err_tag_id)
fix = {}
for detection in back_detections:
id_ = detection.tag_id
center = detection.center
if id_ == 32:
s1_min = center[0]
elif id_ == 11:
s1_max = center[0]
elif id_ == 33:
s2_min = center[0]
elif id_ == 30:
s2_max = center[0]
elif id_ == 9:
s3_min = center[0]
elif id_ == 2:
s3_max = center[0]
corners = detection.corners
point_list = list(corners) + [center]
fix[id_] = point_list
Ex = np.eye(4)
P2 = np.dot(K2, Ex)
P2 = np.dot(P2, Rigid_body)
err = []
to_quit = False
count = 0
total_elapsed = 0
start = time.time()
num_blinks = 0
left_gidx_old = 0
right_gidx_old = 0
while True:
ret, frame = cap.read()
bg_copy = background.copy()
s1_copy = screen1.copy()
s2_copy = screen2.copy()
s3_copy = screen3.copy()
# cv2.imshow("frame", frame)
gray_frame = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)
result = detector.detect(gray_frame)
data = tobiiglasses.get_data()
gaze3 = []
gaze3 = data['gp3']
if 'gp3' in gaze3:
gaze3 = gaze3['gp3']
else:
continue
pd_left = data['left_eye']['pd']['pd']
pd_right = data['right_eye']['pd']['pd']
left_gidx = data['left_eye']['pd']['gidx']
right_gidx = data['right_eye']['pd']['gidx']
if len(gaze3) < 4:
gaze3.append(1)
gaze3 = np.array(gaze3)
if left_gidx != left_gidx_old and right_gidx != right_gidx_old:
left_gidx_old = left_gidx
right_gidx_old = right_gidx
if not(gaze3[0] == -1 and gaze3[1] == -1 and gaze3[2] == -1 or pd_left == -1 or pd_right == -1):
gaze3[0] = -gaze3[0]
gaze3[1] = -gaze3[1]
gaze_mapped = np.dot(P2, gaze3) ## 3X1
gaze_mapped = gaze_mapped/gaze_mapped[2]
gaze_mapped[0] = gaze_mapped[0] / 1920*640 ## TODO: Need to check 1920 -> 640
gaze_mapped[1] = gaze_mapped[1] / 1080*480
H = get_hom(result, fix)
if H is not None:
gaze_mapped_back = np.dot(H, gaze_mapped)
gaze_mapped_back = gaze_mapped_back/gaze_mapped_back[2]
cv2.circle(bg_copy, (int(gaze_mapped_back[0]), int(gaze_mapped_back[1])), 2, (0,0,255), 40)
if gaze_mapped_back[0] > s1_min and gaze_mapped_back[0] < s1_max:
H1 = get_hom(result, fix_s1)
print("Looking at screen 1", end='\r')
if H1 is not None:
gaze_mapped_s1 = np.dot(H1, gaze_mapped)
gaze_mapped_s1 = gaze_mapped_s1/gaze_mapped_s1[2]
# cv2.circle(s1_copy, (int(gaze_mapped_s1[0]), int(gaze_mapped_s1[1])), 2, (0,0,255), 40)
vec = (gaze_mapped_s1[:2] - err_center)
err.append(vec)
elif gaze_mapped_back[0] > s2_min and gaze_mapped_back[0] < s2_max and gaze_mapped_back[1] < 240:
H2 = get_hom(result, fix_s2)
print("Looking at screen 2", end='\r')
if H2 is not None:
gaze_mapped_s2 = np.dot(H2, gaze_mapped)
gaze_mapped_s2 = gaze_mapped_s2/gaze_mapped_s2[2]
# cv2.circle(s2_copy, (int(gaze_mapped_s2[0]), int(gaze_mapped_s2[1])), 2, (0,0,255), 40)
elif gaze_mapped_back[0] > s3_min and gaze_mapped_back[0] < s3_max and gaze_mapped_back[1] > 240:
H3 = get_hom(result, fix_s3)
print("Looking at screen 3", end='\r')
if H3 is not None:
gaze_mapped_s3 = np.dot(H3, gaze_mapped)
gaze_mapped_s3 = gaze_mapped_s3/gaze_mapped_s3[2]
# cv2.circle(s3_copy, (int(gaze_mapped_s3[0]), int(gaze_mapped_s3[1])), 2, (0,0,255), 40)
# cv2.imshow('mapped gaze', bg_copy)
# cv2.imshow("screen1", s1_copy)
# cv2.imshow("screen2", s2_copy)
# cv2.imshow("screen3", s3_copy)
elapsed = time.time() - start
# total_elapsed += elapsed
if elapsed > 10:
print("Done")
break
else:
bcount = 0
while left_gidx == left_gidx_old or right_gidx == right_gidx_old:
bcount += 1
data = tobiiglasses.get_data()
left_gidx = data['left_eye']['pd']['gidx']
right_gidx = data['right_eye']['pd']['gidx']
time.sleep(0.02)
if bcount > 4:
num_blinks += 1
# time.sleep(0.05)
dir_name = "error_data/Screen1/multiple/pos" + str(err_tag_id)
if not os.path.exists(dir_name):
os.makedirs(dir_name)
# np.save(dir_name + "/direct_gaze_1", err)
# np.save(dir_name + "/looking_left_corner_1", err)
# np.save(dir_name + "/looking_right_corner_1", err)
# np.save(dir_name + "/moving_head_1", err)
print()
print("number of blinks", num_blinks)
print()
tobiiglasses.stop_streaming()
cap.release()
cv2.destroyAllWindows()
cv2resizetrial.py deleted 100755 → 0
+ 0
− 23
View file @ fd73ecb3
#! /usr/bin/python3
import cv2
from apriltag import apriltag
ori_img = cv2.imread('screen3.jpg')
resized = cv2.resize(ori_img, (640,480))
cv2.imshow("orig", ori_img)
cv2.imshow("resized", resized)
grey = cv2.cvtColor(ori_img, cv2.COLOR_RGB2GRAY)
grey1 = cv2.cvtColor(resized, cv2.COLOR_RGB2GRAY)
detector = apriltag('tag25h9')
orig_detections = detector.detect(grey)
resized_detections = detector.detect(grey1)
print(len(orig_detections))
print(len(resized_detections))
while True:
if cv2.waitKey(1) & 0xFF == ord('q'):
break
\ No newline at end of file
fixations_using_eye_pos.py deleted 100644 → 0
+ 0
− 422
View file @ fd73ecb3
from pupil_apriltags import Detector
from cv2 import cv2
import sys
import numpy as np
from tobiiglassesctrl import TobiiGlassesController
import time
import os
import math
import config
def main():
ip = '192.168.71.50'
tobiiglasses = TobiiGlassesController(ip, video_scene=True)
project_id = tobiiglasses.create_project("Test live_scene_and_gaze.py")
participant_id = tobiiglasses.create_participant(project_id, "participant_test")
calibration_id = tobiiglasses.create_calibration(project_id, participant_id)
print(tobiiglasses.get_battery_status())
input("Put the calibration marker in front of the user, then press enter to calibrate")
tobiiglasses.start_calibration(calibration_id)
res = tobiiglasses.wait_until_calibration_is_done(calibration_id)
if res is False:
print("Calibration failed!")
exit(1)
tobiiglasses.start_streaming()
eye_data = []
max_len = 1000
min_len = 100
total_fixations = 0
total_saccades = 0
num_fixations_in_window = 0 #number of fixations in the last window
AOIfixations = [0] * 4 #screen1, screen2, screen3, screen4
AOIsaccades = [0] * 4
num_saccades_in_window = 0
AOIblinks = [0] * 4
vel_threshold = 100 #45 # degrees per second
# discard_short_fixations
# merge_close_fixations
test_time = 60 * 3
camera_rate = 30
num_apriltags = 15
cap = cv2.VideoCapture(1)
cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*"MJPG"))
cap.set(cv2.CAP_PROP_FPS, camera_rate)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1920) # set the resolution
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 1080)
if cap.isOpened():
print("Succesfully connected to webcam...")
else:
print("Could not connect to camera...")
tobiiglasses.stop_streaming()
sys.exit(1)
background = cv2.imread('background36h11.jpg')
background = cv2.resize(background, (640, 360))
gray_back = cv2.cvtColor(background, cv2.COLOR_RGB2GRAY)
# detector = Detector(families='tag25h9', nthreads=1, quad_decimate=1.0)
detector = Detector(families='tag36h11', nthreads=1, quad_decimate=1.0)
back_detections = detector.detect(gray_back)
if len(back_detections) < num_apriltags:
print("Could not find all tags in background image, only found:", len(back_detections))
tobiiglasses.stop_streaming()
sys.exit(1)
fix = {}
#######IDS NEED TO CHANGE#############
for detection in back_detections:
id_ = detection.tag_id
center = detection.center
if id_ == 8:
s1_min_x = center[0]
elif id_ == 11:
s1_min_y = center[1]
elif id_ == 14:
s1_max_y = center[1]
elif id_ == 12:
s1_max_x = center[0]
elif id_ == 0:
s2_min_x = center[0]
s2_max_y = center[1]
elif id_ == 1:
s2_min_y = center[1]
elif id_ == 3:
s2_max_x = center[0]
elif id_ == 4:
s3_min_x = center[0]
elif id_ == 5:
s3_min_y = center[1]
elif id_ == 7:
s3_max_x = center[0]
s3_max_y = center[1]
corners = detection.corners
point_list = list(corners) + [center]
fix[id_] = point_list
H = np.eye(3)
Ex = np.eye(4)
P2 = np.dot(config.K2, Ex)
P2 = np.dot(P2, config.Rigid_body)
last_ts = 0
start = time.time()
while True:
comp_start_time = time.time()
data = tobiiglasses.get_data()
if len(eye_data) > max_len:
eye_data.pop(0)
# 3d gaze coordinate data
gaze_data = data['gp3']
# timestamp
ts = gaze_data['ts']
if 'gp3' in gaze_data.keys():
# 3d gaze coordinate
gaze3 = gaze_data['gp3']
else:
gaze3 = [-1] * 3
if len(gaze3) < 4: # Homogenizing
gaze3.append(1)
# print("gaze3 in main", gaze3)
# gaze3 = np.array(gaze3)
# pupil diameter
if 'pd' in data['left_eye']['pd'].keys():
pd_left = data['left_eye']['pd']['pd']
else:
pd_left = -1
if 'pd' in data['right_eye']['pd'].keys():
pd_right = data['right_eye']['pd']['pd']
else:
pd_right = -1
# Right eye pupil postion
Pc_right = data['right_eye']['pc']
if 'pc' in Pc_right.keys():
pc_right = Pc_right['pc']
else:
pc_right = [-1] * 3
_, frame = cap.read()
gray_frame = cv2.resize(frame, (640, 360))
gray_frame = cv2.cvtColor(gray_frame, cv2.COLOR_BGR2GRAY)
result = detector.detect(gray_frame)
if not(gaze3[0] == -1 and gaze3[1] == -1 and gaze3[2] == -1 or pd_left == -1 or pd_right == -1):
gaze3[0] = -gaze3[0]
gaze3[1] = -gaze3[1]
gaze_screen = -1
gaze_mapped = np.dot(P2, gaze3) ## 3X1
gaze_mapped = gaze_mapped/gaze_mapped[2]
gaze_mapped[0] = gaze_mapped[0] / 1920*640 ## TODO: Need to check 1920 -> 640
gaze_mapped[1] = gaze_mapped[1] / 1080*360
H = get_hom(result, fix)
if H is not None:
gaze_mapped = np.dot(H, gaze_mapped)
gaze_mapped = gaze_mapped/gaze_mapped[2]
# cv2.circle(bg_copy, (int(gaze_mapped_back[0]), int(gaze_mapped_back[1])), 2, (0,0,255), 40)
if gaze_mapped[0] > s1_min_x and gaze_mapped[1] > s1_min_y and gaze_mapped[0] < s1_max_x and gaze_mapped[1] < s1_max_y:
## LOOKING AT SCREEN1######
print("Looking at screen 1", end='\r')
gaze_screen = 1
elif gaze_mapped[0] > s2_min_x and gaze_mapped[1] > s2_min_y and gaze_mapped[0] < s2_max_x and gaze_mapped[1] < s2_max_y: # and gaze_mapped[1] > 180:
# LOOKING AT SCREEN2 ######
print("Looking at screen 2", end='\r')
gaze_screen = 2
elif gaze_mapped[0] > s3_min_x and gaze_mapped[1] > s3_min_y and gaze_mapped[0] < s3_max_x and gaze_mapped[1] < s3_max_y:
# LOOKING AT SCREEN 3 ###
print("Looking at screen 3", end='\r')
gaze_screen = 3
else:
print("Looking at screen 0", end='\r')
gaze_screen = -1
if ts not in eye_data:
eye_data.append([ts, gaze3, pc_right, gaze_mapped, gaze_screen])
if len(eye_data) < min_len:
continue
ret_data = computeFixations(eye_data,
vel_threshold,
total_fixations,
total_saccades,
num_fixations_in_window,
AOIfixations,
AOIsaccades,
num_saccades_in_window)
AOIfixations = ret_data[0]
AOIsaccades = ret_data[1]
total_fixations = ret_data[2]
total_saccades = ret_data[3]
num_fixations_in_window = ret_data[4]
num_saccades_in_window = ret_data[5]
if last_ts == ts and gaze_screen > 0:
AOIblinks[gaze_screen - 1] += 1
last_ts = ts
elapsed_one_iter = time.time() - comp_start_time
if elapsed_one_iter < camera_rate * 1e-3:
time.sleep(camera_rate * 1e-3 - elapsed_one_iter)
# else:
# print("Took %2.2f s longer for computations"%(elapsed_one_iter - camera_rate * 1e-3))
elapsed = time.time() - start
if elapsed > test_time:
print()
print()
break
tobiiglasses.stop_streaming()
return total_fixations, total_saccades, AOIfixations, AOIsaccades, AOIblinks
def get_hom(result, fix):
src_pts = np.empty((0, 2))
dst_pts = np.empty((0, 2))
H = None
for detection in result:
if detection.tag_id in fix:
center = detection.center
corners = detection.corners
point_list = list(corners) + [center]
src_pts = np.append(src_pts, point_list, axis=0)
dst_pts = np.append(dst_pts, np.array(fix[detection.tag_id]), axis=0)
if src_pts.shape[0] > 0:
H, _ = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0)
return H
def increment_fixations(data, AOIfixations):
fix_num = data[1][0]
i = 0
length = len(data[1])
while i < length:
screen = -1
single_fix_data = []
while i < length and data[1][i] == fix_num:
single_fix_data.append(data[0][i])
i += 1
if len(single_fix_data) != 0:
# print(single_fix_data[-1])
screen = single_fix_data[-1][-1]
# print("screen", screen)
# print("Screen, fix_num", screen, fix_num)
fix_num += 1
i += 1
if screen > 0:
AOIfixations[screen - 1] += 1
# print("AOI fixations", AOIfixations)
return AOIfixations
def cross(v1, v2):
p1 = v1[1] * v2[2] - v1[2] * v2[1]
p2 = v1[2] * v2[0] - v1[0] * v2[2]
p3 = v1[0] * v2[1] - v1[1] * v2[0]
return math.sqrt(p1**2 + p2**2 + p3**2)
def dot(v1, v2):
dot_prod = 0
for i in range(3):
dot_prod += v1[i] * v2[i]
return dot_prod
def calculateVelocity(ts1, gp3_1, ts2, gp3_2, pc_right_1, pc_right_2):
# print("gp3_1", gp3_1)
# print("gp3_2", gp3_2)
# print("ts1", ts1)
# print("ts2", ts2)
v1 = np.array(gp3_1) - np.array(pc_right_1)
v2 = np.array(gp3_2) - np.array(pc_right_2)
cross_prod = cross(v1, v2)
dot_prod = dot(v1, v2)
angle = math.atan2(cross_prod, dot_prod) * 180 / math.pi
vel = angle / math.fabs(ts1 - ts2) * 1e6
# print("vel", vel)
return vel
def increment_saccades(data, AOIsaccades):
sac_num = data[1][0]
i = 0
length = len(data[1])
while i < length:
screen = -1
single_sac_data = []
while i < length and data[1][i] == sac_num:
single_sac_data.append(data[0][i])
i += 1
if len(single_sac_data) != 0:
# print(single_fix_data[-1])
screen = single_sac_data[-1][-1]
# print("screen", screen)
# print("Screen, fix_num", screen, fix_num)
sac_num += 1
i += 1
if screen > 0:
AOIsaccades[screen - 1] += 1
# print("AOI fixations", AOIfixations)
return AOIsaccades
def computeFixations(eye_data, vel_threshold, total_fixations, total_saccades, num_fixations_in_window, AOIfixations, AOIsaccades, num_saccades_in_window):
fix_num = 1
sac_num = 1
fix_num_changed = True
sac_num_changed = True
fixations = [0] * len(eye_data)
saccades = [0] * len(eye_data)
for i in range(1, len(eye_data)):
ts1 = eye_data[i - 1][0]
gp3_1 = eye_data[i - 1][1][:3]
pc_right_1 = eye_data[i - 1][2]
ts2 = eye_data[i][0]
gp3_2 = eye_data[i][1][:3]
pc_right_2 = eye_data[i][2]
invalid_data_1 = pc_right_1 == [-1, -1, -1]
invalid_data_2 = pc_right_2 == [-1, -1, -1]
if ts1 == ts2 or invalid_data_1 or invalid_data_2:
continue
vel = calculateVelocity(ts1, gp3_1, ts2, gp3_2, pc_right_1, pc_right_2)
if vel < vel_threshold:
fix_num_changed = False
fixations[i - 1] = fix_num
fixations[i] = fix_num
elif not fix_num_changed:
fix_num += 1
fix_num_changed = True
if vel >= vel_threshold:
sac_num_changed = False
saccades[i - 1] = sac_num
saccades[i] = sac_num
elif not sac_num_changed:
sac_num += 1
sac_num_changed = True
# if num_fixations_in_window == fix_num:
# do nothing
if num_fixations_in_window < max(fixations):
# map all the gaze coordinates above the num_fix_in_window
# update num_fix_in_window (to return)
# increment the fixation numbers appropriately
# increment total_fixations by the difference fix_num - num_fixations_in_window
# try:
idx = fixations.index(num_fixations_in_window + 1)
mapped_data = [eye_data[idx:], fixations[idx:]]
AOIfixations = increment_fixations(mapped_data, AOIfixations)
total_fixations = sum(AOIfixations)
num_fixations_in_window = max(fixations)
# except:
# pass
if num_saccades_in_window < max(saccades):
idx = saccades.index(num_saccades_in_window + 1)
mapped_data = [eye_data[idx:], saccades[idx:]]
AOIsaccades = increment_saccades(mapped_data, AOIsaccades)
total_saccades = sum(AOIsaccades)
num_saccades_in_window = max(saccades)
return AOIfixations, AOIsaccades, total_fixations, total_saccades, num_fixations_in_window, num_saccades_in_window
if __name__ == "__main__":
ret_data = main()
total_fixations = ret_data[0]
total_saccades = ret_data[1]
AOIfixations = ret_data[2]
AOIsaccades = ret_data[3]
AOIblinks = ret_data[4]
print()
print()
print("Total number of fixations", total_fixations)
print("Fixations in each AOI", AOIfixations)
print("Total number of saccades", total_saccades)
print("Saccades in each AOI", AOIsaccades)
print("Blinks in each AOI", AOIblinks)
gazeMappingUbuntu.py deleted 100644 → 0
+ 0
− 223
View file @ fd73ecb3
#! /usr/bin/python3
from apriltag import apriltag
import cv2
import sys
import numpy as np
from tobiiglassesctrl import TobiiGlassesController
def get_hom(result, fix):
src_pts = np.empty((0, 2))
dst_pts = np.empty((0, 2))
H = None
for detection in result:
if detection['id'] in fix:
center = detection['center']
corners = detection['lb-rb-rt-lt']
point_list = list(corners) + [center]
src_pts = np.append(src_pts, point_list, axis=0)
dst_pts = np.append(dst_pts, np.array(fix[detection['id']]), axis=0)
if src_pts.shape[0] > 0:
H, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0)
return H
ip = '192.168.71.50'
tobiiglasses = TobiiGlassesController(ip, video_scene=True)
project_id = tobiiglasses.create_project("Test live_scene_and_gaze.py")
participant_id = tobiiglasses.create_participant(project_id, "participant_test")
calibration_id = tobiiglasses.create_calibration(project_id, participant_id)
input("Put the calibration marker in front of the user, then press enter to calibrate")
tobiiglasses.start_calibration(calibration_id)
res = tobiiglasses.wait_until_calibration_is_done(calibration_id)
if res is False:
print("Calibration failed!")
exit(1)
tobiiglasses.start_streaming()
eye_camera_id = "usb-046d_HD_Webcam_C615_29CD8CC0-video-index0"
K1 = [[1146.8119767102853, 0, 995.8472226069734, 0],
[0, 1150.968615728085, 560.9054428070731, 0],
[0, 0, 1, 0]]
K2 = [[1478.0057732719538, 0, 964.3897381426681, 0],
[0, 1478.2826857743617, 559.6902650031882, 0],
[0, 0, 1, 0]]
Rigid_body = [[0.9998443871765447, 0.016930978475685435, 0.004954129531717574, 0.0014797681252487246],
[-0.017536129544264166, 0.9844701379081924, 0.1746740728546538, 0.017710310592413483],
[-0.0019197896155425946, -0.1747337675862396, 0.9846138455622852, -0.017384735431967623],
[0., 0., 0., 1.]]
num_apriltags = 7
screen1_ids = [32, 31, 3]
screen2_ids = [33, 30]
screen3_ids = [9, 2]
try:
cap = cv2.VideoCapture('/dev/v4l/by-id/' + eye_camera_id)
except:
print("Could not connect to webcam")
sys.exit(1)
background = cv2.imread('background.jpg')
background = cv2.resize(background, (640,480))
gray_back = cv2.cvtColor(background, cv2.COLOR_RGB2GRAY)
detector = apriltag('tag25h9')
back_detections = detector.detect(gray_back)
# print(back_detections)
screen1 = cv2.imread('screen1.jpg')
screen1 = cv2.resize(screen1, (640,480))
screen2 = cv2.imread('screen2.jpg')
screen2 = cv2.resize(screen2, (640,480))
screen3 = cv2.imread('screen3.jpg')
screen3 = cv2.resize(screen3, (640,480))
s1_grey = cv2.cvtColor(screen1, cv2.COLOR_RGB2GRAY)
s2_grey = cv2.cvtColor(screen2, cv2.COLOR_RGB2GRAY)
s3_grey = cv2.cvtColor(screen3, cv2.COLOR_RGB2GRAY)
s1_detections = detector.detect(s1_grey)
fix_s1 = {}
for detection in s1_detections:
id_ = detection['id']
center = detection['center']
corners = detection['lb-rb-rt-lt']
point_list = list(corners) + [center]
fix_s1[id_] = point_list
s2_detections = detector.detect(s2_grey)
fix_s2 = {}
for detection in s2_detections:
id_ = detection['id']
center = detection['center']
corners = detection['lb-rb-rt-lt']
point_list = list(corners) + [center]
fix_s2[id_] = point_list
s3_detections = detector.detect(s3_grey)
fix_s3 = {}
for detection in s3_detections:
id_ = detection['id']
center = detection['center']
corners = detection['lb-rb-rt-lt']
point_list = list(corners) + [center]
fix_s3[id_] = point_list
if len(back_detections) < num_apriltags:
print("Could not find all tags in background image, only found:", len(back_detections))
sys.exit(1)
fix = {}
for detection in back_detections:
id_ = detection['id']
center = detection['center']
if id_ == 32:
s1_min = center[0]
elif id_ == 3:
s1_max = center[0]
elif id_ == 33:
s2_min = center[0]
elif id_ == 30:
s2_max = center[0]
elif id_ == 9:
s3_min = center[0]
elif id_ == 2:
s3_max = center[0]
corners = detection['lb-rb-rt-lt']
point_list = list(corners) + [center]
fix[id_] = point_list
Ex = np.eye(4)
P2 = np.dot(K2, Ex)
P2 = np.dot(P2, Rigid_body)
writer = cv2.VideoWriter("output_tobii_1.avi", cv2.VideoWriter_fourcc(*"MJPG"), 30, (640, 480))
writer2 = cv2.VideoWriter("output_tobii_1_2.avi", cv2.VideoWriter_fourcc(*"MJPG"), 30, (640, 480))
while True:
bg_copy = background.copy()
s1_copy = screen1.copy()
s2_copy = screen2.copy()
s3_copy = screen3.copy()
ret, frame = cap.read()
cv2.imshow("frame", frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
gray_frame = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)
result = detector.detect(gray_frame)
data = tobiiglasses.get_data()
gaze3 = []
gaze3 = data['gp3']
# gaze2 = data['gp']
# gaze2 = gaze2['gp']
# print(gaze3)
if 'gp3' in gaze3:
gaze3 = gaze3['gp3']
else:
continue
if len(gaze3) < 4:
gaze3.append(1)
# gaze2 = np.array(gaze2)
gaze3 = np.array(gaze3)
# print(gaze2)
if not(gaze3[0] == -1 and gaze3[1] == -1 and gaze3[2] == -1): #or pd_left == -1 or pd_right == -1):
gaze3[0] = -gaze3[0]
gaze3[1] = -gaze3[1]
gaze_mapped = np.dot(P2, gaze3) ## 3X1
gaze_mapped = gaze_mapped/gaze_mapped[2]
gaze_mapped[0] = gaze_mapped[0] / 1920*640 ## TODO: Need to check 1920 -> 640
gaze_mapped[1] = gaze_mapped[1] / 1080*480
H = get_hom(result, fix)
if H is not None:
gaze_mapped_back = np.dot(H, gaze_mapped)
gaze_mapped_back = gaze_mapped_back/gaze_mapped_back[2]
# print(gaze_mapped)
cv2.circle(bg_copy, (int(gaze_mapped_back[0]), int(gaze_mapped_back[1])), 2, (0,0,255), 40)
# cv2.circle(frame_copy, (int(gaze2[0]*640), int(gaze2[1]*480)), 2, (0,0,255), 40)
if gaze_mapped_back[0] > s1_min and gaze_mapped_back[0] < s1_max:
H1 = get_hom(result, fix_s1)
if H1 is not None:
gaze_mapped_s1 = np.dot(H1, gaze_mapped)
gaze_mapped_s1 = gaze_mapped_s1/gaze_mapped_s1[2]
# print(gaze_mapped_s1)
cv2.circle(s1_copy, (int(gaze_mapped_s1[0]), int(gaze_mapped_s1[1])), 2, (0,0,255), 40)
elif gaze_mapped_back[0] > s2_min and gaze_mapped_back[0] < s2_max and gaze_mapped_back[1] < 240:
H2 = get_hom(result, fix_s2)
if H2 is not None:
gaze_mapped_s2 = np.dot(H2, gaze_mapped)
gaze_mapped_s2 = gaze_mapped_s2/gaze_mapped_s2[2]
# print(gaze_mapped_s2)
cv2.circle(s2_copy, (int(gaze_mapped_s2[0]), int(gaze_mapped_s2[1])), 2, (0,0,255), 40)
elif gaze_mapped_back[0] > s3_min and gaze_mapped_back[0] < s3_max and gaze_mapped_back[1] > 240:
H3 = get_hom(result, fix_s3)
if H3 is not None:
gaze_mapped_s3 = np.dot(H3, gaze_mapped)
gaze_mapped_s3 = gaze_mapped_s3/gaze_mapped_s3[2]
# print(gaze_mapped_s3)
cv2.circle(s3_copy, (int(gaze_mapped_s3[0]), int(gaze_mapped_s3[1])), 2, (0,0,255), 40)
cv2.imshow('mapped gaze', bg_copy)
cv2.imshow("screen1", s1_copy)
cv2.imshow("screen2", s2_copy)
cv2.imshow("screen3", s3_copy)
# writer.write(bg_copy)
# writer2.write(frame_copy)
# print(gaze3)
gazeMappingWindows.py deleted 100644 → 0
+ 0
− 290
View file @ fd73ecb3
from pupil_apriltags import Detector
import cv2
import sys
import numpy as np
from tobiiglassesctrl import TobiiGlassesController
import matplotlib.pyplot as plt
import time
import os
def get_hom(result, fix):
src_pts = np.empty((0, 2))
dst_pts = np.empty((0, 2))
H = None
for detection in result:
if detection.tag_id in fix:
center = detection.center
corners = detection.corners
point_list = list(corners) + [center]
src_pts = np.append(src_pts, point_list, axis=0)
dst_pts = np.append(dst_pts, np.array(fix[detection.tag_id]), axis=0)
if src_pts.shape[0] > 0:
H, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0)
return H
def get_dict_from_image(img, detector):
fix = {}
detections = detector.detect(img)
for detection in detections:
id_ = detection.tag_id
center = detection.center
corners = detection.corners
point_list = list(corners) + [center]
fix[id_] = point_list
return fix
def get_center(img, detector, tag_id):
detections = detector.detect(img)
found = False
for detection in detections:
if detection.tag_id == tag_id:
found = True
return detection.center
if not found:
print("could not find tag with id:", tag_id)
sys.exit(1)
ip = '192.168.71.50'
tobiiglasses = TobiiGlassesController(ip, video_scene=True)
project_id = tobiiglasses.create_project("Test live_scene_and_gaze.py")
participant_id = tobiiglasses.create_participant(project_id, "participant_test")
calibration_id = tobiiglasses.create_calibration(project_id, participant_id)
input("Put the calibration marker in front of the user, then press enter to calibrate")
tobiiglasses.start_calibration(calibration_id)
res = tobiiglasses.wait_until_calibration_is_done(calibration_id)
if res is False:
print("Calibration failed!")
exit(1)
tobiiglasses.start_streaming()
eye_camera_id = "usb-046d_HD_Webcam_C615_29CD8CC0-video-index0"
K1 = [[1146.8119767102853, 0, 995.8472226069734, 0],
[0, 1150.968615728085, 560.9054428070731, 0],
[0, 0, 1, 0]]
K2 = [[1478.0057732719538, 0, 964.3897381426681, 0],
[0, 1478.2826857743617, 559.6902650031882, 0],
[0, 0, 1, 0]]
Rigid_body = [[0.9998443871765447, 0.016930978475685435, 0.004954129531717574, 0.0014797681252487246],
[-0.017536129544264166, 0.9844701379081924, 0.1746740728546538, 0.017710310592413483],
[-0.0019197896155425946, -0.1747337675862396, 0.9846138455622852, -0.017384735431967623],
[0., 0., 0., 1.]]
num_apriltags = 12
# screen1_ids = [32, 31, 3, 5, 10, 7]
# screen2_ids = [33, 30, 17, 6]
# screen3_ids = [9, 2]
print("before webcam")
cap = cv2.VideoCapture(0)
if cap.isOpened():
print("Succesfully connected to webcam...")
else:
print("Could not connect to camera...")
tobiiglasses.stop_streaming()
sys.exit(1)
background = cv2.imread('background.jpg')
background = cv2.resize(background, (640,480))
gray_back = cv2.cvtColor(background, cv2.COLOR_RGB2GRAY)
detector = Detector(families='tag25h9', nthreads=1, quad_decimate=1.0)
back_detections = detector.detect(gray_back)
# print(back_detections)
if len(back_detections) < num_apriltags:
print("Could not find all tags in background image, only found:", len(back_detections))
tobiiglasses.stop_streaming()
sys.exit(1)
screen1 = cv2.imread('screen1.jpg')
screen1 = cv2.resize(screen1, (640,480))
screen2 = cv2.imread('screen2.jpg')
screen2 = cv2.resize(screen2, (640,480))
screen3 = cv2.imread('screen3.jpg')
screen3 = cv2.resize(screen3, (640,480))
s1_grey = cv2.cvtColor(screen1, cv2.COLOR_RGB2GRAY)
s2_grey = cv2.cvtColor(screen2, cv2.COLOR_RGB2GRAY)
s3_grey = cv2.cvtColor(screen3, cv2.COLOR_RGB2GRAY)
fix_s1 = get_dict_from_image(s1_grey, detector)
fix_s2 = get_dict_from_image(s2_grey, detector)
fix_s3 = get_dict_from_image(s3_grey, detector)
# error_measure_tag = 18
# gaze_center = fix_s3[error_measure_tag][-1]
# print(gaze_center)
# del fix_s3[error_measure_tag]
# err_tag_id = 3
# detector_2 = Detector(families="tag36h11", nthreads=1, quad_decimate=1.0)
# err_center = get_center(s1_grey, detector_2, err_tag_id)
fix = {}
for detection in back_detections:
id_ = detection.tag_id
center = detection.center
if id_ == 32:
s1_min = center[0]
elif id_ == 11:
s1_max = center[0]
elif id_ == 33:
s2_min = center[0]
elif id_ == 30:
s2_max = center[0]
elif id_ == 9:
s3_min = center[0]
elif id_ == 2:
s3_max = center[0]
corners = detection.corners
point_list = list(corners) + [center]
fix[id_] = point_list
Ex = np.eye(4)
P2 = np.dot(K2, Ex)
P2 = np.dot(P2, Rigid_body)
err = []
# writer = cv2.VideoWriter("output_tobii_1.avi", cv2.VideoWriter_fourcc(*"MJPG"), 30, (640, 480))
# writer2 = cv2.VideoWriter("output_tobii_1_2.avi", cv2.VideoWriter_fourcc(*"MJPG"), 30, (640, 480))
to_quit = False
count = 0
time_taken = []
total_time = 0
screen1_time = 0
# start_while = time.time()
total_elapsed = 0
while True:
start = time.time()
if to_quit or count > 1e3:
break
ret, frame = cap.read()
# start = time()
# bg_copy = background.copy()
# s1_copy = screen1.copy()
# s2_copy = screen2.copy()
# s3_copy = screen3.copy()
# cv2.imshow("frame", frame)
gray_frame = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)
result = detector.detect(gray_frame)
data = tobiiglasses.get_data()
gaze3 = []
gaze3 = data['gp3']
if 'gp3' in gaze3:
gaze3 = gaze3['gp3']
else:
continue
if len(gaze3) < 4:
gaze3.append(1)
gaze3 = np.array(gaze3)
pd_left = data['left_eye']['pd']['pd']
pd_right = data['right_eye']['pd']['pd']
# print(pd_left, pd_right)
if not(gaze3[0] == -1 and gaze3[1] == -1 and gaze3[2] == -1 or pd_left == -1 or pd_right == -1):
average_pd = 0.5 * (pd_right+pd_left)
gaze3[0] = -gaze3[0]
gaze3[1] = -gaze3[1]
gaze_mapped = np.dot(P2, gaze3) ## 3X1
gaze_mapped = gaze_mapped/gaze_mapped[2]
gaze_mapped[0] = gaze_mapped[0] / 1920*640 ## TODO: Need to check 1920 -> 640
gaze_mapped[1] = gaze_mapped[1] / 1080*480
H = get_hom(result, fix)
if H is not None:
gaze_mapped_back = np.dot(H, gaze_mapped)
gaze_mapped_back = gaze_mapped_back/gaze_mapped_back[2]
# cv2.circle(bg_copy, (int(gaze_mapped_back[0]), int(gaze_mapped_back[1])), 2, (0,0,255), 40)
if gaze_mapped_back[0] > s1_min and gaze_mapped_back[0] < s1_max:
H1 = get_hom(result, fix_s1)
# print("Looking at screen 1", end='\r')
if H1 is not None:
gaze_mapped_s1 = np.dot(H1, gaze_mapped)
gaze_mapped_s1 = gaze_mapped_s1/gaze_mapped_s1[2]
# cv2.circle(s1_copy, (int(gaze_mapped_s1[0]), int(gaze_mapped_s1[1])), 2, (0,0,255), 40)
screen1_time += 1
# print(gaze_mapped, gaze_center)
# vec = (gaze_mapped_s1[:2] - err_center)
# print(vec)
# curr_err = np.sqrt(vec)
# err.append(vec)
elif gaze_mapped_back[0] > s2_min and gaze_mapped_back[0] < s2_max and gaze_mapped_back[1] < 240:
H2 = get_hom(result, fix_s2)
# print("Looking at screen 2", end='\r')
if H2 is not None:
gaze_mapped_s2 = np.dot(H2, gaze_mapped)
gaze_mapped_s2 = gaze_mapped_s2/gaze_mapped_s2[2]
# cv2.circle(s2_copy, (int(gaze_mapped_s2[0]), int(gaze_mapped_s2[1])), 2, (0,0,255), 40)
elif gaze_mapped_back[0] > s3_min and gaze_mapped_back[0] < s3_max and gaze_mapped_back[1] > 240:
H3 = get_hom(result, fix_s3)
# print("Looking at screen 3", end='\r')
if H3 is not None:
gaze_mapped_s3 = np.dot(H3, gaze_mapped)
gaze_mapped_s3 = gaze_mapped_s3/gaze_mapped_s3[2]
# cv2.circle(s3_copy, (int(gaze_mapped_s3[0]), int(gaze_mapped_s3[1])), 2, (0,0,255), 40)
# cv2.imshow('mapped gaze', bg_copy)
# cv2.imshow("screen1", s1_copy)
# cv2.imshow("screen2", s2_copy)
# cv2.imshow("screen3", s3_copy)
# end = time()
# elapsed = end - start
# count += 1
total_time += 1
time_ratio = screen1_time / total_time
# time_taken.append(elapsed)
# print("Time for 1 iteration %1.3f" %elapsed, end='\r')
# if cv2.waitKey(1) & 0xFF == ord('q'):
# to_quit = True
# break
elapsed = time.time() - start
total_elapsed += elapsed
# print(elapsed)
if total_elapsed > 10:
# print("Too slow")
# print(elapsed)
break
time_taken.append(elapsed)
# time.sleep(0.1 - elapsed)
time_taken = np.array(time_taken)
# avg_time_per_iteration = np.sum(time_taken) / count
# print("Average time per iteration", avg_time_per_iteration * 1e3, "millisec")
time_ratio = screen1_time / total_time
print()
print("Time ratio", time_ratio)
print()
print()
# dir_name = "error_data/Screen1/multipe/pos" + str(err_tag_id)
# if not os.path.exists(dir_name):
# os.makedirs(dir_name)
# np.save(dir_name + "/trial1", err)
np.save("time_data", time_taken)
tobiiglasses.stop_streaming()
cap.release()
cv2.destroyAllWindows()
map_and_send.py
+ 11
− 4
View file @ d1c05339
"""Code file to map the gaze coordinates to a background image of the setup using apriltags
It then sends this data over to iMotions via UDP (you may need to change the ip) and add the corresponding xml file in iMotions external events API tab
You may need to change the background image and define the bounding boxes for the AOIs and the number of apriltags variable"""
from pupil_apriltags import Detector
from cv2 import cv2
import sys
......@@ -13,7 +17,7 @@ tobii_ip = '192.168.71.50'
tobiiglasses = TobiiGlassesController(tobii_ip, video_scene=False)
def main():
"""The main function"""
# iMotions connection settings
iMotions_ip = '192.168.71.60'
port = 8125
......@@ -32,9 +36,8 @@ def main():
print("Calibration failed!")
exit(1)
camera_rate = 30
num_apriltags = 11 # Number of apriltags in the background image
test_time = 60
camera_rate = 30 # fps
num_apriltags = 15 # Number of apriltags in the background image
cap = cv2.VideoCapture(0)
cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*"MJPG"))
......@@ -227,6 +230,8 @@ def main():
tobiiglasses.stop_streaming()
def generate_message(ts, gaze3, gaze_mapped, pd, gaze_screen, pc_left, pc_right):
"""Generates the utf-8 message to be sent to iMotions via UDP. The format is as specified in the iMotions programmer's guide"""
msg = 'E;1;EyeTrack;;;;;EyeTrackerSample;'
msg += str(ts) + ';'
......@@ -251,6 +256,7 @@ def generate_message(ts, gaze3, gaze_mapped, pd, gaze_screen, pc_left, pc_right)
return msg
def get_hom(result, fix):
"""Helper function to find the homography between two images based on the apriltag locations"""
src_pts = np.empty((0, 2))
dst_pts = np.empty((0, 2))
......@@ -268,6 +274,7 @@ def get_hom(result, fix):
return H
def handler(signal_received, frame):
"""A handler to cleanly exit if ctrl-c is detected"""
tobiiglasses.stop_streaming()
print("SIGINT or CTRL-C detected. Exiting...")
exit(0)
......
readme.md 0 → 100644
+ 10
− 0
View file @ d1c05339
# Description
The main file to run is map_and_send.py
This file maps the 3d coordinates given by Tobii Glasses 2 eye-tracker to a background image of the setup.
It then computes the AOI that the person is looking at
Changes to be made for the driving simulator setup:
- Define the AOIs
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment