From e0215c4b872c65f0fac469b861c561a8ab783581 Mon Sep 17 00:00:00 2001 From: jadeddelta <101148768+jadeddelta@users.noreply.github.com> Date: Wed, 29 Jun 2022 12:47:44 -0400 Subject: [PATCH 01/10] initial commit --- examples/example-components.html | 125 +++++++++++++++++++++++++++++++ examples/pseudo-components.html | 33 ++++++++ src/Components.ts | 81 ++++++++++++++++++++ 3 files changed, 239 insertions(+) create mode 100644 examples/example-components.html create mode 100644 examples/pseudo-components.html create mode 100644 src/Components.ts diff --git a/examples/example-components.html b/examples/example-components.html new file mode 100644 index 0000000..8b21dd9 --- /dev/null +++ b/examples/example-components.html @@ -0,0 +1,125 @@ + + + + Test Camera + + + + + + + diff --git a/examples/pseudo-components.html b/examples/pseudo-components.html new file mode 100644 index 0000000..5ae326f --- /dev/null +++ b/examples/pseudo-components.html @@ -0,0 +1,33 @@ + + + + + + + + diff --git a/src/Components.ts b/src/Components.ts new file mode 100644 index 0000000..2c45923 --- /dev/null +++ b/src/Components.ts @@ -0,0 +1,81 @@ +import { Eyetracker } from "./Eyetracker"; + +export class Components { + private Eyetracker: Eyetracker | undefined; + private currentComponents: Array = []; + + constructor(et: Eyetracker) { + this.Eyetracker = et; + } + + init() { + // is this even necessary? + } + + //TODO: figure out how to put CSS in here shorthand without giving me a stroke + createLanding(id: string, message: string): HTMLDivElement { + this.currentComponents.push(id); + if (message === undefined) { + message = `Welcome to an experiment that uses eye tracking. + In a few moments you will be asked for permission to use your camera in order to complete the experiment.`; + } + let landing = document.createElement("div"); + landing.id = id; + landing.classList.add("landing"); + landing.innerHTML = ` +
+

${message}

+
+ `; + return landing; + } + + // defaults to true: will bypass if there is only one camera + // TODO: what type is the return???? + async generateSelector( + id: string, + message: string, + bypass: boolean + ): Promise { + this.currentComponents.push(id); + let selector = document.createElement("select"); + selector.id = id; + if (message === undefined) message = "Select a camera"; + if (bypass === undefined) bypass = true; + + await this.Eyetracker!.getCameraPermission(); + const devices = await this.Eyetracker!.getListOfCameras(); + + const blank = document.createElement("option"); + blank.style.display = "none"; + selector.appendChild(blank); + + devices.forEach((d) => { + let option = document.createElement("option"); + option.value = d.deviceId; + option.innerHTML = d.label; + selector.appendChild(option); + }); + + const btn = document.createElement("button"); + btn.id = `${id}-btn`; + btn.innerHTML = `${message}`; + btn.addEventListener("click", async () => { + const cam = selector.options[selector.selectedIndex].value; + if (id !== "") { + await this.Eyetracker!.setCamera( + //@ts-ignore + await navigator.mediaDevices.getUserMedia({ + video: { deviceId: cam }, + }) + ); + } else { + alert("Please select a camera."); + } + }); + + // but like why tho + //@ts-ignore + return new Array(selector, btn); + } +} From 4dbd4eb76d4e8445e31371bf5b5d5ee2021aa653 Mon Sep 17 00:00:00 2001 From: jadeddelta <101148768+jadeddelta@users.noreply.github.com> Date: Wed, 6 Jul 2022 16:09:34 -0400 Subject: [PATCH 02/10] typescript + calibration added --- examples/pseudo-components.html | 2 +- examples/test-components.html | 57 ++++++++++++ src/Components.ts | 156 ++++++++++++++++++++++++++++---- src/Eyetracker.ts | 78 +++++++++------- src/index.ts | 5 + 5 files changed, 242 insertions(+), 56 deletions(-) create mode 100644 examples/test-components.html diff --git a/examples/pseudo-components.html b/examples/pseudo-components.html index 5ae326f..62451f6 100644 --- a/examples/pseudo-components.html +++ b/examples/pseudo-components.html @@ -6,7 +6,7 @@ + + + + + diff --git a/src/Components.ts b/src/Components.ts index 2c45923..6832381 100644 --- a/src/Components.ts +++ b/src/Components.ts @@ -2,23 +2,43 @@ import { Eyetracker } from "./Eyetracker"; export class Components { private Eyetracker: Eyetracker | undefined; - private currentComponents: Array = []; + private currentComponents: Array = []; + private calDivUsed: HTMLDivElement | undefined; + + private DEFAULT_MESSAGE = `Welcome to an experiment that uses eye tracking. + In a few moments you will be asked for permission to use your camera in order to complete the experiment.`; + private DEFAULT_POINTS: Array> = [ + [10, 10], + [10, 50], + [10, 90], + [50, 10], + [50, 50], + [50, 90], + [90, 10], + [90, 50], + [90, 90], + ]; constructor(et: Eyetracker) { this.Eyetracker = et; } - init() { - // is this even necessary? + async preload(): Promise { + console.log("checK"); + let detector = await this.Eyetracker!.init(); + console.log("checK"); + let video = await this.Eyetracker!.createVideo(); + console.log("checK"); + let canvas = this.Eyetracker!.createDisplayCanvas(); + return { detector, video, canvas }; } //TODO: figure out how to put CSS in here shorthand without giving me a stroke - createLanding(id: string, message: string): HTMLDivElement { + createLanding( + id: string = "landing", + message: string = this.DEFAULT_MESSAGE + ): HTMLDivElement { this.currentComponents.push(id); - if (message === undefined) { - message = `Welcome to an experiment that uses eye tracking. - In a few moments you will be asked for permission to use your camera in order to complete the experiment.`; - } let landing = document.createElement("div"); landing.id = id; landing.classList.add("landing"); @@ -30,18 +50,14 @@ export class Components { return landing; } - // defaults to true: will bypass if there is only one camera - // TODO: what type is the return???? - async generateSelector( - id: string, - message: string, - bypass: boolean - ): Promise { + // TODO: what type is the return???? Array + async createSelector( + id: string = "selector", + message: string = "Select a camera" + ): Promise> { this.currentComponents.push(id); let selector = document.createElement("select"); selector.id = id; - if (message === undefined) message = "Select a camera"; - if (bypass === undefined) bypass = true; await this.Eyetracker!.getCameraPermission(); const devices = await this.Eyetracker!.getListOfCameras(); @@ -59,6 +75,7 @@ export class Components { const btn = document.createElement("button"); btn.id = `${id}-btn`; + this.currentComponents.push(btn.id); btn.innerHTML = `${message}`; btn.addEventListener("click", async () => { const cam = selector.options[selector.selectedIndex].value; @@ -69,13 +86,112 @@ export class Components { video: { deviceId: cam }, }) ); + this.clearComponents(); } else { alert("Please select a camera."); } }); + return new Array(selector, btn); + } + //TODO: these instructions- + /* + DONE: Pass in a DIV with the calibration points being other Div elements + DONE: New coordinate system absolutely (%) on the new element + - Calibrate based off of that (let's adapt calibrate point to take absolute coords) + - Different fixation points??? + */ + async calibrate( + div: HTMLDivElement, + points: Array> = this.DEFAULT_POINTS + ): Promise> { + if (div === null) { + div = document.createElement("div"); + div.id = "cal-div"; + } else { + div.innerHTML = ""; + } + this.calDivUsed = div; + + let finishedPoints: Array = []; - // but like why tho - //@ts-ignore - return new Array(selector, btn); + if (points.length < 4) { + console.warn( + "There are not a lot of points to calibrate, consider adding more or using the DEFAULT_POINTS." + ); + } + await this.Eyetracker!.initVideoFrameLoop(); + + let calibrateLoop = setInterval(async () => { + div.innerHTML = ""; + let point = points.shift(); + if (point === undefined) { + clearInterval(calibrateLoop); + //TODO- what exactly should we return? + console.log(await this.Eyetracker!.processCalibrationPoints()); // facial landmarks + return finishedPoints; // imageData + onset time + } + this.drawFixation(point[0], point[1], div); + let onsetTime = performance.now(); + + setTimeout(async () => { + await this.Eyetracker!.detectFace(); + let currentPoint = this.Eyetracker!.calibratePoint( + point![0], + point![1] + ); + //TODO- let's find a way to prevent throwing this error, maybe explicitly defining? + //@ts-ignore + currentPoint.onsetTime = onsetTime; + finishedPoints.push(currentPoint); + }, 1500); + }, 3000); + console.warn("This shouldn't be accessible."); + return finishedPoints; + } + + clearComponents() { + this.currentComponents.forEach((c) => { + let el = document.getElementById(c); + if (el !== null) { + el.remove(); + } else { + console.log(`${c} not found`); + } + }); + this.currentComponents = []; + } + + // TODO- add different fixations: CROSS, FUNNY THING, SQUARE + drawFixation(x: number, y: number, div: HTMLDivElement) { + const circle = document.createElement("div"); + circle.style.width = "10px"; + circle.style.height = "10px"; + circle.style.borderRadius = "50%"; + circle.style.backgroundColor = "red"; + circle.style.position = "absolute"; + circle.style.left = `calc(${x}% - 5px)`; + circle.style.top = `calc(${y}% - 5px)`; + div.appendChild(circle); + } + + translateToStandard(points: Array): Array { + if (this.calDivUsed === null || this.calDivUsed === undefined) { + throw new Error( + "No calibration div was used, please use the calibrate function." + ); + } + let newPoints: Array = []; + points.forEach((p) => { + let newPoint = { + //@ts-ignore + x: p.x * this.calDivUsed!.offsetWidth, + //@ts-ignore + y: p.y * this.calDivUsed!.offsetWidth, + //@ts-ignore + onsetTime: p.onsetTime, + }; // TODO- we should really set a type to these points + newPoints.push(newPoint); + }); + return newPoints; } } diff --git a/src/Eyetracker.ts b/src/Eyetracker.ts index 101d4dc..046945c 100644 --- a/src/Eyetracker.ts +++ b/src/Eyetracker.ts @@ -101,11 +101,14 @@ export class Eyetracker { * @param Id An id attribute to which can be used to identify the video later * @returns An HTMLVideoElement whose source is the stream from the previously selected camera */ - async createVideo(Id: string): Promise { + async createVideo(Id: string = "et-video"): Promise { let video: HTMLVideoElement = document.createElement("video"); video.setAttribute("id", Id); video.style.transform = "scaleX(-1)"; - (video.srcObject as undefined | MediaProvider | null) = this.stream; + if (this.stream === null || this.stream === undefined) { + throw new Error("No camera stream detected, have you run setCamera()?"); + } + video.srcObject = this.stream; video.autoplay = true; return new Promise((resolve) => { video.onloadedmetadata = () => { @@ -122,28 +125,28 @@ export class Eyetracker { * @param Id An id attribute to which can be used to identify the video later * @returns A mirrored HTMLCanvasElement with matching proportions to the created video */ - createDisplayCanvas(Id: string): HTMLCanvasElement | undefined { + createDisplayCanvas(Id: string = "et-canvas"): HTMLCanvasElement | undefined { + if (this.video === null || this.video === undefined) { + throw new Error("No video stream detected, have you run createVideo()?"); + } + let canvas: HTMLCanvasElement = document.createElement("canvas"); canvas.setAttribute("id", Id); this.canvas = canvas; - if (this.video != null) { - canvas.height = this.video.height; - canvas.width = this.video.width; - this.canvas = canvas; - var ctx: CanvasRenderingContext2D | null = canvas.getContext("2d"); - if (ctx != null) { - ctx.translate(canvas.width, 0); - ctx.scale(-1, 1); - ctx.fillStyle = "green"; - (this.ctx as undefined | CanvasRenderingContext2D | null) = ctx; - return canvas; - } else { - console.log("canvas.getContext('2d') return null"); - return canvas; - } - } else { - console.log('Undefined Property "this.video"'); + canvas.height = this.video.height; + canvas.width = this.video.width; + this.canvas = canvas; + var ctx: CanvasRenderingContext2D | null = canvas.getContext("2d"); + if (ctx === null) { + throw new Error( + "Could not create canvas context, have you already declared ctx with a different type of context?" + ); } + ctx.translate(canvas.width, 0); + ctx.scale(-1, 1); + ctx.fillStyle = "green"; + this.ctx = ctx; + return canvas; } // Need to test this out @@ -152,21 +155,26 @@ export class Eyetracker { * @param canvas An existing canvas whose proprotions should match the video */ setDisplayCanvas(canvas: HTMLCanvasElement): void { - let video: HTMLVideoElement | undefined = this.video; + if (this.video === null || this.video === undefined) { + throw new Error("No video stream detected, have you run createVideo()?"); + } + if (this.canvas === null || this.canvas === undefined) { + throw new Error( + "No canvas detected, have you run createDisplayCanvas()?" + ); + } + + let video = this.video; this.canvas = canvas; - if (canvas != undefined && video != undefined) { - canvas.height = video.height; - canvas.width = video.width; - var ctx: CanvasRenderingContext2D | null = canvas.getContext("2d"); - if (ctx != null) { - ctx.translate(canvas.width, 0); - ctx.scale(-1, 1); - ctx.fillStyle = "green"; - } - (this.ctx as undefined | CanvasRenderingContext2D | null) = ctx; - } else { - console.log('/"this.canvas/", /"this.video/" Undefined'); + canvas.height = video.height; + canvas.width = video.width; + var ctx: CanvasRenderingContext2D | null = canvas.getContext("2d"); + if (ctx != null) { + ctx.translate(canvas.width, 0); + ctx.scale(-1, 1); + ctx.fillStyle = "green"; } + (this.ctx as undefined | CanvasRenderingContext2D | null) = ctx; } /** @@ -315,7 +323,7 @@ export class Eyetracker { return point; } - async processCalibrationPoints(): Promise { + async processCalibrationPoints(): Promise> { let processedPoints = []; for (let i = 0; i < this.calibrationPoints.length; i++) { let predictions = await this.model!.estimateFaces({ @@ -330,6 +338,7 @@ export class Eyetracker { }); } this.processedCalibrationPoints = processedPoints; + return processedPoints; } /** @@ -376,7 +385,6 @@ export class Eyetracker { canvas: HTMLCanvasElement = this.canvas! ) { let ctx = canvas.getContext("2d"); - let frames = this.frames; let Eyetracker = this; async function repeatDetection(now: DOMHighResTimeStamp, metadata: object) { if (!Eyetracker.frameUpdatePaused) { diff --git a/src/index.ts b/src/index.ts index d85c202..0114bd6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,5 @@ import { Eyetracker } from "./Eyetracker"; +import { Components } from "./Components"; export function initEyetracker() { if (!("requestVideoFrameCallback" in HTMLVideoElement.prototype)) { @@ -8,3 +9,7 @@ export function initEyetracker() { } return new Eyetracker(); } + +export function initComponents(et: Eyetracker) { + return new Components(et); +} From 7a0e8e908d66126d0f04248c746832cb678c5dca Mon Sep 17 00:00:00 2001 From: jadeddelta <101148768+jadeddelta@users.noreply.github.com> Date: Wed, 6 Jul 2022 16:55:01 -0400 Subject: [PATCH 03/10] error handling + start work on fixation props --- examples/example-components.html | 141 +++++-------------------------- examples/test-components.html | 1 - src/Components.ts | 109 +++++++++++++++++++----- 3 files changed, 113 insertions(+), 138 deletions(-) diff --git a/examples/example-components.html b/examples/example-components.html index 8b21dd9..fdacdb1 100644 --- a/examples/example-components.html +++ b/examples/example-components.html @@ -1,125 +1,30 @@ - Test Camera + An Eyetracking Experiment - - - - - diff --git a/examples/test-components.html b/examples/test-components.html index c677930..0b1ebc3 100644 --- a/examples/test-components.html +++ b/examples/test-components.html @@ -21,7 +21,6 @@ }); async function start() { - console.log("checK"); const landing = await comp.createLanding(); document.body.append(landing); let next = clearComponentsButton; diff --git a/src/Components.ts b/src/Components.ts index 6832381..c65be30 100644 --- a/src/Components.ts +++ b/src/Components.ts @@ -1,12 +1,33 @@ import { Eyetracker } from "./Eyetracker"; +type fixationProps = { + /** The radius of the fixation used. */ + radius: string; + /** The color of the fixation used. */ + color: string; + /** The shape of the fixation used. (ACCEPTED: circle, idk the name, cross, maybe x?) */ + shape: string; +}; + export class Components { + /** The eyetracker object that will be used to carry out most underlying work. */ private Eyetracker: Eyetracker | undefined; + /** The current components created + * and displayed by this class and will be cleared on {@link clearComponents()} */ private currentComponents: Array = []; + /** The calibration div used in the {@link calibrate()} function. */ private calDivUsed: HTMLDivElement | undefined; + /** The properties of the fixation that will be created by {@link drawFixation()}. */ + private props: fixationProps = { + radius: "10px", + color: "red", + shape: "circle", + }; + /** The default message that will be used for the landing page. */ private DEFAULT_MESSAGE = `Welcome to an experiment that uses eye tracking. In a few moments you will be asked for permission to use your camera in order to complete the experiment.`; + /** The default 9 points that will be used for calibration. */ private DEFAULT_POINTS: Array> = [ [10, 10], [10, 50], @@ -20,20 +41,34 @@ export class Components { ]; constructor(et: Eyetracker) { + if (et === undefined || et === null) { + throw new Error("Eyetracker cannot be undefined or null."); + } this.Eyetracker = et; } + /** + * Initializes core components of the Eyetracker class. This must be called AFTER the + * camera is selected, either through the {@link createSelector()} function or + * by manually setting the camera. + * @returns An object containing the Eyetracker's detector, video, and canvas. + */ async preload(): Promise { - console.log("checK"); let detector = await this.Eyetracker!.init(); - console.log("checK"); let video = await this.Eyetracker!.createVideo(); - console.log("checK"); let canvas = this.Eyetracker!.createDisplayCanvas(); return { detector, video, canvas }; } //TODO: figure out how to put CSS in here shorthand without giving me a stroke + /** + * This will create a landing page that will inform the user of the nature + * of the usage of the Eyetracking software. + * + * @param id The id of the div that will be used to display the landing page. + * @param message The message that will be displayed on the landing page. + * @returns The landing page div. + */ createLanding( id: string = "landing", message: string = this.DEFAULT_MESSAGE @@ -51,6 +86,15 @@ export class Components { } // TODO: what type is the return???? Array + /** + * This will create a selector that will allow for one to select a camera. + * This will automatically ask for permission and once finished, use + * {@link Eyetracker.setCamera()}. + * + * @param id The id of the selector. + * @param message The message that will be displayed on the button. + * @returns An array containing the selector and the button, in that order. + */ async createSelector( id: string = "selector", message: string = "Select a camera" @@ -93,13 +137,18 @@ export class Components { }); return new Array(selector, btn); } - //TODO: these instructions- - /* - DONE: Pass in a DIV with the calibration points being other Div elements - DONE: New coordinate system absolutely (%) on the new element - - Calibrate based off of that (let's adapt calibrate point to take absolute coords) - - Different fixation points??? - */ + + /** + * Calibrate the eyetracker. This will automatically cycle through the points + * in 3s intervals, taking facial landmark data 1.5s into the point's appearance. + * + * PLEASE NOTE: To increase calibration accuracy, use more than 4 points. 9 is preferred. + * + * @param div The div that will be used to display the calibration. + * @param points A list of points denoted in absolute coordinates that will be used in calibration. + * @returns An object containing the x and y coordinates of the calibration point, + * along with associated facial landmark data. + */ async calibrate( div: HTMLDivElement, points: Array> = this.DEFAULT_POINTS @@ -127,6 +176,7 @@ export class Components { if (point === undefined) { clearInterval(calibrateLoop); //TODO- what exactly should we return? + // NOTE: the below statement returns only two points, and fires before calibration finishes. console.log(await this.Eyetracker!.processCalibrationPoints()); // facial landmarks return finishedPoints; // imageData + onset time } @@ -149,6 +199,9 @@ export class Components { return finishedPoints; } + /** + * This will clear all components that were created by this class. + */ clearComponents() { this.currentComponents.forEach((c) => { let el = document.getElementById(c); @@ -162,18 +215,36 @@ export class Components { } // TODO- add different fixations: CROSS, FUNNY THING, SQUARE + // TODO- when developing customized fixations, should we pass in an object? or should we split it? + // ** For now, we'll set a generalized object that can be modified. + /** + * This will create a fixation based off of the given coordinates, placing it in the div. + * The properties of the fixation are customizable through the {@link props} field. + * + * @param x The x coordinate of the fixation. + * @param y The y coordinate of the fixation. + * @param div The div that the fixation will be placed in. + */ drawFixation(x: number, y: number, div: HTMLDivElement) { - const circle = document.createElement("div"); - circle.style.width = "10px"; - circle.style.height = "10px"; - circle.style.borderRadius = "50%"; - circle.style.backgroundColor = "red"; - circle.style.position = "absolute"; - circle.style.left = `calc(${x}% - 5px)`; - circle.style.top = `calc(${y}% - 5px)`; - div.appendChild(circle); + if (this.props.shape === "circle") { + const circle = document.createElement("div"); + circle.style.width = "10px"; + circle.style.height = "10px"; + circle.style.borderRadius = "50%"; + circle.style.backgroundColor = "red"; + circle.style.position = "absolute"; + circle.style.left = `calc(${x}% - 5px)`; + circle.style.top = `calc(${y}% - 5px)`; + div.appendChild(circle); + } } + //TODO- update this if we need it or not + /** + * This is a helper function that will translate given absolute coordinates to standard coordinates. + * @param points Points in absolute coordinates, generated by the {@link calibrate()} function. + * @returns + */ translateToStandard(points: Array): Array { if (this.calDivUsed === null || this.calDivUsed === undefined) { throw new Error( From 9cc347b76f68f5bdcc7a0735bc64271bdb6a1c15 Mon Sep 17 00:00:00 2001 From: jadeddelta <101148768+jadeddelta@users.noreply.github.com> Date: Wed, 6 Jul 2022 19:45:39 -0400 Subject: [PATCH 04/10] error handling --- examples/example-components.html | 39 ++++++++++++ examples/test-components.html | 101 ++++++++++++++++++++----------- src/Components.ts | 62 +++++++++++++------ src/Eyetracker.ts | 27 ++++++--- 4 files changed, 167 insertions(+), 62 deletions(-) diff --git a/examples/example-components.html b/examples/example-components.html index fdacdb1..809929d 100644 --- a/examples/example-components.html +++ b/examples/example-components.html @@ -27,4 +27,43 @@ clearComponentsButton.addEventListener("click", () => { comp.clearComponents(); }); + + async function start() { + const landing = await comp.createLanding(); + document.body.append(landing); + let next = clearComponentsButton; + // invokes the next step and clears itself + next.addEventListener("click", () => { + selectorPrompt(); + next.remove(); + }); + document.body.append(next); + } + + async function selectorPrompt() { + // giving the html components allows us to modify the style! + const [selector, btn] = await comp.createSelector(); + selector.style.margin = "50px"; + document.body.append(selector); + btn.style.top = "50px"; + document.body.append(btn); + + btn.addEventListener("click", () => { + // i'm unsure if setTimeout is necessary + // wanted to make sure the components would clear, THEN calibrate. + setTimeout(async () => { + const div = document.createElement("div"); + div.style.width = "100%"; + div.style.height = "100%"; + div.style.position = "fixed"; + div.id = "cal"; + document.body.appendChild(div); + await comp.preload(); + let ret = await comp.calibrate(div); + console.log(ret); + }, 1000); + }); + } + + start(); diff --git a/examples/test-components.html b/examples/test-components.html index 0b1ebc3..5882433 100644 --- a/examples/test-components.html +++ b/examples/test-components.html @@ -14,43 +14,74 @@ const Eyetracker = eyetrack.initEyetracker(); const comp = eyetrack.initComponents(Eyetracker); - const clearComponentsButton = document.createElement("button"); - clearComponentsButton.textContent = "Next"; - clearComponentsButton.addEventListener("click", () => { - comp.clearComponents(); - }); - - async function start() { - const landing = await comp.createLanding(); - document.body.append(landing); - let next = clearComponentsButton; - next.addEventListener("click", () => { - selectorPrompt(); - next.remove(); - }); - document.body.append(next); - } + const div = document.createElement("div"); + div.style.width = "100%"; + div.style.height = "100%"; + div.style.position = "fixed"; + div.id = "cal"; + document.body.appendChild(div); + + test(24, 50, div); + + function test(x, y, div) { + const adjust = parseInt(4) / 2; + const cross1 = document.createElement("div"); + cross1.style.width = "4px"; + cross1.style.height = "100px"; + cross1.style.backgroundColor = "green"; + cross1.style.position = "absolute"; + cross1.style.left = `calc(${x}% - ${adjust}px)`; + cross1.style.top = `calc(${y}% - ${5}px)`; + + const cross2 = document.createElement("div"); + cross2.style.width = "100px"; + cross2.style.height = "4px"; + cross2.style.backgroundColor = "green"; + cross2.style.position = "absolute"; + cross2.style.left = `calc(${x}% - ${5}px)`; + cross2.style.top = `calc(${y}% - ${adjust}px)`; + div.appendChild(cross2); - async function selectorPrompt() { - const [selector, btn] = await comp.createSelector(); - document.body.append(selector); - document.body.append(btn); - - btn.addEventListener("click", () => { - setTimeout(async () => { - const div = document.createElement("div"); - div.style.width = "100%"; - div.style.height = "100%"; - div.style.position = "fixed"; - div.id = "cal"; - document.body.appendChild(div); - await comp.preload(); - let ret = await comp.calibrate(div); - console.log(ret); - }, 1000); - }); + div.appendChild(cross1); } - start(); + // const clearComponentsButton = document.createElement("button"); + // clearComponentsButton.textContent = "Next"; + // clearComponentsButton.addEventListener("click", () => { + // comp.clearComponents(); + // }); + + // async function start() { + // const landing = await comp.createLanding(); + // document.body.append(landing); + // let next = clearComponentsButton; + // next.addEventListener("click", () => { + // selectorPrompt(); + // next.remove(); + // }); + // document.body.append(next); + // } + + // async function selectorPrompt() { + // const [selector, btn] = await comp.createSelector(); + // document.body.append(selector); + // document.body.append(btn); + + // btn.addEventListener("click", () => { + // setTimeout(async () => { + // const div = document.createElement("div"); + // div.style.width = "100%"; + // div.style.height = "100%"; + // div.style.position = "fixed"; + // div.id = "cal"; + // document.body.appendChild(div); + // await comp.preload(); + // let ret = await comp.calibrate(div); + // console.log(ret); + // }, 1000); + // }); + // } + + //start(); diff --git a/src/Components.ts b/src/Components.ts index c65be30..9a33663 100644 --- a/src/Components.ts +++ b/src/Components.ts @@ -1,12 +1,14 @@ import { Eyetracker } from "./Eyetracker"; type fixationProps = { - /** The radius of the fixation used. */ - radius: string; + /** The diameter of the fixation used. */ + diameter: string; /** The color of the fixation used. */ color: string; /** The shape of the fixation used. (ACCEPTED: circle, idk the name, cross, maybe x?) */ shape: string; + /** The thickness of a cross fixation. */ + crossThickness: string; }; export class Components { @@ -19,9 +21,10 @@ export class Components { private calDivUsed: HTMLDivElement | undefined; /** The properties of the fixation that will be created by {@link drawFixation()}. */ private props: fixationProps = { - radius: "10px", + diameter: "10px", color: "red", shape: "circle", + crossThickness: "4px", }; /** The default message that will be used for the landing page. */ @@ -85,7 +88,6 @@ export class Components { return landing; } - // TODO: what type is the return???? Array /** * This will create a selector that will allow for one to select a camera. * This will automatically ask for permission and once finished, use @@ -98,7 +100,7 @@ export class Components { async createSelector( id: string = "selector", message: string = "Select a camera" - ): Promise> { + ): Promise> { this.currentComponents.push(id); let selector = document.createElement("select"); selector.id = id; @@ -135,7 +137,7 @@ export class Components { alert("Please select a camera."); } }); - return new Array(selector, btn); + return new Array(selector, btn); } /** @@ -176,7 +178,6 @@ export class Components { if (point === undefined) { clearInterval(calibrateLoop); //TODO- what exactly should we return? - // NOTE: the below statement returns only two points, and fires before calibration finishes. console.log(await this.Eyetracker!.processCalibrationPoints()); // facial landmarks return finishedPoints; // imageData + onset time } @@ -195,7 +196,6 @@ export class Components { finishedPoints.push(currentPoint); }, 1500); }, 3000); - console.warn("This shouldn't be accessible."); return finishedPoints; } @@ -226,16 +226,42 @@ export class Components { * @param div The div that the fixation will be placed in. */ drawFixation(x: number, y: number, div: HTMLDivElement) { - if (this.props.shape === "circle") { - const circle = document.createElement("div"); - circle.style.width = "10px"; - circle.style.height = "10px"; - circle.style.borderRadius = "50%"; - circle.style.backgroundColor = "red"; - circle.style.position = "absolute"; - circle.style.left = `calc(${x}% - 5px)`; - circle.style.top = `calc(${y}% - 5px)`; - div.appendChild(circle); + switch (this.props.shape) { + case "circle": + const adjustCircle = parseInt(this.props.diameter) / 2; + const circle = document.createElement("div"); + circle.style.width = this.props.diameter; + circle.style.height = this.props.diameter; + circle.style.borderRadius = "50%"; + circle.style.backgroundColor = this.props.color; + circle.style.position = "absolute"; + circle.style.left = `calc(${x}% - ${adjustCircle}px)`; + circle.style.top = `calc(${y}% - ${adjustCircle}px)`; + div.appendChild(circle); + break; + // !! DOES NOT WORK !! + case "cross": + const adjustCross = parseInt(this.props.crossThickness) / 2; + const cross1 = document.createElement("div"); + cross1.style.width = this.props.crossThickness; + cross1.style.height = this.props.diameter; + cross1.style.backgroundColor = this.props.color; + cross1.style.position = "absolute"; + cross1.style.left = `calc(${x}% - ${adjustCross}px)`; + cross1.style.top = `calc(${y}% - ${adjustCross * 2 + 1}px)`; + + const cross2 = document.createElement("div"); + cross2.style.width = this.props.diameter; + cross2.style.height = this.props.crossThickness; + cross2.style.backgroundColor = this.props.color; + cross2.style.position = "absolute"; + cross2.style.left = `calc(${x}% - ${adjustCross * 2 + 1}px)`; + cross2.style.top = `calc(${y}% - ${adjustCross}px)`; + div.appendChild(cross1); + div.appendChild(cross2); + break; + default: + throw new Error("Invalid shape."); } } diff --git a/src/Eyetracker.ts b/src/Eyetracker.ts index 046945c..4f2d232 100644 --- a/src/Eyetracker.ts +++ b/src/Eyetracker.ts @@ -169,12 +169,15 @@ export class Eyetracker { canvas.height = video.height; canvas.width = video.width; var ctx: CanvasRenderingContext2D | null = canvas.getContext("2d"); - if (ctx != null) { - ctx.translate(canvas.width, 0); - ctx.scale(-1, 1); - ctx.fillStyle = "green"; + if (ctx === null || ctx === undefined) { + throw new Error( + "Could not create canvas context, have you already declared ctx with a different type of context?" + ); } - (this.ctx as undefined | CanvasRenderingContext2D | null) = ctx; + ctx.translate(canvas.width, 0); + ctx.scale(-1, 1); + ctx.fillStyle = "green"; + this.ctx = ctx; } /** @@ -187,11 +190,16 @@ export class Eyetracker { video: HTMLVideoElement = this.video! ): void { let ctx: CanvasRenderingContext2D | null = canvas.getContext("2d"); - if (ctx != undefined && video != undefined) { - ctx.drawImage(video, 0, 0); - } else { - console.log('"this.ctx", "this.video" Undefined'); + if (ctx === null || ctx === undefined) { + throw new Error( + "Could not create canvas context, have you already declared ctx with a different type of context?" + ); } + if (video === null || video === undefined) { + throw new Error("No video stream detected, have you run createVideo()?"); + } + + ctx.drawImage(video, 0, 0); } /** @@ -235,6 +243,7 @@ export class Eyetracker { }, canvas: HTMLCanvasElement = this.canvas! ): void { + // ?? why is this in a try-catch? try { let ctx: CanvasRenderingContext2D | null = canvas.getContext("2d"); From b32c721bf0688808166fe8d95bd5a69e476d49e7 Mon Sep 17 00:00:00 2001 From: jadeddelta <101148768+jadeddelta@users.noreply.github.com> Date: Tue, 12 Jul 2022 17:47:37 -0400 Subject: [PATCH 05/10] added cross fixation --- examples/test-components.html | 16 ++++++++++++---- src/Components.ts | 14 +++++++++----- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/examples/test-components.html b/examples/test-components.html index 5882433..cb8fbcd 100644 --- a/examples/test-components.html +++ b/examples/test-components.html @@ -23,6 +23,10 @@ test(24, 50, div); + comp.props.shape = "cross"; + comp.props.diameter = "100px"; + comp.drawFixation(24, 50, div); + function test(x, y, div) { const adjust = parseInt(4) / 2; const cross1 = document.createElement("div"); @@ -30,16 +34,20 @@ cross1.style.height = "100px"; cross1.style.backgroundColor = "green"; cross1.style.position = "absolute"; - cross1.style.left = `calc(${x}% - ${adjust}px)`; - cross1.style.top = `calc(${y}% - ${5}px)`; + cross1.style.left = `calc(${x}% - ${adjust}px + ${ + parseInt(cross1.style.height) / 2 + }px)`; + cross1.style.top = `calc(${y}%)`; const cross2 = document.createElement("div"); cross2.style.width = "100px"; cross2.style.height = "4px"; cross2.style.backgroundColor = "green"; cross2.style.position = "absolute"; - cross2.style.left = `calc(${x}% - ${5}px)`; - cross2.style.top = `calc(${y}% - ${adjust}px)`; + cross2.style.left = `calc(${x}%)`; + cross2.style.top = `calc(${y}% - ${adjust}px + ${ + parseInt(cross2.style.width) / 2 + }px)`; div.appendChild(cross2); div.appendChild(cross1); diff --git a/src/Components.ts b/src/Components.ts index 9a33663..bee8ef6 100644 --- a/src/Components.ts +++ b/src/Components.ts @@ -239,7 +239,7 @@ export class Components { circle.style.top = `calc(${y}% - ${adjustCircle}px)`; div.appendChild(circle); break; - // !! DOES NOT WORK !! + case "cross": const adjustCross = parseInt(this.props.crossThickness) / 2; const cross1 = document.createElement("div"); @@ -247,16 +247,20 @@ export class Components { cross1.style.height = this.props.diameter; cross1.style.backgroundColor = this.props.color; cross1.style.position = "absolute"; - cross1.style.left = `calc(${x}% - ${adjustCross}px)`; - cross1.style.top = `calc(${y}% - ${adjustCross * 2 + 1}px)`; + cross1.style.left = `calc(${x}% - ${adjustCross}px + ${ + parseInt(this.props.diameter) / 2 + }px)`; + cross1.style.top = `calc(${y}%)`; const cross2 = document.createElement("div"); cross2.style.width = this.props.diameter; cross2.style.height = this.props.crossThickness; cross2.style.backgroundColor = this.props.color; cross2.style.position = "absolute"; - cross2.style.left = `calc(${x}% - ${adjustCross * 2 + 1}px)`; - cross2.style.top = `calc(${y}% - ${adjustCross}px)`; + cross2.style.left = `calc(${x}%)`; + cross2.style.top = `calc(${y}% - ${adjustCross}px + ${ + parseInt(this.props.diameter) / 2 + }px)`; div.appendChild(cross1); div.appendChild(cross2); break; From 73c74b9246fd06d2948a6011a4b3da65c8e478d7 Mon Sep 17 00:00:00 2001 From: jadeddelta <101148768+jadeddelta@users.noreply.github.com> Date: Tue, 12 Jul 2022 21:44:42 -0400 Subject: [PATCH 06/10] added square, configured landing as flexbox --- examples/example-components.html | 7 ++++-- examples/test-components.html | 24 +++++--------------- src/Components.ts | 38 +++++++++++++++++++++++++++----- src/Eyetracker.ts | 2 +- 4 files changed, 45 insertions(+), 26 deletions(-) diff --git a/examples/example-components.html b/examples/example-components.html index 809929d..c39decf 100644 --- a/examples/example-components.html +++ b/examples/example-components.html @@ -31,13 +31,16 @@ async function start() { const landing = await comp.createLanding(); document.body.append(landing); + // create a wrapper so that the button doesn't stretch b/c landing is flex + let wrapper = document.createElement("div"); let next = clearComponentsButton; // invokes the next step and clears itself next.addEventListener("click", () => { selectorPrompt(); - next.remove(); + next.parentNode.remove(); }); - document.body.append(next); + wrapper.append(next); + landing.append(wrapper); } async function selectorPrompt() { diff --git a/examples/test-components.html b/examples/test-components.html index cb8fbcd..237fbb3 100644 --- a/examples/test-components.html +++ b/examples/test-components.html @@ -21,7 +21,8 @@ div.id = "cal"; document.body.appendChild(div); - test(24, 50, div); + test(5, 5, div); + test(0, 0, div); comp.props.shape = "cross"; comp.props.diameter = "100px"; @@ -30,25 +31,12 @@ function test(x, y, div) { const adjust = parseInt(4) / 2; const cross1 = document.createElement("div"); - cross1.style.width = "4px"; - cross1.style.height = "100px"; + cross1.style.width = "10px"; + cross1.style.height = "10px"; cross1.style.backgroundColor = "green"; cross1.style.position = "absolute"; - cross1.style.left = `calc(${x}% - ${adjust}px + ${ - parseInt(cross1.style.height) / 2 - }px)`; - cross1.style.top = `calc(${y}%)`; - - const cross2 = document.createElement("div"); - cross2.style.width = "100px"; - cross2.style.height = "4px"; - cross2.style.backgroundColor = "green"; - cross2.style.position = "absolute"; - cross2.style.left = `calc(${x}%)`; - cross2.style.top = `calc(${y}% - ${adjust}px + ${ - parseInt(cross2.style.width) / 2 - }px)`; - div.appendChild(cross2); + cross1.style.left = `calc(${x}% - ${adjust}px)`; + cross1.style.top = `calc(${y}% - ${adjust}px)`; div.appendChild(cross1); } diff --git a/src/Components.ts b/src/Components.ts index bee8ef6..532c19e 100644 --- a/src/Components.ts +++ b/src/Components.ts @@ -1,5 +1,6 @@ import { Eyetracker } from "./Eyetracker"; +/** The properties of any fixation that will be drawn. */ type fixationProps = { /** The diameter of the fixation used. */ diameter: string; @@ -43,6 +44,10 @@ export class Components { [90, 90], ]; + /** + * @constructor + * @param et The eyetracker object that will be used to carry out most underlying work. + */ constructor(et: Eyetracker) { if (et === undefined || et === null) { throw new Error("Eyetracker cannot be undefined or null."); @@ -63,10 +68,11 @@ export class Components { return { detector, video, canvas }; } - //TODO: figure out how to put CSS in here shorthand without giving me a stroke + //TODO: maybe allow multiple lines instead of forcing user to put one big one? /** * This will create a landing page that will inform the user of the nature - * of the usage of the Eyetracking software. + * of the usage of the Eyetracking software. The landing page is contained + * inside of a wrapper div with flex capabilities. * * @param id The id of the div that will be used to display the landing page. * @param message The message that will be displayed on the landing page. @@ -77,6 +83,7 @@ export class Components { message: string = this.DEFAULT_MESSAGE ): HTMLDivElement { this.currentComponents.push(id); + this.currentComponents.push(`${id}-wrapper`); let landing = document.createElement("div"); landing.id = id; landing.classList.add("landing"); @@ -85,7 +92,15 @@ export class Components {

${message}

`; - return landing; + + let wrapper = document.createElement("div"); + wrapper.id = `${id}-wrapper`; + wrapper.append(landing); + wrapper.style.display = "flex"; + wrapper.style.flexDirection = "column"; + wrapper.style.textAlign = "center"; + + return wrapper; } /** @@ -186,12 +201,11 @@ export class Components { setTimeout(async () => { await this.Eyetracker!.detectFace(); - let currentPoint = this.Eyetracker!.calibratePoint( + let currentPoint: any = this.Eyetracker!.calibratePoint( point![0], point![1] ); //TODO- let's find a way to prevent throwing this error, maybe explicitly defining? - //@ts-ignore currentPoint.onsetTime = onsetTime; finishedPoints.push(currentPoint); }, 1500); @@ -264,6 +278,20 @@ export class Components { div.appendChild(cross1); div.appendChild(cross2); break; + + case "square": + const adjustSquare = parseInt(this.props.diameter) / 2; + const square = document.createElement("div"); + square.style.width = this.props.diameter; + square.style.height = this.props.diameter; + square.style.borderRadius = "50%"; + square.style.backgroundColor = this.props.color; + square.style.position = "absolute"; + square.style.left = `calc(${x}% - ${adjustSquare}px)`; + square.style.top = `calc(${y}% - ${adjustSquare}px)`; + div.appendChild(square); + break; + default: throw new Error("Invalid shape."); } diff --git a/src/Eyetracker.ts b/src/Eyetracker.ts index 4f2d232..32e687d 100644 --- a/src/Eyetracker.ts +++ b/src/Eyetracker.ts @@ -32,7 +32,7 @@ export class Eyetracker { }> = []; /** */ public onFrameUpdateCallbackList: Array = []; - /** */ + /** Determines whether or not {@link initVideoFrameLoop()}'s rVFCallbacks are paused or not. */ public frameUpdatePaused: boolean = false; /** From 0a78b02d0d2338729f4411fc67495f8a1adae446 Mon Sep 17 00:00:00 2001 From: jadeddelta <101148768+jadeddelta@users.noreply.github.com> Date: Wed, 13 Jul 2022 12:44:06 -0400 Subject: [PATCH 07/10] reticule added + fixed cross --- examples/test-components.html | 67 ++++++++++++++++++++++------ src/Components.ts | 82 +++++++++++++++++++++++++++++++---- 2 files changed, 128 insertions(+), 21 deletions(-) diff --git a/examples/test-components.html b/examples/test-components.html index 237fbb3..8f769bb 100644 --- a/examples/test-components.html +++ b/examples/test-components.html @@ -21,24 +21,65 @@ div.id = "cal"; document.body.appendChild(div); - test(5, 5, div); + test(10, 10, div); test(0, 0, div); - comp.props.shape = "cross"; - comp.props.diameter = "100px"; - comp.drawFixation(24, 50, div); + for (let i = 0; i < 10; i++) { + test(Math.random() * 100, Math.random() * 100, div); + } + + comp.props.shape = "reticule"; + comp.props.diameter = "20px"; + comp.props.color = "black"; + comp.drawFixation(20, 20, div); function test(x, y, div) { - const adjust = parseInt(4) / 2; - const cross1 = document.createElement("div"); - cross1.style.width = "10px"; - cross1.style.height = "10px"; - cross1.style.backgroundColor = "green"; - cross1.style.position = "absolute"; - cross1.style.left = `calc(${x}% - ${adjust}px)`; - cross1.style.top = `calc(${y}% - ${adjust}px)`; + const color = + div.parentNode.style.backgroundColor === "" + ? "white" + : div.parentNode.style.backgroundColor; + + const outerCircle = document.createElement("div"); + outerCircle.style.width = "20px"; + outerCircle.style.height = "20px"; + outerCircle.style.backgroundColor = "green"; + outerCircle.style.position = "absolute"; + outerCircle.style.left = `calc(${x}% - ${10}px)`; + outerCircle.style.top = `calc(${y}% - ${10}px)`; + outerCircle.style.borderRadius = "50%"; + outerCircle.style.zIndex = "1"; + div.appendChild(outerCircle); + + const blankCross1 = document.createElement("div"); + blankCross1.style.width = "5px"; + blankCross1.style.height = "20px"; + blankCross1.style.backgroundColor = color; + blankCross1.style.position = "absolute"; + blankCross1.style.left = `calc(${x}% - ${2.5}px)`; + blankCross1.style.top = `calc(${y}% - ${10}px)`; + blankCross1.style.zIndex = "2"; + div.appendChild(blankCross1); + + const blankCross2 = document.createElement("div"); + blankCross2.style.width = "20px"; + blankCross2.style.height = "5px"; + blankCross2.style.backgroundColor = color; + blankCross2.style.position = "absolute"; + blankCross2.style.left = `calc(${x}% - ${10}px)`; + blankCross2.style.top = `calc(${y}% - ${2.5}px)`; + blankCross2.style.zIndex = "2"; + div.appendChild(blankCross2); - div.appendChild(cross1); + const innerCircle = document.createElement("div"); + innerCircle.style.width = "10px"; + innerCircle.style.height = "10px"; + innerCircle.style.backgroundColor = "red"; + innerCircle.style.position = "absolute"; + innerCircle.style.left = `calc(${x}% - ${5}px)`; + innerCircle.style.top = `calc(${y}% - ${5}px)`; + innerCircle.style.borderRadius = "50%"; + innerCircle.style.zIndex = "3"; + div.appendChild(innerCircle); } // const clearComponentsButton = document.createElement("button"); diff --git a/src/Components.ts b/src/Components.ts index 532c19e..b2ff19d 100644 --- a/src/Components.ts +++ b/src/Components.ts @@ -6,7 +6,7 @@ type fixationProps = { diameter: string; /** The color of the fixation used. */ color: string; - /** The shape of the fixation used. (ACCEPTED: circle, idk the name, cross, maybe x?) */ + /** The shape of the fixation used. (ACCEPTED: circle, reticule, cross, square) */ shape: string; /** The thickness of a cross fixation. */ crossThickness: string; @@ -228,9 +228,6 @@ export class Components { this.currentComponents = []; } - // TODO- add different fixations: CROSS, FUNNY THING, SQUARE - // TODO- when developing customized fixations, should we pass in an object? or should we split it? - // ** For now, we'll set a generalized object that can be modified. /** * This will create a fixation based off of the given coordinates, placing it in the div. * The properties of the fixation are customizable through the {@link props} field. @@ -240,6 +237,13 @@ export class Components { * @param div The div that the fixation will be placed in. */ drawFixation(x: number, y: number, div: HTMLDivElement) { + if (div === null || div === undefined) { + throw new Error("Please provide a div to draw the fixation in."); + } + if (div.parentElement === null || div.parentElement === undefined) { + throw new Error("Div cannot be document.body."); + } + switch (this.props.shape) { case "circle": const adjustCircle = parseInt(this.props.diameter) / 2; @@ -261,20 +265,20 @@ export class Components { cross1.style.height = this.props.diameter; cross1.style.backgroundColor = this.props.color; cross1.style.position = "absolute"; - cross1.style.left = `calc(${x}% - ${adjustCross}px + ${ + cross1.style.left = `calc(${x}% - ${adjustCross}px)`; + cross1.style.top = `calc(${y}% - ${ parseInt(this.props.diameter) / 2 }px)`; - cross1.style.top = `calc(${y}%)`; const cross2 = document.createElement("div"); cross2.style.width = this.props.diameter; cross2.style.height = this.props.crossThickness; cross2.style.backgroundColor = this.props.color; cross2.style.position = "absolute"; - cross2.style.left = `calc(${x}%)`; - cross2.style.top = `calc(${y}% - ${adjustCross}px + ${ + cross2.style.left = `calc(${x}% - ${ parseInt(this.props.diameter) / 2 }px)`; + cross2.style.top = `calc(${y}% - ${adjustCross}px)`; div.appendChild(cross1); div.appendChild(cross2); break; @@ -292,6 +296,68 @@ export class Components { div.appendChild(square); break; + case "reticule": + if (parseInt(this.props.diameter) <= 10) { + console.warn("Reticule diameter is small and will look odd."); + } + const adjustReticule = parseInt(this.props.diameter) / 2; + const outerCircle = document.createElement("div"); + outerCircle.style.width = this.props.diameter; + outerCircle.style.height = this.props.diameter; + outerCircle.style.borderRadius = "50%"; + outerCircle.style.backgroundColor = this.props.color; + outerCircle.style.position = "absolute"; + outerCircle.style.left = `calc(${x}% - ${adjustReticule}px)`; + outerCircle.style.top = `calc(${y}% - ${adjustReticule}px)`; + outerCircle.style.zIndex = "1"; + div.appendChild(outerCircle); + + const adjustInner = adjustReticule / 2; + const innerCircle = document.createElement("div"); + const innerSize = `${parseInt(this.props.diameter) / 2}px`; + innerCircle.style.width = innerSize; + innerCircle.style.height = innerSize; + innerCircle.style.borderRadius = "50%"; + innerCircle.style.backgroundColor = this.props.color; + innerCircle.style.position = "absolute"; + innerCircle.style.left = `calc(${x}% - ${adjustInner}px)`; + innerCircle.style.top = `calc(${y}% - ${adjustInner}px)`; + innerCircle.style.zIndex = "3"; + div.appendChild(innerCircle); + + const eraseColor = + div.parentElement.style.backgroundColor === "" + ? "white" + : div.parentElement.style.backgroundColor; + const eraseThickness = parseInt(this.props.diameter) / 4; + const eraseSize = `${eraseThickness}px`; + const adjustErase = eraseThickness / 2; + + const eCross1 = document.createElement("div"); + eCross1.style.width = eraseSize; + eCross1.style.height = this.props.diameter; + eCross1.style.backgroundColor = eraseColor; + eCross1.style.position = "absolute"; + eCross1.style.left = `calc(${x}% - ${adjustErase}px)`; + eCross1.style.top = `calc(${y}% - ${ + parseInt(this.props.diameter) / 2 + }px)`; + eCross1.style.zIndex = "2"; + + const eCross2 = document.createElement("div"); + eCross2.style.width = this.props.diameter; + eCross2.style.height = eraseSize; + eCross2.style.backgroundColor = eraseColor; + eCross2.style.position = "absolute"; + eCross2.style.left = `calc(${x}% - ${ + parseInt(this.props.diameter) / 2 + }px)`; + eCross2.style.top = `calc(${y}% - ${adjustErase}px)`; + eCross2.style.zIndex = "2"; + div.appendChild(eCross1); + div.appendChild(eCross2); + break; + default: throw new Error("Invalid shape."); } From 8aeb658ecdf53ba117c662339afddf212296d3b4 Mon Sep 17 00:00:00 2001 From: Josh de Leeuw Date: Tue, 19 Jul 2022 11:47:00 -0400 Subject: [PATCH 08/10] sample edits from code review --- src/Components.ts | 30 +++++++++++++++++++++++------- src/index.ts | 4 ++++ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/Components.ts b/src/Components.ts index b2ff19d..eaf28c7 100644 --- a/src/Components.ts +++ b/src/Components.ts @@ -12,6 +12,8 @@ type fixationProps = { crossThickness: string; }; +type fixationShapeFunction = (x: number, y: number) => HTMLElement; + export class Components { /** The eyetracker object that will be used to carry out most underlying work. */ private Eyetracker: Eyetracker | undefined; @@ -140,13 +142,9 @@ export class Components { btn.innerHTML = `${message}`; btn.addEventListener("click", async () => { const cam = selector.options[selector.selectedIndex].value; + const selectedCamera = devices.filter((d) => d.deviceId === cam)[0]; if (id !== "") { - await this.Eyetracker!.setCamera( - //@ts-ignore - await navigator.mediaDevices.getUserMedia({ - video: { deviceId: cam }, - }) - ); + await this.Eyetracker!.setCamera(selectedCamera); this.clearComponents(); } else { alert("Please select a camera."); @@ -236,7 +234,12 @@ export class Components { * @param y The y coordinate of the fixation. * @param div The div that the fixation will be placed in. */ - drawFixation(x: number, y: number, div: HTMLDivElement) { + drawFixation( + x: number, + y: number, + div: HTMLDivElement, + fixationShape: fixationShapeFunction + ) { if (div === null || div === undefined) { throw new Error("Please provide a div to draw the fixation in."); } @@ -363,6 +366,19 @@ export class Components { } } + drawCircleFixation(x, y) { + const adjustCircle = parseInt(this.props.diameter) / 2; + const circle = document.createElement("div"); + circle.style.width = this.props.diameter; + circle.style.height = this.props.diameter; + circle.style.borderRadius = "50%"; + circle.style.backgroundColor = this.props.color; + circle.style.position = "absolute"; + circle.style.left = `calc(${x}% - ${adjustCircle}px)`; + circle.style.top = `calc(${y}% - ${adjustCircle}px)`; + return circle; + } + //TODO- update this if we need it or not /** * This is a helper function that will translate given absolute coordinates to standard coordinates. diff --git a/src/index.ts b/src/index.ts index 0114bd6..778b0ed 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,10 @@ import { Eyetracker } from "./Eyetracker"; import { Components } from "./Components"; +/** + * Create a new instance of the eye tracker. + * @returns A new instance of the eye tracker. + */ export function initEyetracker() { if (!("requestVideoFrameCallback" in HTMLVideoElement.prototype)) { window.alert( From cb3248ae0144aa0dc876f2ebd6ae07706db6caf2 Mon Sep 17 00:00:00 2001 From: jadeddelta <101148768+jadeddelta@users.noreply.github.com> Date: Thu, 21 Jul 2022 16:51:09 -0400 Subject: [PATCH 09/10] code review changes --- examples/example-components.html | 12 +- src/Components.ts | 315 +++++++++++++++---------------- src/Eyetracker.ts | 2 +- src/index.ts | 5 + 4 files changed, 156 insertions(+), 178 deletions(-) diff --git a/examples/example-components.html b/examples/example-components.html index c39decf..17680f2 100644 --- a/examples/example-components.html +++ b/examples/example-components.html @@ -21,23 +21,17 @@ const Eyetracker = eyetrack.initEyetracker(); const comp = eyetrack.initComponents(Eyetracker); - // basically a "go to the next stage" button - const clearComponentsButton = document.createElement("button"); - clearComponentsButton.textContent = "Next"; - clearComponentsButton.addEventListener("click", () => { - comp.clearComponents(); - }); - async function start() { const landing = await comp.createLanding(); document.body.append(landing); // create a wrapper so that the button doesn't stretch b/c landing is flex let wrapper = document.createElement("div"); - let next = clearComponentsButton; + let next = document.createElement("button"); + next.innerText = "Next"; // invokes the next step and clears itself next.addEventListener("click", () => { + next.parentNode.parentNode.remove(); selectorPrompt(); - next.parentNode.remove(); }); wrapper.append(next); landing.append(wrapper); diff --git a/src/Components.ts b/src/Components.ts index eaf28c7..d548c4f 100644 --- a/src/Components.ts +++ b/src/Components.ts @@ -6,27 +6,25 @@ type fixationProps = { diameter: string; /** The color of the fixation used. */ color: string; - /** The shape of the fixation used. (ACCEPTED: circle, reticule, cross, square) */ - shape: string; + /** The function of the fixation used to generate it. */ + shape: fixationShapeFunction; /** The thickness of a cross fixation. */ crossThickness: string; }; +/** The function type used by {@link calibrate()} in order to draw a fixation. */ type fixationShapeFunction = (x: number, y: number) => HTMLElement; export class Components { /** The eyetracker object that will be used to carry out most underlying work. */ private Eyetracker: Eyetracker | undefined; - /** The current components created - * and displayed by this class and will be cleared on {@link clearComponents()} */ - private currentComponents: Array = []; /** The calibration div used in the {@link calibrate()} function. */ private calDivUsed: HTMLDivElement | undefined; /** The properties of the fixation that will be created by {@link drawFixation()}. */ private props: fixationProps = { diameter: "10px", color: "red", - shape: "circle", + shape: this.drawCircleFixation, crossThickness: "4px", }; @@ -84,16 +82,10 @@ export class Components { id: string = "landing", message: string = this.DEFAULT_MESSAGE ): HTMLDivElement { - this.currentComponents.push(id); - this.currentComponents.push(`${id}-wrapper`); let landing = document.createElement("div"); landing.id = id; landing.classList.add("landing"); - landing.innerHTML = ` -
-

${message}

-
- `; + landing.innerHTML = `

${message}

`; let wrapper = document.createElement("div"); wrapper.id = `${id}-wrapper`; @@ -118,37 +110,34 @@ export class Components { id: string = "selector", message: string = "Select a camera" ): Promise> { - this.currentComponents.push(id); let selector = document.createElement("select"); selector.id = id; await this.Eyetracker!.getCameraPermission(); const devices = await this.Eyetracker!.getListOfCameras(); - const blank = document.createElement("option"); - blank.style.display = "none"; - selector.appendChild(blank); + if (devices.length !== 1) { + const blank = document.createElement("option"); + blank.style.display = "none"; + selector.appendChild(blank); + } - devices.forEach((d) => { + for (const d of devices) { let option = document.createElement("option"); option.value = d.deviceId; option.innerHTML = d.label; selector.appendChild(option); - }); + } const btn = document.createElement("button"); btn.id = `${id}-btn`; - this.currentComponents.push(btn.id); btn.innerHTML = `${message}`; btn.addEventListener("click", async () => { const cam = selector.options[selector.selectedIndex].value; const selectedCamera = devices.filter((d) => d.deviceId === cam)[0]; - if (id !== "") { - await this.Eyetracker!.setCamera(selectedCamera); - this.clearComponents(); - } else { - alert("Please select a camera."); - } + await this.Eyetracker!.setCamera(selectedCamera); + selector.remove(); + btn.remove(); }); return new Array(selector, btn); } @@ -169,8 +158,7 @@ export class Components { points: Array> = this.DEFAULT_POINTS ): Promise> { if (div === null) { - div = document.createElement("div"); - div.id = "cal-div"; + throw new Error("Please provide a div."); } else { div.innerHTML = ""; } @@ -190,7 +178,7 @@ export class Components { let point = points.shift(); if (point === undefined) { clearInterval(calibrateLoop); - //TODO- what exactly should we return? + //TODO- Update this to return what exactly we need for the neural network. console.log(await this.Eyetracker!.processCalibrationPoints()); // facial landmarks return finishedPoints; // imageData + onset time } @@ -198,12 +186,15 @@ export class Components { let onsetTime = performance.now(); setTimeout(async () => { + //TODO- eventually we must update the API so that we can just gather the facial + //landmarks/image data using a field in eyetracker. await this.Eyetracker!.detectFace(); let currentPoint: any = this.Eyetracker!.calibratePoint( point![0], point![1] ); //TODO- let's find a way to prevent throwing this error, maybe explicitly defining? + // fixed with the :any cast, but we should still consider implementing types. currentPoint.onsetTime = onsetTime; finishedPoints.push(currentPoint); }, 1500); @@ -211,21 +202,6 @@ export class Components { return finishedPoints; } - /** - * This will clear all components that were created by this class. - */ - clearComponents() { - this.currentComponents.forEach((c) => { - let el = document.getElementById(c); - if (el !== null) { - el.remove(); - } else { - console.log(`${c} not found`); - } - }); - this.currentComponents = []; - } - /** * This will create a fixation based off of the given coordinates, placing it in the div. * The properties of the fixation are customizable through the {@link props} field. @@ -234,12 +210,7 @@ export class Components { * @param y The y coordinate of the fixation. * @param div The div that the fixation will be placed in. */ - drawFixation( - x: number, - y: number, - div: HTMLDivElement, - fixationShape: fixationShapeFunction - ) { + drawFixation(x: number, y: number, div: HTMLDivElement) { if (div === null || div === undefined) { throw new Error("Please provide a div to draw the fixation in."); } @@ -247,126 +218,18 @@ export class Components { throw new Error("Div cannot be document.body."); } - switch (this.props.shape) { - case "circle": - const adjustCircle = parseInt(this.props.diameter) / 2; - const circle = document.createElement("div"); - circle.style.width = this.props.diameter; - circle.style.height = this.props.diameter; - circle.style.borderRadius = "50%"; - circle.style.backgroundColor = this.props.color; - circle.style.position = "absolute"; - circle.style.left = `calc(${x}% - ${adjustCircle}px)`; - circle.style.top = `calc(${y}% - ${adjustCircle}px)`; - div.appendChild(circle); - break; - - case "cross": - const adjustCross = parseInt(this.props.crossThickness) / 2; - const cross1 = document.createElement("div"); - cross1.style.width = this.props.crossThickness; - cross1.style.height = this.props.diameter; - cross1.style.backgroundColor = this.props.color; - cross1.style.position = "absolute"; - cross1.style.left = `calc(${x}% - ${adjustCross}px)`; - cross1.style.top = `calc(${y}% - ${ - parseInt(this.props.diameter) / 2 - }px)`; - - const cross2 = document.createElement("div"); - cross2.style.width = this.props.diameter; - cross2.style.height = this.props.crossThickness; - cross2.style.backgroundColor = this.props.color; - cross2.style.position = "absolute"; - cross2.style.left = `calc(${x}% - ${ - parseInt(this.props.diameter) / 2 - }px)`; - cross2.style.top = `calc(${y}% - ${adjustCross}px)`; - div.appendChild(cross1); - div.appendChild(cross2); - break; - - case "square": - const adjustSquare = parseInt(this.props.diameter) / 2; - const square = document.createElement("div"); - square.style.width = this.props.diameter; - square.style.height = this.props.diameter; - square.style.borderRadius = "50%"; - square.style.backgroundColor = this.props.color; - square.style.position = "absolute"; - square.style.left = `calc(${x}% - ${adjustSquare}px)`; - square.style.top = `calc(${y}% - ${adjustSquare}px)`; - div.appendChild(square); - break; - - case "reticule": - if (parseInt(this.props.diameter) <= 10) { - console.warn("Reticule diameter is small and will look odd."); - } - const adjustReticule = parseInt(this.props.diameter) / 2; - const outerCircle = document.createElement("div"); - outerCircle.style.width = this.props.diameter; - outerCircle.style.height = this.props.diameter; - outerCircle.style.borderRadius = "50%"; - outerCircle.style.backgroundColor = this.props.color; - outerCircle.style.position = "absolute"; - outerCircle.style.left = `calc(${x}% - ${adjustReticule}px)`; - outerCircle.style.top = `calc(${y}% - ${adjustReticule}px)`; - outerCircle.style.zIndex = "1"; - div.appendChild(outerCircle); - - const adjustInner = adjustReticule / 2; - const innerCircle = document.createElement("div"); - const innerSize = `${parseInt(this.props.diameter) / 2}px`; - innerCircle.style.width = innerSize; - innerCircle.style.height = innerSize; - innerCircle.style.borderRadius = "50%"; - innerCircle.style.backgroundColor = this.props.color; - innerCircle.style.position = "absolute"; - innerCircle.style.left = `calc(${x}% - ${adjustInner}px)`; - innerCircle.style.top = `calc(${y}% - ${adjustInner}px)`; - innerCircle.style.zIndex = "3"; - div.appendChild(innerCircle); - - const eraseColor = - div.parentElement.style.backgroundColor === "" - ? "white" - : div.parentElement.style.backgroundColor; - const eraseThickness = parseInt(this.props.diameter) / 4; - const eraseSize = `${eraseThickness}px`; - const adjustErase = eraseThickness / 2; - - const eCross1 = document.createElement("div"); - eCross1.style.width = eraseSize; - eCross1.style.height = this.props.diameter; - eCross1.style.backgroundColor = eraseColor; - eCross1.style.position = "absolute"; - eCross1.style.left = `calc(${x}% - ${adjustErase}px)`; - eCross1.style.top = `calc(${y}% - ${ - parseInt(this.props.diameter) / 2 - }px)`; - eCross1.style.zIndex = "2"; - - const eCross2 = document.createElement("div"); - eCross2.style.width = this.props.diameter; - eCross2.style.height = eraseSize; - eCross2.style.backgroundColor = eraseColor; - eCross2.style.position = "absolute"; - eCross2.style.left = `calc(${x}% - ${ - parseInt(this.props.diameter) / 2 - }px)`; - eCross2.style.top = `calc(${y}% - ${adjustErase}px)`; - eCross2.style.zIndex = "2"; - div.appendChild(eCross1); - div.appendChild(eCross2); - break; - - default: - throw new Error("Invalid shape."); - } + let fixation = this.props.shape.bind(this)(x, y); + div.appendChild(fixation); } - drawCircleFixation(x, y) { + /** + * Creates a circle fixation centered absolutely on the given coordinates. + * + * @param x The x coordinate of the fixation. + * @param y The y coordinate of the fixation. + * @returns A circle fixation element. + */ + drawCircleFixation(x: number, y: number) { const adjustCircle = parseInt(this.props.diameter) / 2; const circle = document.createElement("div"); circle.style.width = this.props.diameter; @@ -379,6 +242,122 @@ export class Components { return circle; } + /** + * Creates a cross (+) fixation centered absolutely on the given coordinates. + * + * @param x The x coordinate of the fixation. + * @param y The y coordinate of the fixation. + * @returns A cross fixation element. + */ + drawCrossFixation(x: number, y: number) { + const adjustCross = parseInt(this.props.crossThickness) / 2; + const cross1 = document.createElement("div"); + cross1.style.width = this.props.crossThickness; + cross1.style.height = this.props.diameter; + cross1.style.backgroundColor = this.props.color; + cross1.style.position = "absolute"; + cross1.style.left = `calc(${x}% - ${adjustCross}px)`; + cross1.style.top = `calc(${y}% - ${parseInt(this.props.diameter) / 2}px)`; + + const cross2 = document.createElement("div"); + cross2.style.width = this.props.diameter; + cross2.style.height = this.props.crossThickness; + cross2.style.backgroundColor = this.props.color; + cross2.style.position = "absolute"; + cross2.style.left = `calc(${x}% - ${parseInt(this.props.diameter) / 2}px)`; + cross2.style.top = `calc(${y}% - ${adjustCross}px)`; + + const wrapper = document.createElement("div"); + wrapper.appendChild(cross1); + wrapper.appendChild(cross2); + return wrapper; + } + + /** + * Creates a square fixation centered absolutely on the given coordinates. + * + * @param x The x coordinate of the fixation. + * @param y The y coordinate of the fixation. + * @returns A square fixation element. + */ + drawSquareFixation(x: number, y: number) { + const adjustSquare = parseInt(this.props.diameter) / 2; + const square = document.createElement("div"); + square.style.width = this.props.diameter; + square.style.height = this.props.diameter; + square.style.borderRadius = "50%"; + square.style.backgroundColor = this.props.color; + square.style.position = "absolute"; + square.style.left = `calc(${x}% - ${adjustSquare}px)`; + square.style.top = `calc(${y}% - ${adjustSquare}px)`; + return square; + } + + /** + * Creates a reticule fixation centered absolutely on the given coordinates. + * + * @param x The x coordinate of the fixation. + * @param y The y coordinate of the fixation. + * @returns A reticule fixation element. + */ + drawReticuleFixation(x: number, y: number) { + if (parseInt(this.props.diameter) <= 10) { + console.warn("Reticule diameter is small and will look odd."); + } + const adjustReticule = parseInt(this.props.diameter) / 2; + const outerCircle = document.createElement("div"); + outerCircle.style.width = this.props.diameter; + outerCircle.style.height = this.props.diameter; + outerCircle.style.borderRadius = "50%"; + outerCircle.style.backgroundColor = this.props.color; + outerCircle.style.position = "absolute"; + outerCircle.style.left = `calc(${x}% - ${adjustReticule}px)`; + outerCircle.style.top = `calc(${y}% - ${adjustReticule}px)`; + outerCircle.style.zIndex = "1"; + + const adjustInner = adjustReticule / 2; + const innerCircle = document.createElement("div"); + const innerSize = `${parseInt(this.props.diameter) / 2}px`; + innerCircle.style.width = innerSize; + innerCircle.style.height = innerSize; + innerCircle.style.borderRadius = "50%"; + innerCircle.style.backgroundColor = this.props.color; + innerCircle.style.position = "absolute"; + innerCircle.style.left = `calc(${x}% - ${adjustInner}px)`; + innerCircle.style.top = `calc(${y}% - ${adjustInner}px)`; + innerCircle.style.zIndex = "3"; + + const eraseColor = "white"; + const eraseThickness = parseInt(this.props.diameter) / 4; + const eraseSize = `${eraseThickness}px`; + const adjustErase = eraseThickness / 2; + + const eCross1 = document.createElement("div"); + eCross1.style.width = eraseSize; + eCross1.style.height = this.props.diameter; + eCross1.style.backgroundColor = eraseColor; + eCross1.style.position = "absolute"; + eCross1.style.left = `calc(${x}% - ${adjustErase}px)`; + eCross1.style.top = `calc(${y}% - ${parseInt(this.props.diameter) / 2}px)`; + eCross1.style.zIndex = "2"; + + const eCross2 = document.createElement("div"); + eCross2.style.width = this.props.diameter; + eCross2.style.height = eraseSize; + eCross2.style.backgroundColor = eraseColor; + eCross2.style.position = "absolute"; + eCross2.style.left = `calc(${x}% - ${parseInt(this.props.diameter) / 2}px)`; + eCross2.style.top = `calc(${y}% - ${adjustErase}px)`; + eCross2.style.zIndex = "2"; + + const wrapper = document.createElement("div"); + wrapper.appendChild(outerCircle); + wrapper.appendChild(innerCircle); + wrapper.appendChild(eCross1); + wrapper.appendChild(eCross2); + return wrapper; + } + //TODO- update this if we need it or not /** * This is a helper function that will translate given absolute coordinates to standard coordinates. diff --git a/src/Eyetracker.ts b/src/Eyetracker.ts index 32e687d..03c45f7 100644 --- a/src/Eyetracker.ts +++ b/src/Eyetracker.ts @@ -25,7 +25,7 @@ export class Eyetracker { private model: MediaPipeFaceMesh | undefined; /** Shows whether or not the overlay should be displayed. */ private overlay: boolean = true; - /** */ + /** The frames, timestamped with rVF callbacks, generated by {@link initVideoFrameLoop()} */ private frames: Array<{ imageData: ImageData; timestamp: DOMHighResTimeStamp; diff --git a/src/index.ts b/src/index.ts index 778b0ed..931518e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -14,6 +14,11 @@ export function initEyetracker() { return new Eyetracker(); } +/** + * Creates an instance of a components module. + * @param et The eye tracker instance. + * @returns A new instance of the components class. + */ export function initComponents(et: Eyetracker) { return new Components(et); } From 79aa2bf9287166a780098cfb9b0a17c089f55eae Mon Sep 17 00:00:00 2001 From: jadeddelta <101148768+jadeddelta@users.noreply.github.com> Date: Thu, 15 Sep 2022 11:57:28 -0400 Subject: [PATCH 10/10] customizable calibration time + tests --- src/Components.ts | 7 +++++-- tests/components.test.ts | 22 ++++++++++++++++++++++ tests/index.test.ts | 3 ++- 3 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 tests/components.test.ts diff --git a/src/Components.ts b/src/Components.ts index d548c4f..1b3c25b 100644 --- a/src/Components.ts +++ b/src/Components.ts @@ -149,12 +149,15 @@ export class Components { * PLEASE NOTE: To increase calibration accuracy, use more than 4 points. 9 is preferred. * * @param div The div that will be used to display the calibration. + * @param time The time between each point in the calibration process, with data collected at + * the middle of each interval. Default is 3s. * @param points A list of points denoted in absolute coordinates that will be used in calibration. * @returns An object containing the x and y coordinates of the calibration point, * along with associated facial landmark data. */ async calibrate( div: HTMLDivElement, + time: number = 3000, points: Array> = this.DEFAULT_POINTS ): Promise> { if (div === null) { @@ -197,8 +200,8 @@ export class Components { // fixed with the :any cast, but we should still consider implementing types. currentPoint.onsetTime = onsetTime; finishedPoints.push(currentPoint); - }, 1500); - }, 3000); + }, time / 2); + }, time); return finishedPoints; } diff --git a/tests/components.test.ts b/tests/components.test.ts new file mode 100644 index 0000000..bb8516f --- /dev/null +++ b/tests/components.test.ts @@ -0,0 +1,22 @@ +import { initEyetracker, initComponents } from "../src/index"; + +describe("components", () => { + let eye: any; + let comp: any; + + beforeEach(() => { + global.alert = jest.fn(); + eye = initEyetracker(); + comp = initComponents(eye); + }); + + test("landing page generated and customizable", () => { + document.body.innerHTML = `
`; + let landing = comp.createLanding("test-land", "Landing page."); + document.getElementById("test")!.appendChild(landing); + + let innerLanding = document.getElementById("test-land"); + expect(innerLanding).toBeTruthy(); + expect(innerLanding!.innerHTML).toEqual(`

Landing page.

`); + }); +}); diff --git a/tests/index.test.ts b/tests/index.test.ts index 9973466..4966a4a 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -1,6 +1,7 @@ -import { initEyetracker } from "../src/index"; +import { initEyetracker, initComponents } from "../src/index"; test("test adds two numbers", () => { + global.alert = jest.fn(); // TODO: figure out a way to implement alert or just remove it! const eye = initEyetracker(); expect(eye.add(2, 2)).toEqual(4); });