diff --git a/examples/example-components.html b/examples/example-components.html
new file mode 100644
index 0000000..17680f2
--- /dev/null
+++ b/examples/example-components.html
@@ -0,0 +1,66 @@
+
+
+ An Eyetracking Experiment
+
+
+
+
+
diff --git a/examples/pseudo-components.html b/examples/pseudo-components.html
new file mode 100644
index 0000000..62451f6
--- /dev/null
+++ b/examples/pseudo-components.html
@@ -0,0 +1,33 @@
+
+
+
+
+
+
+
+
diff --git a/examples/test-components.html b/examples/test-components.html
new file mode 100644
index 0000000..8f769bb
--- /dev/null
+++ b/examples/test-components.html
@@ -0,0 +1,124 @@
+
+
+
+
+
+
+
+
+
diff --git a/src/Components.ts b/src/Components.ts
new file mode 100644
index 0000000..1b3c25b
--- /dev/null
+++ b/src/Components.ts
@@ -0,0 +1,390 @@
+import { Eyetracker } from "./Eyetracker";
+
+/** The properties of any fixation that will be drawn. */
+type fixationProps = {
+ /** The diameter of the fixation used. */
+ diameter: string;
+ /** The color of the fixation used. */
+ color: string;
+ /** The function of the fixation used to generate it. */
+ shape: fixationShapeFunction;
+ /** The thickness of a cross fixation. */
+ crossThickness: string;
+};
+
+/** The function type used by {@link calibrate()} in order to draw a fixation. */
+type fixationShapeFunction = (x: number, y: number) => HTMLElement;
+
+export class Components {
+ /** The eyetracker object that will be used to carry out most underlying work. */
+ private Eyetracker: Eyetracker | undefined;
+ /** The calibration div used in the {@link calibrate()} function. */
+ private calDivUsed: HTMLDivElement | undefined;
+ /** The properties of the fixation that will be created by {@link drawFixation()}. */
+ private props: fixationProps = {
+ diameter: "10px",
+ color: "red",
+ shape: this.drawCircleFixation,
+ crossThickness: "4px",
+ };
+
+ /** The default message that will be used for the landing page. */
+ private DEFAULT_MESSAGE = `Welcome to an experiment that uses eye tracking.
+ In a few moments you will be asked for permission to use your camera in order to complete the experiment.`;
+ /** The default 9 points that will be used for calibration. */
+ private DEFAULT_POINTS: Array> = [
+ [10, 10],
+ [10, 50],
+ [10, 90],
+ [50, 10],
+ [50, 50],
+ [50, 90],
+ [90, 10],
+ [90, 50],
+ [90, 90],
+ ];
+
+ /**
+ * @constructor
+ * @param et The eyetracker object that will be used to carry out most underlying work.
+ */
+ constructor(et: Eyetracker) {
+ if (et === undefined || et === null) {
+ throw new Error("Eyetracker cannot be undefined or null.");
+ }
+ this.Eyetracker = et;
+ }
+
+ /**
+ * Initializes core components of the Eyetracker class. This must be called AFTER the
+ * camera is selected, either through the {@link createSelector()} function or
+ * by manually setting the camera.
+ * @returns An object containing the Eyetracker's detector, video, and canvas.
+ */
+ async preload(): Promise {
+ let detector = await this.Eyetracker!.init();
+ let video = await this.Eyetracker!.createVideo();
+ let canvas = this.Eyetracker!.createDisplayCanvas();
+ return { detector, video, canvas };
+ }
+
+ //TODO: maybe allow multiple lines instead of forcing user to put one big one?
+ /**
+ * This will create a landing page that will inform the user of the nature
+ * of the usage of the Eyetracking software. The landing page is contained
+ * inside of a wrapper div with flex capabilities.
+ *
+ * @param id The id of the div that will be used to display the landing page.
+ * @param message The message that will be displayed on the landing page.
+ * @returns The landing page div.
+ */
+ createLanding(
+ id: string = "landing",
+ message: string = this.DEFAULT_MESSAGE
+ ): HTMLDivElement {
+ let landing = document.createElement("div");
+ landing.id = id;
+ landing.classList.add("landing");
+ landing.innerHTML = `${message}
`;
+
+ let wrapper = document.createElement("div");
+ wrapper.id = `${id}-wrapper`;
+ wrapper.append(landing);
+ wrapper.style.display = "flex";
+ wrapper.style.flexDirection = "column";
+ wrapper.style.textAlign = "center";
+
+ return wrapper;
+ }
+
+ /**
+ * This will create a selector that will allow for one to select a camera.
+ * This will automatically ask for permission and once finished, use
+ * {@link Eyetracker.setCamera()}.
+ *
+ * @param id The id of the selector.
+ * @param message The message that will be displayed on the button.
+ * @returns An array containing the selector and the button, in that order.
+ */
+ async createSelector(
+ id: string = "selector",
+ message: string = "Select a camera"
+ ): Promise> {
+ let selector = document.createElement("select");
+ selector.id = id;
+
+ await this.Eyetracker!.getCameraPermission();
+ const devices = await this.Eyetracker!.getListOfCameras();
+
+ if (devices.length !== 1) {
+ const blank = document.createElement("option");
+ blank.style.display = "none";
+ selector.appendChild(blank);
+ }
+
+ for (const d of devices) {
+ let option = document.createElement("option");
+ option.value = d.deviceId;
+ option.innerHTML = d.label;
+ selector.appendChild(option);
+ }
+
+ const btn = document.createElement("button");
+ btn.id = `${id}-btn`;
+ btn.innerHTML = `${message}`;
+ btn.addEventListener("click", async () => {
+ const cam = selector.options[selector.selectedIndex].value;
+ const selectedCamera = devices.filter((d) => d.deviceId === cam)[0];
+ await this.Eyetracker!.setCamera(selectedCamera);
+ selector.remove();
+ btn.remove();
+ });
+ return new Array(selector, btn);
+ }
+
+ /**
+ * Calibrate the eyetracker. This will automatically cycle through the points
+ * in 3s intervals, taking facial landmark data 1.5s into the point's appearance.
+ *
+ * PLEASE NOTE: To increase calibration accuracy, use more than 4 points. 9 is preferred.
+ *
+ * @param div The div that will be used to display the calibration.
+ * @param time The time between each point in the calibration process, with data collected at
+ * the middle of each interval. Default is 3s.
+ * @param points A list of points denoted in absolute coordinates that will be used in calibration.
+ * @returns An object containing the x and y coordinates of the calibration point,
+ * along with associated facial landmark data.
+ */
+ async calibrate(
+ div: HTMLDivElement,
+ time: number = 3000,
+ points: Array> = this.DEFAULT_POINTS
+ ): Promise> {
+ if (div === null) {
+ throw new Error("Please provide a div.");
+ } else {
+ div.innerHTML = "";
+ }
+ this.calDivUsed = div;
+
+ let finishedPoints: Array = [];
+
+ if (points.length < 4) {
+ console.warn(
+ "There are not a lot of points to calibrate, consider adding more or using the DEFAULT_POINTS."
+ );
+ }
+ await this.Eyetracker!.initVideoFrameLoop();
+
+ let calibrateLoop = setInterval(async () => {
+ div.innerHTML = "";
+ let point = points.shift();
+ if (point === undefined) {
+ clearInterval(calibrateLoop);
+ //TODO- Update this to return what exactly we need for the neural network.
+ console.log(await this.Eyetracker!.processCalibrationPoints()); // facial landmarks
+ return finishedPoints; // imageData + onset time
+ }
+ this.drawFixation(point[0], point[1], div);
+ let onsetTime = performance.now();
+
+ setTimeout(async () => {
+ //TODO- eventually we must update the API so that we can just gather the facial
+ //landmarks/image data using a field in eyetracker.
+ await this.Eyetracker!.detectFace();
+ let currentPoint: any = this.Eyetracker!.calibratePoint(
+ point![0],
+ point![1]
+ );
+ //TODO- let's find a way to prevent throwing this error, maybe explicitly defining?
+ // fixed with the :any cast, but we should still consider implementing types.
+ currentPoint.onsetTime = onsetTime;
+ finishedPoints.push(currentPoint);
+ }, time / 2);
+ }, time);
+ return finishedPoints;
+ }
+
+ /**
+ * This will create a fixation based off of the given coordinates, placing it in the div.
+ * The properties of the fixation are customizable through the {@link props} field.
+ *
+ * @param x The x coordinate of the fixation.
+ * @param y The y coordinate of the fixation.
+ * @param div The div that the fixation will be placed in.
+ */
+ drawFixation(x: number, y: number, div: HTMLDivElement) {
+ if (div === null || div === undefined) {
+ throw new Error("Please provide a div to draw the fixation in.");
+ }
+ if (div.parentElement === null || div.parentElement === undefined) {
+ throw new Error("Div cannot be document.body.");
+ }
+
+ let fixation = this.props.shape.bind(this)(x, y);
+ div.appendChild(fixation);
+ }
+
+ /**
+ * Creates a circle fixation centered absolutely on the given coordinates.
+ *
+ * @param x The x coordinate of the fixation.
+ * @param y The y coordinate of the fixation.
+ * @returns A circle fixation element.
+ */
+ drawCircleFixation(x: number, y: number) {
+ const adjustCircle = parseInt(this.props.diameter) / 2;
+ const circle = document.createElement("div");
+ circle.style.width = this.props.diameter;
+ circle.style.height = this.props.diameter;
+ circle.style.borderRadius = "50%";
+ circle.style.backgroundColor = this.props.color;
+ circle.style.position = "absolute";
+ circle.style.left = `calc(${x}% - ${adjustCircle}px)`;
+ circle.style.top = `calc(${y}% - ${adjustCircle}px)`;
+ return circle;
+ }
+
+ /**
+ * Creates a cross (+) fixation centered absolutely on the given coordinates.
+ *
+ * @param x The x coordinate of the fixation.
+ * @param y The y coordinate of the fixation.
+ * @returns A cross fixation element.
+ */
+ drawCrossFixation(x: number, y: number) {
+ const adjustCross = parseInt(this.props.crossThickness) / 2;
+ const cross1 = document.createElement("div");
+ cross1.style.width = this.props.crossThickness;
+ cross1.style.height = this.props.diameter;
+ cross1.style.backgroundColor = this.props.color;
+ cross1.style.position = "absolute";
+ cross1.style.left = `calc(${x}% - ${adjustCross}px)`;
+ cross1.style.top = `calc(${y}% - ${parseInt(this.props.diameter) / 2}px)`;
+
+ const cross2 = document.createElement("div");
+ cross2.style.width = this.props.diameter;
+ cross2.style.height = this.props.crossThickness;
+ cross2.style.backgroundColor = this.props.color;
+ cross2.style.position = "absolute";
+ cross2.style.left = `calc(${x}% - ${parseInt(this.props.diameter) / 2}px)`;
+ cross2.style.top = `calc(${y}% - ${adjustCross}px)`;
+
+ const wrapper = document.createElement("div");
+ wrapper.appendChild(cross1);
+ wrapper.appendChild(cross2);
+ return wrapper;
+ }
+
+ /**
+ * Creates a square fixation centered absolutely on the given coordinates.
+ *
+ * @param x The x coordinate of the fixation.
+ * @param y The y coordinate of the fixation.
+ * @returns A square fixation element.
+ */
+ drawSquareFixation(x: number, y: number) {
+ const adjustSquare = parseInt(this.props.diameter) / 2;
+ const square = document.createElement("div");
+ square.style.width = this.props.diameter;
+ square.style.height = this.props.diameter;
+ square.style.borderRadius = "50%";
+ square.style.backgroundColor = this.props.color;
+ square.style.position = "absolute";
+ square.style.left = `calc(${x}% - ${adjustSquare}px)`;
+ square.style.top = `calc(${y}% - ${adjustSquare}px)`;
+ return square;
+ }
+
+ /**
+ * Creates a reticule fixation centered absolutely on the given coordinates.
+ *
+ * @param x The x coordinate of the fixation.
+ * @param y The y coordinate of the fixation.
+ * @returns A reticule fixation element.
+ */
+ drawReticuleFixation(x: number, y: number) {
+ if (parseInt(this.props.diameter) <= 10) {
+ console.warn("Reticule diameter is small and will look odd.");
+ }
+ const adjustReticule = parseInt(this.props.diameter) / 2;
+ const outerCircle = document.createElement("div");
+ outerCircle.style.width = this.props.diameter;
+ outerCircle.style.height = this.props.diameter;
+ outerCircle.style.borderRadius = "50%";
+ outerCircle.style.backgroundColor = this.props.color;
+ outerCircle.style.position = "absolute";
+ outerCircle.style.left = `calc(${x}% - ${adjustReticule}px)`;
+ outerCircle.style.top = `calc(${y}% - ${adjustReticule}px)`;
+ outerCircle.style.zIndex = "1";
+
+ const adjustInner = adjustReticule / 2;
+ const innerCircle = document.createElement("div");
+ const innerSize = `${parseInt(this.props.diameter) / 2}px`;
+ innerCircle.style.width = innerSize;
+ innerCircle.style.height = innerSize;
+ innerCircle.style.borderRadius = "50%";
+ innerCircle.style.backgroundColor = this.props.color;
+ innerCircle.style.position = "absolute";
+ innerCircle.style.left = `calc(${x}% - ${adjustInner}px)`;
+ innerCircle.style.top = `calc(${y}% - ${adjustInner}px)`;
+ innerCircle.style.zIndex = "3";
+
+ const eraseColor = "white";
+ const eraseThickness = parseInt(this.props.diameter) / 4;
+ const eraseSize = `${eraseThickness}px`;
+ const adjustErase = eraseThickness / 2;
+
+ const eCross1 = document.createElement("div");
+ eCross1.style.width = eraseSize;
+ eCross1.style.height = this.props.diameter;
+ eCross1.style.backgroundColor = eraseColor;
+ eCross1.style.position = "absolute";
+ eCross1.style.left = `calc(${x}% - ${adjustErase}px)`;
+ eCross1.style.top = `calc(${y}% - ${parseInt(this.props.diameter) / 2}px)`;
+ eCross1.style.zIndex = "2";
+
+ const eCross2 = document.createElement("div");
+ eCross2.style.width = this.props.diameter;
+ eCross2.style.height = eraseSize;
+ eCross2.style.backgroundColor = eraseColor;
+ eCross2.style.position = "absolute";
+ eCross2.style.left = `calc(${x}% - ${parseInt(this.props.diameter) / 2}px)`;
+ eCross2.style.top = `calc(${y}% - ${adjustErase}px)`;
+ eCross2.style.zIndex = "2";
+
+ const wrapper = document.createElement("div");
+ wrapper.appendChild(outerCircle);
+ wrapper.appendChild(innerCircle);
+ wrapper.appendChild(eCross1);
+ wrapper.appendChild(eCross2);
+ return wrapper;
+ }
+
+ //TODO- update this if we need it or not
+ /**
+ * This is a helper function that will translate given absolute coordinates to standard coordinates.
+ * @param points Points in absolute coordinates, generated by the {@link calibrate()} function.
+ * @returns
+ */
+ translateToStandard(points: Array): Array {
+ if (this.calDivUsed === null || this.calDivUsed === undefined) {
+ throw new Error(
+ "No calibration div was used, please use the calibrate function."
+ );
+ }
+ let newPoints: Array = [];
+ points.forEach((p) => {
+ let newPoint = {
+ //@ts-ignore
+ x: p.x * this.calDivUsed!.offsetWidth,
+ //@ts-ignore
+ y: p.y * this.calDivUsed!.offsetWidth,
+ //@ts-ignore
+ onsetTime: p.onsetTime,
+ }; // TODO- we should really set a type to these points
+ newPoints.push(newPoint);
+ });
+ return newPoints;
+ }
+}
diff --git a/src/Eyetracker.ts b/src/Eyetracker.ts
index 101d4dc..03c45f7 100644
--- a/src/Eyetracker.ts
+++ b/src/Eyetracker.ts
@@ -25,14 +25,14 @@ export class Eyetracker {
private model: MediaPipeFaceMesh | undefined;
/** Shows whether or not the overlay should be displayed. */
private overlay: boolean = true;
- /** */
+ /** The frames, timestamped with rVF callbacks, generated by {@link initVideoFrameLoop()} */
private frames: Array<{
imageData: ImageData;
timestamp: DOMHighResTimeStamp;
}> = [];
/** */
public onFrameUpdateCallbackList: Array = [];
- /** */
+ /** Determines whether or not {@link initVideoFrameLoop()}'s rVFCallbacks are paused or not. */
public frameUpdatePaused: boolean = false;
/**
@@ -101,11 +101,14 @@ export class Eyetracker {
* @param Id An id attribute to which can be used to identify the video later
* @returns An HTMLVideoElement whose source is the stream from the previously selected camera
*/
- async createVideo(Id: string): Promise {
+ async createVideo(Id: string = "et-video"): Promise {
let video: HTMLVideoElement = document.createElement("video");
video.setAttribute("id", Id);
video.style.transform = "scaleX(-1)";
- (video.srcObject as undefined | MediaProvider | null) = this.stream;
+ if (this.stream === null || this.stream === undefined) {
+ throw new Error("No camera stream detected, have you run setCamera()?");
+ }
+ video.srcObject = this.stream;
video.autoplay = true;
return new Promise((resolve) => {
video.onloadedmetadata = () => {
@@ -122,28 +125,28 @@ export class Eyetracker {
* @param Id An id attribute to which can be used to identify the video later
* @returns A mirrored HTMLCanvasElement with matching proportions to the created video
*/
- createDisplayCanvas(Id: string): HTMLCanvasElement | undefined {
+ createDisplayCanvas(Id: string = "et-canvas"): HTMLCanvasElement | undefined {
+ if (this.video === null || this.video === undefined) {
+ throw new Error("No video stream detected, have you run createVideo()?");
+ }
+
let canvas: HTMLCanvasElement = document.createElement("canvas");
canvas.setAttribute("id", Id);
this.canvas = canvas;
- if (this.video != null) {
- canvas.height = this.video.height;
- canvas.width = this.video.width;
- this.canvas = canvas;
- var ctx: CanvasRenderingContext2D | null = canvas.getContext("2d");
- if (ctx != null) {
- ctx.translate(canvas.width, 0);
- ctx.scale(-1, 1);
- ctx.fillStyle = "green";
- (this.ctx as undefined | CanvasRenderingContext2D | null) = ctx;
- return canvas;
- } else {
- console.log("canvas.getContext('2d') return null");
- return canvas;
- }
- } else {
- console.log('Undefined Property "this.video"');
+ canvas.height = this.video.height;
+ canvas.width = this.video.width;
+ this.canvas = canvas;
+ var ctx: CanvasRenderingContext2D | null = canvas.getContext("2d");
+ if (ctx === null) {
+ throw new Error(
+ "Could not create canvas context, have you already declared ctx with a different type of context?"
+ );
}
+ ctx.translate(canvas.width, 0);
+ ctx.scale(-1, 1);
+ ctx.fillStyle = "green";
+ this.ctx = ctx;
+ return canvas;
}
// Need to test this out
@@ -152,21 +155,29 @@ export class Eyetracker {
* @param canvas An existing canvas whose proprotions should match the video
*/
setDisplayCanvas(canvas: HTMLCanvasElement): void {
- let video: HTMLVideoElement | undefined = this.video;
+ if (this.video === null || this.video === undefined) {
+ throw new Error("No video stream detected, have you run createVideo()?");
+ }
+ if (this.canvas === null || this.canvas === undefined) {
+ throw new Error(
+ "No canvas detected, have you run createDisplayCanvas()?"
+ );
+ }
+
+ let video = this.video;
this.canvas = canvas;
- if (canvas != undefined && video != undefined) {
- canvas.height = video.height;
- canvas.width = video.width;
- var ctx: CanvasRenderingContext2D | null = canvas.getContext("2d");
- if (ctx != null) {
- ctx.translate(canvas.width, 0);
- ctx.scale(-1, 1);
- ctx.fillStyle = "green";
- }
- (this.ctx as undefined | CanvasRenderingContext2D | null) = ctx;
- } else {
- console.log('/"this.canvas/", /"this.video/" Undefined');
+ canvas.height = video.height;
+ canvas.width = video.width;
+ var ctx: CanvasRenderingContext2D | null = canvas.getContext("2d");
+ if (ctx === null || ctx === undefined) {
+ throw new Error(
+ "Could not create canvas context, have you already declared ctx with a different type of context?"
+ );
}
+ ctx.translate(canvas.width, 0);
+ ctx.scale(-1, 1);
+ ctx.fillStyle = "green";
+ this.ctx = ctx;
}
/**
@@ -179,11 +190,16 @@ export class Eyetracker {
video: HTMLVideoElement = this.video!
): void {
let ctx: CanvasRenderingContext2D | null = canvas.getContext("2d");
- if (ctx != undefined && video != undefined) {
- ctx.drawImage(video, 0, 0);
- } else {
- console.log('"this.ctx", "this.video" Undefined');
+ if (ctx === null || ctx === undefined) {
+ throw new Error(
+ "Could not create canvas context, have you already declared ctx with a different type of context?"
+ );
}
+ if (video === null || video === undefined) {
+ throw new Error("No video stream detected, have you run createVideo()?");
+ }
+
+ ctx.drawImage(video, 0, 0);
}
/**
@@ -227,6 +243,7 @@ export class Eyetracker {
},
canvas: HTMLCanvasElement = this.canvas!
): void {
+ // ?? why is this in a try-catch?
try {
let ctx: CanvasRenderingContext2D | null = canvas.getContext("2d");
@@ -315,7 +332,7 @@ export class Eyetracker {
return point;
}
- async processCalibrationPoints(): Promise {
+ async processCalibrationPoints(): Promise> {
let processedPoints = [];
for (let i = 0; i < this.calibrationPoints.length; i++) {
let predictions = await this.model!.estimateFaces({
@@ -330,6 +347,7 @@ export class Eyetracker {
});
}
this.processedCalibrationPoints = processedPoints;
+ return processedPoints;
}
/**
@@ -376,7 +394,6 @@ export class Eyetracker {
canvas: HTMLCanvasElement = this.canvas!
) {
let ctx = canvas.getContext("2d");
- let frames = this.frames;
let Eyetracker = this;
async function repeatDetection(now: DOMHighResTimeStamp, metadata: object) {
if (!Eyetracker.frameUpdatePaused) {
diff --git a/src/index.ts b/src/index.ts
index d85c202..931518e 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -1,5 +1,10 @@
import { Eyetracker } from "./Eyetracker";
+import { Components } from "./Components";
+/**
+ * Create a new instance of the eye tracker.
+ * @returns A new instance of the eye tracker.
+ */
export function initEyetracker() {
if (!("requestVideoFrameCallback" in HTMLVideoElement.prototype)) {
window.alert(
@@ -8,3 +13,12 @@ export function initEyetracker() {
}
return new Eyetracker();
}
+
+/**
+ * Creates an instance of a components module.
+ * @param et The eye tracker instance.
+ * @returns A new instance of the components class.
+ */
+export function initComponents(et: Eyetracker) {
+ return new Components(et);
+}
diff --git a/tests/components.test.ts b/tests/components.test.ts
new file mode 100644
index 0000000..bb8516f
--- /dev/null
+++ b/tests/components.test.ts
@@ -0,0 +1,22 @@
+import { initEyetracker, initComponents } from "../src/index";
+
+describe("components", () => {
+ let eye: any;
+ let comp: any;
+
+ beforeEach(() => {
+ global.alert = jest.fn();
+ eye = initEyetracker();
+ comp = initComponents(eye);
+ });
+
+ test("landing page generated and customizable", () => {
+ document.body.innerHTML = `
`;
+ let landing = comp.createLanding("test-land", "Landing page.");
+ document.getElementById("test")!.appendChild(landing);
+
+ let innerLanding = document.getElementById("test-land");
+ expect(innerLanding).toBeTruthy();
+ expect(innerLanding!.innerHTML).toEqual(`Landing page.
`);
+ });
+});
diff --git a/tests/index.test.ts b/tests/index.test.ts
index 9973466..4966a4a 100644
--- a/tests/index.test.ts
+++ b/tests/index.test.ts
@@ -1,6 +1,7 @@
-import { initEyetracker } from "../src/index";
+import { initEyetracker, initComponents } from "../src/index";
test("test adds two numbers", () => {
+ global.alert = jest.fn(); // TODO: figure out a way to implement alert or just remove it!
const eye = initEyetracker();
expect(eye.add(2, 2)).toEqual(4);
});