From a512f5a8163d3701325a3d8b13c2cd2601550296 Mon Sep 17 00:00:00 2001 From: tharvik Date: Wed, 11 Sep 2024 16:13:13 +0200 Subject: [PATCH 01/31] server/tests: bump timeout --- server/tests/validator.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/tests/validator.spec.ts b/server/tests/validator.spec.ts index 0b994b4a7..dd6b84631 100644 --- a/server/tests/validator.spec.ts +++ b/server/tests/validator.spec.ts @@ -79,5 +79,5 @@ describe("validator", () => { } expect(hits / size).to.be.greaterThan(0.3); - }); + }).timeout("10s"); }); From 0968620c2843ccb399ef34a54ed1fb93c17e8641 Mon Sep 17 00:00:00 2001 From: tharvik Date: Tue, 22 Oct 2024 17:09:41 +0200 Subject: [PATCH 02/31] server/test/status: avoid arbitrary wait --- server/tests/e2e/federated.spec.ts | 102 ++++++++++++++--------------- server/tests/e2e/utils.ts | 25 +++++-- 2 files changed, 66 insertions(+), 61 deletions(-) diff --git a/server/tests/e2e/federated.spec.ts b/server/tests/e2e/federated.spec.ts index bb6e2d756..651779493 100644 --- a/server/tests/e2e/federated.spec.ts +++ b/server/tests/e2e/federated.spec.ts @@ -3,12 +3,14 @@ import { List, Repeat } from "immutable"; import type * as http from "node:http"; import path from "node:path"; -import type { RoundLogs, RoundStatus, WeightsContainer } from "@epfml/discojs"; +import type { RoundStatus, WeightsContainer } from "@epfml/discojs"; import { Disco, defaultTasks } from "@epfml/discojs"; import { loadCSV, loadImagesInDir, loadText } from "@epfml/discojs-node"; import { Server } from "../../src/index.js"; +import { Queue } from "./utils.js"; + // Array.fromAsync not yet widely used (2024) async function arrayFromAsync(iter: AsyncIterable): Promise { const ret: T[] = []; @@ -189,9 +191,13 @@ describe("end-to-end federated", () => { ); const lusCovidTask = defaultTasks.lusCovid.getTask(); - lusCovidTask.trainingInformation.epochs = 8; - lusCovidTask.trainingInformation.roundDuration = 2; - lusCovidTask.trainingInformation.minNbOfParticipants = 2; + lusCovidTask.trainingInformation = { + ...lusCovidTask.trainingInformation, + scheme: "federated", + epochs: 8, + roundDuration: 2, + minNbOfParticipants: 2, + } const [positive, negative] = [ ( @@ -225,93 +231,81 @@ describe("end-to-end federated", () => { * - User 3 joins * - User 2 & 3 leave */ - const statusUpdateTime = 500 // Create User 1 - const discoUser1 = new Disco(lusCovidTask, url, { scheme: "federated" }); - let statusUser1: RoundStatus | undefined; - discoUser1.on("status", status => { statusUser1 = status }) + const discoUser1 = new Disco(lusCovidTask, url, { }); + const statusUser1 = new Queue(); + discoUser1.on("status", (status) => statusUser1.put(status)); const generatorUser1 = discoUser1.trainByRound(["image", dataset]) // Have User 1 join the task and train locally for one round - const logUser1Round1 = await generatorUser1.next() - expect(logUser1Round1.done).to.be.false - // User 1 did a) and b) so their status should be Training - expect((logUser1Round1.value as RoundLogs).participants).equal(1) + await generatorUser1.next() + expect(await statusUser1.next()).equal("local training") // Calling next() a 2nd time makes User 1 go to c) where the client should // stay stuck awaiting until another participant joins const logUser1Round2Promise = generatorUser1.next() - await new Promise((res,_) => setTimeout(res, statusUpdateTime)) // Wait some time for the status to update - expect(statusUser1).equal("not enough participants") + expect(await statusUser1.next()).equal("not enough participants") // Create User 2 - const discoUser2 = new Disco(lusCovidTask, url, { scheme: "federated" }); - let statusUser2: RoundStatus | undefined; - discoUser2.on("status", status => { statusUser2 = status }) + const discoUser2 = new Disco(lusCovidTask, url, { }); + const statusUser2 = new Queue(); + discoUser2.on("status", (status) => statusUser2.put(status)); const generatorUser2 = discoUser2.trainByRound(["image", dataset]) // Have User 2 join the task and train for one round - const logUser2Round1 = await generatorUser2.next() - expect(logUser2Round1.done).to.be.false - expect((logUser2Round1.value as RoundLogs).participants).equal(2) + await generatorUser2.next() // User 2 did a) and b) - expect(statusUser2).equal("local training") + expect(await statusUser1.next()).equal("local training") + expect(await statusUser2.next()).equal("local training") // User 1 is still in c) now waiting for user 2 to share their local update // and for the server to aggregate the local updates - expect(statusUser1).equal("updating model") + expect(await statusUser1.next()).equal("updating model") // Proceed with round 2 + // the server should answer with the new global weights // and users should train locally on the new weights - const logUser2Round2 = await generatorUser2.next() - const logUser1Round2 = await logUser1Round2Promise // the promise can resolve now - expect(logUser1Round2.done).to.be.false - expect(logUser2Round2.done).to.be.false - expect((logUser1Round2.value as RoundLogs).participants).equal(2) - expect((logUser2Round2.value as RoundLogs).participants).equal(2) + await Promise.all([logUser1Round2Promise, generatorUser2.next()]) // User 1 and 2 did c), a) and b) - expect(statusUser1).equal("local training") - expect(statusUser2).equal("local training") + expect(await statusUser2.next()).equal("updating model") + expect(await statusUser1.next()).equal("local training") + expect(await statusUser2.next()).equal("local training") + + // Make user 2 go to c) + const logUser2Round3Promise = generatorUser2.next() + expect(await statusUser2.next()).equal("updating model") // Have user 1 quit the session await discoUser1.close() - // Make user 2 go to c) - const logUser2Round3Promise = generatorUser2.next() - await new Promise((res, _) => setTimeout(res, statusUpdateTime)) // Wait some time for the status to update - expect(statusUser2).equal("not enough participants") + expect(await statusUser2.next()).equal("not enough participants") // Create User 3 - const discoUser3 = new Disco(lusCovidTask, url, { scheme: "federated" }); - let statusUser3: RoundStatus | undefined; - discoUser3.on("status", status => { statusUser3 = status }) + const discoUser3 = new Disco(lusCovidTask, url, { }); + const statusUser3 = new Queue(); + discoUser3.on("status", (status) => statusUser3.put(status)); const generatorUser3 = discoUser3.trainByRound(["image", dataset]) // User 3 joins mid-training and trains one local round - const logUser3Round1 = await generatorUser3.next() - expect(logUser3Round1.done).to.be.false - expect((logUser3Round1.value as RoundLogs).participants).equal(2) - // User 3 did a) and b) - expect(statusUser3).equal("local training") + await generatorUser3.next() + expect(await statusUser3.next()).equal("local training") + // User 2 is still in c) waiting for user 3 to share their local update // and for the server to aggregate the local updates - expect(statusUser2).equal("updating model") + expect(await statusUser2.next()).equal("updating model") // User 3 sends their weights to the server - const logUser3Round3 = await generatorUser3.next() + await Promise.all([logUser2Round3Promise, generatorUser3.next()]) + expect(await statusUser3.next()).equal("updating model") + // the server should accept user 3's weights (should not be outdated) and aggregate the global weights - const logUser2Round3 = await logUser2Round3Promise // the promise can resolve now - if (logUser3Round3.done || logUser2Round3.done) - throw Error("User 1 or 2 finished training at the 3nd round") - expect(logUser2Round3.value.participants).equal(2) - expect(logUser3Round3.value.participants).equal(2) // both user 2 and 3 did c), a) and are now in b) - expect(statusUser2).equal("local training") - expect(statusUser3).equal("local training") - + expect(await statusUser2.next()).equal("local training") + expect(await statusUser3.next()).equal("local training") + await discoUser2.close() - await new Promise((res, _) => setTimeout(res, statusUpdateTime)) // Wait some time for the status to update - expect(statusUser3).equal("not enough participants") + expect(await statusUser3.next()).equal("not enough participants") + await discoUser3.close() }).timeout("30s"); }); diff --git a/server/tests/e2e/utils.ts b/server/tests/e2e/utils.ts index 342491a4b..adc76bb88 100644 --- a/server/tests/e2e/utils.ts +++ b/server/tests/e2e/utils.ts @@ -1,20 +1,31 @@ -import { List } from 'immutable'; +import { List } from "immutable"; export class Queue { - #content = List(); + #content = List<[index: number, T]>(); + // keep track of what was added and asked for + #index = { head: 0, tail: 0 }; put(e: T) { - this.#content = this.#content.push(e); + this.#content = this.#content.push([this.#index.tail, e]); + this.#index.tail++; } async next(): Promise { + const index = this.#index.head; + this.#index.head++; + for (;;) { const ret = this.#content.first(); - if (ret !== undefined) { - this.#content = this.#content.shift() - return ret + if (ret !== undefined && ret[0] > index) + throw new Error("assertion failed: head's index bigger than ours"); + + // check that it is intended for us + if (ret?.[0] === index) { + this.#content = this.#content.shift(); + return ret[1]; } + await new Promise((resolve) => setTimeout(resolve, 10)); } } -} \ No newline at end of file +} From c19b42c86c4ddc9eadb2a5f58bfa89b4f4c34218 Mon Sep 17 00:00:00 2001 From: tharvik Date: Fri, 23 Aug 2024 12:29:21 +0200 Subject: [PATCH 03/31] discojs/processing: expand --- cli/src/benchmark_gpt.ts | 2 +- discojs/package.json | 2 + discojs/src/dataset/dataset.ts | 4 +- discojs/src/index.ts | 2 +- discojs/src/processing.ts | 140 ------------------- discojs/src/processing/image.spec.ts | 28 ++++ discojs/src/processing/image.ts | 141 +++++++++++++++++++ discojs/src/processing/index.ts | 90 +++++++++++++ discojs/src/processing/tabular.ts | 40 ++++++ discojs/src/processing/text.ts | 47 +++++++ discojs/src/types.ts | 34 ++++- package-lock.json | 195 +++++++++++++++++++++++++++ 12 files changed, 581 insertions(+), 144 deletions(-) delete mode 100644 discojs/src/processing.ts create mode 100644 discojs/src/processing/image.spec.ts create mode 100644 discojs/src/processing/image.ts create mode 100644 discojs/src/processing/index.ts create mode 100644 discojs/src/processing/tabular.ts create mode 100644 discojs/src/processing/text.ts diff --git a/cli/src/benchmark_gpt.ts b/cli/src/benchmark_gpt.ts index f4faab276..fc99ba8ed 100644 --- a/cli/src/benchmark_gpt.ts +++ b/cli/src/benchmark_gpt.ts @@ -99,7 +99,7 @@ async function main(args: Required): Promise { .map((batch) => tf.tidy(() => ({ xs: tf.tensor2d( - batch.map((tokens) => tokens.slice(0, -1)).toArray(), + batch.map((tokens) => tokens.slice(0, -1).toArray()).toArray(), ), ys: tf.stack( batch diff --git a/discojs/package.json b/discojs/package.json index 7f208fb68..aae6a4790 100644 --- a/discojs/package.json +++ b/discojs/package.json @@ -19,6 +19,8 @@ }, "homepage": "https://github.com/epfml/disco#readme", "dependencies": { + "@jimp/core": "1", + "@jimp/plugin-resize": "1", "@msgpack/msgpack": "^3.0.0-beta2", "@tensorflow/tfjs": "4", "@xenova/transformers": "2", diff --git a/discojs/src/dataset/dataset.ts b/discojs/src/dataset/dataset.ts index 3c8183356..f43a1b344 100644 --- a/discojs/src/dataset/dataset.ts +++ b/discojs/src/dataset/dataset.ts @@ -1,5 +1,7 @@ import { List } from "immutable"; +import type { Batched } from "../index.js"; + type DatasetLike = | AsyncIterable | Iterable @@ -117,7 +119,7 @@ export class Dataset implements AsyncIterable { * * @param size count of element per chunk */ - batch(size: number): Dataset> { + batch(size: number): Dataset> { if (size <= 0 || !Number.isInteger(size)) throw new Error("invalid size"); const content = { diff --git a/discojs/src/index.ts b/discojs/src/index.ts index 014001176..867bcfee4 100644 --- a/discojs/src/index.ts +++ b/discojs/src/index.ts @@ -24,4 +24,4 @@ export { Dataset } from "./dataset/index.js"; export * from "./dataset/types.js"; // TODO merge with above export * from "./types.js"; -export * as processing from "./processing.js"; +export * as processing from "./processing/index.js"; diff --git a/discojs/src/processing.ts b/discojs/src/processing.ts deleted file mode 100644 index 71f5d64ee..000000000 --- a/discojs/src/processing.ts +++ /dev/null @@ -1,140 +0,0 @@ -/** Dataset shapers, convenient to map with */ - -import { PreTrainedTokenizer } from "@xenova/transformers"; -import { List, Repeat, Seq } from "immutable"; -import { Image } from "./dataset/image.js"; - -/** - * Convert a string to a number - * - * @throws if it isn't written as a number - */ -export function convertToNumber(raw: string): number { - const num = Number.parseFloat(raw); - if (Number.isNaN(num)) throw new Error(`unable to parse "${raw}" as number`); - return num; -} - -/** - * Return the named field of an object with string values - * - * @throws if the named field isn't there - */ -export function extractColumn( - row: Partial>, - column: string, -): string { - const raw = row[column]; - if (raw === undefined) throw new Error(`${column} not found in row`); - return raw; -} - -/** - * Return the index of the element in the given list - * - * @throws if not found - */ -export function indexInList(element: string, elements: List): number { - const ret = elements.indexOf(element); - if (ret === -1) throw new Error(`${element} not found in list`); - return ret; -} - -function isArrayOfNumber(raw: unknown): raw is number[] { - return Array.isArray(raw) && raw.every((e) => typeof e === "number"); -} - -/** - * Tokenize and truncates input strings - * - * @param length number of tokens - * @returns encoded string in an array of token, size of max_length - */ -export function tokenizeAndLeftPad( - line: string, - tokenizer: PreTrainedTokenizer, - length: number, -): number[] { - if (!Number.isInteger(length)) throw new Error("length should be an integer"); - - // Transformers.js currently only supports right padding while we need left for text generation - // Right padding should be supported in the future, once it is, we can directly pad while tokenizing - // https://github.com/xenova/transformers.js/blob/8804c36591d11d8456788d1bb4b16489121b3be2/src/tokenizers.js#L2517 - const tokenized: unknown = tokenizer(line, { - padding: false, - truncation: true, - return_tensor: false, - max_length: length, - }); - - if ( - typeof tokenized !== "object" || - tokenized === null || - !("input_ids" in tokenized) || - !isArrayOfNumber(tokenized.input_ids) - ) - throw new Error("tokenizer returns unexcepted type"); - const tokens: number[] = tokenized.input_ids; - - const paddingSize = length - tokens.length; - if (paddingSize < 0) - throw new Error("tokenized returned more token than excepted"); - - const padding = new Array(paddingSize); - padding.fill(tokenizer.pad_token_id); - const padded = padding.concat(tokens); - - return padded; -} - -/** Remove the alpha channel of an image */ -export function removeAlpha( - image: Image<4, W, H>, -): Image<3, W, H>; -export function removeAlpha< - D extends 1 | 3, - W extends number, - H extends number, ->(image: Image): Image; -export function removeAlpha( - image: Image<1 | 3 | 4, W, H>, -): Image<1 | 3, W, H> { - switch (image.depth) { - case 1: - case 3: - return new Image(image.data, image.width, image.height, image.depth); - case 4: - return new Image( - image.data.filter((_, i) => i % 4 !== 3), - image.width, - image.height, - 3, - ); - } -} - -/** Convert monochrome images to multicolor */ -export function expandToMulticolor( - image: Image<1, W, H>, -): Image<3, W, H>; -export function expandToMulticolor< - D extends 3 | 4, - W extends number, - H extends number, ->(image: Image<1 | D, W, H>): Image; -export function expandToMulticolor( - image: Image<1 | 3 | 4, W, H>, -): Image<3 | 4, W, H> { - switch (image.depth) { - case 1: - return new Image( - Uint8Array.from(Seq(image.data).flatMap((v) => Repeat(v, 3))), - image.width, - image.height, - 3, - ); - case 3: - case 4: - return new Image(image.data, image.width, image.height, image.depth); - } -} diff --git a/discojs/src/processing/image.spec.ts b/discojs/src/processing/image.spec.ts new file mode 100644 index 000000000..dff78d5d4 --- /dev/null +++ b/discojs/src/processing/image.spec.ts @@ -0,0 +1,28 @@ +import { expect } from "chai"; +import { Repeat, Seq } from "immutable"; + +import { Image } from "../index.js"; + +import { removeAlpha, resize } from "./image.js"; + +describe("resize", () => { + it("doesn't change with same image dimensions", () => { + const base = new Image(Uint8Array.of(1, 2, 3, 4, 5, 6), 2, 1, 3); + + const resized = resize(base.width, base.height, base); + + expect(removeAlpha(resized)).to.be.deep.equal(base); + }); + + it("copies single pixel image to every pixel", () => { + const base = new Image(Uint8Array.of(1, 2, 3), 1, 1, 3); + + const resized = resize(2, 3, base); + + expect(removeAlpha(resized).data).to.have.same.ordered.members( + Repeat(Seq.Indexed.of(1, 2, 3), 6) + .flatten() + .toArray(), + ); + }); +}); diff --git a/discojs/src/processing/image.ts b/discojs/src/processing/image.ts new file mode 100644 index 000000000..4a78079d1 --- /dev/null +++ b/discojs/src/processing/image.ts @@ -0,0 +1,141 @@ +import { Repeat, Seq } from "immutable"; +import { createJimp } from "@jimp/core"; +import * as jimpResize from "@jimp/plugin-resize"; + +import { Image } from "../index.js"; + +/** Image where intensity is represented in the range 0..1 */ +export class NormalizedImage< + D extends 1 | 3 | 4 = 1 | 3 | 4, + W extends number = number, + H extends number = number, +> { + // private as it doesn't check that array content is valid + private constructor( + public readonly data: Readonly, + public readonly width: W, + public readonly height: H, + public readonly depth: D, + ) { + if (data.length != width * height * depth) + throw new Error("data isn't of expected size"); + } + + static from< + D extends 1 | 3 | 4 = 1 | 3 | 4, + W extends number = number, + H extends number = number, + >(image: Image): NormalizedImage { + return new NormalizedImage( + Float32Array.from(image.data).map((v) => v / 255), + image.width, + image.height, + image.depth, + ); + } +} + +/** Add a full opaque alpha channel to an image */ +function addAlpha( + image: Image<3 | 4, W, H>, +): Image<4, W, H> { + switch (image.depth) { + case 3: + return new Image( + Uint8Array.from( + // we are adding a channel, so for every 3 byte in the base image, + // we need to add a fourth. we choose to "expand" the last channel + // to two value, the channel base value and the transparency. + // let's say we want to add a byte A to the bytestring RGB + // [R, G, B] -> [[R], [G], [B, A]] -> [R, G, B, A] + Seq(image.data).flatMap((v, i) => { + const OPAQUE = 0xff; + if (i % 3 !== 2) return [v]; + else return [v, OPAQUE]; + }), + ), + image.width, + image.height, + 4, + ); + case 4: + return new Image(image.data, image.width, image.height, image.depth); + } +} + +/** Remove the alpha channel of an image */ +export function removeAlpha( + image: Image<4, W, H>, +): Image<3, W, H>; +export function removeAlpha< + D extends 1 | 3, + W extends number, + H extends number, +>(image: Image): Image; +export function removeAlpha( + image: Image<1 | 3 | 4, W, H>, +): Image<1 | 3, W, H> { + switch (image.depth) { + case 1: + case 3: + return new Image(image.data, image.width, image.height, image.depth); + case 4: + return new Image( + image.data.filter((_, i) => i % 4 !== 3), + image.width, + image.height, + 3, + ); + } +} + +/** Convert monochrome images to multicolor */ +export function expandToMulticolor( + image: Image<1, W, H>, +): Image<3, W, H>; +export function expandToMulticolor< + D extends 3 | 4, + W extends number, + H extends number, +>(image: Image<1 | D, W, H>): Image; +export function expandToMulticolor( + image: Image<1 | 3 | 4, W, H>, +): Image<3 | 4, W, H> { + switch (image.depth) { + case 1: + return new Image( + Uint8Array.from(Seq(image.data).flatMap((v) => Repeat(v, 3))), + image.width, + image.height, + 3, + ); + case 3: + case 4: + return new Image(image.data, image.width, image.height, image.depth); + } +} + +export function resize( + width: W, + height: H, + image: Image, +): Image<4, W, H> { + const Jimp = createJimp({ + plugins: [jimpResize.methods], + }); + + const resized = new Jimp(addAlpha(expandToMulticolor(image))).resize({ + w: width, + h: height, + }); + + return new Image(new Uint8Array(resized.bitmap.data), width, height, 4); +} + +export function normalize< + D extends 1 | 3 | 4, + W extends number, + H extends number, +>(image: Image): NormalizedImage { + return NormalizedImage.from(image); +} diff --git a/discojs/src/processing/index.ts b/discojs/src/processing/index.ts new file mode 100644 index 000000000..1a9055733 --- /dev/null +++ b/discojs/src/processing/index.ts @@ -0,0 +1,90 @@ +/** Dataset shapers, convenient to map with */ + +import { List } from "immutable"; +import { AutoTokenizer } from "@xenova/transformers"; + +import type { + Tabular, + Task, + TypedLabeledDataset, + TypedPreprocessedLabeledDataset, +} from "../index.js"; + +import * as processing from "./index.js"; + +export * from "./image.js"; +export * from "./tabular.js"; +export * from "./text.js"; + +export async function preprocess( + task: Task, + [t, dataset]: TypedLabeledDataset, +): Promise { + switch (t) { + case "image": { + const { LABEL_LIST, IMAGE_H, IMAGE_W } = task.trainingInformation; + if ( + IMAGE_H === undefined || + IMAGE_W === undefined || + LABEL_LIST === undefined + ) + throw new Error("task is missing fields for image dataset"); + + return [ + "image", + dataset.map(([image, label]) => [ + processing.normalize( + processing.removeAlpha(processing.resize(IMAGE_W, IMAGE_H, image)), + ), + processing.indexInList(label, LABEL_LIST), + ]), + ]; + } + case "tabular": { + const { inputColumns, outputColumns } = task.trainingInformation; + if (inputColumns === undefined || outputColumns === undefined) + throw new Error("tabular task without input and output columns"); + + return [ + "tabular", + dataset.map((row) => [ + tabularToNumbers(inputColumns, row), + tabularToNumbers(outputColumns, row), + ]), + ]; + } + case "text": { + const tokenizerName = task.trainingInformation.tokenizer; + if (typeof tokenizerName !== "string") + throw Error( + "no tokenizer name specified in the task training information", + ); + const tokenizer = await AutoTokenizer.from_pretrained(tokenizerName); + const totalTokenCount = + task.trainingInformation.maxSequenceLength ?? + (tokenizer.model_max_length as number); + + return [ + "text", + dataset + .map((line) => + processing.tokenizeAndLeftPad(line, tokenizer, totalTokenCount), + ) + .map((tokens) => [tokens.pop(), tokens.shift()]), + ]; + } + } +} + +function tabularToNumbers( + columns: Iterable, + row: Tabular, +): List { + return ( + List(columns) + .map((column) => processing.extractColumn(row, column)) + // TODO sanitization doesn't care about column distribution + .map((v) => (v !== "" ? v : "0")) + .map(processing.convertToNumber) + ); +} diff --git a/discojs/src/processing/tabular.ts b/discojs/src/processing/tabular.ts new file mode 100644 index 000000000..685baca03 --- /dev/null +++ b/discojs/src/processing/tabular.ts @@ -0,0 +1,40 @@ +import { List } from "immutable"; + +/** + * Convert a string to a number + * + * @throws if it isn't written as a number + */ +export function convertToNumber(raw: string): number { + const num = Number.parseFloat(raw); + if (Number.isNaN(num)) throw new Error(`unable to parse "${raw}" as number`); + return num; +} + +/** + * Return the named field of an object with string values + * + * @throws if the named field isn't there + */ +export function extractColumn( + row: Partial>, + column: string, +): string { + const raw = row[column]; + if (raw === undefined) throw new Error(`${column} not found in row`); + return raw; +} + +/** + * Return the index of the element in the given list + * + * @throws if not found + */ +export function indexInList( + element: string, + elements: List | Array, +): number { + const ret = elements.indexOf(element); + if (ret === -1) throw new Error(`${element} not found in list`); + return ret; +} diff --git a/discojs/src/processing/text.ts b/discojs/src/processing/text.ts new file mode 100644 index 000000000..4bdce82e0 --- /dev/null +++ b/discojs/src/processing/text.ts @@ -0,0 +1,47 @@ +import { List, Repeat } from "immutable"; +import { PreTrainedTokenizer } from "@xenova/transformers"; + +function isArrayOfNumber(raw: unknown): raw is number[] { + return Array.isArray(raw) && raw.every((e) => typeof e === "number"); +} + +type Token = number; + +/** + * Tokenize and truncates input strings + * + * @param length number of tokens + * @returns encoded string in an array of token, size of max_length + */ +export function tokenizeAndLeftPad( + line: string, + tokenizer: PreTrainedTokenizer, + length: number, +): List { + if (!Number.isInteger(length)) throw new Error("length should be an integer"); + + // Transformers.js currently only supports right padding while we need left for text generation + // Right padding should be supported in the future, once it is, we can directly pad while tokenizing + // https://github.com/xenova/transformers.js/blob/8804c36591d11d8456788d1bb4b16489121b3be2/src/tokenizers.js#L2517 + const tokenized: unknown = tokenizer(line, { + padding: false, + truncation: true, + return_tensor: false, + max_length: length, + }); + + if ( + typeof tokenized !== "object" || + tokenized === null || + !("input_ids" in tokenized) || + !isArrayOfNumber(tokenized.input_ids) + ) + throw new Error("tokenizer returns unexcepted type"); + const tokens: Token[] = tokenized.input_ids; + + const paddingSize = length - tokens.length; + if (paddingSize < 0) + throw new Error("tokenized returned more token than excepted"); + + return Repeat(tokenizer.pad_token_id, paddingSize).concat(tokens).toList(); +} diff --git a/discojs/src/types.ts b/discojs/src/types.ts index 12a1088fa..9a8e12ff2 100644 --- a/discojs/src/types.ts +++ b/discojs/src/types.ts @@ -1,4 +1,26 @@ -import { Dataset, Image, Tabular, Text } from "./dataset/index.js" +import { List } from "immutable"; + +import type { Dataset, Image, processing, Tabular, Text } from "./index.js"; + +export type DataType = "image" | "tabular" | "text"; + +type Token = number; +export interface DataTypeToPreprocessed { + image: processing.NormalizedImage<3>; + tabular: List; + text: List; +} +export interface DataTypeToPreprocessedLabeled { + image: [DataTypeToPreprocessed["image"], label: number]; + tabular: [features: DataTypeToPreprocessed["tabular"], labels: List]; + text: [DataTypeToPreprocessed["text"], nexts: List]; +} + +export type Batched = List; + +export type DataTypeToBatchedPreprocessedLabeledDataset = { + [D in DataType]: Dataset>; +}; export type TypedDataset = | ["image", Dataset] @@ -9,3 +31,13 @@ export type TypedLabeledDataset = | ["image", Dataset<[Image, label: string]>] | ["tabular", Dataset] | ["text", Dataset]; + +export type TypedPreprocessedLabeledDataset = + | ["image", Dataset] + | ["tabular", Dataset] + | ["text", Dataset]; + +export type TypedBatchedPreprocessedLabeledDataset = + | ["image", Dataset>] + | ["tabular", Dataset>] + | ["text", Dataset>]; diff --git a/package-lock.json b/package-lock.json index 8c0d8f8d0..dd68b064d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -43,6 +43,8 @@ "name": "@epfml/discojs", "version": "3.0.0", "dependencies": { + "@jimp/core": "1", + "@jimp/plugin-resize": "1", "@msgpack/msgpack": "^3.0.0-beta2", "@tensorflow/tfjs": "4", "@xenova/transformers": "2", @@ -1322,6 +1324,84 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, + "node_modules/@jimp/core": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jimp/core/-/core-1.6.0.tgz", + "integrity": "sha512-EQQlKU3s9QfdJqiSrZWNTxBs3rKXgO2W+GxNXDtwchF3a4IqxDheFX1ti+Env9hdJXDiYLp2jTRjlxhPthsk8w==", + "license": "MIT", + "dependencies": { + "@jimp/file-ops": "1.6.0", + "@jimp/types": "1.6.0", + "@jimp/utils": "1.6.0", + "await-to-js": "^3.0.0", + "exif-parser": "^0.1.12", + "file-type": "^16.0.0", + "mime": "3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@jimp/core/node_modules/mime": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", + "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/@jimp/file-ops": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jimp/file-ops/-/file-ops-1.6.0.tgz", + "integrity": "sha512-Dx/bVDmgnRe1AlniRpCKrGRm5YvGmUwbDzt+MAkgmLGf+jvBT75hmMEZ003n9HQI/aPnm/YKnXjg/hOpzNCpHQ==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@jimp/plugin-resize": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jimp/plugin-resize/-/plugin-resize-1.6.0.tgz", + "integrity": "sha512-uSUD1mqXN9i1SGSz5ov3keRZ7S9L32/mAQG08wUwZiEi5FpbV0K8A8l1zkazAIZi9IJzLlTauRNU41Mi8IF9fA==", + "license": "MIT", + "dependencies": { + "@jimp/core": "1.6.0", + "@jimp/types": "1.6.0", + "zod": "^3.23.8" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@jimp/types": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jimp/types/-/types-1.6.0.tgz", + "integrity": "sha512-7UfRsiKo5GZTAATxm2qQ7jqmUXP0DxTArztllTcYdyw6Xi5oT4RaoXynVtCD4UyLK5gJgkZJcwonoijrhYFKfg==", + "license": "MIT", + "dependencies": { + "zod": "^3.23.8" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@jimp/utils": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jimp/utils/-/utils-1.6.0.tgz", + "integrity": "sha512-gqFTGEosKbOkYF/WFj26jMHOI5OH2jeP1MmC/zbK6BF6VJBf8rIC5898dPfSzZEbSA0wbbV5slbntWVc5PKLFA==", + "license": "MIT", + "dependencies": { + "@jimp/types": "1.6.0", + "tinycolor2": "^1.6.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz", @@ -2204,6 +2284,12 @@ "sprintf-js": "~1.0.2" } }, + "node_modules/@tokenizer/token": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", + "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==", + "license": "MIT" + }, "node_modules/@tsconfig/node10": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz", @@ -3957,6 +4043,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/await-to-js": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/await-to-js/-/await-to-js-3.0.0.tgz", + "integrity": "sha512-zJAaP9zxTcvTHRlejau3ZOY4V7SRpiByf3/dxx2uyKxxor19tpmpV2QRsTKikckwhaPmr2dVpxxMr7jOCYVp5g==", + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/aws-sign2": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", @@ -6643,6 +6738,11 @@ "node": ">=4" } }, + "node_modules/exif-parser": { + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/exif-parser/-/exif-parser-0.1.12.tgz", + "integrity": "sha512-c2bQfLNbMzLPmzQuOr8fy0csy84WmwnER81W88DzTp9CYNPJ6yzOj2EZAh9pywYpqHnshVLHQJ8WzldAyfY+Iw==" + }, "node_modules/expand-template": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", @@ -6922,6 +7022,23 @@ "node": "^10.12.0 || >=12.0.0" } }, + "node_modules/file-type": { + "version": "16.5.4", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-16.5.4.tgz", + "integrity": "sha512-/yFHK0aGjFEgDJjEKP0pWCplsPFPhwyfwevf/pVxiN0tmE4L9LmwWxWukdJSHdoCli4VgQLehjJtwQBnqmsKcw==", + "license": "MIT", + "dependencies": { + "readable-web-to-node-stream": "^3.0.0", + "strtok3": "^6.2.4", + "token-types": "^4.1.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/file-type?sponsor=1" + } + }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -10112,6 +10229,19 @@ "node": ">=0.12" } }, + "node_modules/peek-readable": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-4.1.0.tgz", + "integrity": "sha512-ZI3LnwUv5nOGbQzD9c2iDG6toheuXSZP5esSHBjopsXH4dg19soufvpUGA3uohi5anFtGb2lhAVdHzH6R/Evvg==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, "node_modules/pend": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", @@ -10917,6 +11047,22 @@ "node": ">= 6" } }, + "node_modules/readable-web-to-node-stream": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.2.tgz", + "integrity": "sha512-ePeK6cc1EcKLEhJFt/AebMCLL+GgSKhuygrZ/GLaKZYEecIgIECf4UaUuaByiGtzckwR4ain9VzUh95T1exYGw==", + "license": "MIT", + "dependencies": { + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, "node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -11963,6 +12109,23 @@ "url": "https://github.com/sponsors/antfu" } }, + "node_modules/strtok3": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-6.3.0.tgz", + "integrity": "sha512-fZtbhtvI9I48xDSywd/somNqgUHl2L2cstmXCCif0itOf96jeW18MBSyrLuNicYQVkvpOxkZtkzujiTJ9LW5Jw==", + "license": "MIT", + "dependencies": { + "@tokenizer/token": "^0.3.0", + "peek-readable": "^4.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, "node_modules/style-inject": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/style-inject/-/style-inject-0.3.0.tgz", @@ -12377,6 +12540,12 @@ "dev": true, "license": "MIT" }, + "node_modules/tinycolor2": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/tinycolor2/-/tinycolor2-1.6.0.tgz", + "integrity": "sha512-XPaBkWQJdsf3pLKJV9p4qN/S+fm2Oj8AIPo1BTUhg5oxkvm9+SVEGFdhyOz7tTdUTfvxMiAs4sp6/eZO2Ew+pw==", + "license": "MIT" + }, "node_modules/tinypool": { "version": "0.8.4", "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-0.8.4.tgz", @@ -12464,6 +12633,23 @@ "node": ">=0.6" } }, + "node_modules/token-types": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/token-types/-/token-types-4.2.1.tgz", + "integrity": "sha512-6udB24Q737UD/SDsKAHI9FCRP7Bqc9D/MQUV02ORQg5iskjtLJlZJNdN4kKtcdtwCeWIwIHDGaUsTsCCAa8sFQ==", + "license": "MIT", + "dependencies": { + "@tokenizer/token": "^0.3.0", + "ieee754": "^1.2.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, "node_modules/toposort": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/toposort/-/toposort-2.0.2.tgz", @@ -13969,6 +14155,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/zod": { + "version": "3.23.8", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz", + "integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, "server": { "license": "ISC", "dependencies": { From 8266afc179c051af3de3805aa9e251f625ae8d08 Mon Sep 17 00:00:00 2001 From: tharvik Date: Wed, 4 Sep 2024 11:56:19 +0200 Subject: [PATCH 04/31] discojs/dataset: allow subclass --- discojs/src/dataset/dataset.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/discojs/src/dataset/dataset.ts b/discojs/src/dataset/dataset.ts index f43a1b344..44f941f2f 100644 --- a/discojs/src/dataset/dataset.ts +++ b/discojs/src/dataset/dataset.ts @@ -44,7 +44,7 @@ export class Dataset implements AsyncIterable { */ map(mapper: (_: T) => U | Promise): Dataset { const content = { - [Symbol.asyncIterator]: () => this.#content(), + [Symbol.asyncIterator]: () => this[Symbol.asyncIterator](), }; return new Dataset(async function* () { @@ -59,9 +59,7 @@ export class Dataset implements AsyncIterable { chain(other: Dataset | DatasetLike): Dataset { if (!(other instanceof Dataset)) other = new Dataset(other); - const self = { - [Symbol.asyncIterator]: () => this.#content(), - }; + const self = { [Symbol.asyncIterator]: () => this[Symbol.asyncIterator]() }; return new Dataset(async function* () { yield* self; @@ -77,7 +75,7 @@ export class Dataset implements AsyncIterable { if (ratio < 0 || ratio > 1) throw new Error("ratio out of range"); const content = { - [Symbol.asyncIterator]: () => this.#content(), + [Symbol.asyncIterator]: () => this[Symbol.asyncIterator](), }; // to avoid using random sampling or knowing the size beforehand, @@ -123,7 +121,7 @@ export class Dataset implements AsyncIterable { if (size <= 0 || !Number.isInteger(size)) throw new Error("invalid size"); const content = { - [Symbol.asyncIterator]: () => this.#content(), + [Symbol.asyncIterator]: () => this[Symbol.asyncIterator](), }; return new Dataset(async function* () { @@ -152,7 +150,7 @@ export class Dataset implements AsyncIterable { if (!(other instanceof Dataset)) other = new Dataset(other); const content = { - [Symbol.asyncIterator]: () => this.#content(), + [Symbol.asyncIterator]: () => this[Symbol.asyncIterator](), }; return new Dataset(async function* () { From 33e0c7c1f8f535e1aea040d2280325d8f5a5c509 Mon Sep 17 00:00:00 2001 From: tharvik Date: Wed, 4 Sep 2024 11:56:33 +0200 Subject: [PATCH 05/31] discojs/dataset: add cache --- discojs/src/dataset/dataset.ts | 62 ++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/discojs/src/dataset/dataset.ts b/discojs/src/dataset/dataset.ts index 44f941f2f..ee06786d1 100644 --- a/discojs/src/dataset/dataset.ts +++ b/discojs/src/dataset/dataset.ts @@ -1,7 +1,10 @@ +import createDebug from "debug"; import { List } from "immutable"; import type { Batched } from "../index.js"; +const debug = createDebug("discojs:dataset"); + type DatasetLike = | AsyncIterable | Iterable @@ -174,4 +177,63 @@ export class Dataset implements AsyncIterable { for await (const _ of this) ret++; return ret; } + + /** Try to keep generated elements to avoid recomputing + * + * Drops everything when memory pressure is applied. + */ + cached(): Dataset { + return new CachingDataset(this.#content); + } +} + +/** + * Avoid recomputing the parent dataset, without hogging memory + * + * As dataset operations can be time-consuming, this keeps a weak reference to + * the generated elements so that a second iteration might yield theses directly. + **/ +class CachingDataset extends Dataset { + // potential reference to all elements + // tristate: undefined == empty, [false, _] == filling, [true, _] == filled + #cache = new WeakRef<[filled: boolean, List]>([false, List()]); + + override [Symbol.asyncIterator](): AsyncIterator { + const cached = this.#cache.deref(); + + if (cached !== undefined && cached[0]) { + debug("valid cache, reading from it"); + + // eslint-disable-next-line @typescript-eslint/require-await + return (async function* () { + yield* cached[1]; + })(); + } + + debug("cache invalid, reading from dataset"); + + this.#cache = new WeakRef([false, List()]); + const cache = this.#cache; + + const content = { + [Symbol.asyncIterator]: () => super[Symbol.asyncIterator](), + }; + return (async function* () { + for await (const e of content) { + yield e; + + const caching = cache.deref(); + if (caching !== undefined) caching[1] = caching[1].push(e); + } + + const caching = cache.deref(); + if (caching === undefined) { + debug("cache evicted while filling"); + return; + } + + debug("cache filled"); + caching[0] = true; + })(); + } } From 76ea663a497279353426335f47779eba93428ace Mon Sep 17 00:00:00 2001 From: tharvik Date: Wed, 4 Sep 2024 17:17:39 +0200 Subject: [PATCH 06/31] discojs/dataset: resolve batch in parallel --- discojs/src/dataset/dataset.ts | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/discojs/src/dataset/dataset.ts b/discojs/src/dataset/dataset.ts index ee06786d1..e9c87e41e 100644 --- a/discojs/src/dataset/dataset.ts +++ b/discojs/src/dataset/dataset.ts @@ -1,5 +1,5 @@ import createDebug from "debug"; -import { List } from "immutable"; +import { List, Range } from "immutable"; import type { Batched } from "../index.js"; @@ -128,18 +128,23 @@ export class Dataset implements AsyncIterable { }; return new Dataset(async function* () { - let batch = List(); + const iter = content[Symbol.asyncIterator](); - for await (const e of content) { - batch = batch.push(e); + for (;;) { + const batch = List( + await Promise.all(Range(0, size).map(() => iter.next())), + ).flatMap((res) => { + if (res.done) return []; + else return [res.value]; + }); - if (batch.size === size) { - yield batch; - batch = List(); - } - } + if (batch.isEmpty()) break; - if (!batch.isEmpty()) yield batch; + yield batch; + + // iterator couldn't generate more + if (batch.size < size) break; + } }); } From 7433b0bbe50c4924faa6fe504b644a31beaeb453 Mon Sep 17 00:00:00 2001 From: tharvik Date: Thu, 29 Aug 2024 10:09:04 +0200 Subject: [PATCH 07/31] discojs/model: generic on datatype --- cli/src/benchmark_gpt.ts | 41 +---- discojs/src/default_tasks/cifar10.ts | 4 +- discojs/src/default_tasks/lus_covid.ts | 4 +- discojs/src/default_tasks/mnist.ts | 4 +- discojs/src/default_tasks/simple_face.ts | 4 +- discojs/src/default_tasks/titanic.ts | 4 +- discojs/src/default_tasks/wikitext.ts | 2 +- discojs/src/models/gpt/gpt.spec.ts | 65 +++---- discojs/src/models/gpt/index.ts | 89 +++++---- discojs/src/models/model.ts | 24 ++- discojs/src/models/tfjs.ts | 169 +++++++++++++----- discojs/src/serialization/model.spec.ts | 6 +- discojs/src/serialization/model.ts | 34 +++- discojs/src/training/disco.ts | 57 +++++- discojs/src/training/trainer.ts | 23 ++- docs/examples/custom_task.ts | 2 +- server/src/task_set.ts | 5 +- server/tests/e2e/decentralized.spec.ts | 2 +- server/tests/e2e/federated.spec.ts | 4 +- .../task_creation_form/TaskForm.vue | 5 +- 20 files changed, 356 insertions(+), 192 deletions(-) diff --git a/cli/src/benchmark_gpt.ts b/cli/src/benchmark_gpt.ts index fc99ba8ed..ced2a82bd 100644 --- a/cli/src/benchmark_gpt.ts +++ b/cli/src/benchmark_gpt.ts @@ -1,5 +1,5 @@ +import { List } from "immutable"; import { parse } from "ts-command-line-args"; -import * as tf from "@tensorflow/tfjs" import { AutoTokenizer } from "@xenova/transformers"; import { fetchTasks, models, async_iterator, defaultTasks, processing } from "@epfml/discojs"; @@ -41,15 +41,6 @@ const args = { ...defaultArgs, ...parsedArgs } * Benchmark results are reported in https://github.com/epfml/disco/pull/659 */ -function intoTFGenerator( - iter: AsyncIterable, -): tf.data.Dataset { - // @ts-expect-error generator - return tf.data.generator(async function* () { - yield* iter; - }); -} - async function main(args: Required): Promise { const { inference: benchmarkInference, modelType, contextLength, batchSize, modelPath } = args @@ -89,32 +80,10 @@ async function main(args: Required): Promise { const dataset = loadText('../datasets/wikitext/wiki.train.tokens') const maxLength = task.trainingInformation.maxSequenceLength ?? (tokenizer.model_max_length as number) + 1 - // TODO will be easier when preproccessing is redone - const preprocessedDataset = intoTFGenerator( - dataset - .map((line) => - processing.tokenizeAndLeftPad(line, tokenizer, maxLength), - ) - .batch(batchSize) - .map((batch) => - tf.tidy(() => ({ - xs: tf.tensor2d( - batch.map((tokens) => tokens.slice(0, -1).toArray()).toArray(), - ), - ys: tf.stack( - batch - .map( - (tokens) => - tf.oneHot( - tokens.slice(1), - tokenizer.model.vocab.length + 1, - ) as tf.Tensor2D, - ) - .toArray(), - ) as tf.Tensor3D, - })), - ), - ); + const preprocessedDataset = dataset + .map((line) => processing.tokenizeAndLeftPad(line, tokenizer, maxLength)) + .map((tokens) => [tokens.pop(), tokens.shift()] as [List, List]) + .batch(batchSize); // Init and train the model const model = new models.GPT(config) diff --git a/discojs/src/default_tasks/cifar10.ts b/discojs/src/default_tasks/cifar10.ts index 6c5f22235..ade49a4ef 100644 --- a/discojs/src/default_tasks/cifar10.ts +++ b/discojs/src/default_tasks/cifar10.ts @@ -42,7 +42,7 @@ export const cifar10: TaskProvider = { } }, - async getModel (): Promise { + async getModel (): Promise> { const mobilenet = await tf.loadLayersModel({ load: async () => Promise.resolve(baseModel), }) @@ -64,6 +64,6 @@ export const cifar10: TaskProvider = { metrics: ['accuracy'] }) - return new models.TFJS(model) + return new models.TFJS('image', model) } } diff --git a/discojs/src/default_tasks/lus_covid.ts b/discojs/src/default_tasks/lus_covid.ts index 77889eb48..993372d11 100644 --- a/discojs/src/default_tasks/lus_covid.ts +++ b/discojs/src/default_tasks/lus_covid.ts @@ -40,7 +40,7 @@ export const lusCovid: TaskProvider = { // Model architecture from tensorflow.js docs: // https://codelabs.developers.google.com/codelabs/tfjs-training-classfication/index.html#4 - async getModel (): Promise { + async getModel (): Promise> { const imageHeight = 100 const imageWidth = 100 const imageChannels = 3 @@ -93,6 +93,6 @@ export const lusCovid: TaskProvider = { metrics: ['accuracy'] }) - return Promise.resolve(new models.TFJS(model)) + return Promise.resolve(new models.TFJS('image', model)) } } diff --git a/discojs/src/default_tasks/mnist.ts b/discojs/src/default_tasks/mnist.ts index a55c2df7d..240fc8811 100644 --- a/discojs/src/default_tasks/mnist.ts +++ b/discojs/src/default_tasks/mnist.ts @@ -40,7 +40,7 @@ export const mnist: TaskProvider = { } }, - getModel(): Promise { + getModel(): Promise> { // Architecture from the PyTorch MNIST example (I made it slightly smaller, 650kB instead of 5MB) // https://github.com/pytorch/examples/blob/main/mnist/main.py const model = tf.sequential() @@ -68,6 +68,6 @@ export const mnist: TaskProvider = { metrics: ['accuracy'] }) - return Promise.resolve(new models.TFJS(model)) + return Promise.resolve(new models.TFJS('image', model)) } } diff --git a/discojs/src/default_tasks/simple_face.ts b/discojs/src/default_tasks/simple_face.ts index e18d5c726..ce1bd5be2 100644 --- a/discojs/src/default_tasks/simple_face.ts +++ b/discojs/src/default_tasks/simple_face.ts @@ -38,7 +38,7 @@ export const simpleFace: TaskProvider = { } }, - async getModel (): Promise { + async getModel (): Promise> { const model = await tf.loadLayersModel({ load: async () => Promise.resolve(baseModel), }); @@ -49,6 +49,6 @@ export const simpleFace: TaskProvider = { metrics: ['accuracy'] }) - return new models.TFJS(model) + return new models.TFJS('image', model) } } diff --git a/discojs/src/default_tasks/titanic.ts b/discojs/src/default_tasks/titanic.ts index 393ee01c8..776f545e3 100644 --- a/discojs/src/default_tasks/titanic.ts +++ b/discojs/src/default_tasks/titanic.ts @@ -72,7 +72,7 @@ export const titanic: TaskProvider = { } }, - getModel (): Promise { + getModel (): Promise> { const model = tf.sequential() model.add( @@ -93,6 +93,6 @@ export const titanic: TaskProvider = { metrics: ['accuracy'] }) - return Promise.resolve(new models.TFJS(model)) + return Promise.resolve(new models.TFJS('tabular', model)) } } diff --git a/discojs/src/default_tasks/wikitext.ts b/discojs/src/default_tasks/wikitext.ts index 518c060c2..cdcfa3781 100644 --- a/discojs/src/default_tasks/wikitext.ts +++ b/discojs/src/default_tasks/wikitext.ts @@ -42,7 +42,7 @@ export const wikitext: TaskProvider = { } }, - getModel (): Promise { + getModel (): Promise> { return Promise.resolve(new models.GPT()) } } diff --git a/discojs/src/models/gpt/gpt.spec.ts b/discojs/src/models/gpt/gpt.spec.ts index 5ee898ab6..4be871e32 100644 --- a/discojs/src/models/gpt/gpt.spec.ts +++ b/discojs/src/models/gpt/gpt.spec.ts @@ -1,44 +1,49 @@ -import { expect } from 'chai' -import * as tf from '@tensorflow/tfjs-node' -import { AutoTokenizer } from '@xenova/transformers'; -import { GPT } from './index.js' -import { type GPTConfig } from './config.js' +import { expect } from "chai"; +import "@tensorflow/tfjs-node"; // speed up +import { AutoTokenizer } from "@xenova/transformers"; -describe('gpt-tfjs', function() { - const data = "Lorem ipsum dolor sit" +import { Dataset, DataTypeToPreprocessedLabeled } from "../../index.js"; + +import { GPT } from "./index.js"; +import type { GPTConfig } from "./config.js"; +import { List, Repeat } from "immutable"; + +describe("gpt-tfjs", function () { + const data = "Lorem ipsum dolor sit"; const config: GPTConfig = { - modelType: 'gpt-nano', + modelType: "gpt-nano", lr: 0.01, maxIter: 10, - evaluateEvery:10, + evaluateEvery: 10, maxEvalBatches: 10, blockSize: 8, - vocabSize: 50258 - } - - it('can overfit one sentence', async function() { - this.timeout("2m") - - const tokenizer = await AutoTokenizer.from_pretrained('Xenova/gpt2') - const datasetSource = new tf.data.FileDataSource(Buffer.from(data)) - const textDataset = new tf.data.TextLineDataset(datasetSource) - const tokenDataset = textDataset.map((text: string) => { - const { input_ids: tokens } = tokenizer(text, { + vocabSize: 50258, + }; + + it("can overfit one sentence", async function () { + this.timeout("2m"); + + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/gpt2"); + const tokenDataset = new Dataset(Repeat(data)) + .map((text) => { + const { input_ids: tokens } = tokenizer(text, { padding: true, truncation: true, return_tensor: false, max_length: config.blockSize + 1, - }) as { input_ids: number[] } - const ys = tf.oneHot(tokens.slice(1), tokenizer.model.vocab.length + 1) - const xs = tf.tensor(tokens.slice(0, config.blockSize), undefined, 'int32') - return {xs, ys} - }).repeat().batch(64) as tf.data.Dataset<{ xs: tf.Tensor2D, ys: tf.Tensor3D }> + }) as { input_ids: number[] }; - const model = new GPT(config) + return [List(tokens.slice(0, config.blockSize)), List(tokens.slice(1))]; + }) + .batch(64); + + const model = new GPT(config); for (let i = 0; i < 5; i++) for await (const _ of model.train(tokenDataset, undefined)); - const generation = await model.generate("Lorem ipsum dolor", tokenizer, 1) - expect(generation).equal(data) // Assert that the model completes 'Lorem ipsum dolor' with 'sit' - }) -}) + + const generation = await model.generate("Lorem ipsum dolor", tokenizer, 1); + + expect(generation).equal(data); // Assert that the model completes 'Lorem ipsum dolor' with 'sit' + }); +}); diff --git a/discojs/src/models/gpt/index.ts b/discojs/src/models/gpt/index.ts index bb1f0f74d..ca6fcfcd9 100644 --- a/discojs/src/models/gpt/index.ts +++ b/discojs/src/models/gpt/index.ts @@ -3,11 +3,17 @@ **/ import createDebug from "debug"; -import { List } from 'immutable'; +import { List, Range } from "immutable"; import * as tf from '@tensorflow/tfjs' import { PreTrainedTokenizer } from '@xenova/transformers'; -import { WeightsContainer } from '../../index.js' +import { + Batched, + Dataset, + DataTypeToBatchedPreprocessedLabeledDataset, + DataTypeToPreprocessedLabeled, + WeightsContainer, +} from "../../index.js"; import { BatchLogs, Model, EpochLogs } from "../index.js"; import type { Prediction, Sample } from '../model.js' @@ -23,16 +29,21 @@ export type GPTSerialization = { config?: GPTConfig } -export class GPT extends Model { +export class GPT extends Model<"text"> { private readonly model: GPTForCausalLM readonly #maxBatchCount: number + readonly #vocabSize: number constructor (partialConfig?: GPTConfig, layersModel?: tf.LayersModel) { super() - this.model = new GPTForCausalLM(partialConfig, layersModel) + const model = new GPTForCausalLM(partialConfig, layersModel) + model.compile(); + this.model = model; + this.#maxBatchCount = partialConfig?.maxIter ?? DEFAULT_CONFIG.maxIter + this.#vocabSize = partialConfig?.vocabSize ?? DEFAULT_CONFIG.vocabSize } /** @@ -45,38 +56,33 @@ export class GPT extends Model { * @param tracker */ override async *train( - trainingData: tf.data.Dataset<{ xs: tf.Tensor2D, ys: tf.Tensor3D }>, - validationData?: tf.data.Dataset<{ xs: tf.Tensor2D, ys: tf.Tensor3D }>, + trainingDataset: Dataset>, + validationDataset?: Dataset>, ): AsyncGenerator { - this.model.compile(); - - const batches = await trainingData.iterator(); // tf.LazyIterator isn't an AsyncGenerator let batchesLogs = List(); - for ( - let batchNumber = 0; - batchNumber < this.#maxBatchCount; - batchNumber++ - ) { - const iteration = await batches.next(); - if (iteration.done) break; - const batch = iteration.value; + for await (const [batch, _] of trainingDataset.zip( + Range(0, this.#maxBatchCount), + )) { const batchLogs = await this.#runBatch(batch); - tf.dispose(batch); yield batchLogs; batchesLogs = batchesLogs.push(batchLogs); } - const validation = validationData && (await this.#evaluate(validationData)); + const validation = + validationDataset && (await this.#evaluate(validationDataset)); + return new EpochLogs(batchesLogs, validation); } async #runBatch( - batch: tf.TensorContainer, + batch: Batched, ): Promise { + const tfBatch = this.#batchToTF(batch); + let logs: tf.Logs | undefined; - await this.model.fitDataset(tf.data.array([batch]), { + await this.model.fitDataset(tf.data.array([tfBatch]), { epochs: 1, verbose: 0, // don't pollute callbacks: { @@ -85,6 +91,7 @@ export class GPT extends Model { }, }, }); + tf.dispose(tfBatch); if (logs === undefined) throw new Error("batch didn't gave any logs"); const { loss, acc: accuracy } = logs; @@ -99,20 +106,11 @@ export class GPT extends Model { } async #evaluate( - dataset: tf.data.Dataset, + dataset: DataTypeToBatchedPreprocessedLabeledDataset["text"], ): Promise> { const evaluation = await evaluate( this.model, - dataset.map((t) => { - switch (t) { - case null: - case undefined: - throw new Error("nullish value in dataset"); - default: - // TODO unsafe cast - return t as { xs: tf.Tensor2D; ys: tf.Tensor3D }; - } - }), + intoTFDataset(dataset.map((batch) => this.#batchToTF(batch))), this.config.maxEvalBatches, ); @@ -122,6 +120,22 @@ export class GPT extends Model { }; } + #batchToTF(batch: Batched): { + xs: tf.Tensor2D; + ys: tf.Tensor3D; + } { + return tf.tidy(() => ({ + xs: tf.stack( + batch.map(([line]) => tf.tensor1d(line.toArray(), "int32")).toArray(), + ) as tf.Tensor2D, // cast as stack doesn't type + ys: tf.stack( + batch + .map(([_, next]) => tf.oneHot(next.toArray(), this.#vocabSize)) + .toArray(), + ) as tf.Tensor3D, // cast as oneHot/stack doesn't type + })); + } + override predict(input: Sample): Promise { const ret = this.model.predict(input); if (Array.isArray(ret)) { @@ -157,7 +171,7 @@ export class GPT extends Model { this.model.setWeights(ws.weights) } - static deserialize (data: GPTSerialization): Model { + static deserialize (data: GPTSerialization): Model<'text'> { const model = new GPT(data.config) model.weights = data.weights return model @@ -182,3 +196,12 @@ export class GPT extends Model { debug("model not disposed correctly: %o", disposeResults); } } + +function intoTFDataset( + iter: AsyncIterable, +): tf.data.Dataset { + // @ts-expect-error generator + return tf.data.generator(async function* () { + yield* iter; + }); +} diff --git a/discojs/src/models/model.ts b/discojs/src/models/model.ts index 70a8e57bb..50c65cae1 100644 --- a/discojs/src/models/model.ts +++ b/discojs/src/models/model.ts @@ -1,6 +1,12 @@ import type tf from "@tensorflow/tfjs"; -import type { WeightsContainer } from "../index.js"; +import type { + Batched, + Dataset, + DataType, + DataTypeToPreprocessedLabeled, + WeightsContainer, +} from "../index.js"; import type { BatchLogs, EpochLogs } from "./logs.js"; @@ -14,7 +20,9 @@ export type Sample = tf.Tensor; * Allow for various implementation of models (various train function, tensor-library, ...) **/ // TODO make it typesafe: same shape of data/input/weights -export abstract class Model implements Disposable { +export abstract class Model + implements Disposable +{ // TODO don't allow external access but upgrade train to return weights on every epoch /** Return training state */ abstract get weights(): WeightsContainer; @@ -24,15 +32,13 @@ export abstract class Model implements Disposable { /** * Improve predictor * - * @param trainingData dataset to optimize for - * @param validationData dataset to measure how well it is training - * @param epochs number of pass over the training dataset - * @param tracker watch the various steps - * @yields on every epoch, training can be stop by `return`ing it + * @param trainingDataset dataset to optimize for + * @param validationDataset dataset to measure how well it is training + * @yields on every epoch, training can be stop by `return`ing or `throw`ing it */ abstract train( - trainingData: tf.data.Dataset, - validationData?: tf.data.Dataset, + trainingDataset: Dataset>, + validationDataset?: Dataset>, ): AsyncGenerator; /** Predict likely values */ diff --git a/discojs/src/models/tfjs.ts b/discojs/src/models/tfjs.ts index dba93e042..ca4e42192 100644 --- a/discojs/src/models/tfjs.ts +++ b/discojs/src/models/tfjs.ts @@ -1,17 +1,26 @@ -import { List, Map } from 'immutable' +import { List, Map, Range } from "immutable"; import * as tf from '@tensorflow/tfjs' -import { WeightsContainer } from '../index.js' +import { + Batched, + Dataset, + DataType, + DataTypeToPreprocessedLabeled, + WeightsContainer, +} from "../index.js"; import { BatchLogs } from './index.js' import { Model } from './index.js' import { Prediction, Sample } from './model.js' import { EpochLogs } from './logs.js' +type Serialized = [D, tf.io.ModelArtifacts] + /** TensorFlow JavaScript model with standard training */ -export class TFJS extends Model { +export class TFJS extends Model { /** Wrap the given trainable model */ constructor ( + public readonly datatype: D, private readonly model: tf.LayersModel ) { super() @@ -30,69 +39,58 @@ export class TFJS extends Model { } override async *train( - trainingData: tf.data.Dataset, - validationData?: tf.data.Dataset, + trainingDataset: Dataset>, + validationDataset?: Dataset>, ): AsyncGenerator { - const batches = await trainingData.iterator(); // tf.LazyIterator isn't an AsyncGenerator let batchesLogs = List(); - for (let batchNumber = 0; true; batchNumber++) { - const iteration = await batches.next(); - if (iteration.done) break; - const batch = iteration.value; + for await (const [batch, batchNumber] of trainingDataset.zip(Range())) { const batchLogs = { batch: batchNumber, ...(await this.#runBatch(batch)), }; - tf.dispose(batch); yield batchLogs; batchesLogs = batchesLogs.push(batchLogs); } - const validation = validationData && (await this.#evaluate(validationData)); + const validation = validationDataset && (await this.#evaluate(validationDataset)); return new EpochLogs(batchesLogs, validation); } async #runBatch( - batch: tf.TensorContainer, + batch: Batched, ): Promise> { - let logs: tf.Logs | undefined; - await this.model.fitDataset(tf.data.array([batch]), { + const { xs, ys } = this.#batchToTF(batch); + + const { history } = await this.model.fit(xs, ys, { epochs: 1, verbose: 0, // don't pollute - callbacks: { - onEpochEnd: (_, cur) => { - logs = cur; - }, - }, }); - if (logs === undefined) throw new Error("batch didn't gave any logs"); - const { loss, acc: accuracy } = logs; - if (loss === undefined || isNaN(loss)) - throw new Error("training loss is undefined or NaN"); + const { loss: losses, acc: accuracies } = history; + if ( + losses === undefined || + accuracies === undefined || + typeof losses[0] !== "number" || + typeof accuracies[0] !== "number" || + isNaN(losses[0]) || + isNaN(accuracies[0]) + ) + throw new Error("training loss or accuracy is undefined or NaN"); return { - accuracy, - loss, + accuracy: accuracies[0], + loss: losses[0], memoryUsage: tf.memory().numBytes / 1024 / 1024 / 1024, }; } async #evaluate( - dataset: tf.data.Dataset, + dataset: Dataset>, ): Promise> { const evaluation = await this.model.evaluateDataset( - dataset.map((t) => { - switch (t) { - case null: - case undefined: - throw new Error("nullish value in dataset"); - default: - return t as Exclude; - } - }), + intoTFDataset(dataset.map((batch) => this.#batchToTF(batch))), ); const metricToValue = Map( List(this.model.metricsNames).zip( @@ -125,13 +123,20 @@ export class TFJS extends Model { return Promise.resolve(ret) } - static async deserialize (raw: tf.io.ModelArtifacts): Promise { - return new this(await tf.loadLayersModel({ - load: () => Promise.resolve(raw) - })) + static async deserialize([ + datatype, + artifacts, + ]: Serialized): Promise> { + return new this( + datatype, + await tf.loadLayersModel({ + load: () => Promise.resolve(artifacts), + }), + ); } - async serialize (): Promise { + + async serialize (): Promise> { let resolveArtifacts: (_: tf.io.ModelArtifacts) => void const ret = new Promise((resolve) => { resolveArtifacts = resolve }) @@ -149,7 +154,7 @@ export class TFJS extends Model { includeOptimizer: true // keep model compiled }) - return await ret + return [this.datatype, await ret] } [Symbol.dispose](): void{ @@ -164,4 +169,84 @@ export class TFJS extends Model { extract (): tf.LayersModel { return this.model } + + #batchToTF( + batch: Batched, + ): Record<"xs" | "ys", tf.Tensor> { + const outputSize = tf.util.sizeFromShape( + this.model.outputShape.map((dim) => { + if (Array.isArray(dim)) + throw new Error("TODO support multiple outputs"); + return dim ?? 1; + }), + ); + + switch (this.datatype) { + case "image": { + // cast as typescript doesn't reduce generic type + const b = batch as Batched; + + return tf.tidy(() => ({ + xs: tf.stack( + b + .map(([image]) => + tf.tensor3d( + image.data, + [image.width, image.height, 3], + "float32", + ), + ) + .toArray(), + ), + ys: tf.stack( + b + .map(([_, label]) => + tf.oneHot(label, outputSize, 1, 0, "int32"), + ) + .toArray(), + ), + })); + } + case "tabular": { + // cast as typescript doesn't reduce generic type + const b = batch as Batched; + + return { + xs: tf.stack( + b.map(([inputs]) => tf.tensor1d(inputs.toArray())).toArray(), + ), + ys: tf.stack( + b.map(([_, outputs]) => tf.tensor1d(outputs.toArray())).toArray(), + ), + }; + } + case "text": { + // cast as typescript doesn't reduce generic type + const b = batch as Batched; + + return { + xs: tf.stack( + b.map(([line]) => tf.tensor1d(line.toArray())).toArray(), + ), + ys: tf.stack( + b + .map(([_, next]) => tf.oneHot(next.toArray(), outputSize)) + .toArray(), + ), + }; + } + } + + const _: never = this.datatype; + throw new Error("should never happen"); + } +} + +function intoTFDataset( + iter: AsyncIterable, +): tf.data.Dataset { + // @ts-expect-error generator + return tf.data.generator(async function* () { + yield* iter; + }); } diff --git a/discojs/src/serialization/model.spec.ts b/discojs/src/serialization/model.spec.ts index 12e9152d0..caf123cc1 100644 --- a/discojs/src/serialization/model.spec.ts +++ b/discojs/src/serialization/model.spec.ts @@ -1,4 +1,4 @@ -import { assert } from 'chai' +import { assert, expect } from 'chai' import * as tf from '@tensorflow/tfjs' import type { Model } from '../index.js' @@ -26,12 +26,14 @@ describe('serialization', () => { ] }) rawModel.compile({ optimizer: 'sgd', loss: 'hinge' }) - const model = new models.TFJS(rawModel) + const model = new models.TFJS("image", rawModel) const encoded = await serialization.model.encode(model) assert.isTrue(serialization.isEncoded(encoded)) const decoded = await serialization.model.decode(encoded) + expect(decoded).to.be.an.instanceof(models.TFJS); + expect((decoded as models.TFJS).datatype).to.equal("image") assert.sameDeepOrderedMembers( await getRawWeights(model), await getRawWeights(decoded) diff --git a/discojs/src/serialization/model.ts b/discojs/src/serialization/model.ts index 71408b4b5..550dfa83d 100644 --- a/discojs/src/serialization/model.ts +++ b/discojs/src/serialization/model.ts @@ -1,6 +1,6 @@ import type tf from '@tensorflow/tfjs' -import type { Model } from '../index.js' +import type { DataType, Model } from '../index.js' import { models, serialization } from '../index.js' import { GPTConfig } from '../models/index.js' @@ -16,7 +16,7 @@ export async function encode(model: Model): Promise { switch (true) { case model instanceof models.TFJS: { const serialized = await model.serialize(); - return coder.encode([Type.TFJS, serialized]); + return coder.encode([Type.TFJS, ...serialized]); } case model instanceof models.GPT: { const { weights, config } = model.serialize(); @@ -42,12 +42,32 @@ export async function decode (encoded: unknown): Promise { } const rawModel = raw[1] as unknown switch (type) { - case Type.TFJS: - if (raw.length !== 2) { - throw new Error('invalid encoding, TFJS model encoding should be an array of length 2') + case Type.TFJS: { + if (raw.length !== 3) + throw new Error( + "invalid TFJS model encoding: should be an array of length 3", + ); + const [rawDatatype, rawModel] = raw.slice(1) as unknown[]; + + let datatype: DataType; + switch (rawDatatype) { + case "image": + case "tabular": + case "text": + datatype = rawDatatype; + break; + default: + throw new Error( + "invalid TFJS model encoding: invalid DataType", + ); } - // TODO totally unsafe casting - return await models.TFJS.deserialize(rawModel as tf.io.ModelArtifacts) + + return await models.TFJS.deserialize([ + datatype, + // TODO totally unsafe casting + rawModel as tf.io.ModelArtifacts, + ]); + } case Type.GPT: { let config if (raw.length == 2) { diff --git a/discojs/src/training/disco.ts b/discojs/src/training/disco.ts index 274c109bb..8b72f0ecf 100644 --- a/discojs/src/training/disco.ts +++ b/discojs/src/training/disco.ts @@ -7,15 +7,18 @@ import { Logger, Task, TrainingInformation, + processing, +} from "../index.js"; +import type { + TypedBatchedPreprocessedLabeledDataset, + TypedLabeledDataset, } from "../index.js"; -import type { TypedLabeledDataset } from "../index.js"; import type { Aggregator } from "../aggregator/index.js"; import { getAggregator } from "../aggregator/index.js"; import { enumerate, split } from "../utils/async_iterator.js"; import { EventEmitter } from "../utils/event_emitter.js"; import { RoundLogs, Trainer } from "./trainer.js"; -import { labeledDatasetToDataSplit } from "../dataset/data/helpers.js"; interface DiscoConfig { scheme: TrainingInformation["scheme"]; @@ -127,15 +130,14 @@ export class Disco extends EventEmitter<{'status': RoundStatus}>{ > { this.#logger.success("Training started"); - const data = await labeledDatasetToDataSplit(this.#task, dataset); - const trainData = data.train.preprocess().batch().dataset; - const validationData = - data.validation?.preprocess().batch().dataset ?? trainData; + const [trainingDataset, validationDataset] = + await this.#preprocessBatchAndSplit(dataset); + // the client fetches the latest weights upon connection this.trainer.model = await this.#client.connect(); for await (const [round, epochs] of enumerate( - this.trainer.train(trainData, validationData), + this.trainer.train(trainingDataset, validationDataset), )) { yield async function* (this: Disco) { const [gen, returnedRoundLogs] = split(epochs); @@ -174,4 +176,45 @@ export class Disco extends EventEmitter<{'status': RoundStatus}>{ async close(): Promise { await this.#client.disconnect(); } + + async #preprocessBatchAndSplit( + dataset: TypedLabeledDataset, + ): Promise< + [ + TypedBatchedPreprocessedLabeledDataset, + TypedBatchedPreprocessedLabeledDataset, + ] + > { + const preprocessed = await processing.preprocess(this.#task, dataset); + const { batchSize, validationSplit } = this.#task.trainingInformation; + switch (preprocessed[0]) { + case "image": { + const [training, validation] = preprocessed[1] + .split(validationSplit) + .map((d) => d.batch(batchSize).cached()); + return [ + ["image", training], + ["image", validation], + ]; + } + case "tabular": { + const [training, validation] = preprocessed[1] + .split(validationSplit) + .map((d) => d.batch(batchSize).cached()); + return [ + ["tabular", training], + ["tabular", validation], + ]; + } + case "text": { + const [training, validation] = preprocessed[1] + .split(validationSplit) + .map((d) => d.batch(batchSize).cached()); + return [ + ["text", training], + ["text", validation], + ]; + } + } + } } diff --git a/discojs/src/training/trainer.ts b/discojs/src/training/trainer.ts index 4f885b8aa..87fffb5f4 100644 --- a/discojs/src/training/trainer.ts +++ b/discojs/src/training/trainer.ts @@ -2,8 +2,12 @@ import * as tf from "@tensorflow/tfjs"; import { List } from "immutable"; import type { - BatchLogs, EpochLogs, Model, Task, - WeightsContainer + BatchLogs, + EpochLogs, + Model, + Task, + TypedBatchedPreprocessedLabeledDataset, + WeightsContainer, } from "../index.js"; import { privacy } from "../index.js"; import { Client } from "../client/index.js"; @@ -55,8 +59,8 @@ export class Trainer { } async *train( - dataset: tf.data.Dataset, - valDataset: tf.data.Dataset, + dataset: TypedBatchedPreprocessedLabeledDataset, + valDataset?: TypedBatchedPreprocessedLabeledDataset, ): AsyncGenerator< AsyncGenerator, RoundLogs>, void @@ -75,8 +79,8 @@ export class Trainer { } async *#runRounds( - dataset: tf.data.Dataset, - valDataset: tf.data.Dataset, + dataset: TypedBatchedPreprocessedLabeledDataset, + valDataset?: TypedBatchedPreprocessedLabeledDataset, ): AsyncGenerator< AsyncGenerator, RoundLogs>, void @@ -105,13 +109,14 @@ export class Trainer { } async *#runRound( - dataset: tf.data.Dataset, - valDataset: tf.data.Dataset, + dataset: TypedBatchedPreprocessedLabeledDataset, + valDataset?: TypedBatchedPreprocessedLabeledDataset, ): AsyncGenerator, RoundLogs> { let epochsLogs = List(); for (let epoch = 0; epoch < this.#roundDuration; epoch++) { const [gen, epochLogs] = async_iterator.split( - this.model.train(dataset, valDataset), + // TODO check that dataset is of valid type for model + this.model.train(dataset[1], valDataset?.[1]), ); yield gen; diff --git a/docs/examples/custom_task.ts b/docs/examples/custom_task.ts index fded9d788..8113dde9f 100644 --- a/docs/examples/custom_task.ts +++ b/docs/examples/custom_task.ts @@ -57,7 +57,7 @@ const customTask: TaskProvider = { metrics: ['accuracy'] }) - return Promise.resolve(new models.TFJS(model)) + return Promise.resolve(new models.TFJS('tabular', model)) } } diff --git a/server/src/task_set.ts b/server/src/task_set.ts index e40c95ad6..70f040aae 100644 --- a/server/src/task_set.ts +++ b/server/src/task_set.ts @@ -71,7 +71,10 @@ export class TaskSet extends EventEmitter<{ tfModel = await this.loadModelFromTask(taskOrProvider) } else if (model instanceof URL) { // Downloading the model if a URL is given - tfModel = new models.TFJS(await tf.loadLayersModel(model.href)) + tfModel = new models.TFJS( + task.trainingInformation.dataType, + await tf.loadLayersModel(model.href), + ) } else if (model instanceof Model) { // Don't do anything if the model is already specified tfModel = model diff --git a/server/tests/e2e/decentralized.spec.ts b/server/tests/e2e/decentralized.spec.ts index f572c0fe7..bde58987d 100644 --- a/server/tests/e2e/decentralized.spec.ts +++ b/server/tests/e2e/decentralized.spec.ts @@ -308,5 +308,5 @@ describe('end-to-end decentralized', function () { await new Promise((res, _) => setTimeout(res, statusUpdateTime)) // Wait some time for the status to update expect(await statusUser3.next()).equal("not enough participants") await discoUser3.close() - }).timeout("30s"); + }).timeout("5m"); }) diff --git a/server/tests/e2e/federated.spec.ts b/server/tests/e2e/federated.spec.ts index 651779493..0157b5d51 100644 --- a/server/tests/e2e/federated.spec.ts +++ b/server/tests/e2e/federated.spec.ts @@ -172,7 +172,7 @@ describe("end-to-end federated", () => { const [m1, m2] = await Promise.all([lusCovidUser(), lusCovidUser()]); assert.isTrue(m1.equals(m2)); - }).timeout("1m"); + }).timeout("10m"); it("two wikitext reach consensus", async () => { [server, url] = await new Server().serve( @@ -307,5 +307,5 @@ describe("end-to-end federated", () => { expect(await statusUser3.next()).equal("not enough participants") await discoUser3.close() - }).timeout("30s"); + }).timeout("5m"); }); diff --git a/webapp/src/components/task_creation_form/TaskForm.vue b/webapp/src/components/task_creation_form/TaskForm.vue index 612f43bd6..a7d4ed157 100644 --- a/webapp/src/components/task_creation_form/TaskForm.vue +++ b/webapp/src/components/task_creation_form/TaskForm.vue @@ -264,7 +264,10 @@ const onSubmit = async (rawTask: any): Promise => { let model try { - model = new models.TFJS(await tf.loadLayersModel(tf.io.browserFiles(modelFiles.value.toArray()))) + model = new models.TFJS( + task.trainingInformation.dataType, + await tf.loadLayersModel(tf.io.browserFiles(modelFiles.value.toArray())), + ); } catch (e) { debug("while loading model:%o", e); toaster.error('Model loading failed'); From ce187ba652d67641844cf1eda1a20bcca77929d1 Mon Sep 17 00:00:00 2001 From: tharvik Date: Thu, 5 Sep 2024 09:53:46 +0200 Subject: [PATCH 08/31] discojs/disco: option to keep preprocessed --- discojs/src/training/disco.ts | 49 +++++++++++++++++++------- server/tests/e2e/decentralized.spec.ts | 8 ++--- server/tests/e2e/federated.spec.ts | 14 ++++---- 3 files changed, 48 insertions(+), 23 deletions(-) diff --git a/discojs/src/training/disco.ts b/discojs/src/training/disco.ts index 8b72f0ecf..107515fd4 100644 --- a/discojs/src/training/disco.ts +++ b/discojs/src/training/disco.ts @@ -8,8 +8,10 @@ import { Task, TrainingInformation, processing, + Dataset, } from "../index.js"; import type { + Batched, TypedBatchedPreprocessedLabeledDataset, TypedLabeledDataset, } from "../index.js"; @@ -23,6 +25,9 @@ import { RoundLogs, Trainer } from "./trainer.js"; interface DiscoConfig { scheme: TrainingInformation["scheme"]; logger: Logger; + + // keep preprocessed dataset in memory while training + preprocessOnce: boolean; } export type RoundStatus = 'not enough participants' | // Server notification to wait for more participants @@ -40,6 +45,7 @@ export class Disco extends EventEmitter<{'status': RoundStatus}>{ readonly #client: clients.Client; readonly #logger: Logger; readonly #task: Task; + readonly #preprocessOnce: boolean; /** * Connect to the given task and get ready to train. @@ -52,10 +58,11 @@ export class Disco extends EventEmitter<{'status': RoundStatus}>{ clientConfig: clients.Client | URL | { aggregator: Aggregator; url: URL }, config: Partial ) { - super() - const { scheme, logger } = { + super(); + const { scheme, logger, preprocessOnce } = { scheme: task.trainingInformation.scheme, logger: new ConsoleLogger(), + preprocessOnce: false, ...config, }; @@ -76,6 +83,7 @@ export class Disco extends EventEmitter<{'status': RoundStatus}>{ throw new Error("client not setup for given task"); this.#logger = logger; + this.#preprocessOnce = preprocessOnce; this.#client = client; this.#task = task; this.trainer = new Trainer(task, client) @@ -131,7 +139,7 @@ export class Disco extends EventEmitter<{'status': RoundStatus}>{ this.#logger.success("Training started"); const [trainingDataset, validationDataset] = - await this.#preprocessBatchAndSplit(dataset); + await this.#preprocessSplitAndBatch(dataset); // the client fetches the latest weights upon connection this.trainer.model = await this.#client.connect(); @@ -177,7 +185,7 @@ export class Disco extends EventEmitter<{'status': RoundStatus}>{ await this.#client.disconnect(); } - async #preprocessBatchAndSplit( + async #preprocessSplitAndBatch( dataset: TypedLabeledDataset, ): Promise< [ @@ -185,31 +193,39 @@ export class Disco extends EventEmitter<{'status': RoundStatus}>{ TypedBatchedPreprocessedLabeledDataset, ] > { + const splitAndBatch = async ( + d: Dataset, + ): Promise<[Dataset>, Dataset>]> => { + const [training, validation] = ( + this.#preprocessOnce ? new Dataset(await arrayFromAsync(d)) : d + ).split(validationSplit); + + return [ + training.batch(batchSize).cached(), + validation.batch(batchSize).cached(), + ]; + }; + const preprocessed = await processing.preprocess(this.#task, dataset); + const { batchSize, validationSplit } = this.#task.trainingInformation; switch (preprocessed[0]) { case "image": { - const [training, validation] = preprocessed[1] - .split(validationSplit) - .map((d) => d.batch(batchSize).cached()); + const [training, validation] = await splitAndBatch(preprocessed[1]); return [ ["image", training], ["image", validation], ]; } case "tabular": { - const [training, validation] = preprocessed[1] - .split(validationSplit) - .map((d) => d.batch(batchSize).cached()); + const [training, validation] = await splitAndBatch(preprocessed[1]); return [ ["tabular", training], ["tabular", validation], ]; } case "text": { - const [training, validation] = preprocessed[1] - .split(validationSplit) - .map((d) => d.batch(batchSize).cached()); + const [training, validation] = await splitAndBatch(preprocessed[1]); return [ ["text", training], ["text", validation], @@ -218,3 +234,10 @@ export class Disco extends EventEmitter<{'status': RoundStatus}>{ } } } + +// Array.fromAsync not yet widely used (2024) +async function arrayFromAsync(iter: AsyncIterable): Promise { + const ret: T[] = []; + for await (const e of iter) ret.push(e); + return ret; +} diff --git a/server/tests/e2e/decentralized.spec.ts b/server/tests/e2e/decentralized.spec.ts index bde58987d..c43fac0da 100644 --- a/server/tests/e2e/decentralized.spec.ts +++ b/server/tests/e2e/decentralized.spec.ts @@ -202,7 +202,7 @@ describe('end-to-end decentralized', function () { const statusUpdateTime = 500 // allow some time for the client to update their status // Create User 1 - const discoUser1 = new Disco(lusCovidTask, url, { }); + const discoUser1 = new Disco(lusCovidTask, url, { preprocessOnce: true }); const statusUser1 = new Queue(); discoUser1.on("status", status => { statusUser1.put(status) }) const generatorUser1 = discoUser1.trainByRound(["image", dataset]) @@ -225,7 +225,7 @@ describe('end-to-end decentralized', function () { expect(await statusUser1.next()).equal("not enough participants") // but has to wait for more participants // Create User 2 - const discoUser2 = new Disco(lusCovidTask, url, { }); + const discoUser2 = new Disco(lusCovidTask, url, { preprocessOnce: true }); const statusUser2 = new Queue(); discoUser2.on("status", status => { statusUser2.put(status) }) const generatorUser2 = discoUser2.trainByRound(["image", dataset]) @@ -271,7 +271,7 @@ describe('end-to-end decentralized', function () { expect(await statusUser2.next()).equal("not enough participants") // Create User 3 - const discoUser3 = new Disco(lusCovidTask, url, { }); + const discoUser3 = new Disco(lusCovidTask, url, { preprocessOnce: true }); const statusUser3 = new Queue(); discoUser3.on("status", status => { statusUser3.put(status) }) const generatorUser3 = discoUser3.trainByRound(["image", dataset]) @@ -308,5 +308,5 @@ describe('end-to-end decentralized', function () { await new Promise((res, _) => setTimeout(res, statusUpdateTime)) // Wait some time for the status to update expect(await statusUser3.next()).equal("not enough participants") await discoUser3.close() - }).timeout("5m"); + }).timeout("1m"); }) diff --git a/server/tests/e2e/federated.spec.ts b/server/tests/e2e/federated.spec.ts index 0157b5d51..5bc16d7e1 100644 --- a/server/tests/e2e/federated.spec.ts +++ b/server/tests/e2e/federated.spec.ts @@ -48,7 +48,8 @@ describe("end-to-end federated", () => { ).zip(Repeat("cat")); const disco = new Disco(defaultTasks.cifar10.getTask(), url, { - scheme: "federated" + scheme: "federated", + preprocessOnce: true, }) await disco.trainFully(["image", dataset]); await disco.close(); @@ -126,6 +127,7 @@ describe("end-to-end federated", () => { const disco = new Disco(lusCovidTask, url, { scheme: "federated", + preprocessOnce: true, }); const logs = List( @@ -172,7 +174,7 @@ describe("end-to-end federated", () => { const [m1, m2] = await Promise.all([lusCovidUser(), lusCovidUser()]); assert.isTrue(m1.equals(m2)); - }).timeout("10m"); + }).timeout("1m"); it("two wikitext reach consensus", async () => { [server, url] = await new Server().serve( @@ -233,7 +235,7 @@ describe("end-to-end federated", () => { */ // Create User 1 - const discoUser1 = new Disco(lusCovidTask, url, { }); + const discoUser1 = new Disco(lusCovidTask, url, { preprocessOnce: true }); const statusUser1 = new Queue(); discoUser1.on("status", (status) => statusUser1.put(status)); const generatorUser1 = discoUser1.trainByRound(["image", dataset]) @@ -248,7 +250,7 @@ describe("end-to-end federated", () => { expect(await statusUser1.next()).equal("not enough participants") // Create User 2 - const discoUser2 = new Disco(lusCovidTask, url, { }); + const discoUser2 = new Disco(lusCovidTask, url, { preprocessOnce: true }); const statusUser2 = new Queue(); discoUser2.on("status", (status) => statusUser2.put(status)); const generatorUser2 = discoUser2.trainByRound(["image", dataset]) @@ -281,7 +283,7 @@ describe("end-to-end federated", () => { expect(await statusUser2.next()).equal("not enough participants") // Create User 3 - const discoUser3 = new Disco(lusCovidTask, url, { }); + const discoUser3 = new Disco(lusCovidTask, url, { preprocessOnce: true }); const statusUser3 = new Queue(); discoUser3.on("status", (status) => statusUser3.put(status)); const generatorUser3 = discoUser3.trainByRound(["image", dataset]) @@ -307,5 +309,5 @@ describe("end-to-end federated", () => { expect(await statusUser3.next()).equal("not enough participants") await discoUser3.close() - }).timeout("5m"); + }).timeout("1m"); }); From a52865b3234fa13ec974a7567e4cdcc62d4db614 Mon Sep 17 00:00:00 2001 From: tharvik Date: Tue, 10 Sep 2024 14:38:09 +0200 Subject: [PATCH 09/31] discojs/types: use better names --- cli/src/cli.ts | 4 +- cli/src/data.ts | 4 +- discojs/src/dataset/data/helpers.ts | 10 +- discojs/src/dataset/dataset.ts | 2 +- discojs/src/dataset/types.ts | 4 + discojs/src/models/gpt/gpt.spec.ts | 4 +- discojs/src/models/gpt/index.ts | 21 ++-- discojs/src/models/model.ts | 6 +- discojs/src/models/tfjs.ts | 18 ++-- discojs/src/processing/index.ts | 8 +- discojs/src/training/disco.ts | 24 +++-- discojs/src/training/trainer.ts | 14 +-- discojs/src/types.ts | 98 +++++++++++++------ discojs/src/validation/validator.ts | 10 +- docs/examples/training.ts | 6 +- webapp/src/components/training/Trainer.vue | 4 +- .../src/components/training/TrainingSteps.vue | 9 +- 17 files changed, 142 insertions(+), 104 deletions(-) diff --git a/cli/src/cli.ts b/cli/src/cli.ts index 4591a80a6..5ce11f8bb 100644 --- a/cli/src/cli.ts +++ b/cli/src/cli.ts @@ -4,7 +4,7 @@ import "@tensorflow/tfjs-node" import { List, Range } from 'immutable' import fs from 'node:fs/promises' -import type { RoundLogs, Task, TaskProvider, TypedLabeledDataset } from '@epfml/discojs' +import type { RoundLogs, Task, TaskProvider, TypedRawDataset } from '@epfml/discojs' import { Disco, aggregator as aggregators, client as clients } from '@epfml/discojs' import { Server } from 'server' @@ -21,7 +21,7 @@ async function arrayFromAsync(iter: AsyncIterable): Promise { async function runUser( task: Task, url: URL, - data: TypedLabeledDataset, + data: TypedRawDataset, ): Promise> { const trainingScheme = task.trainingInformation.scheme const aggregator = aggregators.getAggregator(task) diff --git a/cli/src/data.ts b/cli/src/data.ts index 37ce1cf17..88eb49525 100644 --- a/cli/src/data.ts +++ b/cli/src/data.ts @@ -1,6 +1,6 @@ import path from "node:path"; -import type { Dataset, Image, Task, TypedLabeledDataset } from "@epfml/discojs"; +import type { Dataset, Image, Task, TypedRawDataset } from "@epfml/discojs"; import { loadCSV, loadImagesInDir } from "@epfml/discojs-node"; import { Repeat } from "immutable"; @@ -30,7 +30,7 @@ async function loadLusCovidData(): Promise> { return positive.chain(negative); } -export async function getTaskData(task: Task): Promise { +export async function getTaskData(task: Task): Promise { switch (task.id) { case "simple_face": return ["image", await loadSimpleFaceData()]; diff --git a/discojs/src/dataset/data/helpers.ts b/discojs/src/dataset/data/helpers.ts index 36bc9b47f..752ab3133 100644 --- a/discojs/src/dataset/data/helpers.ts +++ b/discojs/src/dataset/data/helpers.ts @@ -10,8 +10,8 @@ import type { Image, Tabular, Task, - TypedDataset, - TypedLabeledDataset, + TypedRawDataset, + TypedRawWithoutLabelDataset, } from "../../index.js"; import { processing } from "../../index.js"; @@ -41,7 +41,7 @@ function tabularToNumbers(columns: Iterable, row: Tabular): number[] { export async function datasetToData( task: Task, - [t, dataset]: TypedDataset, + [t, dataset]: TypedRawWithoutLabelDataset, ): Promise { switch (t) { case "image": { @@ -69,7 +69,7 @@ export async function datasetToData( export async function labeledDatasetToData( task: Task, - [t, dataset]: TypedLabeledDataset, + [t, dataset]: TypedRawDataset, ): Promise { switch (t) { case "image": { @@ -117,7 +117,7 @@ export async function labeledDatasetToData( export async function labeledDatasetToDataSplit( task: Task, - [t, dataset]: TypedLabeledDataset, + [t, dataset]: TypedRawDataset, ): Promise { const split = task.trainingInformation.validationSplit; diff --git a/discojs/src/dataset/dataset.ts b/discojs/src/dataset/dataset.ts index e9c87e41e..7a3060b2f 100644 --- a/discojs/src/dataset/dataset.ts +++ b/discojs/src/dataset/dataset.ts @@ -1,7 +1,7 @@ import createDebug from "debug"; import { List, Range } from "immutable"; -import type { Batched } from "../index.js"; +import { Batched } from "./types.js"; const debug = createDebug("discojs:dataset"); diff --git a/discojs/src/dataset/types.ts b/discojs/src/dataset/types.ts index 79c510ea0..86700a28e 100644 --- a/discojs/src/dataset/types.ts +++ b/discojs/src/dataset/types.ts @@ -1,5 +1,9 @@ +import { List } from "immutable"; + import { Image } from "./image.js" +export type Batched = List; + export { Image }; export type Tabular = Partial>; export type Text = string; diff --git a/discojs/src/models/gpt/gpt.spec.ts b/discojs/src/models/gpt/gpt.spec.ts index 4be871e32..b2d2cf06c 100644 --- a/discojs/src/models/gpt/gpt.spec.ts +++ b/discojs/src/models/gpt/gpt.spec.ts @@ -2,7 +2,7 @@ import { expect } from "chai"; import "@tensorflow/tfjs-node"; // speed up import { AutoTokenizer } from "@xenova/transformers"; -import { Dataset, DataTypeToPreprocessedLabeled } from "../../index.js"; +import { Dataset, ModelEncoded } from "../../index.js"; import { GPT } from "./index.js"; import type { GPTConfig } from "./config.js"; @@ -26,7 +26,7 @@ describe("gpt-tfjs", function () { const tokenizer = await AutoTokenizer.from_pretrained("Xenova/gpt2"); const tokenDataset = new Dataset(Repeat(data)) - .map((text) => { + .map((text) => { const { input_ids: tokens } = tokenizer(text, { padding: true, truncation: true, diff --git a/discojs/src/models/gpt/index.ts b/discojs/src/models/gpt/index.ts index ca6fcfcd9..66cbe81a5 100644 --- a/discojs/src/models/gpt/index.ts +++ b/discojs/src/models/gpt/index.ts @@ -7,13 +7,8 @@ import { List, Range } from "immutable"; import * as tf from '@tensorflow/tfjs' import { PreTrainedTokenizer } from '@xenova/transformers'; -import { - Batched, - Dataset, - DataTypeToBatchedPreprocessedLabeledDataset, - DataTypeToPreprocessedLabeled, - WeightsContainer, -} from "../../index.js"; +import type { Batched, Dataset, ModelEncoded } from "../../index.js"; +import { WeightsContainer } from "../../index.js"; import { BatchLogs, Model, EpochLogs } from "../index.js"; import type { Prediction, Sample } from '../model.js' @@ -56,8 +51,8 @@ export class GPT extends Model<"text"> { * @param tracker */ override async *train( - trainingDataset: Dataset>, - validationDataset?: Dataset>, + trainingDataset: Dataset>, + validationDataset?: Dataset>, ): AsyncGenerator { let batchesLogs = List(); @@ -76,9 +71,7 @@ export class GPT extends Model<"text"> { return new EpochLogs(batchesLogs, validation); } - async #runBatch( - batch: Batched, - ): Promise { + async #runBatch(batch: Batched): Promise { const tfBatch = this.#batchToTF(batch); let logs: tf.Logs | undefined; @@ -106,7 +99,7 @@ export class GPT extends Model<"text"> { } async #evaluate( - dataset: DataTypeToBatchedPreprocessedLabeledDataset["text"], + dataset: Dataset>, ): Promise> { const evaluation = await evaluate( this.model, @@ -120,7 +113,7 @@ export class GPT extends Model<"text"> { }; } - #batchToTF(batch: Batched): { + #batchToTF(batch: Batched): { xs: tf.Tensor2D; ys: tf.Tensor3D; } { diff --git a/discojs/src/models/model.ts b/discojs/src/models/model.ts index 50c65cae1..2e5102e80 100644 --- a/discojs/src/models/model.ts +++ b/discojs/src/models/model.ts @@ -4,7 +4,7 @@ import type { Batched, Dataset, DataType, - DataTypeToPreprocessedLabeled, + ModelEncoded, WeightsContainer, } from "../index.js"; @@ -37,8 +37,8 @@ export abstract class Model * @yields on every epoch, training can be stop by `return`ing or `throw`ing it */ abstract train( - trainingDataset: Dataset>, - validationDataset?: Dataset>, + trainingDataset: Dataset>, + validationDataset?: Dataset>, ): AsyncGenerator; /** Predict likely values */ diff --git a/discojs/src/models/tfjs.ts b/discojs/src/models/tfjs.ts index ca4e42192..c6b95d872 100644 --- a/discojs/src/models/tfjs.ts +++ b/discojs/src/models/tfjs.ts @@ -5,7 +5,7 @@ import { Batched, Dataset, DataType, - DataTypeToPreprocessedLabeled, + ModelEncoded, WeightsContainer, } from "../index.js"; @@ -39,8 +39,8 @@ export class TFJS extends Model { } override async *train( - trainingDataset: Dataset>, - validationDataset?: Dataset>, + trainingDataset: Dataset>, + validationDataset?: Dataset>, ): AsyncGenerator { let batchesLogs = List(); for await (const [batch, batchNumber] of trainingDataset.zip(Range())) { @@ -59,7 +59,7 @@ export class TFJS extends Model { } async #runBatch( - batch: Batched, + batch: Batched, ): Promise> { const { xs, ys } = this.#batchToTF(batch); @@ -87,7 +87,7 @@ export class TFJS extends Model { } async #evaluate( - dataset: Dataset>, + dataset: Dataset>, ): Promise> { const evaluation = await this.model.evaluateDataset( intoTFDataset(dataset.map((batch) => this.#batchToTF(batch))), @@ -171,7 +171,7 @@ export class TFJS extends Model { } #batchToTF( - batch: Batched, + batch: Batched, ): Record<"xs" | "ys", tf.Tensor> { const outputSize = tf.util.sizeFromShape( this.model.outputShape.map((dim) => { @@ -184,7 +184,7 @@ export class TFJS extends Model { switch (this.datatype) { case "image": { // cast as typescript doesn't reduce generic type - const b = batch as Batched; + const b = batch as Batched; return tf.tidy(() => ({ xs: tf.stack( @@ -209,7 +209,7 @@ export class TFJS extends Model { } case "tabular": { // cast as typescript doesn't reduce generic type - const b = batch as Batched; + const b = batch as Batched; return { xs: tf.stack( @@ -222,7 +222,7 @@ export class TFJS extends Model { } case "text": { // cast as typescript doesn't reduce generic type - const b = batch as Batched; + const b = batch as Batched; return { xs: tf.stack( diff --git a/discojs/src/processing/index.ts b/discojs/src/processing/index.ts index 1a9055733..040e07d82 100644 --- a/discojs/src/processing/index.ts +++ b/discojs/src/processing/index.ts @@ -6,8 +6,8 @@ import { AutoTokenizer } from "@xenova/transformers"; import type { Tabular, Task, - TypedLabeledDataset, - TypedPreprocessedLabeledDataset, + TypedModelEncodedDataset, + TypedRawDataset, } from "../index.js"; import * as processing from "./index.js"; @@ -18,8 +18,8 @@ export * from "./text.js"; export async function preprocess( task: Task, - [t, dataset]: TypedLabeledDataset, -): Promise { + [t, dataset]: TypedRawDataset, +): Promise { switch (t) { case "image": { const { LABEL_LIST, IMAGE_H, IMAGE_W } = task.trainingInformation; diff --git a/discojs/src/training/disco.ts b/discojs/src/training/disco.ts index 107515fd4..4f197633b 100644 --- a/discojs/src/training/disco.ts +++ b/discojs/src/training/disco.ts @@ -5,15 +5,15 @@ import { ConsoleLogger, EpochLogs, Logger, - Task, TrainingInformation, processing, Dataset, } from "../index.js"; import type { Batched, - TypedBatchedPreprocessedLabeledDataset, - TypedLabeledDataset, + Task, + TypedBatchedModelEncodedDataset, + TypedRawDataset, } from "../index.js"; import type { Aggregator } from "../aggregator/index.js"; import { getAggregator } from "../aggregator/index.js"; @@ -92,7 +92,7 @@ export class Disco extends EventEmitter<{'status': RoundStatus}>{ } /** Train on dataset, yielding logs of every round. */ - async *trainByRound(dataset: TypedLabeledDataset): AsyncGenerator { + async *trainByRound(dataset: TypedRawDataset): AsyncGenerator { for await (const round of this.train(dataset)) { const [roundGen, roundLogs] = async_iterator.split(round); for await (const epoch of roundGen) for await (const _ of epoch); @@ -101,7 +101,7 @@ export class Disco extends EventEmitter<{'status': RoundStatus}>{ } /** Train on dataset, yielding logs of every epoch. */ - async *trainByEpoch(dataset: TypedLabeledDataset): AsyncGenerator { + async *trainByEpoch(dataset: TypedRawDataset): AsyncGenerator { for await (const round of this.train(dataset)) { for await (const epoch of round) { const [epochGen, epochLogs] = async_iterator.split(epoch); @@ -112,15 +112,13 @@ export class Disco extends EventEmitter<{'status': RoundStatus}>{ } /** Train on dataset, yielding logs of every batch. */ - async *trainByBatch( - dataTuple: TypedLabeledDataset, - ): AsyncGenerator { + async *trainByBatch(dataTuple: TypedRawDataset): AsyncGenerator { for await (const round of this.train(dataTuple)) for await (const epoch of round) yield* epoch; } /** Run whole train on dataset. */ - async trainFully(dataTuple: TypedLabeledDataset): Promise { + async trainFully(dataTuple: TypedRawDataset): Promise { for await (const round of this.train(dataTuple)) for await (const epoch of round) for await (const _ of epoch); } @@ -132,7 +130,7 @@ export class Disco extends EventEmitter<{'status': RoundStatus}>{ * If you don't care about the whole process, use one of the other train methods. **/ async *train( - dataset: TypedLabeledDataset, + dataset: TypedRawDataset, ): AsyncGenerator< AsyncGenerator, RoundLogs> > { @@ -186,11 +184,11 @@ export class Disco extends EventEmitter<{'status': RoundStatus}>{ } async #preprocessSplitAndBatch( - dataset: TypedLabeledDataset, + dataset: TypedRawDataset, ): Promise< [ - TypedBatchedPreprocessedLabeledDataset, - TypedBatchedPreprocessedLabeledDataset, + TypedBatchedModelEncodedDataset, + TypedBatchedModelEncodedDataset, ] > { const splitAndBatch = async ( diff --git a/discojs/src/training/trainer.ts b/discojs/src/training/trainer.ts index 87fffb5f4..cc5b1eeb8 100644 --- a/discojs/src/training/trainer.ts +++ b/discojs/src/training/trainer.ts @@ -6,7 +6,7 @@ import type { EpochLogs, Model, Task, - TypedBatchedPreprocessedLabeledDataset, + TypedBatchedModelEncodedDataset, WeightsContainer, } from "../index.js"; import { privacy } from "../index.js"; @@ -59,8 +59,8 @@ export class Trainer { } async *train( - dataset: TypedBatchedPreprocessedLabeledDataset, - valDataset?: TypedBatchedPreprocessedLabeledDataset, + dataset: TypedBatchedModelEncodedDataset, + valDataset?: TypedBatchedModelEncodedDataset, ): AsyncGenerator< AsyncGenerator, RoundLogs>, void @@ -79,8 +79,8 @@ export class Trainer { } async *#runRounds( - dataset: TypedBatchedPreprocessedLabeledDataset, - valDataset?: TypedBatchedPreprocessedLabeledDataset, + dataset: TypedBatchedModelEncodedDataset, + valDataset?: TypedBatchedModelEncodedDataset, ): AsyncGenerator< AsyncGenerator, RoundLogs>, void @@ -109,8 +109,8 @@ export class Trainer { } async *#runRound( - dataset: TypedBatchedPreprocessedLabeledDataset, - valDataset?: TypedBatchedPreprocessedLabeledDataset, + dataset: TypedBatchedModelEncodedDataset, + valDataset?: TypedBatchedModelEncodedDataset, ): AsyncGenerator, RoundLogs> { let epochsLogs = List(); for (let epoch = 0; epoch < this.#roundDuration; epoch++) { diff --git a/discojs/src/types.ts b/discojs/src/types.ts index 9a8e12ff2..6a77f393a 100644 --- a/discojs/src/types.ts +++ b/discojs/src/types.ts @@ -1,43 +1,81 @@ import { List } from "immutable"; -import type { Dataset, Image, processing, Tabular, Text } from "./index.js"; +import type { + Batched, + Dataset, + Image, + processing, + Tabular, + Text, +} from "./index.js"; + +/** + * The data that we handle goes through various stages. + * The labels also gets transformed at each stage. + * + * raw -> model encoded -> inferred + */ export type DataType = "image" | "tabular" | "text"; -type Token = number; -export interface DataTypeToPreprocessed { - image: processing.NormalizedImage<3>; - tabular: List; - text: List; +/** what get's ingested by Disco */ +export interface Raw { + image: [Image, label: string]; + tabular: Tabular; + text: Text; } -export interface DataTypeToPreprocessedLabeled { - image: [DataTypeToPreprocessed["image"], label: number]; - tabular: [features: DataTypeToPreprocessed["tabular"], labels: List]; - text: [DataTypeToPreprocessed["text"], nexts: List]; +/** what get's ingested by the Validator */ +export interface RawWithoutLabel { + image: Image; + tabular: Tabular; + text: Text; } -export type Batched = List; +type Token = number; +export interface ModelEncoded { + image: [processing.NormalizedImage<3>, number]; + tabular: [List, List]; + text: [List, List]; +} +export type ModelEncodedWithoutLabel = { [D in DataType]: ModelEncoded[D][0] }; +export type ModelEncodedOnlyWithLabel = { [D in DataType]: ModelEncoded[D][1] }; -export type DataTypeToBatchedPreprocessedLabeledDataset = { - [D in DataType]: Dataset>; -}; +export interface Inferred { + image: string; + tabular: Partial>; + text: string; +} -export type TypedDataset = - | ["image", Dataset] - | ["tabular", Dataset] - | ["text", Dataset]; +// allow to handle multiple type +// TODO will be removed when fully generic -export type TypedLabeledDataset = - | ["image", Dataset<[Image, label: string]>] - | ["tabular", Dataset] - | ["text", Dataset]; +export type TypedRawDataset = + | ["image", Dataset] + | ["tabular", Dataset] + | ["text", Dataset]; +export type TypedRawWithoutLabelDataset = + | ["image", Dataset] + | ["tabular", Dataset] + | ["text", Dataset]; -export type TypedPreprocessedLabeledDataset = - | ["image", Dataset] - | ["tabular", Dataset] - | ["text", Dataset]; +export type TypedModelEncodedDataset = + | ["image", Dataset] + | ["tabular", Dataset] + | ["text", Dataset]; +export type TypedBatchedModelEncodedDataset = + | ["image", Dataset>] + | ["tabular", Dataset>] + | ["text", Dataset>]; +export type TypedModelEncodedWithoutLabelDataset = + | ["image", Dataset] + | ["tabular", Dataset] + | ["text", Dataset]; +export type TypedModelEncodedOnlyWithLabelDataset = + | ["image", Dataset] + | ["tabular", Dataset] + | ["text", Dataset]; -export type TypedBatchedPreprocessedLabeledDataset = - | ["image", Dataset>] - | ["tabular", Dataset>] - | ["text", Dataset>]; +export type TypedInferredDataset = + | ["image", Dataset] + | ["tabular", Dataset] + | ["text", Dataset]; diff --git a/discojs/src/validation/validator.ts b/discojs/src/validation/validator.ts index 437cbf73a..3194dcde0 100644 --- a/discojs/src/validation/validator.ts +++ b/discojs/src/validation/validator.ts @@ -3,8 +3,8 @@ import * as tf from "@tensorflow/tfjs"; import type { Model, Task, - TypedDataset, - TypedLabeledDataset, + TypedRawDataset, + TypedRawWithoutLabelDataset, } from "../index.js"; import { datasetToData, @@ -31,7 +31,7 @@ export class Validator { } /** infer every line of the dataset and check that it is as labeled */ - async *test(dataset: TypedLabeledDataset): AsyncGenerator { + async *test(dataset: TypedRawDataset): AsyncGenerator { const preprocessed = ( await labeledDatasetToData(this.task, dataset) ).preprocess(); @@ -69,7 +69,9 @@ export class Validator { } /** use the model to predict every line of the dataset */ - async *infer(dataset: TypedDataset): AsyncGenerator { + async *infer( + dataset: TypedRawWithoutLabelDataset, + ): AsyncGenerator { const data = await datasetToData(this.task, dataset); const batched = data.preprocess().batch().dataset; diff --git a/docs/examples/training.ts b/docs/examples/training.ts index d3600deb2..5f0632761 100644 --- a/docs/examples/training.ts +++ b/docs/examples/training.ts @@ -2,7 +2,7 @@ import { Repeat } from 'immutable' import * as path from 'node:path' import '@tensorflow/tfjs-node' -import type { Dataset, Image, Task, TypedLabeledDataset } from '@epfml/discojs' +import type { Dataset, Image, Task, TypedRawDataset } from '@epfml/discojs' import { Disco, fetchTasks, defaultTasks } from '@epfml/discojs' import { loadCSV, loadImagesInDir } from '@epfml/discojs-node' import { Server } from 'server' @@ -11,7 +11,7 @@ import { Server } from 'server' * Example of discojs API, we load data, build the appropriate loggers, the disco object * and finally start training. */ -async function runUser (url: URL, task: Task, dataset: TypedLabeledDataset): Promise { +async function runUser (url: URL, task: Task, dataset: TypedRawDataset): Promise { // Create Disco object associated with the server url, the training scheme const disco = new Disco(task, url, { scheme: 'federated' }) @@ -35,7 +35,7 @@ async function main (): Promise { // Choose the task and load local data // Make sure you first ran ./get_training_data let task: Task | undefined - let dataset: TypedLabeledDataset + let dataset: TypedRawDataset switch (NAME) { case 'titanic': { task = tasks.get('titanic') diff --git a/webapp/src/components/training/Trainer.vue b/webapp/src/components/training/Trainer.vue index 2462fa72a..ffbee4435 100644 --- a/webapp/src/components/training/Trainer.vue +++ b/webapp/src/components/training/Trainer.vue @@ -128,7 +128,7 @@ import type { RoundLogs, RoundStatus, Task, - TypedLabeledDataset, + TypedRawDataset, } from "@epfml/discojs"; import { async_iterator, Disco } from "@epfml/discojs"; @@ -145,7 +145,7 @@ const toaster = useToaster(); const props = defineProps<{ task: Task; - dataset?: TypedLabeledDataset; + dataset?: TypedRawDataset; }>(); const emit = defineEmits<{ model: [Model]; diff --git a/webapp/src/components/training/TrainingSteps.vue b/webapp/src/components/training/TrainingSteps.vue index 9372f46e7..9c8346f79 100644 --- a/webapp/src/components/training/TrainingSteps.vue +++ b/webapp/src/components/training/TrainingSteps.vue @@ -46,11 +46,12 @@ import { useRouter, useRoute } from "vue-router"; import type { Dataset, + Image, Model, Tabular, TaskID, Text, - TypedLabeledDataset, + TypedRawDataset, } from "@epfml/discojs"; import { useTrainingStore } from "@/store"; @@ -112,7 +113,7 @@ const labels = computed(() => Set(task.value?.trainingInformation.LABEL_LIST)); const imageDataset = ref(); const tabularDataset = ref>(); const textDataset = ref>(); -const dataset = computed(() => { +const dataset = computed(() => { if ( Set.of( imageDataset.value, @@ -125,7 +126,9 @@ const dataset = computed(() => { if (imageDataset.value !== undefined) return [ "image", - toRaw(imageDataset.value).map(({ image, label }) => [image, label]), + toRaw(imageDataset.value).map( + ({ image, label }) => [image, label] as [Image, string], + ), ]; if (tabularDataset.value !== undefined) return ["tabular", toRaw(tabularDataset.value)]; From 977b66fee1db381190f08e57f33626201e5d7d6b Mon Sep 17 00:00:00 2001 From: tharvik Date: Tue, 10 Sep 2024 14:43:50 +0200 Subject: [PATCH 10/31] discojs/validator: flatten --- discojs/src/index.ts | 2 +- discojs/src/validation/index.ts | 1 - discojs/src/{validation => }/validator.ts | 7 ++----- 3 files changed, 3 insertions(+), 7 deletions(-) delete mode 100644 discojs/src/validation/index.ts rename discojs/src/{validation => }/validator.ts (97%) diff --git a/discojs/src/index.ts b/discojs/src/index.ts index 867bcfee4..c635074a6 100644 --- a/discojs/src/index.ts +++ b/discojs/src/index.ts @@ -9,7 +9,7 @@ export * as aggregator from './aggregator/index.js' export { WeightsContainer, aggregation } from './weights/index.js' export { Logger, ConsoleLogger } from './logging/index.js' export { Disco, RoundLogs, RoundStatus } from './training/index.js' -export { Validator } from './validation/index.js' +export { Validator } from './validator.js' export { Model, BatchLogs, EpochLogs, ValidationMetrics } from './models/index.js' export * as models from './models/index.js' diff --git a/discojs/src/validation/index.ts b/discojs/src/validation/index.ts deleted file mode 100644 index c1ad9037c..000000000 --- a/discojs/src/validation/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { Validator } from './validator.js' diff --git a/discojs/src/validation/validator.ts b/discojs/src/validator.ts similarity index 97% rename from discojs/src/validation/validator.ts rename to discojs/src/validator.ts index 3194dcde0..0e7648d7a 100644 --- a/discojs/src/validation/validator.ts +++ b/discojs/src/validator.ts @@ -5,11 +5,8 @@ import type { Task, TypedRawDataset, TypedRawWithoutLabelDataset, -} from "../index.js"; -import { - datasetToData, - labeledDatasetToData, -} from "../dataset/data/helpers.js"; +} from "./index.js"; +import { datasetToData, labeledDatasetToData } from "./dataset/data/helpers.js"; function intoTFDataset( iter: AsyncIterable, From 35d09c5eb56e55a8bc0849d4d9538f542d67a6d0 Mon Sep 17 00:00:00 2001 From: tharvik Date: Tue, 10 Sep 2024 17:45:12 +0200 Subject: [PATCH 11/31] discojs/processing: add validator specifics --- discojs/src/processing/index.ts | 110 ++++++++++++++++++++++++++++---- 1 file changed, 96 insertions(+), 14 deletions(-) diff --git a/discojs/src/processing/index.ts b/discojs/src/processing/index.ts index 040e07d82..92102ad4d 100644 --- a/discojs/src/processing/index.ts +++ b/discojs/src/processing/index.ts @@ -1,14 +1,18 @@ /** Dataset shapers, convenient to map with */ -import { List } from "immutable"; -import { AutoTokenizer } from "@xenova/transformers"; +import { List, Map } from "immutable"; import type { Tabular, Task, + TypedInferredDataset, TypedModelEncodedDataset, + TypedModelEncodedOnlyWithLabelDataset, + TypedModelEncodedWithoutLabelDataset, TypedRawDataset, + TypedRawWithoutLabelDataset, } from "../index.js"; +import { models } from "../index.js"; import * as processing from "./index.js"; @@ -48,18 +52,13 @@ export async function preprocess( return [ "tabular", dataset.map((row) => [ - tabularToNumbers(inputColumns, row), - tabularToNumbers(outputColumns, row), + extractToNumbers(inputColumns, row), + extractToNumbers(outputColumns, row), ]), ]; } case "text": { - const tokenizerName = task.trainingInformation.tokenizer; - if (typeof tokenizerName !== "string") - throw Error( - "no tokenizer name specified in the task training information", - ); - const tokenizer = await AutoTokenizer.from_pretrained(tokenizerName); + const tokenizer = await models.getTaskTokenizer(task); const totalTokenCount = task.trainingInformation.maxSequenceLength ?? (tokenizer.model_max_length as number); @@ -76,10 +75,93 @@ export async function preprocess( } } -function tabularToNumbers( - columns: Iterable, - row: Tabular, -): List { +export async function preprocessWithoutLabel( + task: Task, + [t, dataset]: TypedRawWithoutLabelDataset, +): Promise { + switch (t) { + case "image": { + const { IMAGE_H, IMAGE_W } = task.trainingInformation; + if (IMAGE_H === undefined || IMAGE_W === undefined) + throw new Error("task is missing fields for image dataset"); + + return [ + "image", + dataset.map((image) => + processing.normalize( + processing.removeAlpha(processing.resize(IMAGE_W, IMAGE_H, image)), + ), + ), + ]; + } + case "tabular": { + const { inputColumns } = task.trainingInformation; + if (inputColumns === undefined) + throw new Error("tabular task without input columns"); + + return [ + "tabular", + dataset.map((row) => extractToNumbers(inputColumns, row)), + ]; + } + case "text": { + const tokenizer = await models.getTaskTokenizer(task); + const totalTokenCount = + task.trainingInformation.maxSequenceLength ?? + (tokenizer.model_max_length as number); + + return [ + "text", + dataset + .map((line) => + processing.tokenizeAndLeftPad(line, tokenizer, totalTokenCount), + ) + .map((tokens) => tokens.pop()), + ]; + } + } +} + +export async function postprocess( + task: Task, + [t, dataset]: TypedModelEncodedOnlyWithLabelDataset, +): Promise { + switch (t) { + case "image": { + const { LABEL_LIST } = task.trainingInformation; + if (LABEL_LIST === undefined) + throw new Error("task is missing fields for image dataset"); + const labels = List(LABEL_LIST); + + return [ + "image", + dataset.map((index) => { + const v = labels.get(index); + if (v === undefined) throw new Error("index not found in labels"); + return v; + }), + ]; + } + case "tabular": { + const { outputColumns } = task.trainingInformation; + if (outputColumns === undefined) + throw new Error("tabular task without input columns"); + const output = List(outputColumns); + + return ["tabular", dataset.map((row) => Map(output.zip(row)).toObject())]; + } + case "text": { + const tokenizer = await models.getTaskTokenizer(task); + + return [ + "text", + dataset.map((tokens) => tokenizer.decode(tokens.toArray())), + ]; + } + } +} + +function extractToNumbers(columns: Iterable, row: Tabular) { return ( List(columns) .map((column) => processing.extractColumn(row, column)) From e09b6ff7a4ee87c3a4c25f823fac06eb54e19901 Mon Sep 17 00:00:00 2001 From: tharvik Date: Tue, 10 Sep 2024 23:40:44 +0200 Subject: [PATCH 12/31] discojs/types/text: single output token --- cli/src/benchmark_gpt.ts | 2 +- discojs/src/models/gpt/gpt.spec.ts | 2 +- discojs/src/models/gpt/index.ts | 4 +++- discojs/src/models/tfjs.ts | 4 +++- discojs/src/processing/index.ts | 7 ++----- discojs/src/types.ts | 2 +- 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/cli/src/benchmark_gpt.ts b/cli/src/benchmark_gpt.ts index ced2a82bd..b86f69852 100644 --- a/cli/src/benchmark_gpt.ts +++ b/cli/src/benchmark_gpt.ts @@ -82,7 +82,7 @@ async function main(args: Required): Promise { const maxLength = task.trainingInformation.maxSequenceLength ?? (tokenizer.model_max_length as number) + 1 const preprocessedDataset = dataset .map((line) => processing.tokenizeAndLeftPad(line, tokenizer, maxLength)) - .map((tokens) => [tokens.pop(), tokens.shift()] as [List, List]) + .map((tokens) => [tokens.pop(), tokens.last()] as [List, number]) .batch(batchSize); // Init and train the model diff --git a/discojs/src/models/gpt/gpt.spec.ts b/discojs/src/models/gpt/gpt.spec.ts index b2d2cf06c..79cc892e9 100644 --- a/discojs/src/models/gpt/gpt.spec.ts +++ b/discojs/src/models/gpt/gpt.spec.ts @@ -34,7 +34,7 @@ describe("gpt-tfjs", function () { max_length: config.blockSize + 1, }) as { input_ids: number[] }; - return [List(tokens.slice(0, config.blockSize)), List(tokens.slice(1))]; + return [List(tokens.slice(0, config.blockSize)), tokens[config.blockSize]]; }) .batch(64); diff --git a/discojs/src/models/gpt/index.ts b/discojs/src/models/gpt/index.ts index 66cbe81a5..9ad90a7cf 100644 --- a/discojs/src/models/gpt/index.ts +++ b/discojs/src/models/gpt/index.ts @@ -123,7 +123,9 @@ export class GPT extends Model<"text"> { ) as tf.Tensor2D, // cast as stack doesn't type ys: tf.stack( batch - .map(([_, next]) => tf.oneHot(next.toArray(), this.#vocabSize)) + .map(([line, next]) => + tf.oneHot(line.shift().push(next).toArray(), this.#vocabSize), + ) .toArray(), ) as tf.Tensor3D, // cast as oneHot/stack doesn't type })); diff --git a/discojs/src/models/tfjs.ts b/discojs/src/models/tfjs.ts index c6b95d872..c34bacc39 100644 --- a/discojs/src/models/tfjs.ts +++ b/discojs/src/models/tfjs.ts @@ -230,7 +230,9 @@ export class TFJS extends Model { ), ys: tf.stack( b - .map(([_, next]) => tf.oneHot(next.toArray(), outputSize)) + .map(([line, next]) => + tf.oneHot(line.shift().push(next).toArray(), outputSize), + ) .toArray(), ), }; diff --git a/discojs/src/processing/index.ts b/discojs/src/processing/index.ts index 92102ad4d..7e29ccf35 100644 --- a/discojs/src/processing/index.ts +++ b/discojs/src/processing/index.ts @@ -69,7 +69,7 @@ export async function preprocess( .map((line) => processing.tokenizeAndLeftPad(line, tokenizer, totalTokenCount), ) - .map((tokens) => [tokens.pop(), tokens.shift()]), + .map((tokens) => [tokens.pop(), tokens.last()]), ]; } } @@ -153,10 +153,7 @@ export async function postprocess( case "text": { const tokenizer = await models.getTaskTokenizer(task); - return [ - "text", - dataset.map((tokens) => tokenizer.decode(tokens.toArray())), - ]; + return ["text", dataset.map((token) => tokenizer.decode([token]))]; } } } diff --git a/discojs/src/types.ts b/discojs/src/types.ts index 6a77f393a..3a6714528 100644 --- a/discojs/src/types.ts +++ b/discojs/src/types.ts @@ -35,7 +35,7 @@ type Token = number; export interface ModelEncoded { image: [processing.NormalizedImage<3>, number]; tabular: [List, List]; - text: [List, List]; + text: [List, Token]; } export type ModelEncodedWithoutLabel = { [D in DataType]: ModelEncoded[D][0] }; export type ModelEncodedOnlyWithLabel = { [D in DataType]: ModelEncoded[D][1] }; From 654c4b688bdf3c973818a3cec678bcadacb3c462 Mon Sep 17 00:00:00 2001 From: tharvik Date: Fri, 13 Sep 2024 11:03:58 +0200 Subject: [PATCH 13/31] *: bump deps --- discojs-web/package.json | 2 +- discojs/package.json | 2 +- discojs/src/dataset/data/helpers.ts | 1 - discojs/src/models/gpt/index.ts | 15 +- discojs/src/models/tfjs.ts | 15 +- discojs/src/validator.ts | 1 - package-lock.json | 2642 ++++++++++---------------- package.json | 2 +- server/package.json | 2 +- server/src/routes/task_router.ts | 5 +- server/src/routes/training_router.ts | 4 +- webapp/package.json | 4 +- 12 files changed, 994 insertions(+), 1701 deletions(-) diff --git a/discojs-web/package.json b/discojs-web/package.json index e6e82c3dd..a4b52a85c 100644 --- a/discojs-web/package.json +++ b/discojs-web/package.json @@ -27,6 +27,6 @@ "@types/papaparse": "5", "jsdom": "25", "nodemon": "3", - "vitest": "1" + "vitest": "2" } } diff --git a/discojs/package.json b/discojs/package.json index aae6a4790..f500bf8ff 100644 --- a/discojs/package.json +++ b/discojs/package.json @@ -33,7 +33,7 @@ }, "devDependencies": { "@tensorflow/tfjs-node": "4", - "@types/chai": "4", + "@types/chai": "5", "@types/mocha": "10", "@types/simple-peer": "9", "chai": "5", diff --git a/discojs/src/dataset/data/helpers.ts b/discojs/src/dataset/data/helpers.ts index 752ab3133..75a3e3ff4 100644 --- a/discojs/src/dataset/data/helpers.ts +++ b/discojs/src/dataset/data/helpers.ts @@ -21,7 +21,6 @@ import { DataSplit } from "./data_split.js"; function intoTFDataset( iter: AsyncIterable, ): tf.data.Dataset { - // @ts-expect-error generator return tf.data.generator(async function* () { yield* iter; }); diff --git a/discojs/src/models/gpt/index.ts b/discojs/src/models/gpt/index.ts index 9ad90a7cf..6e57c49f2 100644 --- a/discojs/src/models/gpt/index.ts +++ b/discojs/src/models/gpt/index.ts @@ -103,7 +103,11 @@ export class GPT extends Model<"text"> { ): Promise> { const evaluation = await evaluate( this.model, - intoTFDataset(dataset.map((batch) => this.#batchToTF(batch))), + tf.data.generator( + async function* (this: GPT) { + yield* dataset.map((batch) => this.#batchToTF(batch)); + }.bind(this), + ), this.config.maxEvalBatches, ); @@ -191,12 +195,3 @@ export class GPT extends Model<"text"> { debug("model not disposed correctly: %o", disposeResults); } } - -function intoTFDataset( - iter: AsyncIterable, -): tf.data.Dataset { - // @ts-expect-error generator - return tf.data.generator(async function* () { - yield* iter; - }); -} diff --git a/discojs/src/models/tfjs.ts b/discojs/src/models/tfjs.ts index c34bacc39..46a2b00f5 100644 --- a/discojs/src/models/tfjs.ts +++ b/discojs/src/models/tfjs.ts @@ -90,7 +90,11 @@ export class TFJS extends Model { dataset: Dataset>, ): Promise> { const evaluation = await this.model.evaluateDataset( - intoTFDataset(dataset.map((batch) => this.#batchToTF(batch))), + tf.data.generator( + async function* (this: TFJS) { + yield* dataset.map((batch) => this.#batchToTF(batch)); + }.bind(this), + ), ); const metricToValue = Map( List(this.model.metricsNames).zip( @@ -243,12 +247,3 @@ export class TFJS extends Model { throw new Error("should never happen"); } } - -function intoTFDataset( - iter: AsyncIterable, -): tf.data.Dataset { - // @ts-expect-error generator - return tf.data.generator(async function* () { - yield* iter; - }); -} diff --git a/discojs/src/validator.ts b/discojs/src/validator.ts index 0e7648d7a..6a54f5316 100644 --- a/discojs/src/validator.ts +++ b/discojs/src/validator.ts @@ -11,7 +11,6 @@ import { datasetToData, labeledDatasetToData } from "./dataset/data/helpers.js"; function intoTFDataset( iter: AsyncIterable, ): tf.data.Dataset { - // @ts-expect-error generator return tf.data.generator(async function* () { yield* iter; }); diff --git a/package-lock.json b/package-lock.json index dd68b064d..1ce9857ae 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,7 +22,7 @@ "@types/debug": "4", "@typescript-eslint/eslint-plugin": "7", "eslint": "8", - "typescript": "5", + "typescript": "<5.6.0", "typescript-eslint": "7" } }, @@ -57,7 +57,7 @@ }, "devDependencies": { "@tensorflow/tfjs-node": "4", - "@types/chai": "4", + "@types/chai": "5", "@types/mocha": "10", "@types/simple-peer": "9", "chai": "5", @@ -95,7 +95,7 @@ "@types/papaparse": "5", "jsdom": "25", "nodemon": "3", - "vitest": "1" + "vitest": "2" } }, "isomorphic-wrtc": { @@ -118,43 +118,31 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@ampproject/remapping": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", - "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@jridgewell/gen-mapping": "^0.3.5", - "@jridgewell/trace-mapping": "^0.3.24" - }, - "engines": { - "node": ">=6.0.0" - } - }, "node_modules/@babel/helper-string-parser": { - "version": "7.24.8", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.24.8.tgz", - "integrity": "sha512-pO9KhhRcuUyGnJWwyEgnRJTSIZHiT+vMD0kPeD+so0l7mxkMT19g3pjY9GTnHySck/hDzq+dtW/4VgnMkippsQ==", + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.25.9.tgz", + "integrity": "sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==", + "license": "MIT", "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.24.7", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.24.7.tgz", - "integrity": "sha512-rR+PBcQ1SMQDDyF6X0wxtG8QyLCgUB0eRAGguqRLfkCA87l7yAP7ehq8SNj96OOGTO8OBV70KhuFYcIkHXOg0w==", + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz", + "integrity": "sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==", + "license": "MIT", "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/parser": { - "version": "7.25.4", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.25.4.tgz", - "integrity": "sha512-nq+eWrOgdtu3jG5Os4TQP3x3cLA8hR8TvJNjD8vnPa20WGycimcparWnLK4jJhElTK6SDyuJo1weMKO/5LpmLA==", + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.26.0.tgz", + "integrity": "sha512-aP8x5pIw3xvYr/sXT+SEUwyhrXT8rUJRZltK/qN3Db80dcKpTett8cJxHyjk+xYSVXvNnl2SfcJVjbwxpOSscA==", "license": "MIT", "dependencies": { - "@babel/types": "^7.25.4" + "@babel/types": "^7.26.0" }, "bin": { "parser": "bin/babel-parser.js" @@ -164,14 +152,13 @@ } }, "node_modules/@babel/types": { - "version": "7.25.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.25.4.tgz", - "integrity": "sha512-zQ1ijeeCXVEh+aNL0RlmkPkG8HUiDcU2pzQQFjtbntgAczRASFzj4H+6+bV+dy1ntKR14I/DypeuRG1uma98iQ==", + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.0.tgz", + "integrity": "sha512-Z/yiTPj+lDVnF7lWeKCIJzaIkI0vYO87dMpZ4bg4TDrFe4XXLFWL1TbXU27gBP3QccxV9mZICCrnjnYlJjXHOA==", "license": "MIT", "dependencies": { - "@babel/helper-string-parser": "^7.24.8", - "@babel/helper-validator-identifier": "^7.24.7", - "to-fast-properties": "^2.0.0" + "@babel/helper-string-parser": "^7.25.9", + "@babel/helper-validator-identifier": "^7.25.9" }, "engines": { "node": ">=6.9.0" @@ -212,9 +199,9 @@ } }, "node_modules/@cypress/request": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@cypress/request/-/request-3.0.1.tgz", - "integrity": "sha512-TWivJlJi8ZDx2wGOw1dbLuHJKUYX7bWySw377nlnGOW3hP9/MUKIsEdXT/YngWxVdgNCHRBmFlBipE+5/2ZZlQ==", + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@cypress/request/-/request-3.0.5.tgz", + "integrity": "sha512-v+XHd9XmWbufxF1/bTaVm2yhbxY+TB4YtWRqF2zaXBlDNMkls34KiATz0AVDLavL3iB6bQk9/7n3oY1EoLSWGA==", "license": "Apache-2.0", "dependencies": { "aws-sign2": "~0.7.0", @@ -223,14 +210,14 @@ "combined-stream": "~1.0.6", "extend": "~3.0.2", "forever-agent": "~0.6.1", - "form-data": "~2.3.2", - "http-signature": "~1.3.6", + "form-data": "~4.0.0", + "http-signature": "~1.4.0", "is-typedarray": "~1.0.0", "isstream": "~0.1.2", "json-stringify-safe": "~5.0.1", "mime-types": "~2.1.19", "performance-now": "^2.1.0", - "qs": "6.10.4", + "qs": "6.13.0", "safe-buffer": "^5.1.2", "tough-cookie": "^4.1.3", "tunnel-agent": "^0.6.0", @@ -240,20 +227,6 @@ "node": ">= 6" } }, - "node_modules/@cypress/request/node_modules/form-data": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", - "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 0.12" - } - }, "node_modules/@cypress/request/node_modules/uuid": { "version": "8.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", @@ -283,9 +256,10 @@ } }, "node_modules/@emnapi/runtime": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.2.0.tgz", - "integrity": "sha512-bV21/9LQmcQeCPEg3BDFtvwL6cwiTMksYNWQQ4KOxCZikEGalWtenoZ0wCiukJINlGCIi2KXx01g4FoH/LxpzQ==", + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.3.1.tgz", + "integrity": "sha512-kEBmG8KyqtxJZv+ygbEim+KCGtIq1fC22Ms3S4ziXmYKm8uyoLX0MHONVKwp+9opg390VaKRNt4a7A9NwmpNhw==", + "license": "MIT", "optional": true, "dependencies": { "tslib": "^2.4.0" @@ -711,9 +685,9 @@ } }, "node_modules/@eslint-community/regexpp": { - "version": "4.11.0", - "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.11.0.tgz", - "integrity": "sha512-G/M/tIiMrTAxEWRfLfQJMmGNX28IxBg4PBz8XqQhqUHLFI6TL2htpIB1iQCj144V5ee/JaKyT9/WZ0MGZWfA7A==", + "version": "4.11.1", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.11.1.tgz", + "integrity": "sha512-m4DVN9ZqskZoLU5GlWZadwDnYo3vAEydiUayB9widCl9ffWx2IvPnp6n3on5rJmziJSw9Bv+Z3ChDVdMwXCY8Q==", "dev": true, "license": "MIT", "engines": { @@ -769,9 +743,9 @@ } }, "node_modules/@eslint/js": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.0.tgz", - "integrity": "sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g==", + "version": "8.57.1", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.1.tgz", + "integrity": "sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q==", "dev": true, "license": "MIT", "engines": { @@ -805,14 +779,14 @@ } }, "node_modules/@humanwhocodes/config-array": { - "version": "0.11.14", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", - "integrity": "sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==", + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.13.0.tgz", + "integrity": "sha512-DZLEEqFWQFiyK6h5YIeynKx7JlvCYWL0cImfSRXZ9l4Sg2efkFGTuFf6vzXjK1cq6IYkU+Eg/JizXw+TD2vRNw==", "deprecated": "Use @eslint/config-array instead", "dev": true, "license": "Apache-2.0", "dependencies": { - "@humanwhocodes/object-schema": "^2.0.2", + "@humanwhocodes/object-schema": "^2.0.3", "debug": "^4.3.1", "minimatch": "^3.0.5" }, @@ -873,6 +847,7 @@ "cpu": [ "arm64" ], + "license": "Apache-2.0", "optional": true, "os": [ "darwin" @@ -894,6 +869,7 @@ "cpu": [ "x64" ], + "license": "Apache-2.0", "optional": true, "os": [ "darwin" @@ -915,6 +891,7 @@ "cpu": [ "arm64" ], + "license": "LGPL-3.0-or-later", "optional": true, "os": [ "darwin" @@ -930,6 +907,7 @@ "cpu": [ "x64" ], + "license": "LGPL-3.0-or-later", "optional": true, "os": [ "darwin" @@ -945,6 +923,7 @@ "cpu": [ "arm" ], + "license": "LGPL-3.0-or-later", "optional": true, "os": [ "linux" @@ -960,6 +939,7 @@ "cpu": [ "arm64" ], + "license": "LGPL-3.0-or-later", "optional": true, "os": [ "linux" @@ -975,6 +955,7 @@ "cpu": [ "s390x" ], + "license": "LGPL-3.0-or-later", "optional": true, "os": [ "linux" @@ -990,6 +971,7 @@ "cpu": [ "x64" ], + "license": "LGPL-3.0-or-later", "optional": true, "os": [ "linux" @@ -1005,6 +987,7 @@ "cpu": [ "arm64" ], + "license": "LGPL-3.0-or-later", "optional": true, "os": [ "linux" @@ -1020,6 +1003,7 @@ "cpu": [ "x64" ], + "license": "LGPL-3.0-or-later", "optional": true, "os": [ "linux" @@ -1035,6 +1019,7 @@ "cpu": [ "arm" ], + "license": "Apache-2.0", "optional": true, "os": [ "linux" @@ -1056,6 +1041,7 @@ "cpu": [ "arm64" ], + "license": "Apache-2.0", "optional": true, "os": [ "linux" @@ -1077,6 +1063,7 @@ "cpu": [ "s390x" ], + "license": "Apache-2.0", "optional": true, "os": [ "linux" @@ -1098,6 +1085,7 @@ "cpu": [ "x64" ], + "license": "Apache-2.0", "optional": true, "os": [ "linux" @@ -1119,6 +1107,7 @@ "cpu": [ "arm64" ], + "license": "Apache-2.0", "optional": true, "os": [ "linux" @@ -1140,6 +1129,7 @@ "cpu": [ "x64" ], + "license": "Apache-2.0", "optional": true, "os": [ "linux" @@ -1161,6 +1151,7 @@ "cpu": [ "wasm32" ], + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", "optional": true, "dependencies": { "@emnapi/runtime": "^1.2.0" @@ -1179,6 +1170,7 @@ "cpu": [ "ia32" ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", "optional": true, "os": [ "win32" @@ -1197,6 +1189,7 @@ "cpu": [ "x64" ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", "optional": true, "os": [ "win32" @@ -1227,9 +1220,9 @@ } }, "node_modules/@isaacs/cliui/node_modules/ansi-regex": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", - "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", + "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", "dev": true, "license": "MIT", "engines": { @@ -1311,19 +1304,6 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, - "node_modules/@jest/schemas": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz", - "integrity": "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.27.8" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, "node_modules/@jimp/core": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/@jimp/core/-/core-1.6.0.tgz", @@ -1342,18 +1322,6 @@ "node": ">=18" } }, - "node_modules/@jimp/core/node_modules/mime": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", - "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", - "license": "MIT", - "bin": { - "mime": "cli.js" - }, - "engines": { - "node": ">=10.0.0" - } - }, "node_modules/@jimp/file-ops": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/@jimp/file-ops/-/file-ops-1.6.0.tgz", @@ -1547,22 +1515,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@pinia/testing": { - "version": "0.1.5", - "resolved": "https://registry.npmjs.org/@pinia/testing/-/testing-0.1.5.tgz", - "integrity": "sha512-AcGzuotkzhRoF00htuxLfIPBBHVE6HjjB3YC5Y3os8vRgKu6ipknK5GBQq9+pduwYQhZ+BcCZDC9TyLAUlUpoQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "vue-demi": "^0.14.10" - }, - "funding": { - "url": "https://github.com/sponsors/posva" - }, - "peerDependencies": { - "pinia": ">=2.2.1" - } - }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", @@ -1685,15 +1637,15 @@ } }, "node_modules/@rollup/pluginutils": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.1.0.tgz", - "integrity": "sha512-XTIWOPPcpvyKI6L1NHo0lFlCyznUEyPmPY1mc3KpPVDYulHSTvyeLNVW00QTLIAFNhR3kYnJTQHeGqU4M3n09g==", + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.1.3.tgz", + "integrity": "sha512-Pnsb6f32CD2W3uCaLZIzDmeFyQ2b8UWMFI7xtwUezpcGBDVDW6y9XgAWIlARiGAo6eNF5FK5aQTr0LFyNyqq5A==", "dev": true, "license": "MIT", "dependencies": { "@types/estree": "^1.0.0", "estree-walker": "^2.0.2", - "picomatch": "^2.3.1" + "picomatch": "^4.0.2" }, "engines": { "node": ">=14.0.0" @@ -1707,10 +1659,23 @@ } } }, + "node_modules/@rollup/pluginutils/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.20.0.tgz", - "integrity": "sha512-TSpWzflCc4VGAUJZlPpgAJE1+V60MePDQnBd7PPkpuEmOy8i87aL6tinFGKBFKuEDikYpig72QzdT3QPYIi+oA==", + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.24.0.tgz", + "integrity": "sha512-Q6HJd7Y6xdB48x8ZNVDOqsbh2uByBhgK8PiQgPhwkIw/HC/YX5Ghq2mQY5sRMZWHb3VsFkWooUVOZHKr7DmDIA==", "cpu": [ "arm" ], @@ -1722,9 +1687,9 @@ ] }, "node_modules/@rollup/rollup-android-arm64": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.20.0.tgz", - "integrity": "sha512-u00Ro/nok7oGzVuh/FMYfNoGqxU5CPWz1mxV85S2w9LxHR8OoMQBuSk+3BKVIDYgkpeOET5yXkx90OYFc+ytpQ==", + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.24.0.tgz", + "integrity": "sha512-ijLnS1qFId8xhKjT81uBHuuJp2lU4x2yxa4ctFPtG+MqEE6+C5f/+X/bStmxapgmwLwiL3ih122xv8kVARNAZA==", "cpu": [ "arm64" ], @@ -1736,9 +1701,9 @@ ] }, "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.20.0.tgz", - "integrity": "sha512-uFVfvzvsdGtlSLuL0ZlvPJvl6ZmrH4CBwLGEFPe7hUmf7htGAN+aXo43R/V6LATyxlKVC/m6UsLb7jbG+LG39Q==", + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.24.0.tgz", + "integrity": "sha512-bIv+X9xeSs1XCk6DVvkO+S/z8/2AMt/2lMqdQbMrmVpgFvXlmde9mLcbQpztXm1tajC3raFDqegsH18HQPMYtA==", "cpu": [ "arm64" ], @@ -1750,9 +1715,9 @@ ] }, "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.20.0.tgz", - "integrity": "sha512-xbrMDdlev53vNXexEa6l0LffojxhqDTBeL+VUxuuIXys4x6xyvbKq5XqTXBCEUA8ty8iEJblHvFaWRJTk/icAQ==", + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.24.0.tgz", + "integrity": "sha512-X6/nOwoFN7RT2svEQWUsW/5C/fYMBe4fnLK9DQk4SX4mgVBiTA9h64kjUYPvGQ0F/9xwJ5U5UfTbl6BEjaQdBQ==", "cpu": [ "x64" ], @@ -1764,9 +1729,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.20.0.tgz", - "integrity": "sha512-jMYvxZwGmoHFBTbr12Xc6wOdc2xA5tF5F2q6t7Rcfab68TT0n+r7dgawD4qhPEvasDsVpQi+MgDzj2faOLsZjA==", + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.24.0.tgz", + "integrity": "sha512-0KXvIJQMOImLCVCz9uvvdPgfyWo93aHHp8ui3FrtOP57svqrF/roSSR5pjqL2hcMp0ljeGlU4q9o/rQaAQ3AYA==", "cpu": [ "arm" ], @@ -1778,9 +1743,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm-musleabihf": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.20.0.tgz", - "integrity": "sha512-1asSTl4HKuIHIB1GcdFHNNZhxAYEdqML/MW4QmPS4G0ivbEcBr1JKlFLKsIRqjSwOBkdItn3/ZDlyvZ/N6KPlw==", + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.24.0.tgz", + "integrity": "sha512-it2BW6kKFVh8xk/BnHfakEeoLPv8STIISekpoF+nBgWM4d55CZKc7T4Dx1pEbTnYm/xEKMgy1MNtYuoA8RFIWw==", "cpu": [ "arm" ], @@ -1792,9 +1757,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.20.0.tgz", - "integrity": "sha512-COBb8Bkx56KldOYJfMf6wKeYJrtJ9vEgBRAOkfw6Ens0tnmzPqvlpjZiLgkhg6cA3DGzCmLmmd319pmHvKWWlQ==", + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.24.0.tgz", + "integrity": "sha512-i0xTLXjqap2eRfulFVlSnM5dEbTVque/3Pi4g2y7cxrs7+a9De42z4XxKLYJ7+OhE3IgxvfQM7vQc43bwTgPwA==", "cpu": [ "arm64" ], @@ -1806,9 +1771,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.20.0.tgz", - "integrity": "sha512-+it+mBSyMslVQa8wSPvBx53fYuZK/oLTu5RJoXogjk6x7Q7sz1GNRsXWjn6SwyJm8E/oMjNVwPhmNdIjwP135Q==", + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.24.0.tgz", + "integrity": "sha512-9E6MKUJhDuDh604Qco5yP/3qn3y7SLXYuiC0Rpr89aMScS2UAmK1wHP2b7KAa1nSjWJc/f/Lc0Wl1L47qjiyQw==", "cpu": [ "arm64" ], @@ -1820,9 +1785,9 @@ ] }, "node_modules/@rollup/rollup-linux-powerpc64le-gnu": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.20.0.tgz", - "integrity": "sha512-yAMvqhPfGKsAxHN8I4+jE0CpLWD8cv4z7CK7BMmhjDuz606Q2tFKkWRY8bHR9JQXYcoLfopo5TTqzxgPUjUMfw==", + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.24.0.tgz", + "integrity": "sha512-2XFFPJ2XMEiF5Zi2EBf4h73oR1V/lycirxZxHZNc93SqDN/IWhYYSYj8I9381ikUFXZrz2v7r2tOVk2NBwxrWw==", "cpu": [ "ppc64" ], @@ -1834,9 +1799,9 @@ ] }, "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.20.0.tgz", - "integrity": "sha512-qmuxFpfmi/2SUkAw95TtNq/w/I7Gpjurx609OOOV7U4vhvUhBcftcmXwl3rqAek+ADBwSjIC4IVNLiszoj3dPA==", + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.24.0.tgz", + "integrity": "sha512-M3Dg4hlwuntUCdzU7KjYqbbd+BLq3JMAOhCKdBE3TcMGMZbKkDdJ5ivNdehOssMCIokNHFOsv7DO4rlEOfyKpg==", "cpu": [ "riscv64" ], @@ -1848,9 +1813,9 @@ ] }, "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.20.0.tgz", - "integrity": "sha512-I0BtGXddHSHjV1mqTNkgUZLnS3WtsqebAXv11D5BZE/gfw5KoyXSAXVqyJximQXNvNzUo4GKlCK/dIwXlz+jlg==", + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.24.0.tgz", + "integrity": "sha512-mjBaoo4ocxJppTorZVKWFpy1bfFj9FeCMJqzlMQGjpNPY9JwQi7OuS1axzNIk0nMX6jSgy6ZURDZ2w0QW6D56g==", "cpu": [ "s390x" ], @@ -1862,9 +1827,9 @@ ] }, "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.20.0.tgz", - "integrity": "sha512-y+eoL2I3iphUg9tN9GB6ku1FA8kOfmF4oUEWhztDJ4KXJy1agk/9+pejOuZkNFhRwHAOxMsBPLbXPd6mJiCwew==", + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.24.0.tgz", + "integrity": "sha512-ZXFk7M72R0YYFN5q13niV0B7G8/5dcQ9JDp8keJSfr3GoZeXEoMHP/HlvqROA3OMbMdfr19IjCeNAnPUG93b6A==", "cpu": [ "x64" ], @@ -1876,9 +1841,9 @@ ] }, "node_modules/@rollup/rollup-linux-x64-musl": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.20.0.tgz", - "integrity": "sha512-hM3nhW40kBNYUkZb/r9k2FKK+/MnKglX7UYd4ZUy5DJs8/sMsIbqWK2piZtVGE3kcXVNj3B2IrUYROJMMCikNg==", + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.24.0.tgz", + "integrity": "sha512-w1i+L7kAXZNdYl+vFvzSZy8Y1arS7vMgIy8wusXJzRrPyof5LAb02KGr1PD2EkRcl73kHulIID0M501lN+vobQ==", "cpu": [ "x64" ], @@ -1890,9 +1855,9 @@ ] }, "node_modules/@rollup/rollup-win32-arm64-msvc": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.20.0.tgz", - "integrity": "sha512-psegMvP+Ik/Bg7QRJbv8w8PAytPA7Uo8fpFjXyCRHWm6Nt42L+JtoqH8eDQ5hRP7/XW2UiIriy1Z46jf0Oa1kA==", + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.24.0.tgz", + "integrity": "sha512-VXBrnPWgBpVDCVY6XF3LEW0pOU51KbaHhccHw6AS6vBWIC60eqsH19DAeeObl+g8nKAz04QFdl/Cefta0xQtUQ==", "cpu": [ "arm64" ], @@ -1904,9 +1869,9 @@ ] }, "node_modules/@rollup/rollup-win32-ia32-msvc": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.20.0.tgz", - "integrity": "sha512-GabekH3w4lgAJpVxkk7hUzUf2hICSQO0a/BLFA11/RMxQT92MabKAqyubzDZmMOC/hcJNlc+rrypzNzYl4Dx7A==", + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.24.0.tgz", + "integrity": "sha512-xrNcGDU0OxVcPTH/8n/ShH4UevZxKIO6HJFK0e15XItZP2UcaiLFd5kiX7hJnqCbSztUF8Qot+JWBC/QXRPYWQ==", "cpu": [ "ia32" ], @@ -1918,9 +1883,9 @@ ] }, "node_modules/@rollup/rollup-win32-x64-msvc": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.20.0.tgz", - "integrity": "sha512-aJ1EJSuTdGnM6qbVC4B5DSmozPTqIag9fSzXRNNo+humQLG89XpPgdt16Ia56ORD7s+H8Pmyx44uczDQ0yDzpg==", + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.24.0.tgz", + "integrity": "sha512-fbMkAF7fufku0N2dE5TBXcNlg0pt0cJue4xBRE2Qc5Vqikxr4VCgKj/ht6SMdFcOacVA9rqF70APJ8RN/4vMJw==", "cpu": [ "x64" ], @@ -1955,25 +1920,18 @@ "dev": true, "license": "BSD-3-Clause" }, - "node_modules/@sinclair/typebox": { - "version": "0.27.8", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", - "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==", - "dev": true, - "license": "MIT" - }, "node_modules/@tensorflow/tfjs": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs/-/tfjs-4.20.0.tgz", - "integrity": "sha512-+ZLfJq2jyIOE2/+yKPoyD/gfy3RZypbfMrlzvBDgodTK5jnexprihhX38hxilh9HPWvWQXJqiUjKJP5ECCikrw==", + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs/-/tfjs-4.22.0.tgz", + "integrity": "sha512-0TrIrXs6/b7FLhLVNmfh8Sah6JgjBPH4mZ8JGb7NU6WW+cx00qK5BcAZxw7NCzxj6N8MRAIfHq+oNbPUNG5VAg==", "license": "Apache-2.0", "dependencies": { - "@tensorflow/tfjs-backend-cpu": "4.20.0", - "@tensorflow/tfjs-backend-webgl": "4.20.0", - "@tensorflow/tfjs-converter": "4.20.0", - "@tensorflow/tfjs-core": "4.20.0", - "@tensorflow/tfjs-data": "4.20.0", - "@tensorflow/tfjs-layers": "4.20.0", + "@tensorflow/tfjs-backend-cpu": "4.22.0", + "@tensorflow/tfjs-backend-webgl": "4.22.0", + "@tensorflow/tfjs-converter": "4.22.0", + "@tensorflow/tfjs-core": "4.22.0", + "@tensorflow/tfjs-data": "4.22.0", + "@tensorflow/tfjs-layers": "4.22.0", "argparse": "^1.0.10", "chalk": "^4.1.0", "core-js": "3.29.1", @@ -1985,9 +1943,9 @@ } }, "node_modules/@tensorflow/tfjs-backend-cpu": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-4.20.0.tgz", - "integrity": "sha512-1QRQ6AqAa/VB8JOArf5nY3Dc/QQHXbfuxgdIdQhKrABEHgvlaWt2Vv696UhIlVl75YoNY+vWlCwBdGQIKYfFGw==", + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-4.22.0.tgz", + "integrity": "sha512-1u0FmuLGuRAi8D2c3cocHTASGXOmHc/4OvoVDENJayjYkS119fcTcQf4iHrtLthWyDIPy3JiPhRrZQC9EwnhLw==", "license": "Apache-2.0", "dependencies": { "@types/seedrandom": "^2.4.28", @@ -1997,16 +1955,16 @@ "yarn": ">= 1.3.2" }, "peerDependencies": { - "@tensorflow/tfjs-core": "4.20.0" + "@tensorflow/tfjs-core": "4.22.0" } }, "node_modules/@tensorflow/tfjs-backend-webgl": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-4.20.0.tgz", - "integrity": "sha512-M03fJonJGxm2u3SCzRNA2JLh0gxaAye64SEmGAXOehizowxy42l+lMsPWU8xU7r7mN6PEilBNkuKAf5YJ7Xumg==", + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-4.22.0.tgz", + "integrity": "sha512-H535XtZWnWgNwSzv538czjVlbJebDl5QTMOth4RXr2p/kJ1qSIXE0vZvEtO+5EC9b00SvhplECny2yDewQb/Yg==", "license": "Apache-2.0", "dependencies": { - "@tensorflow/tfjs-backend-cpu": "4.20.0", + "@tensorflow/tfjs-backend-cpu": "4.22.0", "@types/offscreencanvas": "~2019.3.0", "@types/seedrandom": "^2.4.28", "seedrandom": "^3.0.5" @@ -2015,22 +1973,22 @@ "yarn": ">= 1.3.2" }, "peerDependencies": { - "@tensorflow/tfjs-core": "4.20.0" + "@tensorflow/tfjs-core": "4.22.0" } }, "node_modules/@tensorflow/tfjs-converter": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-4.20.0.tgz", - "integrity": "sha512-UJ2ntQ1TNtVHB5qGMwB0j306bs3KH1E1HKJ9Dxvrc6PUaivOV+CPKqmbidOFG5LylXeRC36JBdhe+gVT2nFHNw==", + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-4.22.0.tgz", + "integrity": "sha512-PT43MGlnzIo+YfbsjM79Lxk9lOq6uUwZuCc8rrp0hfpLjF6Jv8jS84u2jFb+WpUeuF4K33ZDNx8CjiYrGQ2trQ==", "license": "Apache-2.0", "peerDependencies": { - "@tensorflow/tfjs-core": "4.20.0" + "@tensorflow/tfjs-core": "4.22.0" } }, "node_modules/@tensorflow/tfjs-core": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-4.20.0.tgz", - "integrity": "sha512-m/cc9qDc63al9UhdbXRUYTLGfJJlhuN5tylAX/2pJMLj32c8a6ThGDJYoKzpf32n5g3MQGYLchjClDxeGdXMPQ==", + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-4.22.0.tgz", + "integrity": "sha512-LEkOyzbknKFoWUwfkr59vSB68DMJ4cjwwHgicXN0DUi3a0Vh1Er3JQqCI1Hl86GGZQvY8ezVrtDIvqR1ZFW55A==", "license": "Apache-2.0", "dependencies": { "@types/long": "^4.0.1", @@ -2094,9 +2052,9 @@ } }, "node_modules/@tensorflow/tfjs-data": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-data/-/tfjs-data-4.20.0.tgz", - "integrity": "sha512-k6S8joXhoXkatcoT6mYCxBzRCsnrLfnl6xjLe46SnXO0oEEy4Vuzbmp5Ydl1uU2hHr73zL91EdAC1k8Hng/+oA==", + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-data/-/tfjs-data-4.22.0.tgz", + "integrity": "sha512-dYmF3LihQIGvtgJrt382hSRH4S0QuAp2w1hXJI2+kOaEqo5HnUPG0k5KA6va+S1yUhx7UBToUKCBHeLHFQRV4w==", "license": "Apache-2.0", "dependencies": { "@types/node-fetch": "^2.1.2", @@ -2104,7 +2062,7 @@ "string_decoder": "^1.3.0" }, "peerDependencies": { - "@tensorflow/tfjs-core": "4.20.0", + "@tensorflow/tfjs-core": "4.22.0", "seedrandom": "^3.0.5" } }, @@ -2151,23 +2109,23 @@ } }, "node_modules/@tensorflow/tfjs-layers": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-layers/-/tfjs-layers-4.20.0.tgz", - "integrity": "sha512-SCHZH29Vyw+Y9eoaJHiaNo6yqM9vD3XCKncoczonRRywejm3FFqddg1AuWAfSE9XoNPE21o9PsknvKLl/Uh+Cg==", + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-layers/-/tfjs-layers-4.22.0.tgz", + "integrity": "sha512-lybPj4ZNj9iIAPUj7a8ZW1hg8KQGfqWLlCZDi9eM/oNKCCAgchiyzx8OrYoWmRrB+AM6VNEeIT+2gZKg5ReihA==", "license": "Apache-2.0 AND MIT", "peerDependencies": { - "@tensorflow/tfjs-core": "4.20.0" + "@tensorflow/tfjs-core": "4.22.0" } }, "node_modules/@tensorflow/tfjs-node": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-node/-/tfjs-node-4.20.0.tgz", - "integrity": "sha512-pVSOlzsVqh5ck3aiNPJCltB3ASKjsLqNPvJ28lXn9Xg648U4eHDk8G47m9w4uf0FdVcWDfjPM3hDCbBZ/E2KXg==", + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-node/-/tfjs-node-4.22.0.tgz", + "integrity": "sha512-uHrXeUlfgkMxTZqHkESSV7zSdKdV0LlsBeblqkuKU9nnfxB1pC6DtoyYVaLxznzZy7WQSegjcohxxCjAf6Dc7w==", "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { "@mapbox/node-pre-gyp": "1.0.9", - "@tensorflow/tfjs": "4.20.0", + "@tensorflow/tfjs": "4.22.0", "adm-zip": "^0.5.2", "google-protobuf": "^3.9.2", "https-proxy-agent": "^2.2.1", @@ -2337,9 +2295,9 @@ } }, "node_modules/@types/chai": { - "version": "4.3.17", - "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.3.17.tgz", - "integrity": "sha512-zmZ21EWzR71B4Sscphjief5djsLre50M6lI622OSySTmn9DB3j+C3kWroHfBQWXbOBwbgg/M8CG/hUxDLIloow==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.0.0.tgz", + "integrity": "sha512-+DwhEHAaFPPdJ2ral3kNHFQXnTfscEEFsUxzD+d7nlcLrFK23JtNjH71RGasTcHb88b4vVi4mTyfpf8u2L8bdA==", "dev": true, "license": "MIT" }, @@ -2589,9 +2547,9 @@ "license": "MIT" }, "node_modules/@types/d3-selection": { - "version": "3.0.10", - "resolved": "https://registry.npmjs.org/@types/d3-selection/-/d3-selection-3.0.10.tgz", - "integrity": "sha512-cuHoUgS/V3hLdjJOLTT691+G2QoqAjCVLmr4kJXR4ha56w1Zdu8UUQ5TxLRqudgNjwXeQxKMq4j+lyf9sWuslg==", + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@types/d3-selection/-/d3-selection-3.0.11.tgz", + "integrity": "sha512-bhAXu23DJWsrI45xafYpkQ4NtcKMwWnAC/vKrd2l+nxMFuvOT3XMYTIj2opv8vq8AO5Yh7Qac/nSeP/3zjTK0w==", "dev": true, "license": "MIT" }, @@ -2627,9 +2585,9 @@ "license": "MIT" }, "node_modules/@types/d3-transition": { - "version": "3.0.8", - "resolved": "https://registry.npmjs.org/@types/d3-transition/-/d3-transition-3.0.8.tgz", - "integrity": "sha512-ew63aJfQ/ms7QQ4X7pk5NxQ9fZH/z+i24ZfJ6tJSfqxJMrYLiK01EAs2/Rtw/JreGUsS3pLPNV644qXFGnoZNQ==", + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/@types/d3-transition/-/d3-transition-3.0.9.tgz", + "integrity": "sha512-uZS5shfxzO3rGlu0cC3bjmMFKsXv+SmZZcgp0KD22ts4uGXp5EVYGzu/0YdwZeKmddhcAccYtREJKkPfXkZuCg==", "dev": true, "license": "MIT", "dependencies": { @@ -2658,29 +2616,29 @@ } }, "node_modules/@types/estree": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz", - "integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==", + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz", + "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==", "dev": true, "license": "MIT" }, "node_modules/@types/express": { - "version": "4.17.21", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.21.tgz", - "integrity": "sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@types/express/-/express-5.0.0.tgz", + "integrity": "sha512-DvZriSMehGHL1ZNLzi6MidnsDhUZM/x2pRdDIKdwbUNqqwHxMlRdkxtn6/EPKyqKpHqTl/4nRZsRNLpZxZRpPQ==", "dev": true, "license": "MIT", "dependencies": { "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.33", + "@types/express-serve-static-core": "^5.0.0", "@types/qs": "*", "@types/serve-static": "*" } }, "node_modules/@types/express-serve-static-core": { - "version": "4.19.5", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.5.tgz", - "integrity": "sha512-y6W03tvrACO72aijJ5uF02FRq5cgDR9lUxddQ8vyF+GvmjJQqbzDcJngEjURc+ZsG31VI3hODNZJ2URj86pzmg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-5.0.1.tgz", + "integrity": "sha512-CRICJIl0N5cXDONAdlTv5ShATZ4HEwk6kDDIW2/w9qOWKg+NU/5F8wYRWCrONad0/UKkloNSmmyN/wX4rtpbVA==", "dev": true, "license": "MIT", "dependencies": { @@ -2691,9 +2649,9 @@ } }, "node_modules/@types/express-ws": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@types/express-ws/-/express-ws-3.0.4.tgz", - "integrity": "sha512-Yjj18CaivG5KndgcvzttWe8mPFinPCHJC2wvyQqVzA7hqeufM8EtWMj6mpp5omg3s8XALUexhOu8aXAyi/DyJQ==", + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@types/express-ws/-/express-ws-3.0.5.tgz", + "integrity": "sha512-lbWMjoHrm/v85j81UCmb/GNZFO3genxRYBW1Ob7rjRI+zxUBR+4tcFuOpKKsYQ1LYTYiy3356epLeYi/5zxUwA==", "dev": true, "license": "MIT", "dependencies": { @@ -2742,9 +2700,9 @@ "license": "MIT" }, "node_modules/@types/mocha": { - "version": "10.0.7", - "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-10.0.7.tgz", - "integrity": "sha512-GN8yJ1mNTcFcah/wKEFIJckJx9iJLoMSzWcfRRuxz/Jk+U6KQNnml+etbtxFK8lPjzOw3zp4Ha/kjSst9fsHYw==", + "version": "10.0.9", + "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-10.0.9.tgz", + "integrity": "sha512-sicdRoWtYevwxjOHNMPTl3vSfJM6oyW8o1wXeI7uww6b6xHg8eBznQDNSGBCDJmsE8UMxP05JgZRtsKbTqt//Q==", "dev": true, "license": "MIT" }, @@ -2756,11 +2714,12 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "22.5.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.5.0.tgz", - "integrity": "sha512-DkFrJOe+rfdHTqqMg0bSNlGlQ85hSoh2TPzZyhHsXnMtligRWpxUySiyw8FY14ITt24HVCiQPWxS3KO/QlGmWg==", + "version": "22.8.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.8.0.tgz", + "integrity": "sha512-84rafSBHC/z1i1E3p0cJwKA+CfYDNSXX9WSZBRopjIzLET8oNt6ht2tei4C7izwDeEiLLfdeSVBv1egOH916hg==", + "license": "MIT", "dependencies": { - "undici-types": "~6.19.2" + "undici-types": "~6.19.8" } }, "node_modules/@types/node-fetch": { @@ -2780,9 +2739,9 @@ "license": "MIT" }, "node_modules/@types/papaparse": { - "version": "5.3.14", - "resolved": "https://registry.npmjs.org/@types/papaparse/-/papaparse-5.3.14.tgz", - "integrity": "sha512-LxJ4iEFcpqc6METwp9f6BV6VVc43m6MfH0VqFosHvrUgfXiFe6ww7R3itkOQ+TCK6Y+Iv/+RnnvtRZnkc5Kc9g==", + "version": "5.3.15", + "resolved": "https://registry.npmjs.org/@types/papaparse/-/papaparse-5.3.15.tgz", + "integrity": "sha512-JHe6vF6x/8Z85nCX4yFdDslN11d+1pr12E526X8WAfhadOeaOTx5AuIkvDKIBopfvlzpzkdMx4YyvSKCM9oqtw==", "dev": true, "license": "MIT", "dependencies": { @@ -2790,9 +2749,9 @@ } }, "node_modules/@types/qs": { - "version": "6.9.15", - "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.15.tgz", - "integrity": "sha512-uXHQKES6DQKKCLh441Xv/dwxOq1TVS3JPUMlEqoEglvlhR6Mxnlew/Xq/LRVHpLyk7iK3zODe1qYHIMltO7XGg==", + "version": "6.9.16", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.16.tgz", + "integrity": "sha512-7i+zxXdPD0T4cKDuxCUXJ4wHcsJLwENa6Z3dCu8cfCK743OGy5Nu1RmAGqDPsoTDINVEcdXKRvR/zre+P2Ku1A==", "dev": true, "license": "MIT" }, @@ -2849,9 +2808,9 @@ "license": "MIT" }, "node_modules/@types/sizzle": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/@types/sizzle/-/sizzle-2.3.8.tgz", - "integrity": "sha512-0vWLNK2D5MT9dg0iOo8GlKguPAU02QjmZitPEsXRuJXU/OGIOt9vT9Fc26wtYuavLxtO45v9PGleoL9Z0k1LHg==", + "version": "2.3.9", + "resolved": "https://registry.npmjs.org/@types/sizzle/-/sizzle-2.3.9.tgz", + "integrity": "sha512-xzLEyKB50yqCUPUJkIsrVvoWNfFUbIZI+RspLWt8u+tIW/BetMBZtgV2LY/2o+tYH8dRvQ+eoPf3NdhQCcLE2w==", "license": "MIT" }, "node_modules/@types/tough-cookie": { @@ -3093,6 +3052,7 @@ "resolved": "https://registry.npmjs.org/@vitejs/plugin-vue/-/plugin-vue-5.1.4.tgz", "integrity": "sha512-N2XSI2n3sQqp5w7Y/AN/L2XDjBIRGqXko+eDp42sydYSBeJuSm5a1sLf8zakmo8u7tA8NmBgoDLA1HeOESjp9A==", "dev": true, + "license": "MIT", "engines": { "node": "^18.0.0 || >=20.0.0" }, @@ -3102,99 +3062,63 @@ } }, "node_modules/@vitest/expect": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-1.6.0.tgz", - "integrity": "sha512-ixEvFVQjycy/oNgHjqsL6AZCDduC+tflRluaHIzKIsdbzkLn2U/iBnVeJwB6HsIjQBdfMR8Z0tRxKUsvFJEeWQ==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-2.1.3.tgz", + "integrity": "sha512-SNBoPubeCJhZ48agjXruCI57DvxcsivVDdWz+SSsmjTT4QN/DfHk3zB/xKsJqMs26bLZ/pNRLnCf0j679i0uWQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "1.6.0", - "@vitest/utils": "1.6.0", - "chai": "^4.3.10" + "@vitest/spy": "2.1.3", + "@vitest/utils": "2.1.3", + "chai": "^5.1.1", + "tinyrainbow": "^1.2.0" }, "funding": { "url": "https://opencollective.com/vitest" } }, - "node_modules/@vitest/expect/node_modules/assertion-error": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz", - "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==", - "dev": true, - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/@vitest/expect/node_modules/chai": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/chai/-/chai-4.5.0.tgz", - "integrity": "sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw==", + "node_modules/@vitest/mocker": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-2.1.3.tgz", + "integrity": "sha512-eSpdY/eJDuOvuTA3ASzCjdithHa+GIF1L4PqtEELl6Qa3XafdMLBpBlZCIUCX2J+Q6sNmjmxtosAG62fK4BlqQ==", "dev": true, "license": "MIT", "dependencies": { - "assertion-error": "^1.1.0", - "check-error": "^1.0.3", - "deep-eql": "^4.1.3", - "get-func-name": "^2.0.2", - "loupe": "^2.3.6", - "pathval": "^1.1.1", - "type-detect": "^4.1.0" + "@vitest/spy": "2.1.3", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.11" }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@vitest/expect/node_modules/check-error": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.3.tgz", - "integrity": "sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg==", - "dev": true, - "license": "MIT", - "dependencies": { - "get-func-name": "^2.0.2" + "funding": { + "url": "https://opencollective.com/vitest" }, - "engines": { - "node": "*" - } - }, - "node_modules/@vitest/expect/node_modules/deep-eql": { - "version": "4.1.4", - "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-4.1.4.tgz", - "integrity": "sha512-SUwdGfqdKOwxCPeVYjwSyRpJ7Z+fhpwIAtmCUdZIWZ/YP5R9WAsyuSgpLVDi9bjWoN2LXHNss/dk3urXtdQxGg==", - "dev": true, - "license": "MIT", - "dependencies": { - "type-detect": "^4.0.0" + "peerDependencies": { + "@vitest/spy": "2.1.3", + "msw": "^2.3.5", + "vite": "^5.0.0" }, - "engines": { - "node": ">=6" + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } } }, - "node_modules/@vitest/expect/node_modules/loupe": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/loupe/-/loupe-2.3.7.tgz", - "integrity": "sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA==", + "node_modules/@vitest/mocker/node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", "dev": true, "license": "MIT", "dependencies": { - "get-func-name": "^2.0.1" - } - }, - "node_modules/@vitest/expect/node_modules/pathval": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.1.tgz", - "integrity": "sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "*" + "@types/estree": "^1.0.0" } }, "node_modules/@vitest/pretty-format": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-2.0.5.tgz", - "integrity": "sha512-h8k+1oWHfwTkyTkb9egzwNMfJAEx4veaPSnMeKbVSjp4euqGSbQlm5+6VHwTr7u4FJslVVsUG5nopCaAYdOmSQ==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-2.1.3.tgz", + "integrity": "sha512-XH1XdtoLZCpqV59KRbPrIhFCOO0hErxrQCMcvnQete3Vibb9UeIOX02uFPfVn3Z9ZXsq78etlfyhnkmIZSzIwQ==", "dev": true, "license": "MIT", "dependencies": { @@ -3205,194 +3129,139 @@ } }, "node_modules/@vitest/runner": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-1.6.0.tgz", - "integrity": "sha512-P4xgwPjwesuBiHisAVz/LSSZtDjOTPYZVmNAnpHHSR6ONrf8eCJOFRvUwdHn30F5M1fxhqtl7QZQUk2dprIXAg==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-2.1.3.tgz", + "integrity": "sha512-JGzpWqmFJ4fq5ZKHtVO3Xuy1iF2rHGV4d/pdzgkYHm1+gOzNZtqjvyiaDGJytRyMU54qkxpNzCx+PErzJ1/JqQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "1.6.0", - "p-limit": "^5.0.0", - "pathe": "^1.1.1" + "@vitest/utils": "2.1.3", + "pathe": "^1.1.2" }, "funding": { "url": "https://opencollective.com/vitest" } }, - "node_modules/@vitest/runner/node_modules/p-limit": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-5.0.0.tgz", - "integrity": "sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "yocto-queue": "^1.0.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@vitest/runner/node_modules/yocto-queue": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.1.1.tgz", - "integrity": "sha512-b4JR1PFR10y1mKjhHY9LaGo6tmrgjit7hxVIeAmyMw3jegXR4dhYqLaQF5zMXZxY7tLpMyJeLjr1C4rLmkVe8g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12.20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/@vitest/snapshot": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-1.6.0.tgz", - "integrity": "sha512-+Hx43f8Chus+DCmygqqfetcAZrDJwvTj0ymqjQq4CvmpKFSTVteEOBzCusu1x2tt4OJcvBflyHUE0DZSLgEMtQ==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-2.1.3.tgz", + "integrity": "sha512-qWC2mWc7VAXmjAkEKxrScWHWFyCQx/cmiZtuGqMi+WwqQJ2iURsVY4ZfAK6dVo6K2smKRU6l3BPwqEBvhnpQGg==", "dev": true, "license": "MIT", "dependencies": { - "magic-string": "^0.30.5", - "pathe": "^1.1.1", - "pretty-format": "^29.7.0" + "@vitest/pretty-format": "2.1.3", + "magic-string": "^0.30.11", + "pathe": "^1.1.2" }, "funding": { "url": "https://opencollective.com/vitest" } }, "node_modules/@vitest/spy": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-1.6.0.tgz", - "integrity": "sha512-leUTap6B/cqi/bQkXUu6bQV5TZPx7pmMBKBQiI0rJA8c3pB56ZsaTbREnF7CJfmvAS4V2cXIBAh/3rVwrrCYgw==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-2.1.3.tgz", + "integrity": "sha512-Nb2UzbcUswzeSP7JksMDaqsI43Sj5+Kry6ry6jQJT4b5gAK+NS9NED6mDb8FlMRCX8m5guaHCDZmqYMMWRy5nQ==", "dev": true, "license": "MIT", "dependencies": { - "tinyspy": "^2.2.0" + "tinyspy": "^3.0.0" }, "funding": { "url": "https://opencollective.com/vitest" } }, "node_modules/@vitest/utils": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-1.6.0.tgz", - "integrity": "sha512-21cPiuGMoMZwiOHa2i4LXkMkMkCGzA+MVFV70jRwHo95dL4x/ts5GZhML1QWuy7yfp3WzK3lRvZi3JnXTYqrBw==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-2.1.3.tgz", + "integrity": "sha512-xpiVfDSg1RrYT0tX6czgerkpcKFmFOF/gCr30+Mve5V2kewCy4Prn1/NDMSRwaSmT7PRaOF83wu+bEtsY1wrvA==", "dev": true, "license": "MIT", "dependencies": { - "diff-sequences": "^29.6.3", - "estree-walker": "^3.0.3", - "loupe": "^2.3.7", - "pretty-format": "^29.7.0" + "@vitest/pretty-format": "2.1.3", + "loupe": "^3.1.1", + "tinyrainbow": "^1.2.0" }, "funding": { "url": "https://opencollective.com/vitest" } }, - "node_modules/@vitest/utils/node_modules/estree-walker": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", - "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/estree": "^1.0.0" - } - }, - "node_modules/@vitest/utils/node_modules/loupe": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/loupe/-/loupe-2.3.7.tgz", - "integrity": "sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA==", - "dev": true, - "license": "MIT", - "dependencies": { - "get-func-name": "^2.0.1" - } - }, "node_modules/@volar/language-core": { - "version": "2.4.4", - "resolved": "https://registry.npmjs.org/@volar/language-core/-/language-core-2.4.4.tgz", - "integrity": "sha512-kO9k4kTLfxpg+6lq7/KAIv3m2d62IHuCL6GbVgYZTpfKvIGoAIlDxK7pFcB/eczN2+ydg/vnyaeZ6SGyZrJw2w==", + "version": "2.4.7", + "resolved": "https://registry.npmjs.org/@volar/language-core/-/language-core-2.4.7.tgz", + "integrity": "sha512-G/EB0vkArVB04F8DVBf30AlRK/QAOx63CzsuKKuda2ZIJamQlv4t6gEJrFVmYF560kbslFtaAJcmn8cyg7QmLA==", "dev": true, + "license": "MIT", "dependencies": { - "@volar/source-map": "2.4.4" + "@volar/source-map": "2.4.7" } }, "node_modules/@volar/source-map": { - "version": "2.4.4", - "resolved": "https://registry.npmjs.org/@volar/source-map/-/source-map-2.4.4.tgz", - "integrity": "sha512-xG3PZqOP2haG8XG4Pg3PD1UGDAdqZg24Ru8c/qYjYAnmcj6GBR64mstx+bZux5QOyRaJK+/lNM/RnpvBD3489g==", - "dev": true + "version": "2.4.7", + "resolved": "https://registry.npmjs.org/@volar/source-map/-/source-map-2.4.7.tgz", + "integrity": "sha512-c+7IJrD4mht1s8FLlCf6dAUC1aTUY9leKeLosfUiuMxavcG/sY3IPBiD1rdLL5qrhzYVmUWRGxhWvJeyYa/bsQ==", + "dev": true, + "license": "MIT" }, "node_modules/@volar/typescript": { - "version": "2.4.4", - "resolved": "https://registry.npmjs.org/@volar/typescript/-/typescript-2.4.4.tgz", - "integrity": "sha512-QQMQRVj0fVHJ3XdRKiS1LclhG0VBXdFYlyuHRQF/xLk2PuJuHNWP26MDZNvEVCvnyUQuUQhIAfylwY5TGPgc6w==", + "version": "2.4.7", + "resolved": "https://registry.npmjs.org/@volar/typescript/-/typescript-2.4.7.tgz", + "integrity": "sha512-sp3mFLmMtXY47S8GrMwFnwjGiW7aVtCLMAwnePRJA4P7CfSkrRj2DjoSxl//0pt+KR7oGG/48T2q413b8TvPbg==", "dev": true, + "license": "MIT", "dependencies": { - "@volar/language-core": "2.4.4", + "@volar/language-core": "2.4.7", "path-browserify": "^1.0.1", "vscode-uri": "^3.0.8" } }, "node_modules/@vue/compiler-core": { - "version": "3.4.37", - "resolved": "https://registry.npmjs.org/@vue/compiler-core/-/compiler-core-3.4.37.tgz", - "integrity": "sha512-ZDDT/KiLKuCRXyzWecNzC5vTcubGz4LECAtfGPENpo0nrmqJHwuWtRLxk/Sb9RAKtR9iFflFycbkjkY+W/PZUQ==", + "version": "3.5.12", + "resolved": "https://registry.npmjs.org/@vue/compiler-core/-/compiler-core-3.5.12.tgz", + "integrity": "sha512-ISyBTRMmMYagUxhcpyEH0hpXRd/KqDU4ymofPgl2XAkY9ZhQ+h0ovEZJIiPop13UmR/54oA2cgMDjgroRelaEw==", + "license": "MIT", "dependencies": { - "@babel/parser": "^7.24.7", - "@vue/shared": "3.4.37", - "entities": "^5.0.0", + "@babel/parser": "^7.25.3", + "@vue/shared": "3.5.12", + "entities": "^4.5.0", "estree-walker": "^2.0.2", "source-map-js": "^1.2.0" } }, - "node_modules/@vue/compiler-core/node_modules/entities": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-5.0.0.tgz", - "integrity": "sha512-BeJFvFRJddxobhvEdm5GqHzRV/X+ACeuw0/BuuxsCh1EUZcAIz8+kYmBp/LrQuloy6K1f3a0M7+IhmZ7QnkISA==", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, "node_modules/@vue/compiler-dom": { - "version": "3.4.37", - "resolved": "https://registry.npmjs.org/@vue/compiler-dom/-/compiler-dom-3.4.37.tgz", - "integrity": "sha512-rIiSmL3YrntvgYV84rekAtU/xfogMUJIclUMeIKEtVBFngOL3IeZHhsH3UaFEgB5iFGpj6IW+8YuM/2Up+vVag==", + "version": "3.5.12", + "resolved": "https://registry.npmjs.org/@vue/compiler-dom/-/compiler-dom-3.5.12.tgz", + "integrity": "sha512-9G6PbJ03uwxLHKQ3P42cMTi85lDRvGLB2rSGOiQqtXELat6uI4n8cNz9yjfVHRPIu+MsK6TE418Giruvgptckg==", + "license": "MIT", "dependencies": { - "@vue/compiler-core": "3.4.37", - "@vue/shared": "3.4.37" + "@vue/compiler-core": "3.5.12", + "@vue/shared": "3.5.12" } }, "node_modules/@vue/compiler-sfc": { - "version": "3.4.37", - "resolved": "https://registry.npmjs.org/@vue/compiler-sfc/-/compiler-sfc-3.4.37.tgz", - "integrity": "sha512-vCfetdas40Wk9aK/WWf8XcVESffsbNkBQwS5t13Y/PcfqKfIwJX2gF+82th6dOpnpbptNMlMjAny80li7TaCIg==", - "dependencies": { - "@babel/parser": "^7.24.7", - "@vue/compiler-core": "3.4.37", - "@vue/compiler-dom": "3.4.37", - "@vue/compiler-ssr": "3.4.37", - "@vue/shared": "3.4.37", + "version": "3.5.12", + "resolved": "https://registry.npmjs.org/@vue/compiler-sfc/-/compiler-sfc-3.5.12.tgz", + "integrity": "sha512-2k973OGo2JuAa5+ZlekuQJtitI5CgLMOwgl94BzMCsKZCX/xiqzJYzapl4opFogKHqwJk34vfsaKpfEhd1k5nw==", + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.25.3", + "@vue/compiler-core": "3.5.12", + "@vue/compiler-dom": "3.5.12", + "@vue/compiler-ssr": "3.5.12", + "@vue/shared": "3.5.12", "estree-walker": "^2.0.2", - "magic-string": "^0.30.10", - "postcss": "^8.4.40", + "magic-string": "^0.30.11", + "postcss": "^8.4.47", "source-map-js": "^1.2.0" } }, "node_modules/@vue/compiler-ssr": { - "version": "3.4.37", - "resolved": "https://registry.npmjs.org/@vue/compiler-ssr/-/compiler-ssr-3.4.37.tgz", - "integrity": "sha512-TyAgYBWrHlFrt4qpdACh8e9Ms6C/AZQ6A6xLJaWrCL8GCX5DxMzxyeFAEMfU/VFr4tylHm+a2NpfJpcd7+20XA==", + "version": "3.5.12", + "resolved": "https://registry.npmjs.org/@vue/compiler-ssr/-/compiler-ssr-3.5.12.tgz", + "integrity": "sha512-eLwc7v6bfGBSM7wZOGPmRavSWzNFF6+PdRhE+VFJhNCgHiF8AM7ccoqcv5kBXA2eWUfigD7byekvf/JsOfKvPA==", + "license": "MIT", "dependencies": { - "@vue/compiler-dom": "3.4.37", - "@vue/shared": "3.4.37" + "@vue/compiler-dom": "3.5.12", + "@vue/shared": "3.5.12" } }, "node_modules/@vue/compiler-vue2": { @@ -3400,6 +3269,7 @@ "resolved": "https://registry.npmjs.org/@vue/compiler-vue2/-/compiler-vue2-2.7.16.tgz", "integrity": "sha512-qYC3Psj9S/mfu9uVi5WvNZIzq+xnXMhOwbTFKKDD7b1lhpnn71jXSFdTQ+WsIEk0ONCd7VV2IMm7ONl6tbQ86A==", "dev": true, + "license": "MIT", "dependencies": { "de-indent": "^1.0.2", "he": "^1.2.0" @@ -3408,7 +3278,32 @@ "node_modules/@vue/devtools-api": { "version": "6.6.4", "resolved": "https://registry.npmjs.org/@vue/devtools-api/-/devtools-api-6.6.4.tgz", - "integrity": "sha512-sGhTPMuXqZ1rVOk32RylztWkfXTRhuS7vgAKv0zjqk8gbsHkJ7xfFf+jbySxt7tWObEJwyKaHMikV/WGDiQm8g==" + "integrity": "sha512-sGhTPMuXqZ1rVOk32RylztWkfXTRhuS7vgAKv0zjqk8gbsHkJ7xfFf+jbySxt7tWObEJwyKaHMikV/WGDiQm8g==", + "license": "MIT" + }, + "node_modules/@vue/devtools-kit": { + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/@vue/devtools-kit/-/devtools-kit-7.5.4.tgz", + "integrity": "sha512-0i7WFgc1B2TL52tstn82zlb9opSA0aIiHfkUYFXtZb8CIpmlFMTkHtgwVl6PMWNBj3LNhYou1YJCLpCYvJYYoA==", + "license": "MIT", + "dependencies": { + "@vue/devtools-shared": "^7.5.4", + "birpc": "^0.2.19", + "hookable": "^5.5.3", + "mitt": "^3.0.1", + "perfect-debounce": "^1.0.0", + "speakingurl": "^14.0.1", + "superjson": "^2.2.1" + } + }, + "node_modules/@vue/devtools-shared": { + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/@vue/devtools-shared/-/devtools-shared-7.5.4.tgz", + "integrity": "sha512-dwuq4YmwTyLc7eBOqX63s3JB8il7qnKsNgENglSMkUPwiItHkVAYYfPESN1rxSdYkl1RCux1l5TBidYqfUDNAA==", + "license": "MIT", + "dependencies": { + "rfdc": "^1.4.1" + } }, "node_modules/@vue/eslint-config-prettier": { "version": "9.0.0", @@ -3455,6 +3350,7 @@ "resolved": "https://registry.npmjs.org/@vue/language-core/-/language-core-2.1.6.tgz", "integrity": "sha512-MW569cSky9R/ooKMh6xa2g1D0AtRKbL56k83dzus/bx//RDJk24RHWkMzbAlXjMdDNyxAaagKPRquBIxkxlCkg==", "dev": true, + "license": "MIT", "dependencies": { "@volar/language-core": "~2.4.1", "@vue/compiler-dom": "^3.4.0", @@ -3475,49 +3371,54 @@ } }, "node_modules/@vue/reactivity": { - "version": "3.4.37", - "resolved": "https://registry.npmjs.org/@vue/reactivity/-/reactivity-3.4.37.tgz", - "integrity": "sha512-UmdKXGx0BZ5kkxPqQr3PK3tElz6adTey4307NzZ3whZu19i5VavYal7u2FfOmAzlcDVgE8+X0HZ2LxLb/jgbYw==", + "version": "3.5.12", + "resolved": "https://registry.npmjs.org/@vue/reactivity/-/reactivity-3.5.12.tgz", + "integrity": "sha512-UzaN3Da7xnJXdz4Okb/BGbAaomRHc3RdoWqTzlvd9+WBR5m3J39J1fGcHes7U3za0ruYn/iYy/a1euhMEHvTAg==", + "license": "MIT", "dependencies": { - "@vue/shared": "3.4.37" + "@vue/shared": "3.5.12" } }, "node_modules/@vue/runtime-core": { - "version": "3.4.37", - "resolved": "https://registry.npmjs.org/@vue/runtime-core/-/runtime-core-3.4.37.tgz", - "integrity": "sha512-MNjrVoLV/sirHZoD7QAilU1Ifs7m/KJv4/84QVbE6nyAZGQNVOa1HGxaOzp9YqCG+GpLt1hNDC4RbH+KtanV7w==", + "version": "3.5.12", + "resolved": "https://registry.npmjs.org/@vue/runtime-core/-/runtime-core-3.5.12.tgz", + "integrity": "sha512-hrMUYV6tpocr3TL3Ad8DqxOdpDe4zuQY4HPY3X/VRh+L2myQO8MFXPAMarIOSGNu0bFAjh1yBkMPXZBqCk62Uw==", + "license": "MIT", "dependencies": { - "@vue/reactivity": "3.4.37", - "@vue/shared": "3.4.37" + "@vue/reactivity": "3.5.12", + "@vue/shared": "3.5.12" } }, "node_modules/@vue/runtime-dom": { - "version": "3.4.37", - "resolved": "https://registry.npmjs.org/@vue/runtime-dom/-/runtime-dom-3.4.37.tgz", - "integrity": "sha512-Mg2EwgGZqtwKrqdL/FKMF2NEaOHuH+Ks9TQn3DHKyX//hQTYOun+7Tqp1eo0P4Ds+SjltZshOSRq6VsU0baaNg==", + "version": "3.5.12", + "resolved": "https://registry.npmjs.org/@vue/runtime-dom/-/runtime-dom-3.5.12.tgz", + "integrity": "sha512-q8VFxR9A2MRfBr6/55Q3umyoN7ya836FzRXajPB6/Vvuv0zOPL+qltd9rIMzG/DbRLAIlREmnLsplEF/kotXKA==", + "license": "MIT", "dependencies": { - "@vue/reactivity": "3.4.37", - "@vue/runtime-core": "3.4.37", - "@vue/shared": "3.4.37", + "@vue/reactivity": "3.5.12", + "@vue/runtime-core": "3.5.12", + "@vue/shared": "3.5.12", "csstype": "^3.1.3" } }, "node_modules/@vue/server-renderer": { - "version": "3.4.37", - "resolved": "https://registry.npmjs.org/@vue/server-renderer/-/server-renderer-3.4.37.tgz", - "integrity": "sha512-jZ5FAHDR2KBq2FsRUJW6GKDOAG9lUTX8aBEGq4Vf6B/35I9fPce66BornuwmqmKgfiSlecwuOb6oeoamYMohkg==", + "version": "3.5.12", + "resolved": "https://registry.npmjs.org/@vue/server-renderer/-/server-renderer-3.5.12.tgz", + "integrity": "sha512-I3QoeDDeEPZm8yR28JtY+rk880Oqmj43hreIBVTicisFTx/Dl7JpG72g/X7YF8hnQD3IFhkky5i2bPonwrTVPg==", + "license": "MIT", "dependencies": { - "@vue/compiler-ssr": "3.4.37", - "@vue/shared": "3.4.37" + "@vue/compiler-ssr": "3.5.12", + "@vue/shared": "3.5.12" }, "peerDependencies": { - "vue": "3.4.37" + "vue": "3.5.12" } }, "node_modules/@vue/shared": { - "version": "3.4.37", - "resolved": "https://registry.npmjs.org/@vue/shared/-/shared-3.4.37.tgz", - "integrity": "sha512-nIh8P2fc3DflG8+5Uw8PT/1i17ccFn0xxN/5oE9RfV5SVnd7G0XEFRwakrnNFE/jlS95fpGXDVG5zDETS26nmg==" + "version": "3.5.12", + "resolved": "https://registry.npmjs.org/@vue/shared/-/shared-3.5.12.tgz", + "integrity": "sha512-L2RPSAwUFbgZH20etwrXyVyCBu9OxRSi8T/38QsvnkJyvq2LufW2lDCOzm7t/U9C1mkhJGWYfCuFBCmIuNivrg==", + "license": "MIT" }, "node_modules/@vue/test-utils": { "version": "2.4.6", @@ -3637,9 +3538,9 @@ } }, "node_modules/acorn": { - "version": "8.12.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz", - "integrity": "sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==", + "version": "8.13.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.13.0.tgz", + "integrity": "sha512-8zSiw54Oxrdym50NlZ9sUusyO1Z1ZchgRLWRaK6c86XJFClyCgFKetdowBg5bKxyp/u+CDBJG4Mpp0m3HLZl9w==", "dev": true, "license": "MIT", "bin": { @@ -3660,9 +3561,9 @@ } }, "node_modules/acorn-walk": { - "version": "8.3.3", - "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.3.tgz", - "integrity": "sha512-MxXdReSRhGO7VlFe1bRG/oI7/mdLV9B9JJT0N8vZOhF7gFRR5l3M8W9G8JxmKV+JC5mGqJ0QvqfSOLsCPa4nUw==", + "version": "8.3.4", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz", + "integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==", "dev": true, "license": "MIT", "dependencies": { @@ -3673,9 +3574,9 @@ } }, "node_modules/adm-zip": { - "version": "0.5.15", - "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.15.tgz", - "integrity": "sha512-jYPWSeOA8EFoZnucrKCNihqBjoEGQSU4HKgHYQgKNEQ0pQF9a/DYuo/+fAxY76k4qe75LUlLWpAM1QWcBMTOKw==", + "version": "0.5.16", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.16.tgz", + "integrity": "sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==", "license": "MIT", "engines": { "node": ">=12.0" @@ -3805,9 +3706,10 @@ } }, "node_modules/apexcharts": { - "version": "3.53.0", - "resolved": "https://registry.npmjs.org/apexcharts/-/apexcharts-3.53.0.tgz", - "integrity": "sha512-QESZHZY3w9LPQ64PGh1gEdfjYjJ5Jp+Dfy0D/CLjsLOPTpXzdxwlNMqRj+vPbTcP0nAHgjWv1maDqcEq6u5olw==", + "version": "3.54.1", + "resolved": "https://registry.npmjs.org/apexcharts/-/apexcharts-3.54.1.tgz", + "integrity": "sha512-E4et0h/J1U3r3EwS/WlqJCQIbepKbp6wGUmaAwJOMjHUP4Ci0gxanLa7FR3okx6p9coi4st6J853/Cb1NP0vpA==", + "license": "MIT", "dependencies": { "@yr/monotone-cubic-spline": "^1.0.3", "svg.draggable.js": "^2.2.2", @@ -3969,9 +3871,9 @@ } }, "node_modules/async": { - "version": "3.2.5", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.5.tgz", - "integrity": "sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==", + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", "license": "MIT" }, "node_modules/asynckit": { @@ -4062,26 +3964,34 @@ } }, "node_modules/aws4": { - "version": "1.13.1", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.13.1.tgz", - "integrity": "sha512-u5w79Rd7SU4JaIlA/zFqG+gOiuq25q5VLyZ8E+ijJeILuTxVzZgp2CaGw/UTw6pXYN9XMO9yiqj/nEHmhTG5CA==", + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.13.2.tgz", + "integrity": "sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw==", "license": "MIT" }, "node_modules/axios": { - "version": "1.7.5", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.5.tgz", - "integrity": "sha512-fZu86yCo+svH3uqJ/yTdQ0QHpQu5oL+/QE+QPSv6BZSkDAoky9vytxp7u5qk83OJFS3kEBcesWni9WTZAv3tSw==", + "version": "1.7.7", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz", + "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==", "dev": true, + "license": "MIT", "dependencies": { "follow-redirects": "^1.15.6", "form-data": "^4.0.0", "proxy-from-env": "^1.1.0" } }, + "node_modules/axios/node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "dev": true, + "license": "MIT" + }, "node_modules/b4a": { - "version": "1.6.6", - "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.6.tgz", - "integrity": "sha512-5Tk1HLk6b6ctmjIkAcU/Ujv/1WqiDl0F0JdRCR80VsOcUlHcu7pWeWRlOqQLHfDEsVx9YH/aif5AG4ehoCtTmg==", + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.7.tgz", + "integrity": "sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg==", "license": "Apache-2.0" }, "node_modules/balanced-match": { @@ -4091,16 +4001,16 @@ "license": "MIT" }, "node_modules/bare-events": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.4.2.tgz", - "integrity": "sha512-qMKFd2qG/36aA4GwvKq8MxnPgCQAmBWmSyLWsJcbn8v03wvIPQ/hG1Ms8bPzndZxMDoHpxez5VOS+gC9Yi24/Q==", + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.5.0.tgz", + "integrity": "sha512-/E8dDe9dsbLyh2qrZ64PEPadOQ0F4gbl1sUJOrmph7xOiIxfY8vwab/4bFLh4Y88/Hk/ujKcrQKc+ps0mv873A==", "license": "Apache-2.0", "optional": true }, "node_modules/bare-fs": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-2.3.1.tgz", - "integrity": "sha512-W/Hfxc/6VehXlsgFtbB5B4xFcsCl+pAh30cYhoFyXErf6oGrwjh8SwiPAdHgpmWonKuYpZgGywN0SXt7dgsADA==", + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-2.3.5.tgz", + "integrity": "sha512-SlE9eTxifPDJrT6YgemQ1WGFleevzwY+XAP1Xqgl56HtcrisC2CHCZ2tq6dBpcH2TnNxwUEUGhweo+lrQtYuiw==", "license": "Apache-2.0", "optional": true, "dependencies": { @@ -4110,9 +4020,9 @@ } }, "node_modules/bare-os": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-2.4.0.tgz", - "integrity": "sha512-v8DTT08AS/G0F9xrhyLtepoo9EJBJ85FRSMbu1pQUlAf6A8T0tEEQGMVObWeqpjhSPXsE0VGlluFBJu2fdoTNg==", + "version": "2.4.4", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-2.4.4.tgz", + "integrity": "sha512-z3UiI2yi1mK0sXeRdc4O1Kk8aOa/e+FNWZcTiPB/dfTWyLypuE99LibgRaQki914Jq//yAWylcAt+mknKdixRQ==", "license": "Apache-2.0", "optional": true }, @@ -4127,13 +4037,13 @@ } }, "node_modules/bare-stream": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.1.3.tgz", - "integrity": "sha512-tiDAH9H/kP+tvNO5sczyn9ZAA7utrSMobyDchsnyyXBuUe2FSQWbxhtuHB8jwpHYYevVo2UJpcmvvjrbHboUUQ==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.3.2.tgz", + "integrity": "sha512-EFZHSIBkDgSHIwj2l2QZfP4U5OcD4xFAOwhSb/vlr9PIqyGJGvB/nfClJbcnh3EY4jtPE4zsb5ztae96bVF79A==", "license": "Apache-2.0", "optional": true, "dependencies": { - "streamx": "^2.18.0" + "streamx": "^2.20.0" } }, "node_modules/base64-js": { @@ -4178,7 +4088,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/bl": { + "node_modules/birpc": { + "version": "0.2.19", + "resolved": "https://registry.npmjs.org/birpc/-/birpc-0.2.19.tgz", + "integrity": "sha512-5WeXXAvTmitV1RqJFppT5QtUiz2p1mRSYU000Jkft5ZUCLJIk4uQriYNO50HknxKwM6jd8utNc66K1qGIwwWBQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", @@ -4212,6 +4131,7 @@ "version": "1.20.3", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", + "license": "MIT", "dependencies": { "bytes": "3.1.2", "content-type": "~1.0.5", @@ -4235,6 +4155,7 @@ "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", "dependencies": { "ms": "2.0.0" } @@ -4243,6 +4164,7 @@ "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "license": "MIT", "dependencies": { "safer-buffer": ">= 2.1.2 < 3" }, @@ -4253,21 +4175,8 @@ "node_modules/body-parser/node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" - }, - "node_modules/body-parser/node_modules/qs": { - "version": "6.13.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", - "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", - "dependencies": { - "side-channel": "^1.0.6" - }, - "engines": { - "node": ">=0.6" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" }, "node_modules/boolbase": { "version": "1.0.0", @@ -4365,14 +4274,18 @@ } }, "node_modules/browserify-rsa": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.1.0.tgz", - "integrity": "sha512-AdEER0Hkspgno2aR97SAf6vi0y0k8NuOpGnVH3O99rcA5Q6sh8QxcngtHuJ6uXwnfAXNM4Gn1Gb7/MV1+Ymbog==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.1.1.tgz", + "integrity": "sha512-YBjSAiTqM04ZVei6sXighu679a3SqWORA3qZTEqZImnlkDIFtKc6pNutpjyZ8RJTjQtuYfeetkxM11GwoYXMIQ==", "dev": true, "license": "MIT", "dependencies": { - "bn.js": "^5.0.0", - "randombytes": "^2.0.1" + "bn.js": "^5.2.1", + "randombytes": "^2.1.0", + "safe-buffer": "^5.2.1" + }, + "engines": { + "node": ">= 0.10" } }, "node_modules/browserify-sign": { @@ -4448,9 +4361,9 @@ } }, "node_modules/browserslist": { - "version": "4.23.3", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.23.3.tgz", - "integrity": "sha512-btwCFJVjI4YWDNfau8RhZ+B1Q/VLoUITrm3RlP6y1tYGWIOa+InuYiRGXUBXo8nA1qKmHMyLB/iVQg5TT4eFoA==", + "version": "4.24.2", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.24.2.tgz", + "integrity": "sha512-ZIc+Q62revdMcqC6aChtW4jz3My3klmCO1fEmINZY/8J3EpBg5/A/D0AKmBveUh6pgoeycoMkVMko84tuYS+Gg==", "dev": true, "funding": [ { @@ -4468,10 +4381,10 @@ ], "license": "MIT", "dependencies": { - "caniuse-lite": "^1.0.30001646", - "electron-to-chromium": "^1.5.4", + "caniuse-lite": "^1.0.30001669", + "electron-to-chromium": "^1.5.41", "node-releases": "^2.0.18", - "update-browserslist-db": "^1.1.0" + "update-browserslist-db": "^1.1.1" }, "bin": { "browserslist": "cli.js" @@ -4531,6 +4444,7 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", "engines": { "node": ">= 0.8" } @@ -4607,9 +4521,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001650", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001650.tgz", - "integrity": "sha512-fgEc7hP/LB7iicdXHUI9VsBsMZmUmlVJeQP2qqQW+3lkqVhbmjEU8zp+h5stWeilX+G7uXuIUIIlWlDw9jdt8g==", + "version": "1.0.30001669", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001669.tgz", + "integrity": "sha512-DlWzFDJqstqtIVx1zeSpIMLjunf5SmwOw0N2Ck/QSQdS8PLS4+9HrLaYei4w8BIAL7IB/UEDu889d8vhCTPA0w==", "dev": true, "funding": [ { @@ -4634,9 +4548,9 @@ "license": "Apache-2.0" }, "node_modules/chai": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/chai/-/chai-5.1.1.tgz", - "integrity": "sha512-pT1ZgP8rPNqUgieVaEY+ryQr6Q4HXNg8Ei9UnLUrjN4IA7dvQC5JB+/kxVcPNDHyBcc/26CXPkbNzq3qwrOEKA==", + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-5.1.2.tgz", + "integrity": "sha512-aGtmf24DW6MLHHG5gCx4zaI3uBq3KRtxeVs0DjFH6Z0rDNbsvTxFASFvdj79pxjxZ8/5u3PIiN3IwEIQkiiuPw==", "dev": true, "license": "MIT", "dependencies": { @@ -5061,7 +4975,8 @@ "version": "0.0.1", "resolved": "https://registry.npmjs.org/computeds/-/computeds-0.0.1.tgz", "integrity": "sha512-7CEBgcMjVmitjYo5q8JTJVra6X5mQ20uTThdK+0kR7UEaDrAWEQcRiBtWJzga4eRpP6afNwwLsX2SET2JhVB1Q==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/concat-map": { "version": "0.0.1", @@ -5069,13 +4984,6 @@ "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", "license": "MIT" }, - "node_modules/confbox": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/confbox/-/confbox-0.1.7.tgz", - "integrity": "sha512-uJcB/FKZtBMCJpK8MQji6bJHgu1tixKPxRLeGkNzBoOZzpnZUJm0jm2/sBDWcuBx1dYgxV4JU+g5hmNxCyAmdA==", - "dev": true, - "license": "MIT" - }, "node_modules/config-chain": { "version": "1.1.13", "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.13.tgz", @@ -5129,14 +5037,15 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", "engines": { "node": ">= 0.6" } }, "node_modules/cookie": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", - "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==", + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", + "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", "license": "MIT", "engines": { "node": ">= 0.6" @@ -5148,6 +5057,21 @@ "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", "license": "MIT" }, + "node_modules/copy-anything": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/copy-anything/-/copy-anything-3.0.5.tgz", + "integrity": "sha512-yCEafptTtb4bk7GLEQoM8KVJpxAfdBJYaXyzQEgQQQgYrZiDp8SJmGKlYza6CYjEDNstAdNdKA3UuoULlEbS6w==", + "license": "MIT", + "dependencies": { + "is-what": "^4.1.8" + }, + "engines": { + "node": ">=12.13" + }, + "funding": { + "url": "https://github.com/sponsors/mesqueeb" + } + }, "node_modules/core-js": { "version": "3.29.1", "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.29.1.tgz", @@ -5247,26 +5171,30 @@ } }, "node_modules/crypto-browserify": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-3.12.0.tgz", - "integrity": "sha512-fz4spIh+znjO2VjL+IdhEpRJ3YN6sMzITSBijk6FK2UvTqruSQW+/cCZTSNsMiZNvUeq0CqurF+dAbyiGOY6Wg==", + "version": "3.12.1", + "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-3.12.1.tgz", + "integrity": "sha512-r4ESw/IlusD17lgQi1O20Fa3qNnsckR126TdUuBgAu7GBYSIPvdNyONd3Zrxh0xCwA4+6w/TDArBPsMvhur+KQ==", "dev": true, "license": "MIT", "dependencies": { - "browserify-cipher": "^1.0.0", - "browserify-sign": "^4.0.0", - "create-ecdh": "^4.0.0", - "create-hash": "^1.1.0", - "create-hmac": "^1.1.0", - "diffie-hellman": "^5.0.0", - "inherits": "^2.0.1", - "pbkdf2": "^3.0.3", - "public-encrypt": "^4.0.0", - "randombytes": "^2.0.0", - "randomfill": "^1.0.3" + "browserify-cipher": "^1.0.1", + "browserify-sign": "^4.2.3", + "create-ecdh": "^4.0.4", + "create-hash": "^1.2.0", + "create-hmac": "^1.1.7", + "diffie-hellman": "^5.0.3", + "hash-base": "~3.0.4", + "inherits": "^2.0.4", + "pbkdf2": "^3.1.2", + "public-encrypt": "^4.0.3", + "randombytes": "^2.1.0", + "randomfill": "^1.0.4" }, "engines": { - "node": "*" + "node": ">= 0.10" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/cssesc": { @@ -5283,29 +5211,23 @@ } }, "node_modules/cssstyle": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-4.0.1.tgz", - "integrity": "sha512-8ZYiJ3A/3OkDd093CBT/0UKDWry7ak4BdPTFP2+QEP7cmhouyq/Up709ASSj2cK02BbZiMgk7kYjZNS4QP5qrQ==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-4.1.0.tgz", + "integrity": "sha512-h66W1URKpBS5YMI/V8PyXvTMFT8SupJ1IzoIV8IeBC/ji8WVmrO8dGlTi+2dh6whmdk6BiKJLD/ZBkhWbcg6nA==", "dev": true, "license": "MIT", "dependencies": { - "rrweb-cssom": "^0.6.0" + "rrweb-cssom": "^0.7.1" }, "engines": { "node": ">=18" } }, - "node_modules/cssstyle/node_modules/rrweb-cssom": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.6.0.tgz", - "integrity": "sha512-APM0Gt1KoXBz0iIkkdB/kfvGOwC4UuJFeG/c+yV7wSc7q96cG/kJ0HiYCnzivD9SB53cLV1MlHFNfOuPaadYSw==", - "dev": true, - "license": "MIT" - }, "node_modules/csstype": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz", - "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==" + "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==", + "license": "MIT" }, "node_modules/csv-parse": { "version": "5.5.6", @@ -5314,12 +5236,13 @@ "license": "MIT" }, "node_modules/cypress": { - "version": "13.14.2", - "resolved": "https://registry.npmjs.org/cypress/-/cypress-13.14.2.tgz", - "integrity": "sha512-lsiQrN17vHMB2fnvxIrKLAjOr9bPwsNbPZNrWf99s4u+DVmCY6U+w7O3GGG9FvP4EUVYaDu+guWeNLiUzBrqvA==", + "version": "13.15.1", + "resolved": "https://registry.npmjs.org/cypress/-/cypress-13.15.1.tgz", + "integrity": "sha512-DwUFiKXo4lef9kA0M4iEhixFqoqp2hw8igr0lTqafRb9qtU3X0XGxKbkSYsUFdkrAkphc7MPDxoNPhk5pj9PVg==", "hasInstallScript": true, + "license": "MIT", "dependencies": { - "@cypress/request": "^3.0.1", + "@cypress/request": "^3.0.4", "@cypress/xvfb": "^1.2.4", "@types/sinonjs__fake-timers": "8.1.1", "@types/sizzle": "^2.3.2", @@ -5359,6 +5282,7 @@ "semver": "^7.5.3", "supports-color": "^8.1.1", "tmp": "~0.2.3", + "tree-kill": "1.2.2", "untildify": "^4.0.0", "yauzl": "^2.10.0" }, @@ -5369,12 +5293,6 @@ "node": "^16.0.0 || ^18.0.0 || >=20.0.0" } }, - "node_modules/cypress/node_modules/proxy-from-env": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.0.0.tgz", - "integrity": "sha512-F2JHgJQ1iqwnHDcQjVBsq3n/uoaFL+iPW/eAeL7kVxy/2RrWaN4WroKjjvbsoRtv0ftelNyC01bjRhn/bhcf4A==", - "license": "MIT" - }, "node_modules/cypress/node_modules/supports-color": { "version": "8.1.1", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", @@ -5827,21 +5745,23 @@ } }, "node_modules/dayjs": { - "version": "1.11.12", - "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.12.tgz", - "integrity": "sha512-Rt2g+nTbLlDWZTwwrIXjy9MeiZmSDI375FvZs72ngxx8PDC6YXOeR3q5LAuPzjZQxhiWdRKac7RKV+YyQYfYIg==", + "version": "1.11.13", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.13.tgz", + "integrity": "sha512-oaMBel6gjolK862uaPQOVTA7q3TZhuSvuMQAAglQDOWYO9A91IrAOUJEyKVlqJlHE0vq5p5UXxzdPfMH/x6xNg==", "license": "MIT" }, "node_modules/de-indent": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/de-indent/-/de-indent-1.0.2.tgz", "integrity": "sha512-e/1zu3xH5MQryN2zdVaF0OrdNLUbvWxzMbi+iNA6Bky7l1RoP8a2fIbRocyHclXt/arDrrR6lL3TqFD9pMQTsg==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/debug": { "version": "4.3.7", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "license": "MIT", "dependencies": { "ms": "^2.1.3" }, @@ -5977,6 +5897,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", "engines": { "node": ">= 0.8" } @@ -5996,6 +5917,7 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "license": "MIT", "engines": { "node": ">= 0.8", "npm": "1.2.8000 || >= 1.4.16" @@ -6027,16 +5949,6 @@ "node": ">=0.3.1" } }, - "node_modules/diff-sequences": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz", - "integrity": "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, "node_modules/diffie-hellman": { "version": "5.0.3", "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", @@ -6189,16 +6101,16 @@ "license": "MIT" }, "node_modules/electron-to-chromium": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.5.tgz", - "integrity": "sha512-QR7/A7ZkMS8tZuoftC/jfqNkZLQO779SSW3YuZHP4eXpj3EffGLFcB/Xu9AAZQzLccTiCV+EmUo3ha4mQ9wnlA==", + "version": "1.5.45", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.45.tgz", + "integrity": "sha512-vOzZS6uZwhhbkZbcRyiy99Wg+pYFV5hk+5YaECvx0+Z31NR3Tt5zS6dze2OepT6PCTzVzT0dIJItti+uAW5zmw==", "dev": true, "license": "ISC" }, "node_modules/elliptic": { - "version": "6.5.6", - "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.6.tgz", - "integrity": "sha512-mpzdtpeCLuS3BmE3pO3Cpp5bbjlOPY2Q0PgoF+Od1XZrHLYI28Xe3ossCmYCQt11FQKEYd9+PF8jymTvtWJSHQ==", + "version": "6.5.7", + "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.7.tgz", + "integrity": "sha512-ESVCtTwiA+XhY3wyh24QqRGBoP3rEdDUl3EDUUo9tft074fi19IrdpH7hLCMMP3CIj7jb3W96rn8lt/BqIlt5Q==", "dev": true, "license": "MIT", "dependencies": { @@ -6225,9 +6137,9 @@ "license": "MIT" }, "node_modules/encodeurl": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", "license": "MIT", "engines": { "node": ">= 0.8" @@ -6259,7 +6171,6 @@ "version": "4.5.0", "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", - "dev": true, "license": "BSD-2-Clause", "engines": { "node": ">=0.12" @@ -6365,9 +6276,9 @@ } }, "node_modules/escalade": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz", - "integrity": "sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", "license": "MIT", "engines": { "node": ">=6" @@ -6393,17 +6304,18 @@ } }, "node_modules/eslint": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.0.tgz", - "integrity": "sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ==", + "version": "8.57.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.1.tgz", + "integrity": "sha512-ypowyDxpVSYpkXr9WPv2PAZCtNip1Mv5KTW0SCurXv/9iOpcrH9PaqUElksqEB6pChqHGDRCFTyrZlGhnLNGiA==", + "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", "@eslint/eslintrc": "^2.1.4", - "@eslint/js": "8.57.0", - "@humanwhocodes/config-array": "^0.11.14", + "@eslint/js": "8.57.1", + "@humanwhocodes/config-array": "^0.13.0", "@humanwhocodes/module-importer": "^1.0.1", "@nodelib/fs.walk": "^1.2.8", "@ungap/structured-clone": "^1.2.0", @@ -6462,10 +6374,11 @@ } }, "node_modules/eslint-plugin-cypress": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-cypress/-/eslint-plugin-cypress-3.5.0.tgz", - "integrity": "sha512-JZQ6XnBTNI8h1B9M7wJSFzc48SYbh7VMMKaNTQOFa3BQlnmXPrVc4PKen8R+fpv6VleiPeej6VxloGb42zdRvw==", + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-cypress/-/eslint-plugin-cypress-3.6.0.tgz", + "integrity": "sha512-7IAMcBbTVu5LpWeZRn5a9mQ30y4hKp3AfTz+6nSD/x/7YyLMoBI6X7XjDLYI6zFvuy4Q4QVGl563AGEXGW/aSA==", "dev": true, + "license": "MIT", "dependencies": { "globals": "^13.20.0" }, @@ -6505,9 +6418,9 @@ } }, "node_modules/eslint-plugin-vue": { - "version": "9.27.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-vue/-/eslint-plugin-vue-9.27.0.tgz", - "integrity": "sha512-5Dw3yxEyuBSXTzT5/Ge1X5kIkRTQ3nvBn/VwPwInNiZBSJOO/timWMUaflONnFBzU6NhB68lxnCda7ULV5N7LA==", + "version": "9.29.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-vue/-/eslint-plugin-vue-9.29.1.tgz", + "integrity": "sha512-MH/MbVae4HV/tM8gKAVWMPJbYgW04CK7SuzYRrlNERpxbO0P3+Zdsa2oAcFBW6xNu7W6lIkGOsFAMCRTYmrlWQ==", "dev": true, "license": "MIT", "peer": true, @@ -6517,7 +6430,7 @@ "natural-compare": "^1.4.0", "nth-check": "^2.1.1", "postcss-selector-parser": "^6.0.15", - "semver": "^7.6.0", + "semver": "^7.6.3", "vue-eslint-parser": "^9.4.3", "xml-name-validator": "^4.0.0" }, @@ -6656,6 +6569,7 @@ "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", "engines": { "node": ">= 0.6" } @@ -6753,23 +6667,24 @@ } }, "node_modules/express": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/express/-/express-4.20.0.tgz", - "integrity": "sha512-pLdae7I6QqShF5PnNTCVn4hI91Dx0Grkn2+IAsMTgMIKuQVte2dN9PeGSSAME2FR8anOhVA62QDIUaWVfEXVLw==", + "version": "4.21.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.21.1.tgz", + "integrity": "sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ==", + "license": "MIT", "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", "body-parser": "1.20.3", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.6.0", + "cookie": "0.7.1", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "etag": "~1.8.1", - "finalhandler": "1.2.0", + "finalhandler": "1.3.1", "fresh": "0.5.2", "http-errors": "2.0.0", "merge-descriptors": "1.0.3", @@ -6778,11 +6693,11 @@ "parseurl": "~1.3.3", "path-to-regexp": "0.1.10", "proxy-addr": "~2.0.7", - "qs": "6.11.0", + "qs": "6.13.0", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", "send": "0.19.0", - "serve-static": "1.16.0", + "serve-static": "1.16.2", "setprototypeof": "1.2.0", "statuses": "2.0.1", "type-is": "~1.6.18", @@ -6838,35 +6753,12 @@ "ms": "2.0.0" } }, - "node_modules/express/node_modules/encodeurl": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", - "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/express/node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", "license": "MIT" }, - "node_modules/express/node_modules/qs": { - "version": "6.11.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", - "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", - "license": "BSD-3-Clause", - "dependencies": { - "side-channel": "^1.0.4" - }, - "engines": { - "node": ">=0.6" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -7053,13 +6945,13 @@ } }, "node_modules/finalhandler": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", - "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", + "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==", "license": "MIT", "dependencies": { "debug": "2.6.9", - "encodeurl": "~1.0.2", + "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "on-finished": "2.4.1", "parseurl": "~1.3.3", @@ -7154,9 +7046,9 @@ "license": "ISC" }, "node_modules/follow-redirects": { - "version": "1.15.6", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", - "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", "dev": true, "funding": [ { @@ -7224,9 +7116,9 @@ } }, "node_modules/form-data": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", - "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz", + "integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==", "license": "MIT", "dependencies": { "asynckit": "^0.4.0", @@ -7264,6 +7156,7 @@ "version": "0.5.2", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "license": "MIT", "engines": { "node": ">= 0.6" } @@ -7386,16 +7279,6 @@ "node": "6.* || 8.* || >= 10.*" } }, - "node_modules/get-func-name": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz", - "integrity": "sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "*" - } - }, "node_modules/get-intrinsic": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", @@ -7725,6 +7608,12 @@ "minimalistic-crypto-utils": "^1.0.1" } }, + "node_modules/hookable": { + "version": "5.5.3", + "resolved": "https://registry.npmjs.org/hookable/-/hookable-5.5.3.tgz", + "integrity": "sha512-Yc+BQe8SvoXH1643Qez1zqLRmbA5rCL+sSmk6TVos0LWVfNIB7PGncdlId77WzLGSIB5KaWgTaNTs2lNVEI6VQ==", + "license": "MIT" + }, "node_modules/hosted-git-info": { "version": "2.8.9", "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz", @@ -7748,6 +7637,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "license": "MIT", "dependencies": { "depd": "2.0.0", "inherits": "2.0.4", @@ -7787,14 +7677,14 @@ } }, "node_modules/http-signature": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.3.6.tgz", - "integrity": "sha512-3adrsD6zqo4GsTqtO7FyrejHNv+NgiIfAfv68+jVlFmSr9OGy7zrxONceFRLKvnnZA5jbxQBX1u9PpB6Wi32Gw==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.4.0.tgz", + "integrity": "sha512-G5akfn7eKbpDN+8nPS/cb57YeA1jLTVxjpCj7tmm3QKPdyDy7T+qSC40e9ptydSWvkwjSXw1VbkpyEm39ukeAg==", "license": "MIT", "dependencies": { "assert-plus": "^1.0.0", "jsprim": "^2.0.2", - "sshpk": "^1.14.1" + "sshpk": "^1.18.0" }, "engines": { "node": ">=0.10" @@ -8035,9 +7925,9 @@ } }, "node_modules/is-core-module": { - "version": "2.15.0", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.15.0.tgz", - "integrity": "sha512-Dd+Lb2/zvk9SKy1TGCt1wFJFo/MWBPMX5x7KcvLajWTGuomczdQX61PvY5yK6SVACwpoexWo81IfFyoKY2QnTA==", + "version": "2.15.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.15.1.tgz", + "integrity": "sha512-z0vtXSwucUJtANQWldhbtbt7BnL0vxiFjIdDLAatwhDYty2bad6s+rijD6Ri4YuYJubLzIJLUidCh09e1djEVQ==", "license": "MIT", "dependencies": { "hasown": "^2.0.2" @@ -8218,6 +8108,18 @@ "integrity": "sha512-rMYPYvCzsXywIsldgLaSoPlw5PfoB/ssr7hY4pLfcodrA5M/eArza1a9VmTiNIBNMjOGr1Ow9mTyU2o69U6U9Q==", "license": "MIT" }, + "node_modules/is-what": { + "version": "4.1.16", + "resolved": "https://registry.npmjs.org/is-what/-/is-what-4.1.16.tgz", + "integrity": "sha512-ZhMwEosbFJkA0YhFnNDgTM4ZxDRsS6HqTo7qsZM08fehyRYIYa0yHu5R6mgo1n/8MgaPBXiPimPD77baVFYg+A==", + "license": "MIT", + "engines": { + "node": ">=12.13" + }, + "funding": { + "url": "https://github.com/sponsors/mesqueeb" + } + }, "node_modules/isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", @@ -8389,13 +8291,6 @@ "node": ">=14" } }, - "node_modules/js-tokens": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-9.0.0.tgz", - "integrity": "sha512-WriZw1luRMlmV3LGJaR6QOJjWwgLUTf89OwT2lUOyjX2dJGBwgmIkbcz+7WFZjrZM635JOIR517++e/67CP9dQ==", - "dev": true, - "license": "MIT" - }, "node_modules/js-yaml": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", @@ -8416,12 +8311,13 @@ "license": "MIT" }, "node_modules/jsdom": { - "version": "25.0.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-25.0.0.tgz", - "integrity": "sha512-OhoFVT59T7aEq75TVw9xxEfkXgacpqAhQaYgP9y/fDqWQCMB/b1H66RfmPm/MaeaAIU9nDwMOVTlPN51+ao6CQ==", + "version": "25.0.1", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-25.0.1.tgz", + "integrity": "sha512-8i7LzZj7BF8uplX+ZyOlIz86V6TAsSs+np6m1kpW9u0JWi4z/1t+FzcK1aek+ybTnAC4KhBL4uXCNT0wcUIeCw==", "dev": true, + "license": "MIT", "dependencies": { - "cssstyle": "^4.0.1", + "cssstyle": "^4.1.0", "data-urls": "^5.0.0", "decimal.js": "^10.4.3", "form-data": "^4.0.0", @@ -8434,7 +8330,7 @@ "rrweb-cssom": "^0.7.1", "saxes": "^6.0.0", "symbol-tree": "^3.2.4", - "tough-cookie": "^4.1.4", + "tough-cookie": "^5.0.0", "w3c-xmlserializer": "^5.0.0", "webidl-conversions": "^7.0.0", "whatwg-encoding": "^3.1.1", @@ -8482,6 +8378,19 @@ "node": ">= 14" } }, + "node_modules/jsdom/node_modules/tough-cookie": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.0.0.tgz", + "integrity": "sha512-FRKsF7cz96xIIeMZ82ehjC3xW2E+O2+v11udrDYewUbszngYhsGa8z6YUMMzO9QJZzzyd0nGGXnML/TReX6W8Q==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "tldts": "^6.1.32" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/jsdom/node_modules/webidl-conversions": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", @@ -8681,23 +8590,6 @@ "node": ">=0.10.0" } }, - "node_modules/local-pkg": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/local-pkg/-/local-pkg-0.5.0.tgz", - "integrity": "sha512-ok6z3qlYyCDS4ZEU27HaU6x/xZa9Whf8jD4ptH5UZTQYZVYeb9bnZ3ojVhiJNLiXK1Hfc0GNbLXcmZ5plLDDBg==", - "dev": true, - "license": "MIT", - "dependencies": { - "mlly": "^1.4.2", - "pkg-types": "^1.0.3" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/antfu" - } - }, "node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -8812,14 +8704,11 @@ "license": "Apache-2.0" }, "node_modules/loupe": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.1.1.tgz", - "integrity": "sha512-edNu/8D5MKVfGVFRhFf8aAxiTM6Wumfz5XsaatSxlD3w4R1d/WEKUTydCdPGbl9K7QG/Ca3GnDV2sIKIpXRQcw==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.1.2.tgz", + "integrity": "sha512-23I4pFZHmAemUnz8WZXbYRSKYj801VDaNv9ETuMh7IrMc7VuVVSo+Z9iLE3ni30+U48iDWfi30d3twAXBYmnCg==", "dev": true, - "license": "MIT", - "dependencies": { - "get-func-name": "^2.0.1" - } + "license": "MIT" }, "node_modules/lru-cache": { "version": "10.4.3", @@ -8829,9 +8718,9 @@ "license": "ISC" }, "node_modules/magic-string": { - "version": "0.30.11", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.11.tgz", - "integrity": "sha512-+Wri9p0QHMy+545hKww7YAu5NyzF8iomPL/RQazugQ9+Ez4Ic3mERMd8ZTX5rfK944j+560ZJi8iAwgak1Ac7A==", + "version": "0.30.12", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.12.tgz", + "integrity": "sha512-Ea8I3sQMVXr8JhN4z+H/d8zwo+tYDgHE9+5G4Wnrwhs0gaK9fXTKx0Tw5Xwsd/bCPTTZNRAdpyzvoeORe9LYpw==", "license": "MIT", "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0" @@ -8890,6 +8779,7 @@ "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "license": "MIT", "engines": { "node": ">= 0.6" } @@ -8898,6 +8788,7 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "license": "MIT", "funding": { "url": "https://github.com/sponsors/sindresorhus" } @@ -8928,9 +8819,9 @@ } }, "node_modules/micromatch": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz", - "integrity": "sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==", + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", "dev": true, "license": "MIT", "dependencies": { @@ -8963,14 +8854,15 @@ "license": "MIT" }, "node_modules/mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", + "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", + "license": "MIT", "bin": { "mime": "cli.js" }, "engines": { - "node": ">=4" + "node": ">=10.0.0" } }, "node_modules/mime-db": { @@ -9088,6 +8980,12 @@ "node": ">=8" } }, + "node_modules/mitt": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", + "license": "MIT" + }, "node_modules/mkdirp": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", @@ -9106,24 +9004,12 @@ "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", "license": "MIT" }, - "node_modules/mlly": { - "version": "1.7.1", - "resolved": "https://registry.npmjs.org/mlly/-/mlly-1.7.1.tgz", - "integrity": "sha512-rrVRZRELyQzrIUAVMHxP97kv+G786pHmOKzuFII8zDYahFBS7qnHh2AlYSl1GAHhaMPCz6/oHjVMcfFYgFYHgA==", - "dev": true, - "license": "MIT", - "dependencies": { - "acorn": "^8.11.3", - "pathe": "^1.1.2", - "pkg-types": "^1.1.1", - "ufo": "^1.5.3" - } - }, "node_modules/mocha": { "version": "10.7.3", "resolved": "https://registry.npmjs.org/mocha/-/mocha-10.7.3.tgz", "integrity": "sha512-uQWxAu44wwiACGqjbPYmjo7Lg8sFrS3dQe7PP2FQI+woptP4vZXSMcfMyFL/e1yFEeEpV4RtyTpZROOKmxis+A==", "dev": true, + "license": "MIT", "dependencies": { "ansi-colors": "^4.1.3", "browser-stdout": "^1.3.1", @@ -9207,13 +9093,15 @@ "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" }, "node_modules/muggle-string": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/muggle-string/-/muggle-string-0.4.1.tgz", "integrity": "sha512-VNTrAak/KhO2i8dqqnqnAHOa3cYBwXEZe9h+D5h/1ZqFSTEFHdM65lR7RoIqq3tBBYavsOXV84NoHXZ0AkPyqQ==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/mz": { "version": "2.7.0", @@ -9228,9 +9116,9 @@ } }, "node_modules/nan": { - "version": "2.20.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.20.0.tgz", - "integrity": "sha512-bk3gXBZDGILuuo/6sKtr0DQmSThYHLtNCdSdXk9YkxD/jK6X2vmCyyXBBxyqZ4XcnzTyYEAThfX3DCEnLf6igw==", + "version": "2.22.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.0.tgz", + "integrity": "sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw==", "license": "MIT" }, "node_modules/nanoid": { @@ -9274,9 +9162,9 @@ } }, "node_modules/node-abi": { - "version": "3.65.0", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.65.0.tgz", - "integrity": "sha512-ThjYBfoDNr08AWx6hGaRbfPwxKV9kVzAzOzlLKbk2CuqXE2xnCh+cbAGnwM3t8Lq4v9rUB7VfondlkBckcJrVA==", + "version": "3.71.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.71.0.tgz", + "integrity": "sha512-SZ40vRiy/+wRTf21hxkkEjPJZpARzUMVcJoQse2EF8qkUWbbO2z7vd5oA/H6bVH6SZQ5STGcu0KRDS7biNRfxw==", "license": "MIT", "dependencies": { "semver": "^7.3.5" @@ -9450,9 +9338,9 @@ } }, "node_modules/node-datachannel": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/node-datachannel/-/node-datachannel-0.11.0.tgz", - "integrity": "sha512-8/vAMms32XxgJ9FIRDXbfmmH1ROm0HBdsa/XteIcUWN4VTQN38UITTkuu6YsfQzN/CQp8YhhnfAEzEadQJ2c6Q==", + "version": "0.20.0-alpha.2", + "resolved": "https://registry.npmjs.org/node-datachannel/-/node-datachannel-0.20.0-alpha.2.tgz", + "integrity": "sha512-oIkIoerLuJCYrFi3pDGYVUUPjRNwv75ExDp9FS5V3INzWlkrcFmX/PGsLOGBg/7VrKUHvjGb0VZ1Mz38cDHncw==", "hasInstallScript": true, "license": "MPL 2.0", "peer": true, @@ -9534,9 +9422,9 @@ "license": "MIT" }, "node_modules/node-stdlib-browser": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/node-stdlib-browser/-/node-stdlib-browser-1.2.0.tgz", - "integrity": "sha512-VSjFxUhRhkyed8AtLwSCkMrJRfQ3e2lGtG3sP6FEgaLKBBbxM/dLfjRe1+iLhjvyLFW3tBQ8+c0pcOtXGbAZJg==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/node-stdlib-browser/-/node-stdlib-browser-1.2.1.tgz", + "integrity": "sha512-dZezG3D88Lg22DwyjsDuUs7cCT/XGr8WwJgg/S3ZnkcWuPet2Tt/W1d2Eytb1Z73JpZv+XVCDI5TWv6UMRq0Gg==", "dev": true, "license": "MIT", "dependencies": { @@ -9564,7 +9452,7 @@ "string_decoder": "^1.0.0", "timers-browserify": "^2.0.4", "tty-browserify": "0.0.1", - "url": "^0.11.0", + "url": "^0.11.4", "util": "^0.12.4", "vm-browserify": "^1.0.1" }, @@ -9584,6 +9472,7 @@ "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.1.7.tgz", "integrity": "sha512-hLj7fuMow6f0lbB0cD14Lz2xNjwsyruH251Pk4t/yIitCFJbmY1myuLlHm/q06aST4jg6EgAh74PIBBrRqpVAQ==", "dev": true, + "license": "MIT", "dependencies": { "chokidar": "^3.5.2", "debug": "^4", @@ -9759,9 +9648,9 @@ } }, "node_modules/nwsapi": { - "version": "2.2.12", - "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.12.tgz", - "integrity": "sha512-qXDmcVlZV4XRtKFzddidpfVP4oMSGhga+xdMc25mv8kaLUHtgzCDhUxkrN8exkGdTlLNaXj7CV3GtON7zuGZ+w==", + "version": "2.2.13", + "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.13.tgz", + "integrity": "sha512-cTGB9ptp9dY9A5VbMSe7fQBcl/tt22Vcqdq8+eN93rblOuE0aCFu4aZ2vMwct/2t+lFnosm8RkQW1I0Omb1UtQ==", "dev": true, "license": "MIT" }, @@ -10018,9 +9907,9 @@ } }, "node_modules/package-json-from-dist": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.0.tgz", - "integrity": "sha512-dATvCeZN/8wQsGywez1mzHtTlP22H8OEfPrVMLNr4/eGa+ijtLn/6M5f0dY8UKNrC2O9UCU6SSoG3qRKnt7STw==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", "dev": true, "license": "BlueOak-1.0.0" }, @@ -10088,13 +9977,13 @@ "license": "MIT" }, "node_modules/parse5": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz", - "integrity": "sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.2.0.tgz", + "integrity": "sha512-ZkDsAOcxsUMZ4Lz5fVciOehNcJ+Gb8gTzcA4yl3wnc273BAybYWrQ+Ks/OjCjSEpjvQkDSeZbybK9qj2VHHdGA==", "dev": true, "license": "MIT", "dependencies": { - "entities": "^4.4.0" + "entities": "^4.5.0" }, "funding": { "url": "https://github.com/inikulin/parse5?sponsor=1" @@ -10170,7 +10059,8 @@ "node_modules/path-to-regexp": { "version": "0.1.10", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.10.tgz", - "integrity": "sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==" + "integrity": "sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==", + "license": "MIT" }, "node_modules/path-type": { "version": "4.0.0", @@ -10248,6 +10138,12 @@ "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", "license": "MIT" }, + "node_modules/perfect-debounce": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/perfect-debounce/-/perfect-debounce-1.0.0.tgz", + "integrity": "sha512-xCy9V055GLEqoFaHoC1SoLIaLmWctgCUaBaWxDZ7/Zx4CTyX7cJQLJOok/orfjZAh9kEYpjJa4d0KcJmCbctZA==", + "license": "MIT" + }, "node_modules/performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", @@ -10255,9 +10151,10 @@ "license": "MIT" }, "node_modules/picocolors": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.0.tgz", - "integrity": "sha512-TQ92mBOW0l3LeMeyLV6mzy/kWr8lkd/hp3mTg7wYK7zJhuBStmGMBG0BdeDZS/dZx1IukaX6Bk11zcln25o1Aw==" + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" }, "node_modules/picomatch": { "version": "2.3.1", @@ -10285,6 +10182,7 @@ "version": "2.2.2", "resolved": "https://registry.npmjs.org/pinia/-/pinia-2.2.2.tgz", "integrity": "sha512-ja2XqFWZC36mupU4z1ZzxeTApV7DOw44cV4dhQ9sGwun+N89v/XP7+j7q6TanS1u1tdbK4r+1BUx7heMaIdagA==", + "license": "MIT", "dependencies": { "@vue/devtools-api": "^6.6.3", "vue-demi": "^0.14.10" @@ -10362,18 +10260,6 @@ "node": ">=10" } }, - "node_modules/pkg-types": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/pkg-types/-/pkg-types-1.2.0.tgz", - "integrity": "sha512-+ifYuSSqOQ8CqP4MbZA5hDpb97n3E8SVWdJe+Wms9kj745lmd3b7EZJiqvmLwAlmRfjrI7Hi5z3kdBJ93lFNPA==", - "dev": true, - "license": "MIT", - "dependencies": { - "confbox": "^0.1.7", - "mlly": "^1.7.1", - "pathe": "^1.1.2" - } - }, "node_modules/platform": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", @@ -10408,6 +10294,7 @@ "url": "https://github.com/sponsors/ai" } ], + "license": "MIT", "dependencies": { "nanoid": "^3.3.7", "picocolors": "^1.1.0", @@ -10531,9 +10418,9 @@ } }, "node_modules/postcss-selector-parser": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.1.tgz", - "integrity": "sha512-b4dlw/9V8A71rLIDsSwVmak9z2DuBUB7CA1/wSdelNEzqsjoSPeADTWNO09lpH49Diy3/JIZ2bSPB1dI3LJCHg==", + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz", + "integrity": "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==", "dev": true, "license": "MIT", "dependencies": { @@ -10629,34 +10516,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/pretty-format": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", - "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "^29.6.3", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/pretty-format/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, "node_modules/process": { "version": "0.11.10", "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", @@ -10735,10 +10594,9 @@ } }, "node_modules/proxy-from-env": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", - "dev": true, + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.0.0.tgz", + "integrity": "sha512-F2JHgJQ1iqwnHDcQjVBsq3n/uoaFL+iPW/eAeL7kVxy/2RrWaN4WroKjjvbsoRtv0ftelNyC01bjRhn/bhcf4A==", "license": "MIT" }, "node_modules/ps-tree": { @@ -10793,9 +10651,9 @@ "license": "MIT" }, "node_modules/pump": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", - "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", + "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==", "license": "MIT", "dependencies": { "end-of-stream": "^1.1.0", @@ -10812,12 +10670,12 @@ } }, "node_modules/qs": { - "version": "6.10.4", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.4.tgz", - "integrity": "sha512-OQiU+C+Ds5qiH91qh/mg0w+8nwQuLjM4F4M/PbmhDOoYehPh+Fb0bDjtR1sOvy7YKxvj28Y/M0PhP5uVX0kB+g==", - "license": "BSD-3-Clause", + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "license": "BSD-3-Clause", "dependencies": { - "side-channel": "^1.0.4" + "side-channel": "^1.0.6" }, "engines": { "node": ">=0.6" @@ -10891,6 +10749,7 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", "engines": { "node": ">= 0.6" } @@ -10899,6 +10758,7 @@ "version": "2.5.2", "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", + "license": "MIT", "dependencies": { "bytes": "3.1.2", "http-errors": "2.0.0", @@ -10913,6 +10773,7 @@ "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "license": "MIT", "dependencies": { "safer-buffer": ">= 2.1.2 < 3" }, @@ -10950,13 +10811,6 @@ "node": ">=0.10.0" } }, - "node_modules/react-is": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", - "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", - "dev": true, - "license": "MIT" - }, "node_modules/read-cache": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz", @@ -11213,13 +11067,13 @@ "license": "Unlicense" }, "node_modules/rollup": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.20.0.tgz", - "integrity": "sha512-6rbWBChcnSGzIlXeIdNIZTopKYad8ZG8ajhl78lGRLsI2rX8IkaotQhVas2Ma+GPxJav19wrSzvRvuiv0YKzWw==", + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.24.0.tgz", + "integrity": "sha512-DOmrlGSXNk1DM0ljiQA+i+o0rSLhtii1je5wgk60j49d1jHT5YYttBv1iWOnYSTG+fZZESUOSNiAl89SIet+Cg==", "dev": true, "license": "MIT", "dependencies": { - "@types/estree": "1.0.5" + "@types/estree": "1.0.6" }, "bin": { "rollup": "dist/bin/rollup" @@ -11229,22 +11083,22 @@ "npm": ">=8.0.0" }, "optionalDependencies": { - "@rollup/rollup-android-arm-eabi": "4.20.0", - "@rollup/rollup-android-arm64": "4.20.0", - "@rollup/rollup-darwin-arm64": "4.20.0", - "@rollup/rollup-darwin-x64": "4.20.0", - "@rollup/rollup-linux-arm-gnueabihf": "4.20.0", - "@rollup/rollup-linux-arm-musleabihf": "4.20.0", - "@rollup/rollup-linux-arm64-gnu": "4.20.0", - "@rollup/rollup-linux-arm64-musl": "4.20.0", - "@rollup/rollup-linux-powerpc64le-gnu": "4.20.0", - "@rollup/rollup-linux-riscv64-gnu": "4.20.0", - "@rollup/rollup-linux-s390x-gnu": "4.20.0", - "@rollup/rollup-linux-x64-gnu": "4.20.0", - "@rollup/rollup-linux-x64-musl": "4.20.0", - "@rollup/rollup-win32-arm64-msvc": "4.20.0", - "@rollup/rollup-win32-ia32-msvc": "4.20.0", - "@rollup/rollup-win32-x64-msvc": "4.20.0", + "@rollup/rollup-android-arm-eabi": "4.24.0", + "@rollup/rollup-android-arm64": "4.24.0", + "@rollup/rollup-darwin-arm64": "4.24.0", + "@rollup/rollup-darwin-x64": "4.24.0", + "@rollup/rollup-linux-arm-gnueabihf": "4.24.0", + "@rollup/rollup-linux-arm-musleabihf": "4.24.0", + "@rollup/rollup-linux-arm64-gnu": "4.24.0", + "@rollup/rollup-linux-arm64-musl": "4.24.0", + "@rollup/rollup-linux-powerpc64le-gnu": "4.24.0", + "@rollup/rollup-linux-riscv64-gnu": "4.24.0", + "@rollup/rollup-linux-s390x-gnu": "4.24.0", + "@rollup/rollup-linux-x64-gnu": "4.24.0", + "@rollup/rollup-linux-x64-musl": "4.24.0", + "@rollup/rollup-win32-arm64-msvc": "4.24.0", + "@rollup/rollup-win32-ia32-msvc": "4.24.0", + "@rollup/rollup-win32-x64-msvc": "4.24.0", "fsevents": "~2.3.2" } }, @@ -11355,6 +11209,7 @@ "version": "0.19.0", "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", + "license": "MIT", "dependencies": { "debug": "2.6.9", "depd": "2.0.0", @@ -11378,6 +11233,7 @@ "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", "dependencies": { "ms": "2.0.0" } @@ -11385,7 +11241,29 @@ "node_modules/send/node_modules/debug/node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/send/node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/send/node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } }, "node_modules/serialize-javascript": { "version": "6.0.2", @@ -11398,50 +11276,15 @@ } }, "node_modules/serve-static": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.0.tgz", - "integrity": "sha512-pDLK8zwl2eKaYrs8mrPZBJua4hMplRWJ1tIFksVC3FtBEBnl8dxgeHtsaMS8DhS9i4fLObaon6ABoc4/hQGdPA==", + "version": "1.16.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", + "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", + "license": "MIT", "dependencies": { - "encodeurl": "~1.0.2", + "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "parseurl": "~1.3.3", - "send": "0.18.0" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/serve-static/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/serve-static/node_modules/debug/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" - }, - "node_modules/serve-static/node_modules/send": { - "version": "0.18.0", - "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", - "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", - "dependencies": { - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "0.5.2", - "http-errors": "2.0.0", - "mime": "1.6.0", - "ms": "2.1.3", - "on-finished": "2.4.1", - "range-parser": "~1.2.1", - "statuses": "2.0.1" + "send": "0.19.0" }, "engines": { "node": ">= 0.8.0" @@ -11484,7 +11327,8 @@ "node_modules/setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", - "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" }, "node_modules/sha.js": { "version": "2.4.11", @@ -11505,6 +11349,7 @@ "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.5.tgz", "integrity": "sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw==", "hasInstallScript": true, + "license": "Apache-2.0", "dependencies": { "color": "^4.2.3", "detect-libc": "^2.0.3", @@ -11744,6 +11589,7 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "license": "BSD-3-Clause", "engines": { "node": ">=0.10.0" } @@ -11775,11 +11621,20 @@ } }, "node_modules/spdx-license-ids": { - "version": "3.0.18", - "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.18.tgz", - "integrity": "sha512-xxRs31BqRYHwiMzudOrpSiHtZ8i/GeionCBDSilhYRj+9gIcI8wCZTlXZKu9vZIVqViP3dcp9qE5G6AlIaD+TQ==", + "version": "3.0.20", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.20.tgz", + "integrity": "sha512-jg25NiDV/1fLtSgEgyvVyDunvaNHbuwF9lfNV17gSmPFAlYzdfNBlLtLzXTevwkPj7DhGbmN9VnmJIgLnhvaBw==", "license": "CC0-1.0" }, + "node_modules/speakingurl": { + "version": "14.0.1", + "resolved": "https://registry.npmjs.org/speakingurl/-/speakingurl-14.0.1.tgz", + "integrity": "sha512-1POYv7uv2gXoyGFpBCmpDVSNV74IfsWlDW216UPjbWufNf+bSU6GdbDsxdcxtfwb4xlI3yxzOTKClUosxARYrQ==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/split": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/split/-/split-0.3.3.tgz", @@ -11832,20 +11687,20 @@ "license": "MIT" }, "node_modules/start-server-and-test": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/start-server-and-test/-/start-server-and-test-2.0.5.tgz", - "integrity": "sha512-2CV4pz69NJVJKQmJeSr+O+SPtOreu0yxvhPmSXclzmAKkPREuMabyMh+Txpzemjx0RDzXOcG2XkhiUuxjztSQw==", + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/start-server-and-test/-/start-server-and-test-2.0.8.tgz", + "integrity": "sha512-v2fV6NV2F7tL1ocwfI4Wpait+IKjRbT5l3ZZ+ZikXdMLmxYsS8ynGAsCQAUVXkVyGyS+UibsRnvgHkMvJIvCsw==", "dev": true, "license": "MIT", "dependencies": { "arg": "^5.0.2", "bluebird": "3.7.2", "check-more-types": "2.24.0", - "debug": "4.3.6", + "debug": "4.3.7", "execa": "5.1.1", "lazy-ass": "1.6.0", "ps-tree": "1.2.0", - "wait-on": "7.2.0" + "wait-on": "8.0.1" }, "bin": { "server-test": "src/bin/start.js", @@ -11856,23 +11711,6 @@ "node": ">=16" } }, - "node_modules/start-server-and-test/node_modules/debug": { - "version": "4.3.6", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.6.tgz", - "integrity": "sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg==", - "dev": true, - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, "node_modules/start-server-and-test/node_modules/execa": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", @@ -11920,12 +11758,6 @@ "node": ">=10.17.0" } }, - "node_modules/start-server-and-test/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -11977,9 +11809,9 @@ } }, "node_modules/streamx": { - "version": "2.18.0", - "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.18.0.tgz", - "integrity": "sha512-LLUC1TWdjVdn1weXGcSxyTR3T4+acB6tVGXT95y0nGbca4t4o/ng1wKAGTljm9VicuCVLvRlqFYXYy5GwgM7sQ==", + "version": "2.20.1", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.20.1.tgz", + "integrity": "sha512-uTa0mU6WUC65iUvzKH4X9hEdvSW7rbPxPtwfWiLMSj3qTdQbAiUboZTxauKfpFuGIGa1C2BYijZ7wgdUXICJhA==", "license": "MIT", "dependencies": { "fast-fifo": "^1.3.2", @@ -12096,19 +11928,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/strip-literal": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/strip-literal/-/strip-literal-2.1.0.tgz", - "integrity": "sha512-Op+UycaUt/8FbN/Z2TWPBLge3jWrP3xj10f3fnYxf052bKuS3EKs1ZQcVGjnEMdsNVAM+plXRdmjrZ/KgG3Skw==", - "dev": true, - "license": "MIT", - "dependencies": { - "js-tokens": "^9.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/antfu" - } - }, "node_modules/strtok3": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-6.3.0.tgz", @@ -12197,6 +12016,18 @@ "node": ">=16 || 14 >=14.17" } }, + "node_modules/superjson": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/superjson/-/superjson-2.2.1.tgz", + "integrity": "sha512-8iGv75BYOa0xRJHK5vRLEjE2H/i4lulTjzpUXic3Eg8akftYjkmQDa8JARQ42rlczXyFR3IeRoeFCc7RxHsYZA==", + "license": "MIT", + "dependencies": { + "copy-anything": "^3.0.2" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -12320,9 +12151,9 @@ "license": "MIT" }, "node_modules/synckit": { - "version": "0.9.1", - "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.9.1.tgz", - "integrity": "sha512-7gr8p9TQP6RAHusBOSLs46F4564ZrjV8xFmw5zCmgmhGUcw2hxsShhJ6CEiHQMgPDwAQ1fWHPM0ypc4RMAig4A==", + "version": "0.9.2", + "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.9.2.tgz", + "integrity": "sha512-vrozgXDQwYO72vHjUb/HnFbQx1exDjoKzqx23aXEg2a9VIg2TSFZ8FmeZpTjUCFMYw7mpX4BE2SFu8wI7asYsw==", "dev": true, "license": "MIT", "dependencies": { @@ -12373,10 +12204,11 @@ } }, "node_modules/tailwindcss": { - "version": "3.4.13", - "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.4.13.tgz", - "integrity": "sha512-KqjHOJKogOUt5Bs752ykCeiwvi0fKVkr5oqsFNt/8px/tA8scFPIlkygsf6jXrfCqGHz7VflA6+yytWuM+XhFw==", + "version": "3.4.14", + "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.4.14.tgz", + "integrity": "sha512-IcSvOcTRcUtQQ7ILQL5quRDg7Xs93PdJEk1ZLbhhvJc7uj/OAhYOnruEiwnGgBvUtaUAJ8/mhSw1o8L2jCiENA==", "dev": true, + "license": "MIT", "dependencies": { "@alloc/quick-lru": "^5.2.0", "arg": "^5.0.2", @@ -12461,13 +12293,10 @@ } }, "node_modules/text-decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.1.1.tgz", - "integrity": "sha512-8zll7REEv4GDD3x4/0pW+ppIxSNs7H1J10IKFZsuOMscumCdM2a+toDGLPA3T+1+fLBql4zbt5z83GEQGGV5VA==", - "license": "Apache-2.0", - "dependencies": { - "b4a": "^1.6.4" - } + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.1.tgz", + "integrity": "sha512-x9v3H/lTKIJKQQe7RPQkLfKAnc9lUTkWDypIQgTzPJAq+5/GCDHonmshfvlsNSj58yyshbIJJDLmU15qNERrXQ==", + "license": "Apache-2.0" }, "node_modules/text-table": { "version": "0.2.0", @@ -12546,14 +12375,21 @@ "integrity": "sha512-XPaBkWQJdsf3pLKJV9p4qN/S+fm2Oj8AIPo1BTUhg5oxkvm9+SVEGFdhyOz7tTdUTfvxMiAs4sp6/eZO2Ew+pw==", "license": "MIT" }, + "node_modules/tinyexec": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.1.tgz", + "integrity": "sha512-WiCJLEECkO18gwqIp6+hJg0//p23HXp4S+gGtAKu3mI2F2/sXC4FvHvXvB0zJVVaTPhx1/tOwdbRsa1sOBIKqQ==", + "dev": true, + "license": "MIT" + }, "node_modules/tinypool": { - "version": "0.8.4", - "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-0.8.4.tgz", - "integrity": "sha512-i11VH5gS6IFeLY3gMBQ00/MmLncVP7JLXOw1vlgkytLmJK7QnEr7NXf0LBdxfmNPAeyetukOk0bOYrJrFGjYJQ==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-1.0.1.tgz", + "integrity": "sha512-URZYihUbRPcGv95En+sz6MfghfIc2OJ1sv/RmhWZLouPY0/8Vo80viwPvg3dlaS9fuq7fQMEfgRRK7BBZThBEA==", "dev": true, "license": "MIT", "engines": { - "node": ">=14.0.0" + "node": "^18.0.0 || >=20.0.0" } }, "node_modules/tinyrainbow": { @@ -12567,9 +12403,9 @@ } }, "node_modules/tinyspy": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-2.2.1.tgz", - "integrity": "sha512-KYad6Vy5VDWV4GH3fjpseMQ/XU2BhIYP7Vzd0LG44qRWm/Yt2WCOTicFdvmgo6gWaqooMQCawTtILVQJupKu7A==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-3.0.2.tgz", + "integrity": "sha512-n1cw8k1k0x4pgA2+9XrOkFydTerNcJ1zWCO5Nn9scWHTD+5tp8dghT2x1uduQePZTZgd3Tupf+x9BxJjeJi77Q==", "dev": true, "license": "MIT", "engines": { @@ -12585,6 +12421,26 @@ "@popperjs/core": "^2.9.0" } }, + "node_modules/tldts": { + "version": "6.1.55", + "resolved": "https://registry.npmjs.org/tldts/-/tldts-6.1.55.tgz", + "integrity": "sha512-HxQR/9roQ07Pwc8RyyrJMAxRz5/ssoF3qIPPUiIo3zUt6yMdmYZjM2OZIFMiZ3jHyz9jrGHEHuQZrUhoc1LkDw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tldts-core": "^6.1.55" + }, + "bin": { + "tldts": "bin/cli.js" + } + }, + "node_modules/tldts-core": { + "version": "6.1.55", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-6.1.55.tgz", + "integrity": "sha512-BL+BuKHHaOpntE5BGI6naXjULU6aRlgaYdfDHR3T/hdbNTWkWUZ9yuc11wGnwgpvRwlyUiIK+QohYK3olaVU6Q==", + "dev": true, + "license": "MIT" + }, "node_modules/tmp": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz", @@ -12604,14 +12460,6 @@ "tmp": "^0.2.0" } }, - "node_modules/to-fast-properties": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", - "integrity": "sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==", - "engines": { - "node": ">=4" - } - }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -12629,6 +12477,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", "engines": { "node": ">=0.6" } @@ -12703,6 +12552,15 @@ "node": ">=18" } }, + "node_modules/tree-kill": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz", + "integrity": "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==", + "license": "MIT", + "bin": { + "tree-kill": "cli.js" + } + }, "node_modules/ts-api-utils": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.3.0.tgz", @@ -12801,9 +12659,10 @@ } }, "node_modules/tslib": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", - "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.0.tgz", + "integrity": "sha512-jWVzBLplnCmoaTr13V9dYbiQ99wvZRd0vNWaDRg+aVYRcjDF3nDksxFDE/+fkXnKhpnUUkmx5pK/v8mCtLVqZA==", + "license": "0BSD" }, "node_modules/tty-browserify": { "version": "0.0.1", @@ -12843,16 +12702,6 @@ "node": ">= 0.8.0" } }, - "node_modules/type-detect": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.1.0.tgz", - "integrity": "sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, "node_modules/type-fest": { "version": "0.20.2", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", @@ -12870,6 +12719,7 @@ "version": "1.6.18", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "license": "MIT", "dependencies": { "media-typer": "0.3.0", "mime-types": "~2.1.24" @@ -12879,10 +12729,11 @@ } }, "node_modules/typescript": { - "version": "5.6.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.2.tgz", - "integrity": "sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==", + "version": "5.5.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.4.tgz", + "integrity": "sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==", "devOptional": true, + "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -12928,13 +12779,6 @@ "node": ">=8" } }, - "node_modules/ufo": { - "version": "1.5.4", - "resolved": "https://registry.npmjs.org/ufo/-/ufo-1.5.4.tgz", - "integrity": "sha512-UsUk3byDzKd04EyoZ7U4DOlxQaD14JUKQl6/P7wiX4FNvUfm3XL246n9W5AmqwW5RSFJ27NAuM0iLscAOYUiGQ==", - "dev": true, - "license": "MIT" - }, "node_modules/undefsafe": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.5.tgz", @@ -12945,7 +12789,8 @@ "node_modules/undici-types": { "version": "6.19.8", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", - "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==" + "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", + "license": "MIT" }, "node_modules/universalify": { "version": "2.0.1", @@ -12975,9 +12820,9 @@ } }, "node_modules/update-browserslist-db": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.0.tgz", - "integrity": "sha512-EdRAaAyk2cUE1wOf2DkEhzxqOQvFOoRJFNS6NeyJ01Gp2beMRpBAINjM2iDXE3KCuKhwnvHIQCJm6ThL2Z+HzQ==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.1.tgz", + "integrity": "sha512-R8UzCaa9Az+38REPiJ1tXlImTJXlVfgHZsglwBD/k6nj76ctsH1E3q4doGrukiLQd3sGQYu56r5+lo5r94l29A==", "dev": true, "funding": [ { @@ -12995,8 +12840,8 @@ ], "license": "MIT", "dependencies": { - "escalade": "^3.1.2", - "picocolors": "^1.0.1" + "escalade": "^3.2.0", + "picocolors": "^1.1.0" }, "bin": { "update-browserslist-db": "cli.js" @@ -13046,22 +12891,6 @@ "dev": true, "license": "MIT" }, - "node_modules/url/node_modules/qs": { - "version": "6.13.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", - "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "side-channel": "^1.0.6" - }, - "engines": { - "node": ">=0.6" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/util": { "version": "0.12.5", "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", @@ -13131,22 +12960,31 @@ } }, "node_modules/vee-validate": { - "version": "4.13.2", - "resolved": "https://registry.npmjs.org/vee-validate/-/vee-validate-4.13.2.tgz", - "integrity": "sha512-HlpR/6MJ92TW9f135umMZKUqdd/tFQTxLNSf2ImbU4Y/MlLVAUpF1l64VdjTOhbClAqPjCb5p/SqHDxLpUHXrw==", + "version": "4.14.4", + "resolved": "https://registry.npmjs.org/vee-validate/-/vee-validate-4.14.4.tgz", + "integrity": "sha512-Eg1FK2IqxqmujjN8Oehvqfx+QynyrIVc6Su2h/HQI/m3VEyUXW4wlB0h0sLkNizsEWlTj7Kwxn5Q2UWCr7RTQg==", "license": "MIT", "dependencies": { - "@vue/devtools-api": "^6.6.1", + "@vue/devtools-api": "^7.5.2", "type-fest": "^4.8.3" }, "peerDependencies": { "vue": "^3.4.26" } }, + "node_modules/vee-validate/node_modules/@vue/devtools-api": { + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/@vue/devtools-api/-/devtools-api-7.5.4.tgz", + "integrity": "sha512-j9UC/KeYUNZ6AyCJxBROBCbogB5YHW6PZv9VnCNp2ntE4rq426Lfc8WP5B9V+rXBwqWmrgZTGYBa31CBSxdAUg==", + "license": "MIT", + "dependencies": { + "@vue/devtools-kit": "^7.5.4" + } + }, "node_modules/vee-validate/node_modules/type-fest": { - "version": "4.23.0", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.23.0.tgz", - "integrity": "sha512-ZiBujro2ohr5+Z/hZWHESLz3g08BBdrdLMieYFULJO+tWc437sn8kQsWLJoZErY8alNhxre9K4p3GURAG11n+w==", + "version": "4.26.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.26.1.tgz", + "integrity": "sha512-yOGpmOAL7CkKe/91I5O3gPICmJNLJ1G4zFYVAsRHg7M64biSnPtRj0WNQt++bRkjYOqjWXrhnUw1utzmVErAdg==", "license": "(MIT OR CC0-1.0)", "engines": { "node": ">=16" @@ -13170,10 +13008,11 @@ } }, "node_modules/vite": { - "version": "5.4.6", - "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.6.tgz", - "integrity": "sha512-IeL5f8OO5nylsgzd9tq4qD2QqI0k2CQLGrWD0rCN0EQJZpBK5vJAx0I+GDkMOXxQX/OfFHMuLIx6ddAxGX/k+Q==", + "version": "5.4.10", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.10.tgz", + "integrity": "sha512-1hvaPshuPUtxeQ0hsVH3Mud0ZanOLwVTneA1EgbAM5LhaZEqyPWGRQ7BtaMvUrTDeEaC8pxtj6a6jku3x4z6SQ==", "dev": true, + "license": "MIT", "dependencies": { "esbuild": "^0.21.3", "postcss": "^8.4.43", @@ -13229,16 +13068,15 @@ } }, "node_modules/vite-node": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-1.6.0.tgz", - "integrity": "sha512-de6HJgzC+TFzOu0NTC4RAIsyf/DY/ibWDYQUcuEA84EMHhcefTUGkjFHKKEJhQN4A+6I0u++kr3l36ZF2d7XRw==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-2.1.3.tgz", + "integrity": "sha512-I1JadzO+xYX887S39Do+paRePCKoiDrWRRjp9kkG5he0t7RXNvPAJPCQSJqbGN4uCrFFeS3Kj3sLqY8NMYBEdA==", "dev": true, "license": "MIT", "dependencies": { "cac": "^6.7.14", - "debug": "^4.3.4", - "pathe": "^1.1.1", - "picocolors": "^1.0.0", + "debug": "^4.3.6", + "pathe": "^1.1.2", "vite": "^5.0.0" }, "bin": { @@ -13269,32 +13107,31 @@ } }, "node_modules/vitest": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-1.6.0.tgz", - "integrity": "sha512-H5r/dN06swuFnzNFhq/dnz37bPXnq8xB2xB5JOVk8K09rUtoeNN+LHWkoQ0A/i3hvbUKKcCei9KpbxqHMLhLLA==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-2.1.3.tgz", + "integrity": "sha512-Zrxbg/WiIvUP2uEzelDNTXmEMJXuzJ1kCpbDvaKByFA9MNeO95V+7r/3ti0qzJzrxdyuUw5VduN7k+D3VmVOSA==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/expect": "1.6.0", - "@vitest/runner": "1.6.0", - "@vitest/snapshot": "1.6.0", - "@vitest/spy": "1.6.0", - "@vitest/utils": "1.6.0", - "acorn-walk": "^8.3.2", - "chai": "^4.3.10", - "debug": "^4.3.4", - "execa": "^8.0.1", - "local-pkg": "^0.5.0", - "magic-string": "^0.30.5", - "pathe": "^1.1.1", - "picocolors": "^1.0.0", - "std-env": "^3.5.0", - "strip-literal": "^2.0.0", - "tinybench": "^2.5.1", - "tinypool": "^0.8.3", + "@vitest/expect": "2.1.3", + "@vitest/mocker": "2.1.3", + "@vitest/pretty-format": "^2.1.3", + "@vitest/runner": "2.1.3", + "@vitest/snapshot": "2.1.3", + "@vitest/spy": "2.1.3", + "@vitest/utils": "2.1.3", + "chai": "^5.1.1", + "debug": "^4.3.6", + "magic-string": "^0.30.11", + "pathe": "^1.1.2", + "std-env": "^3.7.0", + "tinybench": "^2.9.0", + "tinyexec": "^0.3.0", + "tinypool": "^1.0.0", + "tinyrainbow": "^1.2.0", "vite": "^5.0.0", - "vite-node": "1.6.0", - "why-is-node-running": "^2.2.2" + "vite-node": "2.1.3", + "why-is-node-running": "^2.3.0" }, "bin": { "vitest": "vitest.mjs" @@ -13308,8 +13145,8 @@ "peerDependencies": { "@edge-runtime/vm": "*", "@types/node": "^18.0.0 || >=20.0.0", - "@vitest/browser": "1.6.0", - "@vitest/ui": "1.6.0", + "@vitest/browser": "2.1.3", + "@vitest/ui": "2.1.3", "happy-dom": "*", "jsdom": "*" }, @@ -13334,295 +13171,78 @@ } } }, - "node_modules/vitest/node_modules/assertion-error": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz", - "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==", + "node_modules/vm-browserify": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-1.1.2.tgz", + "integrity": "sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==", "dev": true, - "license": "MIT", - "engines": { - "node": "*" - } + "license": "MIT" }, - "node_modules/vitest/node_modules/chai": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/chai/-/chai-4.5.0.tgz", - "integrity": "sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw==", + "node_modules/vscode-uri": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-3.0.8.tgz", + "integrity": "sha512-AyFQ0EVmsOZOlAnxoFOGOq1SQDWAB7C6aqMGS23svWAllfOaxbuFvcT8D1i8z3Gyn8fraVeZNNmN6e9bxxXkKw==", "dev": true, + "license": "MIT" + }, + "node_modules/vue": { + "version": "3.5.12", + "resolved": "https://registry.npmjs.org/vue/-/vue-3.5.12.tgz", + "integrity": "sha512-CLVZtXtn2ItBIi/zHZ0Sg1Xkb7+PU32bJJ8Bmy7ts3jxXTcbfsEfBivFYYWz1Hur+lalqGAh65Coin0r+HRUfg==", "license": "MIT", "dependencies": { - "assertion-error": "^1.1.0", - "check-error": "^1.0.3", - "deep-eql": "^4.1.3", - "get-func-name": "^2.0.2", - "loupe": "^2.3.6", - "pathval": "^1.1.1", - "type-detect": "^4.1.0" + "@vue/compiler-dom": "3.5.12", + "@vue/compiler-sfc": "3.5.12", + "@vue/runtime-dom": "3.5.12", + "@vue/server-renderer": "3.5.12", + "@vue/shared": "3.5.12" }, - "engines": { - "node": ">=4" + "peerDependencies": { + "typescript": "*" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } } }, - "node_modules/vitest/node_modules/check-error": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.3.tgz", - "integrity": "sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg==", + "node_modules/vue-component-type-helpers": { + "version": "2.1.6", + "resolved": "https://registry.npmjs.org/vue-component-type-helpers/-/vue-component-type-helpers-2.1.6.tgz", + "integrity": "sha512-ng11B8B/ZADUMMOsRbqv0arc442q7lifSubD0v8oDXIFoMg/mXwAPUunrroIDkY+mcD0dHKccdaznSVp8EoX3w==", "dev": true, + "license": "MIT" + }, + "node_modules/vue-demi": { + "version": "0.14.10", + "resolved": "https://registry.npmjs.org/vue-demi/-/vue-demi-0.14.10.tgz", + "integrity": "sha512-nMZBOwuzabUO0nLgIcc6rycZEebF6eeUfaiQx9+WSk8e29IbLvPU9feI6tqW4kTo3hvoYAJkMh8n8D0fuISphg==", + "hasInstallScript": true, "license": "MIT", - "dependencies": { - "get-func-name": "^2.0.2" + "bin": { + "vue-demi-fix": "bin/vue-demi-fix.js", + "vue-demi-switch": "bin/vue-demi-switch.js" }, "engines": { - "node": "*" + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + }, + "peerDependencies": { + "@vue/composition-api": "^1.0.0-rc.1", + "vue": "^3.0.0-0 || ^2.6.0" + }, + "peerDependenciesMeta": { + "@vue/composition-api": { + "optional": true + } } }, - "node_modules/vitest/node_modules/deep-eql": { - "version": "4.1.4", - "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-4.1.4.tgz", - "integrity": "sha512-SUwdGfqdKOwxCPeVYjwSyRpJ7Z+fhpwIAtmCUdZIWZ/YP5R9WAsyuSgpLVDi9bjWoN2LXHNss/dk3urXtdQxGg==", - "dev": true, - "license": "MIT", - "dependencies": { - "type-detect": "^4.0.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/vitest/node_modules/execa": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/execa/-/execa-8.0.1.tgz", - "integrity": "sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg==", - "dev": true, - "license": "MIT", - "dependencies": { - "cross-spawn": "^7.0.3", - "get-stream": "^8.0.1", - "human-signals": "^5.0.0", - "is-stream": "^3.0.0", - "merge-stream": "^2.0.0", - "npm-run-path": "^5.1.0", - "onetime": "^6.0.0", - "signal-exit": "^4.1.0", - "strip-final-newline": "^3.0.0" - }, - "engines": { - "node": ">=16.17" - }, - "funding": { - "url": "https://github.com/sindresorhus/execa?sponsor=1" - } - }, - "node_modules/vitest/node_modules/get-stream": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-8.0.1.tgz", - "integrity": "sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/vitest/node_modules/human-signals": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-5.0.0.tgz", - "integrity": "sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=16.17.0" - } - }, - "node_modules/vitest/node_modules/is-stream": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-3.0.0.tgz", - "integrity": "sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/vitest/node_modules/loupe": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/loupe/-/loupe-2.3.7.tgz", - "integrity": "sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA==", - "dev": true, - "license": "MIT", - "dependencies": { - "get-func-name": "^2.0.1" - } - }, - "node_modules/vitest/node_modules/mimic-fn": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz", - "integrity": "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/vitest/node_modules/npm-run-path": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-5.3.0.tgz", - "integrity": "sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "path-key": "^4.0.0" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/vitest/node_modules/onetime": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-6.0.0.tgz", - "integrity": "sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "mimic-fn": "^4.0.0" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/vitest/node_modules/path-key": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-4.0.0.tgz", - "integrity": "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/vitest/node_modules/pathval": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.1.tgz", - "integrity": "sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/vitest/node_modules/signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/vitest/node_modules/strip-final-newline": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-3.0.0.tgz", - "integrity": "sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/vm-browserify": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-1.1.2.tgz", - "integrity": "sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/vscode-uri": { - "version": "3.0.8", - "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-3.0.8.tgz", - "integrity": "sha512-AyFQ0EVmsOZOlAnxoFOGOq1SQDWAB7C6aqMGS23svWAllfOaxbuFvcT8D1i8z3Gyn8fraVeZNNmN6e9bxxXkKw==", - "dev": true - }, - "node_modules/vue": { - "version": "3.4.37", - "resolved": "https://registry.npmjs.org/vue/-/vue-3.4.37.tgz", - "integrity": "sha512-3vXvNfkKTBsSJ7JP+LyR7GBuwQuckbWvuwAid3xbqK9ppsKt/DUvfqgZ48fgOLEfpy1IacL5f8QhUVl77RaI7A==", - "dependencies": { - "@vue/compiler-dom": "3.4.37", - "@vue/compiler-sfc": "3.4.37", - "@vue/runtime-dom": "3.4.37", - "@vue/server-renderer": "3.4.37", - "@vue/shared": "3.4.37" - }, - "peerDependencies": { - "typescript": "*" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } - } - }, - "node_modules/vue-component-type-helpers": { - "version": "2.0.29", - "resolved": "https://registry.npmjs.org/vue-component-type-helpers/-/vue-component-type-helpers-2.0.29.tgz", - "integrity": "sha512-58i+ZhUAUpwQ+9h5Hck0D+jr1qbYl4voRt5KffBx8qzELViQ4XdT/Tuo+mzq8u63teAG8K0lLaOiL5ofqW38rg==", - "dev": true, - "license": "MIT" - }, - "node_modules/vue-demi": { - "version": "0.14.10", - "resolved": "https://registry.npmjs.org/vue-demi/-/vue-demi-0.14.10.tgz", - "integrity": "sha512-nMZBOwuzabUO0nLgIcc6rycZEebF6eeUfaiQx9+WSk8e29IbLvPU9feI6tqW4kTo3hvoYAJkMh8n8D0fuISphg==", - "hasInstallScript": true, - "license": "MIT", - "bin": { - "vue-demi-fix": "bin/vue-demi-fix.js", - "vue-demi-switch": "bin/vue-demi-switch.js" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/antfu" - }, - "peerDependencies": { - "@vue/composition-api": "^1.0.0-rc.1", - "vue": "^3.0.0-0 || ^2.6.0" - }, - "peerDependenciesMeta": { - "@vue/composition-api": { - "optional": true - } - } - }, - "node_modules/vue-eslint-parser": { - "version": "9.4.3", - "resolved": "https://registry.npmjs.org/vue-eslint-parser/-/vue-eslint-parser-9.4.3.tgz", - "integrity": "sha512-2rYRLWlIpaiN8xbPiDyXZXRgLGOtWxERV7ND5fFAv5qo1D2N9Fu9MNajBNc6o13lZ+24DAWCkQCvj4klgmcITg==", + "node_modules/vue-eslint-parser": { + "version": "9.4.3", + "resolved": "https://registry.npmjs.org/vue-eslint-parser/-/vue-eslint-parser-9.4.3.tgz", + "integrity": "sha512-2rYRLWlIpaiN8xbPiDyXZXRgLGOtWxERV7ND5fFAv5qo1D2N9Fu9MNajBNc6o13lZ+24DAWCkQCvj4klgmcITg==", "dev": true, "license": "MIT", "dependencies": { @@ -13648,6 +13268,7 @@ "version": "4.4.5", "resolved": "https://registry.npmjs.org/vue-router/-/vue-router-4.4.5.tgz", "integrity": "sha512-4fKZygS8cH1yCyuabAXGUAsyi1b2/o/OKgu/RUb+znIYOxPRxdkytJEx+0wGcpBE1pX6vUgh5jwWOKRGvuA/7Q==", + "license": "MIT", "dependencies": { "@vue/devtools-api": "^6.6.4" }, @@ -13659,9 +13280,9 @@ } }, "node_modules/vue-tippy": { - "version": "6.4.4", - "resolved": "https://registry.npmjs.org/vue-tippy/-/vue-tippy-6.4.4.tgz", - "integrity": "sha512-0C5TSU482FvjhEeKrPkz08tzyC/KJC0CiEbm3yW9oS+n3fa03ajEzU2QcxI9oR6Hwlg8NOP0U6T4EsGuccq6YQ==", + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/vue-tippy/-/vue-tippy-6.5.0.tgz", + "integrity": "sha512-U44UDETTLuZWZGosagslEwgimWQdt1JVSxfWStVPnVdeqo2jo9X5zW3SB04k7JaTFosdgrDhFsUDrd6n42Nh7Q==", "license": "MIT", "dependencies": { "tippy.js": "^6.3.7" @@ -13671,9 +13292,9 @@ } }, "node_modules/vue-toast-notification": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/vue-toast-notification/-/vue-toast-notification-3.1.2.tgz", - "integrity": "sha512-oNRL/W9aaHoeScp+iTIW7k09vM16/+8aptp2maa+7qTB43JuxmAgKdXKFYtf+uvSNOYYq2BIWgLCeJ61pwom/A==", + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/vue-toast-notification/-/vue-toast-notification-3.1.3.tgz", + "integrity": "sha512-XNyWqwLIGBFfX5G9sK+clq3N3IPlhDjzNdbZaXkEElcotPlWs0wWZailk1vqhdtLYT/93Y4FHAVuzyatLmPZRA==", "license": "MIT", "engines": { "node": ">=12.15.0" @@ -13687,6 +13308,7 @@ "resolved": "https://registry.npmjs.org/vue-tsc/-/vue-tsc-2.1.6.tgz", "integrity": "sha512-f98dyZp5FOukcYmbFpuSCJ4Z0vHSOSmxGttZJCsFeX0M4w/Rsq0s4uKXjcSRsZqsRgQa6z7SfuO+y0HVICE57Q==", "dev": true, + "license": "MIT", "dependencies": { "@volar/typescript": "~2.4.1", "@vue/language-core": "2.1.6", @@ -13700,9 +13322,9 @@ } }, "node_modules/vue3-apexcharts": { - "version": "1.5.3", - "resolved": "https://registry.npmjs.org/vue3-apexcharts/-/vue3-apexcharts-1.5.3.tgz", - "integrity": "sha512-yaHTPoj0iVKAtEVg8wEwIwwvf0VG+lPYNufCf3txRzYQOqdKPoZaZ9P3Dj3X+2A1XY9O1kcTk9HVqvLo+rppvQ==", + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/vue3-apexcharts/-/vue3-apexcharts-1.7.0.tgz", + "integrity": "sha512-BmWoS8+x5XLCtk2ml7rLVO+QU+fjgQUUCjUXSFW9cNQpCMa5Z0eRPvZjvYLt5aDKNREtuZoidlG9WRjZ/Af7lA==", "license": "MIT", "peerDependencies": { "apexcharts": "> 3.0.0", @@ -13751,14 +13373,14 @@ } }, "node_modules/wait-on": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/wait-on/-/wait-on-7.2.0.tgz", - "integrity": "sha512-wCQcHkRazgjG5XoAq9jbTMLpNIjoSlZslrJ2+N9MxDsGEv1HnFoVjOCexL0ESva7Y9cu350j+DWADdk54s4AFQ==", + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/wait-on/-/wait-on-8.0.1.tgz", + "integrity": "sha512-1wWQOyR2LVVtaqrcIL2+OM+x7bkpmzVROa0Nf6FryXkS+er5Sa1kzFGjzZRqLnHa3n1rACFLeTwUqE1ETL9Mig==", "dev": true, "license": "MIT", "dependencies": { - "axios": "^1.6.1", - "joi": "^17.11.0", + "axios": "^1.7.7", + "joi": "^17.13.3", "lodash": "^4.17.21", "minimist": "^1.2.8", "rxjs": "^7.8.1" @@ -14043,9 +13665,9 @@ "license": "ISC" }, "node_modules/yaml": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.5.0.tgz", - "integrity": "sha512-2wWLbGbYDiSqqIKoPjar3MPgB94ErzCtrNE1FdqGuaO0pi2JGjmE8aW8TDZwzU7vuxcGRdL/4gPQwQ7hD5AMSw==", + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.6.0.tgz", + "integrity": "sha512-a6ae//JvKDEra2kdi1qzCyrJW/WZCgFi8ydDV+eXExl95t+5R+ijnqHJbz9tmMh8FUjx3iv2fCQ4dclAQlO2UQ==", "dev": true, "license": "ISC", "bin": { @@ -14178,7 +13800,7 @@ "uuid": "10" }, "devDependencies": { - "@types/chai": "4", + "@types/chai": "5", "@types/cors": "2", "@types/express-ws": "3", "@types/mocha": "10", @@ -14199,7 +13821,7 @@ "cypress": "13", "d3": "7", "immutable": "4", - "pinia": "2", + "pinia": "<2.2.3", "pinia-plugin-persistedstate-2": "2", "vee-validate": "4", "vue": "3", @@ -14210,7 +13832,7 @@ "yup": "1" }, "devDependencies": { - "@pinia/testing": "0.1", + "@pinia/testing": "<0.1.6", "@tsconfig/node20": "20", "@types/d3": "7", "@types/jsdom": "21", @@ -14233,340 +13855,18 @@ "vue3-spinners": "1" } }, - "webapp/node_modules/@vitest/expect": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-2.0.5.tgz", - "integrity": "sha512-yHZtwuP7JZivj65Gxoi8upUN2OzHTi3zVfjwdpu2WrvCZPLwsJ2Ey5ILIPccoW23dd/zQBlJ4/dhi7DWNyXCpA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/spy": "2.0.5", - "@vitest/utils": "2.0.5", - "chai": "^5.1.1", - "tinyrainbow": "^1.2.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "webapp/node_modules/@vitest/runner": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-2.0.5.tgz", - "integrity": "sha512-TfRfZa6Bkk9ky4tW0z20WKXFEwwvWhRY+84CnSEtq4+3ZvDlJyY32oNTJtM7AW9ihW90tX/1Q78cb6FjoAs+ig==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/utils": "2.0.5", - "pathe": "^1.1.2" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "webapp/node_modules/@vitest/snapshot": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-2.0.5.tgz", - "integrity": "sha512-SgCPUeDFLaM0mIUHfaArq8fD2WbaXG/zVXjRupthYfYGzc8ztbFbu6dUNOblBG7XLMR1kEhS/DNnfCZ2IhdDew==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/pretty-format": "2.0.5", - "magic-string": "^0.30.10", - "pathe": "^1.1.2" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "webapp/node_modules/@vitest/spy": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-2.0.5.tgz", - "integrity": "sha512-c/jdthAhvJdpfVuaexSrnawxZz6pywlTPe84LUB2m/4t3rl2fTo9NFGBG4oWgaD+FTgDDV8hJ/nibT7IfH3JfA==", - "dev": true, - "license": "MIT", - "dependencies": { - "tinyspy": "^3.0.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "webapp/node_modules/@vitest/utils": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-2.0.5.tgz", - "integrity": "sha512-d8HKbqIcya+GR67mkZbrzhS5kKhtp8dQLcmRZLGTscGVg7yImT82cIrhtn2L8+VujWcy6KZweApgNmPsTAO/UQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/pretty-format": "2.0.5", - "estree-walker": "^3.0.3", - "loupe": "^3.1.1", - "tinyrainbow": "^1.2.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "webapp/node_modules/estree-walker": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", - "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/estree": "^1.0.0" - } - }, - "webapp/node_modules/execa": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/execa/-/execa-8.0.1.tgz", - "integrity": "sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg==", - "dev": true, - "license": "MIT", - "dependencies": { - "cross-spawn": "^7.0.3", - "get-stream": "^8.0.1", - "human-signals": "^5.0.0", - "is-stream": "^3.0.0", - "merge-stream": "^2.0.0", - "npm-run-path": "^5.1.0", - "onetime": "^6.0.0", - "signal-exit": "^4.1.0", - "strip-final-newline": "^3.0.0" - }, - "engines": { - "node": ">=16.17" - }, - "funding": { - "url": "https://github.com/sindresorhus/execa?sponsor=1" - } - }, - "webapp/node_modules/get-stream": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-8.0.1.tgz", - "integrity": "sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "webapp/node_modules/human-signals": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-5.0.0.tgz", - "integrity": "sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=16.17.0" - } - }, - "webapp/node_modules/is-stream": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-3.0.0.tgz", - "integrity": "sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "webapp/node_modules/mimic-fn": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz", - "integrity": "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "webapp/node_modules/npm-run-path": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-5.3.0.tgz", - "integrity": "sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "path-key": "^4.0.0" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "webapp/node_modules/onetime": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-6.0.0.tgz", - "integrity": "sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "mimic-fn": "^4.0.0" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "webapp/node_modules/path-key": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-4.0.0.tgz", - "integrity": "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "webapp/node_modules/signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "webapp/node_modules/strip-final-newline": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-3.0.0.tgz", - "integrity": "sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "webapp/node_modules/tinypool": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-1.0.0.tgz", - "integrity": "sha512-KIKExllK7jp3uvrNtvRBYBWBOAXSX8ZvoaD8T+7KB/QHIuoJW3Pmr60zucywjAlMb5TeXUkcs/MWeWLu0qvuAQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^18.0.0 || >=20.0.0" - } - }, - "webapp/node_modules/tinyspy": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-3.0.0.tgz", - "integrity": "sha512-q5nmENpTHgiPVd1cJDDc9cVoYN5x4vCvwT3FMilvKPKneCBZAxn2YWQjDF0UMcE9k0Cay1gBiDfTMU0g+mPMQA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=14.0.0" - } - }, - "webapp/node_modules/vite-node": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-2.0.5.tgz", - "integrity": "sha512-LdsW4pxj0Ot69FAoXZ1yTnA9bjGohr2yNBU7QKRxpz8ITSkhuDl6h3zS/tvgz4qrNjeRnvrWeXQ8ZF7Um4W00Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "cac": "^6.7.14", - "debug": "^4.3.5", - "pathe": "^1.1.2", - "tinyrainbow": "^1.2.0", - "vite": "^5.0.0" - }, - "bin": { - "vite-node": "vite-node.mjs" - }, - "engines": { - "node": "^18.0.0 || >=20.0.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "webapp/node_modules/vitest": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-2.0.5.tgz", - "integrity": "sha512-8GUxONfauuIdeSl5f9GTgVEpg5BTOlplET4WEDaeY2QBiN8wSm68vxN/tb5z405OwppfoCavnwXafiaYBC/xOA==", + "webapp/node_modules/@pinia/testing": { + "version": "0.1.5", "dev": true, "license": "MIT", "dependencies": { - "@ampproject/remapping": "^2.3.0", - "@vitest/expect": "2.0.5", - "@vitest/pretty-format": "^2.0.5", - "@vitest/runner": "2.0.5", - "@vitest/snapshot": "2.0.5", - "@vitest/spy": "2.0.5", - "@vitest/utils": "2.0.5", - "chai": "^5.1.1", - "debug": "^4.3.5", - "execa": "^8.0.1", - "magic-string": "^0.30.10", - "pathe": "^1.1.2", - "std-env": "^3.7.0", - "tinybench": "^2.8.0", - "tinypool": "^1.0.0", - "tinyrainbow": "^1.2.0", - "vite": "^5.0.0", - "vite-node": "2.0.5", - "why-is-node-running": "^2.3.0" - }, - "bin": { - "vitest": "vitest.mjs" - }, - "engines": { - "node": "^18.0.0 || >=20.0.0" + "vue-demi": "^0.14.10" }, "funding": { - "url": "https://opencollective.com/vitest" + "url": "https://github.com/sponsors/posva" }, "peerDependencies": { - "@edge-runtime/vm": "*", - "@types/node": "^18.0.0 || >=20.0.0", - "@vitest/browser": "2.0.5", - "@vitest/ui": "2.0.5", - "happy-dom": "*", - "jsdom": "*" - }, - "peerDependenciesMeta": { - "@edge-runtime/vm": { - "optional": true - }, - "@types/node": { - "optional": true - }, - "@vitest/browser": { - "optional": true - }, - "@vitest/ui": { - "optional": true - }, - "happy-dom": { - "optional": true - }, - "jsdom": { - "optional": true - } + "pinia": ">=2.2.1" } } } diff --git a/package.json b/package.json index 5b5012252..b1654529f 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "@types/debug": "4", "@typescript-eslint/eslint-plugin": "7", "eslint": "8", - "typescript": "5", + "typescript": "<5.6.0", "typescript-eslint": "7" } } diff --git a/server/package.json b/server/package.json index 3f9926c50..03e7c5900 100644 --- a/server/package.json +++ b/server/package.json @@ -24,7 +24,7 @@ "uuid": "10" }, "devDependencies": { - "@types/chai": "4", + "@types/chai": "5", "@types/cors": "2", "@types/express-ws": "3", "@types/mocha": "10", diff --git a/server/src/routes/task_router.ts b/server/src/routes/task_router.ts index ef608ac42..9948530fa 100644 --- a/server/src/routes/task_router.ts +++ b/server/src/routes/task_router.ts @@ -28,7 +28,10 @@ export class TaskRouter { // POST request to add a new task this.#expressRouter.post('/', (req, res) => { const raw: unknown = req.body - if (typeof raw !== 'object' || raw === null) return res.status(400) + if (typeof raw !== "object" || raw === null) { + res.status(400); + return; + } const { model: encoded, newTask }: Partial> = raw if (!( diff --git a/server/src/routes/training_router.ts b/server/src/routes/training_router.ts index d540d89da..3458558ec 100644 --- a/server/src/routes/training_router.ts +++ b/server/src/routes/training_router.ts @@ -23,7 +23,9 @@ export class TrainingRouter { this.#expressRouter = express.Router() wsApplier.applyTo(this.#expressRouter) - this.#expressRouter.get('/', (_, res) => res.send(`Disco ${this.trainingScheme} server \n`)) + this.#expressRouter.get("/", (_, res) => { + res.send(`Disco ${this.trainingScheme} server\n`); + }); /* delay listener because `this` (object) isn't fully constructed yet. * The lambda function inside process.nextTick is executed after the current operation diff --git a/webapp/package.json b/webapp/package.json index 82449fa03..342144ea7 100644 --- a/webapp/package.json +++ b/webapp/package.json @@ -18,7 +18,7 @@ "cypress": "13", "d3": "7", "immutable": "4", - "pinia": "2", + "pinia": "<2.2.3", "pinia-plugin-persistedstate-2": "2", "vee-validate": "4", "vue": "3", @@ -29,7 +29,7 @@ "yup": "1" }, "devDependencies": { - "@pinia/testing": "0.1", + "@pinia/testing": "<0.1.6", "@tsconfig/node20": "20", "@types/d3": "7", "@types/jsdom": "21", From 160f0e155c9be1bd3b0894eb2922db3901677719 Mon Sep 17 00:00:00 2001 From: tharvik Date: Wed, 11 Sep 2024 14:58:07 +0200 Subject: [PATCH 14/31] discojs/model: generic predict --- discojs/src/models/gpt/index.ts | 27 ++- discojs/src/models/gpt/model.ts | 2 +- discojs/src/models/model.ts | 10 +- discojs/src/models/tfjs.ts | 118 +++++++++-- discojs/src/types.ts | 4 +- discojs/src/validator.ts | 193 ++++++------------ .../src/components/testing/PredictSteps.vue | 4 +- webapp/src/components/testing/TestSteps.vue | 11 +- 8 files changed, 204 insertions(+), 165 deletions(-) diff --git a/discojs/src/models/gpt/index.ts b/discojs/src/models/gpt/index.ts index 6e57c49f2..e42dca124 100644 --- a/discojs/src/models/gpt/index.ts +++ b/discojs/src/models/gpt/index.ts @@ -11,7 +11,6 @@ import type { Batched, Dataset, ModelEncoded } from "../../index.js"; import { WeightsContainer } from "../../index.js"; import { BatchLogs, Model, EpochLogs } from "../index.js"; -import type { Prediction, Sample } from '../model.js' import { GPTForCausalLM } from './model.js' import { DEFAULT_CONFIG, type GPTConfig } from './config.js' @@ -135,17 +134,27 @@ export class GPT extends Model<"text"> { })); } - override predict(input: Sample): Promise { - const ret = this.model.predict(input); - if (Array.isArray(ret)) { - throw new Error( - "prediction yield many Tensors but should have only returned one", - ); - } + override async predict( + batch: Batched, + ): Promise> { + const predictNext = async (tokens: List) => { + const generated = await this.model.generate(tokens.toArray(), { + maxNewTokens: 1, + temperature: 1.0, + doSample: false, + }); + if (generated.length !== 1 && generated[0].length !== 1) + throw new Error( + "generation returned many tokens but should have only returned one", + ); + + return generated[0][0]; + }; - return Promise.resolve(ret); + return List(await Promise.all(batch.map(predictNext).toArray())) } + /** @deprecated use predict instead and pre/post process the values */ async generate(input: string, tokenizer: PreTrainedTokenizer, newTokens: number = 10): Promise { const { input_ids: tokens } = await tokenizer(input, { return_tensor: false}) as { input_ids: number[] } diff --git a/discojs/src/models/gpt/model.ts b/discojs/src/models/gpt/model.ts index c6efcb5de..75666d413 100644 --- a/discojs/src/models/gpt/model.ts +++ b/discojs/src/models/gpt/model.ts @@ -218,7 +218,7 @@ export class GPTForCausalLM extends GPTModel { const output = model.predict(idx) if (Array.isArray(output)) throw new Error('The model outputs too multiple values') - if (output.shape.length !== 3) throw new Error('The model outputs wrong shape') + if (output.rank !== 3) throw new Error('The model outputs wrong shape') const logits = output as tf.Tensor3D const logitsScaled = logits diff --git a/discojs/src/models/model.ts b/discojs/src/models/model.ts index 2e5102e80..4ebe91ca7 100644 --- a/discojs/src/models/model.ts +++ b/discojs/src/models/model.ts @@ -1,5 +1,3 @@ -import type tf from "@tensorflow/tfjs"; - import type { Batched, Dataset, @@ -10,10 +8,6 @@ import type { import type { BatchLogs, EpochLogs } from "./logs.js"; -// TODO still bound to tfjs -export type Prediction = tf.Tensor; -export type Sample = tf.Tensor; - /** * Trainable predictor * @@ -43,7 +37,9 @@ export abstract class Model /** Predict likely values */ // TODO extract in separated TrainedModel? - abstract predict(input: Sample): Promise; + abstract predict( + batch: Batched, + ): Promise>; /** * This method is automatically called to cleanup the memory occupied by the model diff --git a/discojs/src/models/tfjs.ts b/discojs/src/models/tfjs.ts index 46a2b00f5..7117f86bd 100644 --- a/discojs/src/models/tfjs.ts +++ b/discojs/src/models/tfjs.ts @@ -11,7 +11,6 @@ import { import { BatchLogs } from './index.js' import { Model } from './index.js' -import { Prediction, Sample } from './model.js' import { EpochLogs } from './logs.js' type Serialized = [D, tf.io.ModelArtifacts] @@ -28,6 +27,8 @@ export class TFJS extends Model { if (model.loss === undefined) { throw new Error('TFJS models need to be compiled to be used') } + if (model.outputs.length !== 1) + throw new Error("only support single output model") } override get weights (): WeightsContainer { @@ -118,13 +119,61 @@ export class TFJS extends Model { return { accuracy, loss }; } - override predict (input: Sample): Promise { - const ret = this.model.predict(input) - if (Array.isArray(ret)) { - throw new Error('prediction yield many Tensors but should have only returned one') + override async predict( + batch: Batched, + ): Promise> { + async function cleanupPredicted(y: tf.Tensor1D): Promise { + if (y.shape[0] === 1) { + // Binary classification + const threshold = tf.scalar(0.5); + const binaryTensor = y.greaterEqual(threshold); + + const binaryArray = await binaryTensor.data(); + tf.dispose([y, binaryTensor, threshold]); + + return binaryArray[0]; + } + + // Multi-class classification + const indexTensor = y.argMax(); + + const indexArray = await indexTensor.data(); + tf.dispose([y, indexTensor]); + + return indexArray[0]; + + // Multi-label classification is not supported } - return Promise.resolve(ret) + const xs = this.#batchWithoutLabelToTF(batch); + + const prediction = this.model.predict(xs); + if (Array.isArray(prediction)) + throw new Error( + "prediction yield many Tensors but should have only returned one", + ); + tf.dispose(xs); + + if (prediction.rank !== 2) + throw new Error("unexpected batched prediction shape"); + + const ret = List( + await Promise.all( + tf.unstack(prediction).map((y) => + cleanupPredicted( + // cast as unstack reduce by one the rank + y as tf.Tensor1D, + ), + ), + ), + ); + prediction.dispose(); + + // TODO flatten tabular + if (this.datatype === 'tabular') + return ret.map((v) => List.of(v)); + else + return ret; } static async deserialize([ @@ -174,9 +223,7 @@ export class TFJS extends Model { return this.model } - #batchToTF( - batch: Batched, - ): Record<"xs" | "ys", tf.Tensor> { + #batchToTF(batch: Batched): Record<"xs" | "ys", tf.Tensor> { const outputSize = tf.util.sizeFromShape( this.model.outputShape.map((dim) => { if (Array.isArray(dim)) @@ -204,9 +251,7 @@ export class TFJS extends Model { ), ys: tf.stack( b - .map(([_, label]) => - tf.oneHot(label, outputSize, 1, 0, "int32"), - ) + .map(([_, label]) => tf.oneHot(label, outputSize, 1, 0, "int32")) .toArray(), ), })); @@ -215,14 +260,14 @@ export class TFJS extends Model { // cast as typescript doesn't reduce generic type const b = batch as Batched; - return { + return tf.tidy(() => ({ xs: tf.stack( - b.map(([inputs]) => tf.tensor1d(inputs.toArray())).toArray(), + b.map(([inputs, _]) => tf.tensor1d(inputs.toArray())).toArray(), ), ys: tf.stack( b.map(([_, outputs]) => tf.tensor1d(outputs.toArray())).toArray(), ), - }; + })); } case "text": { // cast as typescript doesn't reduce generic type @@ -246,4 +291,47 @@ export class TFJS extends Model { const _: never = this.datatype; throw new Error("should never happen"); } + + #batchWithoutLabelToTF(batch: Batched): tf.Tensor { + switch (this.datatype) { + case "image": { + // cast as typescript doesn't reduce generic type + const b = batch as Batched; + + return tf.tidy(() => tf.stack( + b + .map((image) => + tf.tensor3d( + image.data, + [image.width, image.height, 3], + "float32", + ), + ) + .toArray(), + ), + ); + } + case "tabular": { + // cast as typescript doesn't reduce generic type + const b = batch as Batched; + + return tf.tidy(() => + tf.stack( + b.map((inputs) => tf.tensor1d(inputs.toArray())).toArray(), + ), + ); + } + case "text": { + // cast as typescript doesn't reduce generic type + const b = batch as Batched; + + return tf.stack( + b.map((line) => tf.tensor1d(line.toArray())).toArray(), + ) + } + } + + const _: never = this.datatype; + throw new Error("should never happen"); + } } diff --git a/discojs/src/types.ts b/discojs/src/types.ts index 3a6714528..8bd911fd3 100644 --- a/discojs/src/types.ts +++ b/discojs/src/types.ts @@ -37,8 +37,6 @@ export interface ModelEncoded { tabular: [List, List]; text: [List, Token]; } -export type ModelEncodedWithoutLabel = { [D in DataType]: ModelEncoded[D][0] }; -export type ModelEncodedOnlyWithLabel = { [D in DataType]: ModelEncoded[D][1] }; export interface Inferred { image: string; @@ -62,10 +60,12 @@ export type TypedModelEncodedDataset = | ["image", Dataset] | ["tabular", Dataset] | ["text", Dataset]; +type ModelEncodedOnlyWithLabel = { [D in DataType]: ModelEncoded[D][1] }; export type TypedBatchedModelEncodedDataset = | ["image", Dataset>] | ["tabular", Dataset>] | ["text", Dataset>]; +type ModelEncodedWithoutLabel = { [D in DataType]: ModelEncoded[D][0] }; export type TypedModelEncodedWithoutLabelDataset = | ["image", Dataset] | ["tabular", Dataset] diff --git a/discojs/src/validator.ts b/discojs/src/validator.ts index 6a54f5316..4f2328aca 100644 --- a/discojs/src/validator.ts +++ b/discojs/src/validator.ts @@ -1,20 +1,10 @@ -import * as tf from "@tensorflow/tfjs"; - import type { Model, Task, TypedRawDataset, TypedRawWithoutLabelDataset, } from "./index.js"; -import { datasetToData, labeledDatasetToData } from "./dataset/data/helpers.js"; - -function intoTFDataset( - iter: AsyncIterable, -): tf.data.Dataset { - return tf.data.generator(async function* () { - yield* iter; - }); -} +import { processing } from "./index.js"; export class Validator { readonly #model: Model; @@ -27,137 +17,86 @@ export class Validator { } /** infer every line of the dataset and check that it is as labeled */ - async *test(dataset: TypedRawDataset): AsyncGenerator { - const preprocessed = ( - await labeledDatasetToData(this.task, dataset) - ).preprocess(); - const batched = preprocessed.batch().dataset; - - const iterator = await tf.data - .zip<[tf.Tensor1D | tf.Tensor2D, number]>([ - preprocessed.dataset.map((t) => { - if ( - typeof t !== "object" || - !("ys" in t) || - !(t.ys instanceof tf.Tensor) || - !(t.ys.rank === 1 || t.ys.rank === 2) - ) - throw new Error("unexpected preprocessed dataset"); - if ("xs" in t) tf.dispose(t.xs); - return t.ys; - }), - intoTFDataset(this.#inferOnBatchedData(batched)), - ]) - .iterator(); - for ( - let iter = await iterator.next(); - iter.done !== true; - iter = await iterator.next() - ) { - const zipped = iter.value; - - const label = await getLabel(zipped[0]); - tf.dispose(zipped[0]); - const infered = zipped[1]; - - yield label === infered; - } - } + async *test(dataset: TypedRawDataset): AsyncGenerator { + const preprocessed = await processing.preprocess(this.task, dataset); + + const { batchSize } = this.task.trainingInformation; + switch (preprocessed[0]) { + case "image": { + // TODO unsafe cast, will get solved when fully generic + const model = this.#model as Model<"image">; + + const results = preprocessed[1] + .batch(batchSize) + .map(async (batch) => + (await model.predict(batch.map(([image, _]) => image))) + .zip(batch.map(([_, label]) => label)) + .map(([infered, truth]) => infered === truth), + ); - /** use the model to predict every line of the dataset */ - async *infer( - dataset: TypedRawWithoutLabelDataset, - ): AsyncGenerator { - const data = await datasetToData(this.task, dataset); + for await (const batch of results) for (const e of batch) yield e; - const batched = data.preprocess().batch().dataset; + break; + } + case "tabular": { + // TODO unsafe cast, will get solved when fully generic + const model = this.#model as Model<"tabular">; + + const results = preprocessed[1] + .batch(batchSize) + .map(async (batch) => + (await model.predict(batch.map(([inputs, _]) => inputs))) + .zip(batch.map(([_, outputs]) => outputs)) + .map(([infered, truth]) => infered.equals(truth)), + ); - yield* this.#inferOnBatchedData(batched); - } + for await (const batch of results) for (const e of batch) yield e; - async *#inferOnBatchedData( - batched: tf.data.Dataset, - ): AsyncGenerator { - const iterator = await batched.iterator(); - for ( - let iter = await iterator.next(); - iter.done !== true; - iter = await iterator.next() - ) { - const row = iter.value; - if ( - typeof row !== "object" || - !("xs" in row) || - !(row.xs instanceof tf.Tensor) - ) - throw new Error("unexpected shape of dataset"); - - const prediction = await this.#model.predict(row.xs); - tf.dispose(row); - let predictions: number[]; - switch (prediction.rank) { - case 2: - case 3: - predictions = await getLabels( - // cast as rank was just checked - prediction as tf.Tensor2D | tf.Tensor3D, - ); - prediction.dispose(); - break; - default: - throw new Error("unexpected batched prediction shape"); + break; } - prediction.dispose(); - - for (const prediction of predictions) yield prediction; + case "text": + throw new Error("TODO implement"); } } -} -async function getLabels(ys: tf.Tensor2D | tf.Tensor3D): Promise { - // cast as unstack drop a dimension and tfjs doesn't type correctly - return Promise.all( - tf.unstack(ys).map((y) => { - const ret = getLabel(y as tf.Tensor1D | tf.Tensor2D); - y.dispose(); - return ret; - }), - ); -} - -async function getLabel(ys: tf.Tensor1D | tf.Tensor2D): Promise { - switch (ys.rank) { - case 1: { - if (ys.shape[0] == 1) { - // Binary classification - const threshold = tf.scalar(0.5); - const binaryTensor = ys.greaterEqual(threshold); + /** use the model to predict every line of the dataset */ + async *infer( + dataset: TypedRawWithoutLabelDataset, + ): AsyncGenerator { + const preprocessed = await processing.preprocessWithoutLabel(this.task, dataset); - const binaryArray = await binaryTensor.data(); - tf.dispose([binaryTensor, threshold]); + const { batchSize } = this.task.trainingInformation; + switch (preprocessed[0]) { + case "image": { + // TODO unsafe cast, will get solved when fully generic + const model = this.#model as Model<"image">; - return binaryArray[0]; - } + const gen = preprocessed[1] + .batch(batchSize) + .map((batch) => model.predict(batch)); - // Multi-class classification - const indexTensor = ys.argMax(); + for await (const batch of gen) for await (const e of batch) yield e; - const indexArray = await indexTensor.data(); - tf.dispose([indexTensor]); + break; + } + case "tabular": { + // TODO unsafe cast, will get solved when fully generic + const model = this.#model as Model<"tabular">; - return indexArray[0]; + const gen = preprocessed[1] + .batch(batchSize) + .map((batch) => model.predict(batch)); - // Multi-label classification is not supported - } - case 2: { - // it's LLM, we only extract the next token - const firstToken = tf.tidy(() => ys.gather([0]).squeeze().argMax()); - const raw = await firstToken.data(); - firstToken.dispose(); + for await (const batch of gen) + for await (const e of batch) { + // TODO mutliple output tabular isn't supported, update types to reflect that + yield e.first(); + } - return raw[0]; + break; + } + case "text": + throw new Error("TODO implement"); } - default: - throw new Error("unexpected tensor rank"); } } diff --git a/webapp/src/components/testing/PredictSteps.vue b/webapp/src/components/testing/PredictSteps.vue index 2c0f1efbe..dc6ab041e 100644 --- a/webapp/src/components/testing/PredictSteps.vue +++ b/webapp/src/components/testing/PredictSteps.vue @@ -239,7 +239,7 @@ async function startImageInference(dataset: NamedImageDataset): Promise { let results: (Results & { type: "image" })["results"] = List(); try { - generator.value = await validator.infer([ + generator.value = validator.infer([ "image", dataset.map(({ image }) => image), ]); @@ -277,7 +277,7 @@ async function startTabularInference(dataset: Dataset): Promise { let results: (Results & { type: "tabular" })["results"] = List(); try { - generator.value = await validator.infer(["tabular", dataset]); + generator.value = validator.infer(["tabular", dataset]); for await (const [input, prediction] of dataset.zip( toRaw(generator.value), )) { diff --git a/webapp/src/components/testing/TestSteps.vue b/webapp/src/components/testing/TestSteps.vue index f46fae212..8e1f7393b 100644 --- a/webapp/src/components/testing/TestSteps.vue +++ b/webapp/src/components/testing/TestSteps.vue @@ -123,7 +123,14 @@ import createDebug from "debug"; import { List, Range, Set } from "immutable"; import { computed, ref, toRaw, watch } from "vue"; -import type { Dataset, Model, Tabular, Task, Text } from "@epfml/discojs"; +import type { + Dataset, + Image, + Model, + Tabular, + Task, + Text, +} from "@epfml/discojs"; import { Validator } from "@epfml/discojs"; import { useToaster } from "@/composables/toaster"; @@ -301,7 +308,7 @@ async function startImageTest( try { generator.value = validator.test([ "image", - dataset.map(({ image, label }) => [image, label]), + dataset.map(({ image, label }) => [image, label] as [Image, string]), ]); for await (const [{ filename, image, label }, correct] of dataset.zip( toRaw(generator.value), From 2cb4864d5ed0777ca7919bd946efc22d820e9c97 Mon Sep 17 00:00:00 2001 From: tharvik Date: Fri, 13 Sep 2024 14:28:13 +0200 Subject: [PATCH 15/31] discojs/tabular: single output --- discojs/src/dataset/data/helpers.ts | 6 +- discojs/src/dataset/data/tabular_data.spec.ts | 2 +- discojs/src/default_tasks/titanic.ts | 4 +- discojs/src/models/tfjs.ts | 10 +-- discojs/src/processing/index.ts | 29 ++++--- discojs/src/task/training_information.ts | 14 ++-- discojs/src/types.ts | 2 +- discojs/src/validator.ts | 81 ++++--------------- docs/examples/custom_task.ts | 4 +- .../src/components/testing/PredictSteps.vue | 13 ++- webapp/src/components/testing/TestSteps.vue | 42 +++------- webapp/src/task_creation_form.ts | 2 +- 12 files changed, 68 insertions(+), 141 deletions(-) diff --git a/discojs/src/dataset/data/helpers.ts b/discojs/src/dataset/data/helpers.ts index 75a3e3ff4..aa1a6d370 100644 --- a/discojs/src/dataset/data/helpers.ts +++ b/discojs/src/dataset/data/helpers.ts @@ -94,14 +94,14 @@ export async function labeledDatasetToData( return await ImageData.init(intoTFDataset(converted), task); } case "tabular": { - const { inputColumns, outputColumns } = task.trainingInformation; - if (inputColumns === undefined || outputColumns === undefined) + const { inputColumns, outputColumn } = task.trainingInformation; + if (inputColumns === undefined || outputColumn === undefined) throw new Error("tabular task without input and output columns"); const converted = dataset.map( (row) => ({ xs: tabularToNumbers(inputColumns, row), - ys: tf.tensor1d(tabularToNumbers(outputColumns, row)), + ys: tf.tensor1d(tabularToNumbers([outputColumn], row)), }) satisfies { xs: number[]; ys: tf.Tensor1D; diff --git a/discojs/src/dataset/data/tabular_data.spec.ts b/discojs/src/dataset/data/tabular_data.spec.ts index 24796a953..f75ca6be5 100644 --- a/discojs/src/dataset/data/tabular_data.spec.ts +++ b/discojs/src/dataset/data/tabular_data.spec.ts @@ -12,7 +12,7 @@ describe('tabular data checks', () => { const dataConfig = { features: titanicTask.trainingInformation.inputColumns, - labels: titanicTask.trainingInformation.outputColumns + labels: [titanicTask.trainingInformation.outputColumn] } const columnConfigs = Map( diff --git a/discojs/src/default_tasks/titanic.ts b/discojs/src/default_tasks/titanic.ts index 776f545e3..170e67f2c 100644 --- a/discojs/src/default_tasks/titanic.ts +++ b/discojs/src/default_tasks/titanic.ts @@ -61,9 +61,7 @@ export const titanic: TaskProvider = { 'Fare', 'Pclass' ], - outputColumns: [ - 'Survived' - ], + outputColumn: 'Survived', scheme: 'federated', aggregationStrategy: 'mean', minNbOfParticipants: 2, diff --git a/discojs/src/models/tfjs.ts b/discojs/src/models/tfjs.ts index 7117f86bd..3c0ad8005 100644 --- a/discojs/src/models/tfjs.ts +++ b/discojs/src/models/tfjs.ts @@ -169,11 +169,7 @@ export class TFJS extends Model { ); prediction.dispose(); - // TODO flatten tabular - if (this.datatype === 'tabular') - return ret.map((v) => List.of(v)); - else - return ret; + return ret } static async deserialize([ @@ -264,9 +260,7 @@ export class TFJS extends Model { xs: tf.stack( b.map(([inputs, _]) => tf.tensor1d(inputs.toArray())).toArray(), ), - ys: tf.stack( - b.map(([_, outputs]) => tf.tensor1d(outputs.toArray())).toArray(), - ), + ys: tf.stack(b.map(([_, output]) => tf.tensor1d([output])).toArray()), })); } case "text": { diff --git a/discojs/src/processing/index.ts b/discojs/src/processing/index.ts index 7e29ccf35..6172329ed 100644 --- a/discojs/src/processing/index.ts +++ b/discojs/src/processing/index.ts @@ -1,6 +1,6 @@ /** Dataset shapers, convenient to map with */ -import { List, Map } from "immutable"; +import { List } from "immutable"; import type { Tabular, @@ -45,16 +45,21 @@ export async function preprocess( ]; } case "tabular": { - const { inputColumns, outputColumns } = task.trainingInformation; - if (inputColumns === undefined || outputColumns === undefined) + const { inputColumns, outputColumn } = task.trainingInformation; + if (inputColumns === undefined || outputColumn === undefined) throw new Error("tabular task without input and output columns"); return [ "tabular", - dataset.map((row) => [ - extractToNumbers(inputColumns, row), - extractToNumbers(outputColumns, row), - ]), + dataset.map((row) => { + const output = processing.extractColumn(row, outputColumn); + + return [ + extractToNumbers(inputColumns, row), + // TODO sanitization doesn't care about column distribution + output !== "" ? processing.convertToNumber(output) : 0, + ]; + }), ]; } case "text": { @@ -143,12 +148,14 @@ export async function postprocess( ]; } case "tabular": { - const { outputColumns } = task.trainingInformation; - if (outputColumns === undefined) + const { outputColumn } = task.trainingInformation; + if (outputColumn === undefined) throw new Error("tabular task without input columns"); - const output = List(outputColumns); - return ["tabular", dataset.map((row) => Map(output.zip(row)).toObject())]; + return [ + "tabular", + dataset.map((row) => Object.fromEntries([[outputColumn, row]])), + ]; } case "text": { const tokenizer = await models.getTaskTokenizer(task); diff --git a/discojs/src/task/training_information.ts b/discojs/src/task/training_information.ts index 137b6d57f..15807724a 100644 --- a/discojs/src/task/training_information.ts +++ b/discojs/src/task/training_information.ts @@ -24,8 +24,8 @@ export interface TrainingInformation { dataType: 'image' | 'tabular' | 'text' // inputColumns: for tabular data, the columns to be chosen as input data for the model inputColumns?: string[] - // outputColumns: for tabular data, the columns to be predicted by the model - outputColumns?: string[] + // outputColumn: for tabular data, the column to be predicted by the model + outputColumn?: string // IMAGE_H height of image (or RESIZED_IMAGE_H if ImagePreprocessing.Resize in preprocessingFunctions) IMAGE_H?: number // IMAGE_W width of image (or RESIZED_IMAGE_W if ImagePreprocessing.Resize in preprocessingFunctions) @@ -108,7 +108,7 @@ export function isTrainingInformation (raw: unknown): raw is TrainingInformation inputColumns, maxShareValue, minNbOfParticipants, - outputColumns, + outputColumn, preprocessingFunctions, roundDuration, scheme, @@ -132,9 +132,9 @@ export function isTrainingInformation (raw: unknown): raw is TrainingInformation (maxShareValue !== undefined && typeof maxShareValue !== 'number') || (IMAGE_H !== undefined && typeof IMAGE_H !== 'number') || (IMAGE_W !== undefined && typeof IMAGE_W !== 'number') || + (outputColumn !== undefined && typeof outputColumn !== 'string') || (LABEL_LIST !== undefined && !isStringArray(LABEL_LIST)) || (inputColumns !== undefined && !isStringArray(inputColumns)) || - (outputColumns !== undefined && !isStringArray(outputColumns)) || (preprocessingFunctions !== undefined && !Array.isArray(preprocessingFunctions)) ) { return false @@ -164,9 +164,7 @@ export function isTrainingInformation (raw: unknown): raw is TrainingInformation if (!(Array.isArray(inputColumns) && inputColumns.every((e) => typeof e === 'string'))) { return false } - if (!(Array.isArray(outputColumns) && outputColumns.every((e) => typeof e === 'string'))) { - return false - } + if (outputColumn === undefined) return false } switch (tensorBackend) { @@ -194,7 +192,7 @@ export function isTrainingInformation (raw: unknown): raw is TrainingInformation inputColumns, maxShareValue, minNbOfParticipants, - outputColumns, + outputColumn, preprocessingFunctions, roundDuration, scheme, diff --git a/discojs/src/types.ts b/discojs/src/types.ts index 8bd911fd3..ea1e2f1d6 100644 --- a/discojs/src/types.ts +++ b/discojs/src/types.ts @@ -34,7 +34,7 @@ export interface RawWithoutLabel { type Token = number; export interface ModelEncoded { image: [processing.NormalizedImage<3>, number]; - tabular: [List, List]; + tabular: [List, number]; text: [List, Token]; } diff --git a/discojs/src/validator.ts b/discojs/src/validator.ts index 4f2328aca..df0b3cc4c 100644 --- a/discojs/src/validator.ts +++ b/discojs/src/validator.ts @@ -21,82 +21,33 @@ export class Validator { const preprocessed = await processing.preprocess(this.task, dataset); const { batchSize } = this.task.trainingInformation; - switch (preprocessed[0]) { - case "image": { - // TODO unsafe cast, will get solved when fully generic - const model = this.#model as Model<"image">; - const results = preprocessed[1] - .batch(batchSize) - .map(async (batch) => - (await model.predict(batch.map(([image, _]) => image))) - .zip(batch.map(([_, label]) => label)) - .map(([infered, truth]) => infered === truth), - ); + const results = preprocessed[1] + .batch(batchSize) + .map(async (batch) => + (await this.#model.predict(batch.map(([inputs, _]) => inputs))) + .zip(batch.map(([_, outputs]) => outputs)) + .map(([inferred, truth]) => inferred === truth), + ); - for await (const batch of results) for (const e of batch) yield e; - - break; - } - case "tabular": { - // TODO unsafe cast, will get solved when fully generic - const model = this.#model as Model<"tabular">; - - const results = preprocessed[1] - .batch(batchSize) - .map(async (batch) => - (await model.predict(batch.map(([inputs, _]) => inputs))) - .zip(batch.map(([_, outputs]) => outputs)) - .map(([infered, truth]) => infered.equals(truth)), - ); - - for await (const batch of results) for (const e of batch) yield e; - - break; - } - case "text": - throw new Error("TODO implement"); - } + for await (const batch of results) for (const e of batch) yield e; } /** use the model to predict every line of the dataset */ async *infer( dataset: TypedRawWithoutLabelDataset, ): AsyncGenerator { - const preprocessed = await processing.preprocessWithoutLabel(this.task, dataset); + const preprocessed = await processing.preprocessWithoutLabel( + this.task, + dataset, + ); const { batchSize } = this.task.trainingInformation; - switch (preprocessed[0]) { - case "image": { - // TODO unsafe cast, will get solved when fully generic - const model = this.#model as Model<"image">; - - const gen = preprocessed[1] - .batch(batchSize) - .map((batch) => model.predict(batch)); - - for await (const batch of gen) for await (const e of batch) yield e; - - break; - } - case "tabular": { - // TODO unsafe cast, will get solved when fully generic - const model = this.#model as Model<"tabular">; - - const gen = preprocessed[1] - .batch(batchSize) - .map((batch) => model.predict(batch)); - for await (const batch of gen) - for await (const e of batch) { - // TODO mutliple output tabular isn't supported, update types to reflect that - yield e.first(); - } + const gen = preprocessed[1] + .batch(batchSize) + .map((batch) => this.#model.predict(batch)); - break; - } - case "text": - throw new Error("TODO implement"); - } + for await (const batch of gen) for await (const e of batch) yield e; } } diff --git a/docs/examples/custom_task.ts b/docs/examples/custom_task.ts index 8113dde9f..021705261 100644 --- a/docs/examples/custom_task.ts +++ b/docs/examples/custom_task.ts @@ -25,9 +25,7 @@ const customTask: TaskProvider = { inputColumns: [ 'Age' ], - outputColumns: [ - 'Output' - ], + outputColumn: 'Output', scheme: 'federated', minNbOfParticipants: 2, tensorBackend: 'tfjs', diff --git a/webapp/src/components/testing/PredictSteps.vue b/webapp/src/components/testing/PredictSteps.vue index dc6ab041e..d1bd6d726 100644 --- a/webapp/src/components/testing/PredictSteps.vue +++ b/webapp/src/components/testing/PredictSteps.vue @@ -142,16 +142,13 @@ type Results = | { type: "image"; results: List<{ - input: { - filename: string; - image: ImageData; - }; + input: { filename: string; image: ImageData }; output: string; }>; } | { type: "tabular"; - labels: { input: List; output: List }; + labels: { input: List; output: string }; results: List<{ input: List; output: string }>; }; @@ -266,12 +263,12 @@ async function startImageInference(dataset: NamedImageDataset): Promise { } async function startTabularInference(dataset: Dataset): Promise { - const { inputColumns, outputColumns } = props.task.trainingInformation; - if (inputColumns === undefined || outputColumns === undefined) + const { inputColumns, outputColumn } = props.task.trainingInformation; + if (inputColumns === undefined || outputColumn === undefined) throw new Error("no input and output columns but tabular task needs it"); const labels = { input: List(inputColumns), - output: List(outputColumns).map((c) => `Predicted_${c}`), + output: `Predicted_${outputColumn}`, }; const validator = new Validator(props.task, toRaw(props.model)); diff --git a/webapp/src/components/testing/TestSteps.vue b/webapp/src/components/testing/TestSteps.vue index 8e1f7393b..cb7a8802d 100644 --- a/webapp/src/components/testing/TestSteps.vue +++ b/webapp/src/components/testing/TestSteps.vue @@ -163,31 +163,19 @@ type Results = | { type: "image"; results: List<{ - input: { - filename: string; - image: ImageData; - }; - output: { - truth: string; - correct: boolean; - }; + input: { filename: string; image: ImageData }; + output: { truth: string; correct: boolean }; }>; } | { type: "tabular"; labels: { input: List; - output: { - truth: List; - correct: string; - }; + output: { truth: string; correct: string }; }; results: List<{ input: List; - output: { - truth: List; - correct: boolean; - }; + output: { truth: string; correct: boolean }; }>; } | { @@ -249,10 +237,10 @@ const generator = ref>(); watch(tabularDataset, async (dataset) => { if (dataset === undefined) return; - const { inputColumns, outputColumns } = props.task.trainingInformation; - if (inputColumns === undefined || outputColumns === undefined) + const { inputColumns, outputColumn } = props.task.trainingInformation; + if (inputColumns === undefined || outputColumn === undefined) throw new Error("tabular task without input or output columns"); - const wantedColumns = Set(inputColumns).union(outputColumns); + const wantedColumns = Set(inputColumns).add(outputColumn); try { for await (const [columns, i] of toRaw(dataset) @@ -336,13 +324,13 @@ async function startImageTest( } async function startTabularTest(dataset: Dataset): Promise { - const { inputColumns, outputColumns } = props.task.trainingInformation; - if (inputColumns === undefined || outputColumns === undefined) + const { inputColumns, outputColumn } = props.task.trainingInformation; + if (inputColumns === undefined || outputColumn === undefined) throw new Error("no input and output columns but CSV needs it"); const labels = { input: List(inputColumns), output: { - truth: List(outputColumns).map((c) => `Truth_${c}`), + truth: `Truth_${outputColumn}`, correct: "Correct", }, }; @@ -352,13 +340,9 @@ async function startTabularTest(dataset: Dataset): Promise { try { generator.value = validator.test(["tabular", dataset]); for await (const [row, correct] of dataset.zip(toRaw(generator.value))) { - // TODO we only really support a single output, change Task to reflect that - const truth = List(outputColumns).map((column) => { - const ret = row[column]; - if (ret === undefined) - throw new Error("row doesn't have expected output column"); - return ret; - }); + const truth = row[outputColumn]; + if (truth === undefined) + throw new Error("row doesn't have expected output column"); results = results.push({ input: labels.input.map((label) => { diff --git a/webapp/src/task_creation_form.ts b/webapp/src/task_creation_form.ts index e8b826190..f33420364 100644 --- a/webapp/src/task_creation_form.ts +++ b/webapp/src/task_creation_form.ts @@ -247,7 +247,7 @@ export const trainingInformation: FormSection = { ] }, { - id: 'outputColumns', + id: 'outputColumn', name: 'Output Column', yup: yup.string().when('dataType', otherReq('tabular')), as: 'input', From b73d764f7ecd467e3e01ea5567a3bc42dbf5baf2 Mon Sep 17 00:00:00 2001 From: tharvik Date: Fri, 13 Sep 2024 14:46:12 +0200 Subject: [PATCH 16/31] discojs: rm old processing --- discojs/src/dataset/data/data.ts | 119 -------------- discojs/src/dataset/data/data_split.ts | 9 -- discojs/src/dataset/data/helpers.ts | 147 ------------------ discojs/src/dataset/data/image_data.spec.ts | 23 --- discojs/src/dataset/data/image_data.ts | 54 ------- discojs/src/dataset/data/index.ts | 8 - .../src/dataset/data/preprocessing/base.ts | 19 --- .../data/preprocessing/image_preprocessing.ts | 53 ------- .../src/dataset/data/preprocessing/index.ts | 4 - .../preprocessing/tabular_preprocessing.ts | 52 ------- .../preprocessing/text_preprocessing.spec.ts | 101 ------------ .../data/preprocessing/text_preprocessing.ts | 128 --------------- discojs/src/dataset/data/tabular_data.spec.ts | 45 ------ discojs/src/dataset/data/tabular_data.ts | 35 ----- discojs/src/dataset/data/text_data.ts | 25 --- discojs/src/dataset/index.ts | 13 -- discojs/src/default_tasks/cifar10.ts | 3 +- discojs/src/default_tasks/lus_covid.ts | 3 +- discojs/src/default_tasks/mnist.ts | 4 +- discojs/src/default_tasks/simple_face.ts | 3 +- discojs/src/default_tasks/titanic.ts | 3 +- discojs/src/default_tasks/wikitext.ts | 3 +- discojs/src/index.ts | 3 +- discojs/src/processing/index.spec.ts | 43 +++++ discojs/src/processing/text.spec.ts | 44 ++++++ discojs/src/task/training_information.ts | 8 +- 26 files changed, 95 insertions(+), 857 deletions(-) delete mode 100644 discojs/src/dataset/data/data.ts delete mode 100644 discojs/src/dataset/data/data_split.ts delete mode 100644 discojs/src/dataset/data/helpers.ts delete mode 100644 discojs/src/dataset/data/image_data.spec.ts delete mode 100644 discojs/src/dataset/data/image_data.ts delete mode 100644 discojs/src/dataset/data/index.ts delete mode 100644 discojs/src/dataset/data/preprocessing/base.ts delete mode 100644 discojs/src/dataset/data/preprocessing/image_preprocessing.ts delete mode 100644 discojs/src/dataset/data/preprocessing/index.ts delete mode 100644 discojs/src/dataset/data/preprocessing/tabular_preprocessing.ts delete mode 100644 discojs/src/dataset/data/preprocessing/text_preprocessing.spec.ts delete mode 100644 discojs/src/dataset/data/preprocessing/text_preprocessing.ts delete mode 100644 discojs/src/dataset/data/tabular_data.spec.ts delete mode 100644 discojs/src/dataset/data/tabular_data.ts delete mode 100644 discojs/src/dataset/data/text_data.ts create mode 100644 discojs/src/processing/index.spec.ts create mode 100644 discojs/src/processing/text.spec.ts diff --git a/discojs/src/dataset/data/data.ts b/discojs/src/dataset/data/data.ts deleted file mode 100644 index 00aa8fd78..000000000 --- a/discojs/src/dataset/data/data.ts +++ /dev/null @@ -1,119 +0,0 @@ -import * as tf from '@tensorflow/tfjs' -import type { List } from 'immutable' - -import type { Task } from '../../index.js' - -import type { PreprocessingFunction } from './preprocessing/base.js' - -/** - * Abstract class representing an immutable Disco dataset, including a TF.js dataset, - * Disco task and set of preprocessing functions. - */ -export abstract class Data { - public abstract readonly availablePreprocessing: List - - protected constructor ( - public readonly dataset: tf.data.Dataset, - public readonly task: Task, - public readonly size?: number) {} - - static init ( - _dataset: tf.data.Dataset, - _task: Task, - _size?: number - ): Promise { - return Promise.reject(new Error('abstract')) - } - - /** - * Callable abstract method instead of constructor. - */ - protected abstract create (dataset: tf.data.Dataset, task: Task, size?: number): Data - - /** - * Creates a new Disco data object containing the batched TF.js dataset, according to the - * task's parameters. - * @returns The batched Disco data - */ - batch (): Data { - return this.create(this.batchedDataset, this.task, this.size) - } - - /** - * The TF.js dataset batched according to the task's parameters. - */ - get batchedDataset (): tf.data.Dataset { - const batchSize = this.task.trainingInformation.batchSize - return batchSize === undefined - ? this.dataset - : this.dataset.batch(batchSize) - } - - /** - * Creates a new Disco data object containing the preprocessed TF.js dataset, - * according to the defined set of preprocessing functions and the task's parameters. - * @returns The preprocessed Disco data - */ - preprocess (): Data { - return this.create(this.preprocessedDataset, this.task, this.size) - } - - /** - * Creates a higher level preprocessing function applying the specified set of preprocessing - * functions in a series. The preprocessing functions are chained according to their defined - * priority. - */ - get preprocessing (): (entry: tf.TensorContainer) => Promise { - const params = this.task.trainingInformation - const taskPreprocessing = params.preprocessingFunctions - if ( - taskPreprocessing === undefined || - taskPreprocessing.length === 0 || - this.availablePreprocessing === undefined || - this.availablePreprocessing.size === 0 - ) { - return x => Promise.resolve(x) - } - - const applyPreprocessing = this.availablePreprocessing - .filter((e) => e.type in taskPreprocessing) - .map((e) => e.apply) - - const preprocessingChain = async (input: Promise) => { - let currentContainer = await input; // Start with the initial tensor container - for (const fn of applyPreprocessing) { - const next = await fn(Promise.resolve(currentContainer), this.task); - - // dirty but kinda working way to dispose of converted tensors - if (typeof currentContainer === "object" && typeof next === "object") { - if ( - "xs" in currentContainer && - "xs" in next && - currentContainer.xs !== next.xs - ) - tf.dispose(currentContainer.xs); - if ( - "ys" in currentContainer && - "ys" in next && - currentContainer.ys !== next.ys - ) - tf.dispose(currentContainer.ys); - } - - currentContainer = next - } - - return currentContainer; // Return the final tensor container - }; - - return async (entry) => await preprocessingChain(Promise.resolve(entry)); - } - - /** - * The TF.js dataset preprocessing according to the set of preprocessing functions and the task's - * parameters. - */ - get preprocessedDataset (): tf.data.Dataset { - return this.dataset.mapAsync(this.preprocessing) - } -} diff --git a/discojs/src/dataset/data/data_split.ts b/discojs/src/dataset/data/data_split.ts deleted file mode 100644 index 5cf56bff8..000000000 --- a/discojs/src/dataset/data/data_split.ts +++ /dev/null @@ -1,9 +0,0 @@ -import type { Data } from './data.js' - -/** - * Train-validation split of Disco data. - */ -export interface DataSplit { - train: Data - validation?: Data -} diff --git a/discojs/src/dataset/data/helpers.ts b/discojs/src/dataset/data/helpers.ts deleted file mode 100644 index aa1a6d370..000000000 --- a/discojs/src/dataset/data/helpers.ts +++ /dev/null @@ -1,147 +0,0 @@ -/** Internal functions to help with Dataset to Data/DataSplit conversion - * - * @todo rm when fully using Dataset - */ - -import { List } from "immutable"; -import * as tf from "@tensorflow/tfjs"; - -import type { - Image, - Tabular, - Task, - TypedRawDataset, - TypedRawWithoutLabelDataset, -} from "../../index.js"; -import { processing } from "../../index.js"; - -import { Data, ImageData, TabularData, TextData } from "./index.js"; -import { DataSplit } from "./data_split.js"; - -function intoTFDataset( - iter: AsyncIterable, -): tf.data.Dataset { - return tf.data.generator(async function* () { - yield* iter; - }); -} - -function imageToTensor(image: Image<3>): tf.Tensor3D { - return tf.tensor3d(image.data, [image.width, image.height, 3], "int32"); -} - -function tabularToNumbers(columns: Iterable, row: Tabular): number[] { - return List(columns) - .map((column) => processing.extractColumn(row, column)) - .map((v) => (v !== "" ? v : "0")) // TODO how to specify defaults? - .map(processing.convertToNumber) - .toArray(); -} - -export async function datasetToData( - task: Task, - [t, dataset]: TypedRawWithoutLabelDataset, -): Promise { - switch (t) { - case "image": { - const converted = dataset - .map(processing.removeAlpha) - .map((image) => processing.expandToMulticolor(image)) - .map((image) => ({ - xs: imageToTensor(image), - })); - return await ImageData.init(intoTFDataset(converted), task); - } - case "tabular": { - const inputColumns = task.trainingInformation.inputColumns; - if (inputColumns === undefined) - throw new Error("tabular task without input columns"); - const converted = dataset.map((row) => ({ - xs: tabularToNumbers(inputColumns, row), - })); - return await TabularData.init(intoTFDataset(converted), task); - } - case "text": - return await TextData.init(intoTFDataset(dataset), task); - } -} - -export async function labeledDatasetToData( - task: Task, - [t, dataset]: TypedRawDataset, -): Promise { - switch (t) { - case "image": { - const labels = List(task.trainingInformation.LABEL_LIST); - const converted = dataset - .map( - ([image, label]) => - [ - processing.expandToMulticolor(processing.removeAlpha(image)), - processing.indexInList(label, labels), - ] as const, - ) - .map( - ([image, label]) => - ({ - xs: imageToTensor(image), - ys: tf.oneHot(label, labels.size, 1, 0, "int32") as tf.Tensor1D, - }) satisfies { - xs: tf.Tensor3D; - ys: tf.Tensor1D; - }, - ); - return await ImageData.init(intoTFDataset(converted), task); - } - case "tabular": { - const { inputColumns, outputColumn } = task.trainingInformation; - if (inputColumns === undefined || outputColumn === undefined) - throw new Error("tabular task without input and output columns"); - const converted = dataset.map( - (row) => - ({ - xs: tabularToNumbers(inputColumns, row), - ys: tf.tensor1d(tabularToNumbers([outputColumn], row)), - }) satisfies { - xs: number[]; - ys: tf.Tensor1D; - }, - ); - return await TabularData.init(intoTFDataset(converted), task); - } - case "text": - return await TextData.init(intoTFDataset(dataset), task); - } -} - -export async function labeledDatasetToDataSplit( - task: Task, - [t, dataset]: TypedRawDataset, -): Promise { - const split = task.trainingInformation.validationSplit; - - let train: Data; - let validation: Data; - switch (t) { - case "image": { - [train, validation] = await Promise.all( - dataset.split(split).map((d) => labeledDatasetToData(task, [t, d])), - ); - break; - } - case "tabular": { - [train, validation] = await Promise.all( - dataset.split(split).map((d) => labeledDatasetToData(task, [t, d])), - ); - break; - } - case "text": { - [train, validation] = await Promise.all( - dataset.split(split).map((d) => labeledDatasetToData(task, [t, d])), - ); - break; - } - } - - return { train, validation }; -} diff --git a/discojs/src/dataset/data/image_data.spec.ts b/discojs/src/dataset/data/image_data.spec.ts deleted file mode 100644 index 72386914e..000000000 --- a/discojs/src/dataset/data/image_data.spec.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { assert, expect } from 'chai' -import * as tf from '@tensorflow/tfjs' - -import { ImageData } from './image_data.js' -import { defaultTasks } from '../../index.js' - -describe('image data checks', () => { - const simpleFaceTask = defaultTasks.simpleFace.getTask() - it('throw an error on incorrectly formatted data', async () => { - try { - await ImageData.init(tf.data.array([tf.zeros([150, 150, 3]), tf.zeros([150, 150, 3])]), simpleFaceTask, 3) - } catch (e) { - expect(e).to.be.an.instanceOf(Error) - return - } - // no error means we failed - assert(false) - }) - - it('do nothing on correctly formatted data', async () => { - await ImageData.init(tf.data.array([tf.zeros([200, 200, 3]), tf.zeros([200, 200, 3])]), simpleFaceTask, 3) - }) -}) diff --git a/discojs/src/dataset/data/image_data.ts b/discojs/src/dataset/data/image_data.ts deleted file mode 100644 index c45639cc3..000000000 --- a/discojs/src/dataset/data/image_data.ts +++ /dev/null @@ -1,54 +0,0 @@ -import * as tf from '@tensorflow/tfjs' - -import type { Task } from '../../index.js' - -import { Data } from './data.js' -import { ImagePreprocessing, IMAGE_PREPROCESSING } from './preprocessing/index.js' - -/** - * Disco data made of image samples (.jpg, .png, etc.). - */ -export class ImageData extends Data { - public readonly availablePreprocessing = IMAGE_PREPROCESSING - - static override async init ( - dataset: tf.data.Dataset, - task: Task, - size?: number - ): Promise { - // Here we do our best to check data format before proceeding to training, for - // better error handling. An incorrectly formatted image in the dataset might still - // cause an error during training, because of the lazy aspect of the dataset; we only - // verify the first sample. - if (task.trainingInformation.preprocessingFunctions?.includes(ImagePreprocessing.Resize) !== true) { - const iteration = await dataset.iterator().then((iter) => iter.next()) - if (iteration.done === true) throw new Error("empty dataset") - const sample = iteration.value - - // TODO: We suppose the presence of labels - // TODO: Typing (discojs-node/src/dataset/data_loader/image_loader.spec.ts) - if (typeof sample !== 'object' || sample === null || sample === undefined) { - throw new Error("Image is undefined or is not an object") - } - - let shape - if ('xs' in sample) { - shape = (sample as { xs: tf.Tensor }).xs.shape - } else { - shape = (sample as tf.Tensor3D).shape - } - const {IMAGE_H, IMAGE_W} = task.trainingInformation - if (IMAGE_W !== undefined && IMAGE_H !== undefined && - (shape[0] !== IMAGE_W || shape[1] !== IMAGE_H)) { - throw new Error(`Image doesn't have the dimensions specified in the task's training information. Expected ${IMAGE_H}x${IMAGE_W} but got ${shape[0]}x${shape[1]}.`) - } - - tf.dispose(sample) - } - return new ImageData(dataset, task, size) - } - - protected create (dataset: tf.data.Dataset, task: Task, size: number): ImageData { - return new ImageData(dataset, task, size) - } -} diff --git a/discojs/src/dataset/data/index.ts b/discojs/src/dataset/data/index.ts deleted file mode 100644 index 78d66e506..000000000 --- a/discojs/src/dataset/data/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -export { Data } from './data.js' -export { ImageData } from './image_data.js' -export { TabularData } from './tabular_data.js' -export { TextData } from './text_data.js' -export { - ImagePreprocessing, TabularPreprocessing, TextPreprocessing, - IMAGE_PREPROCESSING, TABULAR_PREPROCESSING, TEXT_PREPROCESSING -} from './preprocessing/index.js' diff --git a/discojs/src/dataset/data/preprocessing/base.ts b/discojs/src/dataset/data/preprocessing/base.ts deleted file mode 100644 index e5823d0d2..000000000 --- a/discojs/src/dataset/data/preprocessing/base.ts +++ /dev/null @@ -1,19 +0,0 @@ -import type tf from '@tensorflow/tfjs' - -import type { Task } from '../../../index.js' -import type { ImagePreprocessing } from './image_preprocessing.js' -import type { TabularPreprocessing } from './tabular_preprocessing.js' -import type { TextPreprocessing } from './text_preprocessing.js' - -/** - * All available preprocessing type enums. - */ -export type Preprocessing = ImagePreprocessing | TextPreprocessing | TabularPreprocessing - -/** - * Preprocessing function associating a preprocessing type enum to a sample transformation. - */ -export interface PreprocessingFunction { - type: Preprocessing - apply: (x: Promise, task: Task) => Promise -} diff --git a/discojs/src/dataset/data/preprocessing/image_preprocessing.ts b/discojs/src/dataset/data/preprocessing/image_preprocessing.ts deleted file mode 100644 index a1ab81f68..000000000 --- a/discojs/src/dataset/data/preprocessing/image_preprocessing.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { List } from 'immutable' -import * as tf from '@tensorflow/tfjs' - -import type { Task } from '../../../index.js' -import type { PreprocessingFunction } from './base.js' - -/** - * Available image preprocessing types. - */ -export enum ImagePreprocessing { - Resize, - Normalize -} - -interface ImageEntry extends tf.TensorContainerObject { - xs: tf.Tensor3D | tf.Tensor4D - ys: tf.Tensor1D | number | undefined -} - -const resize: PreprocessingFunction = { - type: ImagePreprocessing.Resize, - apply: async (entry: Promise, task: Task): Promise => { - const { xs, ys } = await entry as ImageEntry - const params = task.trainingInformation - return { - xs: params.IMAGE_W !== undefined && params.IMAGE_H !== undefined - ? xs.resizeBilinear([params.IMAGE_H, params.IMAGE_W]) - : xs, - ys - } - } -} - -const normalize: PreprocessingFunction = { - type: ImagePreprocessing.Normalize, - apply: async (entry: Promise): Promise => { - const { xs, ys } = await entry as ImageEntry - return tf.tidy(() => { // make sure we dispose the intermediate tensor - return { - xs: xs.div(tf.scalar(255)), - ys - }; - }); - } -} - -/** - * Available image preprocessing functions. - */ -export const AVAILABLE_PREPROCESSING = List([ - resize, - normalize] -).sortBy((e) => e.type) diff --git a/discojs/src/dataset/data/preprocessing/index.ts b/discojs/src/dataset/data/preprocessing/index.ts deleted file mode 100644 index 3e805f897..000000000 --- a/discojs/src/dataset/data/preprocessing/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export type { Preprocessing, PreprocessingFunction } from './base.js' -export { AVAILABLE_PREPROCESSING as IMAGE_PREPROCESSING, ImagePreprocessing } from './image_preprocessing.js' -export { AVAILABLE_PREPROCESSING as TABULAR_PREPROCESSING, TabularPreprocessing } from './tabular_preprocessing.js' -export { AVAILABLE_PREPROCESSING as TEXT_PREPROCESSING, TextPreprocessing } from './text_preprocessing.js' diff --git a/discojs/src/dataset/data/preprocessing/tabular_preprocessing.ts b/discojs/src/dataset/data/preprocessing/tabular_preprocessing.ts deleted file mode 100644 index bb172daf6..000000000 --- a/discojs/src/dataset/data/preprocessing/tabular_preprocessing.ts +++ /dev/null @@ -1,52 +0,0 @@ -import type tf from '@tensorflow/tfjs' -import { List } from 'immutable' - -import type { PreprocessingFunction } from './base.js' - -/** - * Available tabular preprocessing types. - */ -export enum TabularPreprocessing { - Sanitize, - Normalize -} - -interface TabularEntry extends tf.TensorContainerObject { - xs: number[] - ys: tf.Tensor1D | number | undefined -} - -const sanitize: PreprocessingFunction = { - type: TabularPreprocessing.Sanitize, - apply: async (entry: Promise) => { - const entryContainer = await entry - // if preprocessing a dataset without labels, then the entry is an array of numbers - if (Array.isArray(entryContainer)) { - const entry = entryContainer as number[] - return entry.map((i: number) => i ?? 0) - // if it is an object - } else if (typeof entryContainer === 'object' && entry !== null) { - // if the object is a tensor container with features xs and labels ys - if (Object.hasOwn(entryContainer, 'xs')) { - const { xs, ys } = entryContainer as TabularEntry - return { - xs: xs.map(i => i ?? 0), - ys - } - // if the object contains features as a dict of feature names-values - } else { - const entry = Object.values(entryContainer) - return entry.map((i: number) => i ?? 0) - } - } else { - throw new Error('Unrecognized format during tabular preprocessing') - } - } -} - -/** - * Available tabular preprocessing functions. - */ -export const AVAILABLE_PREPROCESSING = List([ - sanitize] -).sortBy((e) => e.type) diff --git a/discojs/src/dataset/data/preprocessing/text_preprocessing.spec.ts b/discojs/src/dataset/data/preprocessing/text_preprocessing.spec.ts deleted file mode 100644 index f75f8b54a..000000000 --- a/discojs/src/dataset/data/preprocessing/text_preprocessing.spec.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { TEXT_PREPROCESSING } from './index.js' -import { expect } from 'chai' - -import type { Task } from '../../../index.js' -import * as tf from '@tensorflow/tfjs' - -describe('text preprocessing', function () { - const [tokenize, leftPadding] = TEXT_PREPROCESSING - // Use a function to create different task object for each test (otherwise the tokenizer gets cached) - function initMockTask(): Task { - return { - id: 'mock-task-id', - displayInformation: { - taskTitle: 'mock title', - summary: { overview: '', preview: '' } - }, - trainingInformation: { - epochs: 1, - roundDuration: 1, - validationSplit: 0, - batchSize: 8, - scheme: 'local', - minNbOfParticipants: 1, - dataType: 'text', - tokenizer: 'Xenova/gpt2', - tensorBackend: 'gpt' - }} - } - - const text = "Hello world, a bc 1 2345, '? 976. Wikipedia is a free content online encyclopedia written and maintained by a community \n of volunteers, known as Wikipedians. Founded by Jimmy Wales and Larry Sanger on January 15, 2001, Wikipedia is hosted by the Wikimedia Foundation, an American nonprofit organization that employs a staff of over 700 people.[7]" - const expectedTokens = [15496, 995, 11, 257, 47125, 352, 2242, 2231, 11, 705, 30, 860, 4304, 13, 15312, 318, 257, 1479, 2695, 2691, 45352, 3194, 290, 9456, 416, 257, 2055, 220, 198, 286, 11661, 11, 1900, 355, 11145, 46647, 1547, 13, 4062, 276, 416, 12963, 11769, 290, 13633, 311, 2564, 319, 3269, 1315, 11, 5878, 11, 15312, 318, 12007, 416, 262, 44877, 5693, 11, 281, 1605, 15346, 4009, 326, 24803, 257, 3085, 286, 625, 13037, 661, 3693, 22, 60] - - it('can tokenize text', async () => { - const { tokens } = await tokenize.apply(Promise.resolve(text), initMockTask()) as { tokens: number[]} - expect(tokens).to.be.deep.equal(expectedTokens) - }).timeout(4000) - - it('can truncate inputs when tokenizing', async () => { - const truncationTask = initMockTask() - truncationTask.trainingInformation.maxSequenceLength = 10 - const { tokens } = await tokenize.apply(Promise.resolve(text), truncationTask) as { tokens: number[] } - const expectedLength = truncationTask.trainingInformation.maxSequenceLength + 1 // + 1 because tokenization includes an extra token label for next label prediction - expect(tokens.length).to.be.equal(expectedLength) - expect(tokens).to.be.deep.equal(expectedTokens.slice(0, expectedLength)) - }).timeout(4000) - - it('can left pad tokens', async () => { - // Create a task where output token sequence should all have length 20 - const paddingTask = initMockTask() - paddingTask.trainingInformation.maxSequenceLength = 20 - - // Create a token sequence of length 10 - const tokens = { tokens: [0,1,2,3,4,5,6,7,8,9] } - const { xs, ys } = await leftPadding.apply(Promise.resolve(tokens), paddingTask) as { xs: tf.Tensor1D, ys: tf.Tensor2D } - const xsArray = await xs.array() - const ysArray = await ys.array() - - // Output sequences should have shape (20) and (20, 50258), 50258 being the size of the vocab for gpt2 - expect(xsArray.length).to.be.equal(paddingTask.trainingInformation.maxSequenceLength) - expect(ysArray.length).to.be.equal(paddingTask.trainingInformation.maxSequenceLength) - expect(ysArray[0].length).to.be.equal(50258) - - // xs should be left pad with gpt2's padding token 50256 to be of length 20. - // We expect the last token of input token sequence (9) to not be included in xs since it doesn't have a next token to be predicted - const paddingToken = 50256 - const expectedXs = Array.from({length:11}).map(_ => paddingToken).concat(tokens.tokens.slice(0,9)) - expect(xsArray).to.be.deep.equal(expectedXs) - - // ys should be a one hot encoding of the next token in xs - // if the input tokens are [0,1,2,3] then the labels are [1,2,3] which are then one-hot encoded - // So the sum of each row should be equal to 1 - const expectedOneHot = Array.from({ length: 20 }).map(_ => 1) - expect(await ys.sum(-1).array()).to.be.deep.equal(expectedOneHot) - - // In each row, the index of the 1 should be the token id - const expectedYs = Array.from({length:10}).map(_ => paddingToken).concat(tokens.tokens) - expect(await ys.argMax(-1).array()).to.be.deep.equal(expectedYs) - }) - - it('throws an error if no tokenizer is specified', async () => { - const invalidTask = initMockTask() - invalidTask.trainingInformation.tokenizer = undefined; - try { - await tokenize.apply(Promise.resolve("input text doesn't matter"), invalidTask) - } catch { - return - } - throw new Error("undefined tokenizer should have thrown an error") - }) - it('throws an error if the tokenizer name is invalid', async () => { - const invalidTask = initMockTask() - invalidTask['trainingInformation']['tokenizer'] = 'invalid-tokenizer-name' - try { - await tokenize.apply(Promise.resolve("input text doesn't matter"), invalidTask) - } catch { - return - } - throw new Error("invalid tokenizer name should have thrown an error") - }) - -}) diff --git a/discojs/src/dataset/data/preprocessing/text_preprocessing.ts b/discojs/src/dataset/data/preprocessing/text_preprocessing.ts deleted file mode 100644 index 3c0c4bb30..000000000 --- a/discojs/src/dataset/data/preprocessing/text_preprocessing.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { List } from 'immutable' -import * as tf from '@tensorflow/tfjs' - -import type { Task } from '../../../index.js' -import type { PreprocessingFunction } from './base.js' -import { models } from '../../../index.js' - -/** - * Available text preprocessing types. - */ -export enum TextPreprocessing { - Tokenize, - LeftPadding -} - -interface TokenizedEntry { - tokens: number [] -} - -function isNumberArray(raw: unknown): raw is number[] { - if (!Array.isArray(raw)) return false; - const arr: unknown[] = raw; // isArray is unsafely guarding with any[] - - return arr.every((e) => typeof e === "number"); -} - -function isTokenizedEntry(raw: unknown): raw is TokenizedEntry { - if (typeof raw !== "object" || raw === null) return false; - - const { tokens }: Partial> = raw; - - if (!isNumberArray(tokens)) return false; - - const _: TokenizedEntry = { tokens } satisfies Record< - keyof TokenizedEntry, - unknown - >; - - return true; -} - -/** - * LeftPadding pads all incoming inputs to be a fixed length, which should be specified - * in `task.trainingInformation.maxSequenceLength`. - * - * We are currently only implementing left padding for text generation - * https://huggingface.co/docs/transformers/en/llm_tutorial#wrong-padding-side - * The function can easily be extended to support right padding if needed - * - * Once Transformers.js supports left padding, it will be possible to pad inputs - * directly when tokenizing - * https://github.com/xenova/transformers.js/blob/8804c36591d11d8456788d1bb4b16489121b3be2/src/tokenizers.js#L2517 - */ -const leftPadding: PreprocessingFunction = { - type: TextPreprocessing.LeftPadding, - apply: async (input: Promise, task: Task): Promise<{ xs: tf.Tensor1D, ys: tf.Tensor2D }> => { - const x = await input - if (!isTokenizedEntry(x)) - throw new Error("The leftPadding preprocessing expects a non empty 1D array of number") - const { tokens } = x - - const tokenizer = await models.getTaskTokenizer(task) - return tf.tidy(() => { - // maxLength is the final length of xs - // Because ys the contains the tokens in xs shifted by one (to predict the next token), we need - // to include one more token than maxSequenceLength in order to have the next token's label of the maxSequenceLength'th token - const maxLength = task.trainingInformation.maxSequenceLength ?? tokenizer.model_max_length as number - const maxLengthPlusLabel = maxLength + 1 - - let fixedLengthTokens = tf.tensor1d(tokens, 'int32') // cast tokens from float to int for gpt-tfjs - if (fixedLengthTokens.size > maxLengthPlusLabel) { // Should never happen because tokenization truncates inputs - throw Error("There are more tokens than expected after tokenization and truncation") - } else if (fixedLengthTokens.size < maxLengthPlusLabel) { // Pad inputs to fixed length - const paddingToken = tokenizer.pad_token_id - fixedLengthTokens = fixedLengthTokens.pad([[Math.max(0, maxLengthPlusLabel - fixedLengthTokens.size), 0]], paddingToken) - } - // if tokens.size == maxLengthPlusLabel we can leave it as it is - - // ys is a one-hot encoding of the next token (i.e. xs shifted by one) - // cast because oneHot isn't size-typing its return value - const ys = tf.oneHot(fixedLengthTokens.slice([1]), tokenizer.model.vocab.length + 1) as tf.Tensor2D - // remove the extra token now that ys is created - const xs = fixedLengthTokens.slice([0], maxLength) - return { xs, ys } - }) - } -} - -interface TokenizerOutput { - input_ids: number[] -} - -/** - * Tokenize and truncates input strings - */ -const tokenize: PreprocessingFunction = { - type: TextPreprocessing.Tokenize, - apply: async (x: Promise, task: Task): Promise<{ tokens: number[] }> => { - const xs = await x - if (typeof xs !== 'string') - throw new Error("The tokenize preprocessing expects a string as input") - - const tokenizer = await models.getTaskTokenizer(task) - // Add plus one to include the next token label of the last token in the input sequence - // The inputs are truncated down to exactly maxSequenceLength in leftPadding - const maxLength = task.trainingInformation.maxSequenceLength ?? (tokenizer.model_max_length as number) - const maxLengthPlusLabel = maxLength + 1 - - const {input_ids: tokens} = tokenizer(xs, { - // Transformers.js currently only supports right padding while we need left for text generation - // Right padding should be supported in the future, once it is, we can directly pad while tokenizing - // https://github.com/xenova/transformers.js/blob/8804c36591d11d8456788d1bb4b16489121b3be2/src/tokenizers.js#L2517 - padding: false, - truncation: true, - return_tensor: false, - max_length: maxLengthPlusLabel, - }) as TokenizerOutput - return { tokens } - } -} - -/** - * Available text preprocessing functions. - */ -export const AVAILABLE_PREPROCESSING = List.of( - tokenize, - leftPadding -).sortBy((e) => e.type) diff --git a/discojs/src/dataset/data/tabular_data.spec.ts b/discojs/src/dataset/data/tabular_data.spec.ts deleted file mode 100644 index f75ca6be5..000000000 --- a/discojs/src/dataset/data/tabular_data.spec.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { assert, expect } from 'chai' -import { Map, Set } from 'immutable' -import * as tf from '@tensorflow/tfjs' - -import { TabularData } from './tabular_data.js' -import { defaultTasks } from '../../index.js' - - -describe('tabular data checks', () => { - const titanicTask = defaultTasks.titanic.getTask() - - - const dataConfig = { - features: titanicTask.trainingInformation.inputColumns, - labels: [titanicTask.trainingInformation.outputColumn] - } - - const columnConfigs = Map( - Set(dataConfig.features).map((feature) => [feature, { required: false, isLabel: false }]) - ).merge( - Set(dataConfig.labels).map((label) => [label, { required: true, isLabel: true }]) - ) - - const csvConfig = { - hasHeader: true, - columnConfigs: columnConfigs.toObject(), - configuredColumnsOnly: true, - delimiter: ',' - } - - it('throw an error on incorrectly formatted data', async () => { - try { - await TabularData.init(tf.data.csv('file://../datasets/cifar10-labels.csv', csvConfig), titanicTask, 3) - } catch (e) { - expect(e).to.be.an.instanceOf(Error) - return - } - // no error means we failed - assert(false) - }) - - it('do nothing on correctly formatted data', async () => { - await TabularData.init(tf.data.csv('file://../datasets/titanic_train.csv', csvConfig), titanicTask, 3) - }) -}) diff --git a/discojs/src/dataset/data/tabular_data.ts b/discojs/src/dataset/data/tabular_data.ts deleted file mode 100644 index 0a3f45651..000000000 --- a/discojs/src/dataset/data/tabular_data.ts +++ /dev/null @@ -1,35 +0,0 @@ -import * as tf from '@tensorflow/tfjs' - -import type { Task } from '../../index.js' - -import { Data } from './data.js' -import { TABULAR_PREPROCESSING } from './preprocessing/index.js' - -/** - * Disco data made of tabular (.csv, .tsv, etc.) files. - */ -export class TabularData extends Data { - public readonly availablePreprocessing = TABULAR_PREPROCESSING - - static override async init ( - dataset: tf.data.Dataset, - task: Task, - size?: number - ): Promise { - // Force the check of the data column format (among other things) before proceeding - // to training, for better error handling. An incorrectly formatted line might still - // cause an error during training, because of the lazy aspect of the dataset; we only - // load/read the tabular file's lines on training. - try { - await dataset.iterator() - } catch (cause) { - throw new Error('data input format not compatible with chosen task', { cause }) - } - - return new TabularData(dataset, task, size) - } - - protected create (dataset: tf.data.Dataset, task: Task, size: number): TabularData { - return new TabularData(dataset, task, size) - } -} diff --git a/discojs/src/dataset/data/text_data.ts b/discojs/src/dataset/data/text_data.ts deleted file mode 100644 index daaee0848..000000000 --- a/discojs/src/dataset/data/text_data.ts +++ /dev/null @@ -1,25 +0,0 @@ -import * as tf from '@tensorflow/tfjs' - -import type { Task } from '../../index.js' - -import { Data } from './data.js' -import { TEXT_PREPROCESSING } from './preprocessing/index.js' - -/** - * Disco data made of textual samples. - */ -export class TextData extends Data { - public readonly availablePreprocessing = TEXT_PREPROCESSING - - static override init ( - dataset: tf.data.Dataset, - task: Task, - size?: number - ): Promise { - return Promise.resolve(new TextData(dataset, task, size)) - } - - protected create (dataset: tf.data.Dataset, task: Task, size?: number): TextData { - return new TextData(dataset, task, size) - } -} diff --git a/discojs/src/dataset/index.ts b/discojs/src/dataset/index.ts index 61da0ee09..3214f4050 100644 --- a/discojs/src/dataset/index.ts +++ b/discojs/src/dataset/index.ts @@ -1,15 +1,2 @@ export { Dataset } from "./dataset.js"; export * from "./types.js"; - -export { - Data, - TabularData, - ImageData, - TextData, - ImagePreprocessing, - TabularPreprocessing, - TextPreprocessing, - IMAGE_PREPROCESSING, - TABULAR_PREPROCESSING, - TEXT_PREPROCESSING, -} from "./data/index.js"; diff --git a/discojs/src/default_tasks/cifar10.ts b/discojs/src/default_tasks/cifar10.ts index ade49a4ef..d7e0259d8 100644 --- a/discojs/src/default_tasks/cifar10.ts +++ b/discojs/src/default_tasks/cifar10.ts @@ -1,7 +1,7 @@ import * as tf from '@tensorflow/tfjs' import type { Model, Task, TaskProvider } from '../index.js' -import { data, models } from '../index.js' +import { models } from '../index.js' import baseModel from '../models/mobileNet_v1_025_224.js' @@ -28,7 +28,6 @@ export const cifar10: TaskProvider = { validationSplit: 0.2, batchSize: 10, dataType: 'image', - preprocessingFunctions: [data.ImagePreprocessing.Resize, data.ImagePreprocessing.Normalize], IMAGE_H: 224, IMAGE_W: 224, LABEL_LIST: ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'], diff --git a/discojs/src/default_tasks/lus_covid.ts b/discojs/src/default_tasks/lus_covid.ts index 993372d11..2cc952b15 100644 --- a/discojs/src/default_tasks/lus_covid.ts +++ b/discojs/src/default_tasks/lus_covid.ts @@ -1,7 +1,7 @@ import * as tf from '@tensorflow/tfjs' import type { Model, Task, TaskProvider } from '../index.js' -import { data, models } from '../index.js' +import { models } from '../index.js' export const lusCovid: TaskProvider = { getTask (): Task { @@ -27,7 +27,6 @@ export const lusCovid: TaskProvider = { batchSize: 5, IMAGE_H: 100, IMAGE_W: 100, - preprocessingFunctions: [data.ImagePreprocessing.Resize, data.ImagePreprocessing.Normalize], LABEL_LIST: ['COVID-Positive', 'COVID-Negative'], dataType: 'image', scheme: 'federated', diff --git a/discojs/src/default_tasks/mnist.ts b/discojs/src/default_tasks/mnist.ts index 240fc8811..c6866994f 100644 --- a/discojs/src/default_tasks/mnist.ts +++ b/discojs/src/default_tasks/mnist.ts @@ -1,7 +1,7 @@ import * as tf from '@tensorflow/tfjs' import type { Model, Task, TaskProvider } from '../index.js' -import { data, models } from '../index.js' +import { models } from '../index.js' export const mnist: TaskProvider = { getTask (): Task { @@ -28,8 +28,6 @@ export const mnist: TaskProvider = { dataType: 'image', IMAGE_H: 28, IMAGE_W: 28, - // Images should already be at the right size but resizing just in case - preprocessingFunctions: [data.ImagePreprocessing.Resize, data.ImagePreprocessing.Normalize], LABEL_LIST: ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'], scheme: 'decentralized', aggregationStrategy: 'secure', diff --git a/discojs/src/default_tasks/simple_face.ts b/discojs/src/default_tasks/simple_face.ts index ce1bd5be2..d6a2e78a4 100644 --- a/discojs/src/default_tasks/simple_face.ts +++ b/discojs/src/default_tasks/simple_face.ts @@ -1,7 +1,7 @@ import * as tf from '@tensorflow/tfjs' import type { Model, Task, TaskProvider } from '../index.js' -import { data, models } from '../index.js' +import { models } from '../index.js' import baseModel from '../models/mobileNetV2_35_alpha_2_classes.js' export const simpleFace: TaskProvider = { @@ -25,7 +25,6 @@ export const simpleFace: TaskProvider = { roundDuration: 1, validationSplit: 0.2, batchSize: 10, - preprocessingFunctions: [data.ImagePreprocessing.Normalize], dataType: 'image', IMAGE_H: 200, IMAGE_W: 200, diff --git a/discojs/src/default_tasks/titanic.ts b/discojs/src/default_tasks/titanic.ts index 170e67f2c..e025b16f0 100644 --- a/discojs/src/default_tasks/titanic.ts +++ b/discojs/src/default_tasks/titanic.ts @@ -1,7 +1,7 @@ import * as tf from '@tensorflow/tfjs' import type { Model, Task, TaskProvider } from '../index.js' -import { data, models } from '../index.js' +import { models } from '../index.js' export const titanic: TaskProvider = { getTask (): Task { @@ -52,7 +52,6 @@ export const titanic: TaskProvider = { roundDuration: 2, validationSplit: 0.2, batchSize: 30, - preprocessingFunctions: [data.TabularPreprocessing.Sanitize], dataType: 'tabular', inputColumns: [ 'Age', diff --git a/discojs/src/default_tasks/wikitext.ts b/discojs/src/default_tasks/wikitext.ts index cdcfa3781..2866beca9 100644 --- a/discojs/src/default_tasks/wikitext.ts +++ b/discojs/src/default_tasks/wikitext.ts @@ -1,5 +1,5 @@ import type { Model, Task, TaskProvider } from '../index.js' -import { data, models } from '../index.js' +import { models } from '../index.js' export const wikitext: TaskProvider = { getTask (): Task { @@ -25,7 +25,6 @@ export const wikitext: TaskProvider = { }, trainingInformation: { dataType: 'text', - preprocessingFunctions: [data.TextPreprocessing.Tokenize, data.TextPreprocessing.LeftPadding], scheme: 'federated', aggregationStrategy: 'mean', minNbOfParticipants: 2, diff --git a/discojs/src/index.ts b/discojs/src/index.ts index c635074a6..7d7aead8c 100644 --- a/discojs/src/index.ts +++ b/discojs/src/index.ts @@ -20,8 +20,7 @@ export * as defaultTasks from './default_tasks/index.js' export * as async_iterator from "./utils/async_iterator.js" export { EventEmitter } from "./utils/event_emitter.js" -export { Dataset } from "./dataset/index.js"; -export * from "./dataset/types.js"; // TODO merge with above +export * from "./dataset/index.js"; export * from "./types.js"; export * as processing from "./processing/index.js"; diff --git a/discojs/src/processing/index.spec.ts b/discojs/src/processing/index.spec.ts new file mode 100644 index 000000000..217ef9433 --- /dev/null +++ b/discojs/src/processing/index.spec.ts @@ -0,0 +1,43 @@ +import { expect } from "chai"; + +import { Dataset, Task } from "../index.js"; + +import { preprocess } from "./index.js"; + +describe("preprocess", () => { + it("throws on missing column in tabular", async () => { + const task: Task = { + id: "task", + displayInformation: { + taskTitle: "", + summary: { preview: "", overview: "" }, + }, + trainingInformation: { + dataType: "tabular", + tensorBackend: "tfjs", + scheme: "local", + minNbOfParticipants: 1, + epochs: 1, + roundDuration: 1, + batchSize: 1, + validationSplit: 0, + inputColumns: ["a", "b"], + outputColumn: "c", + }, + }; + + const dataset = new Dataset([ + { a: "1", b: "2", c: "3" }, + { a: "4", b: "5" }, + ]); + + try { + const preprocessed = await preprocess(task, ["tabular", dataset]); + for await (const _ of preprocessed[1]); + } catch { + return; + } + + expect(false, "should have thrown").to.be.true; + }); +}); diff --git a/discojs/src/processing/text.spec.ts b/discojs/src/processing/text.spec.ts new file mode 100644 index 000000000..992cf1163 --- /dev/null +++ b/discojs/src/processing/text.spec.ts @@ -0,0 +1,44 @@ +import { expect } from "chai"; + +import { tokenizeAndLeftPad } from "./text.js"; +import { AutoTokenizer } from "@xenova/transformers"; +import { Repeat } from "immutable"; + +describe("text processing", () => { + const text = + "Hello world, a bc 1 2345, '? 976. Wikipedia is a free content online encyclopedia written and maintained by a community \n of volunteers, known as Wikipedians. Founded by Jimmy Wales and Larry Sanger on January 15, 2001, Wikipedia is hosted by the Wikimedia Foundation, an American nonprofit organization that employs a staff of over 700 people.[7]"; + const expectedTokens = [ + 15496, 995, 11, 257, 47125, 352, 2242, 2231, 11, 705, 30, 860, 4304, 13, + 15312, 318, 257, 1479, 2695, 2691, 45352, 3194, 290, 9456, 416, 257, 2055, + 220, 198, 286, 11661, 11, 1900, 355, 11145, 46647, 1547, 13, 4062, 276, 416, + 12963, 11769, 290, 13633, 311, 2564, 319, 3269, 1315, 11, 5878, 11, 15312, + 318, 12007, 416, 262, 44877, 5693, 11, 281, 1605, 15346, 4009, 326, 24803, + 257, 3085, 286, 625, 13037, 661, 3693, 22, 60, + ]; + + it("tokenizes text", async () => { + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/gpt2"); + + const tokens = tokenizeAndLeftPad(text, tokenizer, expectedTokens.length); + + expect(tokens.toArray()).to.be.deep.equal(expectedTokens); + }); + + it("tokenizes until wanted size", async () => { + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/gpt2"); + + const tokens = tokenizeAndLeftPad(text, tokenizer, 10); + + expect(tokens.toArray()).to.be.deep.equal(expectedTokens.slice(0, 10)); + }); + + it("pads until enough token are generated", async () => { + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/gpt2"); + + const tokens = tokenizeAndLeftPad("", tokenizer, 10); + + expect(tokens.toArray()).to.be.deep.equal( + Repeat(tokenizer.pad_token_id, 10).toArray(), + ); + }); +}); diff --git a/discojs/src/task/training_information.ts b/discojs/src/task/training_information.ts index 15807724a..9710dee5d 100644 --- a/discojs/src/task/training_information.ts +++ b/discojs/src/task/training_information.ts @@ -1,4 +1,3 @@ -import type { Preprocessing } from '../dataset/data/preprocessing/index.js' import { PreTrainedTokenizer } from '@xenova/transformers'; interface Privacy { @@ -18,8 +17,6 @@ export interface TrainingInformation { validationSplit: number // batchSize: batch size of training data batchSize: number - // preprocessingFunctions: preprocessing functions such as resize and normalize - preprocessingFunctions?: Preprocessing[] // dataType, 'image', 'tabular' or 'text' dataType: 'image' | 'tabular' | 'text' // inputColumns: for tabular data, the columns to be chosen as input data for the model @@ -109,7 +106,6 @@ export function isTrainingInformation (raw: unknown): raw is TrainingInformation maxShareValue, minNbOfParticipants, outputColumn, - preprocessingFunctions, roundDuration, scheme, validationSplit, @@ -134,8 +130,7 @@ export function isTrainingInformation (raw: unknown): raw is TrainingInformation (IMAGE_W !== undefined && typeof IMAGE_W !== 'number') || (outputColumn !== undefined && typeof outputColumn !== 'string') || (LABEL_LIST !== undefined && !isStringArray(LABEL_LIST)) || - (inputColumns !== undefined && !isStringArray(inputColumns)) || - (preprocessingFunctions !== undefined && !Array.isArray(preprocessingFunctions)) + (inputColumns !== undefined && !isStringArray(inputColumns)) ) { return false } @@ -193,7 +188,6 @@ export function isTrainingInformation (raw: unknown): raw is TrainingInformation maxShareValue, minNbOfParticipants, outputColumn, - preprocessingFunctions, roundDuration, scheme, validationSplit, From f02b7e87020fb366e35daebfab5908bd44bbb2ea Mon Sep 17 00:00:00 2001 From: tharvik Date: Fri, 13 Sep 2024 17:26:00 +0200 Subject: [PATCH 17/31] discojs/task: generic on datatype --- cli/src/args.ts | 4 +- cli/src/benchmark_gpt.ts | 4 +- cli/src/cli.ts | 11 +- cli/src/data.ts | 30 +- discojs/src/default_tasks/cifar10.ts | 4 +- discojs/src/default_tasks/lus_covid.ts | 4 +- discojs/src/default_tasks/mnist.ts | 4 +- discojs/src/default_tasks/simple_face.ts | 4 +- discojs/src/default_tasks/titanic.ts | 4 +- discojs/src/default_tasks/wikitext.ts | 4 +- discojs/src/models/tokenizer.ts | 4 +- discojs/src/processing/index.spec.ts | 6 +- discojs/src/processing/index.ts | 224 ++++++------- discojs/src/task/task.ts | 5 +- discojs/src/task/task_provider.ts | 8 +- discojs/src/task/training_information.ts | 277 ++++++++------- discojs/src/training/disco.ts | 98 +++--- discojs/src/training/trainer.ts | 54 +-- discojs/src/validator.ts | 43 +-- docs/examples/training.ts | 39 ++- docs/examples/wikitext.ts | 6 +- server/tests/client/federated.spec.ts | 4 +- server/tests/e2e/decentralized.spec.ts | 6 +- server/tests/e2e/federated.spec.ts | 20 +- server/tests/validator.spec.ts | 6 +- webapp/cypress/e2e/datasetInput.cy.ts | 8 +- webapp/cypress/support/e2e.ts | 20 +- .../dataset_input/FileSelection.vue | 2 +- .../dataset_input/LabeledDatasetInput.vue | 92 +++++ .../LabeledImageDatasetInput/ByGroup.vue | 62 ++-- .../dataset_input/UnlabeledDatasetInput.vue | 80 +++++ webapp/src/components/dataset_input/types.ts | 21 +- .../src/components/dataset_input/validate.ts | 16 + .../src/components/testing/PredictSteps.vue | 231 ++++++------- webapp/src/components/testing/TestSteps.vue | 314 ++++++++---------- webapp/src/components/testing/Testing.vue | 12 +- .../testing/__tests__/Testing.spec.ts | 1 + webapp/src/components/training/Trainer.vue | 12 +- .../src/components/training/TrainingSteps.vue | 106 ++---- .../training/__tests__/Trainer.spec.ts | 13 +- 40 files changed, 1017 insertions(+), 846 deletions(-) create mode 100644 webapp/src/components/dataset_input/LabeledDatasetInput.vue create mode 100644 webapp/src/components/dataset_input/UnlabeledDatasetInput.vue create mode 100644 webapp/src/components/dataset_input/validate.ts diff --git a/cli/src/args.ts b/cli/src/args.ts index 1dfcbbe24..a55aac03e 100644 --- a/cli/src/args.ts +++ b/cli/src/args.ts @@ -37,7 +37,7 @@ const unsafeArgs = parse( ) const supportedTasks = Map( - Set.of( + Set.of | TaskProvider<"tabular">>( defaultTasks.cifar10, defaultTasks.lusCovid, defaultTasks.simpleFace, @@ -67,6 +67,6 @@ export const args: BenchmarkArguments = { return task; }, - getModel: provider.getModel, + getModel: () => provider.getModel(), }, }; diff --git a/cli/src/benchmark_gpt.ts b/cli/src/benchmark_gpt.ts index b86f69852..1d953e837 100644 --- a/cli/src/benchmark_gpt.ts +++ b/cli/src/benchmark_gpt.ts @@ -2,7 +2,7 @@ import { List } from "immutable"; import { parse } from "ts-command-line-args"; import { AutoTokenizer } from "@xenova/transformers"; -import { fetchTasks, models, async_iterator, defaultTasks, processing } from "@epfml/discojs"; +import { fetchTasks, models, async_iterator, defaultTasks, processing, Task } from "@epfml/discojs"; import { loadModelFromDisk, loadText } from '@epfml/discojs-node' import { Server } from "server"; @@ -50,7 +50,7 @@ async function main(args: Required): Promise { // Fetch the wikitext task from the server const tasks = await fetchTasks(url) - const task = tasks.get('llm_task') + const task = tasks.get('llm_task') as Task<'text'> | undefined if (task === undefined) { throw new Error('task not found') } const tokenizerName = task.trainingInformation.tokenizer diff --git a/cli/src/cli.ts b/cli/src/cli.ts index 5ce11f8bb..4e3fd147d 100644 --- a/cli/src/cli.ts +++ b/cli/src/cli.ts @@ -4,7 +4,7 @@ import "@tensorflow/tfjs-node" import { List, Range } from 'immutable' import fs from 'node:fs/promises' -import type { RoundLogs, Task, TaskProvider, TypedRawDataset } from '@epfml/discojs' +import type { Dataset, DataType, Raw, RoundLogs, Task, TaskProvider } from '@epfml/discojs' import { Disco, aggregator as aggregators, client as clients } from '@epfml/discojs' import { Server } from 'server' @@ -18,10 +18,10 @@ async function arrayFromAsync(iter: AsyncIterable): Promise { return ret; } -async function runUser( +async function runUser( task: Task, url: URL, - data: TypedRawDataset, + data: Dataset, ): Promise> { const trainingScheme = task.trainingInformation.scheme const aggregator = aggregators.getAggregator(task) @@ -34,7 +34,10 @@ async function runUser( return logs; } -async function main (provider: TaskProvider, numberOfUsers: number): Promise { +async function main( + provider: TaskProvider, + numberOfUsers: number, +): Promise { const task = provider.getTask() console.log(`Started ${task.trainingInformation.scheme} training of ${task.id}`) console.log({ args }) diff --git a/cli/src/data.ts b/cli/src/data.ts index 88eb49525..20953bb8b 100644 --- a/cli/src/data.ts +++ b/cli/src/data.ts @@ -1,10 +1,10 @@ import path from "node:path"; -import type { Dataset, Image, Task, TypedRawDataset } from "@epfml/discojs"; +import type { Dataset, DataType, Image, Raw, Task } from "@epfml/discojs"; import { loadCSV, loadImagesInDir } from "@epfml/discojs-node"; import { Repeat } from "immutable"; -async function loadSimpleFaceData(): Promise> { +async function loadSimpleFaceData(): Promise> { const folder = path.join("..", "datasets", "simple_face"); const [adults, childs]: Dataset<[Image, string]>[] = [ @@ -15,7 +15,7 @@ async function loadSimpleFaceData(): Promise> { return adults.chain(childs); } -async function loadLusCovidData(): Promise> { +async function loadLusCovidData(): Promise> { const folder = path.join("..", "datasets", "lus_covid"); const [positive, negative]: Dataset<[Image, string]>[] = [ @@ -30,24 +30,22 @@ async function loadLusCovidData(): Promise> { return positive.chain(negative); } -export async function getTaskData(task: Task): Promise { +export async function getTaskData( + task: Task, +): Promise> { switch (task.id) { case "simple_face": - return ["image", await loadSimpleFaceData()]; + return (await loadSimpleFaceData()) as Dataset; case "titanic": - return [ - "tabular", - loadCSV(path.join("..", "datasets", "titanic_train.csv")), - ]; + return loadCSV( + path.join("..", "datasets", "titanic_train.csv"), + ) as Dataset; case "cifar10": - return [ - "image", - (await loadImagesInDir(path.join("..", "datasets", "CIFAR10"))).zip( - Repeat("cat"), - ), - ]; + return ( + await loadImagesInDir(path.join("..", "datasets", "CIFAR10")) + ).zip(Repeat("cat")) as Dataset; case "lus_covid": - return ["image", await loadLusCovidData()]; + return (await loadLusCovidData()) as Dataset; default: throw new Error(`Data loader for ${task.id} not implemented.`); } diff --git a/discojs/src/default_tasks/cifar10.ts b/discojs/src/default_tasks/cifar10.ts index d7e0259d8..b644b6e93 100644 --- a/discojs/src/default_tasks/cifar10.ts +++ b/discojs/src/default_tasks/cifar10.ts @@ -5,8 +5,8 @@ import { models } from '../index.js' import baseModel from '../models/mobileNet_v1_025_224.js' -export const cifar10: TaskProvider = { - getTask (): Task { +export const cifar10: TaskProvider<'image'> = { + getTask (): Task<'image'> { return { id: 'cifar10', displayInformation: { diff --git a/discojs/src/default_tasks/lus_covid.ts b/discojs/src/default_tasks/lus_covid.ts index 2cc952b15..44dd46ed6 100644 --- a/discojs/src/default_tasks/lus_covid.ts +++ b/discojs/src/default_tasks/lus_covid.ts @@ -3,8 +3,8 @@ import * as tf from '@tensorflow/tfjs' import type { Model, Task, TaskProvider } from '../index.js' import { models } from '../index.js' -export const lusCovid: TaskProvider = { - getTask (): Task { +export const lusCovid: TaskProvider<'image'> = { + getTask (): Task<'image'> { return { id: 'lus_covid', displayInformation: { diff --git a/discojs/src/default_tasks/mnist.ts b/discojs/src/default_tasks/mnist.ts index c6866994f..f5b46a1cc 100644 --- a/discojs/src/default_tasks/mnist.ts +++ b/discojs/src/default_tasks/mnist.ts @@ -3,8 +3,8 @@ import * as tf from '@tensorflow/tfjs' import type { Model, Task, TaskProvider } from '../index.js' import { models } from '../index.js' -export const mnist: TaskProvider = { - getTask (): Task { +export const mnist: TaskProvider<'image'> = { + getTask (): Task<'image'> { return { id: 'mnist', displayInformation: { diff --git a/discojs/src/default_tasks/simple_face.ts b/discojs/src/default_tasks/simple_face.ts index d6a2e78a4..a87825e5d 100644 --- a/discojs/src/default_tasks/simple_face.ts +++ b/discojs/src/default_tasks/simple_face.ts @@ -4,8 +4,8 @@ import type { Model, Task, TaskProvider } from '../index.js' import { models } from '../index.js' import baseModel from '../models/mobileNetV2_35_alpha_2_classes.js' -export const simpleFace: TaskProvider = { - getTask (): Task { +export const simpleFace: TaskProvider<'image'> = { + getTask (): Task<'image'> { return { id: 'simple_face', displayInformation: { diff --git a/discojs/src/default_tasks/titanic.ts b/discojs/src/default_tasks/titanic.ts index e025b16f0..b9462ee50 100644 --- a/discojs/src/default_tasks/titanic.ts +++ b/discojs/src/default_tasks/titanic.ts @@ -3,8 +3,8 @@ import * as tf from '@tensorflow/tfjs' import type { Model, Task, TaskProvider } from '../index.js' import { models } from '../index.js' -export const titanic: TaskProvider = { - getTask (): Task { +export const titanic: TaskProvider<'tabular'> = { + getTask (): Task<'tabular'> { return { id: 'titanic', displayInformation: { diff --git a/discojs/src/default_tasks/wikitext.ts b/discojs/src/default_tasks/wikitext.ts index 2866beca9..3d7760b09 100644 --- a/discojs/src/default_tasks/wikitext.ts +++ b/discojs/src/default_tasks/wikitext.ts @@ -1,8 +1,8 @@ import type { Model, Task, TaskProvider } from '../index.js' import { models } from '../index.js' -export const wikitext: TaskProvider = { - getTask (): Task { +export const wikitext: TaskProvider<'text'> = { + getTask (): Task<'text'> { return { id: 'llm_task', displayInformation: { diff --git a/discojs/src/models/tokenizer.ts b/discojs/src/models/tokenizer.ts index 00d1aa967..de27ccc26 100644 --- a/discojs/src/models/tokenizer.ts +++ b/discojs/src/models/tokenizer.ts @@ -12,7 +12,7 @@ import { AutoTokenizer, PreTrainedTokenizer, env } from '@xenova/transformers'; * @param task the task object specifying which tokenizer to use * @returns an initialized tokenizer object */ -export async function getTaskTokenizer(task: Task): Promise { +export async function getTaskTokenizer(task: Task<'text'>): Promise { let tokenizer = task.trainingInformation.tokenizer if (tokenizer === undefined) throw Error('No tokenizer specified in the task training information') if (typeof tokenizer == 'string') { @@ -25,4 +25,4 @@ export async function getTaskTokenizer(task: Task): Promise task.trainingInformation.tokenizer = tokenizer } return tokenizer -} \ No newline at end of file +} diff --git a/discojs/src/processing/index.spec.ts b/discojs/src/processing/index.spec.ts index 217ef9433..8e4e67586 100644 --- a/discojs/src/processing/index.spec.ts +++ b/discojs/src/processing/index.spec.ts @@ -6,7 +6,7 @@ import { preprocess } from "./index.js"; describe("preprocess", () => { it("throws on missing column in tabular", async () => { - const task: Task = { + const task: Task<"tabular"> = { id: "task", displayInformation: { taskTitle: "", @@ -32,8 +32,8 @@ describe("preprocess", () => { ]); try { - const preprocessed = await preprocess(task, ["tabular", dataset]); - for await (const _ of preprocessed[1]); + const preprocessed = await preprocess(task, dataset); + for await (const _ of preprocessed); } catch { return; } diff --git a/discojs/src/processing/index.ts b/discojs/src/processing/index.ts index 6172329ed..94684dad3 100644 --- a/discojs/src/processing/index.ts +++ b/discojs/src/processing/index.ts @@ -3,14 +3,15 @@ import { List } from "immutable"; import type { + Dataset, + DataType, + Inferred, + ModelEncoded, + Raw, + RawWithoutLabel, Tabular, Task, - TypedInferredDataset, - TypedModelEncodedDataset, - TypedModelEncodedOnlyWithLabelDataset, - TypedModelEncodedWithoutLabelDataset, - TypedRawDataset, - TypedRawWithoutLabelDataset, + TrainingInformation, } from "../index.js"; import { models } from "../index.js"; @@ -20,147 +21,142 @@ export * from "./image.js"; export * from "./tabular.js"; export * from "./text.js"; -export async function preprocess( - task: Task, - [t, dataset]: TypedRawDataset, -): Promise { - switch (t) { +export async function preprocess( + task: Task, + dataset: Dataset, +): Promise> { + switch (task.trainingInformation.dataType) { case "image": { - const { LABEL_LIST, IMAGE_H, IMAGE_W } = task.trainingInformation; - if ( - IMAGE_H === undefined || - IMAGE_W === undefined || - LABEL_LIST === undefined - ) - throw new Error("task is missing fields for image dataset"); - - return [ - "image", - dataset.map(([image, label]) => [ - processing.normalize( - processing.removeAlpha(processing.resize(IMAGE_W, IMAGE_H, image)), - ), - processing.indexInList(label, LABEL_LIST), - ]), - ]; + // cast as typescript doesn't reduce generic type + const d = dataset as Dataset; + const { IMAGE_H, IMAGE_W, LABEL_LIST } = + task.trainingInformation as TrainingInformation<"image">; + + return d.map(([image, label]) => [ + processing.normalize( + processing.removeAlpha(processing.resize(IMAGE_W, IMAGE_H, image)), + ), + processing.indexInList(label, LABEL_LIST), + ]) as Dataset; } case "tabular": { - const { inputColumns, outputColumn } = task.trainingInformation; - if (inputColumns === undefined || outputColumn === undefined) - throw new Error("tabular task without input and output columns"); - - return [ - "tabular", - dataset.map((row) => { - const output = processing.extractColumn(row, outputColumn); - - return [ - extractToNumbers(inputColumns, row), - // TODO sanitization doesn't care about column distribution - output !== "" ? processing.convertToNumber(output) : 0, - ]; - }), - ]; + // cast as typescript doesn't reduce generic type + const d = dataset as Dataset; + const { inputColumns, outputColumn } = + task.trainingInformation as TrainingInformation<"tabular">; + + return d.map((row) => { + const output = processing.extractColumn(row, outputColumn); + + return [ + extractToNumbers(inputColumns, row), + // TODO sanitization doesn't care about column distribution + output !== "" ? processing.convertToNumber(output) : 0, + ]; + }) as Dataset; } case "text": { - const tokenizer = await models.getTaskTokenizer(task); + // cast as typescript doesn't reduce generic type + const d = dataset as Dataset; + const t = task as Task<"text">; + + const tokenizer = await models.getTaskTokenizer(t); const totalTokenCount = task.trainingInformation.maxSequenceLength ?? (tokenizer.model_max_length as number); - return [ - "text", - dataset - .map((line) => - processing.tokenizeAndLeftPad(line, tokenizer, totalTokenCount), - ) - .map((tokens) => [tokens.pop(), tokens.last()]), - ]; + return d + .map((line) => + processing.tokenizeAndLeftPad(line, tokenizer, totalTokenCount), + ) + .map((tokens) => [tokens.pop(), tokens.last()]) as Dataset< + ModelEncoded[D] + >; } } } -export async function preprocessWithoutLabel( - task: Task, - [t, dataset]: TypedRawWithoutLabelDataset, -): Promise { - switch (t) { +export async function preprocessWithoutLabel( + task: Task, + dataset: Dataset, +): Promise> { + switch (task.trainingInformation.dataType) { case "image": { - const { IMAGE_H, IMAGE_W } = task.trainingInformation; - if (IMAGE_H === undefined || IMAGE_W === undefined) - throw new Error("task is missing fields for image dataset"); - - return [ - "image", - dataset.map((image) => - processing.normalize( - processing.removeAlpha(processing.resize(IMAGE_W, IMAGE_H, image)), - ), + // cast as typescript doesn't reduce generic type + const d = dataset as Dataset; + const { IMAGE_H, IMAGE_W } = + task.trainingInformation as TrainingInformation<"image">; + + return d.map((image) => + processing.normalize( + processing.removeAlpha(processing.resize(IMAGE_W, IMAGE_H, image)), ), - ]; + ); } case "tabular": { - const { inputColumns } = task.trainingInformation; - if (inputColumns === undefined) - throw new Error("tabular task without input columns"); - - return [ - "tabular", - dataset.map((row) => extractToNumbers(inputColumns, row)), - ]; + // cast as typescript doesn't reduce generic type + const d = dataset as Dataset; + const { inputColumns } = + task.trainingInformation as TrainingInformation<"tabular">; + + return d.map((row) => extractToNumbers(inputColumns, row)); } case "text": { - const tokenizer = await models.getTaskTokenizer(task); + // cast as typescript doesn't reduce generic type + const d = dataset as Dataset; + const t = task as Task<"text">; + + const tokenizer = await models.getTaskTokenizer(t); const totalTokenCount = - task.trainingInformation.maxSequenceLength ?? + t.trainingInformation.maxSequenceLength ?? (tokenizer.model_max_length as number); - return [ - "text", - dataset - .map((line) => - processing.tokenizeAndLeftPad(line, tokenizer, totalTokenCount), - ) - .map((tokens) => tokens.pop()), - ]; + return d + .map((line) => + processing.tokenizeAndLeftPad(line, tokenizer, totalTokenCount), + ) + .map((tokens) => tokens.pop()); } } } -export async function postprocess( - task: Task, - [t, dataset]: TypedModelEncodedOnlyWithLabelDataset, -): Promise { - switch (t) { +export async function postprocess( + task: Task, + dataset: Dataset, +): Promise> { + switch (task.trainingInformation.dataType) { case "image": { - const { LABEL_LIST } = task.trainingInformation; - if (LABEL_LIST === undefined) - throw new Error("task is missing fields for image dataset"); + // cast as typescript doesn't reduce generic type + const d = dataset as Dataset; + const { LABEL_LIST } = + task.trainingInformation as TrainingInformation<"image">; const labels = List(LABEL_LIST); - return [ - "image", - dataset.map((index) => { - const v = labels.get(index); - if (v === undefined) throw new Error("index not found in labels"); - return v; - }), - ]; + return d.map((index) => { + const v = labels.get(index); + if (v === undefined) throw new Error("index not found in labels"); + return v; + }) as Dataset; } case "tabular": { - const { outputColumn } = task.trainingInformation; - if (outputColumn === undefined) - throw new Error("tabular task without input columns"); - - return [ - "tabular", - dataset.map((row) => Object.fromEntries([[outputColumn, row]])), - ]; + // cast as typescript doesn't reduce generic type + const d = dataset as Dataset; + const { outputColumn } = + task.trainingInformation as TrainingInformation<"tabular">; + + return d.map((row) => + Object.fromEntries([[outputColumn, row]]), + ) as Dataset; } case "text": { - const tokenizer = await models.getTaskTokenizer(task); - - return ["text", dataset.map((token) => tokenizer.decode([token]))]; + // cast as typescript doesn't reduce generic type + const d = dataset as Dataset; + const t = task as Task<"text">; + const tokenizer = await models.getTaskTokenizer(t); + + return d.map((token) => tokenizer.decode([token])) as Dataset< + Inferred[D] + >; } } } diff --git a/discojs/src/task/task.ts b/discojs/src/task/task.ts index e5ca12aef..1684ce498 100644 --- a/discojs/src/task/task.ts +++ b/discojs/src/task/task.ts @@ -1,12 +1,13 @@ +import { DataType } from '../types.js' import { isDisplayInformation, type DisplayInformation } from './display_information.js' import { isTrainingInformation, type TrainingInformation } from './training_information.js' export type TaskID = string -export interface Task { +export interface Task { id: TaskID displayInformation: DisplayInformation - trainingInformation: TrainingInformation + trainingInformation: TrainingInformation } export function isTaskID (obj: unknown): obj is TaskID { diff --git a/discojs/src/task/task_provider.ts b/discojs/src/task/task_provider.ts index 64ad74171..c9b4dd31f 100644 --- a/discojs/src/task/task_provider.ts +++ b/discojs/src/task/task_provider.ts @@ -1,7 +1,7 @@ -import type { Model, Task } from '../index.js' +import type { DataType, Model, Task } from "../index.js"; -export interface TaskProvider { - getTask: () => Task +export interface TaskProvider { + getTask(): Task; // Create the corresponding model ready for training (compiled) - getModel: () => Promise + getModel(): Promise>; } diff --git a/discojs/src/task/training_information.ts b/discojs/src/task/training_information.ts index 9710dee5d..b8919d8e2 100644 --- a/discojs/src/task/training_information.ts +++ b/discojs/src/task/training_information.ts @@ -1,4 +1,5 @@ -import { PreTrainedTokenizer } from '@xenova/transformers'; +import { PreTrainedTokenizer } from "@xenova/transformers"; +import { DataType } from "../types.js"; interface Privacy { // maximum weights difference between each round @@ -7,61 +8,66 @@ interface Privacy { noiseScale?: number; } -export interface TrainingInformation { +export type TrainingInformation = { // epochs: number of epochs to run training for - epochs: number - // roundDuration: number of epochs between each weight sharing round. + epochs: number; + // roundDuration: number of epochs between each weight sharing round. // e.g.if 3 then weights are shared every 3 epochs (in the distributed setting). - roundDuration: number + roundDuration: number; // validationSplit: fraction of data to keep for validation, note this only works for image data - validationSplit: number + validationSplit: number; // batchSize: batch size of training data - batchSize: number - // dataType, 'image', 'tabular' or 'text' - dataType: 'image' | 'tabular' | 'text' - // inputColumns: for tabular data, the columns to be chosen as input data for the model - inputColumns?: string[] - // outputColumn: for tabular data, the column to be predicted by the model - outputColumn?: string - // IMAGE_H height of image (or RESIZED_IMAGE_H if ImagePreprocessing.Resize in preprocessingFunctions) - IMAGE_H?: number - // IMAGE_W width of image (or RESIZED_IMAGE_W if ImagePreprocessing.Resize in preprocessingFunctions) - IMAGE_W?: number - // LABEL_LIST of classes, e.g. if two class of images, one with dogs and one with cats, then we would - // define ['dogs', 'cats']. - LABEL_LIST?: string[] + batchSize: number; // scheme: Distributed training scheme, i.e. Federated and Decentralized - scheme: 'decentralized' | 'federated' | 'local' + scheme: "decentralized" | "federated" | "local"; // use Differential Privacy, reduce training accuracy and improve privacy. privacy?: Privacy; // maxShareValue: Secure Aggregation: maximum absolute value of a number in a randomly generated share // default is 100, must be a positive number, check the docs/PRIVACY.md file for more information on significance of maxShareValue selection // only relevant if secure aggregation is true (for either federated or decentralized learning) - maxShareValue?: number + maxShareValue?: number; // minNbOfParticipants: minimum number of participants required to train collaboratively // In decentralized Learning the default is 3, in federated learning it is 2 - minNbOfParticipants: number + minNbOfParticipants: number; // aggregationStrategy: aggregator to be used by the server for federated learning, or by the peers for decentralized learning // default is 'mean' - aggregationStrategy?: 'mean' | 'secure' - // tokenizer (string | PreTrainedTokenizer). This field should be initialized with the name of a Transformers.js pre-trained tokenizer, e.g., 'Xenova/gpt2'. - // When the tokenizer is first called, the actual object will be initialized and loaded into this field for the subsequent tokenizations. - tokenizer?: string | PreTrainedTokenizer - // maxSequenceLength: the maximum length of a input string used as input to a GPT model. It is used during preprocessing to - // truncate strings to a maximum length. The default value is tokenizer.model_max_length - maxSequenceLength?: number + aggregationStrategy?: "mean" | "secure"; // Tensor framework used by the model - tensorBackend: 'tfjs' | 'gpt' -} - -function isStringArray(raw: unknown): raw is string[] { - if (!Array.isArray(raw)) { - return false - } - const arr: unknown[] = raw // isArray is unsafely guarding with any[] - - return arr.every((e) => typeof e === 'string') + tensorBackend: "tfjs" | "gpt"; +} & DataTypeToTrainingInformation[D]; + +interface DataTypeToTrainingInformation { + image: { + dataType: "image"; + + // LABEL_LIST of classes, e.g. if two class of images, one with dogs and one with cats, then we would + // define ['dogs', 'cats']. + LABEL_LIST: string[]; + // IMAGE_H height of image (or RESIZED_IMAGE_H if ImagePreprocessing.Resize in preprocessingFunctions) + IMAGE_H: number; + // IMAGE_W width of image (or RESIZED_IMAGE_W if ImagePreprocessing.Resize in preprocessingFunctions) + IMAGE_W: number; + }; + tabular: { + dataType: "tabular"; + + // inputColumns: for tabular data, the columns to be chosen as input data for the model + inputColumns: string[]; + // outputColumns: for tabular data, the columns to be predicted by the model + outputColumn: string; + }; + text: { + dataType: "text"; + + // tokenizer (string | PreTrainedTokenizer). This field should be initialized with the name of a Transformers.js pre-trained tokenizer, e.g., 'Xenova/gpt2'. + // When the tokenizer is first called, the actual object will be initialized and loaded into this field for the subsequent tokenizations. + tokenizer: string | PreTrainedTokenizer; + + // maxSequenceLength: the maximum length of a input string used as input to a GPT model. It is used during preprocessing to + // truncate strings to a maximum length. The default value is tokenizer.model_max_length + maxSequenceLength?: number; + }; } function isPrivacy(raw: unknown): raw is Privacy { @@ -88,115 +94,158 @@ function isPrivacy(raw: unknown): raw is Privacy { return true; } -export function isTrainingInformation (raw: unknown): raw is TrainingInformation { - if (typeof raw !== 'object' || raw === null) { - return false +export function isTrainingInformation( + raw: unknown, +): raw is TrainingInformation { + if (typeof raw !== "object" || raw === null) { + return false; } const { - IMAGE_H, - IMAGE_W, - LABEL_LIST, aggregationStrategy, batchSize, dataType, privacy, epochs, - inputColumns, maxShareValue, minNbOfParticipants, - outputColumn, roundDuration, scheme, validationSplit, - tokenizer, - maxSequenceLength, - tensorBackend - }: Partial> = raw + tensorBackend, + }: Partial> = raw; if ( - typeof dataType !== 'string' || - typeof epochs !== 'number' || - typeof batchSize !== 'number' || - typeof roundDuration !== 'number' || - typeof validationSplit !== 'number' || - typeof minNbOfParticipants !== 'number' || - (tokenizer !== undefined && typeof tokenizer !== 'string' && !(tokenizer instanceof PreTrainedTokenizer)) || - (maxSequenceLength !== undefined && typeof maxSequenceLength !== 'number') || - (aggregationStrategy !== undefined && typeof aggregationStrategy !== 'string') || + typeof epochs !== "number" || + typeof batchSize !== "number" || + typeof roundDuration !== "number" || + typeof validationSplit !== "number" || + typeof minNbOfParticipants !== "number" || (privacy !== undefined && !isPrivacy(privacy)) || - (maxShareValue !== undefined && typeof maxShareValue !== 'number') || - (IMAGE_H !== undefined && typeof IMAGE_H !== 'number') || - (IMAGE_W !== undefined && typeof IMAGE_W !== 'number') || - (outputColumn !== undefined && typeof outputColumn !== 'string') || - (LABEL_LIST !== undefined && !isStringArray(LABEL_LIST)) || - (inputColumns !== undefined && !isStringArray(inputColumns)) + (maxShareValue !== undefined && typeof maxShareValue !== "number") ) { - return false - } - - if (aggregationStrategy !== undefined) { - switch (aggregationStrategy) { - case 'mean': break - case 'secure': break - default: return false - } - } - - switch (dataType) { - case 'image': break - case 'tabular': break - case 'text': break - default: return false + return false; } - // interdependencies on data type - if (dataType === 'image') { - if (typeof IMAGE_H !== 'number' || typeof IMAGE_W !== 'number') { - return false - } - } else if (dataType in ['text', 'tabular']) { - if (!(Array.isArray(inputColumns) && inputColumns.every((e) => typeof e === 'string'))) { - return false - } - if (outputColumn === undefined) return false + switch (aggregationStrategy) { + case undefined: + case "mean": + case "secure": + break; + default: + return false; } switch (tensorBackend) { - case 'tfjs': break - case 'gpt': break - default: return false + case "tfjs": + case "gpt": + break; + default: + return false; } - + switch (scheme) { - case 'decentralized': break - case 'federated': break - case 'local': break - default: return false + case "decentralized": + case "federated": + case "local": + break; + default: + return false; } const repack = { - IMAGE_W, - IMAGE_H, - LABEL_LIST, aggregationStrategy, batchSize, - dataType, - privacy, epochs, - inputColumns, maxShareValue, minNbOfParticipants, - outputColumn, + privacy, roundDuration, scheme, + tensorBackend, validationSplit, - tokenizer, - maxSequenceLength, - tensorBackend + }; + + switch (dataType) { + case "image": { + type ImageOnly = Omit< + TrainingInformation<"image">, + keyof TrainingInformation + >; + + const { LABEL_LIST, IMAGE_W, IMAGE_H }: Partial = raw; + + if ( + !( + Array.isArray(LABEL_LIST) && + LABEL_LIST.every((e) => typeof e === "string") + ) || + typeof IMAGE_H !== "number" || + typeof IMAGE_W !== "number" + ) + return false; + + const _: TrainingInformation<"image"> = { + ...repack, + dataType, + LABEL_LIST, + IMAGE_W, + IMAGE_H, + } satisfies Record, unknown>; + + return true; + } + case "tabular": { + type TabularOnly = Omit< + TrainingInformation<"tabular">, + keyof TrainingInformation + >; + + const { inputColumns, outputColumn }: Partial = raw; + + if ( + !( + Array.isArray(inputColumns) && + inputColumns.every((e) => typeof e === "string") + ) || + typeof outputColumn !== "string" + ) + return false; + + const _: TrainingInformation<"tabular"> = { + ...repack, + dataType, + inputColumns, + outputColumn, + } satisfies Record, unknown>; + + return true; + } + case "text": { + const { + maxSequenceLength, + tokenizer, + }: Partial, keyof TrainingInformation>> = + raw; + + if ( + (typeof tokenizer !== "string" && + !(tokenizer instanceof PreTrainedTokenizer)) || + (maxSequenceLength !== undefined && + typeof maxSequenceLength !== "number") + ) + return false; + + const _: TrainingInformation<"text"> = { + ...repack, + dataType, + maxSequenceLength, + tokenizer, + } satisfies Record, unknown>; + + return true; + } } - const _correct: TrainingInformation = repack - const _total: Record = repack - return true + return false; } diff --git a/discojs/src/training/disco.ts b/discojs/src/training/disco.ts index 4f197633b..97b7b68cf 100644 --- a/discojs/src/training/disco.ts +++ b/discojs/src/training/disco.ts @@ -11,9 +11,11 @@ import { } from "../index.js"; import type { Batched, + DataType, + Model, + ModelEncoded, + Raw, Task, - TypedBatchedModelEncodedDataset, - TypedRawDataset, } from "../index.js"; import type { Aggregator } from "../aggregator/index.js"; import { getAggregator } from "../aggregator/index.js"; @@ -40,23 +42,26 @@ export type RoundStatus = 'not enough participants' | // Server notification to * a convenient object providing a reduced yet complete API that wraps model training and * communication with nodes. */ -export class Disco extends EventEmitter<{'status': RoundStatus}>{ - public readonly trainer: Trainer; +export class Disco extends EventEmitter<{ + status: RoundStatus; +}> { + public readonly trainer: Trainer; readonly #client: clients.Client; readonly #logger: Logger; - readonly #task: Task; + readonly #task: Task; readonly #preprocessOnce: boolean; /** * Connect to the given task and get ready to train. - * - * @param task + * + * @param task * @param clientConfig client to connect with or parameters on how to create one. * @param config the DiscoConfig */ - constructor(task: Task, + constructor( + task: Task, clientConfig: clients.Client | URL | { aggregator: Aggregator; url: URL }, - config: Partial + config: Partial, ) { super(); const { scheme, logger, preprocessOnce } = { @@ -86,13 +91,13 @@ export class Disco extends EventEmitter<{'status': RoundStatus}>{ this.#preprocessOnce = preprocessOnce; this.#client = client; this.#task = task; - this.trainer = new Trainer(task, client) + this.trainer = new Trainer(task, client); // Simply propagate the training status events emitted by the client - this.#client.on('status', status => this.emit('status', status)) + this.#client.on("status", (status) => this.emit("status", status)); } /** Train on dataset, yielding logs of every round. */ - async *trainByRound(dataset: TypedRawDataset): AsyncGenerator { + async *trainByRound(dataset: Dataset): AsyncGenerator { for await (const round of this.train(dataset)) { const [roundGen, roundLogs] = async_iterator.split(round); for await (const epoch of roundGen) for await (const _ of epoch); @@ -101,7 +106,7 @@ export class Disco extends EventEmitter<{'status': RoundStatus}>{ } /** Train on dataset, yielding logs of every epoch. */ - async *trainByEpoch(dataset: TypedRawDataset): AsyncGenerator { + async *trainByEpoch(dataset: Dataset): AsyncGenerator { for await (const round of this.train(dataset)) { for await (const epoch of round) { const [epochGen, epochLogs] = async_iterator.split(epoch); @@ -112,14 +117,14 @@ export class Disco extends EventEmitter<{'status': RoundStatus}>{ } /** Train on dataset, yielding logs of every batch. */ - async *trainByBatch(dataTuple: TypedRawDataset): AsyncGenerator { - for await (const round of this.train(dataTuple)) + async *trainByBatch(dataset: Dataset): AsyncGenerator { + for await (const round of this.train(dataset)) for await (const epoch of round) yield* epoch; } /** Run whole train on dataset. */ - async trainFully(dataTuple: TypedRawDataset): Promise { - for await (const round of this.train(dataTuple)) + async trainFully(dataset: Dataset): Promise { + for await (const round of this.train(dataset)) for await (const epoch of round) for await (const _ of epoch); } @@ -130,7 +135,7 @@ export class Disco extends EventEmitter<{'status': RoundStatus}>{ * If you don't care about the whole process, use one of the other train methods. **/ async *train( - dataset: TypedRawDataset, + dataset: Dataset, ): AsyncGenerator< AsyncGenerator, RoundLogs> > { @@ -140,7 +145,8 @@ export class Disco extends EventEmitter<{'status': RoundStatus}>{ await this.#preprocessSplitAndBatch(dataset); // the client fetches the latest weights upon connection - this.trainer.model = await this.#client.connect(); + // TODO unsafe cast + this.trainer.model = (await this.#client.connect()) as Model; for await (const [round, epochs] of enumerate( this.trainer.train(trainingDataset, validationDataset), @@ -184,52 +190,24 @@ export class Disco extends EventEmitter<{'status': RoundStatus}>{ } async #preprocessSplitAndBatch( - dataset: TypedRawDataset, + dataset: Dataset, ): Promise< - [ - TypedBatchedModelEncodedDataset, - TypedBatchedModelEncodedDataset, - ] + [Dataset>, Dataset>] > { - const splitAndBatch = async ( - d: Dataset, - ): Promise<[Dataset>, Dataset>]> => { - const [training, validation] = ( - this.#preprocessOnce ? new Dataset(await arrayFromAsync(d)) : d - ).split(validationSplit); - - return [ - training.batch(batchSize).cached(), - validation.batch(batchSize).cached(), - ]; - }; + const { batchSize, validationSplit } = this.#task.trainingInformation; const preprocessed = await processing.preprocess(this.#task, dataset); - const { batchSize, validationSplit } = this.#task.trainingInformation; - switch (preprocessed[0]) { - case "image": { - const [training, validation] = await splitAndBatch(preprocessed[1]); - return [ - ["image", training], - ["image", validation], - ]; - } - case "tabular": { - const [training, validation] = await splitAndBatch(preprocessed[1]); - return [ - ["tabular", training], - ["tabular", validation], - ]; - } - case "text": { - const [training, validation] = await splitAndBatch(preprocessed[1]); - return [ - ["text", training], - ["text", validation], - ]; - } - } + const [training, validation] = ( + this.#preprocessOnce + ? new Dataset(await arrayFromAsync(preprocessed)) + : preprocessed + ).split(validationSplit); + + return [ + training.batch(batchSize).cached(), + validation.batch(batchSize).cached(), + ]; } } diff --git a/discojs/src/training/trainer.ts b/discojs/src/training/trainer.ts index cc5b1eeb8..9681476b1 100644 --- a/discojs/src/training/trainer.ts +++ b/discojs/src/training/trainer.ts @@ -2,11 +2,14 @@ import * as tf from "@tensorflow/tfjs"; import { List } from "immutable"; import type { + Batched, BatchLogs, + Dataset, + DataType, EpochLogs, Model, + ModelEncoded, Task, - TypedBatchedModelEncodedDataset, WeightsContainer, } from "../index.js"; import { privacy } from "../index.js"; @@ -19,30 +22,28 @@ export interface RoundLogs { } /** Train a model and exchange with others **/ -export class Trainer { +export class Trainer { readonly #client: Client; readonly #roundDuration: number; readonly #epochs: number; readonly #privacy: Task["trainingInformation"]["privacy"]; - #model: Model | undefined; + #model: Model | undefined; #training?: AsyncGenerator< AsyncGenerator, RoundLogs>, void >; - - public get model(): Model { - if (this.#model === undefined) throw new Error("trainer's model has not been set") - return this.#model + + public get model(): Model { + if (this.#model === undefined) + throw new Error("trainer's model has not been set"); + return this.#model; } - public set model(model: Model) { - this.#model = model + public set model(model: Model) { + this.#model = model; } - constructor( - task: Task, - client: Client, - ) { + constructor(task: Task, client: Client) { this.#client = client; this.#roundDuration = task.trainingInformation.roundDuration; this.#epochs = task.trainingInformation.epochs; @@ -59,8 +60,8 @@ export class Trainer { } async *train( - dataset: TypedBatchedModelEncodedDataset, - valDataset?: TypedBatchedModelEncodedDataset, + dataset: Dataset>, + validationDataset?: Dataset>, ): AsyncGenerator< AsyncGenerator, RoundLogs>, void @@ -71,7 +72,7 @@ export class Trainer { ); try { - this.#training = this.#runRounds(dataset, valDataset); + this.#training = this.#runRounds(dataset, validationDataset); yield* this.#training; } finally { this.#training = undefined; @@ -79,8 +80,8 @@ export class Trainer { } async *#runRounds( - dataset: TypedBatchedModelEncodedDataset, - valDataset?: TypedBatchedModelEncodedDataset, + dataset: Dataset>, + validationDataset?: Dataset>, ): AsyncGenerator< AsyncGenerator, RoundLogs>, void @@ -90,7 +91,7 @@ export class Trainer { for (let round = 0; round < totalRound; round++) { await this.#client.onRoundBeginCommunication(); - yield this.#runRound(dataset, valDataset); + yield this.#runRound(dataset, validationDataset); let localWeights = this.model.weights; if (this.#privacy !== undefined) @@ -100,23 +101,21 @@ export class Trainer { this.#privacy, ); - const networkWeights = await this.#client.onRoundEndCommunication( - localWeights - ); + const networkWeights = + await this.#client.onRoundEndCommunication(localWeights); this.model.weights = previousRoundWeights = networkWeights; } } async *#runRound( - dataset: TypedBatchedModelEncodedDataset, - valDataset?: TypedBatchedModelEncodedDataset, + dataset: Dataset>, + validationDataset?: Dataset>, ): AsyncGenerator, RoundLogs> { let epochsLogs = List(); for (let epoch = 0; epoch < this.#roundDuration; epoch++) { const [gen, epochLogs] = async_iterator.split( - // TODO check that dataset is of valid type for model - this.model.train(dataset[1], valDataset?.[1]), + this.model.train(dataset, validationDataset), ); yield gen; @@ -138,7 +137,8 @@ async function applyPrivacy( let ret = current; if (options.clippingRadius !== undefined) { - const previousRoundWeights = previous ?? current.map((w) => tf.zerosLike(w)); + const previousRoundWeights = + previous ?? current.map((w) => tf.zerosLike(w)); const weightsProgress = current.sub(previousRoundWeights); ret = previousRoundWeights.add( await privacy.clipNorm(weightsProgress, options.clippingRadius), diff --git a/discojs/src/validator.ts b/discojs/src/validator.ts index df0b3cc4c..39178254b 100644 --- a/discojs/src/validator.ts +++ b/discojs/src/validator.ts @@ -1,29 +1,27 @@ import type { + Dataset, + DataType, Model, + Raw, + RawWithoutLabel, Task, - TypedRawDataset, - TypedRawWithoutLabelDataset, } from "./index.js"; import { processing } from "./index.js"; -export class Validator { - readonly #model: Model; +export class Validator { + readonly #model: Model; constructor( - public readonly task: Task, - model: Model, + public readonly task: Task, + model: Model, ) { this.#model = model; } - /** infer every line of the dataset and check that it is as labeled */ - async *test(dataset: TypedRawDataset): AsyncGenerator { - const preprocessed = await processing.preprocess(this.task, dataset); - - const { batchSize } = this.task.trainingInformation; - - const results = preprocessed[1] - .batch(batchSize) + /** infer every line of the dataset and check that it is as labelled */ + async *test(dataset: Dataset): AsyncGenerator { + const results = (await processing.preprocess(this.task, dataset)) + .batch(this.task.trainingInformation.batchSize) .map(async (batch) => (await this.#model.predict(batch.map(([inputs, _]) => inputs))) .zip(batch.map(([_, outputs]) => outputs)) @@ -35,19 +33,14 @@ export class Validator { /** use the model to predict every line of the dataset */ async *infer( - dataset: TypedRawWithoutLabelDataset, + dataset: Dataset, ): AsyncGenerator { - const preprocessed = await processing.preprocessWithoutLabel( - this.task, - dataset, - ); - - const { batchSize } = this.task.trainingInformation; - - const gen = preprocessed[1] - .batch(batchSize) + const results = ( + await processing.preprocessWithoutLabel(this.task, dataset) + ) + .batch(this.task.trainingInformation.batchSize) .map((batch) => this.#model.predict(batch)); - for await (const batch of gen) for await (const e of batch) yield e; + for await (const batch of results) for await (const e of batch) yield e; } } diff --git a/docs/examples/training.ts b/docs/examples/training.ts index 5f0632761..8dc15bc63 100644 --- a/docs/examples/training.ts +++ b/docs/examples/training.ts @@ -2,7 +2,7 @@ import { Repeat } from 'immutable' import * as path from 'node:path' import '@tensorflow/tfjs-node' -import type { Dataset, Image, Task, TypedRawDataset } from '@epfml/discojs' +import type { Dataset, DataType, Image, Raw, Task } from '@epfml/discojs' import { Disco, fetchTasks, defaultTasks } from '@epfml/discojs' import { loadCSV, loadImagesInDir } from '@epfml/discojs-node' import { Server } from 'server' @@ -11,7 +11,11 @@ import { Server } from 'server' * Example of discojs API, we load data, build the appropriate loggers, the disco object * and finally start training. */ -async function runUser (url: URL, task: Task, dataset: TypedRawDataset): Promise { +async function runUser( + url: URL, + task: Task, + dataset: Dataset, +): Promise { // Create Disco object associated with the server url, the training scheme const disco = new Disco(task, url, { scheme: 'federated' }) @@ -22,6 +26,8 @@ async function runUser (url: URL, task: Task, dataset: TypedRawDataset): Promise await disco.close() } +type TaskAndDataset = [Task, Dataset]; + async function main (): Promise { // Arbitrary chosen Task ID const NAME: string = 'titanic' @@ -34,20 +40,19 @@ async function main (): Promise { // Choose the task and load local data // Make sure you first ran ./get_training_data - let task: Task | undefined - let dataset: TypedRawDataset + let taskAndDataset: TaskAndDataset<'image' | 'tabular'> switch (NAME) { - case 'titanic': { - task = tasks.get('titanic') - if (task === undefined) { throw new Error('task not found') } - dataset = ["tabular", loadCSV("../../datasets/titanic_train.csv")] - break + case "titanic": { + const task = tasks.get("titanic") as Task<"tabular"> | undefined; + if (task === undefined) throw new Error("task not found"); + taskAndDataset = [task, loadCSV("../../datasets/titanic_train.csv")]; + break; } - case 'simple_face': { - task = tasks.get('simple_face') - if (task === undefined) { throw new Error('task not found') } - dataset = ["image", await loadSimpleFaceData()] - break + case "simple_face": { + const task = tasks.get("simple_face") as Task<"image"> | undefined; + if (task === undefined) throw new Error("task not found"); + taskAndDataset = [task, await loadSimpleFaceData()]; + break; } default: throw new Error('task id not found') @@ -55,9 +60,9 @@ async function main (): Promise { // Add more users to the list to simulate more than 3 clients await Promise.all([ - runUser(url, task, dataset), - runUser(url, task, dataset), - runUser(url, task, dataset) + runUser(url, ...taskAndDataset), + runUser(url, ...taskAndDataset), + runUser(url, ...taskAndDataset), ]) // Close server diff --git a/docs/examples/wikitext.ts b/docs/examples/wikitext.ts index 4bc2490f1..69665cbe3 100644 --- a/docs/examples/wikitext.ts +++ b/docs/examples/wikitext.ts @@ -1,6 +1,6 @@ import "@tensorflow/tfjs-node" -import { Disco, fetchTasks, models } from '@epfml/discojs' +import { Disco, fetchTasks, models, Task } from '@epfml/discojs' import { saveModelToDisk, loadModelFromDisk, loadText } from '@epfml/discojs-node' async function main(): Promise { @@ -9,7 +9,7 @@ async function main(): Promise { // Fetch the wikitext task from the server const tasks = await fetchTasks(url) - const task = tasks.get('llm_task') + const task = tasks.get('llm_task') as Task<'text'> | undefined if (task === undefined) { throw new Error('task not found') } let model; @@ -26,7 +26,7 @@ async function main(): Promise { // Initialize a Disco instance and start training a language model const disco = new Disco(task, url, { scheme: 'federated' }) - await disco.trainFully(["text", dataset]); + await disco.trainFully(dataset); // Get the model and save the trained model model = disco.trainer.model as models.GPT diff --git a/server/tests/client/federated.spec.ts b/server/tests/client/federated.spec.ts index 60b36b8ad..cad35c777 100644 --- a/server/tests/client/federated.spec.ts +++ b/server/tests/client/federated.spec.ts @@ -54,7 +54,9 @@ describe("federated client", () => { scheme: "federated", minNbOfParticipants: 2, dataType: "tabular", - tensorBackend: 'tfjs' + tensorBackend: 'tfjs', + inputColumns: ["in"], + outputColumn: "out", }, }, aggregators.getAggregator(TASK), diff --git a/server/tests/e2e/decentralized.spec.ts b/server/tests/e2e/decentralized.spec.ts index c43fac0da..4c774d732 100644 --- a/server/tests/e2e/decentralized.spec.ts +++ b/server/tests/e2e/decentralized.spec.ts @@ -205,7 +205,7 @@ describe('end-to-end decentralized', function () { const discoUser1 = new Disco(lusCovidTask, url, { preprocessOnce: true }); const statusUser1 = new Queue(); discoUser1.on("status", status => { statusUser1.put(status) }) - const generatorUser1 = discoUser1.trainByRound(["image", dataset]) + const generatorUser1 = discoUser1.trainByRound(dataset) // Have User 1 join the task and train locally for one round const logUser1Round1 = await generatorUser1.next() @@ -228,7 +228,7 @@ describe('end-to-end decentralized', function () { const discoUser2 = new Disco(lusCovidTask, url, { preprocessOnce: true }); const statusUser2 = new Queue(); discoUser2.on("status", status => { statusUser2.put(status) }) - const generatorUser2 = discoUser2.trainByRound(["image", dataset]) + const generatorUser2 = discoUser2.trainByRound(dataset) // Have User 2 join the task and train for one round const logUser2Round1 = await generatorUser2.next() @@ -274,7 +274,7 @@ describe('end-to-end decentralized', function () { const discoUser3 = new Disco(lusCovidTask, url, { preprocessOnce: true }); const statusUser3 = new Queue(); discoUser3.on("status", status => { statusUser3.put(status) }) - const generatorUser3 = discoUser3.trainByRound(["image", dataset]) + const generatorUser3 = discoUser3.trainByRound(dataset) // User 3 joins mid-training and trains one local round const logUser3Round1 = await generatorUser3.next() diff --git a/server/tests/e2e/federated.spec.ts b/server/tests/e2e/federated.spec.ts index 5bc16d7e1..31b1de9fb 100644 --- a/server/tests/e2e/federated.spec.ts +++ b/server/tests/e2e/federated.spec.ts @@ -51,7 +51,7 @@ describe("end-to-end federated", () => { scheme: "federated", preprocessOnce: true, }) - await disco.trainFully(["image", dataset]); + await disco.trainFully(dataset); await disco.close(); return disco.trainer.model.weights; @@ -72,7 +72,7 @@ describe("end-to-end federated", () => { }); const logs = List( - await arrayFromAsync(disco.trainByRound(["tabular", dataset])), + await arrayFromAsync(disco.trainByRound(dataset)), ); await disco.close(); @@ -100,7 +100,7 @@ describe("end-to-end federated", () => { const disco = new Disco(task, url, { scheme: "federated" }); const logs = List( - await arrayFromAsync(disco.trainByRound(["text", dataset])), + await arrayFromAsync(disco.trainByRound(dataset)), ); await disco.close(); @@ -131,7 +131,7 @@ describe("end-to-end federated", () => { }); const logs = List( - await arrayFromAsync(disco.trainByRound(["image", dataset])), + await arrayFromAsync(disco.trainByRound(dataset)), ); await disco.close(); @@ -237,8 +237,8 @@ describe("end-to-end federated", () => { // Create User 1 const discoUser1 = new Disco(lusCovidTask, url, { preprocessOnce: true }); const statusUser1 = new Queue(); - discoUser1.on("status", (status) => statusUser1.put(status)); - const generatorUser1 = discoUser1.trainByRound(["image", dataset]) + discoUser1.on("status", (status) => statusUser1.put(status)) + const generatorUser1 = discoUser1.trainByRound(dataset) // Have User 1 join the task and train locally for one round await generatorUser1.next() @@ -252,8 +252,8 @@ describe("end-to-end federated", () => { // Create User 2 const discoUser2 = new Disco(lusCovidTask, url, { preprocessOnce: true }); const statusUser2 = new Queue(); - discoUser2.on("status", (status) => statusUser2.put(status)); - const generatorUser2 = discoUser2.trainByRound(["image", dataset]) + discoUser2.on("status", (status) => statusUser2.put(status)) + const generatorUser2 = discoUser2.trainByRound(dataset) // Have User 2 join the task and train for one round await generatorUser2.next() @@ -285,8 +285,8 @@ describe("end-to-end federated", () => { // Create User 3 const discoUser3 = new Disco(lusCovidTask, url, { preprocessOnce: true }); const statusUser3 = new Queue(); - discoUser3.on("status", (status) => statusUser3.put(status)); - const generatorUser3 = discoUser3.trainByRound(["image", dataset]) + discoUser3.on("status", (status) => statusUser3.put(status)) + const generatorUser3 = discoUser3.trainByRound(dataset) // User 3 joins mid-training and trains one local round await generatorUser3.next() diff --git a/server/tests/validator.spec.ts b/server/tests/validator.spec.ts index dd6b84631..eb85bcb89 100644 --- a/server/tests/validator.spec.ts +++ b/server/tests/validator.spec.ts @@ -25,7 +25,7 @@ describe("validator", () => { let hits = 0; let size = 0; - for await (const correct of validator.test(["image", dataset])) { + for await (const correct of validator.test(dataset)) { if (correct) hits++; size++; } @@ -45,7 +45,7 @@ describe("validator", () => { let hits = 0; let size = 0; - for await (const correct of validator.test(["tabular", dataset])) { + for await (const correct of validator.test(dataset)) { if (correct) hits++; size++; } @@ -73,7 +73,7 @@ describe("validator", () => { let hits = 0; let size = 0; - for await (const correct of validator.test(["image", dataset])) { + for await (const correct of validator.test(dataset)) { if (correct) hits++; size++; } diff --git a/webapp/cypress/e2e/datasetInput.cy.ts b/webapp/cypress/e2e/datasetInput.cy.ts index 51f6d11cc..091dbc905 100644 --- a/webapp/cypress/e2e/datasetInput.cy.ts +++ b/webapp/cypress/e2e/datasetInput.cy.ts @@ -91,7 +91,13 @@ describe("image dataset input by csv", () => { describe("tabular dataset input", () => { it("allows to input CSV", () => { - setupServerWith(basicTask({ dataType: "tabular" })); + setupServerWith( + basicTask({ + dataType: "tabular", + inputColumns: ["a", "b"], + outputColumn: "c", + }), + ); cy.visit("/#/list"); cy.get("button").contains("participate").click(); diff --git a/webapp/cypress/support/e2e.ts b/webapp/cypress/support/e2e.ts index 0cc2e28f3..9b1109615 100644 --- a/webapp/cypress/support/e2e.ts +++ b/webapp/cypress/support/e2e.ts @@ -1,6 +1,7 @@ import { Seq } from "immutable"; import type { + DataType, Model, Task, TaskProvider, @@ -44,13 +45,23 @@ export function setupServerWith(...providers: (Task | TaskProvider)[]): void { }); } -export function basicTask( - info: Partial & Pick, -): Task { +type BasicKeys = + | "epochs" + | "batchSize" + | "roundDuration" + | "validationSplit" + | "tensorBackend" + | "scheme" + | "minNbOfParticipants"; +export function basicTask( + info: { + [K in DataType]: Omit, BasicKeys> & + Partial, BasicKeys>>; + }[D], +): Task { return { id: "task", trainingInformation: { - ...info, epochs: 1, batchSize: 1, roundDuration: 1, @@ -58,6 +69,7 @@ export function basicTask( tensorBackend: "tfjs", scheme: "local", minNbOfParticipants: 1, + ...info, }, displayInformation: { taskTitle: "task", diff --git a/webapp/src/components/dataset_input/FileSelection.vue b/webapp/src/components/dataset_input/FileSelection.vue index 0a7fceb24..fb521ef15 100644 --- a/webapp/src/components/dataset_input/FileSelection.vue +++ b/webapp/src/components/dataset_input/FileSelection.vue @@ -102,7 +102,7 @@ const props = withDefaults( }, ); -const files = defineModel>(); +const files = defineModel | undefined>(); const inputFileElement = ref(null); diff --git a/webapp/src/components/dataset_input/LabeledDatasetInput.vue b/webapp/src/components/dataset_input/LabeledDatasetInput.vue new file mode 100644 index 000000000..95ed12cc8 --- /dev/null +++ b/webapp/src/components/dataset_input/LabeledDatasetInput.vue @@ -0,0 +1,92 @@ + + + diff --git a/webapp/src/components/dataset_input/LabeledImageDatasetInput/ByGroup.vue b/webapp/src/components/dataset_input/LabeledImageDatasetInput/ByGroup.vue index 035e17472..f556367fb 100644 --- a/webapp/src/components/dataset_input/LabeledImageDatasetInput/ByGroup.vue +++ b/webapp/src/components/dataset_input/LabeledImageDatasetInput/ByGroup.vue @@ -1,7 +1,7 @@ - diff --git a/webapp/src/components/testing/Testing.vue b/webapp/src/components/testing/Testing.vue index ca2d96a7d..6c6cdb3c1 100644 --- a/webapp/src/components/testing/Testing.vue +++ b/webapp/src/components/testing/Testing.vue @@ -124,6 +124,7 @@
+ = { mode: "predict" | "test"; - task: Task; + task: Task; // same as in validation store but not undef - model: Model; -}>(); + model: Model; +}; +const selection = ref>(); const federatedTasks = computed(() => tasksStore.tasks diff --git a/webapp/src/components/testing/__tests__/Testing.spec.ts b/webapp/src/components/testing/__tests__/Testing.spec.ts index 70b711f87..79be00a92 100644 --- a/webapp/src/components/testing/__tests__/Testing.spec.ts +++ b/webapp/src/components/testing/__tests__/Testing.spec.ts @@ -20,6 +20,7 @@ const TASK: Task = { }, trainingInformation: { dataType: "text", + tokenizer: "Xenova/gpt2", tensorBackend: "gpt", scheme: "federated", minNbOfParticipants: 1, diff --git a/webapp/src/components/training/Trainer.vue b/webapp/src/components/training/Trainer.vue index ffbee4435..78e3911b7 100644 --- a/webapp/src/components/training/Trainer.vue +++ b/webapp/src/components/training/Trainer.vue @@ -91,7 +91,7 @@ >these steps. @@ -116,19 +116,21 @@
- diff --git a/webapp/src/components/training/__tests__/Trainer.spec.ts b/webapp/src/components/training/__tests__/Trainer.spec.ts index d134b91ab..b77d27493 100644 --- a/webapp/src/components/training/__tests__/Trainer.spec.ts +++ b/webapp/src/components/training/__tests__/Trainer.spec.ts @@ -57,15 +57,12 @@ async function setupForTask() { }, props: { task: provider.getTask(), - dataset: [ - "tabular", - loadCSV( - new File( - [await fs.readFile("../datasets/titanic_train.csv")], - "titanic_train.csv", - ), + dataset: loadCSV( + new File( + [await fs.readFile("../datasets/titanic_train.csv")], + "titanic_train.csv", ), - ], + ), }, }); } From 707447f648b1cbbf1ce574d9778c586b306b66d5 Mon Sep 17 00:00:00 2001 From: tharvik Date: Mon, 23 Sep 2024 20:03:23 +0200 Subject: [PATCH 18/31] discojs: simplify types --- cli/src/args.ts | 4 +- cli/src/cli.ts | 2 +- discojs-node/src/model_loader.ts | 32 +++++++------ discojs/src/aggregator/get.ts | 9 ++-- discojs/src/client/client.ts | 14 ++++-- .../decentralized/decentralized_client.ts | 4 +- .../src/client/federated/federated_client.ts | 4 +- discojs/src/client/utils.ts | 10 +++-- discojs/src/models/model.ts | 4 +- discojs/src/models/tfjs.ts | 6 +-- discojs/src/serialization/model.spec.ts | 8 ++-- discojs/src/serialization/model.ts | 4 +- discojs/src/task/task.ts | 14 +++--- discojs/src/task/task_handler.ts | 12 ++--- discojs/src/task/task_provider.ts | 2 +- discojs/src/task/training_information.ts | 16 ++++--- discojs/src/training/disco.ts | 6 +-- discojs/src/training/trainer.ts | 6 +-- discojs/src/types.ts | 45 +------------------ discojs/src/validator.ts | 2 +- docs/examples/custom_task.ts | 2 +- .../controllers/decentralized_controller.ts | 6 ++- .../src/controllers/federated_controller.ts | 8 ++-- server/src/controllers/training_controller.ts | 6 +-- server/src/routes/task_router.ts | 4 +- server/src/routes/training_router.ts | 9 ++-- server/src/server.ts | 9 ++-- server/src/task_set.ts | 31 ++++++++----- server/tests/client/decentralized.spec.ts | 8 +++- webapp/cypress/support/e2e.ts | 13 +++--- .../dataset_input/DataDescription.vue | 4 +- webapp/src/components/dataset_input/types.ts | 20 +-------- webapp/src/components/pages/TaskList.vue | 8 ++-- .../task_creation_form/TaskForm.vue | 4 +- webapp/src/components/testing/TestSteps.vue | 4 +- webapp/src/components/testing/Testing.vue | 2 +- .../testing/__tests__/Testing.spec.ts | 2 +- .../src/components/training/Description.vue | 4 +- webapp/src/components/training/Finished.vue | 8 ++-- webapp/src/components/training/Trainer.vue | 2 +- .../src/components/training/TrainingSteps.vue | 11 ++++- webapp/src/store/models/index.ts | 9 ++-- webapp/src/store/tasks.ts | 10 +++-- 43 files changed, 193 insertions(+), 195 deletions(-) diff --git a/cli/src/args.ts b/cli/src/args.ts index a55aac03e..74c4ed5c7 100644 --- a/cli/src/args.ts +++ b/cli/src/args.ts @@ -1,11 +1,11 @@ import { parse } from 'ts-command-line-args' import { Map, Set } from 'immutable' -import type { TaskProvider } from '@epfml/discojs' +import type { DataType, TaskProvider } from "@epfml/discojs"; import { defaultTasks } from '@epfml/discojs' interface BenchmarkArguments { - provider: TaskProvider + provider: TaskProvider numberOfUsers: number epochs: number roundDuration: number diff --git a/cli/src/cli.ts b/cli/src/cli.ts index 4e3fd147d..202e2252f 100644 --- a/cli/src/cli.ts +++ b/cli/src/cli.ts @@ -19,7 +19,7 @@ async function arrayFromAsync(iter: AsyncIterable): Promise { } async function runUser( - task: Task, + task: Task, url: URL, data: Dataset, ): Promise> { diff --git a/discojs-node/src/model_loader.ts b/discojs-node/src/model_loader.ts index 6ea5a0db5..1da84def0 100644 --- a/discojs-node/src/model_loader.ts +++ b/discojs-node/src/model_loader.ts @@ -1,17 +1,23 @@ -import fs from 'node:fs/promises' -import { serialization, models } from '@epfml/discojs' +import fs from "node:fs/promises"; -export async function saveModelToDisk(model: models.Model, modelFolder: string, modelFileName: string): Promise { - try { - await fs.access(modelFolder) - } catch { - await fs.mkdir(modelFolder) - } - const encoded = await serialization.model.encode(model) - await fs.writeFile(`${modelFolder}/${modelFileName}`, encoded) +import type { models, DataType } from "@epfml/discojs"; +import { serialization } from "@epfml/discojs"; + +export async function saveModelToDisk( + model: models.Model, + modelFolder: string, + modelFileName: string, +): Promise { + const encoded = await serialization.model.encode(model); + + await fs.mkdir(modelFolder, { recursive: true }); + await fs.writeFile(`${modelFolder}/${modelFileName}`, encoded); } -export async function loadModelFromDisk(modelPath: string): Promise { - const content = await fs.readFile(modelPath) - return await serialization.model.decode(content) as models.GPT +export async function loadModelFromDisk( + modelPath: string, +): Promise> { + const content = await fs.readFile(modelPath); + + return await serialization.model.decode(content); } diff --git a/discojs/src/aggregator/get.ts b/discojs/src/aggregator/get.ts index 076945c58..5c32263ca 100644 --- a/discojs/src/aggregator/get.ts +++ b/discojs/src/aggregator/get.ts @@ -1,8 +1,8 @@ -import type { Task } from '../index.js' +import type { DataType, Task } from '../index.js' import { aggregator } from '../index.js' type AggregatorOptions = Partial<{ - scheme: Task['trainingInformation']['scheme'], // if undefined, fallback on task.trainingInformation.scheme + scheme: Task["trainingInformation"]["scheme"]; // if undefined, fallback on task.trainingInformation.scheme roundCutOff: number, // MeanAggregator threshold: number, // MeanAggregator thresholdType: 'relative' | 'absolute', // MeanAggregator @@ -26,7 +26,10 @@ type AggregatorOptions = Partial<{ * @param options Options passed down to the aggregator's constructor * @returns The aggregator */ -export function getAggregator(task: Task, options: AggregatorOptions = {}): aggregator.Aggregator { +export function getAggregator( + task: Task, + options: AggregatorOptions = {}, +): aggregator.Aggregator { const aggregationStrategy = task.trainingInformation.aggregationStrategy ?? 'mean' const scheme = options.scheme ?? task.trainingInformation.scheme diff --git a/discojs/src/client/client.ts b/discojs/src/client/client.ts index eecf58b18..9f05febc7 100644 --- a/discojs/src/client/client.ts +++ b/discojs/src/client/client.ts @@ -1,6 +1,12 @@ import createDebug from "debug"; -import type { Model, Task, WeightsContainer, RoundStatus } from '../index.js' +import type { + DataType, + Model, + RoundStatus, + Task, + WeightsContainer, +} from "../index.js"; import { serialization } from '../index.js' import type { NodeID } from './types.js' import type { EventConnection } from './event_connection.js' @@ -38,7 +44,7 @@ export abstract class Client extends EventEmitter<{'status': RoundStatus}>{ constructor ( public readonly url: URL, // The network server's URL to connect to - public readonly task: Task, // The client's corresponding task + public readonly task: Task, // The client's corresponding task public readonly aggregator: Aggregator, ) { super() @@ -61,7 +67,7 @@ export abstract class Client extends EventEmitter<{'status': RoundStatus}>{ * This method is overriden by the federated and decentralized clients * By default, it fetches and returns the server's base model */ - async connect(): Promise { + async connect(): Promise> { return this.getLatestModel() } @@ -164,7 +170,7 @@ export abstract class Client extends EventEmitter<{'status': RoundStatus}>{ * Fetches the latest model available on the network's server, for the adequate task. * @returns The latest model */ - async getLatestModel (): Promise { + async getLatestModel (): Promise> { const url = new URL('', this.url.href) if (!url.pathname.endsWith('/')) { url.pathname += '/' diff --git a/discojs/src/client/decentralized/decentralized_client.ts b/discojs/src/client/decentralized/decentralized_client.ts index f89b6c572..5d3e7c79d 100644 --- a/discojs/src/client/decentralized/decentralized_client.ts +++ b/discojs/src/client/decentralized/decentralized_client.ts @@ -1,7 +1,7 @@ import createDebug from "debug"; import { Map, Set } from 'immutable' -import type { Model, WeightsContainer } from "../../index.js"; +import type { DataType, Model, WeightsContainer } from "../../index.js"; import { serialization } from "../../index.js"; import { Client, shortenId } from '../client.js' import { type NodeID } from '../index.js' @@ -44,7 +44,7 @@ export class DecentralizedClient extends Client { * create peer-to-peer WebRTC connections with peers. The server is used to exchange * peers network information. */ - override async connect(): Promise { + override async connect(): Promise> { const model = await super.connect() // Get the server base model const serverURL = new URL('', this.url.href) switch (this.url.protocol) { diff --git a/discojs/src/client/federated/federated_client.ts b/discojs/src/client/federated/federated_client.ts index 556a414ec..e6c89cc79 100644 --- a/discojs/src/client/federated/federated_client.ts +++ b/discojs/src/client/federated/federated_client.ts @@ -1,7 +1,7 @@ import createDebug from "debug"; import { serialization } from "../../index.js"; -import type { Model, WeightsContainer } from "../../index.js"; +import type { DataType, Model, WeightsContainer } from "../../index.js"; import { Client, shortenId } from "../client.js"; import { type, type ClientConnected } from "../messages.js"; import { @@ -39,7 +39,7 @@ export class FederatedClient extends Client { * as well as the latest training information: latest global model, current round and * whether we are waiting for more participants. */ - override async connect(): Promise { + override async connect(): Promise> { const model = await super.connect() // Get the server base model const serverURL = new URL("", this.url.href); diff --git a/discojs/src/client/utils.ts b/discojs/src/client/utils.ts index 57ff2cdd6..5bed14a2a 100644 --- a/discojs/src/client/utils.ts +++ b/discojs/src/client/utils.ts @@ -1,4 +1,4 @@ -import type { Task } from '../index.js' +import type { DataType, Task } from '../index.js' import { client as clients, type aggregator } from '../index.js' // Time to wait for the others in milliseconds. @@ -10,8 +10,12 @@ export async function timeout (ms = MAX_WAIT_PER_ROUND, errorMsg: string = 'time }) } -export function getClient(trainingScheme: Required, - serverURL: URL, task: Task, aggregator: aggregator.Aggregator): clients.Client { +export function getClient( + trainingScheme: Task["trainingInformation"]["scheme"], + serverURL: URL, + task: Task, + aggregator: aggregator.Aggregator, +): clients.Client { switch (trainingScheme) { case 'decentralized': diff --git a/discojs/src/models/model.ts b/discojs/src/models/model.ts index 4ebe91ca7..ec2810992 100644 --- a/discojs/src/models/model.ts +++ b/discojs/src/models/model.ts @@ -14,9 +14,7 @@ import type { BatchLogs, EpochLogs } from "./logs.js"; * Allow for various implementation of models (various train function, tensor-library, ...) **/ // TODO make it typesafe: same shape of data/input/weights -export abstract class Model - implements Disposable -{ +export abstract class Model implements Disposable { // TODO don't allow external access but upgrade train to return weights on every epoch /** Return training state */ abstract get weights(): WeightsContainer; diff --git a/discojs/src/models/tfjs.ts b/discojs/src/models/tfjs.ts index 3c0ad8005..95e7307d9 100644 --- a/discojs/src/models/tfjs.ts +++ b/discojs/src/models/tfjs.ts @@ -13,10 +13,10 @@ import { BatchLogs } from './index.js' import { Model } from './index.js' import { EpochLogs } from './logs.js' -type Serialized = [D, tf.io.ModelArtifacts] +type Serialized = [D, tf.io.ModelArtifacts]; /** TensorFlow JavaScript model with standard training */ -export class TFJS extends Model { +export class TFJS extends Model { /** Wrap the given trainable model */ constructor ( public readonly datatype: D, @@ -172,7 +172,7 @@ export class TFJS extends Model { return ret } - static async deserialize([ + static async deserialize([ datatype, artifacts, ]: Serialized): Promise> { diff --git a/discojs/src/serialization/model.spec.ts b/discojs/src/serialization/model.spec.ts index caf123cc1..8889d4bd0 100644 --- a/discojs/src/serialization/model.spec.ts +++ b/discojs/src/serialization/model.spec.ts @@ -1,11 +1,13 @@ import { assert, expect } from 'chai' import * as tf from '@tensorflow/tfjs' -import type { Model } from '../index.js' +import type { DataType, Model } from "../index.js"; import type { GPTConfig } from '../models/index.js' import { serialization, models } from '../index.js' -async function getRawWeights (model: Model): Promise> { +async function getRawWeights( + model: Model, +): Promise> { return Array.from( (await Promise.all( model.weights.weights.map(async (w) => await w.data<'float32'>())) @@ -33,7 +35,7 @@ describe('serialization', () => { const decoded = await serialization.model.decode(encoded) expect(decoded).to.be.an.instanceof(models.TFJS); - expect((decoded as models.TFJS).datatype).to.equal("image") + expect((decoded as models.TFJS).datatype).to.equal("image") assert.sameDeepOrderedMembers( await getRawWeights(model), await getRawWeights(decoded) diff --git a/discojs/src/serialization/model.ts b/discojs/src/serialization/model.ts index 550dfa83d..726b01f15 100644 --- a/discojs/src/serialization/model.ts +++ b/discojs/src/serialization/model.ts @@ -12,7 +12,7 @@ const Type = { GPT: 1 } as const -export async function encode(model: Model): Promise { +export async function encode(model: Model): Promise { switch (true) { case model instanceof models.TFJS: { const serialized = await model.serialize(); @@ -28,7 +28,7 @@ export async function encode(model: Model): Promise { } } -export async function decode (encoded: unknown): Promise { +export async function decode(encoded: unknown): Promise> { if (!isEncoded(encoded)) throw new Error("Invalid encoding, raw encoding isn't an instance of Uint8Array") const raw = coder.decode(encoded) diff --git a/discojs/src/task/task.ts b/discojs/src/task/task.ts index 1684ce498..a5f6bd983 100644 --- a/discojs/src/task/task.ts +++ b/discojs/src/task/task.ts @@ -4,7 +4,7 @@ import { isTrainingInformation, type TrainingInformation } from './training_info export type TaskID = string -export interface Task { +export interface Task { id: TaskID displayInformation: DisplayInformation trainingInformation: TrainingInformation @@ -14,13 +14,13 @@ export function isTaskID (obj: unknown): obj is TaskID { return typeof obj === 'string' } -export function isTask (raw: unknown): raw is Task { +export function isTask (raw: unknown): raw is Task { if (typeof raw !== 'object' || raw === null) { return false } const { id, displayInformation, trainingInformation }: - Partial> = raw + Partial, unknown>> = raw if (!isTaskID(id) || !isDisplayInformation(displayInformation) || @@ -29,9 +29,11 @@ export function isTask (raw: unknown): raw is Task { return false } - const repack = { id, displayInformation, trainingInformation } - const _correct: Task = repack - const _total: Record = repack + const _: Task = { + id, + displayInformation, + trainingInformation, + } satisfies Record, unknown>; return true } diff --git a/discojs/src/task/task_handler.ts b/discojs/src/task/task_handler.ts index 0b875cb80..7b38a6928 100644 --- a/discojs/src/task/task_handler.ts +++ b/discojs/src/task/task_handler.ts @@ -1,7 +1,7 @@ import createDebug from "debug"; import { Map } from "immutable"; -import type { Model } from "../index.js"; +import type { DataType, Model } from "../index.js"; import { serialization } from "../index.js"; import type { Task, TaskID } from "./task.js"; @@ -15,10 +15,10 @@ function urlToTasks(base: URL): URL { return ret; } -export async function pushTask( +export async function pushTask( base: URL, - task: Task, - model: Model, + task: Task, + model: Model, ): Promise { await fetch(urlToTasks(base), { method: "POST", @@ -30,7 +30,9 @@ export async function pushTask( }); } -export async function fetchTasks(base: URL): Promise> { +export async function fetchTasks( + base: URL, +): Promise>> { const response = await fetch(urlToTasks(base)); const tasks: unknown = await response.json(); diff --git a/discojs/src/task/task_provider.ts b/discojs/src/task/task_provider.ts index c9b4dd31f..fc623eaae 100644 --- a/discojs/src/task/task_provider.ts +++ b/discojs/src/task/task_provider.ts @@ -1,6 +1,6 @@ import type { DataType, Model, Task } from "../index.js"; -export interface TaskProvider { +export interface TaskProvider { getTask(): Task; // Create the corresponding model ready for training (compiled) getModel(): Promise>; diff --git a/discojs/src/task/training_information.ts b/discojs/src/task/training_information.ts index b8919d8e2..d9a0e3767 100644 --- a/discojs/src/task/training_information.ts +++ b/discojs/src/task/training_information.ts @@ -8,7 +8,7 @@ interface Privacy { noiseScale?: number; } -export type TrainingInformation = { +export type TrainingInformation = { // epochs: number of epochs to run training for epochs: number; // roundDuration: number of epochs between each weight sharing round. @@ -96,7 +96,7 @@ function isPrivacy(raw: unknown): raw is Privacy { export function isTrainingInformation( raw: unknown, -): raw is TrainingInformation { +): raw is TrainingInformation { if (typeof raw !== "object" || raw === null) { return false; } @@ -113,7 +113,7 @@ export function isTrainingInformation( scheme, validationSplit, tensorBackend, - }: Partial> = raw; + }: Partial, unknown>> = raw; if ( typeof epochs !== "number" || @@ -170,7 +170,7 @@ export function isTrainingInformation( case "image": { type ImageOnly = Omit< TrainingInformation<"image">, - keyof TrainingInformation + keyof TrainingInformation >; const { LABEL_LIST, IMAGE_W, IMAGE_H }: Partial = raw; @@ -198,7 +198,7 @@ export function isTrainingInformation( case "tabular": { type TabularOnly = Omit< TrainingInformation<"tabular">, - keyof TrainingInformation + keyof TrainingInformation >; const { inputColumns, outputColumn }: Partial = raw; @@ -225,8 +225,10 @@ export function isTrainingInformation( const { maxSequenceLength, tokenizer, - }: Partial, keyof TrainingInformation>> = - raw; + }: Partial< + Omit, + keyof TrainingInformation> + > = raw; if ( (typeof tokenizer !== "string" && diff --git a/discojs/src/training/disco.ts b/discojs/src/training/disco.ts index 97b7b68cf..3998ae225 100644 --- a/discojs/src/training/disco.ts +++ b/discojs/src/training/disco.ts @@ -25,7 +25,7 @@ import { EventEmitter } from "../utils/event_emitter.js"; import { RoundLogs, Trainer } from "./trainer.js"; interface DiscoConfig { - scheme: TrainingInformation["scheme"]; + scheme: TrainingInformation["scheme"]; logger: Logger; // keep preprocessed dataset in memory while training @@ -42,7 +42,7 @@ export type RoundStatus = 'not enough participants' | // Server notification to * a convenient object providing a reduced yet complete API that wraps model training and * communication with nodes. */ -export class Disco extends EventEmitter<{ +export class Disco extends EventEmitter<{ status: RoundStatus; }> { public readonly trainer: Trainer; @@ -151,7 +151,7 @@ export class Disco extends EventEmitter<{ for await (const [round, epochs] of enumerate( this.trainer.train(trainingDataset, validationDataset), )) { - yield async function* (this: Disco) { + yield async function* (this: Disco) { const [gen, returnedRoundLogs] = split(epochs); for await (const [epoch, batches] of enumerate(gen)) { const [gen, returnedEpochLogs] = split(batches); diff --git a/discojs/src/training/trainer.ts b/discojs/src/training/trainer.ts index 9681476b1..b4902d609 100644 --- a/discojs/src/training/trainer.ts +++ b/discojs/src/training/trainer.ts @@ -22,11 +22,11 @@ export interface RoundLogs { } /** Train a model and exchange with others **/ -export class Trainer { +export class Trainer { readonly #client: Client; readonly #roundDuration: number; readonly #epochs: number; - readonly #privacy: Task["trainingInformation"]["privacy"]; + readonly #privacy: Task["trainingInformation"]["privacy"]; #model: Model | undefined; #training?: AsyncGenerator< AsyncGenerator, RoundLogs>, @@ -132,7 +132,7 @@ export class Trainer { async function applyPrivacy( previous: WeightsContainer | undefined, current: WeightsContainer, - options: Exclude, + options: Exclude["trainingInformation"]["privacy"], undefined>, ): Promise { let ret = current; diff --git a/discojs/src/types.ts b/discojs/src/types.ts index ea1e2f1d6..3a7f8d3ae 100644 --- a/discojs/src/types.ts +++ b/discojs/src/types.ts @@ -1,13 +1,6 @@ import { List } from "immutable"; -import type { - Batched, - Dataset, - Image, - processing, - Tabular, - Text, -} from "./index.js"; +import type { Image, processing, Tabular, Text } from "./index.js"; /** * The data that we handle goes through various stages. @@ -43,39 +36,3 @@ export interface Inferred { tabular: Partial>; text: string; } - -// allow to handle multiple type -// TODO will be removed when fully generic - -export type TypedRawDataset = - | ["image", Dataset] - | ["tabular", Dataset] - | ["text", Dataset]; -export type TypedRawWithoutLabelDataset = - | ["image", Dataset] - | ["tabular", Dataset] - | ["text", Dataset]; - -export type TypedModelEncodedDataset = - | ["image", Dataset] - | ["tabular", Dataset] - | ["text", Dataset]; -type ModelEncodedOnlyWithLabel = { [D in DataType]: ModelEncoded[D][1] }; -export type TypedBatchedModelEncodedDataset = - | ["image", Dataset>] - | ["tabular", Dataset>] - | ["text", Dataset>]; -type ModelEncodedWithoutLabel = { [D in DataType]: ModelEncoded[D][0] }; -export type TypedModelEncodedWithoutLabelDataset = - | ["image", Dataset] - | ["tabular", Dataset] - | ["text", Dataset]; -export type TypedModelEncodedOnlyWithLabelDataset = - | ["image", Dataset] - | ["tabular", Dataset] - | ["text", Dataset]; - -export type TypedInferredDataset = - | ["image", Dataset] - | ["tabular", Dataset] - | ["text", Dataset]; diff --git a/discojs/src/validator.ts b/discojs/src/validator.ts index 39178254b..77dce7acf 100644 --- a/discojs/src/validator.ts +++ b/discojs/src/validator.ts @@ -8,7 +8,7 @@ import type { } from "./index.js"; import { processing } from "./index.js"; -export class Validator { +export class Validator { readonly #model: Model; constructor( diff --git a/docs/examples/custom_task.ts b/docs/examples/custom_task.ts index 021705261..609b748ca 100644 --- a/docs/examples/custom_task.ts +++ b/docs/examples/custom_task.ts @@ -5,7 +5,7 @@ import { defaultTasks, models } from '@epfml/discojs' import { Server as DiscoServer } from 'server' // Define your own task provider (task definition + model) -const customTask: TaskProvider = { +const customTask: TaskProvider<"tabular"> = { getTask () { return { id: 'custom-task', diff --git a/server/src/controllers/decentralized_controller.ts b/server/src/controllers/decentralized_controller.ts index 01e706dd6..9307586d0 100644 --- a/server/src/controllers/decentralized_controller.ts +++ b/server/src/controllers/decentralized_controller.ts @@ -4,7 +4,7 @@ import * as msgpack from "@msgpack/msgpack"; import type WebSocket from 'ws' import { Map } from 'immutable' -import { client } from '@epfml/discojs' +import { client, DataType } from "@epfml/discojs"; import { TrainingController } from './training_controller.js' @@ -13,7 +13,9 @@ import MessageTypes = client.messages.type const debug = createDebug("server:controllers:decentralized") -export class DecentralizedController extends TrainingController { +export class DecentralizedController< + D extends DataType, +> extends TrainingController { // Map of nodes who want to join the round. // The boolean value indicates if the node is ready to exchange weight updates (i.e. // the node has already sent a PeerIsReady message) diff --git a/server/src/controllers/federated_controller.ts b/server/src/controllers/federated_controller.ts index 277757ac2..c58a05328 100644 --- a/server/src/controllers/federated_controller.ts +++ b/server/src/controllers/federated_controller.ts @@ -3,7 +3,7 @@ import WebSocket from 'ws' import { v4 as randomUUID } from 'uuid' import * as msgpack from "@msgpack/msgpack"; -import type { Task } from '@epfml/discojs' +import type { DataType, Task } from "@epfml/discojs"; import { aggregator as aggregators, client, @@ -17,7 +17,9 @@ import FederatedMessages = client.federated.messages const debug = createDebug("server:controllers:federated") -export class FederatedController extends TrainingController { +export class FederatedController< + D extends DataType, +> extends TrainingController { /** * Aggregators for each hosted task. By default the server waits for 100% of the nodes to send their contributions before aggregating the updates @@ -30,7 +32,7 @@ export class FederatedController extends TrainingController { */ #latestGlobalWeights: serialization.Encoded; - constructor(task: Task, initialWeights: serialization.Encoded) { + constructor(task: Task, initialWeights: serialization.Encoded) { super(task) this.#latestGlobalWeights = initialWeights diff --git a/server/src/controllers/training_controller.ts b/server/src/controllers/training_controller.ts index a7fb98b89..a104e908f 100644 --- a/server/src/controllers/training_controller.ts +++ b/server/src/controllers/training_controller.ts @@ -4,7 +4,7 @@ import { Map } from 'immutable' import * as msgpack from "@msgpack/msgpack"; import { client } from '@epfml/discojs' -import type { Task } from '@epfml/discojs' +import type { DataType, Task } from "@epfml/discojs"; const debug = createDebug("server:controllers") @@ -23,7 +23,7 @@ const debug = createDebug("server:controllers") * https://developer.mozilla.org/en-US/docs/Learn/Server-side/Express_Nodejs/routes * */ -export abstract class TrainingController { +export abstract class TrainingController { /** * Boolean used to know if we have enough participants to train or if * we should be waiting for more @@ -36,7 +36,7 @@ export abstract class TrainingController { */ protected connections = Map() - constructor(protected readonly task: Task) { } + constructor(protected readonly task: Task) {} abstract handle( ws: WebSocket diff --git a/server/src/routes/task_router.ts b/server/src/routes/task_router.ts index 9948530fa..07e8903df 100644 --- a/server/src/routes/task_router.ts +++ b/server/src/routes/task_router.ts @@ -3,7 +3,7 @@ import type { Request, Response } from 'express' import express from 'express' import { Set } from 'immutable' -import type { Task, TaskID } from '@epfml/discojs' +import type { DataType, Task, TaskID } from '@epfml/discojs' import { serialization, isTask } from '@epfml/discojs' import type { TaskSet } from '../task_set.js' @@ -67,7 +67,7 @@ export class TaskRouter { // When a task has been initialized, // register its GET endpoint - onNewTask(task: Task): void { + onNewTask(task: Task): void { this.#expressRouter.get(`/${task.id}/:file`, (req, res, next) => { this.getLatestModel(task.id, req, res) next() diff --git a/server/src/routes/training_router.ts b/server/src/routes/training_router.ts index 3458558ec..ea2151ccd 100644 --- a/server/src/routes/training_router.ts +++ b/server/src/routes/training_router.ts @@ -1,7 +1,7 @@ import express from 'express' import type expressWS from 'express-ws' import { Set } from 'immutable' -import type { Task } from '@epfml/discojs' +import type { Task, DataType } from '@epfml/discojs' import { serialization } from '@epfml/discojs' import type { TaskSet } from '../task_set.js' @@ -49,12 +49,15 @@ export class TrainingRouter { // Register the task and setup the controller to handle // websocket connections - private async onNewTask (task: Task, encodedModel: serialization.Encoded): Promise { + private async onNewTask( + task: Task, + encodedModel: serialization.Encoded, + ): Promise { this.#tasks = this.#tasks.add(task.id) // The controller handles the actual logic of collaborative training // in its `handle` method. Each task has a dedicated controller which // handles the training logic of this task only - let taskController: TrainingController; + let taskController: TrainingController; if (this.trainingScheme == 'federated') { // The federated controller takes the initial model weights at initialization // so that it can send it to new clients diff --git a/server/src/server.ts b/server/src/server.ts index 9a534c128..dc3e04a1a 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -4,7 +4,7 @@ import express from "express"; import expressWS from "express-ws"; import type * as http from "http"; -import type { TaskProvider } from "@epfml/discojs"; +import type { DataType, TaskProvider } from "@epfml/discojs"; import { TaskRouter, TrainingRouter } from './routes/index.js' import { TaskSet } from "./task_set.js"; @@ -23,7 +23,7 @@ const debug = createDebug("server"); export class Server { readonly #taskSet = new TaskSet(); - async addTask(taskProvider: TaskProvider): Promise { + async addTask(taskProvider: TaskProvider): Promise { await this.#taskSet.addTask(taskProvider); } @@ -35,7 +35,10 @@ export class Server { * @returns a tuple with the server instance and the URL * **/ - async serve(port?: number, ...tasks: TaskProvider[]): Promise<[http.Server, URL]> { + async serve( + port?: number, + ...tasks: TaskProvider[] + ): Promise<[http.Server, URL]> { const wsApplier = expressWS(express(), undefined, { leaveRouterUntouched: true, }); diff --git a/server/src/task_set.ts b/server/src/task_set.ts index 70f040aae..1246e6dbb 100644 --- a/server/src/task_set.ts +++ b/server/src/task_set.ts @@ -3,10 +3,14 @@ import fs from 'node:fs/promises' import tf from '@tensorflow/tfjs' import '@tensorflow/tfjs-node' +import type { DataType, Task, TaskProvider } from "@epfml/discojs"; import { - Task, TaskProvider, isTask, - serialization, models, Model, EventEmitter -} from '@epfml/discojs' + EventEmitter, + isTask, + Model, + models, + serialization, +} from "@epfml/discojs"; type EncodedModel = serialization.Encoded; @@ -33,12 +37,12 @@ type EncodedModel = serialization.Encoded; * the 'newTask' event to run callbacks whenever a new Task and EncodedModel are initialized. */ export class TaskSet extends EventEmitter<{ - "newTask": { task: Task, encodedModel: EncodedModel } + "newTask": { task: Task, encodedModel: EncodedModel } }>{ // Keep track of previously initialized task-model pairs - #tasks = Set<[Task, EncodedModel]>() + #tasks = Set<[Task, EncodedModel]>() - get tasks(): Set<[Task, EncodedModel]> { + get tasks(): Set<[Task, EncodedModel]> { return this.#tasks } @@ -55,8 +59,10 @@ export class TaskSet extends EventEmitter<{ * @param taskOrProvider either a Task or TaskProvider * @param model optional model, can already be an EncodedModel, a Model or a URL for the model */ - async addTask(taskOrProvider: Task | TaskProvider, - model?: Model | URL | EncodedModel): Promise { + async addTask( + taskOrProvider: Task | TaskProvider, + model?: Model | URL | EncodedModel, + ): Promise { // get the task const task = isTask(taskOrProvider) ? taskOrProvider : taskOrProvider.getTask() @@ -65,7 +71,7 @@ export class TaskSet extends EventEmitter<{ if (serialization.isEncoded(model)) { encodedModel = model // don't do anything if already encoded } else { - let tfModel: Model + let tfModel: Model if (model === undefined) { // Get the model if nothing is provided tfModel = await this.loadModelFromTask(taskOrProvider) @@ -96,13 +102,16 @@ export class TaskSet extends EventEmitter<{ * @param taskOrProvider either a Task or a TaskProvider * @returns a promise for the associated model */ - private async loadModelFromTask(taskOrProvider: Task | TaskProvider): Promise { + private async loadModelFromTask( + taskOrProvider: Task | TaskProvider, + ): Promise> { const task = isTask(taskOrProvider) ? taskOrProvider : taskOrProvider.getTask() - let model: Model | undefined + let model: Model | undefined const modelPath = `./models/${task.id}/` try { const content = await fs.readFile(`${modelPath}/model.json`) + // cast as we trust the task ID return await serialization.model.decode(content) } catch { // unable to read file (potentially doesn't exist), continuing diff --git a/server/tests/client/decentralized.spec.ts b/server/tests/client/decentralized.spec.ts index 095cf53da..4b51c8f26 100644 --- a/server/tests/client/decentralized.spec.ts +++ b/server/tests/client/decentralized.spec.ts @@ -1,6 +1,6 @@ import type * as http from 'http' -import type { Task } from '@epfml/discojs' +import type { DataType, Task } from '@epfml/discojs' import { aggregator as aggregators, client as clients, defaultTasks } from '@epfml/discojs' import { Server } from '../../src/index.js' @@ -9,7 +9,11 @@ const TASK = defaultTasks.titanic; function test ( name: string, - Client: new (url: URL, task: Task, aggregator: aggregators.Aggregator) => clients.Client, + Client: new ( + url: URL, + task: Task, + aggregator: aggregators.Aggregator, + ) => clients.Client, Aggregator: new () => aggregators.Aggregator ): void { describe(`decentralized ${name} client`, function () { diff --git a/webapp/cypress/support/e2e.ts b/webapp/cypress/support/e2e.ts index 9b1109615..7d36f290e 100644 --- a/webapp/cypress/support/e2e.ts +++ b/webapp/cypress/support/e2e.ts @@ -2,19 +2,18 @@ import { Seq } from "immutable"; import type { DataType, - Model, Task, TaskProvider, TrainingInformation, } from "@epfml/discojs"; import { isTask, serialization } from "@epfml/discojs"; -export function setupServerWith(...providers: (Task | TaskProvider)[]): void { - const tasksAndModels: Seq.Indexed<[Task, Promise | undefined]> = Seq( - providers, - ).map((p) => { - if (isTask(p)) return [p, undefined]; - return [p.getTask(), p.getModel()]; +export function setupServerWith( + ...providers: (Task | TaskProvider)[] +): void { + const tasksAndModels = Seq(providers).map((p) => { + if (isTask(p)) return [p, undefined] as const; + return [p.getTask(), p.getModel()] as const; }); cy.intercept( diff --git a/webapp/src/components/dataset_input/DataDescription.vue b/webapp/src/components/dataset_input/DataDescription.vue index 845dc3dbf..45f0bc111 100644 --- a/webapp/src/components/dataset_input/DataDescription.vue +++ b/webapp/src/components/dataset_input/DataDescription.vue @@ -43,12 +43,12 @@ diff --git a/webapp/src/components/dataset_input/types.ts b/webapp/src/components/dataset_input/types.ts index ab7b42321..892abd14a 100644 --- a/webapp/src/components/dataset_input/types.ts +++ b/webapp/src/components/dataset_input/types.ts @@ -1,11 +1,4 @@ -import type { - Dataset, - Image, - Raw, - RawWithoutLabel, - Tabular, - Text, -} from "@epfml/discojs"; +import type { Dataset, Image, Raw, RawWithoutLabel } from "@epfml/discojs"; type NamedImage = { image: Image; @@ -18,17 +11,6 @@ type NamedLabeledImage = NamedImage & { export type NamedImageDataset = Dataset; export type NamedLabeledImageDataset = Dataset; -// TODO rm Typed* -type BasicTypedNamedDataset = - | ["tabular", Dataset] - | ["text", Dataset]; -export type TypedNamedLabeledDataset = - | BasicTypedNamedDataset - | ["image", NamedLabeledImageDataset]; -export type TypedNamedDataset = - | BasicTypedNamedDataset - | ["image", NamedImageDataset]; - export interface LabeledDataset { image: NamedLabeledImageDataset; tabular: Dataset; diff --git a/webapp/src/components/pages/TaskList.vue b/webapp/src/components/pages/TaskList.vue index f591e8045..2d5967531 100644 --- a/webapp/src/components/pages/TaskList.vue +++ b/webapp/src/components/pages/TaskList.vue @@ -84,7 +84,7 @@ import { VueSpinner } from 'vue3-spinners'; import { List } from "immutable"; -import type { Task } from '@epfml/discojs' +import type { DataType, Task } from "@epfml/discojs"; import { useTasksStore } from "@/store"; import { useTrainingStore } from "@/store"; @@ -104,7 +104,7 @@ const sortedTasks = computed(() => [...tasks.value.values()].sort( (task1, task2) => task1.displayInformation.taskTitle.localeCompare(task2.displayInformation.taskTitle) )) -function getSchemeColor(task: Task): string { +function getSchemeColor(task: Task): string { switch (task.trainingInformation.scheme) { case 'decentralized': return 'bg-orange-200' @@ -114,7 +114,7 @@ function getSchemeColor(task: Task): string { return 'bg-blue-200' } } -function getDataTypeColor(task: Task): string { +function getDataTypeColor(task: Task): string { switch (task.trainingInformation.dataType) { case 'image': return 'bg-yellow-200' @@ -125,7 +125,7 @@ function getDataTypeColor(task: Task): string { } } -const toTask = (task: Task): void => { +function toTask(task: Task): void { trainingStore.setTask(task.id) trainingStore.setStep(1) router.push(`/${task.id}`) diff --git a/webapp/src/components/task_creation_form/TaskForm.vue b/webapp/src/components/task_creation_form/TaskForm.vue index a7d4ed157..bdae21e8e 100644 --- a/webapp/src/components/task_creation_form/TaskForm.vue +++ b/webapp/src/components/task_creation_form/TaskForm.vue @@ -158,7 +158,7 @@ import { Form as VeeForm, ErrorMessage } from 'vee-validate' import { List, Map } from 'immutable' import * as tf from '@tensorflow/tfjs' -import type { Task } from '@epfml/discojs' +import type { DataType, Task } from "@epfml/discojs"; import { models, pushTask } from '@epfml/discojs' import type { FormDependency, FormField, FormSection } from '@/task_creation_form' @@ -260,7 +260,7 @@ const onSubmit = async (rawTask: any): Promise => { .map((section) => formatSection(section, rawTask)) ) .set('id', rawTask.taskID) - .toObject() as unknown as Task + .toObject() as unknown as Task let model try { diff --git a/webapp/src/components/testing/TestSteps.vue b/webapp/src/components/testing/TestSteps.vue index 942e2b7e0..b1893edaf 100644 --- a/webapp/src/components/testing/TestSteps.vue +++ b/webapp/src/components/testing/TestSteps.vue @@ -135,8 +135,8 @@ const toaster = useToaster(); const validationStore = useValidationStore(); const props = defineProps<{ - task: Task; - model: Model; + task: Task; + model: Model; }>(); interface Tested { diff --git a/webapp/src/components/testing/Testing.vue b/webapp/src/components/testing/Testing.vue index 6c6cdb3c1..8c93c002b 100644 --- a/webapp/src/components/testing/Testing.vue +++ b/webapp/src/components/testing/Testing.vue @@ -225,7 +225,7 @@ onActivated(() => { selectModel(validationStore.modelID, "test"); }); -async function downloadModel(task: Task): Promise { +async function downloadModel(task: Task): Promise { try { toaster.info("Downloading model..."); diff --git a/webapp/src/components/testing/__tests__/Testing.spec.ts b/webapp/src/components/testing/__tests__/Testing.spec.ts index 79be00a92..7cd5d62dd 100644 --- a/webapp/src/components/testing/__tests__/Testing.spec.ts +++ b/webapp/src/components/testing/__tests__/Testing.spec.ts @@ -12,7 +12,7 @@ import { useTasksStore } from "@/store"; import Testing from "../Testing.vue"; -const TASK: Task = { +const TASK: Task<"text"> = { id: "task", displayInformation: { taskTitle: "task title", diff --git a/webapp/src/components/training/Description.vue b/webapp/src/components/training/Description.vue index 2b6e3053b..f23661066 100644 --- a/webapp/src/components/training/Description.vue +++ b/webapp/src/components/training/Description.vue @@ -57,7 +57,7 @@ From d4b1c25794cc1671e9ab6534fbbfeabd82ea7ee2 Mon Sep 17 00:00:00 2001 From: tharvik Date: Mon, 28 Oct 2024 10:44:37 +0100 Subject: [PATCH 30/31] webapp/testing: fix correct color --- webapp/src/components/testing/TestSteps.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/src/components/testing/TestSteps.vue b/webapp/src/components/testing/TestSteps.vue index b1893edaf..3d63c8684 100644 --- a/webapp/src/components/testing/TestSteps.vue +++ b/webapp/src/components/testing/TestSteps.vue @@ -76,7 +76,7 @@