diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 25e3fea..ae994ac 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,8 +32,8 @@ jobs: rustc --version cargo --version cargo build --release -p netsaur - deno run -Ar jsr:@deno/wasmbuild@0.17.1 -p netsaur --out src/backends/wasm/lib - deno run -Ar jsr:@deno/wasmbuild@0.17.1 -p netsaur-tokenizers --out tokenizers/lib + deno run -Ar jsr:@deno/wasmbuild@0.17.2 -p netsaur --out src/backends/wasm/lib + deno run -Ar jsr:@deno/wasmbuild@0.17.2 -p netsaur-tokenizers --out tokenizers/lib - name: Release uses: softprops/action-gh-release@master env: diff --git a/Cargo.lock b/Cargo.lock index 43fcdce..a137cda 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -356,7 +356,7 @@ dependencies = [ [[package]] name = "netsaur" -version = "0.4.0" +version = "0.4.1" dependencies = [ "getrandom", "js-sys", @@ -370,7 +370,7 @@ dependencies = [ [[package]] name = "netsaur-gpu" -version = "0.4.0" +version = "0.4.1" dependencies = [ "cudarc", "ndarray", @@ -383,7 +383,7 @@ dependencies = [ [[package]] name = "netsaur-tokenizers" -version = "0.4.0" +version = "0.4.1" dependencies = [ "getrandom", "js-sys", diff --git a/Cargo.toml b/Cargo.toml index c9397d4..0fe0b9f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -package.version = "0.4.0" +package.version = "0.4.1" members = ["crates/*"] resolver = "2" diff --git a/deno.jsonc b/deno.jsonc index 8e943e8..d591116 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -1,6 +1,6 @@ { "name": "@denosaurs/netsaur", - "version": "0.4.0", + "version": "0.4.1", "exports": { ".": "./mod.ts", "./web": "./web.ts", diff --git a/deno.lock b/deno.lock index 3c921db..74d33ec 100644 --- a/deno.lock +++ b/deno.lock @@ -1,6 +1,7 @@ { "version": "4", "specifiers": { + "jsr:@denosaurs/netsaur@0.4.0": "0.4.0", "jsr:@denosaurs/plug@1.0.3": "1.0.3", "jsr:@std/assert@~0.213.1": "0.213.1", "jsr:@std/encoding@0.213.1": "0.213.1", @@ -10,6 +11,12 @@ "jsr:@std/path@~0.213.1": "0.213.1" }, "jsr": { + "@denosaurs/netsaur@0.4.0": { + "integrity": "91192ee3ffffc2db3eac38a3f01abb7d52d9fd1053591eddb70852423d44f8ae", + "dependencies": [ + "jsr:@denosaurs/plug" + ] + }, "@denosaurs/plug@1.0.3": { "integrity": "b010544e386bea0ff3a1d05e0c88f704ea28cbd4d753439c2f1ee021a85d4640", "dependencies": [ diff --git a/packages/core/src/backends/cpu/mod.ts b/packages/core/src/backends/cpu/mod.ts index 29efd18..53a096b 100644 --- a/packages/core/src/backends/cpu/mod.ts +++ b/packages/core/src/backends/cpu/mod.ts @@ -15,7 +15,7 @@ const options: FetchOptions = { name: "netsaur", url: new URL(import.meta.url).protocol !== "file:" ? new URL( - "https://github.com/denosaurs/netsaur/releases/download/0.4.0/", + "https://github.com/denosaurs/netsaur/releases/download/0.4.1/", import.meta.url, ) : "./target/release/", diff --git a/packages/core/src/backends/gpu/mod.ts b/packages/core/src/backends/gpu/mod.ts index a98b4f3..bac9c99 100644 --- a/packages/core/src/backends/gpu/mod.ts +++ b/packages/core/src/backends/gpu/mod.ts @@ -15,7 +15,7 @@ const options: FetchOptions = { name: "netsaur_gpu", url: new URL(import.meta.url).protocol !== "file:" ? new URL( - "https://github.com/denosaurs/netsaur/releases/download/0.3.2-patch/", + "https://github.com/denosaurs/netsaur/releases/download/0.4.1/", import.meta.url, ) : "./target/release/", diff --git a/packages/core/src/backends/wasm/lib/netsaur.generated.js b/packages/core/src/backends/wasm/lib/netsaur.generated.js index 47661c0..f3523ed 100644 --- a/packages/core/src/backends/wasm/lib/netsaur.generated.js +++ b/packages/core/src/backends/wasm/lib/netsaur.generated.js @@ -217,7 +217,7 @@ const imports = { __wbindgen_object_drop_ref: function (arg0) { takeObject(arg0); }, - __wbg_log_023d7669e382bddf: function (arg0, arg1) { + __wbg_log_6f7dfa87fad40a57: function (arg0, arg1) { console.log(getStringFromWasm0(arg0, arg1)); }, __wbindgen_number_new: function (arg0) { diff --git a/packages/core/src/backends/wasm/lib/netsaur_bg.wasm b/packages/core/src/backends/wasm/lib/netsaur_bg.wasm index 2ffee65..511a32b 100644 Binary files a/packages/core/src/backends/wasm/lib/netsaur_bg.wasm and b/packages/core/src/backends/wasm/lib/netsaur_bg.wasm differ diff --git a/packages/core/src/backends/wasm/mod.ts b/packages/core/src/backends/wasm/mod.ts index 91b1db7..becf182 100644 --- a/packages/core/src/backends/wasm/mod.ts +++ b/packages/core/src/backends/wasm/mod.ts @@ -22,7 +22,7 @@ export class WASMInstance { await instantiate({ url: new URL(import.meta.url).protocol !== "file:" ? new URL( - "https://github.com/denosaurs/netsaur/releases/download/0.3.2-patch/netsaur_bg.wasm", + "https://github.com/denosaurs/netsaur/releases/download/0.4.1/netsaur_bg.wasm", import.meta.url, ) : undefined, diff --git a/packages/core/src/backends/wasm/utils.ts b/packages/core/src/backends/wasm/utils.ts index fc63185..ee8c295 100644 --- a/packages/core/src/backends/wasm/utils.ts +++ b/packages/core/src/backends/wasm/utils.ts @@ -3,19 +3,19 @@ import type { Rank, Shape } from "../../core/api/shape.ts"; /** * Train Options Interface. */ -export type TrainOptions = { +export interface TrainOptions { datasets: number; inputShape: Shape; outputShape: Shape; epochs: number; batches: number; rate: number; -}; +} /** * Predict Options Interface. */ -export type PredictOptions = { +export interface PredictOptions { inputShape: Shape; outputShape: Shape; -}; +} diff --git a/packages/core/src/core/api/layer.ts b/packages/core/src/core/api/layer.ts index a950a62..4a6d191 100644 --- a/packages/core/src/core/api/layer.ts +++ b/packages/core/src/core/api/layer.ts @@ -21,7 +21,7 @@ export type Layer = /** * The configuration for a dense layer. */ -export type DenseLayerConfig = { +export interface DenseLayerConfig { /** * The type of initialization to use. */ @@ -46,7 +46,7 @@ export type DenseLayerConfig = { /** * The configuration for a dropout layer. */ -export type DropoutLayerConfig = { +export interface DropoutLayerConfig { /** * probability of dropping out a value. */ @@ -61,7 +61,7 @@ export type DropoutLayerConfig = { /** * The configuration for an activation layer. */ -export type ActivationLayerConfig = { +export interface ActivationLayerConfig { /** * The activation function to use. */ @@ -71,7 +71,7 @@ export type ActivationLayerConfig = { /** * The configuration for a convolutional layer. */ -export type Conv2DLayerConfig = { +export interface Conv2DLayerConfig { /** * The type of initialization to use. */ @@ -111,7 +111,7 @@ export type Conv2DLayerConfig = { /** * The configuration for a convolutional transpose layer. */ -export type ConvTranspose2DLayerConfig = { +export interface ConvTranspose2DLayerConfig { /** * The type of initialization to use. */ @@ -158,7 +158,7 @@ export enum PoolMode { /** * The configuration for a pooling layer. */ -export type Pool2DLayerConfig = { +export interface Pool2DLayerConfig { /** * The optional strides to use. */ @@ -173,7 +173,7 @@ export type Pool2DLayerConfig = { /** * The configuration for a flatten layer. */ -export type FlattenLayerConfig = { +export interface FlattenLayerConfig { /** * The size of the layer. */ @@ -183,7 +183,7 @@ export type FlattenLayerConfig = { /** * The configuration for a batch normalization layer. */ -export type BatchNormLayerConfig = { +export interface BatchNormLayerConfig { /** * The momentum to use for the batch normalization. * Defaults to 0.99. diff --git a/packages/core/src/core/api/optimizer.ts b/packages/core/src/core/api/optimizer.ts index c8ef7b1..f1fea35 100644 --- a/packages/core/src/core/api/optimizer.ts +++ b/packages/core/src/core/api/optimizer.ts @@ -6,16 +6,16 @@ export type Optimizer = | { type: OptimizerType.Nadam; config: AdamOptimizerConfig } | { type: OptimizerType.RMSProp; config: RMSPropOptimizerConfig }; -export type AdamOptimizerConfig = { +export interface AdamOptimizerConfig { beta1?: number; beta2?: number; epsilon?: number; -}; +} -export type RMSPropOptimizerConfig = { +export interface RMSPropOptimizerConfig { decayRate?: number; epsilon?: number; -}; +} export function SGDOptimizer(): Optimizer { return { type: OptimizerType.SGD }; diff --git a/packages/core/src/core/api/scheduler.ts b/packages/core/src/core/api/scheduler.ts index cfedb81..d49c62d 100644 --- a/packages/core/src/core/api/scheduler.ts +++ b/packages/core/src/core/api/scheduler.ts @@ -8,15 +8,15 @@ export type Scheduler = } | { type: SchedulerType.OneCycle; config: OneCycleSchedulerConfig }; -export type DecaySchedulerConfig = { +export interface DecaySchedulerConfig { rate?: number; step_size?: number; -}; +} -export type OneCycleSchedulerConfig = { +export interface OneCycleSchedulerConfig { max_rate?: number; step_size?: number; -}; +} export function NoScheduler(): Scheduler { return { type: SchedulerType.None }; diff --git a/packages/core/src/core/engine.ts b/packages/core/src/core/engine.ts index 2d0ba5a..e1ce86d 100644 --- a/packages/core/src/core/engine.ts +++ b/packages/core/src/core/engine.ts @@ -15,6 +15,12 @@ export interface BackendInstance { export interface BackendLoader { /** * Whether the backend is supported. + * + * ```ts + * import { WASM } from "https://deno.land/x/netsaur/mod.ts"; + * + * console.log(WASM.isSupported()); + * ``` */ isSupported(): boolean; diff --git a/packages/core/src/core/mod.ts b/packages/core/src/core/mod.ts index 467cddd..454fec1 100644 --- a/packages/core/src/core/mod.ts +++ b/packages/core/src/core/mod.ts @@ -17,11 +17,13 @@ import { SGDOptimizer } from "./api/optimizer.ts"; */ export class Sequential implements NeuralNetwork { backend!: Backend; + config: NetworkConfig; /** * Create a Sequential Neural Network. */ - constructor(public config: NetworkConfig) { + constructor(config: NetworkConfig) { + this.config = config; this.config.cost = this.config.cost || Cost.MSE; this.config.optimizer = this.config.optimizer || SGDOptimizer(); this.config.scheduler = this.config.scheduler || { diff --git a/packages/core/src/core/tensor/tensor.ts b/packages/core/src/core/tensor/tensor.ts index 09e4731..784bc2e 100644 --- a/packages/core/src/core/tensor/tensor.ts +++ b/packages/core/src/core/tensor/tensor.ts @@ -16,7 +16,7 @@ import type { } from "../api/shape.ts"; import { inferShape, length } from "./util.ts"; -export type TensorLike = { +export interface TensorLike { shape: Shape; data: Float32Array; }; @@ -35,6 +35,9 @@ export class Tensor { /** * Creates an empty tensor. + * ```ts + * Tensor.zeros([2, 2]); + * ``` */ static zeroes(shape: Shape): Tensor { return new Tensor(new Float32Array(length(shape)), shape); @@ -42,6 +45,10 @@ export class Tensor { /** * Serialise a tensor into JSON. + * + * ```ts + * tensor([1, 2, 3, 4], [2, 2]).toJSON(); + * ``` */ toJSON(): { data: number[]; shape: Shape } { const data = new Array(this.data.length).fill(1); diff --git a/packages/core/src/core/types.ts b/packages/core/src/core/types.ts index 7901fd9..149c3a3 100644 --- a/packages/core/src/core/types.ts +++ b/packages/core/src/core/types.ts @@ -12,6 +12,10 @@ export interface Backend { * The train method is a function that trains a neural network using a set of training data. * It takes in an array of DataSet objects, the number of epochs to train for, and the learning rate. * The method modifies the weights and biases of the network to minimize the cost function and improve its accuracy on the training data. + * + * ```ts + * net.train(datasets, 100, 10, 0.01); + * ``` */ train( datasets: DataSet[], @@ -24,6 +28,11 @@ export interface Backend { * The predict method is a function that takes in a Tensor object * representing the input to the neural network and returns a Promise that resolves to a Tensor object representing the output of the network. * This method is used to make predictions on new data after the network has been trained. + * + * ```ts + * const input = new Tensor([1, 2, 3, 4]); + * const output = await net.predict(input); + * ``` */ predict( input: Tensor, @@ -34,12 +43,20 @@ export interface Backend { /** * The save method is a function that saves the network to a Uint8Array. * This method is used to save the network after it has been trained. + * + * ```ts + * const buffer = net.save(); + * ``` */ save(): Uint8Array; /** * The saveFile method is a function that takes in a string representing the path to a file and saves the network to that file. * This method is used to save the network after it has been trained. + * + * ```ts + * net.saveFile("model.bin"); + * ``` */ saveFile(path: string): void; } @@ -47,7 +64,7 @@ export interface Backend { /** * NetworkConfig represents the configuration of a neural network. */ -export type NetworkConfig = { +export interface NetworkConfig { /** * Input size of the neural network. */ @@ -87,7 +104,7 @@ export type NetworkConfig = { * Number of disappointing iterations to allow before early stopping */ patience?: number; -}; +} /** * Activation functions are used to transform the output of a layer into a new output. @@ -195,10 +212,10 @@ export enum SchedulerType { /** * DataSet is a container for training data. */ -export type DataSet = { +export interface DataSet { inputs: Tensor; outputs: Tensor; -}; +} export enum LayerType { Activation = "activation", diff --git a/packages/tokenizers/lib/netsaur_tokenizers_bg.wasm b/packages/tokenizers/lib/netsaur_tokenizers_bg.wasm index 817caf3..2f7c7d6 100644 Binary files a/packages/tokenizers/lib/netsaur_tokenizers_bg.wasm and b/packages/tokenizers/lib/netsaur_tokenizers_bg.wasm differ diff --git a/packages/tokenizers/mod.ts b/packages/tokenizers/mod.ts index cda505d..5853c48 100644 --- a/packages/tokenizers/mod.ts +++ b/packages/tokenizers/mod.ts @@ -16,7 +16,7 @@ export async function init(): Promise { await instantiate({ url: new URL(import.meta.url).protocol !== "file:" ? new URL( - "https://github.com/denosaurs/netsaur/releases/download/0.3.2-patch/netsaur_tokenizers_bg.wasm", + "https://github.com/denosaurs/netsaur/releases/download/0.4.1/netsaur_tokenizers_bg.wasm", import.meta.url, ) : undefined, diff --git a/packages/visualizer/mod.ts b/packages/visualizer/mod.ts index 6f0199b..b80537f 100644 --- a/packages/visualizer/mod.ts +++ b/packages/visualizer/mod.ts @@ -5,9 +5,6 @@ import type { Line } from "./types.ts"; * Visualizer for Neural Networks in Jupyter Notebook */ export class Visualizer { - /** - * Graph title - */ #title: string; constructor(title: string) { @@ -76,4 +73,48 @@ export class Visualizer { }, }; } + + /** + * Graph the loss of a Neural Network during training + */ + graphLoss(loss: number[]): { + [x: symbol]: () => { + "application/vnd.plotly.v1+json": { + data: Line[]; + layout: { title: string }; + }; + }; + } { + const lossLine: Line = { + x: [], + y: [], + type: "scatter", + mode: "lines+markers", + name: "Loss", + line: { + color: "blue", + width: 3, + }, + }; + + for (let i = 0; i < loss.length; i++) { + lossLine.x.push(i + 1); + lossLine.y.push(loss[i]); + } + + const title = this.#title; + + return { + [Symbol.for("Jupyter.display")]() { + return { + "application/vnd.plotly.v1+json": { + data: [lossLine], + layout: { + title, + }, + }, + }; + }, + }; + } } diff --git a/utils/buildtools.ts b/utils/buildtools.ts deleted file mode 100644 index 3729e7a..0000000 --- a/utils/buildtools.ts +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright 2018-2024 the Deno authors. MIT license. -export async function cacheToLocalDir( - url: URL, - decompress: (bytes: Uint8Array) => Uint8Array, -) { - const localPath = await getUrlLocalPath(url); - if (localPath == null) { - return undefined; - } - if (!await exists(localPath)) { - const fileBytes = decompress(new Uint8Array(await getUrlBytes(url))); - try { - await Deno.writeFile(localPath, fileBytes); - } catch { - // ignore and return the wasm bytes - return fileBytes; - } - } - return toFileUrl(localPath); -} - -async function getUrlLocalPath(url: URL) { - try { - const dataDirPath = await getInitializedLocalDataDirPath(); - const hash = await getUrlHash(url); - return `${dataDirPath}/${hash}.wasm`; - } catch { - return undefined; - } -} - -async function getInitializedLocalDataDirPath() { - const dataDir = localDataDir(); - if (dataDir == null) { - throw new Error(`Could not find local data directory.`); - } - const dirPath = `${dataDir}/deno-wasmbuild`; - await ensureDir(dirPath); - return dirPath; -} - -async function exists(filePath: string | URL): Promise { - try { - await Deno.lstat(filePath); - return true; - } catch (error) { - if (error instanceof Deno.errors.NotFound) { - return false; - } - throw error; - } -} - -async function ensureDir(dir: string) { - try { - const fileInfo = await Deno.lstat(dir); - if (!fileInfo.isDirectory) { - throw new Error(`Path was not a directory '${dir}'`); - } - } catch (err) { - if (err instanceof Deno.errors.NotFound) { - // if dir not exists. then create it. - await Deno.mkdir(dir, { recursive: true }); - return; - } - throw err; - } -} - -async function getUrlHash(url: URL) { - // Taken from MDN: https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/digest - const hashBuffer = await crypto.subtle.digest( - "SHA-256", - new TextEncoder().encode(url.href), - ); - // convert buffer to byte array - const hashArray = Array.from(new Uint8Array(hashBuffer)); - // convert bytes to hex string - const hashHex = hashArray - .map((b) => b.toString(16).padStart(2, "0")) - .join(""); - return hashHex; -} - -async function getUrlBytes(url: URL) { - const response = await fetchWithRetries(url); - return await response.arrayBuffer(); -} - -// the below is extracted from deno_std/path - -const WHITESPACE_ENCODINGS: Record = { - "\u0009": "%09", - "\u000A": "%0A", - "\u000B": "%0B", - "\u000C": "%0C", - "\u000D": "%0D", - "\u0020": "%20", -}; - -function encodeWhitespace(string: string): string { - return string.replaceAll(/[\s]/g, (c) => { - return WHITESPACE_ENCODINGS[c] ?? c; - }); -} - -function toFileUrl(path: string): URL { - return Deno.build.os === "windows" - ? windowsToFileUrl(path) - : posixToFileUrl(path); -} - -function posixToFileUrl(path: string): URL { - const url = new URL("file:///"); - url.pathname = encodeWhitespace( - path.replace(/%/g, "%25").replace(/\\/g, "%5C"), - ); - return url; -} - -function windowsToFileUrl(path: string): URL { - const [, hostname, pathname] = path.match( - /^(?:[/\\]{2}([^/\\]+)(?=[/\\](?:[^/\\]|$)))?(.*)/, - )!; - const url = new URL("file:///"); - url.pathname = encodeWhitespace(pathname.replace(/%/g, "%25")); - if (hostname != null && hostname != "localhost") { - url.hostname = hostname; - if (!url.hostname) { - throw new TypeError("Invalid hostname."); - } - } - return url; -} - -export async function fetchWithRetries(url: URL | string, maxRetries = 5) { - let sleepMs = 250; - let iterationCount = 0; - while (true) { - iterationCount++; - try { - const res = await fetch(url); - if (res.ok || iterationCount > maxRetries) { - return res; - } - } catch (err) { - if (iterationCount > maxRetries) { - throw err; - } - } - console.warn(`Failed fetching. Retrying in ${sleepMs}ms...`); - await new Promise((resolve) => setTimeout(resolve, sleepMs)); - sleepMs = Math.min(sleepMs * 2, 10_000); - } -} - -// MIT License - Copyright (c) justjavac. -// https://github.com/justjavac/deno_dirs/blob/e8c001bbef558f08fd486d444af391729b0b8068/data_local_dir/mod.ts -function localDataDir(): string | undefined { - switch (Deno.build.os) { - case "linux": { - const xdg = Deno.env.get("XDG_DATA_HOME"); - if (xdg) return xdg; - - const home = Deno.env.get("HOME"); - if (home) return `${home}/.local/share`; - break; - } - - case "darwin": { - const home = Deno.env.get("HOME"); - if (home) return `${home}/Library/Application Support`; - break; - } - - case "windows": - return Deno.env.get("LOCALAPPDATA") ?? undefined; - } - - return undefined; -}