diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..9fc9c2c Binary files /dev/null and b/.DS_Store differ diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 674f1a2..9ac12c3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,8 +31,9 @@ jobs: set -xeuo pipefail rustc --version cargo --version - cargo build --release - deno run -A https://deno.land/x/wasmbuild@0.11.0/main.ts --out src/backend_wasm/lib + cargo build --release -p netsaur + deno run -A https://deno.land/x/wasmbuild@0.11.0/main.ts -p netsaur --out src/backends/wasm/lib + deno run -A https://deno.land/x/wasmbuild@0.11.0/main.ts -p netsaur-tokenizers --out tokenizers/lib - name: Release uses: softprops/action-gh-release@master env: @@ -44,4 +45,5 @@ jobs: target/release/libnetsaur.so target/release/libnetsaur.dylib target/release/netsaur.dll - src/backend_wasm/lib/netsaur_bg.wasm + src/backends/wasm/lib/netsaur_bg.wasm + tokenizers/lib/netsaur_tokenizers_bg.wasm diff --git a/.gitignore b/.gitignore index 0289d11..8357d82 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,5 @@ target/ *.test.bin *.test.st +_test/run.ts +_test/Student_Performance.csv diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c8239ef..79fded4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing -## Building `backend_cpu` +## Building `backends/cpu` Unoptimized: @@ -14,16 +14,16 @@ Optimized: cargo build --release ``` -## Building `backend_wasm` +## Building `backends/wasm` Unoptimized: ```sh -deno run -A https://deno.land/x/wasmbuild@0.11.0/main.ts --out src/backend_wasm/lib --debug +deno run -A https://deno.land/x/wasmbuild@0.11.0/main.ts --out src/backends/wasm/lib --debug ``` Optimized: ```sh -deno run -A https://deno.land/x/wasmbuild@0.11.0/main.ts --out src/backend_wasm/lib +deno run -A https://deno.land/x/wasmbuild@0.11.0/main.ts --out src/backends/wasm/lib ``` diff --git a/Cargo.lock b/Cargo.lock index 7708561..0e37e41 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,12 +2,42 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "aho-corasick" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +dependencies = [ + "memchr", +] + [[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bumpalo" version = "3.13.0" @@ -20,6 +50,139 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "cudarc" +version = "0.9.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc4cab390f4a32340211f015292a4551742a63e528e9ade9e0bde0d1a989d2a1" + +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_builder" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_builder_macro" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" +dependencies = [ + "derive_builder_core", + "syn 1.0.109", +] + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" + +[[package]] +name = "fancy-regex" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b95f7c0680e4142284cf8b22c14a476e87d61b004a3a0861872b32ef7ead40a2" +dependencies = [ + "bit-set", + "regex", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "getrandom" version = "0.2.10" @@ -33,6 +196,21 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.9" @@ -48,6 +226,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + [[package]] name = "libc" version = "0.2.147" @@ -66,6 +250,22 @@ version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" +[[package]] +name = "macro_rules_attribute" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568" + [[package]] name = "matrixmultiply" version = "0.3.7" @@ -76,6 +276,48 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "monostate" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f370ae88093ec6b11a710dec51321a61d420fafd1bad6e30d01bd9c920e8ee" +dependencies = [ + "monostate-impl", + "serde", +] + +[[package]] +name = "monostate-impl" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "371717c0a5543d6a800cac822eac735aa7d2d2fbb41002e9856a4089532dbdce" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.28", +] + [[package]] name = "ndarray" version = "0.15.6" @@ -102,7 +344,7 @@ dependencies = [ [[package]] name = "netsaur" -version = "0.2.7" +version = "0.2.9" dependencies = [ "getrandom", "js-sys", @@ -114,6 +356,43 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "netsaur-gpu" +version = "0.2.9" +dependencies = [ + "cudarc", + "ndarray", + "ndarray-rand", + "safetensors", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "netsaur-tokenizers" +version = "0.2.9" +dependencies = [ + "getrandom", + "js-sys", + "ndarray", + "ndarray-rand", + "serde", + "serde_json", + "tokenizers", + "wasm-bindgen", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "num-complex" version = "0.4.3" @@ -149,6 +428,12 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +[[package]] +name = "paste" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -219,6 +504,66 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" +[[package]] +name = "rayon" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-cond" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "059f538b55efd2309c9794130bc149c6a553db90e9d99c2030785c82f0bd7df9" +dependencies = [ + "either", + "itertools", + "rayon", +] + +[[package]] +name = "rayon-core" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" + [[package]] name = "ryu" version = "1.0.15" @@ -227,14 +572,20 @@ checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "safetensors" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d818a2cb3f564a1844be835011acf5c7ec8ad1986a47f73abc7b5fea91cc3a" +checksum = "f1e5186bd51ae3f90999d243853f5e8cb51f3467f55da42dc611ed2342483dad" dependencies = [ "serde", "serde_json", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "serde" version = "1.0.181" @@ -266,6 +617,30 @@ dependencies = [ "serde", ] +[[package]] +name = "smallvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" + +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64", + "nom", + "serde", + "unicode-segmentation", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "syn" version = "1.0.109" @@ -288,12 +663,84 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "thiserror" +version = "1.0.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.28", +] + +[[package]] +name = "tokenizers" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9be88c795d8b9f9c4002b3a8f26a6d0876103a6f523b32ea3bac52d8560c17c" +dependencies = [ + "aho-corasick", + "derive_builder", + "esaxx-rs", + "fancy-regex", + "getrandom", + "itertools", + "lazy_static", + "log", + "macro_rules_attribute", + "monostate", + "paste", + "rand", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + [[package]] name = "unicode-ident" version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/README.md b/README.md index 5460618..035e1ca 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,9 @@ ### Backends -- [CPU](./src/backend_cpu/) - Native backend written in Rust. -- [WASM](./src/backend_wasm/) - WebAssembly backend written in Rust. -- [GPU](./src/backend_gpu/) (TODO) +- [CPU](./src/backends/cpu/) - Native backend written in Rust. +- [WASM](./src/backends/wasm/) - WebAssembly backend written in Rust. +- [GPU](./src/backends/gpu/) (TODO) ### Examples diff --git a/crates/core-gpu/Cargo.toml b/crates/core-gpu/Cargo.toml new file mode 100644 index 0000000..28eaa10 --- /dev/null +++ b/crates/core-gpu/Cargo.toml @@ -0,0 +1,17 @@ +[package] +edition = "2021" +name = "netsaur-gpu" +version = "0.2.9" + +[lib] +crate-type = ["cdylib"] + + +[dependencies] +ndarray = "0.15.6" +ndarray-rand = "0.14.0" +serde = {version = "1.0", features = ["derive"]} +serde_json = "1.0" +safetensors = "0.4.0" +cudarc = "0.9.14" +thiserror = "1.0.49" diff --git a/crates/core-gpu/src/ffi.rs b/crates/core-gpu/src/ffi.rs new file mode 100644 index 0000000..7a4c00c --- /dev/null +++ b/crates/core-gpu/src/ffi.rs @@ -0,0 +1,109 @@ +use std::slice::{from_raw_parts, from_raw_parts_mut}; + +use crate::{ + decode_array, decode_json, length, Backend, Dataset, Logger, PredictOptions, TrainOptions, + RESOURCES, +}; + +type AllocBufferFn = extern "C" fn(usize) -> *mut u8; + +fn log(string: String) { + println!("{}", string) +} + +#[no_mangle] +pub extern "C" fn ffi_backend_create(ptr: *const u8, len: usize, alloc: AllocBufferFn) -> usize { + let config = decode_json(ptr, len); + let net_backend = Backend::new(config, Logger { log }, None); + let buf: Vec = net_backend.size.iter().map(|x| *x as u8).collect(); + let size_ptr = alloc(buf.len()); + let output_shape = unsafe { from_raw_parts_mut(size_ptr, buf.len()) }; + output_shape.copy_from_slice(buf.as_slice()); + + let mut len = 0; + RESOURCES.with(|cell| { + let mut backend = cell.backend.borrow_mut(); + len = backend.len(); + backend.push(net_backend); + }); + len +} + +#[no_mangle] +pub extern "C" fn ffi_backend_train( + id: usize, + buffer_ptr: *const u64, + buffer_len: usize, + options_ptr: *const u8, + options_len: usize, +) { + let buffer = unsafe { from_raw_parts(buffer_ptr, buffer_len) }; + let options: TrainOptions = decode_json(options_ptr, options_len); + + let mut datasets = Vec::new(); + for i in 0..options.datasets { + let input = buffer[i * 2]; + let output = buffer[i * 2 + 1]; + datasets.push(Dataset { + inputs: decode_array(input as *const f32, options.input_shape.clone()), + outputs: decode_array(output as *const f32, options.output_shape.clone()), + }); + } + + RESOURCES.with(|cell| { + let mut backend = cell.backend.borrow_mut(); + backend[id].train(datasets, options.epochs, options.batches, options.rate) + }); +} + +#[no_mangle] +pub extern "C" fn ffi_backend_predict( + id: usize, + buffer_ptr: *const f32, + options_ptr: *const u8, + options_len: usize, + output_ptr: *mut f32, +) { + let options: PredictOptions = decode_json(options_ptr, options_len); + let inputs = decode_array(buffer_ptr, options.input_shape); + let outputs = unsafe { from_raw_parts_mut(output_ptr, length(options.output_shape)) }; + + RESOURCES.with(|cell| { + let mut backend = cell.backend.borrow_mut(); + let res = backend[id].predict(inputs); + outputs.copy_from_slice(res.as_slice().unwrap()); + }); +} + +#[no_mangle] +pub extern "C" fn ffi_backend_save(id: usize, alloc: AllocBufferFn) { + RESOURCES.with(|cell| { + let backend = cell.backend.borrow_mut(); + let data = backend[id].save(); + let file_ptr = alloc(data.len()); + let file = unsafe { from_raw_parts_mut(file_ptr, data.len()) }; + file.copy_from_slice(data.as_slice()); + }); +} + +#[no_mangle] +pub extern "C" fn ffi_backend_load( + file_ptr: *const u8, + file_len: usize, + alloc: AllocBufferFn, +) -> usize { + let buffer = unsafe { from_raw_parts(file_ptr, file_len) }; + let net_backend = Backend::load(buffer, Logger { log }); + let buf: Vec = net_backend.size.iter().map(|x| *x as u8).collect(); + let size_ptr = alloc(buf.len()); + let output_shape = unsafe { from_raw_parts_mut(size_ptr, buf.len()) }; + output_shape.copy_from_slice(buf.as_slice()); + + let mut len = 0; + RESOURCES.with(|cell| { + let mut backend = cell.backend.borrow_mut(); + len = backend.len(); + backend.push(net_backend); + }); + len +} diff --git a/crates/core-gpu/src/gpu/activation.rs b/crates/core-gpu/src/gpu/activation.rs new file mode 100644 index 0000000..204cd07 --- /dev/null +++ b/crates/core-gpu/src/gpu/activation.rs @@ -0,0 +1,112 @@ +use crate::Activation; +pub struct GPUActivation { + pub activation: Activation, + pub activate: ActivationFn, + pub prime: ActivationFn, +} + +type ActivationFn = fn(x: &f32) -> f32; + +impl GPUActivation { + pub fn from(activation: Activation) -> Self { + let (activate, prime): (ActivationFn, ActivationFn) = match activation { + Activation::Elu => (elu, elu_prime), + Activation::LeakyRelu => (leaky_relu, leaky_relu_prime), + Activation::Linear => (linear, linear_prime), + Activation::Relu => (relu, relu_prime), + Activation::Relu6 => (relu6, relu6_prime), + Activation::Selu => (selu, selu_prime), + Activation::Sigmoid => (sigmoid, sigmoid_prime), + Activation::Tanh => (tanh, tanh_prime), + }; + + Self { + activation, + activate, + prime, + } + } + + pub fn from_option(activation: Option) -> Option { + if let Some(activation) = activation { + Some(GPUActivation::from(activation)) + } else { + None + } + } + + pub fn memoize_output(activation: &GPUActivation) -> bool { + match activation.activation { + Activation::Sigmoid | Activation::Tanh => true, + _ => true, + } + } +} + +fn sigmoid(x: &f32) -> f32 { + return 1.0 / (1.0 + (-x).exp()); +} + +fn sigmoid_prime(x: &f32) -> f32 { + return x * (1.0 - x); +} + +fn tanh(x: &f32) -> f32 { + return x.tanh(); +} + +fn tanh_prime(x: &f32) -> f32 { + return 1.0 - tanh(x).powi(2); +} + +fn linear(x: &f32) -> f32 { + return *x; +} + +fn linear_prime(_x: &f32) -> f32 { + return 1.0; +} + +fn relu(x: &f32) -> f32 { + return x.max(0.0); +} + +fn relu_prime(x: &f32) -> f32 { + return if *x > 0.0 { 1.0 } else { 0.0 }; +} + +fn relu6(x: &f32) -> f32 { + return x.max(0.0).min(6.0); +} + +fn relu6_prime(x: &f32) -> f32 { + return if *x > 0.0 && *x < 6.0 { 1.0 } else { 0.0 }; +} + +fn leaky_relu(x: &f32) -> f32 { + return if *x > 0.0 { *x } else { x.max(0.01 * x) }; +} + +fn leaky_relu_prime(x: &f32) -> f32 { + return if *x > 0.0 { 1.0 } else { 0.01 }; +} + +fn elu(x: &f32) -> f32 { + return if *x >= 0.0 { *x } else { x.exp() - 1.0 }; +} + +fn elu_prime(x: &f32) -> f32 { + return if *x > 0.0 { 1.0 } else { x.exp() }; +} + +fn selu(x: &f32) -> f32 { + return if *x >= 0.0 { + *x + } else { + 1.0507 * (x.exp() - 1.0) + }; +} + +fn selu_prime(x: &f32) -> f32 { + return if *x > 0.0 { 1.0 } else { 1.0507 * x.exp() }; +} diff --git a/crates/core-gpu/src/gpu/backend.rs b/crates/core-gpu/src/gpu/backend.rs new file mode 100644 index 0000000..ed89419 --- /dev/null +++ b/crates/core-gpu/src/gpu/backend.rs @@ -0,0 +1,343 @@ +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use ndarray::{ArrayD, ArrayViewD, IxDyn}; +use safetensors::{serialize, SafeTensors}; + +use crate::{ + to_arr, ActivationGPULayer, BackendConfig, BatchNorm1DGPULayer, BatchNorm2DGPULayer, + BatchNormTensors, Conv2DGPULayer, ConvTensors, ConvTranspose2DGPULayer, Dataset, DenseGPULayer, + DenseTensors, Dropout1DGPULayer, Dropout2DGPULayer, FlattenGPULayer, GPUCost, GPULayer, + GPUOptimizer, GPUScheduler, GetTensor, Layer, Logger, Pool2DGPULayer, SoftmaxGPULayer, Tensor, + Tensors, +}; +pub use cudarc; + +use crate::DType; + +/// cudarc related errors +#[derive(thiserror::Error, Debug)] +pub enum CudaError { + #[error(transparent)] + Cuda(#[from] cudarc::driver::DriverError), + + #[error(transparent)] + Compiler(#[from] cudarc::nvrtc::CompileError), + + #[error(transparent)] + Cublas(#[from] cudarc::cublas::result::CublasError), + + #[error(transparent)] + Curand(#[from] cudarc::curand::result::CurandError), + + #[error("missing kernel '{module_name}'")] + MissingKernel { module_name: String }, + + #[error("unsupported dtype {dtype:?} for {op}")] + UnsupportedDtype { dtype: DType, op: &'static str }, + + #[error("internal error '{0}'")] + InternalError(&'static str), + + #[error("matmul is only supported for contiguous tensors lstride: {lhs_stride:?} rstride: {rhs_stride:?} mnk: {mnk:?}")] + MatMulNonContiguous { + lhs_stride: Vec, + rhs_stride: Vec, + mnk: (usize, usize, usize), + }, + + #[error("{msg}, expected: {expected:?}, got: {got:?}")] + UnexpectedDType { + msg: &'static str, + expected: DType, + got: DType, + }, + + #[error("{cuda} when loading {module_name}")] + Load { + cuda: cudarc::driver::DriverError, + module_name: String, + }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct DeviceId(usize); + +impl DeviceId { + fn new() -> Self { + use std::sync::atomic; + static COUNTER: atomic::AtomicUsize = atomic::AtomicUsize::new(1); + Self(COUNTER.fetch_add(1, atomic::Ordering::Relaxed)) + } +} + +struct CudaRng(cudarc::curand::CudaRng); +unsafe impl Send for CudaRng {} + +#[derive(Clone)] +pub struct CudaDevice { + id: DeviceId, + device: Arc, + #[allow(dead_code)] + blas: Arc, + #[allow(dead_code)] + curand: Arc>, +} + +impl std::fmt::Debug for CudaDevice { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "CudaDevice({:?})", self.id) + } +} + +impl std::ops::Deref for CudaDevice { + type Target = Arc; + + fn deref(&self) -> &Self::Target { + &self.device + } +} + +impl CudaDevice { + pub fn new(ordinal: usize) -> Self { + let device = cudarc::driver::CudaDevice::new(ordinal).unwrap(); + let blas = cudarc::cublas::CudaBlas::new(device.clone()).unwrap(); + let curand = cudarc::curand::CudaRng::new(299792458, device.clone()).unwrap(); + + Self { + id: DeviceId::new(), + device, + blas: Arc::new(blas), + curand: Arc::new(Mutex::new(CudaRng(curand))), + } + } + + pub fn cuda_device(&self) -> Arc { + self.device.clone() + } + + pub fn id(&self) -> DeviceId { + self.id + } +} + +pub struct Backend { + pub silent: bool, + pub config: BackendConfig, + pub cuda_device: CudaDevice, + pub layers: Vec, + pub size: Vec, + pub cost: GPUCost, + pub optimizer: GPUOptimizer, + pub scheduler: GPUScheduler, + pub logger: Logger, +} + +impl Backend { + pub fn new(config: BackendConfig, logger: Logger, mut tensors: Option>) -> Self { + let mut layers = Vec::new(); + let mut size = config.size.clone(); + for layer in config.layers.iter() { + match layer.clone() { + Layer::Activation(config) => { + let layer = ActivationGPULayer::new(config, IxDyn(&size)); + layers.push(GPULayer::Activation(layer)); + } + Layer::Conv2D(config) => { + let layer = Conv2DGPULayer::new(config, IxDyn(&size), tensors.get()); + size = layer.output_size().to_vec(); + layers.push(GPULayer::Conv2D(layer)); + } + Layer::ConvTranspose2D(config) => { + let layer = ConvTranspose2DGPULayer::new(config, IxDyn(&size), tensors.get()); + size = layer.output_size().to_vec(); + layers.push(GPULayer::ConvTranspose2D(layer)); + } + Layer::BatchNorm1D(config) => { + let layer = BatchNorm1DGPULayer::new(config, IxDyn(&size), tensors.get()); + layers.push(GPULayer::BatchNorm1D(layer)); + } + Layer::BatchNorm2D(config) => { + let layer = BatchNorm2DGPULayer::new(config, IxDyn(&size), tensors.get()); + layers.push(GPULayer::BatchNorm2D(layer)); + } + Layer::Dropout1D(config) => { + let layer = Dropout1DGPULayer::new(config, IxDyn(&size)); + layers.push(GPULayer::Dropout1D(layer)); + } + Layer::Dropout2D(config) => { + let layer = Dropout2DGPULayer::new(config, IxDyn(&size)); + layers.push(GPULayer::Dropout2D(layer)); + } + Layer::Dense(config) => { + let layer = DenseGPULayer::new(config, IxDyn(&size), tensors.get()); + size = layer.output_size().to_vec(); + layers.push(GPULayer::Dense(layer)); + } + Layer::Flatten(config) => { + let layer = FlattenGPULayer::new(config, IxDyn(&size)); + size = layer.output_size().to_vec(); + layers.push(GPULayer::Flatten(layer)); + } + Layer::Pool2D(config) => { + let layer = Pool2DGPULayer::new(config, IxDyn(&size)); + size = layer.output_size().to_vec(); + layers.push(GPULayer::Pool2D(layer)); + } + Layer::Softmax => { + let layer = SoftmaxGPULayer::new(IxDyn(&size)); + layers.push(GPULayer::Softmax(layer)); + } + } + } + let optimizer = GPUOptimizer::from(config.optimizer.clone(), &mut layers); + let scheduler = GPUScheduler::from(&config.scheduler); + let cost = GPUCost::from(config.cost.clone()); + let silent = config.silent.is_some_and(|x| x == true); + let cuda_device = CudaDevice::new(0); + Self { + logger, + cuda_device, + silent, + config, + layers, + cost, + optimizer, + scheduler, + size, + } + } + + pub fn forward_propagate(&mut self, mut inputs: ArrayD, training: bool) -> ArrayD { + for layer in &mut self.layers { + inputs = layer.forward_propagate(inputs, training); + } + inputs + } + + pub fn backward_propagate<'b>( + &mut self, + outputs: ArrayViewD<'b, f32>, + data: ArrayViewD<'b, f32>, + ) -> ArrayD { + let mut d_outputs = (self.cost.prime)(data, outputs); + for layer in self.layers.iter_mut().rev() { + d_outputs = layer.backward_propagate(d_outputs); + } + d_outputs + } + + pub fn train(&mut self, datasets: Vec, epochs: usize, batches: usize, rate: f32) { + let mut epoch = 0; + while epoch < epochs { + let mut total = 0.0; + for (i, dataset) in datasets.iter().enumerate() { + let outputs = self.forward_propagate(dataset.inputs.clone(), true); + self.backward_propagate(outputs.view(), dataset.outputs.view()); + self.optimizer + .update_grads(&mut self.layers, &self.scheduler, rate, epoch); + total += (self.cost.cost)(outputs.view(), dataset.outputs.view()); + let minibatch = outputs.dim()[0]; + if !self.silent && ((i + 1) * minibatch) % batches == 0 { + let cost = total / (batches) as f32; + let msg = format!("Epoch={}, Dataset={}, Cost={}", epoch, i * minibatch, cost); + (self.logger.log)(msg); + total = 0.0; + } + } + epoch += 1 + } + } + + pub fn predict(&mut self, data: ArrayD) -> ArrayD { + for layer in &mut self.layers { + layer.reset(1) + } + self.forward_propagate(data, false) + } + + pub fn save(&self) -> Vec { + let mut tensors = Vec::new(); + for (i, layer) in self.layers.iter().enumerate() { + match layer { + GPULayer::BatchNorm1D(layer) => { + let gamma = Tensor::new(layer.gamma.view().into_dyn()); + let beta = Tensor::new(layer.beta.view().into_dyn()); + let running_mean = Tensor::new(layer.running_mean.view().into_dyn()); + let running_var = Tensor::new(layer.running_var.view().into_dyn()); + tensors.push((format!("{}g", i), gamma)); + tensors.push((format!("{}b", i), beta)); + tensors.push((format!("{}m", i), running_mean)); + tensors.push((format!("{}v", i), running_var)); + } + GPULayer::BatchNorm2D(layer) => { + let gamma = Tensor::new(layer.gamma.view().into_dyn()); + let beta = Tensor::new(layer.beta.view().into_dyn()); + let running_mean = Tensor::new(layer.running_mean.view().into_dyn()); + let running_var = Tensor::new(layer.running_var.view().into_dyn()); + tensors.push((format!("{}g", i), gamma)); + tensors.push((format!("{}b", i), beta)); + tensors.push((format!("{}m", i), running_mean)); + tensors.push((format!("{}v", i), running_var)); + } + GPULayer::ConvTranspose2D(layer) => { + let weights = Tensor::new(layer.weights.view().into_dyn()); + let biases = Tensor::new(layer.biases.view().into_dyn()); + tensors.push((format!("{}w", i), weights)); + tensors.push((format!("{}b", i), biases)); + } + GPULayer::Conv2D(layer) => { + let weights = Tensor::new(layer.weights.view().into_dyn()); + let biases = Tensor::new(layer.biases.view().into_dyn()); + tensors.push((format!("{}w", i), weights)); + tensors.push((format!("{}b", i), biases)); + } + GPULayer::Dense(layer) => { + let weights = Tensor::new(layer.weights.view().into_dyn()); + let biases = Tensor::new(layer.biases.view().into_dyn()); + tensors.push((format!("{}w", i), weights)); + tensors.push((format!("{}b", i), biases)); + } + _ => {} + } + } + let config = serde_json::to_string(&self.config).unwrap(); + let metadata = HashMap::from([("metadata".to_string(), config)]); + serialize(tensors, &Some(metadata)).unwrap() + } + + pub fn load(buffer: &[u8], logger: Logger) -> Self { + let tensors = SafeTensors::deserialize(buffer).unwrap(); + let (_, metadata) = SafeTensors::read_metadata(buffer).unwrap(); + let data = metadata.metadata().as_ref().unwrap(); + let json = data.get("metadata").unwrap(); + let config: BackendConfig = serde_json::from_str(json).unwrap(); + let mut layers = Vec::new(); + + for (i, layer) in config.layers.iter().enumerate() { + match layer { + Layer::BatchNorm1D(_) | Layer::BatchNorm2D(_) => { + layers.push(Tensors::BatchNorm(BatchNormTensors { + gamma: to_arr(tensors.tensor(&format!("{}g", i)).unwrap()), + beta: to_arr(tensors.tensor(&format!("{}b", i)).unwrap()), + running_mean: to_arr(tensors.tensor(&format!("{}m", i)).unwrap()), + running_var: to_arr(tensors.tensor(&format!("{}v", i)).unwrap()), + })) + } + Layer::Dense(_) => layers.push(Tensors::Dense(DenseTensors { + weights: to_arr(tensors.tensor(&format!("{}w", i)).unwrap()), + biases: to_arr(tensors.tensor(&format!("{}b", i)).unwrap()), + })), + Layer::Conv2D(_) | Layer::ConvTranspose2D(_) => { + layers.push(Tensors::Conv(ConvTensors { + weights: to_arr(tensors.tensor(&format!("{}w", i)).unwrap()), + biases: to_arr(tensors.tensor(&format!("{}b", i)).unwrap()), + })) + } + _ => {} + }; + } + + Backend::new(config, logger, Some(layers)) + } +} diff --git a/crates/core-gpu/src/gpu/cost.rs b/crates/core-gpu/src/gpu/cost.rs new file mode 100644 index 0000000..53e2cdf --- /dev/null +++ b/crates/core-gpu/src/gpu/cost.rs @@ -0,0 +1,89 @@ +use std::ops::{Div, Mul, Sub}; + +use ndarray::{s, ArrayD, ArrayViewD}; + +use crate::Cost; + +pub struct GPUCost { + pub cost: for<'a> fn(y_hat: ArrayViewD<'a, f32>, y: ArrayViewD<'a, f32>) -> f32, + pub prime: for<'a> fn(y_hat: ArrayViewD<'a, f32>, y: ArrayViewD<'a, f32>) -> ArrayD, +} + +impl GPUCost { + pub fn from(cost: Cost) -> GPUCost { + match cost { + Cost::MSE => GPUCost { + cost: mse, + prime: mse_prime, + }, + Cost::CrossEntropy => GPUCost { + cost: cross_entropy, + prime: cross_entropy_prime, + }, + Cost::BinCrossEntropy => GPUCost { + cost: bin_cross_entropy, + prime: bin_cross_entropy_prime, + }, + Cost::Hinge => GPUCost { + cost: hinge, + prime: hinge_prime, + }, + } + } +} + +fn mse<'a>(y_hat: ArrayViewD<'a, f32>, y: ArrayViewD<'a, f32>) -> f32 { + let sub = y.sub(&y_hat); + return sub.clone().mul(sub).sum(); +} + +fn mse_prime<'a>(y_hat: ArrayViewD<'a, f32>, y: ArrayViewD<'a, f32>) -> ArrayD { + return y.sub(&y_hat); +} + +fn cross_entropy<'a>(y_hat: ArrayViewD<'a, f32>, y: ArrayViewD<'a, f32>) -> f32 { + let batches = y_hat.dim()[0]; + let mut total = 0.0; + for b in 0..batches { + total -= y_hat.slice(s![b, ..]).mul(&y.slice(s![b, ..])).sum().ln() + } + return total / batches as f32; +} + +fn cross_entropy_prime<'a>(y_hat: ArrayViewD<'a, f32>, y: ArrayViewD<'a, f32>) -> ArrayD { + return -y_hat.div(&y); +} + +fn bin_cross_entropy<'a>(y_hat: ArrayViewD<'a, f32>, y: ArrayViewD<'a, f32>) -> f32 { + return -y_hat + .mul(y.map(|x| x.ln())) + .sub(((1.0).sub(&y_hat)).mul(y.map(|x| 1.0 - x.ln()))) + .sum() + / y.len() as f32; +} + +fn bin_cross_entropy_prime<'a>(y_hat: ArrayViewD<'a, f32>, y: ArrayViewD<'a, f32>) -> ArrayD { + return y.sub(&y_hat).div(y.mul(1.0.sub(&y))); +} + +fn hinge<'a>(y_hat: ArrayViewD<'a, f32>, y: ArrayViewD<'a, f32>) -> f32 { + let mut sum = 0.0; + for (y_hat_i, y_i) in y_hat.iter().zip(y.iter()) { + let margin = 1.0 - y_hat_i * y_i; + if margin > 0.0 { + sum += margin; + } + } + return sum; +} + +fn hinge_prime<'a>(y_hat: ArrayViewD<'a, f32>, y: ArrayViewD<'a, f32>) -> ArrayD { + let mut result = ArrayD::zeros(y_hat.shape()); + for ((result_i, y_hat_i), y_i) in result.iter_mut().zip(y_hat.iter()).zip(y.iter()) { + let margin = 1.0 - y_hat_i * y_i; + if margin > 0.0 { + *result_i = -y_i; + } + } + return result; +} diff --git a/crates/core-gpu/src/gpu/init.rs b/crates/core-gpu/src/gpu/init.rs new file mode 100644 index 0000000..8f5fa8c --- /dev/null +++ b/crates/core-gpu/src/gpu/init.rs @@ -0,0 +1,52 @@ +use ndarray::{ArrayD, IxDyn}; +use ndarray_rand::{ + rand_distr::{Normal, Uniform}, + RandomExt, +}; + +use crate::Init; + +pub struct GPUInit { + pub init: Init, +} + +impl GPUInit { + pub fn from(init: Init) -> Self { + Self { init } + } + + pub fn init(&self, size: IxDyn, input_size: usize, output_size: usize) -> ArrayD { + match self.init { + Init::Uniform => uniform(size), + Init::Xavier => xavier(size, input_size), + Init::XavierN => xaviern(size, input_size, output_size), + Init::Kaiming => kaiming(size, input_size), + } + } + + pub fn from_default(init: Option, default: Init) -> Self { + if let Some(init) = init { + return Self { init }; + } + Self { init: default } + } +} + +pub fn uniform(size: IxDyn) -> ArrayD { + ArrayD::random(size, Uniform::new(-1.0, 1.0)) +} + +pub fn xavier(size: IxDyn, input_size: usize) -> ArrayD { + let bounds = 1.0 / (input_size as f32).sqrt(); + ArrayD::random(size, Uniform::new(-bounds, bounds)) +} + +pub fn xaviern(size: IxDyn, input_size: usize, output_size: usize) -> ArrayD { + let bounds = (6.0 as f32).sqrt() / ((input_size + output_size) as f32).sqrt(); + ArrayD::random(size, Uniform::new(-bounds, bounds)) +} + +pub fn kaiming(size: IxDyn, input_size: usize) -> ArrayD { + let deviation = (2.0 / (input_size as f32)).sqrt(); + ArrayD::random(size, Normal::new(0.0, deviation).unwrap()) +} diff --git a/crates/core-gpu/src/gpu/layers/activation.rs b/crates/core-gpu/src/gpu/layers/activation.rs new file mode 100644 index 0000000..0536efc --- /dev/null +++ b/crates/core-gpu/src/gpu/layers/activation.rs @@ -0,0 +1,99 @@ +use ndarray::{s, ArrayD, Dimension, IxDyn}; +use std::ops::{Div, Mul, Sub}; + +use crate::{ActivationLayer, GPUActivation}; + +pub struct ActivationGPULayer { + pub outputs: ArrayD, + pub activation: GPUActivation, +} + +impl ActivationGPULayer { + pub fn new(config: ActivationLayer, size: IxDyn) -> Self { + Self { + outputs: ArrayD::zeros(size), + activation: GPUActivation::from(config.activation), + } + } + + pub fn output_size(&self) -> Vec { + self.outputs.shape().to_vec() + } + + pub fn reset(&mut self, batches: usize) { + let mut output_size = self.outputs.shape().to_vec(); + output_size[0] = batches; + self.outputs = ArrayD::zeros(output_size); + } + + pub fn forward_propagate(&mut self, inputs: ArrayD) -> ArrayD { + let outputs = if GPUActivation::memoize_output(&self.activation) { + self.outputs = inputs.map(self.activation.activate); + self.outputs.clone() + } else { + self.outputs = inputs.clone(); + inputs.map(self.activation.activate) + }; + outputs.into_dyn() + } + + pub fn backward_propagate(&mut self, d_outputs: ArrayD) -> ArrayD { + let d_inputs = d_outputs.mul(self.outputs.map(self.activation.prime)); + d_inputs.into_dyn() + } +} + +pub struct SoftmaxGPULayer { + pub outputs: ArrayD, +} + +impl SoftmaxGPULayer { + pub fn new(size: IxDyn) -> Self { + Self { + outputs: ArrayD::zeros(size), + } + } + + pub fn output_size(&self) -> Vec { + self.outputs.shape().to_vec() + } + + pub fn reset(&mut self, batches: usize) { + let mut output_size = self.outputs.shape().to_vec(); + output_size[0] = batches; + self.outputs = ArrayD::zeros(output_size); + } + + pub fn forward_propagate(&mut self, inputs: ArrayD) -> ArrayD { + let batches = self.outputs.dim()[0]; + for b in 0..batches { + let exp = inputs.slice(s![b, ..]).map(|x| x.exp()); + self.outputs + .slice_mut(s![b, ..]) + .assign(&exp.clone().div(exp.sum())); + } + self.outputs.clone().into_dyn() + } + + pub fn backward_propagate(&mut self, d_outputs: ArrayD) -> ArrayD { + let batches = self.outputs.dim()[0]; + let array_size = self.outputs.dim().size() / batches; + + let mut d_inputs = ArrayD::zeros(self.outputs.dim()); + for b in 0..batches { + for y in 0..array_size { + for x in 0..array_size { + let out1 = self.outputs[[b, y]]; + let out2 = self.outputs[[b, x]]; + let d_out = d_outputs[[b, x]]; + if x == y { + d_inputs[[b, y]] += out1.sub(out1.powi(2)).mul(d_out); + } else { + d_inputs[[b, y]] += -out1.mul(out2).mul(d_out); + } + } + } + } + d_inputs + } +} diff --git a/crates/core-gpu/src/gpu/layers/batchnorm1d.rs b/crates/core-gpu/src/gpu/layers/batchnorm1d.rs new file mode 100644 index 0000000..ef33ae4 --- /dev/null +++ b/crates/core-gpu/src/gpu/layers/batchnorm1d.rs @@ -0,0 +1,158 @@ +use std::ops::{Add, AddAssign, Div, Mul, Sub}; + +use ndarray::{Array2, ArrayD, Axis, Ix2, IxDyn}; + +use crate::{BatchNormLayer, Tensors}; + +macro_rules! axes { + (($array:expr).sum_axes($($axis:literal),+)) => { + $array$( .sum_axis(Axis($axis)).insert_axis(Axis($axis)) )+ + }; + (($array:expr).mean_axes($($axis:literal),+)) => { + $array$( .mean_axis(Axis($axis)).unwrap().insert_axis(Axis($axis)) )+ + }; + (($array:expr).var_axes($($axis:literal),+)) => { + $array$( .var_axis(Axis($axis), 0.0).insert_axis(Axis($axis)) )+ + }; +} + +pub struct BatchNorm1DGPULayer { + // cache + pub inputs: Array2, + pub mean: Array2, + pub var: Array2, + pub std_dev: Array2, + pub normalized: Array2, + pub epsilon: f32, + pub momentum: f32, + + // parameters + pub gamma: Array2, + pub beta: Array2, + pub running_mean: Array2, + pub running_var: Array2, + + // gradients + pub d_gamma: Array2, + pub d_beta: Array2, +} + +impl BatchNorm1DGPULayer { + pub fn new(config: BatchNormLayer, size: IxDyn, tensors: Option) -> Self { + let input_size = [size[0], size[1]]; + + let (gamma, beta, running_mean, running_var) = + if let Some(Tensors::BatchNorm(tensors)) = tensors { + ( + tensors.gamma.into_dimensionality().unwrap(), + tensors.beta.into_dimensionality().unwrap(), + tensors.running_mean.into_dimensionality().unwrap(), + tensors.running_var.into_dimensionality().unwrap(), + ) + } else { + ( + Array2::ones((1, size[1])), + Array2::zeros((1, size[1])), + Array2::zeros((1, size[1])), + Array2::ones((1, size[1])), + ) + }; + + Self { + inputs: Array2::zeros(input_size), + mean: Array2::zeros((1, size[1])), + var: Array2::zeros((1, size[1])), + std_dev: Array2::zeros((1, size[1])), + normalized: Array2::zeros(input_size), + epsilon: config.epsilon, + momentum: config.momentum, + + gamma, + beta, + running_mean, + running_var, + + d_gamma: Array2::zeros((1, size[1])), + d_beta: Array2::zeros((1, size[1])), + } + } + + pub fn output_size(&self) -> Vec { + self.inputs.shape().to_vec() + } + + pub fn reset(&mut self, batches: usize) { + let size = self.inputs.shape(); + self.inputs = Array2::zeros((batches, size[1])); + let size = self.inputs.shape(); + self.normalized = Array2::zeros((batches, size[1])); + } + + pub fn forward_propagate(&mut self, inputs: ArrayD, training: bool) -> ArrayD { + self.inputs = inputs.into_dimensionality::().unwrap(); + + if training { + self.mean = axes!((self.inputs).mean_axes(0)); + self.var = axes!((self.inputs).var_axes(0)); + self.running_mean = self + .running_mean + .view() + .mul(self.momentum) + .add(self.mean.view().mul(1.0 - self.momentum)); + self.running_var = self + .running_var + .view() + .mul(self.momentum) + .add(self.var.view().mul(1.0 - self.momentum)); + } else { + self.mean = self.running_mean.clone(); + self.var = self.running_var.clone(); + }; + self.var.add_assign(self.epsilon); + self.std_dev = self.var.map(|x| x.sqrt()); + self.normalized = self + .inputs + .view() + .sub(&self.mean.view()) + .div(&self.std_dev.view()); + let batch_norm = self + .gamma + .view() + .mul(&self.normalized.view()) + .add(self.beta.view()); + batch_norm.into_dyn() + } + + pub fn backward_propagate(&mut self, d_outputs: ArrayD) -> ArrayD { + let d_outputs = d_outputs.into_dimensionality::().unwrap(); + + let batches = self.inputs.shape()[0] as f32; + + let mean_diff = self.inputs.view().sub(&self.mean.view()); + let d_normalized = d_outputs.view().mul(&self.gamma.view()); + let d_var = axes!((d_normalized + .view() + .mul(&mean_diff.view()) + .mul(-0.5) + .mul(self.var.view().map(|x| x.powf(-1.5)))) + .sum_axes(0)); + let d_mean = &d_normalized + .view() + .mul(axes!(((-1.0).div(&self.std_dev.view())).sum_axes(0))) + .add( + d_var + .view() + .mul(&axes!(((-2.0).mul(&mean_diff.view())).sum_axes(0))) + .div(batches), + ); + self.d_gamma = axes!((d_outputs.view().mul(&self.normalized.view())).sum_axes(0)); + self.d_beta = axes!((d_outputs).sum_axes(0)); + + d_normalized + .view() + .div(&self.std_dev.view()) + .add(&d_var.mul(2.0).mul(&mean_diff.view()).div(batches)) + .add(d_mean.div(batches)) + .into_dyn() + } +} diff --git a/crates/core-gpu/src/gpu/layers/batchnorm2d.rs b/crates/core-gpu/src/gpu/layers/batchnorm2d.rs new file mode 100644 index 0000000..9d1aaad --- /dev/null +++ b/crates/core-gpu/src/gpu/layers/batchnorm2d.rs @@ -0,0 +1,162 @@ +use std::ops::{Add, AddAssign, Div, Mul, Sub}; + +use ndarray::{Array4, ArrayD, Axis, Ix4, IxDyn}; + +use crate::{BatchNormLayer, Tensors}; + +macro_rules! axes { + (($array:expr).sum_axes($($axis:literal),+)) => { + $array$( .sum_axis(Axis($axis)).insert_axis(Axis($axis)) )+ + }; + (($array:expr).mean_axes($($axis:literal),+)) => { + $array$( .mean_axis(Axis($axis)).unwrap().insert_axis(Axis($axis)) )+ + }; +} + +pub struct BatchNorm2DGPULayer { + // cache + pub inputs: Array4, + pub mean: Array4, + pub var: Array4, + pub std_dev: Array4, + pub normalized: Array4, + pub epsilon: f32, + pub momentum: f32, + + // parameters + pub gamma: Array4, + pub beta: Array4, + pub running_mean: Array4, + pub running_var: Array4, + + // gradients + pub d_gamma: Array4, + pub d_beta: Array4, +} + +impl BatchNorm2DGPULayer { + pub fn new(config: BatchNormLayer, size: IxDyn, tensors: Option) -> Self { + let input_size = [size[0], size[1], size[2], size[3]]; + + let (gamma, beta, running_mean, running_var) = + if let Some(Tensors::BatchNorm(tensors)) = tensors { + ( + tensors.gamma.into_dimensionality().unwrap(), + tensors.beta.into_dimensionality().unwrap(), + tensors.running_mean.into_dimensionality().unwrap(), + tensors.running_var.into_dimensionality().unwrap(), + ) + } else { + ( + Array4::ones((1, size[1], 1, 1)), + Array4::zeros((1, size[1], 1, 1)), + Array4::zeros((1, size[1], 1, 1)), + Array4::ones((1, size[1], 1, 1)), + ) + }; + + Self { + inputs: Array4::zeros(input_size), + mean: Array4::zeros((1, size[1], 1, 1)), + var: Array4::zeros((1, size[1], 1, 1)), + std_dev: Array4::zeros((1, size[1], 1, 1)), + normalized: Array4::zeros(input_size), + epsilon: config.epsilon, + momentum: config.momentum, + + gamma, + beta, + running_mean, + running_var, + + d_gamma: Array4::zeros((1, size[1], 1, 1)), + d_beta: Array4::zeros((1, size[1], 1, 1)), + } + } + + pub fn output_size(&self) -> Vec { + self.inputs.shape().to_vec() + } + + pub fn reset(&mut self, batches: usize) { + let size = self.inputs.shape(); + self.inputs = Array4::zeros((batches, size[1], size[2], size[3])); + let size = self.inputs.shape(); + self.normalized = Array4::zeros((batches, size[1], size[2], size[3])); + } + + pub fn forward_propagate(&mut self, inputs: ArrayD, training: bool) -> ArrayD { + self.inputs = inputs.into_dimensionality::().unwrap(); + + if training { + self.mean = axes!((self.inputs).mean_axes(0, 2, 3)); + self.var = axes!((self.inputs.map(|x| x.powi(2))).sum_axes(0, 2, 3)) + .div((self.inputs.len() / self.mean.len()) as f32) + .sub(&self.mean.map(|x| x.powi(2))); + self.running_mean = self + .running_mean + .view() + .mul(self.momentum) + .add(self.mean.view().mul(1.0 - self.momentum)); + self.running_var = self + .running_var + .view() + .mul(self.momentum) + .add(self.var.view().mul(1.0 - self.momentum)); + } else { + self.mean = self.running_mean.clone(); + self.var = self.running_var.clone(); + }; + self.var.add_assign(self.epsilon); + self.std_dev = self.var.map(|x| x.sqrt()); + self.normalized = self + .inputs + .view() + .sub(&self.mean.view()) + .div(&self.std_dev.view()); + let batch_norm = self + .gamma + .view() + .mul(&self.normalized.view()) + .add(self.beta.view()); + batch_norm.into_dyn() + } + + pub fn backward_propagate(&mut self, d_outputs: ArrayD) -> ArrayD { + let d_outputs = d_outputs.into_dimensionality::().unwrap(); + let output_y = self.inputs.shape()[2] as f32; + let output_x = self.inputs.shape()[3] as f32; + + let mean_diff = self.inputs.view().sub(&self.mean.view()); + let d_normalized = d_outputs.view().mul(&self.gamma.view()); + let d_var = axes!((d_normalized + .view() + .mul(&mean_diff.view()) + .mul(-0.5) + .mul(self.var.view().map(|x| x.powf(-1.5)))) + .sum_axes(0, 2, 3)); + let d_mean = &d_normalized + .view() + .mul(axes!(((-1.0).div(&self.std_dev.view())).sum_axes(0, 2, 3))) + .add( + d_var + .view() + .mul(&axes!(((-2.0).mul(&mean_diff.view())).sum_axes(0, 2, 3))) + .div(output_y * output_x), + ); + self.d_gamma = axes!((d_outputs.view().mul(&self.normalized.view())).sum_axes(0, 2, 3)); + self.d_beta = axes!((d_outputs).sum_axes(0, 2, 3)); + + d_normalized + .view() + .div(&self.std_dev.view()) + .add( + &d_var + .mul(2.0) + .mul(&mean_diff.view()) + .div(output_y * output_x), + ) + .add(d_mean.div(output_y * output_x)) + .into_dyn() + } +} diff --git a/crates/core-gpu/src/gpu/layers/conv2d.rs b/crates/core-gpu/src/gpu/layers/conv2d.rs new file mode 100644 index 0000000..27fdab1 --- /dev/null +++ b/crates/core-gpu/src/gpu/layers/conv2d.rs @@ -0,0 +1,144 @@ +use ndarray::{s, Array1, Array4, ArrayD, Dimension, Ix1, Ix4, IxDyn}; +use std::ops::{Add, AddAssign, Mul}; + +use crate::{GPUInit, Conv2DLayer, Init, Tensors}; + +pub struct Conv2DGPULayer { + // cache + pub strides: Vec, + pub padding: Vec, + pub inputs: Array4, + pub output_size: Ix4, + + // parameters + pub weights: Array4, + pub biases: Array1, + + // gradients + pub d_weights: Array4, + pub d_biases: Array1, +} + +impl Conv2DGPULayer { + pub fn new(config: Conv2DLayer, size: IxDyn, tensors: Option) -> Self { + let strides = config.strides.unwrap_or(vec![1, 1]); + let padding = config.padding.unwrap_or(vec![0, 0]); + let input_y = size[2] + 2 * padding[0]; + let input_x = size[3] + 2 * padding[1]; + let output_y = 1 + (input_y - config.kernel_size[2]) / strides[0]; + let output_x = 1 + (input_x - config.kernel_size[3]) / strides[1]; + let input_size = Ix4(size[0], size[1], input_y, input_x); + let weight_size = IxDyn(config.kernel_size.as_slice()); + let output_size = Ix4(size[0], weight_size[0], output_y, output_x); + + let (weights, biases) = if let Some(Tensors::Conv(tensors)) = tensors { + (tensors.weights, tensors.biases) + } else { + let weights = if let Some(tensor) = config.kernel { + ArrayD::from_shape_vec(tensor.shape, tensor.data).unwrap() + } else { + GPUInit::from_default(config.init, Init::Kaiming).init( + weight_size.clone(), + size[1] * input_y * input_x, + weight_size[0] * output_y * output_x, + ) + }; + let biases = ArrayD::zeros(vec![config.kernel_size[0]]); + (weights, biases) + }; + + Self { + strides, + padding, + output_size, + inputs: Array4::zeros(input_size), + weights: weights.into_dimensionality::().unwrap(), + biases: biases.into_dimensionality::().unwrap(), + d_weights: ArrayD::zeros(weight_size) + .into_dimensionality::() + .unwrap(), + d_biases: Array1::zeros(config.kernel_size[0]), + } + } + + pub fn output_size(&self) -> Vec { + self.output_size.as_array_view().to_vec() + } + + pub fn reset(&mut self, batches: usize) { + let input_size = self.inputs.shape(); + self.inputs = Array4::zeros((batches, input_size[1], input_size[2], input_size[3])); + self.output_size[0] = batches; + } + + pub fn forward_propagate(&mut self, inputs: ArrayD) -> ArrayD { + let inputs = inputs.into_dimensionality::().unwrap(); + let (_, _, input_y, input_x) = self.inputs.dim(); + let unpadded_y = self.padding[0]..input_y - self.padding[0]; + let unpadded_x = self.padding[1]..input_x - self.padding[1]; + self.inputs + .slice_mut(s![.., .., unpadded_y, unpadded_x]) + .assign(&inputs); + + let (filters, _, weight_y, weight_x) = self.weights.dim(); + let (batches, _, output_y, output_x) = self.output_size.into_pattern(); + + let mut outputs = Array4::zeros(self.output_size); + for b in 0..batches { + for f in 0..filters { + let mut h = 0; + for y in (0..output_y).step_by(self.strides[0]) { + let mut w = 0; + for x in (0..output_x).step_by(self.strides[1]) { + outputs[(b, f, h, w)] = self + .inputs + .slice(s![b, .., y..y + weight_y, x..x + weight_x]) + .mul(&self.weights.slice(s![f, .., .., ..])) + .sum() + .add(self.biases[f]); + w += 1; + } + h += 1; + } + } + } + + outputs.into_dyn() + } + + pub fn backward_propagate(&mut self, d_outputs: ArrayD) -> ArrayD { + let d_outputs = d_outputs.into_dimensionality::().unwrap(); + + let (filters, _, weight_y, weight_x) = self.weights.dim(); + let (batches, _, output_y, output_x) = self.output_size.into_pattern(); + + let mut d_inputs = Array4::zeros(self.inputs.dim()); + self.d_weights = Array4::zeros(self.weights.dim()); + self.d_biases = Array1::::zeros(self.biases.dim()); + for b in 0..batches { + for f in 0..filters { + for y in (0..output_y).step_by(self.strides[0]) { + for x in (0..output_x).step_by(self.strides[1]) { + d_inputs + .slice_mut(s![b, .., y..y + weight_y, x..x + weight_x]) + .add_assign( + &self + .weights + .slice(s![f, .., .., ..]) + .mul(d_outputs[(b, f, y, x)]), + ); + self.d_weights.slice_mut(s![f, .., .., ..]).add_assign( + &self + .inputs + .slice(s![b, .., y..y + weight_y, x..x + weight_x]) + .mul(d_outputs[(b, f, y, x)]), + ); + self.d_biases[f] += d_outputs[(b, f, y, x)]; + } + } + } + } + + d_inputs.into_dyn() + } +} diff --git a/crates/core-gpu/src/gpu/layers/convtrans2d.rs b/crates/core-gpu/src/gpu/layers/convtrans2d.rs new file mode 100644 index 0000000..b65a6cf --- /dev/null +++ b/crates/core-gpu/src/gpu/layers/convtrans2d.rs @@ -0,0 +1,146 @@ +use ndarray::{s, Array1, Array4, ArrayD, Dimension, Ix1, Ix4, IxDyn}; +use std::ops::{Add, AddAssign, Mul}; + +use crate::{GPUInit, ConvTranspose2DLayer, Init, Tensors}; + +pub struct ConvTranspose2DGPULayer { + // cache + pub strides: Vec, + pub padding: Vec, + pub inputs: Array4, + pub output_size: Ix4, + + // parameters + pub weights: Array4, + pub biases: Array1, + + // gradients + pub d_weights: Array4, + pub d_biases: Array1, +} + +impl ConvTranspose2DGPULayer { + pub fn new(config: ConvTranspose2DLayer, size: IxDyn, tensors: Option) -> Self { + let strides = config.strides.unwrap_or(vec![1, 1]); + let padding = config.padding.unwrap_or(vec![0, 0]); + let input_y = size[2] + 2 * padding[0]; + let input_x = size[3] + 2 * padding[1]; + let output_y = (input_y + config.kernel_size[2]) / strides[0] - 1; + let output_x = (input_x + config.kernel_size[3]) / strides[1] - 1; + let input_size = Ix4(size[0], size[1], input_y, input_x); + let weight_size = IxDyn(config.kernel_size.as_slice()); + let output_size = Ix4(size[0], weight_size[0], output_y, output_x); + + let (weights, biases) = if let Some(Tensors::Conv(tensors)) = tensors { + (tensors.weights, tensors.biases) + } else { + let weights = if let Some(tensor) = config.kernel { + ArrayD::from_shape_vec(tensor.shape, tensor.data).unwrap() + } else { + GPUInit::from_default(config.init, Init::Xavier).init( + weight_size.clone(), + size[1] * input_y * input_x, + weight_size[0] * output_y * output_x, + ) + }; + let biases = ArrayD::zeros(vec![config.kernel_size[0]]); + (weights, biases) + }; + + Self { + strides, + padding, + output_size, + inputs: Array4::zeros(input_size), + weights: weights.into_dimensionality::().unwrap(), + biases: biases.into_dimensionality::().unwrap(), + d_weights: ArrayD::zeros(weight_size) + .into_dimensionality::() + .unwrap(), + d_biases: Array1::zeros(config.kernel_size[0]), + } + } + + pub fn output_size(&self) -> Vec { + self.output_size.as_array_view().to_vec() + } + + pub fn reset(&mut self, batches: usize) { + let input_size = self.inputs.shape(); + self.inputs = Array4::zeros((batches, input_size[1], input_size[2], input_size[3])); + self.output_size[0] = batches; + } + + pub fn forward_propagate(&mut self, inputs: ArrayD) -> ArrayD { + let inputs = inputs.into_dimensionality::().unwrap(); + let (batches, _, input_y, input_x) = self.inputs.dim(); + let unpadded_y = self.padding[0]..input_y - self.padding[0]; + let unpadded_x = self.padding[1]..input_x - self.padding[1]; + self.inputs + .slice_mut(s![.., .., unpadded_y, unpadded_x]) + .assign(&inputs); + + let (filters, _, weight_y, weight_x) = self.weights.dim(); + + let mut outputs = Array4::zeros(self.output_size); + for b in 0..batches { + for f in 0..filters { + let mut h = 0; + for y in (0..input_y).step_by(self.strides[0]) { + let mut w = 0; + for x in (0..input_x).step_by(self.strides[1]) { + outputs + .slice_mut(s![b, .., y..y + weight_y, x..x + weight_x]) + .add_assign( + &self.inputs[(b, f, h, w)] + .mul(&self.weights.slice(s![f, .., .., ..])) + .add(self.biases[f]), + ); + w += 1; + } + h += 1; + } + } + } + + outputs.into_dyn() + } + + pub fn backward_propagate(&mut self, d_outputs: ArrayD) -> ArrayD { + let d_outputs = d_outputs.into_dimensionality::().unwrap(); + + let (batches, _, input_y, input_x) = self.inputs.dim(); + let (filters, _, weight_y, weight_x) = self.weights.dim(); + let unpadded_y = input_y - self.padding[0]; + let unpadded_x = input_x - self.padding[1]; + + let mut d_inputs = Array4::zeros(self.inputs.dim()); + self.d_weights = Array4::zeros(self.weights.dim()); + self.d_biases = Array1::::zeros(self.biases.dim()); + for b in 0..batches { + for f in 0..filters { + for y in (self.padding[0]..unpadded_y).step_by(self.strides[0]) { + for x in (self.padding[1]..unpadded_x).step_by(self.strides[1]) { + d_inputs.slice_mut(s![b, .., y, x]).add_assign( + &self + .weights + .slice(s![f, .., .., ..]) + .mul(&d_outputs.slice(s![b, f, y..y + weight_y, x..x + weight_x])), + ); + self.d_weights.slice_mut(s![f, .., .., ..]).add_assign( + &self.inputs.slice(s![b, .., y, x]).mul(&d_outputs.slice(s![ + b, + f, + y..y + weight_y, + x..x + weight_x + ])), + ); + self.d_biases[f] += d_outputs[(b, f, y, x)]; + } + } + } + } + + d_inputs.into_dyn() + } +} diff --git a/crates/core-gpu/src/gpu/layers/dense.rs b/crates/core-gpu/src/gpu/layers/dense.rs new file mode 100644 index 0000000..7516f36 --- /dev/null +++ b/crates/core-gpu/src/gpu/layers/dense.rs @@ -0,0 +1,71 @@ +use ndarray::{Array1, Array2, ArrayD, Axis, Dimension, Ix1, Ix2, IxDyn}; +use std::ops::Add; + +use crate::{GPUInit, DenseLayer, Init, Tensors}; + +pub struct DenseGPULayer { + // cache + pub output_size: Ix2, + pub inputs: Array2, + + // parameters + pub weights: Array2, + pub biases: Array1, + + // gradients + pub d_weights: Array2, + pub d_biases: Array1, +} + +impl DenseGPULayer { + pub fn new(config: DenseLayer, size: IxDyn, tensors: Option) -> Self { + let init = GPUInit::from_default(config.init, Init::Uniform); + let input_size = Ix2(size[0], size[1]); + let weight_size = Ix2(size[1], config.size[0]); + let output_size = Ix2(size[0], config.size[0]); + + let (weights, biases) = if let Some(Tensors::Dense(tensors)) = tensors { + (tensors.weights, tensors.biases) + } else { + let weights = init.init(weight_size.into_dyn(), size[1], config.size[0]); + let biases = ArrayD::zeros(config.size.clone()); + (weights, biases) + }; + + Self { + output_size, + inputs: Array2::zeros(input_size), + weights: weights.into_dimensionality::().unwrap(), + biases: biases.into_dimensionality::().unwrap(), + d_weights: Array2::zeros(weight_size), + d_biases: Array1::zeros(config.size[0]), + } + } + + pub fn output_size(&self) -> Vec { + self.output_size.as_array_view().to_vec() + } + + pub fn reset(&mut self, batches: usize) { + let input_size = self.inputs.dim().1; + self.inputs = Array2::zeros((batches, input_size)); + self.output_size[0] = batches; + } + + pub fn forward_propagate(&mut self, inputs: ArrayD) -> ArrayD { + self.inputs = inputs.into_dimensionality::().unwrap(); + self.inputs.dot(&self.weights).add(&self.biases).into_dyn() + } + + pub fn backward_propagate(&mut self, d_outputs: ArrayD) -> ArrayD { + let d_outputs = d_outputs.into_dimensionality::().unwrap(); + let mut weights_t = self.weights.view(); + weights_t.swap_axes(0, 1); + let d_inputs = d_outputs.dot(&weights_t); + let mut inputs_t = self.inputs.view(); + inputs_t.swap_axes(0, 1); + self.d_weights = inputs_t.dot(&d_outputs); + self.d_biases = d_outputs.sum_axis(Axis(0)); + d_inputs.into_dyn() + } +} diff --git a/crates/core-gpu/src/gpu/layers/dropout.rs b/crates/core-gpu/src/gpu/layers/dropout.rs new file mode 100644 index 0000000..a0d200e --- /dev/null +++ b/crates/core-gpu/src/gpu/layers/dropout.rs @@ -0,0 +1,87 @@ +use std::ops::Mul; + +use ndarray::{Array2, Array4, ArrayD, Axis, IxDyn}; +use ndarray_rand::{rand_distr::Uniform, RandomExt}; + +use crate::DropoutLayer; + +pub struct Dropout1DGPULayer { + mask: ArrayD, + probability: f32, +} + +impl Dropout1DGPULayer { + pub fn new(config: DropoutLayer, size: IxDyn) -> Self { + Self { + mask: ArrayD::zeros(size), + probability: config.probability, + } + } + + pub fn output_size(&self) -> Vec { + self.mask.shape().to_vec() + } + + pub fn reset(&mut self, batches: usize) { + let mut output_size = self.mask.shape().to_vec(); + output_size[0] = batches; + self.mask = ArrayD::zeros(output_size); + } + + pub fn forward_propagate(&mut self, inputs: ArrayD, training: bool) -> ArrayD { + if training { + self.mask = ArrayD::random(self.mask.dim(), Uniform::new(0.0, 1.0)) + .map(|x| (if x > &self.probability { 1.0 } else { 0.0 })); + inputs.mul(&self.mask).mul(1.0 / 1.0 - self.probability) + } else { + inputs + } + } + + pub fn backward_propagate(&mut self, d_outputs: ArrayD) -> ArrayD { + d_outputs.mul(&self.mask).mul(1.0 / 1.0 - self.probability) + } +} + +pub struct Dropout2DGPULayer { + mask: Array4, + probability: f32, +} + +impl Dropout2DGPULayer { + pub fn new(config: DropoutLayer, size: IxDyn) -> Self { + Self { + mask: Array4::zeros([size[0], size[1], size[2], size[3]]), + probability: config.probability, + } + } + + pub fn output_size(&self) -> Vec { + self.mask.shape().to_vec() + } + + pub fn reset(&mut self, batches: usize) { + let size = self.mask.dim(); + self.mask = Array4::zeros([batches, size.1, size.2, size.3]); + } + + pub fn forward_propagate(&mut self, inputs: ArrayD, training: bool) -> ArrayD { + if training { + let size = self.mask.dim(); + self.mask = Array2::random([size.0, size.1], Uniform::new(0.0, 1.0)) + .map(|x| (if x > &self.probability { 1.0 } else { 0.0 })) + .insert_axis(Axis(2)) + .insert_axis(Axis(3)) + .broadcast(size) + .unwrap() + .to_owned(); + inputs.mul(&self.mask).mul(1.0 / 1.0 - self.probability) + } else { + inputs + } + } + + pub fn backward_propagate(&mut self, d_outputs: ArrayD) -> ArrayD { + d_outputs.mul(&self.mask).mul(1.0 / 1.0 - self.probability) + } +} diff --git a/crates/core-gpu/src/gpu/layers/flatten.rs b/crates/core-gpu/src/gpu/layers/flatten.rs new file mode 100644 index 0000000..ea738cf --- /dev/null +++ b/crates/core-gpu/src/gpu/layers/flatten.rs @@ -0,0 +1,43 @@ +use ndarray::{ArrayD, Dimension, IxDyn}; + +use crate::FlattenLayer; + +pub struct FlattenGPULayer { + pub input_size: IxDyn, + pub output_size: Vec, +} + +impl FlattenGPULayer { + pub fn new(config: FlattenLayer, size: IxDyn) -> Self { + let mut new_size = config.size.clone(); + new_size.insert(0, size[0]); + let output_size = IxDyn(&new_size); + if output_size.size() != size.size() { + panic!( + "Shape {:#?} is incompatible with shape {:#?}", + output_size, size + ) + } + Self { + input_size: size, + output_size: new_size, + } + } + + pub fn output_size(&self) -> Vec { + self.output_size.clone() + } + + pub fn reset(&mut self, batches: usize) { + self.output_size[0] = batches + } + + pub fn forward_propagate(&mut self, inputs: ArrayD) -> ArrayD { + let output_size = IxDyn(&self.output_size); + inputs.into_shape(output_size).unwrap() + } + + pub fn backward_propagate(&mut self, d_outputs: ArrayD) -> ArrayD { + d_outputs.into_shape(self.input_size.clone()).unwrap() + } +} diff --git a/crates/core-gpu/src/gpu/layers/mod.rs b/crates/core-gpu/src/gpu/layers/mod.rs new file mode 100644 index 0000000..5840fc1 --- /dev/null +++ b/crates/core-gpu/src/gpu/layers/mod.rs @@ -0,0 +1,101 @@ +mod activation; +mod batchnorm1d; +mod batchnorm2d; +mod conv2d; +mod convtrans2d; +mod dense; +mod dropout; +mod flatten; +mod pool2d; + +pub use activation::*; +pub use batchnorm1d::*; +pub use batchnorm2d::*; +pub use conv2d::*; +pub use convtrans2d::*; +pub use dense::*; +pub use dropout::*; +pub use flatten::*; +pub use pool2d::*; + +use ndarray::ArrayD; + +pub enum GPULayer { + Activation(ActivationGPULayer), + Conv2D(Conv2DGPULayer), + ConvTranspose2D(ConvTranspose2DGPULayer), + Dense(DenseGPULayer), + Dropout1D(Dropout1DGPULayer), + Dropout2D(Dropout2DGPULayer), + Flatten(FlattenGPULayer), + Pool2D(Pool2DGPULayer), + Softmax(SoftmaxGPULayer), + BatchNorm1D(BatchNorm1DGPULayer), + BatchNorm2D(BatchNorm2DGPULayer), +} + +impl GPULayer { + pub fn output_size(&mut self) -> Vec { + match self { + GPULayer::Activation(layer) => layer.output_size(), + GPULayer::BatchNorm1D(layer) => layer.output_size(), + GPULayer::BatchNorm2D(layer) => layer.output_size(), + GPULayer::Conv2D(layer) => layer.output_size(), + GPULayer::ConvTranspose2D(layer) => layer.output_size(), + GPULayer::Dense(layer) => layer.output_size(), + GPULayer::Dropout1D(layer) => layer.output_size(), + GPULayer::Dropout2D(layer) => layer.output_size(), + GPULayer::Flatten(layer) => layer.output_size(), + GPULayer::Pool2D(layer) => layer.output_size(), + GPULayer::Softmax(layer) => layer.output_size(), + } + } + + pub fn forward_propagate(&mut self, inputs: ArrayD, training: bool) -> ArrayD { + match self { + GPULayer::Activation(layer) => layer.forward_propagate(inputs), + GPULayer::BatchNorm1D(layer) => layer.forward_propagate(inputs, training), + GPULayer::BatchNorm2D(layer) => layer.forward_propagate(inputs, training), + GPULayer::Conv2D(layer) => layer.forward_propagate(inputs), + GPULayer::ConvTranspose2D(layer) => layer.forward_propagate(inputs), + GPULayer::Dense(layer) => layer.forward_propagate(inputs), + GPULayer::Dropout1D(layer) => layer.forward_propagate(inputs, training), + GPULayer::Dropout2D(layer) => layer.forward_propagate(inputs, training), + GPULayer::Flatten(layer) => layer.forward_propagate(inputs), + GPULayer::Pool2D(layer) => layer.forward_propagate(inputs), + GPULayer::Softmax(layer) => layer.forward_propagate(inputs), + } + } + + pub fn backward_propagate(&mut self, d_outputs: ArrayD) -> ArrayD { + match self { + GPULayer::Activation(layer) => layer.backward_propagate(d_outputs), + GPULayer::BatchNorm1D(layer) => layer.backward_propagate(d_outputs), + GPULayer::BatchNorm2D(layer) => layer.backward_propagate(d_outputs), + GPULayer::Conv2D(layer) => layer.backward_propagate(d_outputs), + GPULayer::ConvTranspose2D(layer) => layer.backward_propagate(d_outputs), + GPULayer::Dense(layer) => layer.backward_propagate(d_outputs), + GPULayer::Dropout1D(layer) => layer.backward_propagate(d_outputs), + GPULayer::Dropout2D(layer) => layer.backward_propagate(d_outputs), + GPULayer::Flatten(layer) => layer.backward_propagate(d_outputs), + GPULayer::Pool2D(layer) => layer.backward_propagate(d_outputs), + GPULayer::Softmax(layer) => layer.backward_propagate(d_outputs), + } + } + + pub fn reset(&mut self, batches: usize) { + match self { + GPULayer::Activation(layer) => layer.reset(batches), + GPULayer::BatchNorm1D(layer) => layer.reset(batches), + GPULayer::BatchNorm2D(layer) => layer.reset(batches), + GPULayer::Conv2D(layer) => layer.reset(batches), + GPULayer::Dense(layer) => layer.reset(batches), + GPULayer::Dropout1D(layer) => layer.reset(batches), + GPULayer::Dropout2D(layer) => layer.reset(batches), + GPULayer::Flatten(layer) => layer.reset(batches), + GPULayer::Pool2D(layer) => layer.reset(batches), + GPULayer::Softmax(layer) => layer.reset(batches), + GPULayer::ConvTranspose2D(layer) => layer.reset(batches), + } + } +} diff --git a/crates/core-gpu/src/gpu/layers/pool2d.rs b/crates/core-gpu/src/gpu/layers/pool2d.rs new file mode 100644 index 0000000..be49fc6 --- /dev/null +++ b/crates/core-gpu/src/gpu/layers/pool2d.rs @@ -0,0 +1,126 @@ +use ndarray::{s, Array4, Array5, ArrayD, Ix4, Ix5, IxDyn}; + +use crate::Pool2DLayer; + +pub struct Pool2DGPULayer { + pub strides: Vec, + pub inputs: Array4, + pub indices: Array5, + pub outputs: Array4, + pub max: bool, +} + +impl Pool2DGPULayer { + pub fn new(config: Pool2DLayer, size: IxDyn) -> Self { + let strides = config.strides.unwrap_or(vec![1, 1]); + let input_size = Ix4(size[0], size[1], size[2], size[3]); + let output_y = size[2] / strides[0]; + let output_x = size[3] / strides[1]; + let indice_size = Ix5(size[0], size[1], output_y, output_x, 2); + let output_size = Ix4(size[0], size[1], output_y, output_x); + let max = config.mode == 1; + Self { + strides, + inputs: Array4::zeros(input_size), + indices: Array5::zeros(indice_size), + outputs: Array4::zeros(output_size), + max, + } + } + + pub fn output_size(&self) -> Vec { + self.outputs.shape().to_vec() + } + + pub fn reset(&mut self, batches: usize) { + let input_size = self.inputs.shape(); + self.inputs = Array4::zeros((batches, input_size[1], input_size[2], input_size[3])); + let indice_size = self.outputs.shape(); + self.indices = Array5::zeros((batches, indice_size[1], indice_size[2], indice_size[3], 2)); + let output_size = self.outputs.shape(); + self.outputs = Array4::zeros((batches, output_size[1], output_size[2], output_size[3])); + } + + pub fn forward_propagate(&mut self, inputs: ArrayD) -> ArrayD { + self.inputs = inputs.into_dimensionality::().unwrap(); + + let (batches, channels, output_y, output_x) = self.outputs.dim(); + + for b in 0..batches { + for c in 0..channels { + for y in 0..output_y { + for x in 0..output_x { + let input_y = y * self.strides[0]; + let input_x = x * self.strides[1]; + let stride_y = (y + 1) * self.strides[0]; + let stride_x = (x + 1) * self.strides[1]; + if self.max { + let mut max_index = (0, 0); + let mut max_value = 0.0; + self.inputs + .slice(s![b, c, input_y..stride_y, input_x..stride_x]) + .indexed_iter() + .for_each(|(index, value)| { + if value > &max_value { + max_value = *value; + max_index = index; + } + }); + let mut position = self.indices.slice_mut(s![b, c, y, x, ..]); + position[0] = max_index.0.into(); + position[1] = max_index.1.into(); + self.outputs[[b, c, y, x]] = max_value; + } else { + self.outputs[[b, c, y, x]] = self + .inputs + .slice(s![b, c, input_y..stride_y, input_x..stride_x]) + .mean() + .unwrap(); + } + } + } + } + } + + self.outputs.clone().into_dyn() + } + + pub fn backward_propagate(&mut self, d_outputs: ArrayD) -> ArrayD { + let d_outputs = d_outputs.into_dimensionality::().unwrap(); + + let (batches, channels, output_y, output_x) = self.outputs.dim(); + + let mut d_inputs = Array4::zeros(self.inputs.dim()); + for b in 0..batches { + for c in 0..channels { + for y in 0..output_y { + for x in 0..output_x { + let input_y = y * self.strides[0]; + let input_x = x * self.strides[1]; + let stride_y = (y + 1) * self.strides[0]; + let stride_x = (x + 1) * self.strides[1]; + if self.max { + let index = self.indices.slice(s![b, c, y, x, ..]); + d_inputs[[ + b, + c, + input_y + index[0] as usize, + input_x + index[1] as usize, + ]] = d_outputs[[b, c, y, x]]; + } else { + d_inputs + .slice_mut(s![b, c, input_y..stride_y, input_x..stride_x]) + .fill( + d_outputs[[b, c, y, x]] + / self.strides[0] as f32 + / self.strides[1] as f32, + ); + } + } + } + } + } + + d_inputs.into_dyn() + } +} diff --git a/crates/core-gpu/src/gpu/mod.rs b/crates/core-gpu/src/gpu/mod.rs new file mode 100644 index 0000000..e12191b --- /dev/null +++ b/crates/core-gpu/src/gpu/mod.rs @@ -0,0 +1,15 @@ +mod activation; +mod backend; +mod cost; +mod init; +mod layers; +mod optimizers; +mod schedulers; + +pub use activation::*; +pub use backend::*; +pub use cost::*; +pub use init::*; +pub use layers::*; +pub use optimizers::*; +pub use schedulers::*; diff --git a/crates/core-gpu/src/gpu/optimizers/adam.rs b/crates/core-gpu/src/gpu/optimizers/adam.rs new file mode 100644 index 0000000..f77365f --- /dev/null +++ b/crates/core-gpu/src/gpu/optimizers/adam.rs @@ -0,0 +1,74 @@ +use std::ops::{Add, Div, Mul, SubAssign}; + +use ndarray::{ArrayD, ArrayViewD, ArrayViewMutD}; + +use crate::{AdamOptimizer, GPUScheduler}; + +pub struct GPUAdamOptimizer { + pub beta1: f32, + pub beta2: f32, + pub epsilon: f32, + pub m: Vec>>, + pub v: Vec>>, + pub t: f32, +} + +impl GPUAdamOptimizer { + pub fn new(config: AdamOptimizer, params: Vec>>) -> Self { + let mut m = Vec::new(); + let mut v = Vec::new(); + for params in params { + m.push( + params + .iter() + .map(|param| ArrayD::zeros(param.dim())) + .collect(), + ); + v.push( + params + .iter() + .map(|param| ArrayD::zeros(param.dim())) + .collect(), + ); + } + Self { + beta1: config.beta1, + beta2: config.beta2, + epsilon: config.epsilon, + m, + v, + t: 0.0, + } + } + + pub fn update_grads( + &mut self, + mut params: Vec>, + grads: Vec>, + idx: usize, + scheduler: &GPUScheduler, + rate: f32, + ) { + for (j, (param, grad)) in params.iter_mut().zip(grads).enumerate() { + self.m[idx][j] = self + .beta1 + .mul(&self.m[idx][j]) + .add((1.0 - self.beta1).mul(&grad)); + self.v[idx][j] = self + .beta2 + .mul(&self.v[idx][j]) + .add((1.0 - self.beta2).mul(&grad.map(|x| x.powi(2)))); + + let m_hat = self.m[idx][j].view().div(1.0 - self.beta1.powf(self.t)); + let v_hat = self.v[idx][j].view().div(1.0 - self.beta2.powf(self.t)); + + let rate = scheduler.eta(rate, self.t as usize); + + param.sub_assign( + &rate + .mul(m_hat) + .div(v_hat.map(|x| x.sqrt()).add(self.epsilon)), + ) + } + } +} diff --git a/crates/core-gpu/src/gpu/optimizers/mod.rs b/crates/core-gpu/src/gpu/optimizers/mod.rs new file mode 100644 index 0000000..22f582b --- /dev/null +++ b/crates/core-gpu/src/gpu/optimizers/mod.rs @@ -0,0 +1,115 @@ +mod adam; +mod sgd; + +pub use adam::*; +use ndarray::{ArrayViewD, ArrayViewMutD}; +pub use sgd::*; + +use crate::{GPULayer, GPUScheduler, Optimizer}; + +pub enum GPUOptimizer { + SGD(GPUSGDOptimizer), + Adam(GPUAdamOptimizer), +} + +impl GPUOptimizer { + pub fn from(optimizer: Optimizer, layers: &mut Vec) -> Self { + let mut all_params = Vec::new(); + for layer in layers { + if let Some((params, _)) = GPUOptimizer::get_params(layer) { + all_params.push(params) + } + } + match optimizer { + Optimizer::SGD => GPUOptimizer::SGD(GPUSGDOptimizer::new()), + Optimizer::Adam(config) => { + GPUOptimizer::Adam(GPUAdamOptimizer::new(config, all_params)) + } + } + } + + pub fn update_grads( + &mut self, + layers: &mut Vec, + scheduler: &GPUScheduler, + rate: f32, + epoch: usize, + ) { + match self { + GPUOptimizer::Adam(adam) => adam.t += 1.0, + _ => {} + } + let mut idx = 0; + for layer in layers.iter_mut() { + if let Some((params, grads)) = GPUOptimizer::get_params(layer) { + match self { + GPUOptimizer::SGD(sgd) => { + sgd.update_grads(params, grads, scheduler, rate, epoch) + } + GPUOptimizer::Adam(adam) => { + adam.update_grads(params, grads, idx, scheduler, rate) + } + } + idx += 1; + } + } + } + + pub fn get_params<'a>( + layer: &'a mut GPULayer, + ) -> Option<(Vec>, Vec>)> { + match layer { + GPULayer::Dense(layer) => Some(( + vec![ + layer.weights.view_mut().into_dyn(), + layer.biases.view_mut().into_dyn(), + ], + vec![ + layer.d_weights.view().into_dyn(), + layer.d_biases.view().into_dyn(), + ], + )), + GPULayer::Conv2D(layer) => Some(( + vec![ + layer.weights.view_mut().into_dyn(), + layer.biases.view_mut().into_dyn(), + ], + vec![ + layer.d_weights.view().into_dyn(), + layer.d_biases.view().into_dyn(), + ], + )), + GPULayer::ConvTranspose2D(layer) => Some(( + vec![ + layer.weights.view_mut().into_dyn(), + layer.biases.view_mut().into_dyn(), + ], + vec![ + layer.d_weights.view().into_dyn(), + layer.d_biases.view().into_dyn(), + ], + )), + GPULayer::BatchNorm1D(layer) => Some(( + vec![ + layer.gamma.view_mut().into_dyn(), + layer.beta.view_mut().into_dyn(), + ], + vec![ + layer.d_gamma.view().into_dyn(), + layer.d_beta.view().into_dyn(), + ], + )), + GPULayer::BatchNorm2D(layer) => Some(( + vec![ + layer.gamma.view_mut().into_dyn(), + layer.beta.view_mut().into_dyn(), + ], + vec![ + layer.d_gamma.view().into_dyn(), + layer.d_beta.view().into_dyn(), + ], + )), + _ => return None, + } + } +} diff --git a/crates/core-gpu/src/gpu/optimizers/sgd.rs b/crates/core-gpu/src/gpu/optimizers/sgd.rs new file mode 100644 index 0000000..995679c --- /dev/null +++ b/crates/core-gpu/src/gpu/optimizers/sgd.rs @@ -0,0 +1,27 @@ +use std::ops::{Mul, SubAssign}; + +use ndarray::{ArrayViewD, ArrayViewMutD}; + +use crate::GPUScheduler; + +pub struct GPUSGDOptimizer {} + +impl GPUSGDOptimizer { + pub fn new() -> Self { + Self {} + } + + pub fn update_grads( + &mut self, + mut params: Vec>, + grads: Vec>, + scheduler: &GPUScheduler, + rate: f32, + epoch: usize, + ) { + let eta = scheduler.eta(rate, epoch); + for (param, grad) in params.iter_mut().zip(grads) { + param.sub_assign(&grad.mul(eta)); + } + } +} diff --git a/crates/core-gpu/src/gpu/schedulers/decay.rs b/crates/core-gpu/src/gpu/schedulers/decay.rs new file mode 100644 index 0000000..b6b3b70 --- /dev/null +++ b/crates/core-gpu/src/gpu/schedulers/decay.rs @@ -0,0 +1,21 @@ +use crate::DecayScheduler; + +pub struct GPUDecayScheduler { + pub rate: f32, + pub step_size: usize, +} + +impl GPUDecayScheduler { + pub fn new(config: &DecayScheduler) -> Self { + GPUDecayScheduler { + rate: config.rate, + step_size: config.step_size, + } + } + pub fn exponential(&self, rate: f32, step: usize) -> f32 { + rate * self.rate.powi((step / self.step_size) as i32) + } + pub fn linear(&self, rate: f32, step: usize) -> f32 { + rate / (self.rate * (1 + step / self.step_size) as f32) + } +} diff --git a/crates/core-gpu/src/gpu/schedulers/mod.rs b/crates/core-gpu/src/gpu/schedulers/mod.rs new file mode 100644 index 0000000..5fe263a --- /dev/null +++ b/crates/core-gpu/src/gpu/schedulers/mod.rs @@ -0,0 +1,38 @@ +mod decay; +mod oc; + +use crate::Scheduler; + +pub use decay::*; +pub use oc::*; +pub enum GPUScheduler { + None, + LinearDecay(GPUDecayScheduler), + ExponentialDecay(GPUDecayScheduler), + OneCycle(GPUOneCycleScheduler), +} + +impl GPUScheduler { + pub fn from(scheduler: &Scheduler) -> Self { + match scheduler { + Scheduler::None => GPUScheduler::None, + Scheduler::LinearDecay(config) => { + GPUScheduler::LinearDecay(GPUDecayScheduler::new(config)) + } + Scheduler::ExponentialDecay(config) => { + GPUScheduler::ExponentialDecay(GPUDecayScheduler::new(config)) + } + Scheduler::OneCycle(config) => { + GPUScheduler::OneCycle(GPUOneCycleScheduler::new(config)) + } + } + } + pub fn eta(&self, rate: f32, step: usize) -> f32 { + match self { + GPUScheduler::None => rate, + GPUScheduler::LinearDecay(scheduler) => scheduler.linear(rate, step), + GPUScheduler::ExponentialDecay(scheduler) => scheduler.exponential(rate, step), + GPUScheduler::OneCycle(scheduler) => scheduler.eta(rate, step), + } + } +} diff --git a/crates/core-gpu/src/gpu/schedulers/oc.rs b/crates/core-gpu/src/gpu/schedulers/oc.rs new file mode 100644 index 0000000..573a6e7 --- /dev/null +++ b/crates/core-gpu/src/gpu/schedulers/oc.rs @@ -0,0 +1,24 @@ +use crate::OneCycleScheduler; + +pub struct GPUOneCycleScheduler { + pub max_rate: f32, + pub step_size: usize, +} + +impl GPUOneCycleScheduler { + pub fn new(config: &OneCycleScheduler) -> Self { + GPUOneCycleScheduler { + max_rate: config.max_rate, + step_size: config.step_size, + } + } + pub fn eta(&self, rate: f32, step: usize) -> f32 { + let steps = self.step_size as f32; + let step = step % (2 * self.step_size); + if step < self.step_size { + rate + (self.max_rate - rate) * (step as f32) / (steps) + } else { + self.max_rate - (self.max_rate - rate) * ((step - self.step_size) as f32) / (steps) + } + } +} diff --git a/crates/core-gpu/src/lib.rs b/crates/core-gpu/src/lib.rs new file mode 100644 index 0000000..1864a64 --- /dev/null +++ b/crates/core-gpu/src/lib.rs @@ -0,0 +1,30 @@ +mod gpu; +mod ffi; +mod tensor; +mod types; +mod util; + +pub use gpu::*; + +pub use ffi::*; +pub use tensor::*; +pub use types::*; +pub use util::*; + +use std::cell::RefCell; + +pub struct Resources { + pub backend: RefCell>, +} + +impl Resources { + pub fn new() -> Self { + Self { + backend: RefCell::new(Vec::new()), + } + } +} + +thread_local! { + pub static RESOURCES: Resources = Resources::new(); +} diff --git a/crates/core-gpu/src/tensor.rs b/crates/core-gpu/src/tensor.rs new file mode 100644 index 0000000..f2a0d2c --- /dev/null +++ b/crates/core-gpu/src/tensor.rs @@ -0,0 +1,75 @@ +use std::{borrow::Cow, slice::from_raw_parts}; + +use ndarray::{ArrayD, ArrayViewD}; +use safetensors::{Dtype, View}; + +pub struct Tensor<'a> { + pub data: ArrayViewD<'a, f32>, +} + +impl<'a> Tensor<'a> { + pub fn new(data: ArrayViewD<'a, f32>) -> Self { + Self { data } + } +} + +impl<'a> View for Tensor<'a> { + fn dtype(&self) -> Dtype { + Dtype::F32 + } + + fn shape(&self) -> &[usize] { + self.data.shape() + } + + fn data(&self) -> Cow<[u8]> { + let slice = self.data.as_slice().expect("Non contiguous tensors"); + let new_slice: &[u8] = + unsafe { from_raw_parts(slice.as_ptr() as *const u8, slice.len() * 4) }; + Cow::from(new_slice) + } + + fn data_len(&self) -> usize { + self.data.len() * 4 + } +} + +#[derive(Debug)] +pub struct DenseTensors { + pub weights: ArrayD, + pub biases: ArrayD, +} + +#[derive(Debug)] +pub struct ConvTensors { + pub weights: ArrayD, + pub biases: ArrayD, +} + +#[derive(Debug)] +pub struct BatchNormTensors { + pub gamma: ArrayD, + pub beta: ArrayD, + pub running_mean: ArrayD, + pub running_var: ArrayD, +} + +#[derive(Debug)] +pub enum Tensors { + Dense(DenseTensors), + Conv(ConvTensors), + BatchNorm(BatchNormTensors), +} + +pub trait GetTensor { + fn get(&mut self) -> Option; +} + +impl GetTensor for Option> { + fn get(&mut self) -> Option { + if let Some(tensors) = self { + return Some(tensors.remove(0)); + } + None + } +} diff --git a/crates/core-gpu/src/types.rs b/crates/core-gpu/src/types.rs new file mode 100644 index 0000000..f3d6ec4 --- /dev/null +++ b/crates/core-gpu/src/types.rs @@ -0,0 +1,195 @@ +use ndarray::ArrayD; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct BackendConfig { + pub silent: Option, + pub size: Vec, + pub layers: Vec, + pub cost: Cost, + pub optimizer: Optimizer, + pub scheduler: Scheduler, +} + +#[derive(Debug)] +pub struct Dataset { + pub inputs: ArrayD, + pub outputs: ArrayD, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(tag = "type", content = "config")] +#[serde(rename_all = "lowercase")] +pub enum Layer { + Activation(ActivationLayer), + Dense(DenseLayer), + BatchNorm1D(BatchNormLayer), + BatchNorm2D(BatchNormLayer), + Conv2D(Conv2DLayer), + ConvTranspose2D(ConvTranspose2DLayer), + Pool2D(Pool2DLayer), + Flatten(FlattenLayer), + Dropout1D(DropoutLayer), + Dropout2D(DropoutLayer), + Softmax, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "lowercase")] +pub enum Activation { + Elu, + LeakyRelu, + Linear, + Relu, + Relu6, + Selu, + Sigmoid, + Tanh, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct JSTensor { + pub data: Vec, + pub shape: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct DenseLayer { + pub size: Vec, + pub init: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Conv2DLayer { + pub init: Option, + pub kernel: Option, + pub kernel_size: Vec, + pub padding: Option>, + pub strides: Option>, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct ConvTranspose2DLayer { + pub init: Option, + pub kernel: Option, + pub kernel_size: Vec, + pub padding: Option>, + pub strides: Option>, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct Pool2DLayer { + pub mode: usize, // 0 = avg, 1 = max + pub strides: Option>, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct FlattenLayer { + pub size: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct DropoutLayer { + pub probability: f32, + pub inplace: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct BatchNormLayer { + pub momentum: f32, + pub epsilon: f32, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ActivationLayer { + pub activation: Activation, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "lowercase")] +pub enum Cost { + CrossEntropy, + Hinge, + MSE, + BinCrossEntropy, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "lowercase")] +pub enum Init { + Uniform, + Xavier, + XavierN, + Kaiming, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "lowercase")] +pub struct AdamOptimizer { + pub beta1: f32, + pub beta2: f32, + pub epsilon: f32, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(tag = "type", content = "config")] +#[serde(rename_all = "lowercase")] +pub enum Optimizer { + SGD, + Adam(AdamOptimizer), +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "lowercase")] +pub struct DecayScheduler { + pub rate: f32, + pub step_size: usize, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "lowercase")] +pub struct OneCycleScheduler { + pub max_rate: f32, + pub step_size: usize, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(tag = "type", content = "config")] +#[serde(rename_all = "lowercase")] +pub enum Scheduler { + None, + LinearDecay(DecayScheduler), + ExponentialDecay(DecayScheduler), + OneCycle(OneCycleScheduler), +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct TrainOptions { + pub datasets: usize, + pub input_shape: Vec, + pub output_shape: Vec, + pub epochs: usize, + pub batches: usize, + pub rate: f32, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct PredictOptions { + pub input_shape: Vec, + pub output_shape: Vec, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub enum DType { + U8, + U32, + I64, + BF16, + F16, + F32, + F64, +} diff --git a/crates/core-gpu/src/util.rs b/crates/core-gpu/src/util.rs new file mode 100644 index 0000000..edee1a0 --- /dev/null +++ b/crates/core-gpu/src/util.rs @@ -0,0 +1,34 @@ +use std::slice::from_raw_parts; + +use ndarray::ArrayD; +use safetensors::tensor::TensorView; +use serde::Deserialize; + +pub struct Logger { + pub log: fn(string: String) -> (), +} + +pub fn length(shape: Vec) -> usize { + return shape.iter().fold(1, |i, x| i * x); +} + +pub fn decode_array(ptr: *const f32, shape: Vec) -> ArrayD { + let buffer = unsafe { from_raw_parts(ptr, length(shape.clone())) }; + let vec = Vec::from(buffer); + return ArrayD::from_shape_vec(shape, vec).unwrap(); +} + +pub fn decode_json<'a, T>(ptr: *const u8, len: usize) -> T +where + T: Deserialize<'a>, +{ + let buffer = unsafe { from_raw_parts(ptr, len) }; + let json = std::str::from_utf8(&buffer[0..len]).unwrap(); + return serde_json::from_str(&json).unwrap(); +} + +pub fn to_arr(view: TensorView) -> ArrayD { + let slice: &[f32] = + unsafe { from_raw_parts(view.data().as_ptr() as *const f32, view.data().len() / 4) }; + return ArrayD::from_shape_vec(view.shape(), slice.to_vec()).unwrap(); +} diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index ca2879c..7968a5a 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -1,7 +1,7 @@ [package] edition = "2021" name = "netsaur" -version = "0.2.7" +version = "0.2.9" [lib] crate-type = ["cdylib"] @@ -11,7 +11,7 @@ ndarray = "0.15.6" ndarray-rand = "0.14.0" serde = {version = "1.0", features = ["derive"]} serde_json = "1.0" -safetensors = "0.3.1" +safetensors = "0.4.0" [target.'cfg(target_arch = "wasm32")'.dependencies] wasm-bindgen = "=0.2.84" diff --git a/crates/core/src/cpu/backend.rs b/crates/core/src/cpu/backend.rs index c559e40..cb36f7e 100644 --- a/crates/core/src/cpu/backend.rs +++ b/crates/core/src/cpu/backend.rs @@ -5,23 +5,24 @@ use safetensors::{serialize, SafeTensors}; use crate::{ to_arr, ActivationCPULayer, BackendConfig, BatchNorm1DCPULayer, BatchNorm2DCPULayer, - BatchNormTensors, CPUCost, CPULayer, CPUOptimizer, Conv2DCPULayer, ConvTensors, + BatchNormTensors, CPUCost, CPULayer, CPUOptimizer, CPUScheduler, Conv2DCPULayer, ConvTensors, ConvTranspose2DCPULayer, Dataset, DenseCPULayer, DenseTensors, Dropout1DCPULayer, Dropout2DCPULayer, FlattenCPULayer, GetTensor, Layer, Logger, Pool2DCPULayer, SoftmaxCPULayer, Tensor, Tensors, }; -pub struct CPUBackend { +pub struct Backend { pub silent: bool, pub config: BackendConfig, pub layers: Vec, pub size: Vec, pub cost: CPUCost, pub optimizer: CPUOptimizer, + pub scheduler: CPUScheduler, pub logger: Logger, } -impl CPUBackend { +impl Backend { pub fn new(config: BackendConfig, logger: Logger, mut tensors: Option>) -> Self { let mut layers = Vec::new(); let mut size = config.size.clone(); @@ -79,8 +80,9 @@ impl CPUBackend { } } let optimizer = CPUOptimizer::from(config.optimizer.clone(), &mut layers); + let scheduler = CPUScheduler::from(&config.scheduler); let cost = CPUCost::from(config.cost.clone()); - let silent = config.silent.is_some(); + let silent = config.silent.is_some_and(|x| x == true); Self { logger, silent, @@ -88,6 +90,7 @@ impl CPUBackend { layers, cost, optimizer, + scheduler, size, } } @@ -118,19 +121,17 @@ impl CPUBackend { for (i, dataset) in datasets.iter().enumerate() { let outputs = self.forward_propagate(dataset.inputs.clone(), true); self.backward_propagate(outputs.view(), dataset.outputs.view()); - self.optimizer.update_grads(&mut self.layers, rate); + self.optimizer + .update_grads(&mut self.layers, &self.scheduler, rate, epoch); total += (self.cost.cost)(outputs.view(), dataset.outputs.view()); let minibatch = outputs.dim()[0]; - if !self.silent && (i * minibatch) % batches == 0 { + if !self.silent && ((i + 1) * minibatch) % batches == 0 { let cost = total / (batches) as f32; let msg = format!("Epoch={}, Dataset={}, Cost={}", epoch, i * minibatch, cost); - if i != 0 { - (self.logger.log)(msg); - } + (self.logger.log)(msg); total = 0.0; } } - epoch += 1 } } @@ -224,6 +225,6 @@ impl CPUBackend { }; } - CPUBackend::new(config, logger, Some(layers)) + Backend::new(config, logger, Some(layers)) } } diff --git a/crates/core/src/cpu/layers/conv2d.rs b/crates/core/src/cpu/layers/conv2d.rs index 60a6cc7..9fd8cf8 100644 --- a/crates/core/src/cpu/layers/conv2d.rs +++ b/crates/core/src/cpu/layers/conv2d.rs @@ -1,4 +1,4 @@ -use ndarray::{s, Array1, Array4, ArrayD, Ix1, Ix4, IxDyn, Dimension}; +use ndarray::{s, Array1, Array4, ArrayD, Dimension, Ix1, Ix4, IxDyn}; use std::ops::{Add, AddAssign, Mul}; use crate::{CPUInit, Conv2DLayer, Init, Tensors}; diff --git a/crates/core/src/cpu/mod.rs b/crates/core/src/cpu/mod.rs index 098ba1d..e12191b 100644 --- a/crates/core/src/cpu/mod.rs +++ b/crates/core/src/cpu/mod.rs @@ -4,10 +4,12 @@ mod cost; mod init; mod layers; mod optimizers; +mod schedulers; pub use activation::*; pub use backend::*; pub use cost::*; pub use init::*; pub use layers::*; -pub use optimizers::*; \ No newline at end of file +pub use optimizers::*; +pub use schedulers::*; diff --git a/crates/core/src/cpu/optimizers/adam.rs b/crates/core/src/cpu/optimizers/adam.rs index afb41fd..26958b0 100644 --- a/crates/core/src/cpu/optimizers/adam.rs +++ b/crates/core/src/cpu/optimizers/adam.rs @@ -2,7 +2,7 @@ use std::ops::{Add, Div, Mul, SubAssign}; use ndarray::{ArrayD, ArrayViewD, ArrayViewMutD}; -use crate::AdamOptimizer; +use crate::{AdamOptimizer, CPUScheduler}; pub struct CPUAdamOptimizer { pub beta1: f32, @@ -46,6 +46,7 @@ impl CPUAdamOptimizer { mut params: Vec>, grads: Vec>, idx: usize, + scheduler: &CPUScheduler, rate: f32, ) { for (j, (param, grad)) in params.iter_mut().zip(grads).enumerate() { @@ -61,6 +62,8 @@ impl CPUAdamOptimizer { let m_hat = self.m[idx][j].view().div(1.0 - self.beta1.powf(self.t)); let v_hat = self.v[idx][j].view().div(1.0 - self.beta2.powf(self.t)); + let rate = scheduler.eta(rate, self.t as usize); + param.sub_assign( &rate .mul(m_hat) diff --git a/crates/core/src/cpu/optimizers/mod.rs b/crates/core/src/cpu/optimizers/mod.rs index ea564ba..de0dbe4 100644 --- a/crates/core/src/cpu/optimizers/mod.rs +++ b/crates/core/src/cpu/optimizers/mod.rs @@ -5,7 +5,7 @@ pub use adam::*; use ndarray::{ArrayViewD, ArrayViewMutD}; pub use sgd::*; -use crate::{CPULayer, Optimizer}; +use crate::{CPULayer, CPUScheduler, Optimizer}; pub enum CPUOptimizer { SGD(CPUSGDOptimizer), @@ -28,7 +28,13 @@ impl CPUOptimizer { } } - pub fn update_grads(&mut self, layers: &mut Vec, rate: f32) { + pub fn update_grads( + &mut self, + layers: &mut Vec, + scheduler: &CPUScheduler, + rate: f32, + epoch: usize, + ) { match self { CPUOptimizer::Adam(adam) => adam.t += 1.0, _ => {} @@ -37,8 +43,12 @@ impl CPUOptimizer { for layer in layers.iter_mut() { if let Some((params, grads)) = CPUOptimizer::get_params(layer) { match self { - CPUOptimizer::SGD(sgd) => sgd.update_grads(params, grads, rate), - CPUOptimizer::Adam(adam) => adam.update_grads(params, grads, idx, rate), + CPUOptimizer::SGD(sgd) => { + sgd.update_grads(params, grads, scheduler, rate, epoch) + } + CPUOptimizer::Adam(adam) => { + adam.update_grads(params, grads, idx, scheduler, rate) + } } idx += 1; } diff --git a/crates/core/src/cpu/optimizers/sgd.rs b/crates/core/src/cpu/optimizers/sgd.rs index a1a1a1f..8903be9 100644 --- a/crates/core/src/cpu/optimizers/sgd.rs +++ b/crates/core/src/cpu/optimizers/sgd.rs @@ -1,7 +1,9 @@ -use std::ops::{SubAssign, Mul}; +use std::ops::{Mul, SubAssign}; use ndarray::{ArrayViewD, ArrayViewMutD}; +use crate::CPUScheduler; + pub struct CPUSGDOptimizer {} impl CPUSGDOptimizer { @@ -13,10 +15,13 @@ impl CPUSGDOptimizer { &mut self, mut params: Vec>, grads: Vec>, + scheduler: &CPUScheduler, rate: f32, + epoch: usize, ) { + let eta = scheduler.eta(rate, epoch); for (param, grad) in params.iter_mut().zip(grads) { - param.sub_assign(&grad.mul(rate)) + param.sub_assign(&grad.mul(eta)); } } } diff --git a/crates/core/src/cpu/schedulers/decay.rs b/crates/core/src/cpu/schedulers/decay.rs new file mode 100644 index 0000000..fb969cc --- /dev/null +++ b/crates/core/src/cpu/schedulers/decay.rs @@ -0,0 +1,21 @@ +use crate::DecayScheduler; + +pub struct CPUDecayScheduler { + pub rate: f32, + pub step_size: usize, +} + +impl CPUDecayScheduler { + pub fn new(config: &DecayScheduler) -> Self { + CPUDecayScheduler { + rate: config.rate, + step_size: config.step_size, + } + } + pub fn exponential(&self, rate: f32, step: usize) -> f32 { + rate * self.rate.powi((step / self.step_size) as i32) + } + pub fn linear(&self, rate: f32, step: usize) -> f32 { + rate / (self.rate * (1 + step / self.step_size) as f32) + } +} diff --git a/crates/core/src/cpu/schedulers/mod.rs b/crates/core/src/cpu/schedulers/mod.rs new file mode 100644 index 0000000..2f218b0 --- /dev/null +++ b/crates/core/src/cpu/schedulers/mod.rs @@ -0,0 +1,38 @@ +mod decay; +mod oc; + +use crate::Scheduler; + +pub use decay::*; +pub use oc::*; +pub enum CPUScheduler { + None, + LinearDecay(CPUDecayScheduler), + ExponentialDecay(CPUDecayScheduler), + OneCycle(CPUOneCycleScheduler), +} + +impl CPUScheduler { + pub fn from(scheduler: &Scheduler) -> Self { + match scheduler { + Scheduler::None => CPUScheduler::None, + Scheduler::LinearDecay(config) => { + CPUScheduler::LinearDecay(CPUDecayScheduler::new(config)) + } + Scheduler::ExponentialDecay(config) => { + CPUScheduler::ExponentialDecay(CPUDecayScheduler::new(config)) + } + Scheduler::OneCycle(config) => { + CPUScheduler::OneCycle(CPUOneCycleScheduler::new(config)) + } + } + } + pub fn eta(&self, rate: f32, step: usize) -> f32 { + match self { + CPUScheduler::None => rate, + CPUScheduler::LinearDecay(scheduler) => scheduler.linear(rate, step), + CPUScheduler::ExponentialDecay(scheduler) => scheduler.exponential(rate, step), + CPUScheduler::OneCycle(scheduler) => scheduler.eta(rate, step), + } + } +} diff --git a/crates/core/src/cpu/schedulers/oc.rs b/crates/core/src/cpu/schedulers/oc.rs new file mode 100644 index 0000000..aa8dbb6 --- /dev/null +++ b/crates/core/src/cpu/schedulers/oc.rs @@ -0,0 +1,24 @@ +use crate::OneCycleScheduler; + +pub struct CPUOneCycleScheduler { + pub max_rate: f32, + pub step_size: usize, +} + +impl CPUOneCycleScheduler { + pub fn new(config: &OneCycleScheduler) -> Self { + CPUOneCycleScheduler { + max_rate: config.max_rate, + step_size: config.step_size, + } + } + pub fn eta(&self, rate: f32, step: usize) -> f32 { + let steps = self.step_size as f32; + let step = step % (2 * self.step_size); + if step < self.step_size { + rate + (self.max_rate - rate) * (step as f32) / (steps) + } else { + self.max_rate - (self.max_rate - rate) * ((step - self.step_size) as f32) / (steps) + } + } +} diff --git a/crates/core/src/ffi.rs b/crates/core/src/ffi.rs index 26214e7..7a4c00c 100644 --- a/crates/core/src/ffi.rs +++ b/crates/core/src/ffi.rs @@ -1,7 +1,7 @@ use std::slice::{from_raw_parts, from_raw_parts_mut}; use crate::{ - decode_array, decode_json, length, CPUBackend, Dataset, Logger, PredictOptions, TrainOptions, + decode_array, decode_json, length, Backend, Dataset, Logger, PredictOptions, TrainOptions, RESOURCES, }; @@ -14,7 +14,7 @@ fn log(string: String) { #[no_mangle] pub extern "C" fn ffi_backend_create(ptr: *const u8, len: usize, alloc: AllocBufferFn) -> usize { let config = decode_json(ptr, len); - let net_backend = CPUBackend::new(config, Logger { log }, None); + let net_backend = Backend::new(config, Logger { log }, None); let buf: Vec = net_backend.size.iter().map(|x| *x as u8).collect(); let size_ptr = alloc(buf.len()); let output_shape = unsafe { from_raw_parts_mut(size_ptr, buf.len()) }; @@ -93,7 +93,7 @@ pub extern "C" fn ffi_backend_load( alloc: AllocBufferFn, ) -> usize { let buffer = unsafe { from_raw_parts(file_ptr, file_len) }; - let net_backend = CPUBackend::load(buffer, Logger { log }); + let net_backend = Backend::load(buffer, Logger { log }); let buf: Vec = net_backend.size.iter().map(|x| *x as u8).collect(); let size_ptr = alloc(buf.len()); let output_shape = unsafe { from_raw_parts_mut(size_ptr, buf.len()) }; diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index fb3ae0e..b03ae3b 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -8,18 +8,21 @@ mod util; mod wasm; pub use cpu::*; + + #[cfg(not(target_arch = "wasm32"))] pub use ffi::*; pub use tensor::*; pub use types::*; pub use util::*; + #[cfg(target_arch = "wasm32")] pub use wasm::*; use std::cell::RefCell; pub struct Resources { - pub backend: RefCell>, + pub backend: RefCell>, } impl Resources { diff --git a/crates/core/src/types.rs b/crates/core/src/types.rs index c30135a..4ac4bce 100644 --- a/crates/core/src/types.rs +++ b/crates/core/src/types.rs @@ -8,6 +8,7 @@ pub struct BackendConfig { pub layers: Vec, pub cost: Cost, pub optimizer: Optimizer, + pub scheduler: Scheduler, } #[derive(Debug)] @@ -126,7 +127,7 @@ pub enum Init { #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "lowercase")] -pub struct AdamOptimizer { +pub struct AdamOptimizer { pub beta1: f32, pub beta2: f32, pub epsilon: f32, @@ -137,7 +138,31 @@ pub struct AdamOptimizer { #[serde(rename_all = "lowercase")] pub enum Optimizer { SGD, - Adam(AdamOptimizer) + Adam(AdamOptimizer), +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "lowercase")] +pub struct DecayScheduler { + pub rate: f32, + pub step_size: usize, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "lowercase")] +pub struct OneCycleScheduler { + pub max_rate: f32, + pub step_size: usize, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(tag = "type", content = "config")] +#[serde(rename_all = "lowercase")] +pub enum Scheduler { + None, + LinearDecay(DecayScheduler), + ExponentialDecay(DecayScheduler), + OneCycle(OneCycleScheduler), } #[derive(Serialize, Deserialize, Debug, Clone)] diff --git a/crates/core/src/wasm.rs b/crates/core/src/wasm.rs index cf39cfe..f17435f 100644 --- a/crates/core/src/wasm.rs +++ b/crates/core/src/wasm.rs @@ -3,7 +3,7 @@ use ndarray::ArrayD; use wasm_bindgen::{prelude::wasm_bindgen, JsValue}; -use crate::{CPUBackend, Dataset, Logger, PredictOptions, TrainOptions, RESOURCES}; +use crate::{Backend, Dataset, Logger, PredictOptions, TrainOptions, RESOURCES}; #[wasm_bindgen] extern "C" { @@ -20,7 +20,7 @@ pub fn wasm_backend_create(config: String, shape: Array) -> usize { let config = serde_json::from_str(&config).unwrap(); let mut len = 0; let logger = Logger { log: console_log }; - let net_backend = CPUBackend::new(config, logger, None); + let net_backend = Backend::new(config, logger, None); shape.set_length(net_backend.size.len() as u32); for (i, s) in net_backend.size.iter().enumerate() { shape.set(i as u32, JsValue::from(*s)) @@ -82,7 +82,7 @@ pub fn wasm_backend_save(id: usize) -> Uint8Array { pub fn wasm_backend_load(buffer: Uint8Array, shape: Array) -> usize { let mut len = 0; let logger = Logger { log: console_log }; - let net_backend = CPUBackend::load(buffer.to_vec().as_slice(), logger); + let net_backend = Backend::load(buffer.to_vec().as_slice(), logger); shape.set_length(net_backend.size.len() as u32); for (i, s) in net_backend.size.iter().enumerate() { shape.set(i as u32, JsValue::from(*s)) diff --git a/crates/tokenizers/Cargo.toml b/crates/tokenizers/Cargo.toml new file mode 100644 index 0000000..3e19155 --- /dev/null +++ b/crates/tokenizers/Cargo.toml @@ -0,0 +1,17 @@ +[package] +edition = "2021" +name = "netsaur-tokenizers" +version = "0.2.9" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +ndarray = "0.15.6" +ndarray-rand = "0.14.0" +serde = {version = "1.0", features = ["derive"]} +serde_json = "1.0" +tokenizers = { version="0.14.1", default-features=false, features = ["unstable_wasm"]} +wasm-bindgen = "=0.2.84" +getrandom = { version = "0.2", features = ["js"] } +js-sys = "0.3.61" diff --git a/crates/tokenizers/src/lib.rs b/crates/tokenizers/src/lib.rs new file mode 100644 index 0000000..6576093 --- /dev/null +++ b/crates/tokenizers/src/lib.rs @@ -0,0 +1,27 @@ +mod util; +mod wasm; +use tokenizers::{ModelWrapper, Tokenizer}; + +pub use util::*; + +pub use wasm::*; + +use std::cell::RefCell; + +pub struct Resources { + pub tokenizer: RefCell>, + pub model: RefCell>, +} + +impl Resources { + pub fn new() -> Self { + Self { + tokenizer: RefCell::new(Vec::new()), + model: RefCell::new(Vec::new()), + } + } +} + +thread_local! { + pub static RESOURCES: Resources = Resources::new(); +} diff --git a/crates/tokenizers/src/util.rs b/crates/tokenizers/src/util.rs new file mode 100644 index 0000000..8c3e702 --- /dev/null +++ b/crates/tokenizers/src/util.rs @@ -0,0 +1,27 @@ +use std::slice::from_raw_parts; + +use ndarray::ArrayD; +use serde::Deserialize; + +pub struct Logger { + pub log: fn(string: String) -> (), +} + +pub fn length(shape: Vec) -> usize { + return shape.iter().fold(1, |i, x| i * x); +} + +pub fn decode_array(ptr: *const f32, shape: Vec) -> ArrayD { + let buffer = unsafe { from_raw_parts(ptr, length(shape.clone())) }; + let vec = Vec::from(buffer); + return ArrayD::from_shape_vec(shape, vec).unwrap(); +} + +pub fn decode_json<'a, T>(ptr: *const u8, len: usize) -> T +where + T: Deserialize<'a>, +{ + let buffer = unsafe { from_raw_parts(ptr, len) }; + let json = std::str::from_utf8(&buffer[0..len]).unwrap(); + return serde_json::from_str(&json).unwrap(); +} diff --git a/crates/tokenizers/src/wasm.rs b/crates/tokenizers/src/wasm.rs new file mode 100644 index 0000000..0e1dc0a --- /dev/null +++ b/crates/tokenizers/src/wasm.rs @@ -0,0 +1,55 @@ +use crate::RESOURCES; +use std::str::FromStr; +use tokenizers::{models::bpe::BPE, tokenizer::Tokenizer}; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +pub fn wasm_tokenizer_from_json(json: String) -> usize { + let tokenizer = Tokenizer::from_str(json.as_str()).unwrap().into(); + let mut len = 0; + RESOURCES.with(|cell| { + let mut tokenizers = cell.tokenizer.borrow_mut(); + len = tokenizers.len(); + tokenizers.push(tokenizer); + }); + len +} + +#[wasm_bindgen] +pub fn wasm_tokenizer_save(id: usize, pretty: bool) -> String { + let mut data: String = String::new(); + RESOURCES.with(|cell| { + let tokenizers = cell.tokenizer.borrow_mut(); + let tokenizer = &tokenizers[id]; + data = tokenizer.to_string(pretty).unwrap(); + }); + data +} + +#[wasm_bindgen] +pub fn wasm_bpe_default() -> usize { + let model = BPE::default(); + let mut len = 0; + RESOURCES.with(|cell| { + let mut models = cell.model.borrow_mut(); + len = models.len(); + models.push(tokenizers::ModelWrapper::BPE(model)); + }); + len +} + +#[wasm_bindgen] +pub fn wasm_tokenizer_tokenize(id: usize, string: String) -> Vec { + let mut data: Vec = Vec::new(); + RESOURCES.with(|cell| { + let tokenizers = cell.tokenizer.borrow_mut(); + data = tokenizers[id] + .encode(string, false) + .unwrap() + .get_ids() + .into_iter() + .cloned() + .collect() + }); + data +} diff --git a/data/data.ts b/data/data.ts index 3185b5b..177282d 100644 --- a/data/data.ts +++ b/data/data.ts @@ -1,5 +1,5 @@ import { Rank, Tensor } from "../mod.ts"; -import { CsvLoaderConfig, loadCsv } from "./csv.ts"; +import { CsvLoaderConfig, loadCsv } from "./datasets/csv.ts"; import type { DataLike } from "./types.ts"; export class Data { diff --git a/data/csv.ts b/data/datasets/csv.ts similarity index 91% rename from data/csv.ts rename to data/datasets/csv.ts index 7e95b30..e66e624 100644 --- a/data/csv.ts +++ b/data/datasets/csv.ts @@ -1,6 +1,6 @@ -import { tensor2D } from "../mod.ts"; -import type { DataLike } from "./types.ts"; -import { CsvParseStream } from "./deps.ts"; +import { tensor2D } from "../../mod.ts"; +import type { DataLike } from "../types.ts"; +import { CsvParseStream } from "../deps.ts"; export interface CsvColumnConfig { /** diff --git a/data/datasets/text.ts b/data/datasets/text.ts new file mode 100644 index 0000000..e69de29 diff --git a/data/deps.ts b/data/deps.ts index 1992b7a..e045d1b 100644 --- a/data/deps.ts +++ b/data/deps.ts @@ -1 +1 @@ -export { CsvParseStream } from "https://deno.land/std@0.190.0/csv/csv_parse_stream.ts"; +export { CsvParseStream } from "https://deno.land/std@0.198.0/csv/csv_parse_stream.ts"; diff --git a/data/mod.ts b/data/mod.ts index c88c9af..2186a8e 100644 --- a/data/mod.ts +++ b/data/mod.ts @@ -1,2 +1,2 @@ -export * from "./csv.ts"; +export * from "./datasets/csv.ts"; export * from "./data.ts"; diff --git a/deno.json b/deno.json index 61b19be..eade2d4 100644 --- a/deno.json +++ b/deno.json @@ -3,6 +3,7 @@ "example:xor": "deno run -A --unstable ./examples/xor_auto.ts", "example:xor-option": "deno run -A --unstable ./examples/xor_option.ts", "example:xor-cpu": "deno run -A --unstable ./examples/xor_cpu.ts", + "example:xor-gpu": "deno run -A --unstable ./examples/xor_gpu.ts", "example:xor-wasm": "deno run -A --unstable ./examples/xor_wasm.ts", "example:linear": "deno run -A --unstable ./examples/linear.ts", "example:filters": "deno run -A --unstable examples/filters/conv.ts ", @@ -11,8 +12,11 @@ "example:mnist-download": "deno run -A --unstable examples/mnist/download.ts ", "example:mnist-train": "deno run -A --unstable examples/mnist/train.ts ", "example:mnist-predict": "deno run -A --unstable examples/mnist/predict.ts ", - "build": "deno task build:cpu && deno task build:wasm", - "build:cpu": "cargo build --release", - "build:wasm": "deno run -A https://deno.land/x/wasmbuild@0.11.0/main.ts --out src/backend_wasm/lib" + "example:tokenizers-basic": "deno run -A --unstable examples/tokenizers/basic.ts", + "build": "deno task build:cpu && deno task build:wasm && deno task build:gpu && deno task build:tokenizers", + "build:cpu": "cargo build --release -p netsaur", + "build:gpu": "cargo build --release -p netsaur-gpu", + "build:wasm": "deno run -A https://deno.land/x/wasmbuild@0.11.0/main.ts -p netsaur --out src/backends/wasm/lib", + "build:tokenizers": "deno run -A https://deno.land/x/wasmbuild@0.11.0/main.ts -p netsaur-tokenizers --out tokenizers/lib" } } diff --git a/deno.lock b/deno.lock index caab55b..afd9e68 100644 --- a/deno.lock +++ b/deno.lock @@ -1,6 +1,735 @@ { - "version": "2", + "version": "3", + "packages": { + "specifiers": { + "npm:chartjs-node-canvas": "npm:chartjs-node-canvas@4.1.6_chart.js@3.9.1", + "npm:graphviz": "npm:graphviz@0.0.9", + "npm:tokenizers": "npm:tokenizers@0.13.1" + }, + "npm": { + "@mapbox/node-pre-gyp@1.0.10": { + "integrity": "sha512-4ySo4CjzStuprMwk35H5pPbkymjv1SF3jGLj6rAHp/xT/RF7TL7bd9CTm1xDY49K2qF7jmR/g7k+SkLETP6opA==", + "dependencies": { + "detect-libc": "detect-libc@2.0.1", + "https-proxy-agent": "https-proxy-agent@5.0.1", + "make-dir": "make-dir@3.1.0", + "node-fetch": "node-fetch@2.7.0", + "nopt": "nopt@5.0.0", + "npmlog": "npmlog@5.0.1", + "rimraf": "rimraf@3.0.2", + "semver": "semver@7.5.4", + "tar": "tar@6.2.0" + } + }, + "@types/node@13.13.52": { + "integrity": "sha512-s3nugnZumCC//n4moGGe6tkNMyYEdaDBitVjwPxXmR5lnMG5dHePinH2EdxkG3Rh1ghFHHixAG4NJhpJW1rthQ==", + "dependencies": {} + }, + "abbrev@1.1.1": { + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", + "dependencies": {} + }, + "agent-base@6.0.2": { + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "dependencies": { + "debug": "debug@4.3.4" + } + }, + "ansi-regex@2.1.1": { + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", + "dependencies": {} + }, + "ansi-regex@5.0.1": { + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dependencies": {} + }, + "aproba@1.2.0": { + "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", + "dependencies": {} + }, + "are-we-there-yet@1.1.7": { + "integrity": "sha512-nxwy40TuMiUGqMyRHgCSWZ9FM4VAoRP4xUYSTv5ImRog+h9yISPbVH7H8fASCIzYn9wlEv4zvFL7uKDMCFQm3g==", + "dependencies": { + "delegates": "delegates@1.0.0", + "readable-stream": "readable-stream@2.3.7" + } + }, + "are-we-there-yet@2.0.0": { + "integrity": "sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==", + "dependencies": { + "delegates": "delegates@1.0.0", + "readable-stream": "readable-stream@3.6.2" + } + }, + "balanced-match@1.0.2": { + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dependencies": {} + }, + "brace-expansion@1.1.11": { + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dependencies": { + "balanced-match": "balanced-match@1.0.2", + "concat-map": "concat-map@0.0.1" + } + }, + "canvas@2.11.2": { + "integrity": "sha512-ItanGBMrmRV7Py2Z+Xhs7cT+FNt5K0vPL4p9EZ/UX/Mu7hFbkxSjKF2KVtPwX7UYWp7dRKnrTvReflgrItJbdw==", + "dependencies": { + "@mapbox/node-pre-gyp": "@mapbox/node-pre-gyp@1.0.10", + "nan": "nan@2.17.0", + "simple-get": "simple-get@3.1.1" + } + }, + "chart.js@3.9.1": { + "integrity": "sha512-Ro2JbLmvg83gXF5F4sniaQ+lTbSv18E+TIf2cOeiH1Iqd2PGFOtem+DUufMZsCJwFE7ywPOpfXFBwRTGq7dh6w==", + "dependencies": {} + }, + "chartjs-node-canvas@4.1.6_chart.js@3.9.1": { + "integrity": "sha512-UQJbPWrvqB/FoLclGA9BaLQmZbzSYlujF4w8NZd6Xzb+sqgACBb2owDX6m7ifCXLjUW5Nz0Qx0qqrTtQkkSoYw==", + "dependencies": { + "canvas": "canvas@2.11.2", + "chart.js": "chart.js@3.9.1", + "tslib": "tslib@2.6.2" + } + }, + "chownr@1.1.4": { + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "dependencies": {} + }, + "chownr@2.0.0": { + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "dependencies": {} + }, + "code-point-at@1.1.0": { + "integrity": "sha512-RpAVKQA5T63xEj6/giIbUEtZwJ4UFIc3ZtvEkiaUERylqe8xb5IvqcgOurZLahv93CLKfxcw5YI+DZcUBRyLXA==", + "dependencies": {} + }, + "color-support@1.1.3": { + "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==", + "dependencies": {} + }, + "concat-map@0.0.1": { + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dependencies": {} + }, + "console-control-strings@1.1.0": { + "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==", + "dependencies": {} + }, + "core-util-is@1.0.3": { + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "dependencies": {} + }, + "debug@3.2.7": { + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dependencies": { + "ms": "ms@2.1.3" + } + }, + "debug@4.3.4": { + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dependencies": { + "ms": "ms@2.1.2" + } + }, + "decompress-response@4.2.1": { + "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==", + "dependencies": { + "mimic-response": "mimic-response@2.1.0" + } + }, + "deep-extend@0.6.0": { + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "dependencies": {} + }, + "delegates@1.0.0": { + "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==", + "dependencies": {} + }, + "detect-libc@1.0.3": { + "integrity": "sha512-pGjwhsmsp4kL2RTz08wcOlGN83otlqHeD/Z5T8GXZB+/YcpQ/dgo+lbU8ZsGxV0HIvqqxo9l7mqYwyYMD9bKDg==", + "dependencies": {} + }, + "detect-libc@2.0.1": { + "integrity": "sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==", + "dependencies": {} + }, + "emoji-regex@8.0.0": { + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dependencies": {} + }, + "fs-minipass@1.2.7": { + "integrity": "sha512-GWSSJGFy4e9GUeCcbIkED+bgAoFyj7XF1mV8rma3QW4NIqX9Kyx79N/PF61H5udOV3aY1IaMLs6pGbH71nlCTA==", + "dependencies": { + "minipass": "minipass@2.9.0" + } + }, + "fs-minipass@2.1.0": { + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dependencies": { + "minipass": "minipass@3.3.6" + } + }, + "fs.realpath@1.0.0": { + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dependencies": {} + }, + "gauge@2.7.4": { + "integrity": "sha512-14x4kjc6lkD3ltw589k0NrPD6cCNTD6CWoVUNpB85+DrtONoZn+Rug6xZU5RvSC4+TZPxA5AnBibQYAvZn41Hg==", + "dependencies": { + "aproba": "aproba@1.2.0", + "console-control-strings": "console-control-strings@1.1.0", + "has-unicode": "has-unicode@2.0.1", + "object-assign": "object-assign@4.1.1", + "signal-exit": "signal-exit@3.0.7", + "string-width": "string-width@1.0.2", + "strip-ansi": "strip-ansi@3.0.1", + "wide-align": "wide-align@1.1.5" + } + }, + "gauge@3.0.2": { + "integrity": "sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==", + "dependencies": { + "aproba": "aproba@1.2.0", + "color-support": "color-support@1.1.3", + "console-control-strings": "console-control-strings@1.1.0", + "has-unicode": "has-unicode@2.0.1", + "object-assign": "object-assign@4.1.1", + "signal-exit": "signal-exit@3.0.7", + "string-width": "string-width@4.2.3", + "strip-ansi": "strip-ansi@6.0.1", + "wide-align": "wide-align@1.1.5" + } + }, + "glob@7.2.3": { + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dependencies": { + "fs.realpath": "fs.realpath@1.0.0", + "inflight": "inflight@1.0.6", + "inherits": "inherits@2.0.4", + "minimatch": "minimatch@3.1.2", + "once": "once@1.4.0", + "path-is-absolute": "path-is-absolute@1.0.1" + } + }, + "graphviz@0.0.9": { + "integrity": "sha512-SmoY2pOtcikmMCqCSy2NO1YsRfu9OO0wpTlOYW++giGjfX1a6gax/m1Fo8IdUd0/3H15cTOfR1SMKwohj4LKsg==", + "dependencies": { + "temp": "temp@0.4.0" + } + }, + "has-unicode@2.0.1": { + "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==", + "dependencies": {} + }, + "https-proxy-agent@5.0.1": { + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "dependencies": { + "agent-base": "agent-base@6.0.2", + "debug": "debug@4.3.4" + } + }, + "iconv-lite@0.4.24": { + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dependencies": { + "safer-buffer": "safer-buffer@2.1.2" + } + }, + "ignore-walk@3.0.4": { + "integrity": "sha512-PY6Ii8o1jMRA1z4F2hRkH/xN59ox43DavKvD3oDpfurRlOJyAHpifIwpbdv1n4jt4ov0jSpw3kQ4GhJnpBL6WQ==", + "dependencies": { + "minimatch": "minimatch@3.1.2" + } + }, + "inflight@1.0.6": { + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dependencies": { + "once": "once@1.4.0", + "wrappy": "wrappy@1.0.2" + } + }, + "inherits@2.0.4": { + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dependencies": {} + }, + "ini@1.3.8": { + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "dependencies": {} + }, + "is-fullwidth-code-point@1.0.0": { + "integrity": "sha512-1pqUqRjkhPJ9miNq9SwMfdvi6lBJcd6eFxvfaivQhaH3SgisfiuudvFntdKOmxuee/77l+FPjKrQjWvmPjWrRw==", + "dependencies": { + "number-is-nan": "number-is-nan@1.0.1" + } + }, + "is-fullwidth-code-point@3.0.0": { + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dependencies": {} + }, + "isarray@1.0.0": { + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dependencies": {} + }, + "lru-cache@6.0.0": { + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dependencies": { + "yallist": "yallist@4.0.0" + } + }, + "make-dir@3.1.0": { + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "dependencies": { + "semver": "semver@6.3.1" + } + }, + "mimic-response@2.1.0": { + "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==", + "dependencies": {} + }, + "minimatch@3.1.2": { + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dependencies": { + "brace-expansion": "brace-expansion@1.1.11" + } + }, + "minimist@1.2.7": { + "integrity": "sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g==", + "dependencies": {} + }, + "minipass@2.9.0": { + "integrity": "sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg==", + "dependencies": { + "safe-buffer": "safe-buffer@5.2.1", + "yallist": "yallist@3.1.1" + } + }, + "minipass@3.3.6": { + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dependencies": { + "yallist": "yallist@4.0.0" + } + }, + "minipass@5.0.0": { + "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==", + "dependencies": {} + }, + "minizlib@1.3.3": { + "integrity": "sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q==", + "dependencies": { + "minipass": "minipass@2.9.0" + } + }, + "minizlib@2.1.2": { + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "dependencies": { + "minipass": "minipass@3.3.6", + "yallist": "yallist@4.0.0" + } + }, + "mkdirp@0.5.6": { + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "dependencies": { + "minimist": "minimist@1.2.7" + } + }, + "mkdirp@1.0.4": { + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dependencies": {} + }, + "ms@2.1.2": { + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dependencies": {} + }, + "ms@2.1.3": { + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dependencies": {} + }, + "nan@2.17.0": { + "integrity": "sha512-2ZTgtl0nJsO0KQCjEpxcIr5D+Yv90plTitZt9JBfQvVJDS5seMl3FOvsh3+9CoYWXf/1l5OaZzzF6nDm4cagaQ==", + "dependencies": {} + }, + "needle@2.9.1": { + "integrity": "sha512-6R9fqJ5Zcmf+uYaFgdIHmLwNldn5HbK8L5ybn7Uz+ylX/rnOsSp1AHcvQSrCaFN+qNM1wpymHqD7mVasEOlHGQ==", + "dependencies": { + "debug": "debug@3.2.7", + "iconv-lite": "iconv-lite@0.4.24", + "sax": "sax@1.2.4" + } + }, + "node-fetch@2.7.0": { + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dependencies": { + "whatwg-url": "whatwg-url@5.0.0" + } + }, + "node-pre-gyp@0.14.0": { + "integrity": "sha512-+CvDC7ZttU/sSt9rFjix/P05iS43qHCOOGzcr3Ry99bXG7VX953+vFyEuph/tfqoYu8dttBkE86JSKBO2OzcxA==", + "dependencies": { + "detect-libc": "detect-libc@1.0.3", + "mkdirp": "mkdirp@0.5.6", + "needle": "needle@2.9.1", + "nopt": "nopt@4.0.3", + "npm-packlist": "npm-packlist@1.4.8", + "npmlog": "npmlog@4.1.2", + "rc": "rc@1.2.8", + "rimraf": "rimraf@2.7.1", + "semver": "semver@5.7.1", + "tar": "tar@4.4.19" + } + }, + "nopt@4.0.3": { + "integrity": "sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==", + "dependencies": { + "abbrev": "abbrev@1.1.1", + "osenv": "osenv@0.1.5" + } + }, + "nopt@5.0.0": { + "integrity": "sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==", + "dependencies": { + "abbrev": "abbrev@1.1.1" + } + }, + "npm-bundled@1.1.2": { + "integrity": "sha512-x5DHup0SuyQcmL3s7Rx/YQ8sbw/Hzg0rj48eN0dV7hf5cmQq5PXIeioroH3raV1QC1yh3uTYuMThvEQF3iKgGQ==", + "dependencies": { + "npm-normalize-package-bin": "npm-normalize-package-bin@1.0.1" + } + }, + "npm-normalize-package-bin@1.0.1": { + "integrity": "sha512-EPfafl6JL5/rU+ot6P3gRSCpPDW5VmIzX959Ob1+ySFUuuYHWHekXpwdUZcKP5C+DS4GEtdJluwBjnsNDl+fSA==", + "dependencies": {} + }, + "npm-packlist@1.4.8": { + "integrity": "sha512-5+AZgwru5IevF5ZdnFglB5wNlHG1AOOuw28WhUq8/8emhBmLv6jX5by4WJCh7lW0uSYZYS6DXqIsyZVIXRZU9A==", + "dependencies": { + "ignore-walk": "ignore-walk@3.0.4", + "npm-bundled": "npm-bundled@1.1.2", + "npm-normalize-package-bin": "npm-normalize-package-bin@1.0.1" + } + }, + "npmlog@4.1.2": { + "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", + "dependencies": { + "are-we-there-yet": "are-we-there-yet@1.1.7", + "console-control-strings": "console-control-strings@1.1.0", + "gauge": "gauge@2.7.4", + "set-blocking": "set-blocking@2.0.0" + } + }, + "npmlog@5.0.1": { + "integrity": "sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==", + "dependencies": { + "are-we-there-yet": "are-we-there-yet@2.0.0", + "console-control-strings": "console-control-strings@1.1.0", + "gauge": "gauge@3.0.2", + "set-blocking": "set-blocking@2.0.0" + } + }, + "number-is-nan@1.0.1": { + "integrity": "sha512-4jbtZXNAsfZbAHiiqjLPBiCl16dES1zI4Hpzzxw61Tk+loF+sBDBKx1ICKKKwIqQ7M0mFn1TmkN7euSncWgHiQ==", + "dependencies": {} + }, + "object-assign@4.1.1": { + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "dependencies": {} + }, + "once@1.4.0": { + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dependencies": { + "wrappy": "wrappy@1.0.2" + } + }, + "os-homedir@1.0.2": { + "integrity": "sha512-B5JU3cabzk8c67mRRd3ECmROafjYMXbuzlwtqdM8IbS8ktlTix8aFGb2bAGKrSRIlnfKwovGUUr72JUPyOb6kQ==", + "dependencies": {} + }, + "os-tmpdir@1.0.2": { + "integrity": "sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==", + "dependencies": {} + }, + "osenv@0.1.5": { + "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==", + "dependencies": { + "os-homedir": "os-homedir@1.0.2", + "os-tmpdir": "os-tmpdir@1.0.2" + } + }, + "path-is-absolute@1.0.1": { + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dependencies": {} + }, + "process-nextick-args@2.0.1": { + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "dependencies": {} + }, + "rc@1.2.8": { + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dependencies": { + "deep-extend": "deep-extend@0.6.0", + "ini": "ini@1.3.8", + "minimist": "minimist@1.2.7", + "strip-json-comments": "strip-json-comments@2.0.1" + } + }, + "readable-stream@2.3.7": { + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "core-util-is@1.0.3", + "inherits": "inherits@2.0.4", + "isarray": "isarray@1.0.0", + "process-nextick-args": "process-nextick-args@2.0.1", + "safe-buffer": "safe-buffer@5.1.2", + "string_decoder": "string_decoder@1.1.1", + "util-deprecate": "util-deprecate@1.0.2" + } + }, + "readable-stream@3.6.2": { + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": { + "inherits": "inherits@2.0.4", + "string_decoder": "string_decoder@1.1.1", + "util-deprecate": "util-deprecate@1.0.2" + } + }, + "rimraf@2.7.1": { + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "dependencies": { + "glob": "glob@7.2.3" + } + }, + "rimraf@3.0.2": { + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dependencies": { + "glob": "glob@7.2.3" + } + }, + "safe-buffer@5.1.2": { + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dependencies": {} + }, + "safe-buffer@5.2.1": { + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dependencies": {} + }, + "safer-buffer@2.1.2": { + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dependencies": {} + }, + "sax@1.2.4": { + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==", + "dependencies": {} + }, + "semver@5.7.1": { + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dependencies": {} + }, + "semver@6.3.1": { + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dependencies": {} + }, + "semver@7.5.4": { + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", + "dependencies": { + "lru-cache": "lru-cache@6.0.0" + } + }, + "set-blocking@2.0.0": { + "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==", + "dependencies": {} + }, + "signal-exit@3.0.7": { + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dependencies": {} + }, + "simple-concat@1.0.1": { + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "dependencies": {} + }, + "simple-get@3.1.1": { + "integrity": "sha512-CQ5LTKGfCpvE1K0n2us+kuMPbk/q0EKl82s4aheV9oXjFEz6W/Y7oQFVJuU6QG77hRT4Ghb5RURteF5vnWjupA==", + "dependencies": { + "decompress-response": "decompress-response@4.2.1", + "once": "once@1.4.0", + "simple-concat": "simple-concat@1.0.1" + } + }, + "string-width@1.0.2": { + "integrity": "sha512-0XsVpQLnVCXHJfyEs8tC0zpTVIr5PKKsQtkT29IwupnPTjtPmQ3xT/4yCREF9hYkV/3M3kzcUTSAZT6a6h81tw==", + "dependencies": { + "code-point-at": "code-point-at@1.1.0", + "is-fullwidth-code-point": "is-fullwidth-code-point@1.0.0", + "strip-ansi": "strip-ansi@3.0.1" + } + }, + "string-width@4.2.3": { + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "emoji-regex@8.0.0", + "is-fullwidth-code-point": "is-fullwidth-code-point@3.0.0", + "strip-ansi": "strip-ansi@6.0.1" + } + }, + "string_decoder@1.1.1": { + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "safe-buffer@5.1.2" + } + }, + "strip-ansi@3.0.1": { + "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", + "dependencies": { + "ansi-regex": "ansi-regex@2.1.1" + } + }, + "strip-ansi@6.0.1": { + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "ansi-regex@5.0.1" + } + }, + "strip-json-comments@2.0.1": { + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "dependencies": {} + }, + "tar@4.4.19": { + "integrity": "sha512-a20gEsvHnWe0ygBY8JbxoM4w3SJdhc7ZAuxkLqh+nvNQN2IOt0B5lLgM490X5Hl8FF0dl0tOf2ewFYAlIFgzVA==", + "dependencies": { + "chownr": "chownr@1.1.4", + "fs-minipass": "fs-minipass@1.2.7", + "minipass": "minipass@2.9.0", + "minizlib": "minizlib@1.3.3", + "mkdirp": "mkdirp@0.5.6", + "safe-buffer": "safe-buffer@5.2.1", + "yallist": "yallist@3.1.1" + } + }, + "tar@6.2.0": { + "integrity": "sha512-/Wo7DcT0u5HUV486xg675HtjNd3BXZ6xDbzsCUZPt5iw8bTQ63bP0Raut3mvro9u+CUyq7YQd8Cx55fsZXxqLQ==", + "dependencies": { + "chownr": "chownr@2.0.0", + "fs-minipass": "fs-minipass@2.1.0", + "minipass": "minipass@5.0.0", + "minizlib": "minizlib@2.1.2", + "mkdirp": "mkdirp@1.0.4", + "yallist": "yallist@4.0.0" + } + }, + "temp@0.4.0": { + "integrity": "sha512-IsFisGgDKk7qzK9erMIkQe/XwiSUdac7z3wYOsjcLkhPBy3k1SlvLoIh2dAHIlEpgA971CgguMrx9z8fFg7tSA==", + "dependencies": {} + }, + "tokenizers@0.13.1": { + "integrity": "sha512-2jBdLK++8lwUOYeZ2z46PLZbEvioeCvkcWt1mcFA+ivGzLtMczUCma92XWokR36l1YfZ4jXNz9AM2OrYnmcrZA==", + "dependencies": { + "@types/node": "@types/node@13.13.52", + "node-pre-gyp": "node-pre-gyp@0.14.0" + } + }, + "tr46@0.0.3": { + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dependencies": {} + }, + "tslib@2.6.2": { + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", + "dependencies": {} + }, + "util-deprecate@1.0.2": { + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dependencies": {} + }, + "webidl-conversions@3.0.1": { + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dependencies": {} + }, + "whatwg-url@5.0.0": { + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dependencies": { + "tr46": "tr46@0.0.3", + "webidl-conversions": "webidl-conversions@3.0.1" + } + }, + "wide-align@1.1.5": { + "integrity": "sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==", + "dependencies": { + "string-width": "string-width@1.0.2" + } + }, + "wrappy@1.0.2": { + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dependencies": {} + }, + "yallist@3.1.1": { + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dependencies": {} + }, + "yallist@4.0.0": { + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dependencies": {} + } + } + }, + "redirects": { + "https://deno.land/std/encoding/base64.ts": "https://deno.land/std@0.173.0/encoding/base64.ts", + "https://deno.land/x/canvas/mod.ts": "https://deno.land/x/canvas@v1.4.1/mod.ts", + "https://deno.land/x/netsaur/mod.ts": "https://deno.land/x/netsaur@0.2.8/mod.ts" + }, "remote": { + "https://cdn.jsdelivr.net/npm/chart.js@3.9.1/dist/chart.esm.js": "fd917fb668977fd12433a30ab7e730b526a64b30c4f5f80686ac97fa135923ef", + "https://cdn.jsdelivr.net/npm/chart.js@3.9.1/dist/chunks/helpers.segment.js": "d34357e69f1f76d07321e93bf8a907352cf20f8ddf6f3bad51776dc4960f5be7", + "https://deno.land/std@0.106.0/encoding/base64.ts": "eecae390f1f1d1cae6f6c6d732ede5276bf4b9cd29b1d281678c054dc5cc009e", + "https://deno.land/std@0.145.0/encoding/base64.ts": "c8c16b4adaa60d7a8eee047c73ece26844435e8f7f1328d74593dbb2dd58ea4f", + "https://deno.land/std@0.157.0/_util/assert.ts": "e94f2eb37cebd7f199952e242c77654e43333c1ac4c5c700e929ea3aa5489f74", + "https://deno.land/std@0.157.0/_util/os.ts": "8a33345f74990e627b9dfe2de9b040004b08ea5146c7c9e8fe9a29070d193934", + "https://deno.land/std@0.157.0/encoding/hex.ts": "4cc5324417cbb4ac9b828453d35aed45b9cc29506fad658f1f138d981ae33795", + "https://deno.land/std@0.157.0/fmt/colors.ts": "ff7dc9c9f33a72bd48bc24b21bbc1b4545d8494a431f17894dbc5fe92a938fc4", + "https://deno.land/std@0.157.0/fs/_util.ts": "fdc156f897197f261a1c096dcf8ff9267ed0ff42bd5b31f55053a4763a4bae3b", + "https://deno.land/std@0.157.0/fs/copy.ts": "73bdf24f4322648d9bc38ef983b818637ba368351d17aa03644209d3ce3eac31", + "https://deno.land/std@0.157.0/fs/empty_dir.ts": "c15a0aaaf40f8c21cca902aa1e01a789ad0c2fd1b7e2eecf4957053c5dbf707f", + "https://deno.land/std@0.157.0/fs/ensure_dir.ts": "76395fc1c989ca8d2de3aedfa8240eb8f5225cde20f926de957995b063135b80", + "https://deno.land/std@0.157.0/fs/ensure_file.ts": "b8e32ea63aa21221d0219760ba3f741f682d7f7d68d0d24a3ec067c338568152", + "https://deno.land/std@0.157.0/fs/ensure_link.ts": "5cc1c04f18487d7d1edf4c5469705f30b61390ffd24ad7db6df85e7209b32bb2", + "https://deno.land/std@0.157.0/fs/ensure_symlink.ts": "5273557b8c50be69477aa9cb003b54ff2240a336db52a40851c97abce76b96ab", + "https://deno.land/std@0.157.0/fs/eol.ts": "b92f0b88036de507e7e6fbedbe8f666835ea9dcbf5ac85917fa1fadc919f83a5", + "https://deno.land/std@0.157.0/fs/exists.ts": "3661a679d9018338df5df7cd9fc7cd918f5e447e7304a5391f849a893ce24107", + "https://deno.land/std@0.157.0/fs/expand_glob.ts": "333a8b9b0726b6909e5af30fb99c68e5b0e734d37af8cbc2ad1f062f26ca4d50", + "https://deno.land/std@0.157.0/fs/mod.ts": "354a6f972ef4e00c4dd1f1339a8828ef0764c1c23d3c0010af3fcc025d8655b0", + "https://deno.land/std@0.157.0/fs/move.ts": "6d7fa9da60dbc7a32dd7fdbc2ff812b745861213c8e92ba96dace0669b0c378c", + "https://deno.land/std@0.157.0/fs/walk.ts": "d6c73a2a2fb5fde60150ce27cff3fff420e72e5bb84131b4919c9a41d74712ce", + "https://deno.land/std@0.157.0/path/_constants.ts": "df1db3ffa6dd6d1252cc9617e5d72165cd2483df90e93833e13580687b6083c3", + "https://deno.land/std@0.157.0/path/_interface.ts": "ee3b431a336b80cf445441109d089b70d87d5e248f4f90ff906820889ecf8d09", + "https://deno.land/std@0.157.0/path/_util.ts": "d16be2a16e1204b65f9d0dfc54a9bc472cafe5f4a190b3c8471ec2016ccd1677", + "https://deno.land/std@0.157.0/path/common.ts": "bee563630abd2d97f99d83c96c2fa0cca7cee103e8cb4e7699ec4d5db7bd2633", + "https://deno.land/std@0.157.0/path/glob.ts": "cb5255638de1048973c3e69e420c77dc04f75755524cb3b2e160fe9277d939ee", + "https://deno.land/std@0.157.0/path/mod.ts": "56fec03ad0ebd61b6ab39ddb9b0ddb4c4a5c9f2f4f632e09dd37ec9ebfd722ac", + "https://deno.land/std@0.157.0/path/posix.ts": "c1f7afe274290ea0b51da07ee205653b2964bd74909a82deb07b69a6cc383aaa", + "https://deno.land/std@0.157.0/path/separator.ts": "fe1816cb765a8068afb3e8f13ad272351c85cbc739af56dacfc7d93d710fe0f9", + "https://deno.land/std@0.157.0/path/win32.ts": "bd7549042e37879c68ff2f8576a25950abbfca1d696d41d82c7bca0b7e6f452c", + "https://deno.land/std@0.159.0/_util/assert.ts": "e94f2eb37cebd7f199952e242c77654e43333c1ac4c5c700e929ea3aa5489f74", + "https://deno.land/std@0.159.0/_util/os.ts": "8a33345f74990e627b9dfe2de9b040004b08ea5146c7c9e8fe9a29070d193934", + "https://deno.land/std@0.159.0/fs/_util.ts": "fdc156f897197f261a1c096dcf8ff9267ed0ff42bd5b31f55053a4763a4bae3b", + "https://deno.land/std@0.159.0/fs/copy.ts": "73bdf24f4322648d9bc38ef983b818637ba368351d17aa03644209d3ce3eac31", + "https://deno.land/std@0.159.0/fs/empty_dir.ts": "c15a0aaaf40f8c21cca902aa1e01a789ad0c2fd1b7e2eecf4957053c5dbf707f", + "https://deno.land/std@0.159.0/fs/ensure_dir.ts": "76395fc1c989ca8d2de3aedfa8240eb8f5225cde20f926de957995b063135b80", + "https://deno.land/std@0.159.0/fs/ensure_file.ts": "b8e32ea63aa21221d0219760ba3f741f682d7f7d68d0d24a3ec067c338568152", + "https://deno.land/std@0.159.0/fs/ensure_link.ts": "5cc1c04f18487d7d1edf4c5469705f30b61390ffd24ad7db6df85e7209b32bb2", + "https://deno.land/std@0.159.0/fs/ensure_symlink.ts": "5273557b8c50be69477aa9cb003b54ff2240a336db52a40851c97abce76b96ab", + "https://deno.land/std@0.159.0/fs/eol.ts": "b92f0b88036de507e7e6fbedbe8f666835ea9dcbf5ac85917fa1fadc919f83a5", + "https://deno.land/std@0.159.0/fs/exists.ts": "6a447912e49eb79cc640adacfbf4b0baf8e17ede6d5bed057062ce33c4fa0d68", + "https://deno.land/std@0.159.0/fs/expand_glob.ts": "333a8b9b0726b6909e5af30fb99c68e5b0e734d37af8cbc2ad1f062f26ca4d50", + "https://deno.land/std@0.159.0/fs/mod.ts": "354a6f972ef4e00c4dd1f1339a8828ef0764c1c23d3c0010af3fcc025d8655b0", + "https://deno.land/std@0.159.0/fs/move.ts": "6d7fa9da60dbc7a32dd7fdbc2ff812b745861213c8e92ba96dace0669b0c378c", + "https://deno.land/std@0.159.0/fs/walk.ts": "d6c73a2a2fb5fde60150ce27cff3fff420e72e5bb84131b4919c9a41d74712ce", + "https://deno.land/std@0.159.0/path/_constants.ts": "df1db3ffa6dd6d1252cc9617e5d72165cd2483df90e93833e13580687b6083c3", + "https://deno.land/std@0.159.0/path/_interface.ts": "ee3b431a336b80cf445441109d089b70d87d5e248f4f90ff906820889ecf8d09", + "https://deno.land/std@0.159.0/path/_util.ts": "d16be2a16e1204b65f9d0dfc54a9bc472cafe5f4a190b3c8471ec2016ccd1677", + "https://deno.land/std@0.159.0/path/common.ts": "bee563630abd2d97f99d83c96c2fa0cca7cee103e8cb4e7699ec4d5db7bd2633", + "https://deno.land/std@0.159.0/path/glob.ts": "cb5255638de1048973c3e69e420c77dc04f75755524cb3b2e160fe9277d939ee", + "https://deno.land/std@0.159.0/path/mod.ts": "56fec03ad0ebd61b6ab39ddb9b0ddb4c4a5c9f2f4f632e09dd37ec9ebfd722ac", + "https://deno.land/std@0.159.0/path/posix.ts": "c1f7afe274290ea0b51da07ee205653b2964bd74909a82deb07b69a6cc383aaa", + "https://deno.land/std@0.159.0/path/separator.ts": "fe1816cb765a8068afb3e8f13ad272351c85cbc739af56dacfc7d93d710fe0f9", + "https://deno.land/std@0.159.0/path/win32.ts": "bd7549042e37879c68ff2f8576a25950abbfca1d696d41d82c7bca0b7e6f452c", + "https://deno.land/std@0.173.0/encoding/base64.ts": "7de04c2f8aeeb41453b09b186480be90f2ff357613b988e99fabb91d2eeceba1", "https://deno.land/std@0.184.0/_util/asserts.ts": "178dfc49a464aee693a7e285567b3d0b555dc805ff490505a8aae34f9cfb1462", "https://deno.land/std@0.184.0/_util/os.ts": "d932f56d41e4f6a6093d56044e29ce637f8dcc43c5a90af43504a889cf1775e3", "https://deno.land/std@0.184.0/encoding/hex.ts": "b4b1a7cb678745b0bf181ed8cf2498c7be00d121a7de244b752fbf9c7d9c48cd", @@ -27,10 +756,104 @@ "https://deno.land/std@0.184.0/path/posix.ts": "8b7c67ac338714b30c816079303d0285dd24af6b284f7ad63da5b27372a2c94d", "https://deno.land/std@0.184.0/path/separator.ts": "0fb679739d0d1d7bf45b68dacfb4ec7563597a902edbaf3c59b50d5bcadd93b1", "https://deno.land/std@0.184.0/path/win32.ts": "d186344e5583bcbf8b18af416d13d82b35a317116e6460a5a3953508c3de5bba", + "https://deno.land/std@0.188.0/_util/asserts.ts": "178dfc49a464aee693a7e285567b3d0b555dc805ff490505a8aae34f9cfb1462", + "https://deno.land/std@0.188.0/csv/_io.ts": "0f90b154e0f9c574a025f24d35d7ef617944809a66d277716243e68523a816b2", + "https://deno.land/std@0.188.0/csv/parse.ts": "96f7be0b5b5c0778bbd1c3f6f4ec689aecc406b6ce66d5059da12ca2268d2b44", + "https://deno.land/std@0.76.0/encoding/base64.ts": "b1d8f99b778981548457ec74bc6273ad785ffd6f61b2233bd5b30925345b565d", + "https://deno.land/x/canvas@v1.2.2/canvaskit.ts": "ab95583a578e3283bf66ac527a575482fbaf968dea2f59372846c83c6ecd1e22", + "https://deno.land/x/canvas@v1.2.2/deps.ts": "d6e023604ea156fbcb2bd997e2c957b119addf2bf782087e2358bd4705bab274", + "https://deno.land/x/canvas@v1.2.2/lib.js": "d65843a9e8f66489cce58d499618cd3c7bbc8d9fa31175ddc820785caf5686bc", + "https://deno.land/x/canvas@v1.2.2/mod.ts": "d926acdc90fb84200aaed1023fe60f88cad849e00d0cf39dff715c87b35012c1", + "https://deno.land/x/canvas@v1.2.2/types.ts": "2f59b69b890368cf37916e59f8e92bebe76f3d01412ee1dbc2e2162dd917702d", + "https://deno.land/x/canvas@v1.2.2/wasm.js": "192938164d9f0028fdd5813534efeca951793ff89dfdcbfec0104e71d2b6193a", + "https://deno.land/x/canvas@v1.4.1/deps.ts": "e956026d98094946166e06d7b799290b732db015813870d84e04e33ab88e98f3", + "https://deno.land/x/canvas@v1.4.1/mod.ts": "a4e16972647ceafef58612a377a218372454c99d2c9da615a132694597114f80", + "https://deno.land/x/canvas@v1.4.1/src/base64.ts": "0928031fdba0c43b617154fbe2eb7578366460c04da1422933ae5e936d3d0349", + "https://deno.land/x/canvas@v1.4.1/src/canvas.ts": "58119999b04f68ebeed2627485c5c24c5b0c029707edde0b6568814f9049a3a8", + "https://deno.land/x/canvas@v1.4.1/src/canvaskit.ts": "c3d807472cbb3e1d9fc01bb43ff974ef796c4b010178d1595be5fa793cce5e7d", + "https://deno.land/x/canvas@v1.4.1/src/color_util.ts": "28f1072f0a5acbe7add7fac2f452311a47b44c080806fc4057de2d2e405c6c1c", + "https://deno.land/x/canvas@v1.4.1/src/lib.js": "bb21711589bfbc8997b455cdf53e3150e23289f3b44809188041b1d2fc7924fa", + "https://deno.land/x/canvas@v1.4.1/src/types.ts": "67d5800f8f4b0a407e0251676a03ae91b5f50a3ed53e6b72dc5984113cb93128", + "https://deno.land/x/canvas@v1.4.1/src/wasm.js": "449d72cc14fc4142a5853f944df49a744d852981d09c5515528ede8aebb0afda", + "https://deno.land/x/deno_chart@1.1.0/config/CanvasInstance.ts": "d8439c6e86299ab57f470e78f6e5dcda43a40e0c6b3139ab4f5ec687d263c0e6", + "https://deno.land/x/deno_chart@1.1.0/config/index.ts": "754a3fd0cc4ac33a194ca62d5aca915ab0c86c3c28823b470f04d2be60f9a1a6", + "https://deno.land/x/deno_chart@1.1.0/deps.ts": "525a6d767c6c073125a9bd2a0b447c39dca2dea635835913c1711891d80fbc1e", + "https://deno.land/x/deno_chart@1.1.0/graph/GraphInstance.ts": "07d373ad6971b6ed7e130bb5fa5c24a490be04b3fd79d143405432b8c3bbcfe0", + "https://deno.land/x/deno_chart@1.1.0/graph/index.ts": "a18a2ddc1cae8e8dc2448505508fd69ad629aa83fdc07bda9c57e16362fc8fbb", + "https://deno.land/x/deno_chart@1.1.0/mod.ts": "3ae79662d47fcccd34bd2243a72b87f4a2ee8d7ad11eeafc4415d4fabc56ddaa", + "https://deno.land/x/deno_chart@1.1.0/utils/Helpers.ts": "3b84c657e51d92adc3aa648f0b65bd5fab5ded82471f266a795eb1cac670ba6d", + "https://deno.land/x/deno_chart@1.1.0/utils/index.ts": "99729fe252672869e2f39d0517fcc777f8826285d5ffa018b1ef6bbc6d294ff1", + "https://deno.land/x/denouse@v0.0.6/mod.ts": "f3d2d16c6a9338b69ccd9a8446a19452e09d8c9b8129f79ef7eb5669ca613e0b", + "https://deno.land/x/denouse@v0.0.6/use/array/mod.ts": "a95df5671fcb09123d6e8a9aaa46ff2f935ad43966f3dc4d32051da31577ed0f", + "https://deno.land/x/denouse@v0.0.6/use/array/range.ts": "a72d2355492889e739a1987b5a3a6c859ac71f2601eb67138b58e6983a24d9d1", + "https://deno.land/x/denouse@v0.0.6/use/array/split.ts": "f0c8b4ecd5fe7dfd8a0bb6aef21885220a73b2b139c3b0ee36f9ecc814cad11e", + "https://deno.land/x/denouse@v0.0.6/use/array/unique.ts": "d81d591c077ad81123ecf612445bcfc657a409b0e10a1667c184c675d7ee895a", + "https://deno.land/x/denouse@v0.0.6/use/datetime/duration.ts": "ad1704be1a47b75d581e8030a677b881c92d8f4a23348669066c01b1e478ed19", + "https://deno.land/x/denouse@v0.0.6/use/datetime/mod.ts": "30f93d011754ff647495d38fa38eb9790ae1c55258df9ba5ee615b96dfd30958", + "https://deno.land/x/denouse@v0.0.6/use/datetime/time.ts": "e67edba8d60c682ea87fd6457a6460fbbc92a61e78afa41f0536bab8c26b38ac", + "https://deno.land/x/denouse@v0.0.6/use/mod.ts": "48c15a8d7a498353cc0cc6162d83630d6eda711663db9bea848c63c9ec3174c8", + "https://deno.land/x/denouse@v0.0.6/use/random/mod.ts": "8b1d03fd04bad86ece5c1b8b2efa20f56231d9a8bac962cf10571d321d989116", + "https://deno.land/x/denouse@v0.0.6/use/random/normal.ts": "c6f0bac64e0e147f8df6f9882eeb7c0bc335bd7a00e06897c6147118333af497", + "https://deno.land/x/denouse@v0.0.6/use/random/rearrange.ts": "1c1571664845896663b563029f3f2572fccd38c6d92ec3fd11dc6d7ea866cb4b", + "https://deno.land/x/denouse@v0.0.6/use/random/rng.ts": "213dff1b1a39ae4621d35330eb19a3f96170949e8ca429e8028a1fdf86c61f22", + "https://deno.land/x/denouse@v0.0.6/use/random/shuffle.ts": "66f966851d5b6dd4c81117d32311742c319ea3042aad22a7874d2b10da4e743f", + "https://deno.land/x/denouse@v0.0.6/use/random/weighted.ts": "c4eeb309477ecf431a0b9fd1bf0f04540825194c7ed30ba8c71c9e45e14112e6", + "https://deno.land/x/fetchbase64@1.0.0/deps.ts": "2f44013a40540650b346d36b6aae141207b39426fca02ec5292d9d4d5f33f4ac", + "https://deno.land/x/fetchbase64@1.0.0/mod.ts": "f4551a2ecf41f560dcd12c6c8cf65b5246458b2b9b2b2836693a3410b15ca82f", + "https://deno.land/x/fetchbase64@1.0.0/src/auto.ts": "77a20763dd250b71af0ff7abe11a37722c0ee637cd1feafe4afd081f7d04e868", + "https://deno.land/x/fetchbase64@1.0.0/src/local.ts": "53a9d0157419f3bc1aac6440549adac529c475042d2391532618242a883f96c9", + "https://deno.land/x/fetchbase64@1.0.0/src/remote.ts": "95320d1e2cba524099f18a446195388de31281e713ec6c86e46f9d20bb6db736", + "https://deno.land/x/huggingface@0.2.4/tokenizers/mod.ts": "7402637c83e95bfce3a83f4ffd10733a6b3587d329ea281a383dd4049a8a6384", + "https://deno.land/x/huggingface@0.2.4/tokenizers/src/wasm.js": "d797f93da26460b667add22fae682f566790d7f1360a83cf4104f42458d328c3", + "https://deno.land/x/lz4@v0.1.2/mod.ts": "4decfc1a3569d03fd1813bd39128b71c8f082850fe98ecfdde20025772916582", + "https://deno.land/x/lz4@v0.1.2/wasm.js": "b9c65605327ba273f0c76a6dc596ec534d4cda0f0225d7a94ebc606782319e46", + "https://deno.land/x/netsaur@0.2.8/deps.ts": "ecf8d69bb639cea2aeac1c69730b5a0f5f5fd3518090449c027176cd0fdc4415", + "https://deno.land/x/netsaur@0.2.8/mod.ts": "ed16d242a2792677c47d0082b82321090eaf64e262e017e741373789bcc6c11e", + "https://deno.land/x/netsaur@0.2.8/src/backend_cpu/backend.ts": "5ef0911e6fcd682b891dc64173cafde76b141490c73cedcf66d9ba152cf87ec0", + "https://deno.land/x/netsaur@0.2.8/src/backend_cpu/mod.ts": "5871a2e5b7f6cf7619294d8761080f19ba5de1a6f55ef6e08b76a1b594b370f8", + "https://deno.land/x/netsaur@0.2.8/src/backend_cpu/util.ts": "fd51f7868ed9d7eddeec01559b60684f2898ff72a2abe11d6a2267f63239c6d0", + "https://deno.land/x/netsaur@0.2.8/src/backend_wasm/backend.ts": "1a8957ecb3219d0b02e7c3144ab6619a0d79ef548001838c7fa09f3cb8e767ac", + "https://deno.land/x/netsaur@0.2.8/src/backend_wasm/lib/netsaur.generated.js": "f91e0dd8528a13f6bf91a98908ece7d9192d2a4918180a448b98a2e967033e54", + "https://deno.land/x/netsaur@0.2.8/src/backend_wasm/mod.ts": "b07ff70c68ce179cb8a3ab7d28135fe7f0ecc727c0dbd34857a376cae17f84d4", + "https://deno.land/x/netsaur@0.2.8/src/backend_wasm/utils.ts": "cbbca54c7fa37f6cf71375833cec37d367fb0bc677af1cfa913e2832d7f1f475", + "https://deno.land/x/netsaur@0.2.8/src/core/api/error.ts": "b58811e114b9ffc9d5fa77ddca7a72e3b21d009efb060440de5b5ea5f630b0ef", + "https://deno.land/x/netsaur@0.2.8/src/core/api/layer.ts": "27c34b83c677ce2e92cf7d31211925b1df301dc19c0c76d04e283eb2e102cd34", + "https://deno.land/x/netsaur@0.2.8/src/core/api/layers.ts": "5120bd93eff110999da895172a9e8a4f77b49fc5aa26b5f88aa55678d300f696", + "https://deno.land/x/netsaur@0.2.8/src/core/api/network.ts": "de3ee840b0b169a394accfd28abcac699c731853198eda920315895a4c8ce538", + "https://deno.land/x/netsaur@0.2.8/src/core/api/optimizer.ts": "30d5f2f6e7a469910e6735dee1bb3e06e2c8ebaed48d6c55d4662bb998b473f9", + "https://deno.land/x/netsaur@0.2.8/src/core/api/shape.ts": "a65b381937751ab5e0016fb5c8afb023aff480f39612f77d200fea85249026d4", + "https://deno.land/x/netsaur@0.2.8/src/core/engine.ts": "d63750b7b6e9d5f4cffe45d47859ad2a8f3279c682fdb4c646a8f263df55c9ec", + "https://deno.land/x/netsaur@0.2.8/src/core/mod.ts": "a12c92154b5f5189eed295f3395fb6048412a88bd298ad4f6c14b2b7490ee534", + "https://deno.land/x/netsaur@0.2.8/src/core/tensor/tensor.ts": "2db0c0e5b6cdece8c50b36cc8d683c73fd38cccb101ce12375f1bf2c8859155b", + "https://deno.land/x/netsaur@0.2.8/src/core/tensor/util.ts": "d76a96380354085992731599eaec98f7cc59c2af821e03531b72296277249b34", + "https://deno.land/x/netsaur@0.2.8/src/core/types.ts": "4a2f33769cded405cfb134609cbc906202180f9d87acb3033187d59da4b23b75", + "https://deno.land/x/plug@1.0.0-rc.3/deps.ts": "ea73ca429e71624fdae0fe9393fe5056b9404d058ef45688eda4150112cf85d3", + "https://deno.land/x/plug@1.0.0-rc.3/download.ts": "81d1aef0595e018514f1d70a5c52b0267d81685e2dccb5459f8427b29f3f3736", + "https://deno.land/x/plug@1.0.0-rc.3/mod.ts": "6db7edf982efedf8c9e1002754fbb109c22201fd8fb89ba715fe70171b5e821c", + "https://deno.land/x/plug@1.0.0-rc.3/types.ts": "d8eb738fc6ed883e6abf77093442c2f0b71af9090f15c7613621d4039e410ee1", + "https://deno.land/x/plug@1.0.0-rc.3/util.ts": "e53018a487292d11036efeae9da81eb82ef47f9e53d0ba83f545b0220350155a", "https://deno.land/x/plug@1.0.2/deps.ts": "36846a478fafaa1d8ca18aafd91584a748aa58de47ad0f45f008881dad82f374", "https://deno.land/x/plug@1.0.2/download.ts": "b92bc1c1ae35fdb75828847f1ebfc7e51bf462f339729740e1cffe78384e1509", "https://deno.land/x/plug@1.0.2/mod.ts": "32e0006ed6142e7becdb4103c2aa4e1e9ef28459d7243d6cb404a028f7c4eb7e", "https://deno.land/x/plug@1.0.2/types.ts": "0490359117c53783138f2b6692a34f85bca9237314ba8cdef8ad682d81218d21", - "https://deno.land/x/plug@1.0.2/util.ts": "ded3db6e9bb16b8003899d9073fb310e13231ca617b35d6b7dfd53f38762cc76" + "https://deno.land/x/plug@1.0.2/util.ts": "ded3db6e9bb16b8003899d9073fb310e13231ca617b35d6b7dfd53f38762cc76", + "https://deno.land/x/pngs@0.1.1/mod.ts": "9dc8a7daed1497b94a77b68c954164a9f0b2a6f40866481bdfdbbaf015b5f764", + "https://deno.land/x/pngs@0.1.1/wasm.js": "e3d4a8f293b267c9859a2164ca7b4603869bc92fe0d5ad4f109925858bce0c4c", + "https://deno.land/x/skia_canvas@0.5.1/deno.json": "21a1509b7fd7f8f03fe7157135e2456918f5032c99b8916a983e37a34f18ca40", + "https://deno.land/x/skia_canvas@0.5.1/deps.ts": "34e46a90f0477bc96e22a771c335d4fadeb80bf220f25864d52bfa763f86b68b", + "https://deno.land/x/skia_canvas@0.5.1/mod.ts": "62b65707c46334f15f94647be6280c818d90d9976ef8ca70eb607954b9539e5e", + "https://deno.land/x/skia_canvas@0.5.1/src/canvas.ts": "20a84b2a4b6a3e9960ec1ab017069c37b10d38fa8c69c2fbcb5daed480c54748", + "https://deno.land/x/skia_canvas@0.5.1/src/context2d.ts": "42589da0e6bb5d11bc3cbd5c5e8df08a44b09df757a275dd980a6a395588158a", + "https://deno.land/x/skia_canvas@0.5.1/src/dommatrix.ts": "1f09349bceb483deb2dffa514c8a940a7cd7d61a6a6d0269a3f9837ce5cbe7d6", + "https://deno.land/x/skia_canvas@0.5.1/src/ffi.ts": "f8fe2512d8f57a70a3fc8aa0e536dd5846aec299d76affdc2842c7beaee72f99", + "https://deno.land/x/skia_canvas@0.5.1/src/filter.ts": "ef6ee5d0325678b9b4bf9bdc02635a7cbf4e9f5af040490818e072a6b5b68cb3", + "https://deno.land/x/skia_canvas@0.5.1/src/font.ts": "26fc6b01549639a6382b64d13834c3f25ad4a52796adf4edab766b9409683e1a", + "https://deno.land/x/skia_canvas@0.5.1/src/gradient.ts": "9da3f7f09d1e45cd4da811cb796fdd60d80743788ed15e0a08105003d10ad3bb", + "https://deno.land/x/skia_canvas@0.5.1/src/image.ts": "71681f42d330647594addd5235e1d694c55981d441be8ac45afba2fc9d3e8dec", + "https://deno.land/x/skia_canvas@0.5.1/src/parse_font.ts": "c395bcc38f76b38b8c9d2fdbf3d37a9813962bfd6d632f6e58dea89df2eab713", + "https://deno.land/x/skia_canvas@0.5.1/src/path2d.ts": "f7dd1e5d134ae6f669c7836dcf379610da9b12514472f754a5aa71ac2dd1cfd7", + "https://deno.land/x/skia_canvas@0.5.1/src/pattern.ts": "ac63815273ebd2cc5eb90cce067dabf9f1f5dba1fe9cd38951c7f1d74dbc9f5f", + "https://deno.land/x/skia_canvas@0.5.1/src/pdfdocument.ts": "1e325bc6c0178109be8cdb929a1362a928e0fad9725de82402a9011da0db4412", + "https://deno.land/x/skia_canvas@0.5.1/src/svgcanvas.ts": "e788d4a9599c660f0aab85ff27b63b651a690f059522f2bb2bd2fa1821a37f7f" } } diff --git a/examples/tokenizers/basic.ts b/examples/tokenizers/basic.ts new file mode 100644 index 0000000..4245000 --- /dev/null +++ b/examples/tokenizers/basic.ts @@ -0,0 +1,11 @@ +import { init, Tokenizer } from "../../tokenizers/mod.ts"; + +await init(); + +const tokenizer = Tokenizer.fromJson( + await (await fetch( + `https://huggingface.co/satvikag/chatbot/resolve/main/tokenizer.json`, + )).text(), +); + +console.log(tokenizer.tokenize("Hello World!")); \ No newline at end of file diff --git a/examples/visualize.ipynb b/examples/visualize.ipynb new file mode 100644 index 0000000..2ee3bff --- /dev/null +++ b/examples/visualize.ipynb @@ -0,0 +1,151 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.plotly.v1+json": { + "data": [ + { + "line": { + "color": "blue", + "width": 3 + }, + "mode": "lines+markers", + "name": "Expected", + "type": "scatter", + "x": [ + 1, + 2, + 3, + 4 + ], + "y": [ + 0, + 1, + 1, + 0 + ] + }, + { + "line": { + "color": "red", + "width": 3 + }, + "mode": "lines+markers", + "name": "Results", + "type": "scatter", + "x": [ + 1, + 2, + 3, + 4 + ], + "y": [ + 0.007869369350373745, + 0.9794451594352722, + 0.9797343611717224, + 0.024822847917675972 + ] + } + ], + "layout": { + "title": "XOR Example" + } + } + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "WASM Backend Initialized\n" + ] + } + ], + "source": [ + "import { tensor1D } from \"https://deno.land/x/netsaur@0.2.9/mod.ts\";\n", + "import { Visualizer } from \"https://deno.land/x/netsaur@0.2.9/visualizer/mod.ts\";\n", + "\n", + "import {\n", + " Cost,\n", + " WASM,\n", + " DenseLayer,\n", + " Sequential,\n", + " setupBackend,\n", + " SigmoidLayer,\n", + " tensor2D,\n", + "} from \"../mod.ts\";\n", + " \n", + "await setupBackend(WASM);\n", + "\n", + "const net = new Sequential({\n", + " size: [4, 2],\n", + "\n", + " silent: true,\n", + "\n", + " layers: [\n", + " DenseLayer({ size: [3] }),\n", + " SigmoidLayer(),\n", + " DenseLayer({ size: [1] }),\n", + " SigmoidLayer(),\n", + " ],\n", + "\n", + " cost: Cost.MSE,\n", + "});\n", + "\n", + "net.train(\n", + " [\n", + " {\n", + " inputs: tensor2D([\n", + " [0, 0],\n", + " [1, 0],\n", + " [0, 1],\n", + " [1, 1],\n", + " ]),\n", + " outputs: tensor2D([[0], [1], [1], [0]]),\n", + " },\n", + " ],\n", + " 50000,\n", + ");\n", + " \n", + "const visualizer = new Visualizer(\"XOR Example\");\n", + "await visualizer.graph(net,\n", + " [\n", + " tensor1D([0, 0]),\n", + " tensor1D([1, 0]),\n", + " tensor1D([0, 1]),\n", + " tensor1D([1, 1]),\n", + "], [\n", + " tensor1D([0]),\n", + " tensor1D([1]),\n", + " tensor1D([1]),\n", + " tensor1D([0])\n", + "])" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Deno", + "language": "typescript", + "name": "deno" + }, + "language_info": { + "file_extension": ".ts", + "mimetype": "text/x.typescript", + "name": "typescript", + "nb_converter": "script", + "pygments_lexer": "typescript", + "version": "5.2.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/xor.ipynb b/examples/xor.ipynb new file mode 100644 index 0000000..a6df67b --- /dev/null +++ b/examples/xor.ipynb @@ -0,0 +1,280 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### This example shows how to train a neural network to predict the output of the XOR function." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### CPU Backend" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Setup the CPU backend. This backend is fast but doesn't work on the Edge." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[32mDownloading\u001b[39m https://github.com/denosaurs/netsaur/releases/download/0.2.8/netsaur.dll\n", + "CPU Backend Initialised\n" + ] + } + ], + "source": [ + "import { setupBackend, CPU } from \"https://deno.land/x/netsaur@0.2.9/mod.ts\";\n", + "await setupBackend(CPU);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### WASM Backend" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Setup the WASM backend. This backend is blazing fast in the browser or the edge." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "WASM Backend Initialised\n" + ] + } + ], + "source": [ + "import { setupBackend, WASM } from \"https://deno.land/x/netsaur@0.2.9/mod.ts\";\n", + "await setupBackend(WASM);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### GPU Backend" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Eventually this will be the way to setup the GPU backend but it's not ready yet." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import { setupBackend, GPU } from \"https://deno.land/x/netsaur@0.2.9/mod.ts\";\n", + "await setupBackend(GPU);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create a new Sequential neural network. A Sequential model is a linear stack of layers." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import { Sequential, DenseLayer, SigmoidLayer, Cost, tensor2D, tensor1D } from \"https://deno.land/x/netsaur@0.2.9/mod.ts\";\n", + "const net = new Sequential({\n", + " /**\n", + " * The number of minibatches is set to 4 and the output size is set to 2.\n", + " */\n", + " size: [4, 2],\n", + " \n", + " /**\n", + " * The silent option is set to false, which means that the network will output logs during training\n", + " */\n", + " silent: false,\n", + " \n", + " /**\n", + " * Defines the layers of a neural network in the XOR function example.\n", + " * The neural network has two input neurons and one output neuron.\n", + " * The layers are defined as follows:\n", + " * - A dense layer with 3 neurons.\n", + " * - sigmoid activation layer.\n", + " * - A dense layer with 1 neuron.\n", + " * -A sigmoid activation layer.\n", + " */\n", + " layers: [\n", + " DenseLayer({ size: [3] }),\n", + " SigmoidLayer(),\n", + " DenseLayer({ size: [1] }),\n", + " SigmoidLayer(),\n", + " ],\n", + " \n", + " /**\n", + " * The cost function used for training the network is the mean squared error (MSE).\n", + " */\n", + " cost: Cost.MSE,\n", + " });\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Once the network is created and configured we can begin training it. We will train it for 10000 epochs." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "/**\n", + " * Train the network on the given data.\n", + " */\n", + "net.train(\n", + " [\n", + " {\n", + " inputs: tensor2D([\n", + " [0, 0],\n", + " [1, 0],\n", + " [0, 1],\n", + " [1, 1],\n", + " ]),\n", + " outputs: tensor2D([[0], [1], [1], [0]]),\n", + " },\n", + " ],\n", + " /**\n", + " * The number of iterations is set to 10000.\n", + " */\n", + " 10000,\n", + " );" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Once we have trained the network we can test it by passing in the input values into the predict method." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 xor 0 = 0.11617888510227203 (should be close to 0)\n" + ] + } + ], + "source": [ + "const out1 = (await net.predict(tensor1D([0, 0]))).data;\n", + "console.log(`0 xor 0 = ${out1[0]} (should be close to 0)`);" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1 xor 0 = 0.8679627180099487 (should be close to 1)\n" + ] + } + ], + "source": [ + "const out2 = (await net.predict(tensor1D([1, 0]))).data;\n", + "console.log(`1 xor 0 = ${out2[0]} (should be close to 1)`);" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 xor 1 = 0.8693848848342896 (should be close to 1)\n" + ] + } + ], + "source": [ + "const out3 = (await net.predict(tensor1D([0, 1]))).data;\n", + "console.log(`0 xor 1 = ${out3[0]} (should be close to 1)`);" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1 xor 1 = 0.10980527102947235 (should be close to 0)\n" + ] + } + ], + "source": [ + "const out4 = (await net.predict(tensor1D([1, 1]))).data;\n", + "console.log(`1 xor 1 = ${out4[0]} (should be close to 0)`);\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Deno", + "language": "typescript", + "name": "deno" + }, + "language_info": { + "file_extension": ".ts", + "mimetype": "text/x.typescript", + "name": "typescript", + "nb_converter": "script", + "pygments_lexer": "typescript", + "version": "5.2.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/xor_gpu.ts b/examples/xor_gpu.ts new file mode 100644 index 0000000..527a603 --- /dev/null +++ b/examples/xor_gpu.ts @@ -0,0 +1,96 @@ +/** + * This example shows how to train a neural network to predict the output of the XOR function. + */ + +import { + Cost, + GPU, + DenseLayer, + Sequential, + setupBackend, + SigmoidLayer, + tensor1D, + tensor2D, + } from "../mod.ts"; + + /** + * Setup the GPU backend. This backend is fast but doesn't work on the Edge. + */ + await setupBackend(GPU); + + /** + * Creates a sequential neural network. + */ + const net = new Sequential({ + /** + * The number of minibatches is set to 4 and the output size is set to 2. + */ + size: [4, 2], + + /** + * The silent option is set to true, which means that the network will not output any logs during trainin + */ + silent: true, + + /** + * Defines the layers of a neural network in the XOR function example. + * The neural network has two input neurons and one output neuron. + * The layers are defined as follows: + * - A dense layer with 3 neurons. + * - sigmoid activation layer. + * - A dense layer with 1 neuron. + * -A sigmoid activation layer. + */ + layers: [ + DenseLayer({ size: [3] }), + SigmoidLayer(), + DenseLayer({ size: [1] }), + SigmoidLayer(), + ], + + /** + * The cost function used for training the network is the mean squared error (MSE). + */ + cost: Cost.MSE, + }); + + const time = performance.now(); + + /** + * Train the network on the given data. + */ + net.train( + [ + { + inputs: tensor2D([ + [0, 0], + [1, 0], + [0, 1], + [1, 1], + ]), + outputs: tensor2D([[0], [1], [1], [0]]), + }, + ], + /** + * The number of iterations is set to 10000. + */ + 10000, + ); + + console.log(`training time: ${performance.now() - time}ms`); + + /** + * Predict the output of the XOR function for the given inputs. + */ + const out1 = (await net.predict(tensor1D([0, 0]))).data; + console.log(`0 xor 0 = ${out1[0]} (should be close to 0)`); + + const out2 = (await net.predict(tensor1D([1, 0]))).data; + console.log(`1 xor 0 = ${out2[0]} (should be close to 1)`); + + const out3 = (await net.predict(tensor1D([0, 1]))).data; + console.log(`0 xor 1 = ${out3[0]} (should be close to 1)`); + + const out4 = (await net.predict(tensor1D([1, 1]))).data; + console.log(`1 xor 1 = ${out4[0]} (should be close to 0)`); + \ No newline at end of file diff --git a/mod.ts b/mod.ts index d0f5d81..dbe7d66 100644 --- a/mod.ts +++ b/mod.ts @@ -5,9 +5,10 @@ export * from "./src/core/tensor/tensor.ts"; export * from "./src/core/api/layers.ts"; export * from "./src/core/api/shape.ts"; export * from "./src/core/api/network.ts"; +export { GPU } from "./src/backends/gpu/mod.ts"; -import { CPU } from "./src/backend_cpu/mod.ts"; -import { WASM } from "./src/backend_wasm/mod.ts"; +import { CPU } from "./src/backends/cpu/mod.ts"; +import { WASM } from "./src/backends/wasm/mod.ts"; import { BackendLoader } from "./src/core/engine.ts"; /** diff --git a/src/.DS_Store b/src/.DS_Store new file mode 100644 index 0000000..f356cc5 Binary files /dev/null and b/src/.DS_Store differ diff --git a/src/backend_wasm/lib/netsaur_bg.wasm b/src/backend_wasm/lib/netsaur_bg.wasm deleted file mode 100644 index 638e2bf..0000000 Binary files a/src/backend_wasm/lib/netsaur_bg.wasm and /dev/null differ diff --git a/src/backends/.DS_Store b/src/backends/.DS_Store new file mode 100644 index 0000000..a7b4e6c Binary files /dev/null and b/src/backends/.DS_Store differ diff --git a/src/backend_cpu/backend.ts b/src/backends/cpu/backend.ts similarity index 91% rename from src/backend_cpu/backend.ts rename to src/backends/cpu/backend.ts index 60a9dba..4efbbfb 100644 --- a/src/backend_cpu/backend.ts +++ b/src/backends/cpu/backend.ts @@ -1,7 +1,7 @@ -import { Rank, Shape } from "../core/api/shape.ts"; -import { Tensor } from "../core/tensor/tensor.ts"; -import { length } from "../core/tensor/util.ts"; -import { Backend, DataSet, NetworkConfig } from "../core/types.ts"; +import { Rank, Shape } from "../../core/api/shape.ts"; +import { Tensor } from "../../core/tensor/tensor.ts"; +import { length } from "../../core/tensor/util.ts"; +import { Backend, DataSet, NetworkConfig } from "../../core/types.ts"; import { Library } from "./mod.ts"; import { Buffer, diff --git a/src/backend_cpu/mod.ts b/src/backends/cpu/mod.ts similarity index 86% rename from src/backend_cpu/mod.ts rename to src/backends/cpu/mod.ts index b79c324..8fd9476 100644 --- a/src/backend_cpu/mod.ts +++ b/src/backends/cpu/mod.ts @@ -1,9 +1,9 @@ -import { dlopen, FetchOptions } from "../../deps.ts"; +import { dlopen, FetchOptions } from "../../../deps.ts"; import { CPUBackend } from "./backend.ts"; -import { NoBackendError } from "../core/api/error.ts"; -import { BackendLoader, Engine } from "../core/engine.ts"; -import { Backend, BackendType, Cost, NetworkConfig } from "../core/types.ts"; -import { Sequential } from "../core/mod.ts"; +import { NoBackendError } from "../../core/api/error.ts"; +import { BackendLoader, Engine } from "../../core/engine.ts"; +import { Backend, BackendType, Cost, NetworkConfig } from "../../core/types.ts"; +import { Sequential } from "../../core/mod.ts"; const options: FetchOptions = { name: "netsaur", @@ -50,7 +50,7 @@ export class CPUInstance { CPUInstance.library = await dlopen(options, symbols); CPUInstance.initialized = true; - if (!silent) console.log("CPU Backend Initialised"); + if (!silent) console.log("CPU Backend Initialized"); return true; } } diff --git a/src/backend_cpu/util.ts b/src/backends/cpu/util.ts similarity index 92% rename from src/backend_cpu/util.ts rename to src/backends/cpu/util.ts index fa622fe..5d35da5 100644 --- a/src/backend_cpu/util.ts +++ b/src/backends/cpu/util.ts @@ -1,5 +1,5 @@ -import { Rank, Shape } from "../core/api/shape.ts"; -import { DataSet } from "../core/types.ts"; +import { Rank, Shape } from "../../core/api/shape.ts"; +import { DataSet } from "../../core/types.ts"; export class Buffer { buffer = new Uint8Array(); diff --git a/src/backends/gpu/backend.ts b/src/backends/gpu/backend.ts new file mode 100644 index 0000000..f9a5b42 --- /dev/null +++ b/src/backends/gpu/backend.ts @@ -0,0 +1,107 @@ +import { Rank, Shape } from "../../core/api/shape.ts"; +import { Tensor } from "../../core/tensor/tensor.ts"; +import { length } from "../../core/tensor/util.ts"; +import { Backend, DataSet, NetworkConfig } from "../../core/types.ts"; +import { Library } from "./mod.ts"; +import { + Buffer, + encodeDatasets, + encodeJSON, + PredictOptions, + TrainOptions, +} from "./util.ts"; + +/** + * GPU Backend. + */ +export class GPUBackend implements Backend { + library: Library; + outputShape: Shape[Rank]; + #id: bigint; + + constructor( + library: Library, + outputShape: Shape[Rank], + id: bigint, + ) { + this.library = library; + this.outputShape = outputShape; + this.#id = id; + } + + static create(config: NetworkConfig, library: Library) { + const buffer = encodeJSON(config); + const shape = new Buffer(); + const id = library.symbols.ffi_backend_create( + buffer, + buffer.length, + shape.allocBuffer, + ) as bigint; + const outputShape = Array.from(shape.buffer.slice(1)) as Shape[Rank]; + + return new GPUBackend(library, outputShape, id); + } + + train(datasets: DataSet[], epochs: number, batches: number, rate: number) { + const buffer = encodeDatasets(datasets); + const options = encodeJSON({ + datasets: datasets.length, + inputShape: datasets[0].inputs.shape, + outputShape: datasets[0].outputs.shape, + epochs, + batches, + rate, + } as TrainOptions); + + this.library.symbols.ffi_backend_train( + this.#id, + buffer, + buffer.byteLength, + options, + options.byteLength, + ); + } + + //deno-lint-ignore require-await + async predict(input: Tensor): Promise> { + const options = encodeJSON({ + inputShape: [1, ...input.shape], + outputShape: this.outputShape, + } as PredictOptions); + const output = new Float32Array(length(this.outputShape)); + this.library.symbols.ffi_backend_predict( + this.#id, + input.data as Float32Array, + options, + options.length, + output, + ); + return new Tensor(output, this.outputShape); + } + + save(): Uint8Array { + const shape = new Buffer(); + this.library.symbols.ffi_backend_save(this.#id, shape.allocBuffer); + return shape.buffer; + } + + saveFile(path: string): void { + Deno.writeFileSync(path, this.save()); + } + + static load(buffer: Uint8Array, library: Library): GPUBackend { + const shape = new Buffer(); + const id = library.symbols.ffi_backend_load( + buffer, + buffer.length, + shape.allocBuffer, + ) as bigint; + const outputShape = Array.from(shape.buffer.slice(1)) as Shape[Rank]; + + return new GPUBackend(library, outputShape, id); + } + + static loadFile(path: string, library: Library): GPUBackend { + return this.load(Deno.readFileSync(path), library); + } +} diff --git a/src/backends/gpu/mod.ts b/src/backends/gpu/mod.ts new file mode 100644 index 0000000..7e34749 --- /dev/null +++ b/src/backends/gpu/mod.ts @@ -0,0 +1,97 @@ +import { dlopen, FetchOptions } from "../../../deps.ts"; +import { GPUBackend } from "./backend.ts"; +import { NoBackendError } from "../../core/api/error.ts"; +import { BackendLoader, Engine } from "../../core/engine.ts"; +import { Backend, BackendType, Cost, NetworkConfig } from "../../core/types.ts"; +import { Sequential } from "../../core/mod.ts"; + +const options: FetchOptions = { + name: "netsaur_gpu", + url: new URL(import.meta.url).protocol !== "file:" + ? new URL( + "https://github.com/denosaurs/netsaur/releases/download/0.2.8/", + import.meta.url, + ) + : "./target/release/", + cache: "reloadAll", +}; + +const symbols = { + ffi_backend_create: { + parameters: ["buffer", "usize", "pointer"], + result: "usize", + } as const, + ffi_backend_train: { + parameters: ["usize", "buffer", "usize", "buffer", "usize"], + result: "void", + } as const, + ffi_backend_predict: { + parameters: ["usize", "buffer", "buffer", "usize", "buffer"], + result: "void", + } as const, + ffi_backend_save: { + parameters: ["usize", "pointer"], + result: "void", + } as const, + ffi_backend_load: { + parameters: ["buffer", "usize", "pointer"], + result: "usize", + } as const, +}; + +export type Library = Deno.DynamicLibrary; + +export class GPUInstance { + static library?: Library; + static initialized = false; + + static async init(silent = false) { + if (GPUInstance.initialized) return true; + + GPUInstance.library = await dlopen(options, symbols); + GPUInstance.initialized = true; + if (!silent) console.log("GPU Backend Initialized"); + return true; + } +} + +export class GPUBackendLoader implements BackendLoader { + backend?: GPUBackend; + + isSupported(): boolean { + return Deno.dlopen !== undefined; + } + + async setup(silent = false) { + Engine.type = BackendType.GPU; + return await GPUInstance.init(silent); + } + + loadBackend(config: NetworkConfig): Backend { + if (!GPUInstance.initialized) { + throw new NoBackendError(BackendType.GPU); + } + return this.backend + ? this.backend + : GPUBackend.create(config, GPUInstance.library!); + } + + load(buffer: Uint8Array): Sequential { + this.backend = GPUBackend.load(buffer, GPUInstance.library!); + const net = new Sequential({ size: [0], layers: [], cost: Cost.MSE }); + this.backend = undefined; + return net; + } + + loadFile(path: string): Sequential { + this.backend = GPUBackend.loadFile(path, GPUInstance.library!); + const net = new Sequential({ size: [0], layers: [], cost: Cost.MSE }); + this.backend = undefined; + return net; + } +} + +/** + * GPU Backend written in Rust. + */ +export const GPU = new GPUBackendLoader(); diff --git a/src/backends/gpu/util.ts b/src/backends/gpu/util.ts new file mode 100644 index 0000000..5d35da5 --- /dev/null +++ b/src/backends/gpu/util.ts @@ -0,0 +1,59 @@ +import { Rank, Shape } from "../../core/api/shape.ts"; +import { DataSet } from "../../core/types.ts"; + +export class Buffer { + buffer = new Uint8Array(); + allocBuffer = new Deno.UnsafeCallback({ + parameters: ["usize"], + result: "buffer", + }, (length) => { + this.buffer = new Uint8Array(Number(length)); + return this.buffer; + }).pointer; +} + +/** + * Train Options Interface. + */ +export type TrainOptions = { + datasets: number; + inputShape: Shape[Rank]; + outputShape: Shape[Rank]; + epochs: number; + batches: number; + rate: number; +}; + +/** + * Predict Options Interface. + */ +export type PredictOptions = { + inputShape: Shape[Rank]; + outputShape: Shape[Rank]; +}; + +/** + * Encode JSON data. + */ +export function encodeJSON(json: unknown) { + return new TextEncoder().encode(JSON.stringify(json)); +} + +/** + * Returns the BigInt value of a pointer. + */ +export function pointer(arr: BufferSource) { + return BigInt(Deno.UnsafePointer.value(Deno.UnsafePointer.of(arr))); +} + +/** + * Encode datasets. + */ +export function encodeDatasets(datasets: DataSet[]) { + const pointers: bigint[] = []; + for (const dataset of datasets) { + pointers.push(pointer(dataset.inputs.data as Float32Array)); + pointers.push(pointer(dataset.outputs.data as Float32Array)); + } + return new BigUint64Array(pointers); +} diff --git a/src/backend_wasm/backend.ts b/src/backends/wasm/backend.ts similarity index 92% rename from src/backend_wasm/backend.ts rename to src/backends/wasm/backend.ts index 4eb415a..72886bf 100644 --- a/src/backend_wasm/backend.ts +++ b/src/backends/wasm/backend.ts @@ -1,6 +1,6 @@ -import { Rank, Shape } from "../core/api/shape.ts"; -import { Tensor } from "../core/tensor/tensor.ts"; -import { Backend, DataSet, NetworkConfig } from "../core/types.ts"; +import { Rank, Shape } from "../../core/api/shape.ts"; +import { Tensor } from "../../core/tensor/tensor.ts"; +import { Backend, DataSet, NetworkConfig } from "../../core/types.ts"; import { wasm_backend_create, wasm_backend_load, diff --git a/src/backend_wasm/lib/netsaur.generated.js b/src/backends/wasm/lib/netsaur.generated.js similarity index 98% rename from src/backend_wasm/lib/netsaur.generated.js rename to src/backends/wasm/lib/netsaur.generated.js index 3985b33..e50fb58 100644 --- a/src/backend_wasm/lib/netsaur.generated.js +++ b/src/backends/wasm/lib/netsaur.generated.js @@ -1,7 +1,7 @@ // @generated file from wasmbuild -- do not edit // deno-lint-ignore-file // deno-fmt-ignore-file -// source-hash: 4ef76dfd7d7fcf8612b0d01b5365bc93bcb6327d +// source-hash: cd2ea424aa05fea9255bfa891fb3409cad9fde77 let wasm; let cachedInt32Memory0; @@ -203,20 +203,30 @@ const imports = { __wbindgen_object_drop_ref: function (arg0) { takeObject(arg0); }, - __wbg_log_9e8bb240c2e49b91: function (arg0, arg1) { + __wbg_log_94d5088905279a11: function (arg0, arg1) { console.log(getStringFromWasm0(arg0, arg1)); }, __wbindgen_number_new: function (arg0) { const ret = arg0; return addHeapObject(ret); }, + __wbg_getRandomValues_37fa2ca9e4e07fab: function () { + return handleError(function (arg0, arg1) { + getObject(arg0).getRandomValues(getObject(arg1)); + }, arguments); + }, + __wbg_randomFillSync_dc1e9a60c158336d: function () { + return handleError(function (arg0, arg1) { + getObject(arg0).randomFillSync(takeObject(arg1)); + }, arguments); + }, __wbg_crypto_c48a774b022d20ac: function (arg0) { const ret = getObject(arg0).crypto; return addHeapObject(ret); }, __wbindgen_is_object: function (arg0) { const val = getObject(arg0); - const ret = typeof (val) === "object" && val !== null; + const ret = typeof val === "object" && val !== null; return ret; }, __wbg_process_298734cf255a885d: function (arg0) { @@ -253,16 +263,6 @@ const imports = { const ret = getStringFromWasm0(arg0, arg1); return addHeapObject(ret); }, - __wbg_getRandomValues_37fa2ca9e4e07fab: function () { - return handleError(function (arg0, arg1) { - getObject(arg0).getRandomValues(getObject(arg1)); - }, arguments); - }, - __wbg_randomFillSync_dc1e9a60c158336d: function () { - return handleError(function (arg0, arg1) { - getObject(arg0).randomFillSync(takeObject(arg1)); - }, arguments); - }, __wbg_newnoargs_2b8b6bd7753c76ba: function (arg0, arg1) { const ret = new Function(getStringFromWasm0(arg0, arg1)); return addHeapObject(ret); diff --git a/src/backends/wasm/lib/netsaur_bg.wasm b/src/backends/wasm/lib/netsaur_bg.wasm new file mode 100644 index 0000000..576f6a3 Binary files /dev/null and b/src/backends/wasm/lib/netsaur_bg.wasm differ diff --git a/src/backend_wasm/mod.ts b/src/backends/wasm/mod.ts similarity index 84% rename from src/backend_wasm/mod.ts rename to src/backends/wasm/mod.ts index 4234094..4d70686 100644 --- a/src/backend_wasm/mod.ts +++ b/src/backends/wasm/mod.ts @@ -1,9 +1,9 @@ import { WASMBackend } from "./backend.ts"; -import { NoBackendError } from "../core/api/error.ts"; -import { BackendLoader, Engine } from "../core/engine.ts"; -import { Backend, BackendType, Cost, NetworkConfig } from "../core/types.ts"; +import { NoBackendError } from "../../core/api/error.ts"; +import { BackendLoader, Engine } from "../../core/engine.ts"; +import { Backend, BackendType, Cost, NetworkConfig } from "../../core/types.ts"; import { instantiate } from "./lib/netsaur.generated.js"; -import { Sequential } from "../core/mod.ts"; +import { Sequential } from "../../core/mod.ts"; /** * Web Assembly backend instance. @@ -22,7 +22,7 @@ export class WASMInstance { : undefined, }); WASMInstance.initialized = true; - if (!silent) console.log("WASM Backend Initialised"); + if (!silent) console.log("WASM Backend Initialized"); return true; } } diff --git a/src/backend_wasm/utils.ts b/src/backends/wasm/utils.ts similarity index 85% rename from src/backend_wasm/utils.ts rename to src/backends/wasm/utils.ts index fa950d2..08bc9fe 100644 --- a/src/backend_wasm/utils.ts +++ b/src/backends/wasm/utils.ts @@ -1,4 +1,4 @@ -import { Rank, Shape } from "../core/api/shape.ts"; +import { Rank, Shape } from "../../core/api/shape.ts"; /** * Train Options Interface. diff --git a/src/core/api/scheduler.ts b/src/core/api/scheduler.ts new file mode 100644 index 0000000..225f91b --- /dev/null +++ b/src/core/api/scheduler.ts @@ -0,0 +1,41 @@ +import { SchedulerType } from "../types.ts"; + +export type Scheduler = + | { type: SchedulerType.None } + | { + type: SchedulerType.LinearDecay | SchedulerType.ExponentialDecay; + config: DecaySchedulerConfig; + } + | { type: SchedulerType.OneCycle; config: OneCycleSchedulerConfig }; + +export type DecaySchedulerConfig = { + rate?: number; + step_size?: number; +}; + +export type OneCycleSchedulerConfig = { + max_rate?: number; + step_size?: number; +}; + +export function NoScheduler(): Scheduler { + return { type: SchedulerType.None }; +} + +export function LinearDecay(config: DecaySchedulerConfig = {}): Scheduler { + config.rate = config.rate || 0.99; + config.step_size = config.step_size || 100; + return { type: SchedulerType.LinearDecay, config }; +} + +export function ExponentialDecay(config: DecaySchedulerConfig = {}): Scheduler { + config.rate = config.rate || 0.99; + config.step_size = config.step_size || 100; + return { type: SchedulerType.ExponentialDecay, config }; +} + +export function OneCycle(config: OneCycleSchedulerConfig = {}): Scheduler { + config.max_rate = config.max_rate || 0.01; + config.step_size = config.step_size || 100; + return { type: SchedulerType.OneCycle, config }; + } diff --git a/src/core/engine.ts b/src/core/engine.ts index 70c30ec..6553aba 100644 --- a/src/core/engine.ts +++ b/src/core/engine.ts @@ -1,4 +1,4 @@ -import { WASM } from "../backend_wasm/mod.ts"; +import { WASM } from "../backends/wasm/mod.ts"; import { Sequential } from "./mod.ts"; import { Backend, BackendType, NetworkConfig } from "./types.ts"; diff --git a/src/core/mod.ts b/src/core/mod.ts index 167b346..ba100d2 100644 --- a/src/core/mod.ts +++ b/src/core/mod.ts @@ -1,4 +1,4 @@ -import { Backend, Cost, DataSet, NetworkConfig } from "./types.ts"; +import { Backend, Cost, DataSet, NetworkConfig, SchedulerType } from "./types.ts"; import { Engine } from "./engine.ts"; import { Rank } from "./api/shape.ts"; import { Tensor } from "./tensor/tensor.ts"; @@ -17,6 +17,7 @@ export class Sequential implements NeuralNetwork { constructor(public config: NetworkConfig) { this.config.cost = this.config.cost || Cost.MSE this.config.optimizer = this.config.optimizer || SGDOptimizer() + this.config.scheduler = this.config.scheduler || { type: SchedulerType.None } this.backend = Engine.backendLoader.loadBackend(this.config); } diff --git a/src/core/types.ts b/src/core/types.ts index 8ae2301..1a36df0 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -2,6 +2,7 @@ import { Tensor } from "./tensor/tensor.ts"; import { Rank, Shape } from "./api/shape.ts"; import { Layer } from "./api/layer.ts"; import { Optimizer } from "./api/optimizer.ts"; +import { Scheduler } from "./api/scheduler.ts"; /** * The Backend is responsible for eveything related to the neural network. @@ -58,8 +59,16 @@ export type NetworkConfig = { */ cost?: Cost; + /** + * Optimizer to update parameters. + */ optimizer?: Optimizer; + /** + * Learning rate scheduler. + */ + scheduler?: Scheduler; + /** * Whether or not to silence the verbose messages. */ @@ -135,7 +144,14 @@ export enum Cost { export enum OptimizerType { SGD = "sgd", - Adam = "adam" + Adam = "adam", +} + +export enum SchedulerType { + None = "none", + LinearDecay = "lineardecay", + ExponentialDecay = "exponentialdecay", + OneCycle = "onecycle", } /** diff --git a/tokenizers/lib/netsaur_tokenizers.generated.js b/tokenizers/lib/netsaur_tokenizers.generated.js new file mode 100644 index 0000000..54e5437 --- /dev/null +++ b/tokenizers/lib/netsaur_tokenizers.generated.js @@ -0,0 +1,469 @@ +// @generated file from wasmbuild -- do not edit +// deno-lint-ignore-file +// deno-fmt-ignore-file +// source-hash: 64705faeaa7d050716ef5711a8e4eeaf4224b864 +let wasm; + +const heap = new Array(128).fill(undefined); + +heap.push(undefined, null, true, false); + +function getObject(idx) { + return heap[idx]; +} + +let heap_next = heap.length; + +function dropObject(idx) { + if (idx < 132) return; + heap[idx] = heap_next; + heap_next = idx; +} + +function takeObject(idx) { + const ret = getObject(idx); + dropObject(idx); + return ret; +} + +const cachedTextDecoder = new TextDecoder("utf-8", { + ignoreBOM: true, + fatal: true, +}); + +cachedTextDecoder.decode(); + +let cachedUint8Memory0 = null; + +function getUint8Memory0() { + if (cachedUint8Memory0 === null || cachedUint8Memory0.byteLength === 0) { + cachedUint8Memory0 = new Uint8Array(wasm.memory.buffer); + } + return cachedUint8Memory0; +} + +function getStringFromWasm0(ptr, len) { + return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len)); +} + +function addHeapObject(obj) { + if (heap_next === heap.length) heap.push(heap.length + 1); + const idx = heap_next; + heap_next = heap[idx]; + + heap[idx] = obj; + return idx; +} + +let WASM_VECTOR_LEN = 0; + +const cachedTextEncoder = new TextEncoder("utf-8"); + +const encodeString = function (arg, view) { + return cachedTextEncoder.encodeInto(arg, view); +}; + +function passStringToWasm0(arg, malloc, realloc) { + if (realloc === undefined) { + const buf = cachedTextEncoder.encode(arg); + const ptr = malloc(buf.length); + getUint8Memory0().subarray(ptr, ptr + buf.length).set(buf); + WASM_VECTOR_LEN = buf.length; + return ptr; + } + + let len = arg.length; + let ptr = malloc(len); + + const mem = getUint8Memory0(); + + let offset = 0; + + for (; offset < len; offset++) { + const code = arg.charCodeAt(offset); + if (code > 0x7F) break; + mem[ptr + offset] = code; + } + + if (offset !== len) { + if (offset !== 0) { + arg = arg.slice(offset); + } + ptr = realloc(ptr, len, len = offset + arg.length * 3); + const view = getUint8Memory0().subarray(ptr + offset, ptr + len); + const ret = encodeString(arg, view); + + offset += ret.written; + } + + WASM_VECTOR_LEN = offset; + return ptr; +} +/** + * @param {string} json + * @returns {number} + */ +export function wasm_tokenizer_from_json(json) { + const ptr0 = passStringToWasm0( + json, + wasm.__wbindgen_malloc, + wasm.__wbindgen_realloc, + ); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.wasm_tokenizer_from_json(ptr0, len0); + return ret >>> 0; +} + +let cachedInt32Memory0 = null; + +function getInt32Memory0() { + if (cachedInt32Memory0 === null || cachedInt32Memory0.byteLength === 0) { + cachedInt32Memory0 = new Int32Array(wasm.memory.buffer); + } + return cachedInt32Memory0; +} +/** + * @param {number} id + * @param {boolean} pretty + * @returns {string} + */ +export function wasm_tokenizer_save(id, pretty) { + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + wasm.wasm_tokenizer_save(retptr, id, pretty); + var r0 = getInt32Memory0()[retptr / 4 + 0]; + var r1 = getInt32Memory0()[retptr / 4 + 1]; + return getStringFromWasm0(r0, r1); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + wasm.__wbindgen_free(r0, r1); + } +} + +/** + * @returns {number} + */ +export function wasm_bpe_default() { + const ret = wasm.wasm_bpe_default(); + return ret >>> 0; +} + +let cachedUint32Memory0 = null; + +function getUint32Memory0() { + if (cachedUint32Memory0 === null || cachedUint32Memory0.byteLength === 0) { + cachedUint32Memory0 = new Uint32Array(wasm.memory.buffer); + } + return cachedUint32Memory0; +} + +function getArrayU32FromWasm0(ptr, len) { + return getUint32Memory0().subarray(ptr / 4, ptr / 4 + len); +} +/** + * @param {number} id + * @param {string} string + * @returns {Uint32Array} + */ +export function wasm_tokenizer_tokenize(id, string) { + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + const ptr0 = passStringToWasm0( + string, + wasm.__wbindgen_malloc, + wasm.__wbindgen_realloc, + ); + const len0 = WASM_VECTOR_LEN; + wasm.wasm_tokenizer_tokenize(retptr, id, ptr0, len0); + var r0 = getInt32Memory0()[retptr / 4 + 0]; + var r1 = getInt32Memory0()[retptr / 4 + 1]; + var v1 = getArrayU32FromWasm0(r0, r1).slice(); + wasm.__wbindgen_free(r0, r1 * 4); + return v1; + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + } +} + +function handleError(f, args) { + try { + return f.apply(this, args); + } catch (e) { + wasm.__wbindgen_exn_store(addHeapObject(e)); + } +} + +const imports = { + __wbindgen_placeholder__: { + __wbg_crypto_c48a774b022d20ac: function (arg0) { + const ret = getObject(arg0).crypto; + return addHeapObject(ret); + }, + __wbindgen_is_object: function (arg0) { + const val = getObject(arg0); + const ret = typeof val === "object" && val !== null; + return ret; + }, + __wbg_process_298734cf255a885d: function (arg0) { + const ret = getObject(arg0).process; + return addHeapObject(ret); + }, + __wbg_versions_e2e78e134e3e5d01: function (arg0) { + const ret = getObject(arg0).versions; + return addHeapObject(ret); + }, + __wbg_node_1cd7a5d853dbea79: function (arg0) { + const ret = getObject(arg0).node; + return addHeapObject(ret); + }, + __wbindgen_is_string: function (arg0) { + const ret = typeof (getObject(arg0)) === "string"; + return ret; + }, + __wbindgen_object_drop_ref: function (arg0) { + takeObject(arg0); + }, + __wbg_msCrypto_bcb970640f50a1e8: function (arg0) { + const ret = getObject(arg0).msCrypto; + return addHeapObject(ret); + }, + __wbg_require_8f08ceecec0f4fee: function () { + return handleError(function () { + const ret = module.require; + return addHeapObject(ret); + }, arguments); + }, + __wbindgen_is_function: function (arg0) { + const ret = typeof (getObject(arg0)) === "function"; + return ret; + }, + __wbindgen_string_new: function (arg0, arg1) { + const ret = getStringFromWasm0(arg0, arg1); + return addHeapObject(ret); + }, + __wbg_getRandomValues_37fa2ca9e4e07fab: function () { + return handleError(function (arg0, arg1) { + getObject(arg0).getRandomValues(getObject(arg1)); + }, arguments); + }, + __wbg_randomFillSync_dc1e9a60c158336d: function () { + return handleError(function (arg0, arg1) { + getObject(arg0).randomFillSync(takeObject(arg1)); + }, arguments); + }, + __wbg_newnoargs_2b8b6bd7753c76ba: function (arg0, arg1) { + const ret = new Function(getStringFromWasm0(arg0, arg1)); + return addHeapObject(ret); + }, + __wbg_call_95d1ea488d03e4e8: function () { + return handleError(function (arg0, arg1) { + const ret = getObject(arg0).call(getObject(arg1)); + return addHeapObject(ret); + }, arguments); + }, + __wbindgen_object_clone_ref: function (arg0) { + const ret = getObject(arg0); + return addHeapObject(ret); + }, + __wbg_self_e7c1f827057f6584: function () { + return handleError(function () { + const ret = self.self; + return addHeapObject(ret); + }, arguments); + }, + __wbg_window_a09ec664e14b1b81: function () { + return handleError(function () { + const ret = window.window; + return addHeapObject(ret); + }, arguments); + }, + __wbg_globalThis_87cbb8506fecf3a9: function () { + return handleError(function () { + const ret = globalThis.globalThis; + return addHeapObject(ret); + }, arguments); + }, + __wbg_global_c85a9259e621f3db: function () { + return handleError(function () { + const ret = global.global; + return addHeapObject(ret); + }, arguments); + }, + __wbindgen_is_undefined: function (arg0) { + const ret = getObject(arg0) === undefined; + return ret; + }, + __wbg_call_9495de66fdbe016b: function () { + return handleError(function (arg0, arg1, arg2) { + const ret = getObject(arg0).call(getObject(arg1), getObject(arg2)); + return addHeapObject(ret); + }, arguments); + }, + __wbg_buffer_cf65c07de34b9a08: function (arg0) { + const ret = getObject(arg0).buffer; + return addHeapObject(ret); + }, + __wbg_newwithbyteoffsetandlength_9fb2f11355ecadf5: function ( + arg0, + arg1, + arg2, + ) { + const ret = new Uint8Array(getObject(arg0), arg1 >>> 0, arg2 >>> 0); + return addHeapObject(ret); + }, + __wbg_new_537b7341ce90bb31: function (arg0) { + const ret = new Uint8Array(getObject(arg0)); + return addHeapObject(ret); + }, + __wbg_set_17499e8aa4003ebd: function (arg0, arg1, arg2) { + getObject(arg0).set(getObject(arg1), arg2 >>> 0); + }, + __wbg_newwithlength_b56c882b57805732: function (arg0) { + const ret = new Uint8Array(arg0 >>> 0); + return addHeapObject(ret); + }, + __wbg_subarray_7526649b91a252a6: function (arg0, arg1, arg2) { + const ret = getObject(arg0).subarray(arg1 >>> 0, arg2 >>> 0); + return addHeapObject(ret); + }, + __wbindgen_throw: function (arg0, arg1) { + throw new Error(getStringFromWasm0(arg0, arg1)); + }, + __wbindgen_memory: function () { + const ret = wasm.memory; + return addHeapObject(ret); + }, + }, +}; + +/** + * Decompression callback + * + * @callback DecompressCallback + * @param {Uint8Array} compressed + * @return {Uint8Array} decompressed + */ + +/** + * Options for instantiating a Wasm instance. + * @typedef {Object} InstantiateOptions + * @property {URL=} url - Optional url to the Wasm file to instantiate. + * @property {DecompressCallback=} decompress - Callback to decompress the + * raw Wasm file bytes before instantiating. + */ + +/** Instantiates an instance of the Wasm module returning its functions. + * @remarks It is safe to call this multiple times and once successfully + * loaded it will always return a reference to the same object. + * @param {InstantiateOptions=} opts + */ +export async function instantiate(opts) { + return (await instantiateWithInstance(opts)).exports; +} + +let instanceWithExports; +let lastLoadPromise; + +/** Instantiates an instance of the Wasm module along with its exports. + * @remarks It is safe to call this multiple times and once successfully + * loaded it will always return a reference to the same object. + * @param {InstantiateOptions=} opts + * @returns {Promise<{ + * instance: WebAssembly.Instance; + * exports: { wasm_tokenizer_from_json: typeof wasm_tokenizer_from_json; wasm_tokenizer_save: typeof wasm_tokenizer_save; wasm_bpe_default: typeof wasm_bpe_default; wasm_tokenizer_tokenize: typeof wasm_tokenizer_tokenize } + * }>} + */ +export function instantiateWithInstance(opts) { + if (instanceWithExports != null) { + return Promise.resolve(instanceWithExports); + } + if (lastLoadPromise == null) { + lastLoadPromise = (async () => { + try { + const instance = (await instantiateModule(opts ?? {})).instance; + wasm = instance.exports; + cachedInt32Memory0 = new Int32Array(wasm.memory.buffer); + cachedUint8Memory0 = new Uint8Array(wasm.memory.buffer); + instanceWithExports = { + instance, + exports: getWasmInstanceExports(), + }; + return instanceWithExports; + } finally { + lastLoadPromise = null; + } + })(); + } + return lastLoadPromise; +} + +function getWasmInstanceExports() { + return { + wasm_tokenizer_from_json, + wasm_tokenizer_save, + wasm_bpe_default, + wasm_tokenizer_tokenize, + }; +} + +/** Gets if the Wasm module has been instantiated. */ +export function isInstantiated() { + return instanceWithExports != null; +} + +/** + * @param {InstantiateOptions} opts + */ +async function instantiateModule(opts) { + const wasmUrl = opts.url ?? + new URL("netsaur_tokenizers_bg.wasm", import.meta.url); + const decompress = opts.decompress; + const isFile = wasmUrl.protocol === "file:"; + + // make file urls work in Node via dnt + const isNode = globalThis.process?.versions?.node != null; + if (isNode && isFile) { + // the deno global will be shimmed by dnt + const wasmCode = await Deno.readFile(wasmUrl); + return WebAssembly.instantiate( + decompress ? decompress(wasmCode) : wasmCode, + imports, + ); + } + + switch (wasmUrl.protocol) { + case "file:": + case "https:": + case "http:": { + if (isFile) { + if (typeof Deno !== "object") { + throw new Error("file urls are not supported in this environment"); + } + if ("permissions" in Deno) { + await Deno.permissions.request({ name: "read", path: wasmUrl }); + } + } else if (typeof Deno === "object" && "permissions" in Deno) { + await Deno.permissions.request({ name: "net", host: wasmUrl.host }); + } + const wasmResponse = await fetch(wasmUrl); + if (decompress) { + const wasmCode = new Uint8Array(await wasmResponse.arrayBuffer()); + return WebAssembly.instantiate(decompress(wasmCode), imports); + } + if ( + isFile || + wasmResponse.headers.get("content-type")?.toLowerCase() + .startsWith("application/wasm") + ) { + return WebAssembly.instantiateStreaming(wasmResponse, imports); + } else { + return WebAssembly.instantiate( + await wasmResponse.arrayBuffer(), + imports, + ); + } + } + default: + throw new Error(`Unsupported protocol: ${wasmUrl.protocol}`); + } +} diff --git a/tokenizers/lib/netsaur_tokenizers_bg.wasm b/tokenizers/lib/netsaur_tokenizers_bg.wasm new file mode 100644 index 0000000..1638b08 Binary files /dev/null and b/tokenizers/lib/netsaur_tokenizers_bg.wasm differ diff --git a/tokenizers/mod.ts b/tokenizers/mod.ts new file mode 100644 index 0000000..ccbcb00 --- /dev/null +++ b/tokenizers/mod.ts @@ -0,0 +1,60 @@ +import { + instantiate, + wasm_tokenizer_from_json, + wasm_tokenizer_save, + wasm_tokenizer_tokenize, +} from "./lib/netsaur_tokenizers.generated.js"; + +let initialized = false; +export async function init() { + if (initialized) return; + await instantiate({ + url: new URL(import.meta.url).protocol !== "file:" + ? new URL( + "https://github.com/denosaurs/netsaur/releases/download/0.2.10/netsaur_tokenizers_bg.wasm", + import.meta.url, + ) + : undefined, + }); + initialized = true; +} + +/** + * Tokenizer class + */ +export class Tokenizer { + #id; + constructor(id: number) { + this.#id = id; + } + + /** + * Tokenize a sentence + * @param sentence sentence to tokenize + * @returns + */ + tokenize(sentence: string) { + return wasm_tokenizer_tokenize(this.#id, sentence); + } + + /** + * Save the tokenizer as json + */ + save(): string; + /** + * Save the tokenizer as json + * @param pretty pretty print the json + */ + save(pretty: boolean): string; + save(...args: [boolean?]) { + return wasm_tokenizer_save(this.#id, args[0] ?? false); + } + /** + * Load a tokenizer from json data + * @param json string + * @returns + */ + static fromJson(json: string) { + return new Tokenizer(wasm_tokenizer_from_json(json)); + } +} diff --git a/visualizer/mod.ts b/visualizer/mod.ts new file mode 100644 index 0000000..57e4354 --- /dev/null +++ b/visualizer/mod.ts @@ -0,0 +1,70 @@ +import type { NeuralNetwork, Rank, Tensor } from "../mod.ts"; +import { Line } from "./types.ts"; + +/** + * Visualizer for Neural Networks in Jupyter Notebook + */ +export class Visualizer { + /** + * Graph title + */ + #title: string; + + constructor(title: string) { + this.#title = title; + } + + /** + * Graph the results of a Neural Network + */ + async graph( + net: NeuralNetwork, + inputs: Tensor[], + expectedResults: Tensor[], + ) { + const expected: Line = { + x: [], + y: [], + type: "scatter", + mode: "lines+markers", + name: "Expected", + line: { + color: "blue", + width: 3, + }, + }; + const results: Line = { + x: [], + y: [], + type: "scatter", + mode: "lines+markers", + name: "Results", + line: { + color: "red", + width: 3, + }, + }; + + for (let i = 0; i < inputs.length; i++) { + expected.x.push(i + 1); + results.x.push(i + 1); + const output = (await net.predict(inputs[i])).data; + expected.y.push(expectedResults[i].data[0]); + results.y.push(output[0]); + } + const title = this.#title; + + return { + [Symbol.for("Jupyter.display")]() { + return { + "application/vnd.plotly.v1+json": { + data: [expected, results], + layout: { + title, + }, + }, + }; + }, + }; + } +} diff --git a/visualizer/types.ts b/visualizer/types.ts new file mode 100644 index 0000000..d8c5a04 --- /dev/null +++ b/visualizer/types.ts @@ -0,0 +1,14 @@ +/** + * Line Type for Jupyter Notebook + */ +export interface Line { + x: number[]; + y: number[]; + type?: "scatter" | "bar"; + mode?: "markers" | "lines" | "lines+markers"; + name?: string; + line?: { + color?: string; + width?: number; + }; +} diff --git a/web.ts b/web.ts index bbbbed2..953106c 100644 --- a/web.ts +++ b/web.ts @@ -6,4 +6,4 @@ export * from "./src/core/api/layers.ts"; export * from "./src/core/api/shape.ts"; export * from "./src/core/api/network.ts"; -export { WASM } from "./src/backend_wasm/mod.ts"; +export { WASM } from "./src/backends/wasm/mod.ts";