feat: Implement wnfs-unixfs-file crate for encoding big byte arrays…

… in IPLD (#375)

This adopts some code from iroh-unixfs (from the beetle codebase, previously called "iroh").

The WNFS spec uses UnixFS files as the byte array encoding for public WNFS files.

We've previously put the burden on anyone using rs-wnfs to encode byte arrays and get a CID themselves, now we've got a mechanism inside rs-wnfs to do that. (E.g. previously we've done byte array en/decoding from javascript via js-ipfs-unixfs)

---

* feat: Copy over iroh-unixfs without prost codegen

* chore: Remove `hamt` stuff

* refactor: Remove directory support

* refactor: Remove Symlink support

* refactor: Remove ability to read from file path

* refactor: Replace `ContentLoader` with WNFS's `BlockStore`

* refactor: Rename `UnixfsContentReader` to `..FileReader`

* refactor: Borrow `BlockStore` instead of cloning

* refactor: Delete unused structs & code

* refactor: minor rename `Unixfs` -> `UnixFs`

* refactor: Remove need to provide name for files

* refactor: Write a round-trip proptest

* refactor: Use `BlockStore` to compute hashes & store blocks

* chore: Make sure `async_std` runtime also works

* chore: Write README (add add a proptest for seeking)

* chore: Write more readme

* chore: Fix typo

* chore: Fix lint

* chore: Remove unused dependencies

* chore: Document crate dependencies in the readme

* fix: More accurate lifetimes for `AsyncRead` and `AsyncSeek`

* feat: Add  constructor

Cargo.toml

@@ -5,6 +5,7 @@ members = [
 	"wnfs-common",
 	"wnfs-hamt",
 	"wnfs-nameaccumulator",
+	"wnfs-unixfs-file",
 	"wnfs-wasm",
 ]
 

README.md

@@ -53,10 +53,27 @@ This library is designed with WebAssembly in mind. You can follow instructions o
 ## Crates
 
 - [wnfs](https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs)
+- [wnfs-wasm](https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs-wasm)
 - [wnfs-common](https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs-common)
 - [wnfs-hamt](https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs-hamt)
 - [wnfs-nameaccumulator](https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs-nameaccumulator)
-- [wnfs-wasm](https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs-wasm)
+- [wnfs-unixfs-file](https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs-unixfs-file)
+
+This is the dependency graph between these crates:
+```mermaid
+flowchart TD
+    wnfs-wasm --> wnfs
+    wnfs-wasm --> wnfs-nameaccumulator
+    %% wnfs-bench --> wnfs
+    %% wnfs-bench --> wnfs-hamt
+    %% wnfs-bench --> wnfs-nameaccumulator
+    wnfs --> wnfs-hamt
+    wnfs --> wnfs-common
+    wnfs --> wnfs-unixfs-file
+    wnfs --> wnfs-nameaccumulator
+    wnfs-unixfs-file --> wnfs-common
+    wnfs-hamt --> wnfs-common
+```
 
 ## Building the Project
 

wnfs-hamt/src/strategies/changes.rs

@@ -34,7 +34,7 @@ pub(crate) fn generate_changes<K: Debug + Clone, V: Debug + Clone>(
         pairs
             .clone()
             .into_iter()
-            .zip(randoms.into_iter())
+            .zip(randoms)
             .filter(|(_, (num, _))| *num != 0)
             .map(|((k, _), (num, val))| match num {
                 1 => Change::Add(k, val),

wnfs-unixfs-file/CHANGELOG.md

@@ -0,0 +1 @@
+# Changelog

wnfs-unixfs-file/Cargo.toml

@@ -0,0 +1,44 @@
+[package]
+name = "wnfs-unixfs-file"
+version = "0.1.25"
+description = "IPLD UnixFS File implementation for Webnative Filesystem"
+keywords = ["wnfs", "unixfs", "webnative", "ipfs", "decentralisation"]
+categories = [
+  "filesystem",
+  "cryptography",
+  "web-programming",
+  "wasm",
+]
+license = "Apache-2.0"
+readme = "README.md"
+edition = "2021"
+repository = "https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs-unixfs-file"
+homepage = "https://fission.codes"
+authors = ["The Fission Authors"]
+
+[dependencies]
+anyhow = "1.0"
+async-stream = "0.3"
+bytes = "1.5"
+futures = "0.3"
+libipld = { version = "0.16", features = [] }
+num_enum = "0.5"
+proptest = { version = "1.1", optional = true }
+prost = "0.12"
+rand_core = "0.6"
+testresult = "0.3"
+tokio = { version = "1.34", features = ["io-util"] }
+wnfs-common = { path = "../wnfs-common", version = "=0.1.25" }
+
+[dev-dependencies]
+async-std = { version = "1.11", features = ["attributes"] }
+data-encoding = "2.5.0"
+proptest = "1.1"
+rand = "0.8"
+rand_chacha = "0.3.1"
+tempfile = "3.8.1"
+test-strategy = "0.3"
+tokio = { version = "1.34", features = ["fs", "rt", "macros"] }
+tokio-test = "0.4.3"
+tokio-util = { version = "0.7", features = ["io"] }
+wnfs-common = { path = "../wnfs-common", features = ["test_utils"] }

wnfs-unixfs-file/README.md

@@ -0,0 +1,79 @@
+<div align="center">
+  <a href="https://github.com/wnfs-wg" target="_blank">
+    <img src="https://raw.githubusercontent.com/wnfs-wg/rs-wnfs/main/assets/logo.png" alt="WNFS Logo" width="100" height="100"></img>
+  </a>
+
+  <h1 align="center">wnfs-unixfs-file</h1>
+
+  <p>
+    <a href="https://crates.io/crates/wnfs-unixfs-file">
+      <img src="https://img.shields.io/crates/v/wnfs-unixfs-file?label=crates" alt="Docs">
+    </a>
+    <a href="https://codecov.io/gh/wnfs-wg/rs-wnfs">
+      <img src="https://codecov.io/gh/wnfs-wg/rs-wnfs/branch/main/graph/badge.svg?token=95YHXFMFF4" alt="Code Coverage"/>
+    </a>
+    <a href="https://github.com/wnfs-wg/rs-wnfs/actions?query=">
+      <img src="https://github.com/wnfs-wg/rs-wnfs/actions/workflows/checks.yaml/badge.svg" alt="Build Status">
+    </a>
+    <a href="https://github.com/wnfs-wg/rs-wnfs/blob/main/LICENSE">
+      <img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg" alt="License">
+    </a>
+    <a href="https://docs.rs/wnfs">
+      <img src="https://img.shields.io/static/v1?label=Docs&message=docs.rs&color=blue" alt="Docs">
+    </a>
+    <a href="https://discord.gg/zAQBDEq">
+      <img src="https://img.shields.io/static/v1?label=Discord&message=join%20us!&color=mediumslateblue" alt="Discord">
+    </a>
+  </p>
+</div>
+
+<div align="center"><sub>:warning: Work in progress :warning:</sub></div>
+
+##
+
+This Rust crate provides an implementation of UnixFs files. WNFS uses the UnixFs file encoding purely to chunk big byte arrays into multiple blocks and produce a single CID for them to link to from WNFS structures.
+
+This crate is a fork from beetle (previously "iroh")'s [iroh-unixfs crate](https://github.com/n0-computer/beetle/tree/3e137cb2bc18e1d458c3f72d5e817b03d9537d5d/iroh-unixfs).
+
+Major changes relative to that implementation include:
+- Removed prost code generation, instead it's some hard-coded structs with prost annotations
+- Removed support for any UnixFs structures other than files (no directories, directory shards or symlinks)
+- Removed parallelization for hashing to make the crate async runtime-independent (so it can be used in wasm with wasm-bindgen-futures!)
+- Doesn't hard-code use of SHA-256 anymore
+- Integrated with the wnfs-common `BlockStore` trait
+
+## Usage
+
+```rs
+use wnfs_unixfs_file::builder::FileBuilder;
+use wnfs_common::MemoryBlockStore;
+use tokio::io::AsyncReadExt;
+
+// Where data is stored
+let store = &MemoryBlockStore::new();
+
+// Encoding byte arrays, getting a CID
+let data = vec![1u8; 1_000_000]; // 1MiB of ones
+let root_cid = FileBuilder::new()
+    .content_bytes(data)
+    .build()?
+    .store(store)
+    .await?;
+
+// Taking a CID, reading back a byte array:
+let file = UnixFsFile::load(&root_cid, store).await?;
+println!("filesize: {}", file.filesize());
+let mut buffer = Vec::new();
+let mut reader = file.into_content_reader(store, None)?;
+reader.read_to_end(&mut buffer).await?;
+// buffer now has 1 million ones
+
+// You can also seek
+use tokio::io::AsyncSeekExt;
+use std::io::SeekFrom;
+let mut reader = file.into_content_reader(store, None)?;
+reader.seek(SeekFrom::Start(10_000)).await?;
+let mut slice = [0u8; 10_000];
+reader.read_exact(&mut slice).await?;
+// slice now has 10_000 ones
+```

wnfs-unixfs-file/proptest-regressions/builder.txt

@@ -0,0 +1,8 @@
+# Seeds for failure cases proptest has generated in the past. It is
+# automatically read and these particular cases re-run before any
+# novel cases are generated.
+#
+# It is recommended to check this file in to source control so that
+# everyone who runs the test benefits from these saved cases.
+cc 376480a4a772236f8ba598f595ea690a0a12ff4456b4fe61a9d8caccc03b9a17 # shrinks to input = _TestEncodeDecodeRoundtripArgs { data: [] }
+cc 77c264b383e78eb742f9da87cbff6aa4e6de7d87991ef955792e5637b8134e46 # shrinks to input = _TestSeekSubarrayArgs { seed: 0, degree: 2, len: 87383, seek_start: 43680, seek_len: 23, chunker: Rabin }