diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 0000000..a844f92 --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,76 @@ +project_name: substreams-sink-kv + +release: + draft: true + github: + owner: streamingfast + name: substreams-sink-kv + name_template: '{{.Tag}}' +builds: + - id: substreams-sink-kv + goos: + - linux + - darwin + goarch: + - arm64 + - amd64 + targets: + - linux_amd64 + - darwin_amd64 + - darwin_arm64 + main: ./cmd/substreams-sink-kv + ldflags: -s -w -X main.version={{.Version}} + binary: substreams-sink-kv + +archives: + # fireeth + - name_template: '{{ .Binary }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}' + id: substreams-sink-kv + builds: + - substreams-sink-kv + replacements: + amd64: x86_64 + darwin: macOS + linux: linux + format: tar.gz + files: + - LICENSE + - README.md +snapshot: + name_template: '{{ .Tag }}-next' +checksum: + name_template: checksums.txt +changelog: + filters: + exclude: + - '^docs:' + - '^test:' + sort: asc +dist: dist +signs: +- cmd: keybase + args: + - sign + - --infile + - $artifact + - --binary + - --outfile + - $signature + - --detached + signature: ${artifact}.sig + artifacts: checksum +env_files: + github_token: ~/.config/goreleaser/github_token +brews: + - name: substreams-sink-kv + ids: + - substreams-sink-kv + tap: + owner: streamingfast + name: homebrew-tap + commit_author: + name: goreleaserbot + email: goreleaser@streamingfast.io + homepage: "https://github.com/streamingfast/substreams-sink-kv" + description: "Substreams Sink tool to extract information to a kv store" + license: "Apache-2.0" diff --git a/README.md b/README.md index 6647307..34ea4f0 100644 --- a/README.md +++ b/README.md @@ -1,73 +1,126 @@ -# substreams-sink-kv +# Substreams sink key-value store -This is a command line tool to quickly sync a substreams with a kv database. -It exposes this data through Connect-Web protocol (compatible with GRPC) +## Description + +`substreams-sink-kv` is a tool that allows developers to pipe data extracted from a blockchain into a key-value store and expose it through Connect-Web protocol (compatible with GRPC) + +## Prerequisites + +- Go installation and compiler +- Rust installation and compiler +- Cloned `substreams-sink-kv` repository +- A Substreams module prepared for a kv-sink +- Knowledge of blockchain and substreams development + +## Installation + +* Clone this repository, then run `go install -v ./cmd/substreams-sink-kv` +* Check your installation by running `substreams-sink-kv --version` + +> **Note** the binary file will be installed in your *GO_PATH*, usually `$HOME/go/bin`. Ensure that this folder is included in your *PATH* environment variable. + +## Running with an example substreams + +> **Note** To connect to substreams you will need an authentication token, follow this [guide](https://substreams.streamingfast.io/reference-and-specs/authentication) to obtain one, -### Quickstart +```bash +substreams-sink-kv \ + run \ + "badger3://$(pwd)/badger_data.db" \ + mainnet.eth.streamingfast.io:443 \ + https://github.com/streamingfast/substreams-eth-block-meta/releases/download/v0.4.0/substreams-eth-block-meta-v0.4.0.spkg \ + kv_out +``` -1. Install `substreams-sink-kv` (Installation from source required for now): +You should see output similar to this one: +```bash +2023-01-12T10:08:31.803-0500 INFO (sink-kv) starting prometheus metrics server {"listen_addr": "localhost:9102"} +2023-01-12T10:08:31.803-0500 INFO (sink-kv) sink to kv {"dsn": "badger3:///Users/stepd/repos/substreams-sink-kv/badger_data.db", "endpoint": "mainnet.eth.streamingfast.io:443", "manifest_path": "https://github.com/streamingfast/substreams-eth-block-meta/releases/download/v0.4.0/substreams-eth-block-meta-v0.4.0.spkg", "output_module_name": "kv_out", "block_range": ""} +2023-01-12T10:08:31.803-0500 INFO (sink-kv) starting pprof server {"listen_addr": "localhost:6060"} +2023-01-12T10:08:31.826-0500 INFO (sink-kv) reading substreams manifest {"manifest_path": "https://github.com/streamingfast/substreams-eth-block-meta/releases/download/v0.4.0/substreams-eth-block-meta-v0.4.0.spkg"} +2023-01-12T10:08:32.186-0500 INFO (sink-kv) validating output store {"output_store": "kv_out"} +2023-01-12T10:08:32.186-0500 INFO (sink-kv) resolved block range {"start_block": 0, "stop_block": 0} +2023-01-12T10:08:32.186-0500 INFO (sink-kv) starting to listen on {"addr": "localhost:8000"} +2023-01-12T10:08:32.186-0500 INFO (sink-kv) starting stats service {"runs_each": "2s"} +2023-01-12T10:08:32.186-0500 INFO (sink-kv) no block data buffer provided. since undo steps are possible, using default buffer size {"size": 12} +2023-01-12T10:08:32.186-0500 INFO (sink-kv) starting stats service {"runs_each": "2s"} +2023-01-12T10:08:32.186-0500 INFO (sink-kv) ready, waiting for signal to quit +2023-01-12T10:08:32.186-0500 INFO (sink-kv) launching server {"listen_addr": "localhost:8000"} +2023-01-12T10:08:32.187-0500 INFO (sink-kv) serving plaintext {"listen_addr": "localhost:8000"} +2023-01-12T10:08:32.278-0500 INFO (sink-kv) session init {"trace_id": "a3c59bd7992c433402b70f9541565d2d"} +2023-01-12T10:08:34.186-0500 INFO (sink-kv) substreams sink stats {"db_flush_rate": "10.500 flush/s (21 total)", "data_msg_rate": "0.000 msg/s (0 total)", "progress_msg_rate": "0.000 msg/s (0 total)", "block_rate": "0.000 blocks/s (0 total)", "flushed_entries": 0, "last_block": "None"} +2023-01-12T10:08:34.186-0500 INFO (sink-kv) substreams sink stats {"progress_msg_rate": "16551.500 msg/s (33103 total)", "block_rate": "10941.500 blocks/s (21883 total)", "last_block": "#291883 (66d03f819dde948b297c8d582889246d7ba11a5b947335497f8716a7b608f78e)"} +``` - ```bash - go install ./cmd/substreams-sink-kv - ``` +> **Note** Alternatively, you can simply run `./devel/local/start.sh` which runs this command for you. -2. Add a 'map' module to your `substreams.yaml` with an output type of `proto:substreams.kv.v1.KVOperations`: +## Running with your own substreams - ```yaml - modules: - - name: kv_out - kind: map - initialBlock: 0 - inputs: - - store: store_something - output: - type: proto:substreams.kv.v1.KVOperations - ``` +1. Add a 'map' module to your `substreams.yaml` with an output type of `proto:sf.substreams.kv.v1.KVOperations`: -3. Run the sink to a local 'badger' database +```yaml +modules: + - name: kv_out + kind: map + inputs: + - store: your_store + output: + type: proto:substreams.kv.v1.KVOperations +``` - > To connect to substreams you will need an authentication token, follow this [guide](https://substreams.streamingfast.io/reference-and-specs/authentication) to obtain one, +2. Write the `kv_out` module in your substreams code (and recompile) - ```shell - substreams-sink-kv run \ - "badger3:///home/user/sf-data/my-badger.db" \ - "mainnet.eth.streamingfast.io:443" \ - "substreams.yaml" \ - kv_out - ``` +* See [substreams-eth-block-meta](https://github.com/streamingfast/substreams-eth-block-meta) for an example integration -### Even quicker iteration -* Run `devel/local/start.sh` +3. Run your substreams with the sink-kv, writing a local 'badger' database -### Querying the data +> **Note** To connect to substreams you will need an authentication token, follow this [guide](https://substreams.streamingfast.io/reference-and-specs/authentication) to obtain one, -#### In command-line +```shell +substreams-sink-kv run \ + "badger3:///$(pwd)/my-badger.db" \ + "mainnet.eth.streamingfast.io:443" \ + "substreams.yaml" \ + kv_out +``` -(these commands have been tested while running the 'devel/local/start.sh' in another terminal) +## Querying the data + +### In command-line * Get single block data: `grpcurl --plaintext -d '{"key":"month:last:201511"}' localhost:8000 sf.substreams.sink.kv.v1.Kv/GetMany` * Get many: `grpcurl --plaintext -d '{"keys":["day:first:20151201","day:first:20151202"]}' localhost:8000 sf.substreams.sink.kv.v1.Kv/GetMany` * By prefix: `grpcurl --plaintext -d '{"prefix": "day:first:201511", "limit":31}' localhost:8000 sf.substreams.sink.kv.v1.Kv/GetByPrefix` * Scan: `grpcurl --plaintext -d '{"begin": "day:first:201501", "exclusive_end": "day:first:2016", "limit":400}' localhost:8000 sf.substreams.sink.kv.v1.Kv/Scan` -#### From your browser - -(this has been tested while running the 'devel/local/start.sh' in a terminal) +> **Note** These commands have been tested with `devel/local/start.sh` running in another terminal, which runs [substreams-eth-block-meta](https://github.com/streamingfast/substreams-eth-block-meta) +### From your browser * Run `npm run dev` from within `/connect-web-example` to see a Vite JS example connecting to the KV GRPC Connect-Web interface -### Protobuf generation +> **Note** The 'connect-web-version' works best with `devel/local/start.sh` running in another terminal, which runs [substreams-eth-block-meta](https://github.com/streamingfast/substreams-eth-block-meta) + + +## Protobuf generation * Requires 'buf.build' with protoc-gen-go and protoc-gen-go-grpc (https://docs.buf.build/tour/generate-go-code) * Run `cd proto && buf generate` ## Cargo crate -Note: this crate is a stub, mostly to contain the protobuf bindings, feel free to use it or not. +A library containing the protobuf bindings and a few helpers has been published as a crate for your convenience. * https://crates.io/crates/substreams-sink-kv -### Example integration +See [substreams-eth-block-meta](https://github.com/streamingfast/substreams-eth-block-meta) for an example integration + +## Contributing -* See https://github.com/streamingfast/substreams-eth-block-meta +For additional information, [refer to the general StreamingFast contribution guide](https://github.com/streamingfast/streamingfast/blob/master/CONTRIBUTING.md). + +## License + +The `substreams-sink-kv` tool [uses the Apache 2.0 license](https://github.com/streamingfast/substreams/blob/develop/LICENSE/README.md). + +This is a command line tool to quickly sync a substreams with a kv database. diff --git a/bin/release.sh b/bin/release.sh index 4656fa7..8faf6d4 100755 --- a/bin/release.sh +++ b/bin/release.sh @@ -65,7 +65,7 @@ main() { trap cleanup_tag EXIT fi - # goreleaser release $args + goreleaser release $args args="${CARGO_PUBISH_ARGS:-}" if [[ "$force" == "false" ]]; then @@ -81,16 +81,16 @@ cleanup_tag() { fi } -#verify_github_token() { -# if [[ ! -f "$HOME/.config/goreleaser/github_token" && "$GITHUB_TOKEN" = "" ]]; then -# echo "No GitHub token could be found in environment variable GITHUB_TOKEN" -# echo "nor at ~/.config/goreleaser/github_token." -# echo "" -# echo "You will need to create one on GitHub website and make it available through" -# echo "one of the accept way mentioned above." -# exit 1 -# fi -#} +verify_github_token() { + if [[ ! -f "$HOME/.config/goreleaser/github_token" && "$GITHUB_TOKEN" = "" ]]; then + echo "No GitHub token could be found in environment variable GITHUB_TOKEN" + echo "nor at ~/.config/goreleaser/github_token." + echo "" + echo "You will need to create one on GitHub website and make it available through" + echo "one of the accept way mentioned above." + exit 1 + fi +} verify_keybase() { if ! command keybase &> /dev/null; then @@ -139,13 +139,13 @@ usage() { echo "Perform the necessary commands to perform a release of the project to crates.io." echo "The is optional, if not provided, you'll be asked the question." echo "" - #echo "The release being performed against GitHub, you need a valid GitHub API token" - #echo "with the necessary rights to upload release and push to repositories. It needs to" - #echo "be provided in file ~/.config/goreleaser/github_token or through an environment" - #echo "variable GITHUB_TOKEN." - #echo "" - #echo "Keybase is required to sign the release (the checksum of all the artifacts" - #echo "to be precise)." + echo "The release being performed against GitHub, you need a valid GitHub API token" + echo "with the necessary rights to upload release and push to repositories. It needs to" + echo "be provided in file ~/.config/goreleaser/github_token or through an environment" + echo "variable GITHUB_TOKEN." + echo "" + echo "Keybase is required to sign the release (the checksum of all the artifacts" + echo "to be precise)." echo "" echo "You will need to have it available ('brew install keybase' on Mac OS X) and" echo "configure it, just setting your Git username and a password should be enough." diff --git a/go.mod b/go.mod index 20ba49c..49bf4e4 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/streamingfast/kvdb v0.1.1-0.20230106211814-335aada11ecd github.com/streamingfast/logging v0.0.0-20220813175024-b4fbb0e893df github.com/streamingfast/substreams v0.0.22-0.20221124192438-6ef4954ac865 - github.com/streamingfast/substreams-sink v0.0.0-20230106215604-5562ee4e267f + github.com/streamingfast/substreams-sink v0.0.0-20230111182819-e90f6922406a go.uber.org/zap v1.23.0 google.golang.org/protobuf v1.28.1 ) diff --git a/go.sum b/go.sum index 33fc1ff..46314eb 100644 --- a/go.sum +++ b/go.sum @@ -557,8 +557,8 @@ github.com/streamingfast/shutter v1.5.0 h1:NpzDYzj0HVpSiDJVO/FFSL6QIK/YKOxY0gJAt github.com/streamingfast/shutter v1.5.0/go.mod h1:B/T6efqdeMGbGwjzPS1ToXzYZI4kDzI5/u4I+7qbjY8= github.com/streamingfast/substreams v0.0.22-0.20221124192438-6ef4954ac865 h1:Hml1HCVfu0u+hi1Nh6eXKOwg+YMfFnob7Hpx9Edp70Y= github.com/streamingfast/substreams v0.0.22-0.20221124192438-6ef4954ac865/go.mod h1:6dNovv1UDtmASublixIAGbiTlYVsf42z9nt8/PqoLHE= -github.com/streamingfast/substreams-sink v0.0.0-20230106215604-5562ee4e267f h1:FQnayox0nNq/5+6eMWYoZtRPvlz4pAgIS+Z3WvXm/L4= -github.com/streamingfast/substreams-sink v0.0.0-20230106215604-5562ee4e267f/go.mod h1:dL/WbtABZVye0WXSngqkWn0b73ZNWXSAUcJ4t0mSOWA= +github.com/streamingfast/substreams-sink v0.0.0-20230111182819-e90f6922406a h1:jtlER3vxFrOwIH3hgX3iw3ur4XtDouZCb6/LAPrBZDc= +github.com/streamingfast/substreams-sink v0.0.0-20230111182819-e90f6922406a/go.mod h1:dL/WbtABZVye0WXSngqkWn0b73ZNWXSAUcJ4t0mSOWA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=