Merge branch 'develop' into giu/diataxis

.github/workflows/docker.yml

@@ -3,7 +3,7 @@ name: Build docker image
 on:
   push:
     tags:
-      - "[a-z0-9]+-v*"
+      - "v*"
     branches:
       - "dockerbuild"
       - "develop"
@@ -16,59 +16,69 @@ env:
 
 jobs:
   build-and-push-image:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     if: "${{ !startsWith(github.event.head_commit.message, 'GitBook: [#') }}"
     permissions:
       contents: read
       packages: write
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Install Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.21.x
-
-      - uses: actions/cache@v2
+      - uses: actions/cache@v4
         with:
           path: |
-            ~/go/pkg/mod
             ~/.cache/go-build
+            ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-
 
-      - name: Build Binary
-        run: go build -v -o ./substreams ./cmd/substreams
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
 
       - name: Log in to the Container registry
-        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Generate docker tags/labels from github build context
         id: meta
-        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
             type=ref,event=tag
             type=sha,prefix=
             type=raw,enable=${{ github.ref == 'refs/heads/develop' }},value=develop
-            type=raw,enable=${{ startsWith(github.ref, 'refs/heads/release/v') }},value=${{ steps.extract_branch.outputs.release_train }}
+            type=edge,branch=develop
           flavor: |
             latest=${{ startsWith(github.ref, 'refs/tags/') }}
 
+      - name: Extract version
+        id: extract-version
+        run: |
+          version=edge
+          commit_date="$(git show -s --format=%cI)"
+          if [[ "${GITHUB_REF}" == refs/tags/* ]]; then
+            version=${GITHUB_REF#refs/tags/}
+          fi
+
+          echo "VERSION=$version (Commit ${GITHUB_SHA::7}, Commit Date $commit_date)" >> "$GITHUB_OUTPUT"
+
       - name: Build and push Docker image
-        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        uses: docker/build-push-action@v6
         with:
-          context: .
-          file: ./Dockerfile.github
+          file: ./Dockerfile
+          platforms: linux/amd64,linux/arm64
           push: true
           tags: ${{ steps.meta.outputs.tags }}
-          # org.opencontainers.image.version will match the tag name
           labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            VERSION=${{ steps.extract-version.outputs.VERSION }}

Dockerfile

@@ -1,31 +1,19 @@
-# syntax=docker/dockerfile:1.2
+FROM --platform=$BUILDPLATFORM golang:1.23-bullseye AS build
 
-FROM rust:1.60-bullseye as rust
+WORKDIR /src
 
-FROM ubuntu:20.04
+ARG TARGETOS TARGETARCH VERSION=dev
 
-RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get -y install \
-    gcc libssl-dev pkg-config protobuf-compiler \
-    ca-certificates libssl1.1 vim strace lsof curl jq && \
-    rm -rf /var/cache/apt /var/lib/apt/lists/*
-
-ENV RUSTUP_HOME=/usr/local/rustup \
-    CARGO_HOME=/usr/local/cargo \
-    PATH=/usr/local/cargo/bin:$PATH \
-    RUST_VERSION=1.60.0
+RUN --mount=target=. \
+      --mount=type=cache,target=/root/.cache/go-build \
+      --mount=type=cache,target=/go/pkg \
+      GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags "-X \"main.version=$VERSION\"" -o /app/substreams ./cmd/substreams
 
-COPY --from=rust /usr/local/cargo /usr/local/cargo/
-COPY --from=rust /usr/local/rustup /usr/local/rustup/
+FROM ubuntu:22.04
 
-# The `cargo install rustfmt || true` part serves the purposes of updating the crate registry, it's really
-# hard to update the registry standalone without a package, so we take a detour by installing a component
-# that will requires to update the crate registry
-RUN rustup target install wasm32-unknown-unknown && rustup component add rustfmt && cargo install rustfmt || true
-
-ADD /substreams /app/substreams
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
+      apt-get -y install -y ca-certificates libssl3
 
-# ENV PATH "/app:$HOME/.cargo/bin:$PATH"
-ENV PATH "/app:/usr/local/cargo/bin:$PATH"
+COPY --from=build /app/substreams /app/substreams
 
 ENTRYPOINT ["/app/substreams"]

app/tier1.go

@@ -60,6 +60,7 @@ type Tier1Config struct {
 	StateStoreDefaultTag string
 	BlockType            string
 	StateBundleSize      uint64
+	EnforceCompression   bool // refuse incoming requests that do not accept gzip compression (ConnectRPC or GRPC)
 
 	MaxSubrequests       uint64
 	SubrequestsEndpoint  string
@@ -199,6 +200,7 @@ func (a *Tier1App) Run() error {
 		a.config.BlockType,
 		subrequestsClientConfig,
 		tier2RequestParameters,
+		a.config.EnforceCompression,
 		opts...,
 	)
 	if err != nil {

block/ranges.go

@@ -1,7 +1,6 @@
 package block
 
 import (
-	"sort"
 	"strings"
 )
 
@@ -46,28 +45,6 @@ func (r Ranges) Contains(input *Range) bool {
 	return false
 }
 
-func (r Ranges) SortAndDedupe() (out Ranges) {
-	if r == nil {
-		return nil
-	}
-
-	m := make(map[string]*Range)
-
-	for _, rr := range r {
-		m[rr.String()] = rr
-	}
-
-	out = make(Ranges, len(m))
-	i := 0
-	for _, v := range m {
-		out[i] = v
-		i++
-	}
-
-	sort.Sort(out)
-	return
-}
-
 func (r Ranges) Merged() (out Ranges) {
 	if r == nil {
 		return nil

cmd/substreams/build.go

@@ -397,5 +397,9 @@ func runCommandInDir(ctx context.Context, dir string, cmdArgs []string) error {
 		return fmt.Errorf("error running `%s`: %w", strings.Join(cmdArgs, " "), err)
 	}
 
+	if cmd.ProcessState.ExitCode() != 0 {
+		return fmt.Errorf("command %q failed with exit code %d", strings.Join(cmdArgs, " "), cmd.ProcessState.ExitCode())
+	}
+
 	return nil
 }

cmd/substreams/main.go

@@ -33,7 +33,7 @@ func computeVersionString(version string, settings []debug.BuildSetting) string
 	}
 
 	if date != "" {
-		labels = append(labels, fmt.Sprintf("Built %s", date))
+		labels = append(labels, fmt.Sprintf("Commit Date %s", date))
 	}
 
 	if len(labels) == 0 {

cmd/substreams/pack.go

@@ -51,10 +51,6 @@ func runPack(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("manifest reader: %w", err)
 	}
 
-	if !manifestReader.IsLocalManifest() {
-		return fmt.Errorf(`"pack" can only be used to pack local manifest file`)
-	}
-
 	pkgBundle, err := manifestReader.Read()
 	if err != nil {
 		return fmt.Errorf("reading manifest %q: %w", manifestPath, err)
@@ -69,9 +65,18 @@ func runPack(cmd *cobra.Command, args []string) error {
 
 	originalOutputFile, _ := sflags.GetString(cmd, "output-file")
 
+	manifestDir := filepath.Dir(manifestPath)
+	if manifestReader.IsRemotePackage(manifestPath) {
+		manifestDir = "."
+
+		if !manifestReader.IsLocalManifest() {
+			fmt.Printf("Re-packaging existing .spkg file...")
+		}
+	}
+
 	packageMetadata := pkgBundle.Package.PackageMeta[0]
 	resolvedOutputFile := resolveOutputFile(originalOutputFile, map[string]string{
-		"manifestDir":     filepath.Dir(manifestPath),
+		"manifestDir":     manifestDir,
 		"spkgDefaultName": fmt.Sprintf("%s-%s.spkg", strings.Replace(packageMetadata.Name, "_", "-", -1), packageMetadata.Version),
 		"version":         packageMetadata.Version,
 	})

codegen/proto_generator.go

@@ -36,7 +36,14 @@ func NewProtoGenerator(outputPath string, excludedPaths []string, generateMod bo
 }
 
 func (g *ProtoGenerator) GenerateProto(pkg *pbsubstreams.Package) error {
-	spkgTemporaryFilePath := filepath.Join(os.TempDir(), pkg.PackageMeta[0].Name+".tmp.spkg")
+
+	tmpDir, err := os.MkdirTemp("", "substreams_protogen")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tmpDir)
+
+	spkgTemporaryFilePath := filepath.Join(tmpDir, pkg.PackageMeta[0].Name+".tmp.spkg")
 	cnt, err := proto.Marshal(pkg)
 	if err != nil {
 		return fmt.Errorf("marshalling package: %w", err)

docs/new/how-to-guides/mantra.md

@@ -0,0 +1,56 @@
+In this guide, you'll learn how to initialize a MANTRA-based Substreams project. You’ll learn how to set up a simple project to extract raw data or filter events from a smart contract.
+
+## Prerequisites
+
+- Docker and VS Code installed and up-to-date.
+- Visit the [Getting Started Guide](https://github.com/streamingfast/substreams-starter) to initialize your Dev Container.
+
+## Step 1: Initialize Your MANTRA Substreams Project
+
+1. Open your Dev Container and run the following command to initialize your project:
+
+    ```bash
+    substreams init
+    ```
+
+2. You will be given the option to choose between two MANTRA project options. Select the one that best fits your requirements:
+    - **MANTRA-minimal**: Creates a simple Substreams that extracts raw data from the block and generates Rust code.
+    - **MANTRA-events**: Creates a Substreams that extracts MANTRA events using the cached [MANTRA Foundational Module](https://substreams.dev/packages/mantra-common/v0.1.0), filtered by one or more smart contract addresses.
+
+
+## Step 2: Visualize the Data
+
+1. Create your account [here](https://thegraph.market/) to generate an authentification token (JWT) and pass it as input to: 
+
+    ```bash
+    substreams auth
+    ```
+
+2. Run the following command to visualize and itterate on your filtered data model:
+
+    ```bash
+    substreams gui
+    ````
+
+## Step 3: Customize your Project 
+
+After initialization, you can:
+
+- Modify your Substreams manifest to include additional filters or configurations.
+- Implement custom processing logic in Rust based on the filtered data retrieved by the foundational module.
+
+## Additional Resources
+
+You may find these additional resources helpful for developing your first EVM application.
+
+### Dev Container Reference
+
+The [Dev Container Reference](../../references/devcontainer-ref.md) helps you navigate the complete container and its common errors. 
+
+### GUI Reference
+
+The [GUI reference](../../references/gui.md) lets you explore all the tools available in the Substreams GUI.
+
+### Manifests Reference
+
+The [Manifests Reference](../../references/manifests.md) helps you with editing the `substreams.yaml`.

docs/new/references/cli/installing-the-cli.md

@@ -12,6 +12,17 @@ Used for connecting to endpoints, streaming data in real time, and packaging cus
 brew install streamingfast/tap/substreams
 ```
 
+### Docker Alias
+
+You can use our published Substreams CLI Docker image an assign an alias to Docker. We mount the API token as `SF_API_TOKEN` in the alias so that credentials are known to the CLI running inside Docker.
+
+```bash
+alias substreams='docker run --rm -it -e="SF_API_TOKEN=$SF_API_TOKEN" ghcr.io/streamingfast/substreams'
+```
+
+> [!NOTE]
+> Expansion of `$SF_API_TOKEN` above happens at command runtime, so you must ensure that it is set correctly in your own host environment.
+
 ### Pre-compiled binary installation
 
 There are several CLI binaries available for different operating systems. Choose the correct platform in the [CLI releases page](https://github.com/streamingfast/substreams/releases).
@@ -57,10 +68,6 @@ A successful installation will print the version that you have installed.
 substreams version dev
 ```
 
-{% hint style="info" %}
-**Note**: You can [also use Gitpod](../developers-guide/installation-requirements.md) instead of a local installation.
-{% endhint %}
-
 ## Install Other Developer Dependencies (Only for Substreams Developers)
 
 If you plan to build your own Substreams (i.e. write Rust code to extract data from the blockchain), you will need several dependencies to set up your developer environment:

docs/release-notes/change-log.md

@@ -9,6 +9,30 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## v1.11.3
+
+### Server-side
+
+* Fixed: detection of gzip compression on 'connect' protocol (js/ts clients)
+* Added: tier1.Config `EnforceCompression` to refuse incoming connections that do not support GZIP compression (default: false)
+
+## v1.11.2
+
+### Server-side
+
+* Fix too many memory allocations impacting performance when stores are used
+
+### CLI
+
+* Force topological ordering of protobuf descriptors when 'packing' an spkg (affecting current substreams-js clients)
+* Allow `substreams pack` to be able to do a "re-packing" of an existing spkg file. Useful to apply the protobuf descriptor ordering fix.
+
+### Docker image
+
+* Rebuilt of v1.11.1 to generate Docker `latest` tag with revamp Docker image building.
+* Substreams CLI is now built with using Ubuntu 22, previous releases were built using Ubuntu 20.
+* Substreams Docker image is now using `ubuntu:22` as its base, previous releases were built using `ubuntu:20.04`.
+
 ## v1.11.1
 
 - Fix the `gui` breaking when the network field is not set in the spkg