refactor(indexer-alt): framework from sui-indexer-alt (#20502)

## Description

Pull out the parts of `sui-indexer-alt` that can be used to create any
indexer, in `sui-indexer-alt-framework`. Also take this opportunity to
clean up dependencies and lock down visibilities between projects.

## Test plan

CI

## Stack

- #20499 

---

## Release notes

Check each box that your changes affect. If none of the boxes relate to
your changes, release notes aren't required.

For each box you select, include information after the relevant heading
that describes the impact of your changes that a user might notice and
any actions they must take to implement updates.

- [ ] Protocol: 
- [ ] Nodes (Validators and Full nodes): 
- [ ] Indexer: 
- [ ] JSON-RPC: 
- [ ] GraphQL: 
- [ ] CLI: 
- [ ] Rust SDK:
- [ ] REST API:

.github/workflows/rust.yml

@@ -166,6 +166,9 @@ jobs:
       - name: Indexer Alt schema
         run: |
           ./crates/sui-indexer-alt/generate_schema.sh
+      - name: Indexer Alt Framework schema
+        run: |
+          ./crates/sui-indexer-alt-framework/generate_schema.sh
       # Ensure there are no uncommitted changes in the repo after running tests
       - run: scripts/changed-files.sh
         shell: bash

Cargo.toml

@@ -117,6 +117,7 @@ members = [
     "crates/sui-graphql-rpc-headers",
     "crates/sui-indexer",
     "crates/sui-indexer-alt",
+    "crates/sui-indexer-alt-framework",
     "crates/sui-indexer-builder",
     "crates/sui-json",
     "crates/sui-json-rpc",
@@ -646,6 +647,7 @@ sui-graphql-rpc-client = { path = "crates/sui-graphql-rpc-client" }
 sui-graphql-rpc-headers = { path = "crates/sui-graphql-rpc-headers" }
 sui-genesis-builder = { path = "crates/sui-genesis-builder" }
 sui-indexer = { path = "crates/sui-indexer" }
+sui-indexer-alt-framework = { path = "crates/sui-indexer-alt-framework" }
 sui-indexer-builder = { path = "crates/sui-indexer-builder" }
 sui-json = { path = "crates/sui-json" }
 sui-json-rpc = { path = "crates/sui-json-rpc" }

crates/sui-indexer-alt-framework/Cargo.toml

@@ -0,0 +1,41 @@
+[package]
+name = "sui-indexer-alt-framework"
+version.workspace = true
+authors = ["Mysten Labs <[email protected]>"]
+license = "Apache-2.0"
+publish = false
+edition = "2021"
+
+[dependencies]
+anyhow.workspace = true
+async-trait.workspace = true
+axum.workspace = true
+backoff.workspace = true
+bb8 = "0.8.5"
+chrono.workspace = true
+clap.workspace = true
+diesel = { workspace = true, features = ["chrono"] }
+diesel-async = { workspace = true, features = ["bb8", "postgres", "async-connection-wrapper"] }
+diesel_migrations.workspace = true
+futures.workspace = true
+prometheus.workspace = true
+reqwest.workspace = true
+serde.workspace = true
+thiserror.workspace = true
+tokio.workspace = true
+tokio-stream.workspace = true
+tokio-util.workspace = true
+tracing.workspace = true
+url.workspace = true
+
+sui-field-count.workspace = true
+sui-storage.workspace = true
+sui-types.workspace = true
+
+[dev-dependencies]
+rand.workspace = true
+telemetry-subscribers.workspace = true
+tempfile.workspace = true
+wiremock.workspace = true
+
+sui-pg-temp-db.workspace = true

crates/sui-indexer-alt-framework/diesel.toml

@@ -0,0 +1,6 @@
+[print_schema]
+file = "src/schema.rs"
+patch_file = "schema.patch"
+
+[migrations_directory]
+dir = "migrations"

crates/sui-indexer-alt-framework/generate_schema.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+# Copyright (c) Mysten Labs, Inc.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Update sui-indexer's generated src/schema.rs based on the schema after
+# running all its migrations on a clean database. Expects the first argument to
+# be a port to run the temporary database on (defaults to 5433).
+
+set -x
+set -e
+
+if ! command -v git &> /dev/null; then
+    echo "Please install git: e.g. brew install git" >&2
+    exit 1
+fi
+
+for PG in psql initdb postgres pg_isready pg_ctl; do
+    if ! command -v $PG &> /dev/null; then
+        echo "Could not find $PG. Please install postgres: e.g. brew install postgresql@15" >&2
+        exit 1
+    fi
+done
+
+if ! command -v diesel &> /dev/null; then
+    echo "Please install diesel: e.g. cargo install diesel_cli --features postgres" >&2
+    exit 1
+fi
+
+REPO=$(git rev-parse --show-toplevel)
+
+# Create a temporary directory to store the ephemeral DB.
+TMP=$(mktemp -d)
+
+# Set-up a trap to clean everything up on EXIT (stop DB, delete temp directory)
+function cleanup {
+  pg_ctl stop -D "$TMP" -mfast
+  set +x
+  echo "Postgres STDOUT:"
+  cat "$TMP/db.stdout"
+  echo "Postgres STDERR:"
+  cat "$TMP/db.stderr"
+  set -x
+  rm -rf "$TMP"
+}
+trap cleanup EXIT
+
+# Create a new database in the temporary directory
+initdb -D "$TMP" --user postgres
+
+# Run the DB in the background, on the port provided and capture its output
+PORT=${1:-5433}
+postgres -D "$TMP" -p "$PORT" -c unix_socket_directories=                      \
+   > "$TMP/db.stdout"                                                          \
+  2> "$TMP/db.stderr"                                                          &
+
+# Wait for postgres to report as ready
+RETRIES=0
+while ! pg_isready -p "$PORT" --host "localhost" --username "postgres"; do
+  if [ $RETRIES -gt 5 ]; then
+    echo "Postgres failed to start" >&2
+    exit 1
+  fi
+  sleep 1
+  RETRIES=$((RETRIES + 1))
+done
+
+# Run all migrations on the new database
+diesel migration run                                                          \
+  --database-url "postgres://postgres:postgrespw@localhost:$PORT"             \
+  --migration-dir "$REPO/crates/sui-indexer-alt-framework/migrations"
+
+# Generate the schema.rs file, excluding partition tables and including the
+# copyright notice.
+diesel print-schema                                                           \
+  --database-url "postgres://postgres:postgrespw@localhost:$PORT"             \
+  --patch-file "$REPO/crates/sui-indexer-alt-framework/schema.patch"          \
+  > "$REPO/crates/sui-indexer-alt-framework/src/schema.rs"

crates/sui-indexer-alt-framework/migrations/00000000000000_diesel_initial_setup/down.sql

@@ -0,0 +1,6 @@
+-- This file was automatically created by Diesel to setup helper functions
+-- and other internal bookkeeping. This file is safe to edit, any future
+-- changes will be added to existing projects as new migrations.
+
+DROP FUNCTION IF EXISTS diesel_manage_updated_at(_tbl regclass);
+DROP FUNCTION IF EXISTS diesel_set_updated_at();

crates/sui-indexer-alt-framework/migrations/00000000000000_diesel_initial_setup/up.sql

@@ -0,0 +1,36 @@
+-- This file was automatically created by Diesel to setup helper functions
+-- and other internal bookkeeping. This file is safe to edit, any future
+-- changes will be added to existing projects as new migrations.
+
+
+
+
+-- Sets up a trigger for the given table to automatically set a column called
+-- `updated_at` whenever the row is modified (unless `updated_at` was included
+-- in the modified columns)
+--
+-- # Example
+--
+-- ```sql
+-- CREATE TABLE users (id SERIAL PRIMARY KEY, updated_at TIMESTAMP NOT NULL DEFAULT NOW());
+--
+-- SELECT diesel_manage_updated_at('users');
+-- ```
+CREATE OR REPLACE FUNCTION diesel_manage_updated_at(_tbl regclass) RETURNS VOID AS $$
+BEGIN
+    EXECUTE format('CREATE TRIGGER set_updated_at BEFORE UPDATE ON %s
+                    FOR EACH ROW EXECUTE PROCEDURE diesel_set_updated_at()', _tbl);
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION diesel_set_updated_at() RETURNS trigger AS $$
+BEGIN
+    IF (
+        NEW IS DISTINCT FROM OLD AND
+        NEW.updated_at IS NOT DISTINCT FROM OLD.updated_at
+    ) THEN
+        NEW.updated_at := current_timestamp;
+    END IF;
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;

crates/sui-indexer-alt-framework/schema.patch

@@ -0,0 +1,7 @@
+diff --git a/crates/sui-indexer-alt-framework/src/schema.rs b/crates/sui-indexer-alt-framework/src/schema.rs
+--- a/crates/sui-indexer-alt-framework/src/schema.rs
++++ b/crates/sui-indexer-alt-framework/src/schema.rs
+@@ -1 +1,3 @@
++// Copyright (c) Mysten Labs, Inc.
++// SPDX-License-Identifier: Apache-2.0
+ // @generated automatically by Diesel CLI.

crates/sui-indexer-alt/src/db.rs → crates/sui-indexer-alt-framework/src/db.rs

@@ -2,7 +2,8 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use anyhow::anyhow;
-use diesel::migration::MigrationVersion;
+use diesel::migration::{self, Migration, MigrationSource, MigrationVersion};
+use diesel::pg::Pg;
 use diesel_async::async_connection_wrapper::AsyncConnectionWrapper;
 use diesel_async::{
     pooled_connection::{
@@ -16,6 +17,8 @@ use std::time::Duration;
 use tracing::info;
 use url::Url;
 
+/// Migrations for schema that the indexer framework needs, regardless of the specific data being
+/// indexed.
 const MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations");
 
 #[derive(clap::Args, Debug, Clone)]
@@ -62,8 +65,8 @@ impl Db {
     }
 
     /// Retrieves a connection from the pool. Can fail with a timeout if a connection cannot be
-    /// established before the [DbConfig::connection_timeout] has elapsed.
-    pub(crate) async fn connect(&self) -> Result<Connection<'_>, RunError> {
+    /// established before the [DbArgs::connection_timeout] has elapsed.
+    pub async fn connect(&self) -> Result<Connection<'_>, RunError> {
         self.pool.get().await
     }
 
@@ -123,19 +126,35 @@ impl Db {
         Ok(())
     }
 
+    /// Run migrations on the database. Use the `migrations` parameter to pass in the migrations
+    /// that are specific to the indexer being run. Migrations that the indexer framework needs
+    /// will be added automatically.
+    ///
+    /// Use Diesel's `embed_migrations!` macro to generate the `migrations` parameter for your
+    /// indexer.
     pub(crate) async fn run_migrations(
         &self,
+        migrations: &'static EmbeddedMigrations,
     ) -> Result<Vec<MigrationVersion<'static>>, anyhow::Error> {
         use diesel_migrations::MigrationHarness;
 
+        struct WithFrameworkMigrations(&'static EmbeddedMigrations);
+        impl MigrationSource<Pg> for WithFrameworkMigrations {
+            fn migrations(&self) -> migration::Result<Vec<Box<dyn Migration<Pg>>>> {
+                let mut migrations = self.0.migrations()?;
+                migrations.extend(MIGRATIONS.migrations()?);
+                Ok(migrations)
+            }
+        }
+
         info!("Running migrations ...");
         let conn = self.pool.dedicated_connection().await?;
         let mut wrapper: AsyncConnectionWrapper<AsyncPgConnection> =
             diesel_async::async_connection_wrapper::AsyncConnectionWrapper::from(conn);
 
         let finished_migrations = tokio::task::spawn_blocking(move || {
             wrapper
-                .run_pending_migrations(MIGRATIONS)
+                .run_pending_migrations(WithFrameworkMigrations(migrations))
                 .map(|versions| versions.iter().map(MigrationVersion::as_owned).collect())
         })
         .await?
@@ -158,13 +177,18 @@ impl Default for DbArgs {
     }
 }
 
-/// Drop all tables and rerunning migrations.
-pub async fn reset_database(db_config: DbArgs, skip_migrations: bool) -> Result<(), anyhow::Error> {
+/// Drop all tables, and re-run migrations if supplied.
+pub async fn reset_database(
+    db_config: DbArgs,
+    migrations: Option<&'static EmbeddedMigrations>,
+) -> Result<(), anyhow::Error> {
     let db = Db::new(db_config).await?;
     db.clear_database().await?;
-    if !skip_migrations {
-        db.run_migrations().await?;
+
+    if let Some(migrations) = migrations {
+        db.run_migrations(migrations).await?;
     }
+
     Ok(())
 }
 
@@ -230,7 +254,7 @@ mod tests {
         .unwrap();
         assert_eq!(cnt.cnt, 1);
 
-        reset_database(db_args, true).await.unwrap();
+        reset_database(db_args, None).await.unwrap();
 
         let mut conn = db.connect().await.unwrap();
         let cnt = diesel::sql_query(