diff --git a/cmd/substreams/init.go b/cmd/substreams/init.go index 352721020..de2f10275 100644 --- a/cmd/substreams/init.go +++ b/cmd/substreams/init.go @@ -41,11 +41,13 @@ var errInitUnsupportedProtocol = errors.New("unsupported protocol") var initCmd = &cobra.Command{ Use: "init []", - Short: "Initialize a new, working Substreams project from scratch.", + Short: "Initialize a new, working Substreams project from scratch", Long: cli.Dedent(` Initialize a new, working Substreams project from scratch. The path parameter is optional, with your current working directory being the default value. - If you have an Etherscan API Key, you can set it to "ETHERSCAN_API_KEY" environment variable, it will be used to fetch the ABIs and contract information. + + If you have an Etherscan API Key, you can set it to "ETHERSCAN_API_KEY" environment variable, it will be used to + fetch the ABIs and contract information. `), RunE: runSubstreamsInitE, Args: cobra.RangeArgs(0, 1), diff --git a/codegen/templates/ethereum/Cargo.lock b/codegen/templates/ethereum/Cargo.lock index 49aab0cfa..e63c903a6 100644 --- a/codegen/templates/ethereum/Cargo.lock +++ b/codegen/templates/ethereum/Cargo.lock @@ -767,9 +767,9 @@ dependencies = [ [[package]] name = "substreams-database-change" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ea09c700498fea3e50eb9aab5b0637d8bfce888be899aa68f987132923e46cc" +checksum = "ed32ca6fc7fa4b7a684d3abd5bb0545aadd2df82402e7336443cdbb6f8b350c3" dependencies = [ "prost", "prost-types", diff --git a/codegen/templates/ethereum/Makefile b/codegen/templates/ethereum/Makefile index 5fca602e1..168d70120 100644 --- a/codegen/templates/ethereum/Makefile +++ b/codegen/templates/ethereum/Makefile @@ -11,11 +11,11 @@ endif .PHONY: run run: build - substreams run substreams.yaml map_events $(if $(START_BLOCK),-s $(START_BLOCK)) $(if $(STOP_BLOCK),-t $(STOP_BLOCK)) + substreams run substreams.yaml $(if $(MODULE),$(MODULE),map_events) $(if $(START_BLOCK),-s $(START_BLOCK)) $(if $(STOP_BLOCK),-t $(STOP_BLOCK)) .PHONY: gui gui: build - substreams gui substreams.yaml map_events $(if $(START_BLOCK),-s $(START_BLOCK)) $(if $(STOP_BLOCK),-t $(STOP_BLOCK)) + substreams gui substreams.yaml $(if $(MODULE),$(MODULE),map_events) $(if $(START_BLOCK),-s $(START_BLOCK)) $(if $(STOP_BLOCK),-t $(STOP_BLOCK)) .PHONY: protogen protogen: diff --git a/codegen/templates/ethereum/Makefile.gotmpl b/codegen/templates/ethereum/Makefile.gotmpl index 5fca602e1..168d70120 100644 --- a/codegen/templates/ethereum/Makefile.gotmpl +++ b/codegen/templates/ethereum/Makefile.gotmpl @@ -11,11 +11,11 @@ endif .PHONY: run run: build - substreams run substreams.yaml map_events $(if $(START_BLOCK),-s $(START_BLOCK)) $(if $(STOP_BLOCK),-t $(STOP_BLOCK)) + substreams run substreams.yaml $(if $(MODULE),$(MODULE),map_events) $(if $(START_BLOCK),-s $(START_BLOCK)) $(if $(STOP_BLOCK),-t $(STOP_BLOCK)) .PHONY: gui gui: build - substreams gui substreams.yaml map_events $(if $(START_BLOCK),-s $(START_BLOCK)) $(if $(STOP_BLOCK),-t $(STOP_BLOCK)) + substreams gui substreams.yaml $(if $(MODULE),$(MODULE),map_events) $(if $(START_BLOCK),-s $(START_BLOCK)) $(if $(STOP_BLOCK),-t $(STOP_BLOCK)) .PHONY: protogen protogen: diff --git a/codegen/templates/ethereum/src/lib.rs.gotmpl b/codegen/templates/ethereum/src/lib.rs.gotmpl index fd23a981a..0caeb5219 100644 --- a/codegen/templates/ethereum/src/lib.rs.gotmpl +++ b/codegen/templates/ethereum/src/lib.rs.gotmpl @@ -114,8 +114,8 @@ fn db_{{ $contract.GetName }}_out(events: &contract::Events, tables: &mut Databa .set("evt_block_number", evt.evt_block_number) {{- $numberOfAttributes := len $rust.ProtoFieldTableChangesMap }}{{ if eq $numberOfAttributes 0 }};{{ end }} {{- $i := 0 }} - {{- range $protoField, $databaseChangesToProtoConversion := $rust.ProtoFieldTableChangesMap }} - {{ $i = add $i 1 }}.set("{{$protoField}}", {{$databaseChangesToProtoConversion}}){{if eq $i $numberOfAttributes}};{{ end }} + {{- range $protoField, $changesToProtoConversion := $rust.ProtoFieldTableChangesMap }} + {{ $i = add $i 1 }}.{{$changesToProtoConversion.Setter}}("{{$protoField}}", {{$changesToProtoConversion.ValueAccessCode}}){{if eq $i $numberOfAttributes}};{{ end }} {{- end}} }); {{- end}} @@ -133,8 +133,8 @@ fn db_{{ $ddsContract.GetName }}_out(events: &contract::Events, tables: &mut Dat .set("evt_address", &evt.evt_address) {{- $numberOfAttributes := len $rust.ProtoFieldTableChangesMap }}{{ if eq $numberOfAttributes 0 }};{{ end }} {{- $i := 0 }} - {{- range $protoField, $databaseChangesToProtoConversion := $rust.ProtoFieldTableChangesMap }} - {{ $i = add $i 1 }}.set("{{$protoField}}", {{$databaseChangesToProtoConversion}}){{if eq $i $numberOfAttributes}};{{ end }} + {{- range $protoField, $changesToProtoConversion := $rust.ProtoFieldTableChangesMap }} + {{ $i = add $i 1 }}.{{$changesToProtoConversion.Setter}}("{{$protoField}}", {{$changesToProtoConversion.ValueAccessCode}}){{if eq $i $numberOfAttributes}};{{ end }} {{- end}} }); {{- end}} @@ -157,8 +157,8 @@ fn graph_{{ $contract.GetName }}_out(events: &contract::Events, tables: &mut Ent .set("evt_block_number", evt.evt_block_number) {{- $numberOfAttributes := len $rust.ProtoFieldTableChangesMap }}{{ if eq $numberOfAttributes 0 }};{{ end }} {{- $i := 0 }} - {{- range $protoField, $databaseChangesToProtoConversion := $rust.ProtoFieldTableChangesMap }} - {{ $i = add $i 1 }}.set("{{$protoField}}", {{$databaseChangesToProtoConversion}}){{if eq $i $numberOfAttributes}};{{ end }} + {{- range $protoField, $changesToProtoConversion := $rust.ProtoFieldTableChangesMap }} + {{ $i = add $i 1 }}.set("{{$protoField}}", {{$changesToProtoConversion.ValueAccessCode}}){{if eq $i $numberOfAttributes}};{{ end }} {{- end}} }); {{- end}} @@ -178,8 +178,8 @@ fn graph_{{ $ddsContract.GetName }}_out(events: &contract::Events, tables: &mut .set("evt_address", &evt.evt_address) {{- $numberOfAttributes := len $rust.ProtoFieldTableChangesMap }}{{ if eq $numberOfAttributes 0 }};{{ end }} {{- $i := 0 }} - {{- range $protoField, $databaseChangesToProtoConversion := $rust.ProtoFieldTableChangesMap }} - {{ $i = add $i 1 }}.set("{{$protoField}}", {{$databaseChangesToProtoConversion}}){{if eq $i $numberOfAttributes}};{{ end }} + {{- range $protoField, $changesToProtoConversion := $rust.ProtoFieldTableChangesMap }} + {{ $i = add $i 1 }}.set("{{$protoField}}", {{$changesToProtoConversion.ValueAccessCode}}){{if eq $i $numberOfAttributes}};{{ end }} {{- end}} }); {{- end}} diff --git a/codegen/templates/ethereum/src/multiple_contracts_lib.rs.gotmpl b/codegen/templates/ethereum/src/multiple_contracts_lib.rs.gotmpl new file mode 100644 index 000000000..a607fddf9 --- /dev/null +++ b/codegen/templates/ethereum/src/multiple_contracts_lib.rs.gotmpl @@ -0,0 +1,124 @@ +mod abi; +mod pb; +use hex_literal::hex; +use pb::contract::v1 as contract; +use substreams::Hex; +use substreams_database_change::pb::database::DatabaseChanges; +use substreams_database_change::tables::Tables as DatabaseChangeTables; +use substreams_entity_change::pb::entity::EntityChanges; +use substreams_entity_change::tables::Tables as EntityChangesTables; +use substreams_ethereum::pb::eth::v2 as eth; +use substreams_ethereum::Event; + +#[allow(unused_imports)] +use num_traits::cast::ToPrimitive; +use std::str::FromStr; +use substreams::scalar::BigDecimal; + +substreams_ethereum::init!(); + +{{ range $i, $contract := .ethereumContracts -}} +const {{ toUpper $contract.GetName }}_TRACKED_CONTRACT: [u8; 20] = hex!("{{ $contract.GetAddress }}"); +{{ end -}} +{{ $numberOfContracts := len .ethereumContracts }} + +{{- range $i, $contract := .ethereumContracts }} +fn map_{{ $contract.GetName }}_events(blk: ð::Block, events: &mut contract::Events) { + {{- range $event := $contract.GetEvents }} + {{- $rust := $event.Rust }} + events.{{ $contract.GetName }}_{{ $rust.ProtoOutputModuleFieldName }}.append(&mut blk + .receipts() + .flat_map(|view| { + view.receipt.logs.iter() + .filter(|log| log.address == {{ toUpper $contract.GetName }}_TRACKED_CONTRACT) + .filter_map(|log| { + if let Some(event) = abi::{{ if eq $numberOfContracts 1 }}contract{{ else }}{{ $contract.GetName }}_contract{{ end }}::events::{{$rust.ABIStructName}}::match_and_decode(log) { + return Some(contract::{{ capitalizeFirst $contract.GetName }}{{$rust.ProtoMessageName}} { + evt_tx_hash: Hex(&view.transaction.hash).to_string(), + evt_index: log.block_index, + evt_block_time: Some(blk.timestamp().to_owned()), + evt_block_number: blk.number, + {{- range $protoField, $abiToProtoConversion := $rust.ProtoFieldABIConversionMap }} + {{$protoField}}: {{$abiToProtoConversion}}, + {{- end}} + }); + } + + None + }) + }) + .collect()); + {{- end }} +} +{{ end }} + +{{- range $i, $contract := .ethereumContracts }} +fn db_{{ $contract.GetName }}_out(events: &contract::Events, tables: &mut DatabaseChangeTables) { + // Loop over all the abis events to create table changes + {{- range $event := $contract.GetEvents }} + {{- $rust := $event.Rust }} + events.{{ $contract.GetName }}_{{ $rust.ProtoOutputModuleFieldName }}.iter().for_each(|evt| { + tables + .create_row("{{ if eq $numberOfContracts 1 }}{{ $rust.TableChangeEntityName }}{{ else }}{{ $contract.GetName }}_{{ $rust.TableChangeEntityName }}{{ end }}", [("evt_tx_hash", evt.evt_tx_hash.to_string()),("evt_index", evt.evt_index.to_string())]) + .set("evt_block_time", evt.evt_block_time.as_ref().unwrap()) + .set("evt_block_number", evt.evt_block_number) + {{- $numberOfAttributes := len $rust.ProtoFieldTableChangesMap }}{{ if eq $numberOfAttributes 0 }};{{ end }} + {{- $i := 0 }} + {{- range $protoField, $changesToProtoConversion := $rust.ProtoFieldTableChangesMap }} + {{ $i = add $i 1 }}.{{$changesToProtoConversion.Setter}}("{{$protoField}}", {{$changesToProtoConversion.ValueAccessCode}}){{if eq $i $numberOfAttributes}};{{ end }} + {{- end}} + }); + {{- end}} +} +{{- end }} + +{{ range $i, $contract := .ethereumContracts }} +fn graph_{{ $contract.GetName }}_out(events: &contract::Events, tables: &mut EntityChangesTables) { + // Loop over all the abis events to create table changes + {{- range $event := $contract.GetEvents }} + {{- $rust := $event.Rust }} + events.{{ $contract.GetName }}_{{ $rust.ProtoOutputModuleFieldName }}.iter().for_each(|evt| { + tables + .create_row("{{ if eq $numberOfContracts 1 }}{{ $rust.TableChangeEntityName }}{{ else }}{{ $contract.GetName }}_{{ $rust.TableChangeEntityName }}{{ end }}", format!("{}-{}", evt.evt_tx_hash, evt.evt_index)) + .set("evt_tx_hash", &evt.evt_tx_hash) + .set("evt_index", evt.evt_index) + .set("evt_block_time", evt.evt_block_time.as_ref().unwrap()) + .set("evt_block_number", evt.evt_block_number) + {{- $numberOfAttributes := len $rust.ProtoFieldTableChangesMap }}{{ if eq $numberOfAttributes 0 }};{{ end }} + {{- $i := 0 }} + {{- range $protoField, $changesToProtoConversion := $rust.ProtoFieldTableChangesMap }} + {{ $i = add $i 1 }}.set("{{$protoField}}", {{$changesToProtoConversion.ValueAccessCode}}){{if eq $i $numberOfAttributes}};{{ end }} + {{- end}} + }); +{{- end}} +} +{{- end }} + +#[substreams::handlers::map] +fn map_events(blk: eth::Block) -> Result { + let mut events = contract::Events::default(); + {{- range $i, $contract := .ethereumContracts }} + map_{{ $contract.GetName }}_events(&blk, &mut events); + {{- end }} + Ok(events) +} + +#[substreams::handlers::map] +fn db_out(events: contract::Events) -> Result { + // Initialize Database Changes container + let mut tables = DatabaseChangeTables::new(); + {{- range $i, $contract := .ethereumContracts }} + db_{{ $contract.GetName }}_out(&events, &mut tables); + {{- end }} + Ok(tables.to_database_changes()) +} + +#[substreams::handlers::map] +fn graph_out(events: contract::Events) -> Result { + // Initialize Database Changes container + let mut tables = EntityChangesTables::new(); + {{- range $i, $contract := .ethereumContracts }} + graph_{{ $contract.GetName }}_out(&events, &mut tables); + {{- end }} + Ok(tables.to_entity_changes()) +} diff --git a/codegen/templates/ethereum_project.go b/codegen/templates/ethereum_project.go index b3f8aacdb..96a14bf19 100644 --- a/codegen/templates/ethereum_project.go +++ b/codegen/templates/ethereum_project.go @@ -314,19 +314,24 @@ type rustEventModel struct { ProtoOutputModuleFieldName string TableChangeEntityName string ProtoFieldABIConversionMap map[string]string - ProtoFieldTableChangesMap map[string]string + ProtoFieldTableChangesMap map[string]tableChangeSetField ProtoFieldSqlmap map[string]string ProtoFieldClickhouseMap map[string]string ProtoFieldGraphQLMap map[string]string } +type tableChangeSetField struct { + Setter string + ValueAccessCode string +} + func (e *rustEventModel) populateFields(log *eth.LogEventDef) error { if len(log.Parameters) == 0 { return nil } e.ProtoFieldABIConversionMap = map[string]string{} - e.ProtoFieldTableChangesMap = map[string]string{} + e.ProtoFieldTableChangesMap = map[string]tableChangeSetField{} e.ProtoFieldSqlmap = map[string]string{} e.ProtoFieldClickhouseMap = map[string]string{} e.ProtoFieldGraphQLMap = map[string]string{} @@ -348,11 +353,11 @@ func (e *rustEventModel) populateFields(log *eth.LogEventDef) error { return fmt.Errorf("transform - field type %q on parameter with name %q is not supported right now", parameter.TypeName, parameter.Name) } - toDatabaseChangeCode := generateFieldTableChangeCode(parameter.Type, "evt."+name) + toDatabaseChangeSetter, toDatabaseChangeCode := generateFieldTableChangeCode(parameter.Type, "evt."+name) if toDatabaseChangeCode == SKIP_FIELD { continue } - if toDatabaseChangeCode == "" { + if toDatabaseChangeSetter == "" { return fmt.Errorf("table change - field type %q on parameter with name %q is not supported right now", parameter.TypeName, parameter.Name) } @@ -380,7 +385,7 @@ func (e *rustEventModel) populateFields(log *eth.LogEventDef) error { columnName := sanitizeTableChangesColumnNames(name) e.ProtoFieldABIConversionMap[name] = toProtoCode - e.ProtoFieldTableChangesMap[name] = toDatabaseChangeCode + e.ProtoFieldTableChangesMap[name] = tableChangeSetField{Setter: toDatabaseChangeSetter, ValueAccessCode: toDatabaseChangeCode} e.ProtoFieldSqlmap[columnName] = toSqlCode e.ProtoFieldClickhouseMap[columnName] = toClickhouseCode e.ProtoFieldGraphQLMap[name] = toGraphQLCode @@ -476,12 +481,13 @@ func generateFieldClickhouseTypes(fieldType eth.SolidityType) string { case eth.StructType: return SKIP_FIELD - //case eth.ArrayType: - // elemType := generateFieldClickhouseTypes(v.ElementType) - // if elemType == "" || elemType == SKIP_FIELD { - // return elemType - // } - // return fmt.Sprintf("Array(%s)", elemType) + case eth.ArrayType: + elemType := generateFieldClickhouseTypes(v.ElementType) + if elemType == "" || elemType == SKIP_FIELD { + return SKIP_FIELD + } + + return fmt.Sprintf("Array(%s)", elemType) default: return "" @@ -517,52 +523,59 @@ func generateFieldSqlTypes(fieldType eth.SolidityType) string { case eth.StructType: return SKIP_FIELD - //case eth.ArrayType: - // elemType := generateFieldClickhouseTypes(v.ElementType) - // if elemType == "" || elemType == SKIP_FIELD { - // return elemType - // } - // return fmt.Sprintf("%s ARRAY", elemType) + case eth.ArrayType: + elemType := generateFieldSqlTypes(v.ElementType) + if elemType == "" || elemType == SKIP_FIELD { + return SKIP_FIELD + } + + return elemType + "[]" default: return "" } } -func generateFieldTableChangeCode(fieldType eth.SolidityType, fieldAccess string) string { +func generateFieldTableChangeCode(fieldType eth.SolidityType, fieldAccess string) (setter string, valueAccessCode string) { switch v := fieldType.(type) { case eth.AddressType, eth.BytesType, eth.FixedSizeBytesType: - return fmt.Sprintf("Hex(&%s).to_string()", fieldAccess) + return "set", fmt.Sprintf("Hex(&%s).to_string()", fieldAccess) case eth.BooleanType: - return fieldAccess + return "set", fieldAccess case eth.StringType: - return fmt.Sprintf("&%s", fieldAccess) + return "set", fmt.Sprintf("&%s", fieldAccess) case eth.SignedIntegerType: if v.ByteSize <= 8 { - return fieldAccess + return "set", fieldAccess } - return fmt.Sprintf("BigDecimal::from_str(&%s).unwrap()", fieldAccess) + return "set", fmt.Sprintf("BigDecimal::from_str(&%s).unwrap()", fieldAccess) case eth.UnsignedIntegerType: if v.ByteSize <= 8 { - return fieldAccess + return "set", fieldAccess } - return fmt.Sprintf("BigDecimal::from_str(&%s).unwrap()", fieldAccess) + return "set", fmt.Sprintf("BigDecimal::from_str(&%s).unwrap()", fieldAccess) case eth.SignedFixedPointType, eth.UnsignedFixedPointType: - return fmt.Sprintf("BigDecimal::from_str(&%s).unwrap()", fieldAccess) + return "set", fmt.Sprintf("BigDecimal::from_str(&%s).unwrap()", fieldAccess) case eth.ArrayType: - return SKIP_FIELD + // FIXME: Implement multiple contract support, check what is the actual semantics there + _, inner := generateFieldTableChangeCode(v.ElementType, "x") + if inner == SKIP_FIELD { + return SKIP_FIELD, SKIP_FIELD + } + + return "set_psql_array", fmt.Sprintf("%s.into_iter().map(|x| %s).collect::>()", fieldAccess, inner) case eth.StructType: - return SKIP_FIELD + return SKIP_FIELD, SKIP_FIELD default: - return "" + return "", "" } } @@ -636,10 +649,10 @@ func generateFieldGraphQLTypes(fieldType eth.SolidityType) string { case eth.SignedFixedPointType, eth.UnsignedFixedPointType: return "BigDecimal!" - case eth.StructType: - return SKIP_FIELD - case eth.ArrayType: + return "[" + generateFieldGraphQLTypes(v.ElementType) + "]!" + + case eth.StructType: return SKIP_FIELD default: diff --git a/docs/release-notes/change-log.md b/docs/release-notes/change-log.md index 5ed011fa3..6c7ae02b9 100644 --- a/docs/release-notes/change-log.md +++ b/docs/release-notes/change-log.md @@ -13,6 +13,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Codegen (`substreams init`) now always prefixes the tables and entities with the project name +## v1.3.3 + +* Fixed `substreams init` generated code when dealing with Ethereum ABI events containing array types. + + > [!NOTE] + > For now, the generated code only works with Postgres, an upcoming revision is going to lift that constraint. + ## v1.3.2 * Fixed `store.has_at` Wazero signature which was defined as `has_at(storeIdx: i32, ord: i32, key_ptr: i32, key_len: i32)` but should have been `has_at(storeIdx: i32, ord: i64, key_ptr: i32, key_len: i32)`.