Skip to content

Commit

Permalink
Merge branch 'develop' into feature/dds
Browse files Browse the repository at this point in the history
also fix tiny Makefile template issue in test case
  • Loading branch information
sduchesneau committed Feb 1, 2024
2 parents 77ef810 + 5c9067d commit e339f24
Show file tree
Hide file tree
Showing 8 changed files with 194 additions and 48 deletions.
6 changes: 4 additions & 2 deletions cmd/substreams/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,13 @@ var errInitUnsupportedProtocol = errors.New("unsupported protocol")

var initCmd = &cobra.Command{
Use: "init [<path>]",
Short: "Initialize a new, working Substreams project from scratch.",
Short: "Initialize a new, working Substreams project from scratch",
Long: cli.Dedent(`
Initialize a new, working Substreams project from scratch. The path parameter is optional,
with your current working directory being the default value.
If you have an Etherscan API Key, you can set it to "ETHERSCAN_API_KEY" environment variable, it will be used to fetch the ABIs and contract information.
If you have an Etherscan API Key, you can set it to "ETHERSCAN_API_KEY" environment variable, it will be used to
fetch the ABIs and contract information.
`),
RunE: runSubstreamsInitE,
Args: cobra.RangeArgs(0, 1),
Expand Down
4 changes: 2 additions & 2 deletions codegen/templates/ethereum/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions codegen/templates/ethereum/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,11 @@ endif

.PHONY: run
run: build
substreams run substreams.yaml map_events $(if $(START_BLOCK),-s $(START_BLOCK)) $(if $(STOP_BLOCK),-t $(STOP_BLOCK))
substreams run substreams.yaml $(if $(MODULE),$(MODULE),map_events) $(if $(START_BLOCK),-s $(START_BLOCK)) $(if $(STOP_BLOCK),-t $(STOP_BLOCK))

.PHONY: gui
gui: build
substreams gui substreams.yaml map_events $(if $(START_BLOCK),-s $(START_BLOCK)) $(if $(STOP_BLOCK),-t $(STOP_BLOCK))
substreams gui substreams.yaml $(if $(MODULE),$(MODULE),map_events) $(if $(START_BLOCK),-s $(START_BLOCK)) $(if $(STOP_BLOCK),-t $(STOP_BLOCK))

.PHONY: protogen
protogen:
Expand Down
4 changes: 2 additions & 2 deletions codegen/templates/ethereum/Makefile.gotmpl
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,11 @@ endif

.PHONY: run
run: build
substreams run substreams.yaml map_events $(if $(START_BLOCK),-s $(START_BLOCK)) $(if $(STOP_BLOCK),-t $(STOP_BLOCK))
substreams run substreams.yaml $(if $(MODULE),$(MODULE),map_events) $(if $(START_BLOCK),-s $(START_BLOCK)) $(if $(STOP_BLOCK),-t $(STOP_BLOCK))

.PHONY: gui
gui: build
substreams gui substreams.yaml map_events $(if $(START_BLOCK),-s $(START_BLOCK)) $(if $(STOP_BLOCK),-t $(STOP_BLOCK))
substreams gui substreams.yaml $(if $(MODULE),$(MODULE),map_events) $(if $(START_BLOCK),-s $(START_BLOCK)) $(if $(STOP_BLOCK),-t $(STOP_BLOCK))

.PHONY: protogen
protogen:
Expand Down
16 changes: 8 additions & 8 deletions codegen/templates/ethereum/src/lib.rs.gotmpl
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,8 @@ fn db_{{ $contract.GetName }}_out(events: &contract::Events, tables: &mut Databa
.set("evt_block_number", evt.evt_block_number)
{{- $numberOfAttributes := len $rust.ProtoFieldTableChangesMap }}{{ if eq $numberOfAttributes 0 }};{{ end }}
{{- $i := 0 }}
{{- range $protoField, $databaseChangesToProtoConversion := $rust.ProtoFieldTableChangesMap }}
{{ $i = add $i 1 }}.set("{{$protoField}}", {{$databaseChangesToProtoConversion}}){{if eq $i $numberOfAttributes}};{{ end }}
{{- range $protoField, $changesToProtoConversion := $rust.ProtoFieldTableChangesMap }}
{{ $i = add $i 1 }}.{{$changesToProtoConversion.Setter}}("{{$protoField}}", {{$changesToProtoConversion.ValueAccessCode}}){{if eq $i $numberOfAttributes}};{{ end }}
{{- end}}
});
{{- end}}
Expand All @@ -133,8 +133,8 @@ fn db_{{ $ddsContract.GetName }}_out(events: &contract::Events, tables: &mut Dat
.set("evt_address", &evt.evt_address)
{{- $numberOfAttributes := len $rust.ProtoFieldTableChangesMap }}{{ if eq $numberOfAttributes 0 }};{{ end }}
{{- $i := 0 }}
{{- range $protoField, $databaseChangesToProtoConversion := $rust.ProtoFieldTableChangesMap }}
{{ $i = add $i 1 }}.set("{{$protoField}}", {{$databaseChangesToProtoConversion}}){{if eq $i $numberOfAttributes}};{{ end }}
{{- range $protoField, $changesToProtoConversion := $rust.ProtoFieldTableChangesMap }}
{{ $i = add $i 1 }}.{{$changesToProtoConversion.Setter}}("{{$protoField}}", {{$changesToProtoConversion.ValueAccessCode}}){{if eq $i $numberOfAttributes}};{{ end }}
{{- end}}
});
{{- end}}
Expand All @@ -157,8 +157,8 @@ fn graph_{{ $contract.GetName }}_out(events: &contract::Events, tables: &mut Ent
.set("evt_block_number", evt.evt_block_number)
{{- $numberOfAttributes := len $rust.ProtoFieldTableChangesMap }}{{ if eq $numberOfAttributes 0 }};{{ end }}
{{- $i := 0 }}
{{- range $protoField, $databaseChangesToProtoConversion := $rust.ProtoFieldTableChangesMap }}
{{ $i = add $i 1 }}.set("{{$protoField}}", {{$databaseChangesToProtoConversion}}){{if eq $i $numberOfAttributes}};{{ end }}
{{- range $protoField, $changesToProtoConversion := $rust.ProtoFieldTableChangesMap }}
{{ $i = add $i 1 }}.set("{{$protoField}}", {{$changesToProtoConversion.ValueAccessCode}}){{if eq $i $numberOfAttributes}};{{ end }}
{{- end}}
});
{{- end}}
Expand All @@ -178,8 +178,8 @@ fn graph_{{ $ddsContract.GetName }}_out(events: &contract::Events, tables: &mut
.set("evt_address", &evt.evt_address)
{{- $numberOfAttributes := len $rust.ProtoFieldTableChangesMap }}{{ if eq $numberOfAttributes 0 }};{{ end }}
{{- $i := 0 }}
{{- range $protoField, $databaseChangesToProtoConversion := $rust.ProtoFieldTableChangesMap }}
{{ $i = add $i 1 }}.set("{{$protoField}}", {{$databaseChangesToProtoConversion}}){{if eq $i $numberOfAttributes}};{{ end }}
{{- range $protoField, $changesToProtoConversion := $rust.ProtoFieldTableChangesMap }}
{{ $i = add $i 1 }}.set("{{$protoField}}", {{$changesToProtoConversion.ValueAccessCode}}){{if eq $i $numberOfAttributes}};{{ end }}
{{- end}}
});
{{- end}}
Expand Down
124 changes: 124 additions & 0 deletions codegen/templates/ethereum/src/multiple_contracts_lib.rs.gotmpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
mod abi;
mod pb;
use hex_literal::hex;
use pb::contract::v1 as contract;
use substreams::Hex;
use substreams_database_change::pb::database::DatabaseChanges;
use substreams_database_change::tables::Tables as DatabaseChangeTables;
use substreams_entity_change::pb::entity::EntityChanges;
use substreams_entity_change::tables::Tables as EntityChangesTables;
use substreams_ethereum::pb::eth::v2 as eth;
use substreams_ethereum::Event;

#[allow(unused_imports)]
use num_traits::cast::ToPrimitive;
use std::str::FromStr;
use substreams::scalar::BigDecimal;

substreams_ethereum::init!();

{{ range $i, $contract := .ethereumContracts -}}
const {{ toUpper $contract.GetName }}_TRACKED_CONTRACT: [u8; 20] = hex!("{{ $contract.GetAddress }}");
{{ end -}}
{{ $numberOfContracts := len .ethereumContracts }}

{{- range $i, $contract := .ethereumContracts }}
fn map_{{ $contract.GetName }}_events(blk: &eth::Block, events: &mut contract::Events) {
{{- range $event := $contract.GetEvents }}
{{- $rust := $event.Rust }}
events.{{ $contract.GetName }}_{{ $rust.ProtoOutputModuleFieldName }}.append(&mut blk
.receipts()
.flat_map(|view| {
view.receipt.logs.iter()
.filter(|log| log.address == {{ toUpper $contract.GetName }}_TRACKED_CONTRACT)
.filter_map(|log| {
if let Some(event) = abi::{{ if eq $numberOfContracts 1 }}contract{{ else }}{{ $contract.GetName }}_contract{{ end }}::events::{{$rust.ABIStructName}}::match_and_decode(log) {
return Some(contract::{{ capitalizeFirst $contract.GetName }}{{$rust.ProtoMessageName}} {
evt_tx_hash: Hex(&view.transaction.hash).to_string(),
evt_index: log.block_index,
evt_block_time: Some(blk.timestamp().to_owned()),
evt_block_number: blk.number,
{{- range $protoField, $abiToProtoConversion := $rust.ProtoFieldABIConversionMap }}
{{$protoField}}: {{$abiToProtoConversion}},
{{- end}}
});
}

None
})
})
.collect());
{{- end }}
}
{{ end }}

{{- range $i, $contract := .ethereumContracts }}
fn db_{{ $contract.GetName }}_out(events: &contract::Events, tables: &mut DatabaseChangeTables) {
// Loop over all the abis events to create table changes
{{- range $event := $contract.GetEvents }}
{{- $rust := $event.Rust }}
events.{{ $contract.GetName }}_{{ $rust.ProtoOutputModuleFieldName }}.iter().for_each(|evt| {
tables
.create_row("{{ if eq $numberOfContracts 1 }}{{ $rust.TableChangeEntityName }}{{ else }}{{ $contract.GetName }}_{{ $rust.TableChangeEntityName }}{{ end }}", [("evt_tx_hash", evt.evt_tx_hash.to_string()),("evt_index", evt.evt_index.to_string())])
.set("evt_block_time", evt.evt_block_time.as_ref().unwrap())
.set("evt_block_number", evt.evt_block_number)
{{- $numberOfAttributes := len $rust.ProtoFieldTableChangesMap }}{{ if eq $numberOfAttributes 0 }};{{ end }}
{{- $i := 0 }}
{{- range $protoField, $changesToProtoConversion := $rust.ProtoFieldTableChangesMap }}
{{ $i = add $i 1 }}.{{$changesToProtoConversion.Setter}}("{{$protoField}}", {{$changesToProtoConversion.ValueAccessCode}}){{if eq $i $numberOfAttributes}};{{ end }}
{{- end}}
});
{{- end}}
}
{{- end }}

{{ range $i, $contract := .ethereumContracts }}
fn graph_{{ $contract.GetName }}_out(events: &contract::Events, tables: &mut EntityChangesTables) {
// Loop over all the abis events to create table changes
{{- range $event := $contract.GetEvents }}
{{- $rust := $event.Rust }}
events.{{ $contract.GetName }}_{{ $rust.ProtoOutputModuleFieldName }}.iter().for_each(|evt| {
tables
.create_row("{{ if eq $numberOfContracts 1 }}{{ $rust.TableChangeEntityName }}{{ else }}{{ $contract.GetName }}_{{ $rust.TableChangeEntityName }}{{ end }}", format!("{}-{}", evt.evt_tx_hash, evt.evt_index))
.set("evt_tx_hash", &evt.evt_tx_hash)
.set("evt_index", evt.evt_index)
.set("evt_block_time", evt.evt_block_time.as_ref().unwrap())
.set("evt_block_number", evt.evt_block_number)
{{- $numberOfAttributes := len $rust.ProtoFieldTableChangesMap }}{{ if eq $numberOfAttributes 0 }};{{ end }}
{{- $i := 0 }}
{{- range $protoField, $changesToProtoConversion := $rust.ProtoFieldTableChangesMap }}
{{ $i = add $i 1 }}.set("{{$protoField}}", {{$changesToProtoConversion.ValueAccessCode}}){{if eq $i $numberOfAttributes}};{{ end }}
{{- end}}
});
{{- end}}
}
{{- end }}

#[substreams::handlers::map]
fn map_events(blk: eth::Block) -> Result<contract::Events, substreams::errors::Error> {
let mut events = contract::Events::default();
{{- range $i, $contract := .ethereumContracts }}
map_{{ $contract.GetName }}_events(&blk, &mut events);
{{- end }}
Ok(events)
}

#[substreams::handlers::map]
fn db_out(events: contract::Events) -> Result<DatabaseChanges, substreams::errors::Error> {
// Initialize Database Changes container
let mut tables = DatabaseChangeTables::new();
{{- range $i, $contract := .ethereumContracts }}
db_{{ $contract.GetName }}_out(&events, &mut tables);
{{- end }}
Ok(tables.to_database_changes())
}

#[substreams::handlers::map]
fn graph_out(events: contract::Events) -> Result<EntityChanges, substreams::errors::Error> {
// Initialize Database Changes container
let mut tables = EntityChangesTables::new();
{{- range $i, $contract := .ethereumContracts }}
graph_{{ $contract.GetName }}_out(&events, &mut tables);
{{- end }}
Ok(tables.to_entity_changes())
}
77 changes: 45 additions & 32 deletions codegen/templates/ethereum_project.go
Original file line number Diff line number Diff line change
Expand Up @@ -314,19 +314,24 @@ type rustEventModel struct {
ProtoOutputModuleFieldName string
TableChangeEntityName string
ProtoFieldABIConversionMap map[string]string
ProtoFieldTableChangesMap map[string]string
ProtoFieldTableChangesMap map[string]tableChangeSetField
ProtoFieldSqlmap map[string]string
ProtoFieldClickhouseMap map[string]string
ProtoFieldGraphQLMap map[string]string
}

type tableChangeSetField struct {
Setter string
ValueAccessCode string
}

func (e *rustEventModel) populateFields(log *eth.LogEventDef) error {
if len(log.Parameters) == 0 {
return nil
}

e.ProtoFieldABIConversionMap = map[string]string{}
e.ProtoFieldTableChangesMap = map[string]string{}
e.ProtoFieldTableChangesMap = map[string]tableChangeSetField{}
e.ProtoFieldSqlmap = map[string]string{}
e.ProtoFieldClickhouseMap = map[string]string{}
e.ProtoFieldGraphQLMap = map[string]string{}
Expand All @@ -348,11 +353,11 @@ func (e *rustEventModel) populateFields(log *eth.LogEventDef) error {
return fmt.Errorf("transform - field type %q on parameter with name %q is not supported right now", parameter.TypeName, parameter.Name)
}

toDatabaseChangeCode := generateFieldTableChangeCode(parameter.Type, "evt."+name)
toDatabaseChangeSetter, toDatabaseChangeCode := generateFieldTableChangeCode(parameter.Type, "evt."+name)
if toDatabaseChangeCode == SKIP_FIELD {
continue
}
if toDatabaseChangeCode == "" {
if toDatabaseChangeSetter == "" {
return fmt.Errorf("table change - field type %q on parameter with name %q is not supported right now", parameter.TypeName, parameter.Name)
}

Expand Down Expand Up @@ -380,7 +385,7 @@ func (e *rustEventModel) populateFields(log *eth.LogEventDef) error {
columnName := sanitizeTableChangesColumnNames(name)

e.ProtoFieldABIConversionMap[name] = toProtoCode
e.ProtoFieldTableChangesMap[name] = toDatabaseChangeCode
e.ProtoFieldTableChangesMap[name] = tableChangeSetField{Setter: toDatabaseChangeSetter, ValueAccessCode: toDatabaseChangeCode}
e.ProtoFieldSqlmap[columnName] = toSqlCode
e.ProtoFieldClickhouseMap[columnName] = toClickhouseCode
e.ProtoFieldGraphQLMap[name] = toGraphQLCode
Expand Down Expand Up @@ -476,12 +481,13 @@ func generateFieldClickhouseTypes(fieldType eth.SolidityType) string {
case eth.StructType:
return SKIP_FIELD

//case eth.ArrayType:
// elemType := generateFieldClickhouseTypes(v.ElementType)
// if elemType == "" || elemType == SKIP_FIELD {
// return elemType
// }
// return fmt.Sprintf("Array(%s)", elemType)
case eth.ArrayType:
elemType := generateFieldClickhouseTypes(v.ElementType)
if elemType == "" || elemType == SKIP_FIELD {
return SKIP_FIELD
}

return fmt.Sprintf("Array(%s)", elemType)

default:
return ""
Expand Down Expand Up @@ -517,52 +523,59 @@ func generateFieldSqlTypes(fieldType eth.SolidityType) string {
case eth.StructType:
return SKIP_FIELD

//case eth.ArrayType:
// elemType := generateFieldClickhouseTypes(v.ElementType)
// if elemType == "" || elemType == SKIP_FIELD {
// return elemType
// }
// return fmt.Sprintf("%s ARRAY", elemType)
case eth.ArrayType:
elemType := generateFieldSqlTypes(v.ElementType)
if elemType == "" || elemType == SKIP_FIELD {
return SKIP_FIELD
}

return elemType + "[]"

default:
return ""
}
}

func generateFieldTableChangeCode(fieldType eth.SolidityType, fieldAccess string) string {
func generateFieldTableChangeCode(fieldType eth.SolidityType, fieldAccess string) (setter string, valueAccessCode string) {
switch v := fieldType.(type) {
case eth.AddressType, eth.BytesType, eth.FixedSizeBytesType:
return fmt.Sprintf("Hex(&%s).to_string()", fieldAccess)
return "set", fmt.Sprintf("Hex(&%s).to_string()", fieldAccess)

case eth.BooleanType:
return fieldAccess
return "set", fieldAccess

case eth.StringType:
return fmt.Sprintf("&%s", fieldAccess)
return "set", fmt.Sprintf("&%s", fieldAccess)

case eth.SignedIntegerType:
if v.ByteSize <= 8 {
return fieldAccess
return "set", fieldAccess
}
return fmt.Sprintf("BigDecimal::from_str(&%s).unwrap()", fieldAccess)
return "set", fmt.Sprintf("BigDecimal::from_str(&%s).unwrap()", fieldAccess)

case eth.UnsignedIntegerType:
if v.ByteSize <= 8 {
return fieldAccess
return "set", fieldAccess
}
return fmt.Sprintf("BigDecimal::from_str(&%s).unwrap()", fieldAccess)
return "set", fmt.Sprintf("BigDecimal::from_str(&%s).unwrap()", fieldAccess)

case eth.SignedFixedPointType, eth.UnsignedFixedPointType:
return fmt.Sprintf("BigDecimal::from_str(&%s).unwrap()", fieldAccess)
return "set", fmt.Sprintf("BigDecimal::from_str(&%s).unwrap()", fieldAccess)

case eth.ArrayType:
return SKIP_FIELD
// FIXME: Implement multiple contract support, check what is the actual semantics there
_, inner := generateFieldTableChangeCode(v.ElementType, "x")
if inner == SKIP_FIELD {
return SKIP_FIELD, SKIP_FIELD
}

return "set_psql_array", fmt.Sprintf("%s.into_iter().map(|x| %s).collect::<Vec<_>>()", fieldAccess, inner)

case eth.StructType:
return SKIP_FIELD
return SKIP_FIELD, SKIP_FIELD

default:
return ""
return "", ""
}
}

Expand Down Expand Up @@ -636,10 +649,10 @@ func generateFieldGraphQLTypes(fieldType eth.SolidityType) string {
case eth.SignedFixedPointType, eth.UnsignedFixedPointType:
return "BigDecimal!"

case eth.StructType:
return SKIP_FIELD

case eth.ArrayType:
return "[" + generateFieldGraphQLTypes(v.ElementType) + "]!"

case eth.StructType:
return SKIP_FIELD

default:
Expand Down
Loading

0 comments on commit e339f24

Please sign in to comment.