Skip to content

Commit

Permalink
WIP: dynamic datasource in sql/lib templates
Browse files Browse the repository at this point in the history
  • Loading branch information
@sduchesneau
sduchesneau committed Jan 30, 2024
1 parent 2ea9458 commit 7a2ba31
Show file tree
Hide file tree
Showing 7 changed files with 200 additions and 43 deletions.
6 changes: 6 additions & 0 deletions codegen/templates/ethereum/build.rs.gotmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,17 @@ fn main() -> Result<(), anyhow::Error> {
let file_names = [
{{- range $i, $contract := .ethereumContracts }}
"abi/{{ $contract.GetName }}_contract.abi.json",
{{- range $ddsContract := $contract.GetDDS }}
"abi/{{ $ddsContract.GetName }}_contract.abi.json",
{{- end }}
{{- end }}
];
let file_output_names = [
{{- range $i, $contract := .ethereumContracts }}
"src/abi/{{ $contract.GetName }}_contract.rs",
{{- range $ddsContract := $contract.GetDDS }}
"src/abi/{{ $ddsContract.GetName }}_contract.rs",
{{- end }}
{{- end }}
];

Expand Down
21 changes: 21 additions & 0 deletions codegen/templates/ethereum/proto/contract.proto.gotmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,12 @@ message Events {
{{- $eventsCounter = add $eventsCounter 1 }}
repeated {{ $contract.GetName }}_{{$proto.MessageName}} {{ $contract.GetName }}_{{$proto.OutputModuleFieldName}} = {{ $eventsCounter }};
{{- end}}
{{- range $ddsContract := $contract.GetDDS -}}
{{- range $index, $event := $ddsContract.GetEvents -}}
{{- $proto := $event.Proto }}
repeated {{ $ddsContract.GetName }}_{{$proto.MessageName}} {{ $ddsContract.GetName }}_{{$proto.OutputModuleFieldName}} = {{ $eventsCounter }};
{{- end}}
{{- end}}
{{- end}}
}

Expand All @@ -27,4 +33,19 @@ message {{ $contract.GetName }}_{{ $proto.MessageName }} {
{{- end}}
}
{{- end}}
{{- range $i, $ddsContract := $contract.GetDDS -}}
{{- range $event := $ddsContract.GetEvents -}}
{{- $proto := $event.Proto }}
message {{ $ddsContract.GetName }}_{{ $proto.MessageName }} {
string evt_tx_hash = 1;
uint32 evt_index = 2;
google.protobuf.Timestamp evt_block_time = 3;
uint64 evt_block_number = 4;
string evt_address = 5;
{{- range $index, $protoField := $proto.Fields }}
{{ $protoField.Type }} {{ sanitizeProtoFieldName $protoField.Name }} = {{ add $index 6 }};
{{- end}}
}
{{- end}}
{{- end}}
{{- end}}
21 changes: 21 additions & 0 deletions codegen/templates/ethereum/schema.clickhouse.sql.gotmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,25 @@ CREATE TABLE IF NOT EXISTS {{ $contract.GetName }}_{{ $rust.TableChangeEntityNam
{{- end}}
) ENGINE = MergeTree PRIMARY KEY ("evt_tx_hash","evt_index");
{{ end }}

{{- range $ddsContract := $contract.GetDDS -}}
{{- range $event := $ddsContract.GetEvents -}}
{{- $rust := $event.Rust -}}
{{- $numberOfAttributes := len $rust.ProtoFieldTableChangesMap -}}
CREATE TABLE IF NOT EXISTS {{ $ddsContract.GetName }}_{{ $rust.TableChangeEntityName }} (
"evt_tx_hash" VARCHAR(64),
"evt_index" INT,
"evt_block_time" TIMESTAMP,
"evt_block_number" Uint64,
"evt_address" VARCHAR(40){{ if ne $numberOfAttributes 0 }},{{ end -}}
{{- $i := 0 }}
{{- range $fieldName, $sqlType := $rust.ProtoFieldSqlmap }}
{{ $i = add $i 1 }}{{ $fieldName }} {{ $sqlType }},
{{- end}}
PRIMARY KEY(evt_tx_hash,evt_index)
);
{{ end }}
{{ end }}


{{- end }}
18 changes: 18 additions & 0 deletions codegen/templates/ethereum/schema.sql.gotmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,22 @@ CREATE TABLE IF NOT EXISTS {{ $contract.GetName }}_{{ $rust.TableChangeEntityNam
PRIMARY KEY(evt_tx_hash,evt_index)
);
{{ end }}
{{- range $ddsContract := $contract.GetDDS -}}
{{- range $event := $ddsContract.GetEvents -}}
{{- $rust := $event.Rust -}}
{{- $numberOfAttributes := len $rust.ProtoFieldTableChangesMap -}}
CREATE TABLE IF NOT EXISTS {{ $ddsContract.GetName }}_{{ $rust.TableChangeEntityName }} (
"evt_tx_hash" VARCHAR(64),
"evt_index" INT,
"evt_block_time" TIMESTAMP,
"evt_block_number" DECIMAL,
"evt_address" VARCHAR(40),
{{- $i := 0 }}
{{- range $fieldName, $sqlType := $rust.ProtoFieldSqlmap }}
{{ $i = add $i 1 }}{{ $fieldName }} {{ $sqlType }},
{{- end}}
PRIMARY KEY(evt_tx_hash,evt_index)
);
{{- end -}}
{{- end -}}
{{- end }}
60 changes: 60 additions & 0 deletions codegen/templates/ethereum/src/lib.rs.gotmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,46 @@ fn map_{{ $contract.GetName }}_events(blk: &eth::Block, events: &mut contract::E
.collect());
{{- end }}
}

{{- if $.hadDDS }}
fn is_declared_dds_address(addr: &Vec<u8>, ordinal: u64, dds_store: &store::StoreGetInt64) -> bool {
// substreams::log::info!("Checking if address {} is declared dds address", Hex(addr).to_string());
if dds_store.get_at(ordinal, Hex(addr).to_string()).is_some() {
return true;
}
return false;
}
{{ end -}}

{{- range $ddsContract := $contract.GetDDS }}
fn map_{{ $ddsContract.GetName }}_events(blk: &eth::Block, dds_store: &store::StoreGetInt64, events: &mut contract::Events) {
{{- range $event := $ddsContract.GetEvents }}
{{- $rust := $event.Rust }}
events.{{ $ddsContract.GetName }}_{{ $rust.ProtoOutputModuleFieldName }}.append(&mut blk
.receipts()
.flat_map(|view| {
view.receipt.logs.iter()
.filter(|log| is_declared_dds_address(&log.address, log.ordinal, dds_store))
.filter_map(|log| {
if let Some(event) = abi::{{ $contract.GetName }}_contract::events::{{$rust.ABIStructName}}::match_and_decode(log) {
return Some(contract::{{ capitalizeFirst $contract.GetName }}{{$rust.ProtoMessageName}} {
evt_tx_hash: Hex(&view.transaction.hash).to_string(),
evt_index: log.block_index,
evt_block_time: Some(blk.timestamp().to_owned()),
evt_block_number: blk.number,
{{- range $protoField, $abiToProtoConversion := $rust.ProtoFieldABIConversionMap }}
{{$protoField}}: {{$abiToProtoConversion}},
{{- end}}
});
}

None
})
})
.collect());
{{- end }}
}
{{ end }}
{{ end }}

{{- range $i, $contract := .ethereumContracts }}
Expand Down Expand Up @@ -93,6 +133,26 @@ fn graph_{{ $contract.GetName }}_out(events: &contract::Events, tables: &mut Ent
}
{{- end }}

{{- range $contract := .ethereumContracts }}
{{- range $ddsContract := $contract.GetDDS }}
#[substreams::handlers::store]
fn store_{{ $contract.GetName }}_{{ $ddsContract.GetName }}_created(blk: eth::Block, store: StoreSetInt64) {
for rcpt in blk.receipts() {
for log in rcpt
.receipt
.logs
.iter()
.filter(|log| log.address == {{ toUpper $contract.GetName }}_TRACKED_CONTRACT)
{
if let Some(event) = abi::factory_contract::events::{{ $ddsContract.GetCreationEvent }}::match_and_decode(log) {
store.set(log.ordinal, Hex(event.{{ $ddsContract.GetCreationAddressField }}).to_string(), &1);
}
}
}
}
{{- end -}}
{{- end }}

#[substreams::handlers::map]
fn map_events(blk: eth::Block) -> Result<contract::Events, substreams::errors::Error> {
let mut events = contract::Events::default();
Expand Down
39 changes: 35 additions & 4 deletions codegen/templates/ethereum_project.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,10 @@ type EthereumContract struct {
events []codegenEvent
abi *eth.ABI
abiContent string
dynamicDataSources []ddsContract
dynamicDataSources []*DDSContract
}

type ddsContract struct {
type DDSContract struct {
name string
events []codegenEvent
abi *eth.ABI
Expand All @@ -51,6 +51,21 @@ type ddsContract struct {
creationAddressField string
}

func (c *DDSContract) GetName() string {
return c.name
}

func (c *DDSContract) GetEvents() []codegenEvent {
return c.events
}

func (c *DDSContract) GetCreationEvent() string {
return c.creationEvent
}
func (c *DDSContract) GetCreationAddressField() string {
return c.creationAddressField
}

func NewEthereumContract(name string, address eth.Address, abi *eth.ABI, abiContent string) *EthereumContract {
return &EthereumContract{
name: name,
Expand All @@ -62,7 +77,6 @@ func NewEthereumContract(name string, address eth.Address, abi *eth.ABI, abiCont

func (e *EthereumContract) AddDynamicDataSource(
name string,
// events []codegenEvent,
abi *eth.ABI,
abiContent string,
creationEvent string,
Expand All @@ -74,7 +88,7 @@ func (e *EthereumContract) AddDynamicDataSource(
return fmt.Errorf("build ABI event models for dynamic datasource contract %s: %w", name, err)
}

e.dynamicDataSources = append(e.dynamicDataSources, ddsContract{
e.dynamicDataSources = append(e.dynamicDataSources, &DDSContract{
name: name,
events: events,
abi: abi,
Expand All @@ -85,6 +99,10 @@ func (e *EthereumContract) AddDynamicDataSource(
return nil
}

func (e *EthereumContract) GetDDS() []*DDSContract {
return e.dynamicDataSources
}

func (e *EthereumContract) GetAddress() eth.Address {
return e.address
}
Expand Down Expand Up @@ -145,6 +163,15 @@ func NewEthereumProject(name string, moduleName string, chain *EthereumChain, co
}, nil
}

func (p *EthereumProject) HasDDS() bool {
for _, contract := range p.ethereumContracts {
if len(contract.dynamicDataSources) > 0 {
return true
}
}
return false
}

func (p *EthereumProject) Render() (map[string][]byte, error) {
entries := map[string][]byte{}

Expand Down Expand Up @@ -199,6 +226,7 @@ func (p *EthereumProject) Render() (map[string][]byte, error) {
"databaseChangeImportVersion": p.databaseChangeImportVersion,
"entityChangeImportVersion": p.entityChangeImportVersion,
"network": p.network,
"hasDDS": p.HasDDS(),
}

zlog.Debug("rendering templated file", zap.String("filename", finalFileName), zap.Any("model", model))
Expand All @@ -217,6 +245,9 @@ func (p *EthereumProject) Render() (map[string][]byte, error) {

for _, contract := range p.ethereumContracts {
entries[fmt.Sprintf("abi/%s_contract.abi.json", contract.GetName())] = []byte(contract.abiContent)
for _, dds := range contract.dynamicDataSources {
entries[fmt.Sprintf("abi/%s_contract.abi.json", dds.name)] = []byte(dds.abiContent)
}
}

return entries, nil
Expand Down
78 changes: 39 additions & 39 deletions codegen/templates/ethereum_project_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,45 +152,45 @@ func TestNewEthereumTemplateProject(t *testing.T) {
},
assertion: require.NoError,
},
//{
// name: "dynamic datasource",
// args: []args{
// {
// address: "0x1f98431c8ad98523631ae4a59f267346ea31f984",
// abi: fileContent(t, "ethereum/results/dynamic_datasource/abi/factory_contract.abi.json"),
// shortName: "factory",
// dynamicDataSources: []*dds{
// {
// targetTypeName: "pool",
// addressField: "pool",
// targetABI: fileContent(t, "ethereum/results/dynamic_datasource/abi/pool_contract.abi.json"),
// },
// },
// },
// },
// want: map[string][]byte{
// "abi/factory_contract.abi.json": fileContent(t, "ethereum/results/dynamic_datasource/abi/factory_contract.abi.json"),
// "abi/pool_contract.abi.json": fileContent(t, "ethereum/results/dynamic_datasource/abi/pool_contract.abi.json"),
// "proto/contract.proto": fileContent(t, "./ethereum/results/dynamic_datasource/proto/contract.proto"),
// "src/abi/mod.rs": fileContent(t, "./ethereum/results/dynamic_datasource/src/abi/mod.rs"),
// "src/pb/mod.rs": fileContent(t, "./ethereum/results/dynamic_datasource/src/pb/mod.rs"),
// "src/lib.rs": fileContent(t, "./ethereum/results/dynamic_datasource/src/lib.rs"),
// "build.rs": fileContent(t, "./ethereum/results/dynamic_datasource/build.rs"),
// "Cargo.lock": fileContent(t, "./ethereum/Cargo.lock"),
// "Cargo.toml": fileContent(t, "./ethereum/Cargo.toml"),
// "Makefile": fileContent(t, "./ethereum/Makefile"),
// "substreams.yaml": fileContent(t, "./ethereum/results/dynamic_datasource/substreams.yaml"),
// "substreams.sql.yaml": fileContent(t, "./ethereum/substreams.sql.yaml"),
// "substreams.clickhouse.yaml": fileContent(t, "./ethereum/substreams.clickhouse.yaml"),
// "substreams.subgraph.yaml": fileContent(t, "./ethereum/substreams.subgraph.yaml"),
// "rust-toolchain.toml": fileContent(t, "./ethereum/rust-toolchain.toml"),
// "schema.sql": fileContent(t, "./ethereum/results/dynamic_datasource/schema.sql"),
// "schema.clickhouse.sql": fileContent(t, "./ethereum/results/dynamic_datasource/schema.clickhouse.sql"),
// "schema.graphql": fileContent(t, "./ethereum/results/dynamic_datasource/schema.graphql"),
// "subgraph.yaml": fileContent(t, "./ethereum/subgraph.yaml"),
// },
// assertion: require.NoError,
//},
{
name: "dynamic datasource",
args: []args{
{
address: "0x1f98431c8ad98523631ae4a59f267346ea31f984",
abi: fileContent(t, "ethereum/results/dynamic_datasource/abi/factory_contract.abi.json"),
shortName: "factory",
dynamicDataSources: []*dds{
{
targetTypeName: "pool",
addressField: "pool",
targetABI: fileContent(t, "ethereum/results/dynamic_datasource/abi/pool_contract.abi.json"),
},
},
},
},
want: map[string][]byte{
"abi/factory_contract.abi.json": fileContent(t, "ethereum/results/dynamic_datasource/abi/factory_contract.abi.json"),
"abi/pool_contract.abi.json": fileContent(t, "ethereum/results/dynamic_datasource/abi/pool_contract.abi.json"),
"proto/contract.proto": fileContent(t, "./ethereum/results/dynamic_datasource/proto/contract.proto"),
"src/abi/mod.rs": fileContent(t, "./ethereum/results/dynamic_datasource/src/abi/mod.rs"),
"src/pb/mod.rs": fileContent(t, "./ethereum/results/dynamic_datasource/src/pb/mod.rs"),
"src/lib.rs": fileContent(t, "./ethereum/results/dynamic_datasource/src/lib.rs"),
"build.rs": fileContent(t, "./ethereum/results/dynamic_datasource/build.rs"),
"Cargo.lock": fileContent(t, "./ethereum/Cargo.lock"),
"Cargo.toml": fileContent(t, "./ethereum/Cargo.toml"),
"Makefile": fileContent(t, "./ethereum/Makefile"),
"substreams.yaml": fileContent(t, "./ethereum/results/dynamic_datasource/substreams.yaml"),
"substreams.sql.yaml": fileContent(t, "./ethereum/substreams.sql.yaml"),
"substreams.clickhouse.yaml": fileContent(t, "./ethereum/substreams.clickhouse.yaml"),
"substreams.subgraph.yaml": fileContent(t, "./ethereum/substreams.subgraph.yaml"),
"rust-toolchain.toml": fileContent(t, "./ethereum/rust-toolchain.toml"),
"schema.sql": fileContent(t, "./ethereum/results/dynamic_datasource/schema.sql"),
"schema.clickhouse.sql": fileContent(t, "./ethereum/results/dynamic_datasource/schema.clickhouse.sql"),
"schema.graphql": fileContent(t, "./ethereum/results/dynamic_datasource/schema.graphql"),
"subgraph.yaml": fileContent(t, "./ethereum/subgraph.yaml"),
},
assertion: require.NoError,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down

0 comments on commit 7a2ba31

Please sign in to comment.