From ccfbe38c08ea7ffd7ad5dd76905f1475e1b697e5 Mon Sep 17 00:00:00 2001 From: Jonathan LEI Date: Thu, 11 Apr 2024 09:50:05 +0800 Subject: [PATCH] feat: init command support for starknet --- cmd/substreams/init.go | 76 +++++++++++ codegen/protocol.go | 1 + codegen/protocol_enum.go | 20 ++- codegen/starknet_chain.go | 12 ++ codegen/starknet_chain_enum.go | 96 ++++++++++++++ codegen/templates/starknet/.gitignore | 4 + codegen/templates/starknet/Cargo.toml.gotmpl | 21 +++ codegen/templates/starknet/Makefile.gotmpl | 26 ++++ .../starknet/proto/block.proto.gotmpl | 14 ++ .../templates/starknet/rust-toolchain.toml | 4 + .../starknet/schema.clickhouse.sql.gotmpl | 8 ++ .../templates/starknet/schema.graphql.gotmpl | 8 ++ codegen/templates/starknet/schema.sql.gotmpl | 9 ++ codegen/templates/starknet/src/lib.rs.gotmpl | 55 ++++++++ codegen/templates/starknet/src/pb/mod.rs | 8 ++ .../templates/starknet/subgraph.yaml.gotmpl | 17 +++ .../substreams.clickhouse.yaml.gotmpl | 59 +++++++++ .../starknet/substreams.sql.yaml.gotmpl | 57 ++++++++ .../starknet/substreams.subgraph.yaml.gotmpl | 56 ++++++++ .../templates/starknet/substreams.yaml.gotmpl | 48 +++++++ codegen/templates/starknet_chain.go | 24 ++++ codegen/templates/starknet_project.go | 123 ++++++++++++++++++ 22 files changed, 739 insertions(+), 7 deletions(-) create mode 100644 codegen/starknet_chain.go create mode 100644 codegen/starknet_chain_enum.go create mode 100644 codegen/templates/starknet/.gitignore create mode 100644 codegen/templates/starknet/Cargo.toml.gotmpl create mode 100644 codegen/templates/starknet/Makefile.gotmpl create mode 100644 codegen/templates/starknet/proto/block.proto.gotmpl create mode 100644 codegen/templates/starknet/rust-toolchain.toml create mode 100644 codegen/templates/starknet/schema.clickhouse.sql.gotmpl create mode 100644 codegen/templates/starknet/schema.graphql.gotmpl create mode 100644 codegen/templates/starknet/schema.sql.gotmpl create mode 100644 codegen/templates/starknet/src/lib.rs.gotmpl create mode 100644 codegen/templates/starknet/src/pb/mod.rs create mode 100644 codegen/templates/starknet/subgraph.yaml.gotmpl create mode 100644 codegen/templates/starknet/substreams.clickhouse.yaml.gotmpl create mode 100644 codegen/templates/starknet/substreams.sql.yaml.gotmpl create mode 100644 codegen/templates/starknet/substreams.subgraph.yaml.gotmpl create mode 100644 codegen/templates/starknet/substreams.yaml.gotmpl create mode 100644 codegen/templates/starknet_chain.go create mode 100644 codegen/templates/starknet_project.go diff --git a/cmd/substreams/init.go b/cmd/substreams/init.go index d397f229b..37783b508 100644 --- a/cmd/substreams/init.go +++ b/cmd/substreams/init.go @@ -33,6 +33,7 @@ var ( devInitProtocol = os.Getenv("SUBSTREAMS_DEV_INIT_PROTOCOL") devInitEthereumTrackedContract = os.Getenv("SUBSTREAMS_DEV_INIT_ETHEREUM_TRACKED_CONTRACT") devInitEthereumChain = os.Getenv("SUBSTREAMS_DEV_INIT_ETHEREUM_CHAIN") + devInitStarknetChain = os.Getenv("SUBSTREAMS_DEV_INIT_STARKNET_CHAIN") ) var errInitUnsupportedChain = errors.New("unsupported chain") @@ -173,6 +174,43 @@ func runSubstreamsInitE(cmd *cobra.Command, args []string) error { return fmt.Errorf("render Ethereum %s project: %w", chain.DisplayName, err) } + case codegen.ProtocolStarknet: + chainSelected, err := promptStarknetChain() + if err != nil { + return fmt.Errorf("running chain prompt: %w", err) + } + if chainSelected == codegen.StarknetChainOther { + fmt.Println() + fmt.Println("We haven't added any templates for your selected chain quite yet") + fmt.Println() + fmt.Println("Come join us in discord at https://discord.gg/u8amUbGBgF and suggest templates/chains you want to see!") + fmt.Println() + return errInitUnsupportedChain + } + + // Since Starknet contract ABI decoding has not been implemented yet, for any chain selected we + // would simply generate a template that collects block information. Unlike with Ethereum, We + // don't need to prompt users for any additional info here. + + chain := templates.StarknetChainsByID[chainSelected.String()] + if chain == nil { + return fmt.Errorf("unknown chain: %s", chainSelected.String()) + } + + fmt.Println("Writing project files") + project, err := templates.NewStarknetProject( + projectName, + moduleName, + chain, + ) + if err != nil { + return fmt.Errorf("new Starknet %s project: %w", chain.DisplayName, err) + } + + if err := renderProjectFilesIn(project, absoluteProjectDir); err != nil { + return fmt.Errorf("render Starknet %s project: %w", chain.DisplayName, err) + } + case codegen.ProtocolOther: fmt.Println() fmt.Println("We haven't added any templates for your selected protocol quite yet") @@ -579,6 +617,44 @@ func promptEthereumChain() (codegen.EthereumChain, error) { return chain, nil } +func promptStarknetChain() (codegen.StarknetChain, error) { + if devInitStarknetChain != "" { + // It's ok to panic, we expect the dev to put in a valid Starknet chain + chain, err := codegen.ParseStarknetChain(devInitStarknetChain) + if err != nil { + panic(fmt.Errorf("invalid chain: %w", err)) + } + + return chain, nil + } + + choice := promptui.Select{ + Label: "Select Starknet chain", + Items: codegen.StarknetChainNames(), + Templates: &promptui.SelectTemplates{ + Selected: `{{ "Starknet chain:" | faint }} {{ . }}`, + }, + HideHelp: true, + } + + _, selection, err := choice.Run() + if err != nil { + if errors.Is(err, promptui.ErrInterrupt) { + // We received Ctrl-C, users wants to abort, nothing else to do, quit immediately + os.Exit(1) + } + + return codegen.StarknetChainOther, fmt.Errorf("running chain prompt: %w", err) + } + + var chain codegen.StarknetChain + if err := chain.UnmarshalText([]byte(selection)); err != nil { + panic(fmt.Errorf("impossible, selecting hard-coded value from enum itself, something is really wrong here")) + } + + return chain, nil +} + type promptOptions struct { Validate promptui.ValidateFunc IsConfirm bool diff --git a/codegen/protocol.go b/codegen/protocol.go index 7fd5001f4..efa846b6a 100644 --- a/codegen/protocol.go +++ b/codegen/protocol.go @@ -5,6 +5,7 @@ package codegen // ENUM( // // Ethereum +// Starknet // Other // // ) diff --git a/codegen/protocol_enum.go b/codegen/protocol_enum.go index 4513f37b1..25cbe0313 100644 --- a/codegen/protocol_enum.go +++ b/codegen/protocol_enum.go @@ -14,17 +14,20 @@ import ( const ( // ProtocolEthereum is a Protocol of type Ethereum. ProtocolEthereum Protocol = iota + // ProtocolStarknet is a Protocol of type Starknet. + ProtocolStarknet // ProtocolOther is a Protocol of type Other. ProtocolOther ) var ErrInvalidProtocol = fmt.Errorf("not a valid Protocol, try [%s]", strings.Join(_ProtocolNames, ", ")) -const _ProtocolName = "EthereumOther" +const _ProtocolName = "EthereumStarknetOther" var _ProtocolNames = []string{ _ProtocolName[0:8], - _ProtocolName[8:13], + _ProtocolName[8:16], + _ProtocolName[16:21], } // ProtocolNames returns a list of possible string values of Protocol. @@ -36,7 +39,8 @@ func ProtocolNames() []string { var _ProtocolMap = map[Protocol]string{ ProtocolEthereum: _ProtocolName[0:8], - ProtocolOther: _ProtocolName[8:13], + ProtocolStarknet: _ProtocolName[8:16], + ProtocolOther: _ProtocolName[16:21], } // String implements the Stringer interface. @@ -55,10 +59,12 @@ func (x Protocol) IsValid() bool { } var _ProtocolValue = map[string]Protocol{ - _ProtocolName[0:8]: ProtocolEthereum, - strings.ToLower(_ProtocolName[0:8]): ProtocolEthereum, - _ProtocolName[8:13]: ProtocolOther, - strings.ToLower(_ProtocolName[8:13]): ProtocolOther, + _ProtocolName[0:8]: ProtocolEthereum, + strings.ToLower(_ProtocolName[0:8]): ProtocolEthereum, + _ProtocolName[8:16]: ProtocolStarknet, + strings.ToLower(_ProtocolName[8:16]): ProtocolStarknet, + _ProtocolName[16:21]: ProtocolOther, + strings.ToLower(_ProtocolName[16:21]): ProtocolOther, } // ParseProtocol attempts to convert a string to a Protocol. diff --git a/codegen/starknet_chain.go b/codegen/starknet_chain.go new file mode 100644 index 000000000..031429597 --- /dev/null +++ b/codegen/starknet_chain.go @@ -0,0 +1,12 @@ +package codegen + +//go:generate go-enum -f=$GOFILE --marshal --names --nocase + +// ENUM( +// +// Mainnet +// Sepolia +// Other +// +// ) +type StarknetChain uint diff --git a/codegen/starknet_chain_enum.go b/codegen/starknet_chain_enum.go new file mode 100644 index 000000000..62f372989 --- /dev/null +++ b/codegen/starknet_chain_enum.go @@ -0,0 +1,96 @@ +// Code generated by go-enum DO NOT EDIT. +// Version: +// Revision: +// Build Date: +// Built By: + +package codegen + +import ( + "fmt" + "strings" +) + +const ( + // StarknetChainMainnet is a StarknetChain of type Mainnet. + StarknetChainMainnet StarknetChain = iota + // StarknetChainSepolia is a StarknetChain of type Sepolia. + StarknetChainSepolia + // StarknetChainOther is a StarknetChain of type Other. + StarknetChainOther +) + +var ErrInvalidStarknetChain = fmt.Errorf("not a valid StarknetChain, try [%s]", strings.Join(_StarknetChainNames, ", ")) + +const _StarknetChainName = "MainnetSepoliaOther" + +var _StarknetChainNames = []string{ + _StarknetChainName[0:7], + _StarknetChainName[7:14], + _StarknetChainName[14:19], +} + +// StarknetChainNames returns a list of possible string values of StarknetChain. +func StarknetChainNames() []string { + tmp := make([]string, len(_StarknetChainNames)) + copy(tmp, _StarknetChainNames) + return tmp +} + +var _StarknetChainMap = map[StarknetChain]string{ + StarknetChainMainnet: _StarknetChainName[0:7], + StarknetChainSepolia: _StarknetChainName[7:14], + StarknetChainOther: _StarknetChainName[14:19], +} + +// String implements the Stringer interface. +func (x StarknetChain) String() string { + if str, ok := _StarknetChainMap[x]; ok { + return str + } + return fmt.Sprintf("StarknetChain(%d)", x) +} + +// IsValid provides a quick way to determine if the typed value is +// part of the allowed enumerated values +func (x StarknetChain) IsValid() bool { + _, ok := _StarknetChainMap[x] + return ok +} + +var _StarknetChainValue = map[string]StarknetChain{ + _StarknetChainName[0:7]: StarknetChainMainnet, + strings.ToLower(_StarknetChainName[0:7]): StarknetChainMainnet, + _StarknetChainName[7:14]: StarknetChainSepolia, + strings.ToLower(_StarknetChainName[7:14]): StarknetChainSepolia, + _StarknetChainName[14:19]: StarknetChainOther, + strings.ToLower(_StarknetChainName[14:19]): StarknetChainOther, +} + +// ParseStarknetChain attempts to convert a string to a StarknetChain. +func ParseStarknetChain(name string) (StarknetChain, error) { + if x, ok := _StarknetChainValue[name]; ok { + return x, nil + } + // Case insensitive parse, do a separate lookup to prevent unnecessary cost of lowercasing a string if we don't need to. + if x, ok := _StarknetChainValue[strings.ToLower(name)]; ok { + return x, nil + } + return StarknetChain(0), fmt.Errorf("%s is %w", name, ErrInvalidStarknetChain) +} + +// MarshalText implements the text marshaller method. +func (x StarknetChain) MarshalText() ([]byte, error) { + return []byte(x.String()), nil +} + +// UnmarshalText implements the text unmarshaller method. +func (x *StarknetChain) UnmarshalText(text []byte) error { + name := string(text) + tmp, err := ParseStarknetChain(name) + if err != nil { + return err + } + *x = tmp + return nil +} diff --git a/codegen/templates/starknet/.gitignore b/codegen/templates/starknet/.gitignore new file mode 100644 index 000000000..a907f08ad --- /dev/null +++ b/codegen/templates/starknet/.gitignore @@ -0,0 +1,4 @@ +/build/ +/target/ +buf.gen.yaml +*.spkg diff --git a/codegen/templates/starknet/Cargo.toml.gotmpl b/codegen/templates/starknet/Cargo.toml.gotmpl new file mode 100644 index 000000000..79e0e191c --- /dev/null +++ b/codegen/templates/starknet/Cargo.toml.gotmpl @@ -0,0 +1,21 @@ +[package] +name = "{{ .name }}" +version = "0.0.1" +edition = "2021" + +[lib] +name = "substreams" +crate-type = ["cdylib"] + +[dependencies] +prost = "0.11" +prost-types = "0.11" +substreams = "0.5" +substreams-database-change = "1" +substreams-entity-change = "1" +substreams-starknet = "0.1" + +[profile.release] +lto = true +opt-level = 's' +strip = "debuginfo" diff --git a/codegen/templates/starknet/Makefile.gotmpl b/codegen/templates/starknet/Makefile.gotmpl new file mode 100644 index 000000000..5115a7d07 --- /dev/null +++ b/codegen/templates/starknet/Makefile.gotmpl @@ -0,0 +1,26 @@ +CARGO_VERSION := $(shell cargo version 2>/dev/null) + +.PHONY: build +build: +ifdef CARGO_VERSION + cargo build --target wasm32-unknown-unknown --release +else + @echo "Building substreams target using Docker. To speed up this step, install a Rust development environment." + docker run --rm -ti --init -v ${PWD}:/usr/src --workdir /usr/src/ rust:bullseye cargo build --target wasm32-unknown-unknown --release +endif + +.PHONY: run +run: build + substreams run substreams.yaml $(if $(MODULE),$(MODULE),map_blocks) $(if $(START_BLOCK),-s $(START_BLOCK)) $(if $(STOP_BLOCK),-t $(STOP_BLOCK)) + +.PHONY: gui +gui: build + substreams gui substreams.yaml $(if $(MODULE),$(MODULE),map_blocks) $(if $(START_BLOCK),-s $(START_BLOCK)) $(if $(STOP_BLOCK),-t $(STOP_BLOCK)) + +.PHONY: protogen +protogen: + substreams protogen ./substreams.yaml --exclude-paths="sf/substreams,google" + +.PHONY: pack +pack: build + substreams pack substreams.yaml diff --git a/codegen/templates/starknet/proto/block.proto.gotmpl b/codegen/templates/starknet/proto/block.proto.gotmpl new file mode 100644 index 000000000..35ae797e9 --- /dev/null +++ b/codegen/templates/starknet/proto/block.proto.gotmpl @@ -0,0 +1,14 @@ +syntax = "proto3"; + +import "google/protobuf/timestamp.proto"; + +package block.v1; + +message Block { + uint64 height = 1; + bytes hash = 2; + bytes prev_hash = 3; + uint64 timestamp = 4; + uint64 tx_count = 5; + uint64 event_count = 6; +} diff --git a/codegen/templates/starknet/rust-toolchain.toml b/codegen/templates/starknet/rust-toolchain.toml new file mode 100644 index 000000000..be6387022 --- /dev/null +++ b/codegen/templates/starknet/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.77" +components = ["rustfmt"] +targets = ["wasm32-unknown-unknown"] diff --git a/codegen/templates/starknet/schema.clickhouse.sql.gotmpl b/codegen/templates/starknet/schema.clickhouse.sql.gotmpl new file mode 100644 index 000000000..8b80e341a --- /dev/null +++ b/codegen/templates/starknet/schema.clickhouse.sql.gotmpl @@ -0,0 +1,8 @@ +CREATE TABLE IF NOT EXISTS block ( + "height" INT, + "hash" VARCHAR(64), + "prev_hash" VARCHAR(64), + "timestamp" INT, + "tx_count" INT, + "event_count" INT +) ENGINE = MergeTree PRIMARY KEY ("hash"); diff --git a/codegen/templates/starknet/schema.graphql.gotmpl b/codegen/templates/starknet/schema.graphql.gotmpl new file mode 100644 index 000000000..96a32aaf4 --- /dev/null +++ b/codegen/templates/starknet/schema.graphql.gotmpl @@ -0,0 +1,8 @@ +type Block @entity { + id: ID! + height: BigInt! + prev_hash: Bytes! + timestamp: BigInt! + tx_count: BigInt! + event_count: BigInt! +} diff --git a/codegen/templates/starknet/schema.sql.gotmpl b/codegen/templates/starknet/schema.sql.gotmpl new file mode 100644 index 000000000..8b80a7f27 --- /dev/null +++ b/codegen/templates/starknet/schema.sql.gotmpl @@ -0,0 +1,9 @@ +CREATE TABLE IF NOT EXISTS block ( + "height" INT, + "hash" VARCHAR(64), + "prev_hash" VARCHAR(64), + "timestamp" INT, + "tx_count" INT, + "event_count" INT, + PRIMARY KEY(hash) +); diff --git a/codegen/templates/starknet/src/lib.rs.gotmpl b/codegen/templates/starknet/src/lib.rs.gotmpl new file mode 100644 index 000000000..95a1fe992 --- /dev/null +++ b/codegen/templates/starknet/src/lib.rs.gotmpl @@ -0,0 +1,55 @@ +mod pb; +use pb::block::v1 as block; +use substreams::Hex; +use substreams_database_change::pb::database::DatabaseChanges; +use substreams_database_change::tables::Tables as DatabaseChangeTables; +use substreams_entity_change::pb::entity::EntityChanges; +use substreams_entity_change::tables::Tables as EntityChangesTables; +use substreams_starknet::types as starknet; + +#[substreams::handlers::map] +fn map_blocks(raw_block: starknet::Block) -> Result { + Ok(block::Block { + height: raw_block.height, + hash: raw_block.hash, + prev_hash: raw_block.prev_hash, + timestamp: raw_block.timestamp, + tx_count: raw_block.transactions.len() as u64, + event_count: raw_block + .transactions + .iter() + .fold(0, |acc, tx| acc + tx.events.len() as u64), + }) +} + +#[substreams::handlers::map] +fn db_out(processed_block: block::Block) -> Result { + // Initialize Database Changes container + let mut tables = DatabaseChangeTables::new(); + + tables + .create_row("block", [("hash", Hex(&processed_block.hash).to_string())]) + .set("height", processed_block.height) + .set("prev_hash", processed_block.prev_hash) + .set("timestamp", processed_block.timestamp) + .set("tx_count", processed_block.tx_count) + .set("event_count", processed_block.event_count); + + Ok(tables.to_database_changes()) +} + +#[substreams::handlers::map] +fn graph_out(processed_block: block::Block) -> Result { + // Initialize Database Changes container + let mut tables = EntityChangesTables::new(); + + tables + .create_row("Block", format!("{}", Hex(&processed_block.hash))) + .set("height", processed_block.height) + .set("prev_hash", processed_block.prev_hash) + .set("timestamp", processed_block.timestamp) + .set("tx_count", processed_block.tx_count) + .set("event_count", processed_block.event_count); + + Ok(tables.to_entity_changes()) +} diff --git a/codegen/templates/starknet/src/pb/mod.rs b/codegen/templates/starknet/src/pb/mod.rs new file mode 100644 index 000000000..bc35066f4 --- /dev/null +++ b/codegen/templates/starknet/src/pb/mod.rs @@ -0,0 +1,8 @@ +// @generated +pub mod block { + // @@protoc_insertion_point(attribute:block.v1) + pub mod v1 { + include!("block.v1.rs"); + // @@protoc_insertion_point(block.v1) + } +} diff --git a/codegen/templates/starknet/subgraph.yaml.gotmpl b/codegen/templates/starknet/subgraph.yaml.gotmpl new file mode 100644 index 000000000..036282344 --- /dev/null +++ b/codegen/templates/starknet/subgraph.yaml.gotmpl @@ -0,0 +1,17 @@ +specVersion: 0.0.6 +description: {{ .name }} substreams based subgraph +repository: # fill in with git remote url +schema: + file: ./schema.graphql + +dataSources: + - kind: substreams + name: {{ .name }} + network: {{ .network }} + source: + package: + moduleName: graph_out + file: {{ .name }}-v0.1.0.spkg + mapping: + kind: substreams/graph-entities + apiVersion: 0.0.5 diff --git a/codegen/templates/starknet/substreams.clickhouse.yaml.gotmpl b/codegen/templates/starknet/substreams.clickhouse.yaml.gotmpl new file mode 100644 index 000000000..d6870b2fe --- /dev/null +++ b/codegen/templates/starknet/substreams.clickhouse.yaml.gotmpl @@ -0,0 +1,59 @@ +specVersion: v0.1.0 +package: + name: {{ .moduleName }} + version: v0.1.0 + +imports: + sql: https://github.com/streamingfast/substreams-sink-sql/releases/download/protodefs-v{{ .sqlImportVersion }}/substreams-sink-sql-protodefs-v{{ .sqlImportVersion }}.spkg + graph: https://github.com/streamingfast/substreams-sink-subgraph/releases/download/v{{ .graphImportVersion }}/substreams-sink-subgraph-protodefs-v{{ .graphImportVersion }}.spkg + database_change: https://github.com/streamingfast/substreams-sink-database-changes/releases/download/v{{ .databaseChangeImportVersion }}/substreams-database-change-v{{ .databaseChangeImportVersion }}.spkg + entity: https://github.com/streamingfast/substreams-entity-change/releases/download/v{{ .entityChangeImportVersion }}/substreams-entity-change-v{{ .entityChangeImportVersion }}.spkg + +protobuf: + files: + - block.proto + importPaths: + - ./proto + +binaries: + default: + type: wasm/rust-v1 + file: ./target/wasm32-unknown-unknown/release/substreams.wasm + +modules: + - name: map_blocks + kind: map + initialBlock: {{ .initialBlock }} + inputs: + - source: zklend.starknet.type.v1.Block + output: + type: proto:contract.v1.Events + + - name: db_out + kind: map + initialBlock: {{ .initialBlock }} + inputs: + - map: map_blocks + output: + type: proto:sf.substreams.sink.database.v1.DatabaseChanges + + - name: graph_out + kind: map + initialBlock: {{ .initialBlock }} + inputs: + - map: map_blocks + output: + type: proto:sf.substreams.entity.v1.EntityChanges + +network: {{ .network }} + +sink: + module: db_out + type: sf.substreams.sink.sql.v1.Service + config: + schema: "./schema.clickhouse.sql" + engine: clickhouse + postgraphile_frontend: + enabled: false + rest_frontend: + enabled: false diff --git a/codegen/templates/starknet/substreams.sql.yaml.gotmpl b/codegen/templates/starknet/substreams.sql.yaml.gotmpl new file mode 100644 index 000000000..2df09835d --- /dev/null +++ b/codegen/templates/starknet/substreams.sql.yaml.gotmpl @@ -0,0 +1,57 @@ +specVersion: v0.1.0 +package: + name: {{ .moduleName }} + version: v0.1.0 + +imports: + sql: https://github.com/streamingfast/substreams-sink-sql/releases/download/protodefs-v{{ .sqlImportVersion }}/substreams-sink-sql-protodefs-v{{ .sqlImportVersion }}.spkg + graph: https://github.com/streamingfast/substreams-sink-subgraph/releases/download/v{{ .graphImportVersion }}/substreams-sink-subgraph-protodefs-v{{ .graphImportVersion }}.spkg + database_change: https://github.com/streamingfast/substreams-sink-database-changes/releases/download/v{{ .databaseChangeImportVersion }}/substreams-database-change-v{{ .databaseChangeImportVersion }}.spkg + entity: https://github.com/streamingfast/substreams-entity-change/releases/download/v{{ .entityChangeImportVersion }}/substreams-entity-change-v{{ .entityChangeImportVersion }}.spkg + +protobuf: + files: + - block.proto + importPaths: + - ./proto + +binaries: + default: + type: wasm/rust-v1 + file: ./target/wasm32-unknown-unknown/release/substreams.wasm + +modules: + - name: map_blocks + kind: map + initialBlock: {{ .initialBlock }} + inputs: + - source: zklend.starknet.type.v1.Block + output: + type: proto:contract.v1.Events + + - name: db_out + kind: map + initialBlock: {{ .initialBlock }} + inputs: + - map: map_blocks + output: + type: proto:sf.substreams.sink.database.v1.DatabaseChanges + + - name: graph_out + kind: map + initialBlock: {{ .initialBlock }} + inputs: + - map: map_blocks + output: + type: proto:sf.substreams.entity.v1.EntityChanges + +network: {{ .network }} + +sink: + module: db_out + type: sf.substreams.sink.sql.v1.Service + config: + schema: "./schema.sql" + engine: postgres + postgraphile_frontend: + enabled: true diff --git a/codegen/templates/starknet/substreams.subgraph.yaml.gotmpl b/codegen/templates/starknet/substreams.subgraph.yaml.gotmpl new file mode 100644 index 000000000..ff990275f --- /dev/null +++ b/codegen/templates/starknet/substreams.subgraph.yaml.gotmpl @@ -0,0 +1,56 @@ +specVersion: v0.1.0 +package: + name: {{ .moduleName }} + version: v0.1.0 + +imports: + sql: https://github.com/streamingfast/substreams-sink-sql/releases/download/protodefs-v{{ .sqlImportVersion }}/substreams-sink-sql-protodefs-v{{ .sqlImportVersion }}.spkg + graph: https://github.com/streamingfast/substreams-sink-subgraph/releases/download/v{{ .graphImportVersion }}/substreams-sink-subgraph-protodefs-v{{ .graphImportVersion }}.spkg + database_change: https://github.com/streamingfast/substreams-sink-database-changes/releases/download/v{{ .databaseChangeImportVersion }}/substreams-database-change-v{{ .databaseChangeImportVersion }}.spkg + entity: https://github.com/streamingfast/substreams-entity-change/releases/download/v{{ .entityChangeImportVersion }}/substreams-entity-change-v{{ .entityChangeImportVersion }}.spkg + +protobuf: + files: + - block.proto + importPaths: + - ./proto + +binaries: + default: + type: wasm/rust-v1 + file: ./target/wasm32-unknown-unknown/release/substreams.wasm + +modules: + - name: map_blocks + kind: map + initialBlock: {{ .initialBlock }} + inputs: + - source: zklend.starknet.type.v1.Block + output: + type: proto:contract.v1.Events + + - name: db_out + kind: map + initialBlock: {{ .initialBlock }} + inputs: + - map: map_blocks + output: + type: proto:sf.substreams.sink.database.v1.DatabaseChanges + + - name: graph_out + kind: map + initialBlock: {{ .initialBlock }} + inputs: + - map: map_blocks + output: + type: proto:sf.substreams.entity.v1.EntityChanges + +network: {{ .network }} + +sink: + module: graph_out + type: sf.substreams.sink.subgraph.v1.Service + config: + schema: "./schema.graphql" + subgraph_yaml: "./subgraph.yaml" + postgres_direct_protocol_access: true diff --git a/codegen/templates/starknet/substreams.yaml.gotmpl b/codegen/templates/starknet/substreams.yaml.gotmpl new file mode 100644 index 000000000..4d4b9eec6 --- /dev/null +++ b/codegen/templates/starknet/substreams.yaml.gotmpl @@ -0,0 +1,48 @@ +specVersion: v0.1.0 +package: + name: {{ .moduleName }} + version: v0.1.0 + +imports: + sql: https://github.com/streamingfast/substreams-sink-sql/releases/download/protodefs-v{{ .sqlImportVersion }}/substreams-sink-sql-protodefs-v{{ .sqlImportVersion }}.spkg + graph: https://github.com/streamingfast/substreams-sink-subgraph/releases/download/v{{ .graphImportVersion }}/substreams-sink-subgraph-protodefs-v{{ .graphImportVersion }}.spkg + database_change: https://github.com/streamingfast/substreams-sink-database-changes/releases/download/v{{ .databaseChangeImportVersion }}/substreams-database-change-v{{ .databaseChangeImportVersion }}.spkg + entity: https://github.com/streamingfast/substreams-entity-change/releases/download/v{{ .entityChangeImportVersion }}/substreams-entity-change-v{{ .entityChangeImportVersion }}.spkg + +protobuf: + files: + - block.proto + importPaths: + - ./proto + +binaries: + default: + type: wasm/rust-v1 + file: ./target/wasm32-unknown-unknown/release/substreams.wasm + +modules: + - name: map_blocks + kind: map + initialBlock: {{ .initialBlock }} + inputs: + - source: zklend.starknet.type.v1.Block + output: + type: proto:contract.v1.Events + + - name: db_out + kind: map + initialBlock: {{ .initialBlock }} + inputs: + - map: map_blocks + output: + type: proto:sf.substreams.sink.database.v1.DatabaseChanges + + - name: graph_out + kind: map + initialBlock: {{ .initialBlock }} + inputs: + - map: map_blocks + output: + type: proto:sf.substreams.entity.v1.EntityChanges + +network: {{ .network }} diff --git a/codegen/templates/starknet_chain.go b/codegen/templates/starknet_chain.go new file mode 100644 index 000000000..05f6e1709 --- /dev/null +++ b/codegen/templates/starknet_chain.go @@ -0,0 +1,24 @@ +package templates + +type StarknetChain struct { + ID string + DisplayName string + Network string +} + +var StarknetChainsByID = map[string]*StarknetChain{ + "Mainnet": { + DisplayName: "Starknet Mainnet", + Network: "starknet-mainnet", + }, + "Sepolia": { + DisplayName: "Starknet Sepolia", + Network: "starknet-sepolia", + }, +} + +func init() { + for k, v := range StarknetChainsByID { + v.ID = k + } +} diff --git a/codegen/templates/starknet_project.go b/codegen/templates/starknet_project.go new file mode 100644 index 000000000..2d16eb3af --- /dev/null +++ b/codegen/templates/starknet_project.go @@ -0,0 +1,123 @@ +package templates + +import ( + "bytes" + "embed" + "fmt" + "strings" + "text/template" + + "github.com/huandu/xstrings" + "go.uber.org/zap" +) + +//go:embed starknet/.gitignore +//go:embed starknet/proto/block.proto.gotmpl +//go:embed starknet/src/pb/mod.rs +//go:embed starknet/src/lib.rs.gotmpl +//go:embed starknet/Cargo.toml.gotmpl +//go:embed starknet/Makefile.gotmpl +//go:embed starknet/substreams.yaml.gotmpl +//go:embed starknet/substreams.sql.yaml.gotmpl +//go:embed starknet/substreams.clickhouse.yaml.gotmpl +//go:embed starknet/substreams.subgraph.yaml.gotmpl +//go:embed starknet/rust-toolchain.toml +//go:embed starknet/schema.sql.gotmpl +//go:embed starknet/schema.clickhouse.sql.gotmpl +//go:embed starknet/schema.graphql.gotmpl +//go:embed starknet/subgraph.yaml.gotmpl +var starknetProject embed.FS + +type StarknetProject struct { + name string + moduleName string + chain *StarknetChain + sqlImportVersion string + graphImportVersion string + databaseChangeImportVersion string + entityChangeImportVersion string + network string +} + +func NewStarknetProject(name string, moduleName string, chain *StarknetChain) (*StarknetProject, error) { + return &StarknetProject{ + name: name, + moduleName: moduleName, + chain: chain, + sqlImportVersion: "1.0.7", + graphImportVersion: "0.1.0", + databaseChangeImportVersion: "1.2.1", + entityChangeImportVersion: "1.1.0", + network: chain.Network, + }, nil +} + +func (p *StarknetProject) Render() (map[string][]byte, error) { + entries := map[string][]byte{} + + for _, starknetProjectEntry := range []string{ + ".gitignore", + "proto/block.proto.gotmpl", + "src/pb/mod.rs", + "src/lib.rs.gotmpl", + "Cargo.toml.gotmpl", + "Makefile.gotmpl", + "substreams.yaml.gotmpl", + "substreams.sql.yaml.gotmpl", + "substreams.clickhouse.yaml.gotmpl", + "substreams.subgraph.yaml.gotmpl", + "rust-toolchain.toml", + "schema.sql.gotmpl", + "schema.clickhouse.sql.gotmpl", + "schema.graphql.gotmpl", + "subgraph.yaml.gotmpl", + } { + // We use directly "/" here as `starknetProject` is an embed FS and always uses "/" + content, err := starknetProject.ReadFile("starknet" + "/" + starknetProjectEntry) + if err != nil { + return nil, fmt.Errorf("embed read entry %q: %w", starknetProjectEntry, err) + } + + finalFileName := starknetProjectEntry + + zlog.Debug("reading starknet project entry", zap.String("filename", finalFileName)) + + if strings.HasSuffix(finalFileName, ".gotmpl") { + tmpl, err := template.New(finalFileName).Funcs(ProjectGeneratorFuncs).Parse(string(content)) + if err != nil { + return nil, fmt.Errorf("embed parse entry template %q: %w", finalFileName, err) + } + + name := p.name + if finalFileName == "subgraph.yaml.gotmpl" { + name = xstrings.ToKebabCase(p.name) + } + + model := map[string]any{ + "name": name, + "moduleName": p.moduleName, + "chain": p.chain, + "initialBlock": 0, + "sqlImportVersion": p.sqlImportVersion, + "graphImportVersion": p.graphImportVersion, + "databaseChangeImportVersion": p.databaseChangeImportVersion, + "entityChangeImportVersion": p.entityChangeImportVersion, + "network": p.network, + } + + zlog.Debug("rendering templated file", zap.String("filename", finalFileName), zap.Any("model", model)) + + buffer := bytes.NewBuffer(make([]byte, 0, uint64(float64(len(content))*1.10))) + if err := tmpl.Execute(buffer, model); err != nil { + return nil, fmt.Errorf("embed render entry template %q: %w", finalFileName, err) + } + + finalFileName = strings.TrimSuffix(finalFileName, ".gotmpl") + content = buffer.Bytes() + } + + entries[finalFileName] = content + } + + return entries, nil +}