From 9035f940b60c65f531590166ed198fbf90504899 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Duchesneau?= Date: Thu, 27 Jul 2023 13:59:10 -0400 Subject: [PATCH] bump substreams to v1.1.10, prepare release 0.2.1 --- CHANGELOG.md | 50 +++++++++++++++++++++++++++++ cmd/firesol/cli/firehose.go | 2 +- cmd/firesol/cli/substreams-tier1.go | 5 +-- go.mod | 4 +-- go.sum | 8 ++--- 5 files changed, 58 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9403ea44..93107d58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,56 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). See [MAINTAINERS.md](./MAINTAINERS.md) for instructions to keep up to date. +## v0.2.1 + +### Highlights + +#### Substreams Scheduler Improvements for Parallel Processing + +The `substreams` scheduler has been improved to reduce the number of required jobs for parallel processing. This affects `backprocessing` (preparing the states of modules up to a "start-block") and `forward processing` (preparing the states and the outputs to speed up streaming in production-mode). + +Jobs on `tier2` workers are now divided in "stages", each stage generating the partial states for all the modules that have the same dependencies. A `substreams` that has a single store won't be affected, but one that has 3 top-level stores, which used to run 3 jobs for every segment now only runs a single job per segment to get all the states ready. + + +#### Substreams State Store Selection + +The `substreams` server now accepts `X-Sf-Substreams-Cache-Tag` header to select which Substreams state store URL should be used by the request. When performing a Substreams request, the servers will optionally pick the state store based on the header. This enable consumers to stay on the same cache version when the operators needs to bump the data version (reasons for this could be a bug in Substreams software that caused some cached data to be corrupted on invalid). + +To benefit from this, operators that have a version currently in their state store URL should move the version part from `--substreams-state-store-url` to the new flag `--substreams-state-store-default-tag`. For example if today you have in your config: + +```yaml +start: + ... + flags: + substreams-state-store-url: ///v3 +``` + +You should convert to: + +```yaml +start: + ... + flags: + substreams-state-store-url: // + substreams-state-store-default-tag: v3 +``` + +### Operators Upgrade + +* The app `substreams-tier1` and `substreams-tier2` should be upgraded concurrently. Some calls will fail while versions are misaligned. + +* Remove the flag `--substreams-tier1-subrequests-size` from your config, it is not used anymore. + + +### Backend Changes + +* Authentication plugin `trust` can now specify an exclusive list of `allowed` headers (all lowercase), ex: `trust://?allowed=x-sf-user-id,x-sf-api-key-id,x-real-ip,x-sf-substreams-cache-tag` + +* The `tier2` app no longer uses the `common-auth-plugin`, `trust` will always be used, so that `tier1` can pass down its headers (ex: `X-Sf-Substreams-Cache-Tag`). + +* Added support for *continuous authentication* via the grpc auth plugin (allowing cutoff triggered by the auth system). + + ## v0.2.0 ### BREAKING CHANGES diff --git a/cmd/firesol/cli/firehose.go b/cmd/firesol/cli/firehose.go index 48e8e025..8a6e8b68 100644 --- a/cmd/firesol/cli/firehose.go +++ b/cmd/firesol/cli/firehose.go @@ -41,7 +41,7 @@ func init() { blockstreamAddr := viper.GetString("common-live-blocks-addr") // FIXME: That should be a shared dependencies across `EOSIO on StreamingFast` - authenticator, err := dauth.New(viper.GetString("common-auth-plugin")) + authenticator, err := dauth.New(viper.GetString("common-auth-plugin"), appLogger) if err != nil { return nil, fmt.Errorf("unable to initialize dauth: %w", err) } diff --git a/cmd/firesol/cli/substreams-tier1.go b/cmd/firesol/cli/substreams-tier1.go index e72dd9ce..bd1e9490 100644 --- a/cmd/firesol/cli/substreams-tier1.go +++ b/cmd/firesol/cli/substreams-tier1.go @@ -60,7 +60,6 @@ func init() { cmd.Flags().Bool("substreams-tier1-subrequests-insecure", false, "Connect to tier2 without checking certificate validity") cmd.Flags().Bool("substreams-tier1-subrequests-plaintext", true, "Connect to tier2 without client in plaintext mode") cmd.Flags().Int("substreams-tier1-max-subrequests", 4, "number of parallel subrequests that the tier1 can make to the tier2 per request") - cmd.Flags().Uint64("substreams-tier1-subrequests-size", 10000, "substreams subrequest block range size value for the scheduler") // all substreams registerCommonSubstreamsFlags(cmd) @@ -70,7 +69,7 @@ func init() { FactoryFunc: func(runtime *launcher.Runtime) (launcher.App, error) { blockstreamAddr := viper.GetString("common-live-blocks-addr") - authenticator, err := dauthAuthenticator.New(viper.GetString("common-auth-plugin")) + authenticator, err := dauthAuthenticator.New(viper.GetString("common-auth-plugin"), appLogger) if err != nil { return nil, fmt.Errorf("unable to initialize dauth: %w", err) } @@ -94,7 +93,6 @@ func init() { subrequestsInsecure := viper.GetBool("substreams-tier1-subrequests-insecure") subrequestsPlaintext := viper.GetBool("substreams-tier1-subrequests-plaintext") maxSubrequests := viper.GetUint64("substreams-tier1-max-subrequests") - subrequestsSize := viper.GetUint64("substreams-tier1-subrequests-size") tracing := os.Getenv("SUBSTREAMS_TRACING") == "modules_exec" @@ -122,7 +120,6 @@ func init() { StateBundleSize: stateBundleSize, BlockType: "sf.solana.type.v1.Block", MaxSubrequests: maxSubrequests, - SubrequestsSize: subrequestsSize, SubrequestsEndpoint: subrequestsEndpoint, SubrequestsInsecure: subrequestsInsecure, SubrequestsPlaintext: subrequestsPlaintext, diff --git a/go.mod b/go.mod index 3727e849..3fe1eb9c 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/spf13/viper v1.10.1 github.com/streamingfast/bstream v0.0.2-0.20230510131449-6b591d74130d github.com/streamingfast/cli v0.0.4-0.20220630165922-bc58c6666fc8 - github.com/streamingfast/dauth v0.0.0-20230719201351-ed7e87520891 + github.com/streamingfast/dauth v0.0.0-20230726175303-fc1d7198cb33 github.com/streamingfast/dbin v0.9.1-0.20220513054835-1abebbb944ad // indirect github.com/streamingfast/derr v0.0.0-20230515163924-8570aaa43fe1 github.com/streamingfast/dgrpc v0.0.0-20230621153617-bc715cdb9fd1 @@ -42,7 +42,7 @@ require ( github.com/streamingfast/sf-tools v0.0.0-20230217210059-2ab577f61e8e github.com/streamingfast/shutter v1.5.0 github.com/streamingfast/solana-go v0.5.1-0.20220502224452-432fbe84aee8 - github.com/streamingfast/substreams v1.1.9 + github.com/streamingfast/substreams v1.1.10 github.com/stretchr/testify v1.8.3 github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125 // indirect github.com/test-go/testify v1.1.4 diff --git a/go.sum b/go.sum index be620c4c..71a5cc15 100644 --- a/go.sum +++ b/go.sum @@ -2262,8 +2262,8 @@ github.com/streamingfast/cli v0.0.4-0.20220630165922-bc58c6666fc8 h1:1W2VZY4jRTh github.com/streamingfast/cli v0.0.4-0.20220630165922-bc58c6666fc8/go.mod h1:Lmirl0ABgXzidX33syiUG6lX64mJMK5pDNTBFuz8xZI= github.com/streamingfast/dauth v0.0.0-20210812020920-1c83ba29add1/go.mod h1:FIYpVqt+ICVuNBoOH3ZIicIctpVoCq3393+RpfXsPEM= github.com/streamingfast/dauth v0.0.0-20230711181128-22be70e6ca49/go.mod h1:zfq+mtesfbaZnNeh1BF+vo+zEFP1sat4pm3lvt40nRw= -github.com/streamingfast/dauth v0.0.0-20230719201351-ed7e87520891 h1:4zdqs+l7lB+bx6rMqR9AGF9g7Z4qVx5BIU7AJpZ7f1w= -github.com/streamingfast/dauth v0.0.0-20230719201351-ed7e87520891/go.mod h1:zfq+mtesfbaZnNeh1BF+vo+zEFP1sat4pm3lvt40nRw= +github.com/streamingfast/dauth v0.0.0-20230726175303-fc1d7198cb33 h1:MyOFvos5szhJWpoLYUeJ3UMlXBlL9kOhdD5LgEfDK48= +github.com/streamingfast/dauth v0.0.0-20230726175303-fc1d7198cb33/go.mod h1:zfq+mtesfbaZnNeh1BF+vo+zEFP1sat4pm3lvt40nRw= github.com/streamingfast/dbin v0.0.0-20210809205249-73d5eca35dc5/go.mod h1:YStE7K5/GH47JsWpY7LMKsDaXXpMLU/M26vYFzXHYRk= github.com/streamingfast/dbin v0.9.1-0.20220513054835-1abebbb944ad h1:6z4uS6TlD9KoHdyE1vzoGsELVCCcviTFT/3/vqCylh8= github.com/streamingfast/dbin v0.9.1-0.20220513054835-1abebbb944ad/go.mod h1:YStE7K5/GH47JsWpY7LMKsDaXXpMLU/M26vYFzXHYRk= @@ -2340,8 +2340,8 @@ github.com/streamingfast/shutter v1.5.0 h1:NpzDYzj0HVpSiDJVO/FFSL6QIK/YKOxY0gJAt github.com/streamingfast/shutter v1.5.0/go.mod h1:B/T6efqdeMGbGwjzPS1ToXzYZI4kDzI5/u4I+7qbjY8= github.com/streamingfast/solana-go v0.5.1-0.20220502224452-432fbe84aee8 h1:M9HcIwlbl7PipqrKPgNzfEWD9oxAA0mmjdRd2dTk9/4= github.com/streamingfast/solana-go v0.5.1-0.20220502224452-432fbe84aee8/go.mod h1:9NfZWSK0zqA+M1YU2pTI8sr1BfijCpqBFceLQARQiNw= -github.com/streamingfast/substreams v1.1.9 h1:477zJWpvADeZL8s9gUuB830Di3mzx8b24AjN+o8Nrpk= -github.com/streamingfast/substreams v1.1.9/go.mod h1:U/wDfXapixXmpnBwzQRMGBXhXJGaLZe6XbFhyh5dF18= +github.com/streamingfast/substreams v1.1.10 h1:ov/qmVmhvpRKEBRgxpmNzyKexJkZMKVX4xJ8kb205ho= +github.com/streamingfast/substreams v1.1.10/go.mod h1:9usbgylKcAAYUS+ADhvw05xTbzF9XqB/C+en3Na8r1Y= github.com/streamingfast/validator v0.0.0-20210812013448-b9da5752ce14/go.mod h1:t4h97mWfTs6v0zjEFuGDOoW5wLtu9+ttegIx99i7gsM= github.com/streamingfast/wasmtime-go/v4 v4.0.0-freemem3/go.mod h1:rOffzhrBM87FuXgj23Ss35uFDahjAauERq60QpyCzpE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=