diff --git a/devel/sink-local/start.sh b/devel/sink-local/start.sh index ac150f0..ab851c7 100755 --- a/devel/sink-local/start.sh +++ b/devel/sink-local/start.sh @@ -2,6 +2,7 @@ ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +project="$ROOT/../.." sink="$ROOT/../substreams-sink-files" main() { @@ -28,8 +29,8 @@ main() { "--encoder=lines" \ "--state-store=$output_dir/working/state.yaml" \ "${SUBSTREAMS_ENDPOINT:-"mainnet.eth.streamingfast.io:443"}" \ - "gs://staging.dfuseio-global.appspot.com/substreams/eth-token-transfers/spkg/substreams-v0.3.0.spkg" \ - "${SUBSTREAMS_MODULE:-"map_json_transfers"}" \ + "${project}/docs/tutorial/substreams.yaml" \ + "${SUBSTREAMS_MODULE:-"jsonl_out"}" \ "$output_dir/out" \ "$@" } diff --git a/encoder/lines.go b/encoder/lines.go index 6dd9e27..ad7d29f 100644 --- a/encoder/lines.go +++ b/encoder/lines.go @@ -7,7 +7,7 @@ import ( "github.com/golang/protobuf/proto" "github.com/streamingfast/substreams-sink-files/bundler/writer" - pbsinkfiles "github.com/streamingfast/substreams-sink-files/pb/substreams/sink/files/v1" + pbsinkfiles "github.com/streamingfast/substreams-sink-files/pb/sf/substreams/sink/files/v1" pbsubstreams "github.com/streamingfast/substreams/pb/sf/substreams/v1" ) diff --git a/encoder/lines_test.go b/encoder/lines_test.go index 0ccfd91..b96cd1b 100644 --- a/encoder/lines_test.go +++ b/encoder/lines_test.go @@ -6,7 +6,7 @@ import ( "github.com/streamingfast/bstream" "github.com/streamingfast/substreams-sink-files/bundler/writer" - pbsinkfiles "github.com/streamingfast/substreams-sink-files/pb/substreams/sink/files/v1" + pbsinkfiles "github.com/streamingfast/substreams-sink-files/pb/sf/substreams/sink/files/v1" pbsubstreams "github.com/streamingfast/substreams/pb/sf/substreams/v1" "github.com/stretchr/testify/assert" "github.com/test-go/testify/require" diff --git a/pb/last_generate.txt b/pb/last_generate.txt index 6c216f8..8b6e326 100644 --- a/pb/last_generate.txt +++ b/pb/last_generate.txt @@ -1,2 +1,2 @@ -generate.sh - Fri Jan 6 14:05:20 EST 2023 - maoueh -streamingfast/substreams-sink-files revision: cc76c8b3a7ab3e8053006a2831e8d27b7a1f82bb +generate.sh - Tue Jan 10 11:00:42 EST 2023 - maoueh +streamingfast/substreams-sink-files revision: c1fb775ba17785a7839593b41564c09ed92a307b diff --git a/pb/sf/substreams/sink/files/v1/files.pb.go b/pb/sf/substreams/sink/files/v1/files.pb.go new file mode 100644 index 0000000..785862c --- /dev/null +++ b/pb/sf/substreams/sink/files/v1/files.pb.go @@ -0,0 +1,167 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.27.0 +// protoc (unknown) +// source: sf/substreams/sink/files/v1/files.proto + +// The `sf.substreams.sink.files.v1` package can be use in your Substreams to defined a `map` module +// whose output type will be one of the message defined in the package. The `substreams-sink-files` +// binary will then consume your module's output to create the files containing your extracted data. +// +// The current package supports a single output format which is the `Lines` message which represents +// a list of plain-text "line" that should be appended together in a single bundle. + +package pbsinkfiles + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Lines represents an ordered list of lines that have been extracted of a single block. You are +// free the format each line as you please, the `substream-sink-files` tool does not make any +// assumption about the content and simply write the content to the current bundle with a trailing +// new line. +// +// It is expected that your line do **not** contain a new line character as it will be managed +// manually by the `substream-sink-files` tool. +// +// The most common use case is to use CSV or JSONL format for your line. For example, you can +// extract each transaction out of the block as a single line in JSON format as an object. The +// `substream-sink-files` will then package them in a bundle for N blocks. +type Lines struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Lines []string `protobuf:"bytes,1,rep,name=lines,proto3" json:"lines,omitempty"` +} + +func (x *Lines) Reset() { + *x = Lines{} + if protoimpl.UnsafeEnabled { + mi := &file_sf_substreams_sink_files_v1_files_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Lines) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Lines) ProtoMessage() {} + +func (x *Lines) ProtoReflect() protoreflect.Message { + mi := &file_sf_substreams_sink_files_v1_files_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Lines.ProtoReflect.Descriptor instead. +func (*Lines) Descriptor() ([]byte, []int) { + return file_sf_substreams_sink_files_v1_files_proto_rawDescGZIP(), []int{0} +} + +func (x *Lines) GetLines() []string { + if x != nil { + return x.Lines + } + return nil +} + +var File_sf_substreams_sink_files_v1_files_proto protoreflect.FileDescriptor + +var file_sf_substreams_sink_files_v1_files_proto_rawDesc = []byte{ + 0x0a, 0x27, 0x73, 0x66, 0x2f, 0x73, 0x75, 0x62, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x2f, + 0x73, 0x69, 0x6e, 0x6b, 0x2f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x2f, 0x76, 0x31, 0x2f, 0x66, 0x69, + 0x6c, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x1b, 0x73, 0x66, 0x2e, 0x73, 0x75, + 0x62, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x2e, 0x66, 0x69, + 0x6c, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x22, 0x1d, 0x0a, 0x05, 0x4c, 0x69, 0x6e, 0x65, 0x73, 0x12, + 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, + 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x42, 0x5b, 0x5a, 0x59, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x66, 0x61, 0x73, + 0x74, 0x2f, 0x73, 0x75, 0x62, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x2d, 0x73, 0x69, 0x6e, + 0x6b, 0x2d, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x66, 0x2f, 0x73, 0x75, + 0x62, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x2f, 0x73, 0x69, 0x6e, 0x6b, 0x2f, 0x66, 0x69, + 0x6c, 0x65, 0x73, 0x2f, 0x76, 0x31, 0x3b, 0x70, 0x62, 0x73, 0x69, 0x6e, 0x6b, 0x66, 0x69, 0x6c, + 0x65, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_sf_substreams_sink_files_v1_files_proto_rawDescOnce sync.Once + file_sf_substreams_sink_files_v1_files_proto_rawDescData = file_sf_substreams_sink_files_v1_files_proto_rawDesc +) + +func file_sf_substreams_sink_files_v1_files_proto_rawDescGZIP() []byte { + file_sf_substreams_sink_files_v1_files_proto_rawDescOnce.Do(func() { + file_sf_substreams_sink_files_v1_files_proto_rawDescData = protoimpl.X.CompressGZIP(file_sf_substreams_sink_files_v1_files_proto_rawDescData) + }) + return file_sf_substreams_sink_files_v1_files_proto_rawDescData +} + +var file_sf_substreams_sink_files_v1_files_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_sf_substreams_sink_files_v1_files_proto_goTypes = []interface{}{ + (*Lines)(nil), // 0: sf.substreams.sink.files.v1.Lines +} +var file_sf_substreams_sink_files_v1_files_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_sf_substreams_sink_files_v1_files_proto_init() } +func file_sf_substreams_sink_files_v1_files_proto_init() { + if File_sf_substreams_sink_files_v1_files_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_sf_substreams_sink_files_v1_files_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Lines); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_sf_substreams_sink_files_v1_files_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_sf_substreams_sink_files_v1_files_proto_goTypes, + DependencyIndexes: file_sf_substreams_sink_files_v1_files_proto_depIdxs, + MessageInfos: file_sf_substreams_sink_files_v1_files_proto_msgTypes, + }.Build() + File_sf_substreams_sink_files_v1_files_proto = out.File + file_sf_substreams_sink_files_v1_files_proto_rawDesc = nil + file_sf_substreams_sink_files_v1_files_proto_goTypes = nil + file_sf_substreams_sink_files_v1_files_proto_depIdxs = nil +} diff --git a/pb/substreams/sink/files/v1/files.pb.go b/pb/substreams/sink/files/v1/files.pb.go deleted file mode 100644 index bd5a355..0000000 --- a/pb/substreams/sink/files/v1/files.pb.go +++ /dev/null @@ -1,167 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.27.0 -// protoc (unknown) -// source: substreams/sink/files/v1/files.proto - -// The substreams.sink.files.v1 package can be use in your Substreams to defined a `map` module -// whose output type will be one of the message defined in the package. The `substreams-sink-files` -// binary will then consume your module's output to create the files containing your extracted data. -// -// The current package supports a single output format which is the `Lines` message which represents -// a list of plain-text "line" that should be appended together in a single bundle. - -package pbsinkfiles - -import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -// Lines represents an ordered list of lines that have been extracted of a single block. You are -// free the format each line as you please, the `substream-sink-files` tool does not make any -// assumption about the content and simply write the content to the current bundle with a trailing -// new line. -// -// It is expected that your line do **not** contain a new line character as it will be managed -// manually by the `substream-sink-files` tool. -// -// The most common use case is to use CSV or JSONL format for your line. For example, you can -// extract each transaction out of the block as a single line in JSON format as an object. The -// `substream-sink-files` will then package them in a bundle for N blocks. -type Lines struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Lines []string `protobuf:"bytes,1,rep,name=lines,proto3" json:"lines,omitempty"` -} - -func (x *Lines) Reset() { - *x = Lines{} - if protoimpl.UnsafeEnabled { - mi := &file_substreams_sink_files_v1_files_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Lines) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Lines) ProtoMessage() {} - -func (x *Lines) ProtoReflect() protoreflect.Message { - mi := &file_substreams_sink_files_v1_files_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Lines.ProtoReflect.Descriptor instead. -func (*Lines) Descriptor() ([]byte, []int) { - return file_substreams_sink_files_v1_files_proto_rawDescGZIP(), []int{0} -} - -func (x *Lines) GetLines() []string { - if x != nil { - return x.Lines - } - return nil -} - -var File_substreams_sink_files_v1_files_proto protoreflect.FileDescriptor - -var file_substreams_sink_files_v1_files_proto_rawDesc = []byte{ - 0x0a, 0x24, 0x73, 0x75, 0x62, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x2f, 0x73, 0x69, 0x6e, - 0x6b, 0x2f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x2f, 0x76, 0x31, 0x2f, 0x66, 0x69, 0x6c, 0x65, 0x73, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x18, 0x73, 0x75, 0x62, 0x73, 0x74, 0x72, 0x65, 0x61, - 0x6d, 0x73, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x2e, 0x76, 0x31, - 0x22, 0x1d, 0x0a, 0x05, 0x4c, 0x69, 0x6e, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6e, - 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x42, - 0x5b, 0x5a, 0x59, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x74, - 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x66, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x75, 0x62, 0x73, - 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x2d, 0x73, 0x69, 0x6e, 0x6b, 0x2d, 0x66, 0x69, 0x6c, 0x65, - 0x73, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x66, 0x2f, 0x73, 0x75, 0x62, 0x73, 0x74, 0x72, 0x65, 0x61, - 0x6d, 0x73, 0x2f, 0x73, 0x69, 0x6e, 0x6b, 0x2f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x2f, 0x76, 0x31, - 0x3b, 0x70, 0x62, 0x73, 0x69, 0x6e, 0x6b, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_substreams_sink_files_v1_files_proto_rawDescOnce sync.Once - file_substreams_sink_files_v1_files_proto_rawDescData = file_substreams_sink_files_v1_files_proto_rawDesc -) - -func file_substreams_sink_files_v1_files_proto_rawDescGZIP() []byte { - file_substreams_sink_files_v1_files_proto_rawDescOnce.Do(func() { - file_substreams_sink_files_v1_files_proto_rawDescData = protoimpl.X.CompressGZIP(file_substreams_sink_files_v1_files_proto_rawDescData) - }) - return file_substreams_sink_files_v1_files_proto_rawDescData -} - -var file_substreams_sink_files_v1_files_proto_msgTypes = make([]protoimpl.MessageInfo, 1) -var file_substreams_sink_files_v1_files_proto_goTypes = []interface{}{ - (*Lines)(nil), // 0: substreams.sink.files.v1.Lines -} -var file_substreams_sink_files_v1_files_proto_depIdxs = []int32{ - 0, // [0:0] is the sub-list for method output_type - 0, // [0:0] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name -} - -func init() { file_substreams_sink_files_v1_files_proto_init() } -func file_substreams_sink_files_v1_files_proto_init() { - if File_substreams_sink_files_v1_files_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_substreams_sink_files_v1_files_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Lines); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_substreams_sink_files_v1_files_proto_rawDesc, - NumEnums: 0, - NumMessages: 1, - NumExtensions: 0, - NumServices: 0, - }, - GoTypes: file_substreams_sink_files_v1_files_proto_goTypes, - DependencyIndexes: file_substreams_sink_files_v1_files_proto_depIdxs, - MessageInfos: file_substreams_sink_files_v1_files_proto_msgTypes, - }.Build() - File_substreams_sink_files_v1_files_proto = out.File - file_substreams_sink_files_v1_files_proto_rawDesc = nil - file_substreams_sink_files_v1_files_proto_goTypes = nil - file_substreams_sink_files_v1_files_proto_depIdxs = nil -} diff --git a/proto/substreams/sink/files/v1/files.proto b/proto/sf/substreams/sink/files/v1/files.proto similarity index 90% rename from proto/substreams/sink/files/v1/files.proto rename to proto/sf/substreams/sink/files/v1/files.proto index 9618333..5dab042 100644 --- a/proto/substreams/sink/files/v1/files.proto +++ b/proto/sf/substreams/sink/files/v1/files.proto @@ -1,12 +1,12 @@ syntax = "proto3"; -// The substreams.sink.files.v1 package can be use in your Substreams to defined a `map` module +// The `sf.substreams.sink.files.v1` package can be use in your Substreams to defined a `map` module // whose output type will be one of the message defined in the package. The `substreams-sink-files` // binary will then consume your module's output to create the files containing your extracted data. // // The current package supports a single output format which is the `Lines` message which represents // a list of plain-text "line" that should be appended together in a single bundle. -package substreams.sink.files.v1; +package sf.substreams.sink.files.v1; option go_package = "github.com/streamingfast/substreams-sink-files/pb/sf/substreams/sink/files/v1;pbsinkfiles"; diff --git a/substreams.yaml b/substreams.yaml index b0259ec..94d03c0 100644 --- a/substreams.yaml +++ b/substreams.yaml @@ -1,13 +1,13 @@ specVersion: v0.1.0 package: name: substreams_sink_files - version: v0.1.0 + version: v0.2.0 url: https://github.com/streamingfast/substreams-sink-files doc: | Protobuf definitions for interacting with Substreams Sink Files binary tool protobuf: files: - - substreams/sink/files/v1/files.proto + - sf/substreams/sink/files/v1/files.proto importPaths: - ./proto