From 10b6e2898d4feddd0a2aac787c5923023fc099f5 Mon Sep 17 00:00:00 2001 From: Dustin Decker Date: Fri, 28 Jul 2023 14:30:43 -0700 Subject: [PATCH 1/8] Increase log level of engine messages (#1576) --- pkg/engine/engine.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index b7a03ce596de..5119b4dc1557 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -126,7 +126,7 @@ func Start(ctx context.Context, options ...EngineOption) *Engine { ctx.Logger().Info("No concurrency specified, defaulting to max", "cpu", numCPU) e.concurrency = numCPU } - ctx.Logger().V(2).Info("engine started", "workers", e.concurrency) + ctx.Logger().V(3).Info("engine started", "workers", e.concurrency) // Limit number of concurrent goroutines dedicated to chunking a source. e.sourcesWg.SetLimit(e.concurrency) @@ -156,8 +156,8 @@ func Start(ctx context.Context, options ...EngineOption) *Engine { } e.prefilter = *ahocorasick.NewTrieBuilder().AddStrings(keywords).Build() - ctx.Logger().Info("loaded decoders", "count", len(e.decoders)) - ctx.Logger().Info("loaded detectors", + ctx.Logger().V(3).Info("loaded decoders", "count", len(e.decoders)) + ctx.Logger().V(3).Info("loaded detectors", "total", len(e.detectors[true])+len(e.detectors[false]), "verification_enabled", len(e.detectors[true]), "verification_disabled", len(e.detectors[false]), From 070014f380d770f565bdca4ec51d66b1366de807 Mon Sep 17 00:00:00 2001 From: Miccah Date: Mon, 31 Jul 2023 03:26:19 -0500 Subject: [PATCH 2/8] Initialize the default logger to output to stderr (#1569) --- pkg/context/context.go | 14 ++++++++++++-- pkg/context/context_test.go | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/pkg/context/context.go b/pkg/context/context.go index 61a9a2e62998..890ad1b3457c 100644 --- a/pkg/context/context.go +++ b/pkg/context/context.go @@ -3,18 +3,26 @@ package context import ( "context" "fmt" + "os" "runtime/debug" "sync" "time" "github.com/go-logr/logr" + "github.com/trufflesecurity/trufflehog/v3/pkg/log" ) var ( // defaultLogger can be set via SetDefaultLogger. - defaultLogger logr.Logger = logr.Discard() + // It is initialized to write to stderr. To disable, you can call + // SetDefaultLogger with logr.Discard(). + defaultLogger logr.Logger ) +func init() { + defaultLogger, _ = log.New("context", log.WithConsoleSink(os.Stderr)) +} + // Context wraps context.Context and includes an additional Logger() method. type Context interface { context.Context @@ -136,7 +144,9 @@ func AddLogger(parent context.Context) Context { } // SetupDefaultLogger sets the package-level global default logger that will be -// used for Background and TODO contexts. +// used for Background and TODO contexts. On startup, the default logger will +// be configured to output logs to stderr. Use logr.Discard() to disable all +// logs from Contexts. func SetDefaultLogger(l logr.Logger) { defaultLogger = l } diff --git a/pkg/context/context_test.go b/pkg/context/context_test.go index 6ef93227adde..5438540abd22 100644 --- a/pkg/context/context_test.go +++ b/pkg/context/context_test.go @@ -158,7 +158,7 @@ func TestWithValues(t *testing.T) { assert.NotContains(t, logs[6], `what does this do?`) } -func TestDiscardLogger(t *testing.T) { +func TestDefaultLogger(t *testing.T) { var panicked bool defer func() { if r := recover(); r != nil { From e0faac8d1c65ce15dffaeb827fa7586550dc793a Mon Sep 17 00:00:00 2001 From: Richard Gomez <32133502+rgmz@users.noreply.github.com> Date: Mon, 31 Jul 2023 09:57:42 -0400 Subject: [PATCH 3/8] Fix runtime error when scanning Gist comments (#1552) * fix(github): fix runtime error from gist comments * fix(github): add flag to scan Gist comments --- pkg/pb/sourcespb/sources.pb.go | 503 ++++++++++++------------ pkg/pb/sourcespb/sources.pb.validate.go | 2 + pkg/sources/github/github.go | 178 ++++++--- proto/sources.proto | 1 + 4 files changed, 387 insertions(+), 297 deletions(-) diff --git a/pkg/pb/sourcespb/sources.pb.go b/pkg/pb/sourcespb/sources.pb.go index c7419a93306c..cd629971c79b 100644 --- a/pkg/pb/sourcespb/sources.pb.go +++ b/pkg/pb/sourcespb/sources.pb.go @@ -1428,6 +1428,7 @@ type GitHub struct { IncludeRepos []string `protobuf:"bytes,12,rep,name=includeRepos,proto3" json:"includeRepos,omitempty"` IncludePullRequestComments bool `protobuf:"varint,14,opt,name=includePullRequestComments,proto3" json:"includePullRequestComments,omitempty"` IncludeIssueComments bool `protobuf:"varint,15,opt,name=includeIssueComments,proto3" json:"includeIssueComments,omitempty"` + IncludeGistComments bool `protobuf:"varint,16,opt,name=includeGistComments,proto3" json:"includeGistComments,omitempty"` } func (x *GitHub) Reset() { @@ -1574,6 +1575,13 @@ func (x *GitHub) GetIncludeIssueComments() bool { return false } +func (x *GitHub) GetIncludeGistComments() bool { + if x != nil { + return x.IncludeGistComments + } + return false +} + type isGitHub_Credential interface { isGitHub_Credential() } @@ -3328,7 +3336,7 @@ var file_sources_proto_rawDesc = []byte{ 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x42, 0x0c, - 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xfe, 0x04, 0x0a, + 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xb0, 0x05, 0x0a, 0x06, 0x47, 0x69, 0x74, 0x48, 0x75, 0x62, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, @@ -3367,264 +3375,267 @@ var file_sources_proto_rawDesc = []byte{ 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x32, 0x0a, 0x14, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x49, 0x73, 0x73, 0x75, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x69, 0x6e, 0x63, 0x6c, 0x75, - 0x64, 0x65, 0x49, 0x73, 0x73, 0x75, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x42, - 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x42, 0x0a, - 0x0b, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76, 0x65, 0x12, 0x25, 0x0a, 0x0d, - 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0c, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x6f, - 0x6b, 0x65, 0x6e, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, - 0x6c, 0x22, 0xc7, 0x02, 0x0a, 0x04, 0x4a, 0x49, 0x52, 0x41, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, - 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, - 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, - 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, - 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, - 0x62, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, - 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, - 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, - 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, - 0x74, 0x65, 0x64, 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, - 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, - 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x48, - 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, - 0x65, 0x63, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, - 0x65, 0x63, 0x74, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x70, - 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x69, - 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x42, 0x0c, 0x0a, - 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x73, 0x0a, 0x19, 0x4e, - 0x50, 0x4d, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, - 0x64, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, - 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x64, 0x65, 0x49, 0x73, 0x73, 0x75, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, + 0x30, 0x0a, 0x13, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x47, 0x69, 0x73, 0x74, 0x43, 0x6f, + 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x10, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x69, 0x6e, + 0x63, 0x6c, 0x75, 0x64, 0x65, 0x47, 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, + 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, + 0x42, 0x0a, 0x0b, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x44, 0x72, 0x69, 0x76, 0x65, 0x12, 0x25, + 0x0a, 0x0d, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0c, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, + 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, + 0x69, 0x61, 0x6c, 0x22, 0xc7, 0x02, 0x0a, 0x04, 0x4a, 0x49, 0x52, 0x41, 0x12, 0x24, 0x0a, 0x08, + 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, + 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, + 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, + 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, + 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x48, 0x0a, 0x0f, 0x75, + 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, + 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, + 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, + 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, + 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, + 0x74, 0x68, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, + 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, + 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, + 0x5f, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x0e, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x42, + 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x73, 0x0a, + 0x19, 0x4e, 0x50, 0x4d, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, + 0x74, 0x65, 0x64, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, + 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, + 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, + 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, + 0x61, 0x74, 0x65, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, + 0x61, 0x6c, 0x22, 0x74, 0x0a, 0x1a, 0x50, 0x79, 0x50, 0x49, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, + 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, + 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, + 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, + 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, + 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, + 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, + 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xf0, 0x02, 0x0a, 0x02, 0x53, 0x33, 0x12, + 0x37, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, + 0x73, 0x2e, 0x4b, 0x65, 0x79, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x48, 0x00, 0x52, 0x09, 0x61, + 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, + 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, - 0x65, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, - 0x22, 0x74, 0x0a, 0x1a, 0x50, 0x79, 0x50, 0x49, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, - 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, 0x48, - 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, - 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, - 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, - 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, - 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xf0, 0x02, 0x0a, 0x02, 0x53, 0x33, 0x12, 0x37, 0x0a, - 0x0a, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x65, 0x64, 0x12, 0x4c, 0x0a, 0x11, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x5f, 0x65, 0x6e, 0x76, 0x69, + 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, + 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x43, 0x6c, 0x6f, 0x75, + 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x10, + 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, + 0x12, 0x49, 0x0a, 0x0d, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x6f, 0x6b, 0x65, + 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, + 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x41, 0x57, 0x53, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x48, 0x00, 0x52, 0x0c, 0x73, + 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x62, + 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75, + 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6d, 0x61, 0x78, 0x5f, 0x6f, 0x62, 0x6a, + 0x65, 0x63, 0x74, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, + 0x6d, 0x61, 0x78, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x42, 0x0c, 0x0a, + 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xc3, 0x01, 0x0a, 0x05, + 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, + 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74, + 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, + 0x6b, 0x65, 0x6e, 0x12, 0x32, 0x0a, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, + 0x73, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x48, 0x00, 0x52, + 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, + 0x65, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, + 0x65, 0x6c, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, + 0x74, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, + 0x69, 0x73, 0x74, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, + 0x6c, 0x22, 0x06, 0x0a, 0x04, 0x54, 0x65, 0x73, 0x74, 0x22, 0x31, 0x0a, 0x09, 0x42, 0x75, 0x69, + 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x42, 0x0c, + 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xdb, 0x01, 0x0a, + 0x06, 0x47, 0x65, 0x72, 0x72, 0x69, 0x74, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, + 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, + 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, + 0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, - 0x4b, 0x65, 0x79, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x48, 0x00, 0x52, 0x09, 0x61, 0x63, 0x63, - 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, - 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, + 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, + 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, - 0x12, 0x4c, 0x0a, 0x11, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x5f, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, - 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x63, 0x72, - 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x45, - 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x10, 0x63, 0x6c, - 0x6f, 0x75, 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x49, - 0x0a, 0x0d, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, - 0x61, 0x6c, 0x73, 0x2e, 0x41, 0x57, 0x53, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, - 0x6b, 0x65, 0x6e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x48, 0x00, 0x52, 0x0c, 0x73, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75, 0x63, - 0x6b, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75, 0x63, 0x6b, - 0x65, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6d, 0x61, 0x78, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, - 0x74, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x6d, 0x61, - 0x78, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x42, 0x0c, 0x0a, 0x0a, 0x63, - 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xc3, 0x01, 0x0a, 0x05, 0x53, 0x6c, - 0x61, 0x63, 0x6b, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, - 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, - 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, - 0x6e, 0x12, 0x32, 0x0a, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x18, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, - 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x06, 0x74, - 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, - 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, - 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x18, - 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, - 0x74, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, - 0x06, 0x0a, 0x04, 0x54, 0x65, 0x73, 0x74, 0x22, 0x31, 0x0a, 0x09, 0x42, 0x75, 0x69, 0x6c, 0x64, - 0x6b, 0x69, 0x74, 0x65, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x42, 0x0c, 0x0a, 0x0a, - 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xdb, 0x01, 0x0a, 0x06, 0x47, - 0x65, 0x72, 0x72, 0x69, 0x74, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, - 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, - 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, 0x62, - 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, - 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, - 0x41, 0x75, 0x74, 0x68, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, - 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, - 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, - 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, - 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x1a, - 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, - 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xa5, 0x01, 0x0a, 0x07, 0x4a, 0x65, 0x6e, - 0x6b, 0x69, 0x6e, 0x73, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, + 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x42, 0x0c, 0x0a, 0x0a, + 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xa5, 0x01, 0x0a, 0x07, 0x4a, + 0x65, 0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, + 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, + 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, + 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, + 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, + 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x2d, 0x0a, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, + 0x61, 0x6c, 0x73, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x48, 0x00, 0x52, 0x06, 0x68, 0x65, + 0x61, 0x64, 0x65, 0x72, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, + 0x61, 0x6c, 0x22, 0x9f, 0x02, 0x0a, 0x05, 0x54, 0x65, 0x61, 0x6d, 0x73, 0x12, 0x24, 0x0a, 0x08, + 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, + 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, + 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x46, 0x0a, 0x0d, 0x61, 0x75, + 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1e, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, + 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, + 0x73, 0x48, 0x00, 0x52, 0x0d, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, + 0x65, 0x64, 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, + 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, + 0x1a, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x69, + 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x74, + 0x65, 0x61, 0x6d, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x74, + 0x65, 0x61, 0x6d, 0x49, 0x64, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, + 0x74, 0x69, 0x61, 0x6c, 0x22, 0xc3, 0x01, 0x0a, 0x0b, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, + 0x74, 0x6f, 0x72, 0x79, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, 0x41, - 0x75, 0x74, 0x68, 0x12, 0x2d, 0x0a, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, - 0x73, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x48, 0x00, 0x52, 0x06, 0x68, 0x65, 0x61, 0x64, - 0x65, 0x72, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, - 0x22, 0x9f, 0x02, 0x0a, 0x05, 0x54, 0x65, 0x61, 0x6d, 0x73, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, - 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, - 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, - 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, - 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x46, 0x0a, 0x0d, 0x61, 0x75, 0x74, 0x68, - 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1e, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x43, 0x6c, - 0x69, 0x65, 0x6e, 0x74, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x48, - 0x00, 0x52, 0x0d, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, - 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, - 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x1a, 0x0a, - 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x67, 0x6e, - 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x69, - 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x65, 0x61, - 0x6d, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x74, 0x65, 0x61, - 0x6d, 0x49, 0x64, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, - 0x61, 0x6c, 0x22, 0xc3, 0x01, 0x0a, 0x0b, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, - 0x72, 0x79, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, - 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61, 0x73, 0x69, - 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, - 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63, - 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, - 0x68, 0x12, 0x23, 0x0a, 0x0c, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x74, 0x6f, 0x6b, 0x65, - 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0b, 0x61, 0x63, 0x63, 0x65, 0x73, - 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, - 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, - 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, - 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x94, 0x01, 0x0a, 0x06, 0x53, 0x79, 0x73, - 0x6c, 0x6f, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, - 0x24, 0x0a, 0x0d, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x41, 0x64, - 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x6c, 0x73, 0x43, 0x65, 0x72, 0x74, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x74, 0x6c, 0x73, 0x43, 0x65, 0x72, 0x74, 0x12, - 0x16, 0x0a, 0x06, 0x74, 0x6c, 0x73, 0x4b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x74, 0x6c, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, - 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x22, - 0xd6, 0x01, 0x0a, 0x15, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4d, - 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, - 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, - 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, - 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, - 0x74, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x6d, - 0x61, 0x78, 0x5f, 0x64, 0x65, 0x70, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, - 0x6d, 0x61, 0x78, 0x44, 0x65, 0x70, 0x74, 0x68, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x69, 0x6e, 0x63, - 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, - 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x51, 0x0a, 0x0d, 0x53, 0x6c, 0x61, 0x63, - 0x6b, 0x52, 0x65, 0x61, 0x6c, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x32, 0x0a, 0x06, 0x74, 0x6f, 0x6b, - 0x65, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x63, 0x72, 0x65, 0x64, - 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x54, 0x6f, 0x6b, - 0x65, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x42, 0x0c, 0x0a, - 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x62, 0x0a, 0x0a, 0x53, - 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, - 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, - 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, - 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x69, 0x74, 0x65, 0x5f, 0x75, - 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x69, 0x74, 0x65, 0x55, 0x72, - 0x6c, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, - 0xa7, 0x03, 0x0a, 0x0a, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x24, - 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, - 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x2b, 0x0a, 0x05, - 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, - 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, - 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x70, - 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x12, 0x24, 0x0a, - 0x0d, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x05, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, - 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, - 0x22, 0x0a, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x46, 0x6f, 0x72, 0x6b, 0x73, 0x18, - 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x46, 0x6f, - 0x72, 0x6b, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x70, - 0x6f, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, - 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, - 0x52, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x69, 0x6e, 0x63, - 0x6c, 0x75, 0x64, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x28, 0x0a, 0x0f, 0x69, 0x6e, 0x63, - 0x6c, 0x75, 0x64, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x0a, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x0f, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, - 0x63, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0e, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, - 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x69, 0x67, 0x6e, - 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, - 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x2a, 0x84, 0x07, 0x0a, 0x0a, 0x53, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x53, 0x4f, 0x55, 0x52, - 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x53, 0x54, - 0x4f, 0x52, 0x41, 0x47, 0x45, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, 0x52, 0x43, - 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x49, 0x54, 0x42, 0x55, 0x43, 0x4b, 0x45, 0x54, - 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, - 0x45, 0x5f, 0x43, 0x49, 0x52, 0x43, 0x4c, 0x45, 0x43, 0x49, 0x10, 0x02, 0x12, 0x1a, 0x0a, 0x16, - 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4f, 0x4e, 0x46, - 0x4c, 0x55, 0x45, 0x4e, 0x43, 0x45, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, - 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x43, 0x4b, 0x45, 0x52, 0x10, 0x04, - 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, - 0x45, 0x43, 0x52, 0x10, 0x05, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, - 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x10, 0x06, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, + 0x75, 0x74, 0x68, 0x12, 0x23, 0x0a, 0x0c, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x74, 0x6f, + 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0b, 0x61, 0x63, 0x63, + 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x70, 0x6f, + 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, + 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x42, 0x0c, 0x0a, 0x0a, + 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x94, 0x01, 0x0a, 0x06, 0x53, + 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, + 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, + 0x6c, 0x12, 0x24, 0x0a, 0x0d, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x41, 0x64, 0x64, 0x72, 0x65, + 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, + 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x6c, 0x73, 0x43, 0x65, + 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x74, 0x6c, 0x73, 0x43, 0x65, 0x72, + 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x6c, 0x73, 0x4b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x74, 0x6c, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x72, + 0x6d, 0x61, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, + 0x74, 0x22, 0xd6, 0x01, 0x0a, 0x15, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x12, 0x48, 0x0a, 0x0f, 0x75, + 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, + 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, + 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, + 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x1b, 0x0a, + 0x09, 0x6d, 0x61, 0x78, 0x5f, 0x64, 0x65, 0x70, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x08, 0x6d, 0x61, 0x78, 0x44, 0x65, 0x70, 0x74, 0x68, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x69, + 0x6e, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x42, 0x0c, 0x0a, 0x0a, + 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x51, 0x0a, 0x0d, 0x53, 0x6c, + 0x61, 0x63, 0x6b, 0x52, 0x65, 0x61, 0x6c, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x32, 0x0a, 0x06, 0x74, + 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x63, 0x72, + 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x54, + 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x42, + 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x62, 0x0a, + 0x0a, 0x53, 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2b, 0x0a, 0x05, 0x6f, + 0x61, 0x75, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, + 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, + 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x69, 0x74, 0x65, + 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x69, 0x74, 0x65, + 0x55, 0x72, 0x6c, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, + 0x6c, 0x22, 0xa7, 0x03, 0x0a, 0x0a, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, + 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, + 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x2b, + 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, + 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, + 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x22, 0x0a, 0x0c, 0x72, + 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x12, + 0x24, 0x0a, 0x0d, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, + 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, + 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x46, 0x6f, 0x72, 0x6b, + 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, + 0x46, 0x6f, 0x72, 0x6b, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x52, + 0x65, 0x70, 0x6f, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x67, 0x6e, 0x6f, + 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, + 0x64, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x69, + 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x28, 0x0a, 0x0f, 0x69, + 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x0a, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x72, 0x6f, + 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0e, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x50, + 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x69, + 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x42, 0x0c, 0x0a, + 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x2a, 0x84, 0x07, 0x0a, 0x0a, + 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x53, 0x4f, + 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, + 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, + 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x49, 0x54, 0x42, 0x55, 0x43, 0x4b, + 0x45, 0x54, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, + 0x59, 0x50, 0x45, 0x5f, 0x43, 0x49, 0x52, 0x43, 0x4c, 0x45, 0x43, 0x49, 0x10, 0x02, 0x12, 0x1a, + 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4f, + 0x4e, 0x46, 0x4c, 0x55, 0x45, 0x4e, 0x43, 0x45, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, + 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x43, 0x4b, 0x45, 0x52, + 0x10, 0x04, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x45, 0x43, 0x52, 0x10, 0x05, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, + 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x10, 0x06, 0x12, 0x16, 0x0a, 0x12, + 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, + 0x55, 0x42, 0x10, 0x07, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, + 0x59, 0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x47, 0x49, 0x54, 0x10, 0x08, + 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, + 0x47, 0x49, 0x54, 0x4c, 0x41, 0x42, 0x10, 0x09, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52, + 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x49, 0x52, 0x41, 0x10, 0x0a, 0x12, 0x24, + 0x0a, 0x20, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4e, 0x50, + 0x4d, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, + 0x45, 0x53, 0x10, 0x0b, 0x12, 0x25, 0x0a, 0x21, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, + 0x59, 0x50, 0x45, 0x5f, 0x50, 0x59, 0x50, 0x49, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, + 0x5f, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, 0x10, 0x0c, 0x12, 0x12, 0x0a, 0x0e, 0x53, + 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x10, 0x0d, 0x12, + 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, + 0x4c, 0x41, 0x43, 0x4b, 0x10, 0x0e, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, + 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, + 0x10, 0x0f, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x47, 0x49, 0x54, 0x10, 0x10, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52, 0x43, + 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x53, 0x54, 0x10, 0x11, 0x12, 0x1b, 0x0a, + 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x5f, + 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x12, 0x12, 0x2a, 0x0a, 0x26, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, - 0x10, 0x07, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, - 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x47, 0x49, 0x54, 0x10, 0x08, 0x12, 0x16, - 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, - 0x54, 0x4c, 0x41, 0x42, 0x10, 0x09, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, - 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x49, 0x52, 0x41, 0x10, 0x0a, 0x12, 0x24, 0x0a, 0x20, - 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4e, 0x50, 0x4d, 0x5f, - 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, - 0x10, 0x0b, 0x12, 0x25, 0x0a, 0x21, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, - 0x45, 0x5f, 0x50, 0x59, 0x50, 0x49, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, - 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, 0x10, 0x0c, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x4f, 0x55, - 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x10, 0x0d, 0x12, 0x15, 0x0a, - 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41, - 0x43, 0x4b, 0x10, 0x0e, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, 0x10, 0x0f, - 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, - 0x47, 0x49, 0x54, 0x10, 0x10, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, - 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x53, 0x54, 0x10, 0x11, 0x12, 0x1b, 0x0a, 0x17, 0x53, - 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x5f, 0x55, 0x4e, - 0x41, 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x12, 0x12, 0x2a, 0x0a, 0x26, 0x53, 0x4f, 0x55, 0x52, - 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x5f, 0x55, - 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, 0x49, 0x43, 0x41, 0x54, 0x45, 0x44, 0x5f, 0x4f, - 0x52, 0x47, 0x10, 0x13, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x42, 0x55, 0x49, 0x4c, 0x44, 0x4b, 0x49, 0x54, 0x45, 0x10, 0x14, 0x12, - 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, - 0x45, 0x52, 0x52, 0x49, 0x54, 0x10, 0x15, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55, 0x52, 0x43, - 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x45, 0x4e, 0x4b, 0x49, 0x4e, 0x53, 0x10, 0x16, - 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, - 0x54, 0x45, 0x41, 0x4d, 0x53, 0x10, 0x17, 0x12, 0x21, 0x0a, 0x1d, 0x53, 0x4f, 0x55, 0x52, 0x43, - 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x46, 0x52, 0x4f, 0x47, 0x5f, 0x41, 0x52, 0x54, - 0x49, 0x46, 0x41, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10, 0x18, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, - 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x53, 0x4c, 0x4f, 0x47, - 0x10, 0x19, 0x12, 0x27, 0x0a, 0x23, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, - 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x4d, - 0x4f, 0x4e, 0x49, 0x54, 0x4f, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x1a, 0x12, 0x1e, 0x0a, 0x1a, 0x53, - 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41, 0x43, 0x4b, - 0x5f, 0x52, 0x45, 0x41, 0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x1b, 0x12, 0x1c, 0x0a, 0x18, 0x53, - 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x4f, 0x4f, 0x47, 0x4c, - 0x45, 0x5f, 0x44, 0x52, 0x49, 0x56, 0x45, 0x10, 0x1c, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, - 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x45, 0x50, 0x4f, - 0x49, 0x4e, 0x54, 0x10, 0x1d, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, - 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, - 0x44, 0x10, 0x1e, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, - 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x52, 0x45, 0x50, 0x4f, 0x53, 0x10, 0x1f, - 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, - 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, - 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, - 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, 0x49, 0x43, 0x41, 0x54, 0x45, 0x44, + 0x5f, 0x4f, 0x52, 0x47, 0x10, 0x13, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, + 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x55, 0x49, 0x4c, 0x44, 0x4b, 0x49, 0x54, 0x45, 0x10, + 0x14, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, + 0x5f, 0x47, 0x45, 0x52, 0x52, 0x49, 0x54, 0x10, 0x15, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55, + 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x45, 0x4e, 0x4b, 0x49, 0x4e, 0x53, + 0x10, 0x16, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x54, 0x45, 0x41, 0x4d, 0x53, 0x10, 0x17, 0x12, 0x21, 0x0a, 0x1d, 0x53, 0x4f, 0x55, + 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x46, 0x52, 0x4f, 0x47, 0x5f, 0x41, + 0x52, 0x54, 0x49, 0x46, 0x41, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10, 0x18, 0x12, 0x16, 0x0a, 0x12, + 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x53, 0x4c, + 0x4f, 0x47, 0x10, 0x19, 0x12, 0x27, 0x0a, 0x23, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, + 0x59, 0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, + 0x5f, 0x4d, 0x4f, 0x4e, 0x49, 0x54, 0x4f, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x1a, 0x12, 0x1e, 0x0a, + 0x1a, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41, + 0x43, 0x4b, 0x5f, 0x52, 0x45, 0x41, 0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x1b, 0x12, 0x1c, 0x0a, + 0x18, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x4f, 0x4f, + 0x47, 0x4c, 0x45, 0x5f, 0x44, 0x52, 0x49, 0x56, 0x45, 0x10, 0x1c, 0x12, 0x1a, 0x0a, 0x16, 0x53, + 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x45, + 0x50, 0x4f, 0x49, 0x4e, 0x54, 0x10, 0x1d, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x55, 0x52, 0x43, + 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, + 0x48, 0x45, 0x44, 0x10, 0x1e, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, + 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x52, 0x45, 0x50, 0x4f, 0x53, + 0x10, 0x1f, 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, + 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, + 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x70, 0x62, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/pkg/pb/sourcespb/sources.pb.validate.go b/pkg/pb/sourcespb/sources.pb.validate.go index ffb6c60d9628..70991754780e 100644 --- a/pkg/pb/sourcespb/sources.pb.validate.go +++ b/pkg/pb/sourcespb/sources.pb.validate.go @@ -1884,6 +1884,8 @@ func (m *GitHub) validate(all bool) error { // no validation rules for IncludeIssueComments + // no validation rules for IncludeGistComments + switch m.Credential.(type) { case *GitHub_GithubApp: diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index 5ccee2df2163..b959ca009ff4 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -70,6 +70,7 @@ type Source struct { publicMap map[string]source_metadatapb.Visibility includePRComments bool includeIssueComments bool + includeGistComments bool sources.Progress sources.CommonSourceUnitUnmarshaller } @@ -217,6 +218,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID, sourceID int64, s.includeIssueComments = s.conn.IncludeIssueComments s.includePRComments = s.conn.IncludePullRequestComments + s.includeGistComments = s.conn.IncludeGistComments s.orgsCache = memory.New() for _, org := range s.conn.Organizations { @@ -697,15 +699,16 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch logger.V(2).Info(fmt.Sprintf("scanned %d/%d repos", scanned, len(s.repos)), "repo_size", repoSize, "duration_seconds", time.Since(start).Seconds()) }(now) - if err = s.scanComments(ctx, repoURL, chunksChan); err != nil { - scanErrs.Add(fmt.Errorf("error scanning comments in repo %s: %w", repoURL, err)) + if err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan); err != nil { + scanErrs.Add(fmt.Errorf("error scanning repo %s: %w", repoURL, err)) return nil } - if err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan); err != nil { - scanErrs.Add(fmt.Errorf("error scanning repo %s: %w", repoURL, err)) + if err = s.scanComments(ctx, repoURL, chunksChan); err != nil { + scanErrs.Add(fmt.Errorf("error scanning comments in repo %s: %w", repoURL, err)) return nil } + atomic.AddUint64(&scanned, 1) return nil @@ -966,79 +969,118 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan c } trimmedURL := removeURLAndSplit(repoURL.String()) - owner := trimmedURL[1] - repo := trimmedURL[2] - - var ( - sortType = "created" - directionType = "desc" - allComments = 0 - ) - - if s.includeIssueComments { - - issueOpts := &github.IssueListCommentsOptions{ - Sort: &sortType, - Direction: &directionType, - ListOptions: github.ListOptions{ - PerPage: defaultPagination, - Page: 1, - }, + if repoURL.Host == "gist.github.com" && s.includeGistComments { + // GitHub Gist URL. + var gistId string + if len(trimmedURL) == 2 { + // https://gist.github.com/ + gistId = trimmedURL[1] + } else if len(trimmedURL) == 3 { + // https://gist.github.com// + gistId = trimmedURL[2] + } else { + return fmt.Errorf("failed to parse Gist URL: '%s'", repoURL.String()) } + options := &github.ListOptions{ + PerPage: defaultPagination, + Page: 1, + } for { - issueComments, resp, err := s.apiClient.Issues.ListComments(ctx, owner, repo, allComments, issueOpts) + comments, resp, err := s.apiClient.Gists.ListComments(ctx, gistId, options) if s.handleRateLimit(err, resp) { break } - if err != nil { return err } - err = s.chunkIssueComments(ctx, repo, issueComments, chunksChan, repoPath) + err = s.chunkGistComments(ctx, repoURL.String(), comments, chunksChan) if err != nil { return err } - issueOpts.ListOptions.Page++ - - if len(issueComments) < defaultPagination { + options.Page++ + if len(comments) < options.PerPage { break } } + } else { + // Normal repository URL (https://github.com//). + owner := trimmedURL[1] + repo := trimmedURL[2] + + var ( + sortType = "created" + directionType = "desc" + allComments = 0 + ) - } - - if s.includePRComments { - prOpts := &github.PullRequestListCommentsOptions{ - Sort: sortType, - Direction: directionType, - ListOptions: github.ListOptions{ - PerPage: defaultPagination, - Page: 1, - }, - } + if s.includeIssueComments { - for { - prComments, resp, err := s.apiClient.PullRequests.ListComments(ctx, owner, repo, allComments, prOpts) - if s.handleRateLimit(err, resp) { - break + issueOpts := &github.IssueListCommentsOptions{ + Sort: &sortType, + Direction: &directionType, + ListOptions: github.ListOptions{ + PerPage: defaultPagination, + Page: 1, + }, } - if err != nil { - return err + for { + issueComments, resp, err := s.apiClient.Issues.ListComments(ctx, owner, repo, allComments, issueOpts) + if s.handleRateLimit(err, resp) { + break + } + + if err != nil { + return err + } + + err = s.chunkIssueComments(ctx, repo, issueComments, chunksChan, repoPath) + if err != nil { + return err + } + + issueOpts.ListOptions.Page++ + + if len(issueComments) < defaultPagination { + break + } } - err = s.chunkPullRequestComments(ctx, repo, prComments, chunksChan, repoPath) - if err != nil { - return err + } + + if s.includePRComments { + prOpts := &github.PullRequestListCommentsOptions{ + Sort: sortType, + Direction: directionType, + ListOptions: github.ListOptions{ + PerPage: defaultPagination, + Page: 1, + }, } - prOpts.ListOptions.Page++ + for { + prComments, resp, err := s.apiClient.PullRequests.ListComments(ctx, owner, repo, allComments, prOpts) + if s.handleRateLimit(err, resp) { + break + } - if len(prComments) < defaultPagination { - break + if err != nil { + return err + } + + err = s.chunkPullRequestComments(ctx, repo, prComments, chunksChan, repoPath) + if err != nil { + return err + } + + prOpts.ListOptions.Page++ + + if len(prComments) < defaultPagination { + break + } } } } @@ -1109,6 +1151,40 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comm return nil } +func (s *Source) chunkGistComments(ctx context.Context, gistUrl string, comments []*github.GistComment, chunksChan chan *sources.Chunk) error { + for _, comment := range comments { + // Create chunk and send it to the channel. + chunk := &sources.Chunk{ + SourceName: s.name, + SourceID: s.SourceID(), + SourceType: s.Type(), + SourceMetadata: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Github{ + Github: &source_metadatapb.Github{ + Link: sanitizer.UTF8(comment.GetURL()), + Username: sanitizer.UTF8(comment.GetUser().GetLogin()), + Email: sanitizer.UTF8(comment.GetUser().GetEmail()), + Repository: sanitizer.UTF8(gistUrl), + Timestamp: sanitizer.UTF8(comment.GetCreatedAt().String()), + // Fetching this information requires making an additional API call. + // We may want to include this in the future. + //Visibility: s.visibilityOf(ctx, repoPath), + }, + }, + }, + Data: []byte(sanitizer.UTF8(comment.GetBody())), + Verify: s.verify, + } + + select { + case <-ctx.Done(): + return ctx.Err() + case chunksChan <- chunk: + } + } + return nil +} + func removeURLAndSplit(url string) []string { trimmedURL := strings.TrimPrefix(url, "https://") trimmedURL = strings.TrimSuffix(trimmedURL, ".git") diff --git a/proto/sources.proto b/proto/sources.proto index d9e1ae7c3787..6c70b9a26b5b 100644 --- a/proto/sources.proto +++ b/proto/sources.proto @@ -185,6 +185,7 @@ message GitHub { repeated string includeRepos = 12; bool includePullRequestComments = 14; bool includeIssueComments = 15; + bool includeGistComments = 16; } message GoogleDrive { From ad57de50cd5cbb795f3330d6f3bc9757c9b3aa69 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Mon, 31 Jul 2023 11:31:16 -0400 Subject: [PATCH 4/8] Do not nest transports for Github installation client (#1564) #1454 modified one of the Github enumeration code paths in a way that broke an integration test by causing one client's transport to be used for the construction of a different client, causing authentication failures. This saves the original transport for use, fixing the test. --- pkg/sources/github/github.go | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index b959ca009ff4..9d6d0532991e 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -556,38 +556,41 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app * return nil, errors.New(err) } - // This client is used for most APIs. - itr, err := ghinstallation.New( + // This client is required to create installation tokens for cloning. + // Otherwise, the required JWT is not in the request for the token :/ + // This client uses the source's original HTTP transport, so make sure + // to build it before modifying that transport (such as is done during + // the creation of the other API client below). + appItr, err := ghinstallation.NewAppsTransport( s.httpClient.Transport, appID, - installationID, []byte(app.PrivateKey)) if err != nil { return nil, errors.New(err) } - itr.BaseURL = apiEndpoint + appItr.BaseURL = apiEndpoint - s.httpClient.Transport = itr - s.apiClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, s.httpClient) + // Does this need to be separate from |s.httpClient|? + instHttpClient := common.RetryableHttpClientTimeout(60) + instHttpClient.Transport = appItr + installationClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, instHttpClient) if err != nil { return nil, errors.New(err) } - // This client is required to create installation tokens for cloning. - // Otherwise, the required JWT is not in the request for the token :/ - appItr, err := ghinstallation.NewAppsTransport( + // This client is used for most APIs. + itr, err := ghinstallation.New( s.httpClient.Transport, appID, + installationID, []byte(app.PrivateKey)) if err != nil { return nil, errors.New(err) } - appItr.BaseURL = apiEndpoint + itr.BaseURL = apiEndpoint - // Does this need to be separate from |s.httpClient|? - instHttpClient := common.RetryableHttpClientTimeout(60) - instHttpClient.Transport = appItr - installationClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, instHttpClient) + s.httpClient.Transport = itr + s.apiClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, s.httpClient) if err != nil { return nil, errors.New(err) } From 61bee6c8b100d63fa066df1bac25c9cc09d1c6dc Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Mon, 31 Jul 2023 12:06:11 -0400 Subject: [PATCH 5/8] Identify transient AWS verification failures (#1563) It turns out that GetCallerIdentity returns a surprising quantity of transient, false-negative 403 responses that carry the SignatureDoesNotMatch error reason. I don't know why this is happening, but their transient nature makes them indeterminate verification failures and they should be flagged as such. The AWS detector has therefore been modified to specifically look for the InvalidClientTokenId error reason in 403 responses and mark all other responses as indeterminate. In addition to the functional changes this PR contains some updates to the test code that allow us to test them. --- pkg/common/http.go | 4 +- pkg/detectors/alchemy/alchemy_test.go | 2 +- pkg/detectors/aws/aws.go | 43 +++++++++++++++++++--- pkg/detectors/aws/aws_test.go | 53 ++++++++++++++++++++++----- 4 files changed, 84 insertions(+), 18 deletions(-) diff --git a/pkg/common/http.go b/pkg/common/http.go index 56e587fde354..1929ca927897 100644 --- a/pkg/common/http.go +++ b/pkg/common/http.go @@ -100,14 +100,14 @@ func NewCustomTransport(T http.RoundTripper) *CustomTransport { return &CustomTransport{T} } -func ConstantStatusHttpClient(statusCode int) *http.Client { +func ConstantResponseHttpClient(statusCode int, body string) *http.Client { return &http.Client{ Timeout: DefaultResponseTimeout, Transport: FakeTransport{ CreateResponse: func(req *http.Request) (*http.Response, error) { return &http.Response{ Request: req, - Body: io.NopCloser(strings.NewReader("")), + Body: io.NopCloser(strings.NewReader(body)), StatusCode: statusCode, }, nil }, diff --git a/pkg/detectors/alchemy/alchemy_test.go b/pkg/detectors/alchemy/alchemy_test.go index ed2ae7b8dbef..0e14eff65336 100644 --- a/pkg/detectors/alchemy/alchemy_test.go +++ b/pkg/detectors/alchemy/alchemy_test.go @@ -104,7 +104,7 @@ func TestAlchemy_FromChunk(t *testing.T) { }, { name: "found, verified but unexpected api surface", - s: Scanner{client: common.ConstantStatusHttpClient(404)}, + s: Scanner{client: common.ConstantResponseHttpClient(404, "")}, args: args{ ctx: context.Background(), data: []byte(fmt.Sprintf("You can find a alchemy secret %s within", secret)), diff --git a/pkg/detectors/aws/aws.go b/pkg/detectors/aws/aws.go index b9f96c0619b2..a0177de13ede 100644 --- a/pkg/detectors/aws/aws.go +++ b/pkg/detectors/aws/aws.go @@ -18,7 +18,8 @@ import ( ) type scanner struct { - skipIDs map[string]struct{} + verificationClient *http.Client + skipIDs map[string]struct{} } func New(opts ...func(*scanner)) *scanner { @@ -48,7 +49,7 @@ func WithSkipIDs(skipIDs []string) func(*scanner) { var _ detectors.Detector = (*scanner)(nil) var ( - client = common.SaneHttpClient() + defaultVerificationClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. // Key types are from this list https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids @@ -123,8 +124,9 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result host := "sts.amazonaws.com" region := "us-east-1" endpoint := "https://sts.amazonaws.com" - datestamp := time.Now().UTC().Format("20060102") - amzDate := time.Now().UTC().Format("20060102T150405Z0700") + now := time.Now().UTC() + datestamp := now.Format("20060102") + amzDate := now.Format("20060102T150405Z0700") req, err := http.NewRequestWithContext(ctx, method, endpoint, nil) if err != nil { @@ -171,6 +173,10 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8") req.URL.RawQuery = params.Encode() + client := s.verificationClient + if client == nil { + client = defaultVerificationClient + } res, err := client.Do(req) if err == nil { @@ -194,7 +200,25 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result continue } - if res.StatusCode != 403 { + if res.StatusCode == 403 { + var body awsErrorResponseBody + err = json.NewDecoder(res.Body).Decode(&body) + res.Body.Close() + if err == nil { + // All instances of the code I've seen in the wild are PascalCased but this check is + // case-insensitive out of an abundance of caution + if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") { + // determinate failure - nothing to do + } else { + // We see a surprising number of false-negative signature mismatch errors here + // (The official SDK somehow elicits even more than just making the request + // ourselves) + s1.VerificationError = fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message) + } + } else { + s1.VerificationError = fmt.Errorf("couldn't parse the sts response body (%v)", err) + } + } else { s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode) } } @@ -245,6 +269,15 @@ func awsCustomCleanResults(results []detectors.Result) []detectors.Result { return out } +type awsError struct { + Code string `json:"Code"` + Message string `json:"Message"` +} + +type awsErrorResponseBody struct { + Error awsError `json:"Error"` +} + type identityRes struct { GetCallerIdentityResponse struct { GetCallerIdentityResult struct { diff --git a/pkg/detectors/aws/aws_test.go b/pkg/detectors/aws/aws_test.go index 1dfecccd3d1c..582c1ff3c88d 100644 --- a/pkg/detectors/aws/aws_test.go +++ b/pkg/detectors/aws/aws_test.go @@ -17,6 +17,8 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) +var unverifiedSecretClient = common.ConstantResponseHttpClient(403, `{"Error": {"Code": "InvalidClientTokenId"} }`) + func TestAWS_FromChunk(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() @@ -69,7 +71,7 @@ func TestAWS_FromChunk(t *testing.T) { }, { name: "found, unverified", - s: scanner{}, + s: scanner{verificationClient: unverifiedSecretClient}, args: args{ ctx: context.Background(), data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation @@ -163,7 +165,7 @@ func TestAWS_FromChunk(t *testing.T) { }, { name: "found, unverified, with leading +", - s: scanner{}, + s: scanner{verificationClient: unverifiedSecretClient}, args: args{ ctx: context.Background(), data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation @@ -194,9 +196,45 @@ func TestAWS_FromChunk(t *testing.T) { }, { name: "found, would be verified if not for http timeout", - s: scanner{}, + s: scanner{verificationClient: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, args: args{ - ctx: timeoutContext(1 * time.Microsecond), + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AWS, + Verified: false, + Redacted: "AKIASP2TPHJSQH3FJRUX", + }, + }, + wantErr: false, + wantVerificationError: true, + }, + { + name: "found, unverified due to unexpected http response status", + s: scanner{verificationClient: common.ConstantResponseHttpClient(500, "internal server error")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AWS, + Verified: false, + Redacted: "AKIASP2TPHJSQH3FJRUX", + }, + }, + wantErr: false, + wantVerificationError: true, + }, + { + name: "found, unverified due to unexpected 403 response reason", + s: scanner{verificationClient: common.ConstantResponseHttpClient(403, `{"Error": {"Code": "SignatureDoesNotMatch"} }`)}, + args: args{ + ctx: context.Background(), data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)), verify: true, }, @@ -224,7 +262,7 @@ func TestAWS_FromChunk(t *testing.T) { t.Fatalf("no raw secret present: \n %+v", got[i]) } if (got[i].VerificationError != nil) != tt.wantVerificationError { - t.Fatalf("verification error = %v, wantVerificationError %v", got[i].VerificationError, tt.wantVerificationError) + t.Fatalf("wantVerificationError %v, verification error = %v", tt.wantVerificationError, got[i].VerificationError) } } ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "VerificationError") @@ -249,8 +287,3 @@ func BenchmarkFromData(benchmark *testing.B) { }) } } - -func timeoutContext(timeout time.Duration) context.Context { - c, _ := context.WithTimeout(context.Background(), timeout) - return c -} From a07b6664f82c89a292870df2460d5be2a94f4f24 Mon Sep 17 00:00:00 2001 From: Miccah Date: Mon, 31 Jul 2023 11:28:30 -0500 Subject: [PATCH 6/8] Support fatal errors in job reports (#1562) * Support fatal errors in job reports * WIP: JobReporter and JobInspector * WIP: JobReportHook and JobReportRef * Add ChunkError type and asyncRun helper method * Rename JobReport to JobProgress * Return a closed channel from Done when the JobProgress is nil * Comment catchFirstFatal function --- go.mod | 2 + go.sum | 2 + pkg/sources/errors.go | 9 + pkg/sources/job_progress.go | 272 ++++++++++++++++++++++++++ pkg/sources/job_progress_test.go | 117 +++++++++++ pkg/sources/job_report.go | 37 ---- pkg/sources/mock_job_progress_test.go | 155 +++++++++++++++ pkg/sources/source_manager.go | 243 ++++++++++------------- pkg/sources/source_manager_test.go | 33 ++-- 9 files changed, 680 insertions(+), 190 deletions(-) create mode 100644 pkg/sources/job_progress.go create mode 100644 pkg/sources/job_progress_test.go delete mode 100644 pkg/sources/job_report.go create mode 100644 pkg/sources/mock_job_progress_test.go diff --git a/go.mod b/go.mod index f2b78234e107..faff10cde342 100644 --- a/go.mod +++ b/go.mod @@ -34,6 +34,7 @@ require ( github.com/go-sql-driver/mysql v1.7.1 github.com/gobwas/glob v0.2.3 github.com/golang-jwt/jwt v3.2.2+incompatible + github.com/golang/mock v1.4.3 github.com/google/go-cmp v0.5.9 github.com/google/go-containerregistry v0.15.2 github.com/google/go-github/v42 v42.0.0 @@ -56,6 +57,7 @@ require ( github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 github.com/xanzy/go-gitlab v0.88.0 go.mongodb.org/mongo-driver v1.12.0 + go.uber.org/mock v0.2.0 go.uber.org/zap v1.24.0 golang.org/x/crypto v0.11.0 golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7 diff --git a/go.sum b/go.sum index 62f9da20c35b..1185501099c6 100644 --- a/go.sum +++ b/go.sum @@ -486,6 +486,8 @@ go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/mock v0.2.0 h1:TaP3xedm7JaAgScZO7tlvlKrqT0p7I6OsdGB5YNSMDU= +go.uber.org/mock v0.2.0/go.mod h1:J0y0rp9L3xiff1+ZBfKxlC1fz2+aO16tw0tsDOixfuM= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= diff --git a/pkg/sources/errors.go b/pkg/sources/errors.go index 01e464e4401f..3ec62a74d6af 100644 --- a/pkg/sources/errors.go +++ b/pkg/sources/errors.go @@ -1,6 +1,7 @@ package sources import ( + "errors" "fmt" "sync" ) @@ -32,5 +33,13 @@ func (s *ScanErrors) Count() uint64 { } func (s *ScanErrors) String() string { + s.mu.RLock() + defer s.mu.RUnlock() return fmt.Sprintf("%v", s.errors) } + +func (s *ScanErrors) Errors() error { + s.mu.RLock() + defer s.mu.RUnlock() + return errors.Join(s.errors...) +} diff --git a/pkg/sources/job_progress.go b/pkg/sources/job_progress.go new file mode 100644 index 000000000000..50605485db66 --- /dev/null +++ b/pkg/sources/job_progress.go @@ -0,0 +1,272 @@ +package sources + +//go:generate mockgen --source=./job_progress.go --destination=mock_job_progress_test.go --package=sources + +import ( + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + "time" +) + +type JobProgressHook interface { + // Start and End marks the overall start and end time for this job. + Start(JobProgressRef, time.Time) + End(JobProgressRef, time.Time) + // StartEnumerating and EndEnumerating marks the start and end time for + // calling the source's Enumerate method. If the source does not + // support enumeration these methods will never be called. + StartEnumerating(JobProgressRef, time.Time) + EndEnumerating(JobProgressRef, time.Time) + // StartUnitChunking and EndUnitChunking marks the start and end time + // for calling the source's ChunkUnit method for a given unit. If the + // source does not support enumeration these methods will never be + // called. + StartUnitChunking(JobProgressRef, SourceUnit, time.Time) + EndUnitChunking(JobProgressRef, SourceUnit, time.Time) + // ReportError is called when any general error is encountered, usually + // from enumeration. + ReportError(JobProgressRef, error) + // ReportUnit is called when a unit has been enumerated. If the source + // does not support enumeration this method will never be called. + ReportUnit(JobProgressRef, SourceUnit) + // ReportChunk is called when a chunk has been produced for the given + // unit. The unit will be nil if the source does not support + // enumeration. + ReportChunk(JobProgressRef, SourceUnit, *Chunk) + // Finish marks the job as done. + Finish(JobProgressRef) +} + +// JobProgressRef is a wrapper of a JobProgress for read-only access to its state. +type JobProgressRef struct { + SourceID int64 + JobID int64 + jobProgress *JobProgress +} + +// Snapshot returns a snapshot of the job's current metrics. +func (r *JobProgressRef) Snapshot() JobProgressMetrics { + if r.jobProgress == nil { + return JobProgressMetrics{} + } + return r.jobProgress.Snapshot() +} + +// Done returns a channel that will block until the job has completed. +func (r *JobProgressRef) Done() <-chan struct{} { + if r.jobProgress == nil { + // Return a closed channel so it does not block. + ch := make(chan struct{}) + close(ch) + return ch + } + return r.jobProgress.Done() +} + +// Fatal is a wrapper around error to differentiate non-fatal errors from fatal +// ones. A fatal error is typically from a finished context or any error +// returned from a source's Init, Chunks, Enumerate, or ChunkUnit methods. +type Fatal struct{ error } + +func (f Fatal) Error() string { return fmt.Sprintf("fatal: %s", f.error.Error()) } +func (f Fatal) Unwrap() error { return f.error } + +// ChunkError is a custom error type for errors encountered during chunking of +// a specific unit. +type ChunkError struct { + unit SourceUnit + err error +} + +func (f ChunkError) Error() string { + return fmt.Sprintf("error chunking unit %q: %s", f.unit.SourceUnitID(), f.err.Error()) +} +func (f ChunkError) Unwrap() error { return f.err } + +// JobProgress aggregates information about a run of a Source. +type JobProgress struct { + // Unique identifiers for this job. + SourceID int64 + JobID int64 + // Tracks whether the job is finished or not. + ctx context.Context + cancel context.CancelFunc + // Metrics. + metrics JobProgressMetrics + metricsLock sync.Mutex + // Coarse grained hooks for adding extra functionality when events trigger. + hooks []JobProgressHook +} + +// JobProgressMetrics tracks the metrics of a job. +type JobProgressMetrics struct { + StartTime time.Time + EndTime time.Time + TotalUnits uint64 + FinishedUnits uint64 + TotalChunks uint64 + Errors []error + DoneEnumerating bool +} + +// WithHooks adds hooks to be called when an event triggers. +func WithHooks(hooks ...JobProgressHook) func(*JobProgress) { + return func(jp *JobProgress) { jp.hooks = append(jp.hooks, hooks...) } +} + +// NewJobProgress creates a new job report for the given source and job ID. +func NewJobProgress(sourceID, jobID int64, opts ...func(*JobProgress)) *JobProgress { + ctx, cancel := context.WithCancel(context.Background()) + jp := &JobProgress{ + SourceID: sourceID, + JobID: jobID, + ctx: ctx, + cancel: cancel, + } + for _, opt := range opts { + opt(jp) + } + return jp +} + +// executeHooks is a helper method to execute all the hooks for the given +// closure. +func (jp *JobProgress) executeHooks(todo func(hook JobProgressHook)) { + for _, hook := range jp.hooks { + // TODO: Non-blocking? + todo(hook) + } +} + +// TODO: Comment all this mess. They are mostly implementing JobProgressHook but +// without the JobProgressRef parameter. +func (jp *JobProgress) Start(start time.Time) { + jp.metricsLock.Lock() + jp.metrics.StartTime = start + jp.metricsLock.Unlock() + + jp.executeHooks(func(hook JobProgressHook) { hook.Start(jp.Ref(), start) }) +} +func (jp *JobProgress) End(end time.Time) { + jp.metricsLock.Lock() + jp.metrics.EndTime = end + jp.metricsLock.Unlock() + + jp.executeHooks(func(hook JobProgressHook) { hook.End(jp.Ref(), end) }) +} +func (jp *JobProgress) Finish() { + jp.cancel() + jp.executeHooks(func(hook JobProgressHook) { hook.Finish(jp.Ref()) }) +} +func (jp *JobProgress) Done() <-chan struct{} { return jp.ctx.Done() } +func (jp *JobProgress) ReportUnit(unit SourceUnit) { + atomic.AddUint64(&jp.metrics.TotalUnits, 1) + jp.executeHooks(func(hook JobProgressHook) { hook.ReportUnit(jp.Ref(), unit) }) +} +func (jp *JobProgress) ReportChunk(unit SourceUnit, chunk *Chunk) { + atomic.AddUint64(&jp.metrics.TotalChunks, 1) + jp.executeHooks(func(hook JobProgressHook) { hook.ReportChunk(jp.Ref(), unit, chunk) }) +} +func (jp *JobProgress) StartUnitChunking(unit SourceUnit, start time.Time) { + // TODO: Record time. + jp.executeHooks(func(hook JobProgressHook) { hook.StartUnitChunking(jp.Ref(), unit, start) }) +} +func (jp *JobProgress) EndUnitChunking(unit SourceUnit, end time.Time) { + // TODO: Record time. + atomic.AddUint64(&jp.metrics.FinishedUnits, 1) + jp.executeHooks(func(hook JobProgressHook) { hook.EndUnitChunking(jp.Ref(), unit, end) }) +} +func (jp *JobProgress) StartEnumerating(start time.Time) { + // TODO: Record time. + jp.executeHooks(func(hook JobProgressHook) { hook.StartEnumerating(jp.Ref(), start) }) +} +func (jp *JobProgress) EndEnumerating(end time.Time) { + jp.metricsLock.Lock() + // TODO: Record time. + jp.metrics.DoneEnumerating = true + jp.metricsLock.Unlock() + + jp.executeHooks(func(hook JobProgressHook) { hook.EndEnumerating(jp.Ref(), end) }) +} + +// Snapshot safely gets the job's current metrics. +func (jp *JobProgress) Snapshot() JobProgressMetrics { + jp.metricsLock.Lock() + defer jp.metricsLock.Unlock() + return jp.metrics +} + +// ReportError adds a non-nil error to the aggregate of errors +// encountered during scanning. +func (jp *JobProgress) ReportError(err error) { + if err == nil { + return + } + jp.metricsLock.Lock() + jp.metrics.Errors = append(jp.metrics.Errors, err) + jp.metricsLock.Unlock() + + jp.executeHooks(func(hook JobProgressHook) { hook.ReportError(jp.Ref(), err) }) +} + +// Ref provides a read-only reference to the JobProgress. +func (jp *JobProgress) Ref() JobProgressRef { + return JobProgressRef{ + SourceID: jp.SourceID, + JobID: jp.JobID, + jobProgress: jp, + } +} + +// EnumerationErrors joins all errors encountered during initialization or +// enumeration. +func (m JobProgressMetrics) EnumerationError() error { + return errors.Join(m.Errors...) +} + +// ChunkErrors joins all errors encountered during chunking. +func (m JobProgressMetrics) ChunkError() error { + var aggregate []error + for _, err := range m.Errors { + var chunkErr ChunkError + if ok := errors.As(err, &chunkErr); ok { + aggregate = append(aggregate, err) + } + } + return errors.Join(aggregate...) +} + +// FatalError returns the first Fatal error, if any, encountered in the scan. +func (m JobProgressMetrics) FatalError() error { + for _, err := range m.Errors { + var fatalErr Fatal + if found := errors.As(err, &fatalErr); found { + return fatalErr + } + } + return nil +} + +// FatalErrors returns all of the encountered fatal errors joined together. +func (m JobProgressMetrics) FatalErrors() error { + var aggregate []error + for _, err := range m.Errors { + var fatalErr Fatal + if found := errors.As(err, &fatalErr); found { + aggregate = append(aggregate, fatalErr) + } + } + return errors.Join(aggregate...) +} + +func (m JobProgressMetrics) PercentComplete() int { + num := m.FinishedUnits + den := m.TotalUnits + if num == 0 && den == 0 { + return 0 + } + return int(num * 100 / den) +} diff --git a/pkg/sources/job_progress_test.go b/pkg/sources/job_progress_test.go new file mode 100644 index 000000000000..fd1c2972c41c --- /dev/null +++ b/pkg/sources/job_progress_test.go @@ -0,0 +1,117 @@ +package sources + +import ( + "errors" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" +) + +func TestJobProgressFatalErrors(t *testing.T) { + var jp JobProgress + + // Add a non-fatal error. + jp.ReportError(fmt.Errorf("oh no")) + assert.Greater(t, len(jp.Snapshot().Errors), 0) + assert.NoError(t, jp.Snapshot().FatalError()) + assert.NoError(t, jp.Snapshot().ChunkError()) + + // Add a fatal error and make sure we can test comparison. + err := fmt.Errorf("fatal error") + jp.ReportError(Fatal{err}) + assert.Greater(t, len(jp.Snapshot().Errors), 0) + assert.Error(t, jp.Snapshot().FatalError()) + assert.NoError(t, jp.Snapshot().ChunkError()) + assert.True(t, errors.Is(jp.Snapshot().FatalError(), err)) + + // Add another fatal error and test we still return the first. + jp.ReportError(Fatal{fmt.Errorf("second fatal error")}) + assert.Greater(t, len(jp.Snapshot().Errors), 0) + assert.Error(t, jp.Snapshot().FatalError()) + assert.NoError(t, jp.Snapshot().ChunkError()) + assert.True(t, errors.Is(jp.Snapshot().FatalError(), err)) +} + +func TestJobProgressRef(t *testing.T) { + jp := NewJobProgress(123, 456) + ref := jp.Ref() + assert.Equal(t, int64(123), ref.SourceID) + assert.Equal(t, int64(456), ref.JobID) + + // Test Done() blocks until Finish() is called. + select { + case <-jp.Done(): + assert.FailNow(t, "job should not be finished") + default: + } + + jp.Finish() + select { + case <-jp.Done(): + default: + assert.FailNow(t, "job should be finished") + } +} + +func TestJobProgressHook(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + hook := NewMockJobProgressHook(ctrl) + jp := NewJobProgress(123, 456, WithHooks(hook)) + + // Start(JobProgressRef, time.Time) + // End(JobProgressRef, time.Time) + // StartEnumerating(JobProgressRef, time.Time) + // EndEnumerating(JobProgressRef, time.Time) + // StartUnitChunking(JobProgressRef, SourceUnit, time.Time) + // EndUnitChunking(JobProgressRef, SourceUnit, time.Time) + // ReportError(JobProgressRef, error) + // ReportUnit(JobProgressRef, SourceUnit) + // ReportChunk(JobProgressRef, SourceUnit, *Chunk) + // Finish(JobProgressRef) + + startTime := time.Now() + endTime := time.Now().Add(10 * time.Second) + startEnum := time.Now().Add(20 * time.Second) + endEnum := time.Now().Add(30 * time.Second) + startChunk := time.Now().Add(40 * time.Second) + endChunk := time.Now().Add(50 * time.Second) + reportErr := fmt.Errorf("reporting error") + reportUnit := CommonSourceUnit{"reporting unit"} + reportChunk := &Chunk{Data: []byte("reporting chunk")} + + hook.EXPECT().Start(gomock.Any(), startTime) + hook.EXPECT().End(gomock.Any(), endTime) + hook.EXPECT().StartEnumerating(gomock.Any(), startEnum) + hook.EXPECT().EndEnumerating(gomock.Any(), endEnum) + hook.EXPECT().StartUnitChunking(gomock.Any(), reportUnit, startChunk) + hook.EXPECT().EndUnitChunking(gomock.Any(), reportUnit, endChunk) + hook.EXPECT().ReportError(gomock.Any(), reportErr) + hook.EXPECT().ReportUnit(gomock.Any(), reportUnit) + hook.EXPECT().ReportChunk(gomock.Any(), reportUnit, reportChunk) + hook.EXPECT().Finish(gomock.Any()) + + jp.Start(startTime) + jp.End(endTime) + jp.StartEnumerating(startEnum) + jp.EndEnumerating(endEnum) + jp.StartUnitChunking(reportUnit, startChunk) + jp.EndUnitChunking(reportUnit, endChunk) + jp.ReportError(reportErr) + jp.ReportUnit(reportUnit) + jp.ReportChunk(reportUnit, reportChunk) + jp.Finish() +} + +func TestJobProgressDone(t *testing.T) { + ref := JobProgressRef{} + select { + case <-ref.Done(): + default: + assert.FailNow(t, "done should not block for a nil job") + } +} diff --git a/pkg/sources/job_report.go b/pkg/sources/job_report.go deleted file mode 100644 index 01a7734c3579..000000000000 --- a/pkg/sources/job_report.go +++ /dev/null @@ -1,37 +0,0 @@ -package sources - -import ( - "errors" - "sync" - "time" -) - -// JobReport aggregates information about a run of a Source. -type JobReport struct { - SourceID int64 - JobID int64 - StartTime time.Time - EndTime time.Time - TotalChunks uint64 - errors []error - errorsLock sync.Mutex -} - -// AddError adds a non-nil error to the aggregate of errors encountered during -// scanning. -func (jr *JobReport) AddError(err error) { - if err == nil { - return - } - jr.errorsLock.Lock() - defer jr.errorsLock.Unlock() - jr.errors = append(jr.errors, err) -} - -// Errors joins all aggregated errors into one. If there were no errors, nil is -// returned. errors.Is can be used to check for specific errors. -func (jr *JobReport) Errors() error { - jr.errorsLock.Lock() - defer jr.errorsLock.Unlock() - return errors.Join(jr.errors...) -} diff --git a/pkg/sources/mock_job_progress_test.go b/pkg/sources/mock_job_progress_test.go new file mode 100644 index 000000000000..de3e608fccd2 --- /dev/null +++ b/pkg/sources/mock_job_progress_test.go @@ -0,0 +1,155 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ./job_progress.go + +// Package sources is a generated GoMock package. +package sources + +import ( + reflect "reflect" + time "time" + + gomock "go.uber.org/mock/gomock" +) + +// MockJobProgressHook is a mock of JobProgressHook interface. +type MockJobProgressHook struct { + ctrl *gomock.Controller + recorder *MockJobProgressHookMockRecorder +} + +// MockJobProgressHookMockRecorder is the mock recorder for MockJobProgressHook. +type MockJobProgressHookMockRecorder struct { + mock *MockJobProgressHook +} + +// NewMockJobProgressHook creates a new mock instance. +func NewMockJobProgressHook(ctrl *gomock.Controller) *MockJobProgressHook { + mock := &MockJobProgressHook{ctrl: ctrl} + mock.recorder = &MockJobProgressHookMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockJobProgressHook) EXPECT() *MockJobProgressHookMockRecorder { + return m.recorder +} + +// End mocks base method. +func (m *MockJobProgressHook) End(arg0 JobProgressRef, arg1 time.Time) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "End", arg0, arg1) +} + +// End indicates an expected call of End. +func (mr *MockJobProgressHookMockRecorder) End(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "End", reflect.TypeOf((*MockJobProgressHook)(nil).End), arg0, arg1) +} + +// EndEnumerating mocks base method. +func (m *MockJobProgressHook) EndEnumerating(arg0 JobProgressRef, arg1 time.Time) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "EndEnumerating", arg0, arg1) +} + +// EndEnumerating indicates an expected call of EndEnumerating. +func (mr *MockJobProgressHookMockRecorder) EndEnumerating(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EndEnumerating", reflect.TypeOf((*MockJobProgressHook)(nil).EndEnumerating), arg0, arg1) +} + +// EndUnitChunking mocks base method. +func (m *MockJobProgressHook) EndUnitChunking(arg0 JobProgressRef, arg1 SourceUnit, arg2 time.Time) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "EndUnitChunking", arg0, arg1, arg2) +} + +// EndUnitChunking indicates an expected call of EndUnitChunking. +func (mr *MockJobProgressHookMockRecorder) EndUnitChunking(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EndUnitChunking", reflect.TypeOf((*MockJobProgressHook)(nil).EndUnitChunking), arg0, arg1, arg2) +} + +// Finish mocks base method. +func (m *MockJobProgressHook) Finish(arg0 JobProgressRef) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Finish", arg0) +} + +// Finish indicates an expected call of Finish. +func (mr *MockJobProgressHookMockRecorder) Finish(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Finish", reflect.TypeOf((*MockJobProgressHook)(nil).Finish), arg0) +} + +// ReportChunk mocks base method. +func (m *MockJobProgressHook) ReportChunk(arg0 JobProgressRef, arg1 SourceUnit, arg2 *Chunk) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "ReportChunk", arg0, arg1, arg2) +} + +// ReportChunk indicates an expected call of ReportChunk. +func (mr *MockJobProgressHookMockRecorder) ReportChunk(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportChunk", reflect.TypeOf((*MockJobProgressHook)(nil).ReportChunk), arg0, arg1, arg2) +} + +// ReportError mocks base method. +func (m *MockJobProgressHook) ReportError(arg0 JobProgressRef, arg1 error) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "ReportError", arg0, arg1) +} + +// ReportError indicates an expected call of ReportError. +func (mr *MockJobProgressHookMockRecorder) ReportError(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportError", reflect.TypeOf((*MockJobProgressHook)(nil).ReportError), arg0, arg1) +} + +// ReportUnit mocks base method. +func (m *MockJobProgressHook) ReportUnit(arg0 JobProgressRef, arg1 SourceUnit) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "ReportUnit", arg0, arg1) +} + +// ReportUnit indicates an expected call of ReportUnit. +func (mr *MockJobProgressHookMockRecorder) ReportUnit(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportUnit", reflect.TypeOf((*MockJobProgressHook)(nil).ReportUnit), arg0, arg1) +} + +// Start mocks base method. +func (m *MockJobProgressHook) Start(arg0 JobProgressRef, arg1 time.Time) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Start", arg0, arg1) +} + +// Start indicates an expected call of Start. +func (mr *MockJobProgressHookMockRecorder) Start(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockJobProgressHook)(nil).Start), arg0, arg1) +} + +// StartEnumerating mocks base method. +func (m *MockJobProgressHook) StartEnumerating(arg0 JobProgressRef, arg1 time.Time) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "StartEnumerating", arg0, arg1) +} + +// StartEnumerating indicates an expected call of StartEnumerating. +func (mr *MockJobProgressHookMockRecorder) StartEnumerating(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartEnumerating", reflect.TypeOf((*MockJobProgressHook)(nil).StartEnumerating), arg0, arg1) +} + +// StartUnitChunking mocks base method. +func (m *MockJobProgressHook) StartUnitChunking(arg0 JobProgressRef, arg1 SourceUnit, arg2 time.Time) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "StartUnitChunking", arg0, arg1, arg2) +} + +// StartUnitChunking indicates an expected call of StartUnitChunking. +func (mr *MockJobProgressHookMockRecorder) StartUnitChunking(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartUnitChunking", reflect.TypeOf((*MockJobProgressHook)(nil).StartUnitChunking), arg0, arg1, arg2) +} diff --git a/pkg/sources/source_manager.go b/pkg/sources/source_manager.go index f061c66b62d0..890099478d40 100644 --- a/pkg/sources/source_manager.go +++ b/pkg/sources/source_manager.go @@ -1,7 +1,6 @@ package sources import ( - "errors" "fmt" "sync" "sync/atomic" @@ -23,14 +22,11 @@ type handle int64 type SourceInitFunc func(ctx context.Context, sourceID int64, jobID int64) (Source, error) type SourceManager struct { - api apiClient + api apiClient + hooks []JobProgressHook // Map of handle to source initializer. handles map[handle]SourceInitFunc handlesLock sync.Mutex - // Map of handle to job reports. - // TODO: Manage culling and flushing to the API. - report map[handle]*JobReport - reportLock sync.Mutex // Pool limiting the amount of concurrent sources running. pool errgroup.Group concurrentUnits int @@ -39,8 +35,7 @@ type SourceManager struct { // Downstream chunks channel to be scanned. outputChunks chan *Chunk // Set when Wait() returns. - done bool - doneErr error + done bool } // apiClient is an interface for optionally communicating with an external API. @@ -51,11 +46,18 @@ type apiClient interface { GetJobID(ctx context.Context, id int64) (int64, error) } -// WithAPI adds an API client to the manager for tracking jobs and progress. +// WithAPI adds an API client to the manager for tracking jobs and progress. If +// the API is also a JobProgressHook, it will be added to the list of event hooks. func WithAPI(api apiClient) func(*SourceManager) { return func(mgr *SourceManager) { mgr.api = api } } +func WithReportHook(hook JobProgressHook) func(*SourceManager) { + return func(mgr *SourceManager) { + mgr.hooks = append(mgr.hooks, hook) + } +} + // WithConcurrency limits the concurrent number of sources a manager can run. func WithConcurrency(concurrency int) func(*SourceManager) { return func(mgr *SourceManager) { mgr.pool.SetLimit(concurrency) } @@ -84,7 +86,6 @@ func NewManager(opts ...func(*SourceManager)) *SourceManager { // Default to the headless API. Can be overwritten by the WithAPI option. api: &headlessAPI{}, handles: make(map[handle]SourceInitFunc), - report: make(map[handle]*JobReport), outputChunks: make(chan *Chunk), } for _, opt := range opts { @@ -114,52 +115,44 @@ func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb. } // Run blocks until a resource is available to run the source, then -// synchronously runs it. -func (s *SourceManager) Run(ctx context.Context, handle handle) error { - // Do preflight checks before waiting on the pool. - if err := s.preflightChecks(ctx, handle); err != nil { - return err +// synchronously runs it. The first fatal error, if any, will be returned. +func (s *SourceManager) Run(ctx context.Context, handle handle) (JobProgressRef, error) { + report, err := s.asyncRun(ctx, handle) + if err != nil { + return report, err } - ch := make(chan error) - s.pool.Go(func() error { - defer common.Recover(ctx) - report, err := s.run(ctx, handle) - if report != nil { - s.reportLock.Lock() - s.report[handle] = report - s.reportLock.Unlock() - } - if err != nil { - ch <- err - return nil - } - ch <- report.Errors() - return nil - }) - return <-ch + <-report.Done() + return report, report.Snapshot().FatalError() } // ScheduleRun blocks until a resource is available to run the source, then -// asynchronously runs it. Error information is stored and returned by Wait(). -func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) error { +// asynchronously runs it. Error information is stored and accessible via the +// JobProgressRef as it becomes available. +func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) (JobProgressRef, error) { + return s.asyncRun(ctx, handle) +} + +// asyncRun is a helper method to asynchronously run the Source. It calls out +// to the API to get a job ID for this run, creates a report, then waits for an +// available goroutine to asynchronously run it. +func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgressRef, error) { // Do preflight checks before waiting on the pool. if err := s.preflightChecks(ctx, handle); err != nil { - return err + return JobProgressRef{}, err } + // Get a Job ID. + jobID, err := s.api.GetJobID(ctx, int64(handle)) + if err != nil { + return JobProgressRef{SourceID: int64(handle)}, err + } + // Start a report for this job. + report := NewJobProgress(int64(handle), jobID, WithHooks(s.hooks...)) s.pool.Go(func() error { defer common.Recover(ctx) - // The error is already saved in the report, so we can ignore - // it here. - report, _ := s.run(ctx, handle) - if report != nil { - s.reportLock.Lock() - s.report[handle] = report - s.reportLock.Unlock() - } + _ = s.run(ctx, handle, jobID, report) return nil }) - // TODO: Maybe wait for a signal here that initialization was successful? - return nil + return report.Ref(), nil } // Chunks returns the read only channel of all the chunks produced by all of @@ -168,13 +161,14 @@ func (s *SourceManager) Chunks() <-chan *Chunk { return s.outputChunks } -// Wait blocks until all running sources are completed and returns an error if -// any of the sources had fatal errors. It also closes the channel returned by -// Chunks(). The manager should not be reused after calling this method. -func (s *SourceManager) Wait() error { +// Wait blocks until all running sources are completed and closes the channel +// returned by Chunks(). The manager should not be reused after calling this +// method. This current implementation is not thread safe and should only be +// called by one thread. +func (s *SourceManager) Wait() { // Check if the manager has been Waited. if s.done { - return s.doneErr + return } defer close(s.outputChunks) defer func() { s.done = true }() @@ -182,26 +176,6 @@ func (s *SourceManager) Wait() error { // We are only using the errgroup for limiting concurrency. // TODO: Maybe switch to using a semaphore.Weighted. _ = s.pool.Wait() - - // Aggregate all errors from all job reports. - // TODO: This should probably only be the fatal errors. We'll also need - // to rewrite this for when the reports start getting culled. - s.reportLock.Lock() - defer s.reportLock.Unlock() - errs := make([]error, 0, len(s.report)) - for _, report := range s.report { - errs = append(errs, report.Errors()) - } - s.doneErr = errors.Join(errs...) - return s.doneErr -} - -// Report retrieves a scan report for a given handle. If no report exists or -// the Source has not finished, nil will be returned. -func (s *SourceManager) Report(handle handle) *JobReport { - s.reportLock.Lock() - defer s.reportLock.Unlock() - return s.report[handle] } // preflightChecks is a helper method to check the Manager or the context isn't @@ -219,40 +193,25 @@ func (s *SourceManager) preflightChecks(ctx context.Context, handle handle) erro } // run is a helper method to sychronously run the source. It does not check for -// acquired resources. Possible return values are: -// -// - *JobReport, nil -// Successfully ran the source, but the report could have errors. -// -// - *JobReport, error -// There was an error calling Init or Chunks. This sort of error indicates -// a fatal error and is also recorded in the report. -// -// - nil, error: -// There was an error from the API or the handle is invalid. The latter of -// which should never happen due to the preflightChecks. -func (s *SourceManager) run(ctx context.Context, handle handle) (*JobReport, error) { - jobID, err := s.api.GetJobID(ctx, int64(handle)) - if err != nil { - return nil, err - } +// acquired resources. An error is returned if there was a fatal error during +// the run. This information is also recorded in the JobProgress. +func (s *SourceManager) run(ctx context.Context, handle handle, jobID int64, report *JobProgress) error { + defer report.Finish() + report.Start(time.Now()) + defer func() { report.End(time.Now()) }() + + // Initialize the source. initFunc, ok := s.getInitFunc(handle) if !ok { - return nil, fmt.Errorf("unrecognized handle") + // Shouldn't happen due to preflight checks. + err := fmt.Errorf("unrecognized handle") + report.ReportError(Fatal{err}) + return Fatal{err} } - // Create a report for this run. - report := &JobReport{ - SourceID: int64(handle), - JobID: jobID, - StartTime: time.Now(), - } - defer func() { report.EndTime = time.Now() }() - - // Initialize the source. - source, err := initFunc(ctx, jobID, int64(handle)) + source, err := initFunc(ctx, int64(handle), jobID) if err != nil { - report.AddError(err) - return report, err + report.ReportError(Fatal{err}) + return Fatal{err} } // Check for the preferred method of tracking source units. if enumChunker, ok := source.(SourceUnitEnumChunker); ok && s.useSourceUnits { @@ -263,7 +222,7 @@ func (s *SourceManager) run(ctx context.Context, handle handle) (*JobReport, err // runWithoutUnits is a helper method to run a Source. It has coarse-grained // job reporting. -func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, source Source, report *JobReport) (*JobReport, error) { +func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, source Source, report *JobProgress) error { // Introspect on the chunks we get from the Chunks method. ch := make(chan *Chunk) var wg sync.WaitGroup @@ -272,7 +231,7 @@ func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, sour go func() { defer wg.Done() for chunk := range ch { - atomic.AddUint64(&report.TotalChunks, 1) + report.ReportChunk(nil, chunk) _ = common.CancellableWrite(ctx, s.outputChunks, chunk) } }() @@ -283,25 +242,38 @@ func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, sour defer wg.Wait() defer close(ch) if err := source.Chunks(ctx, ch); err != nil { - report.AddError(err) - return report, err + report.ReportError(Fatal{err}) + return Fatal{err} } - return report, nil + return nil } // runWithUnits is a helper method to run a Source that is also a // SourceUnitEnumChunker. This allows better introspection of what is getting // scanned and any errors encountered. -func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source SourceUnitEnumChunker, report *JobReport) (*JobReport, error) { - reporter := &mgrUnitReporter{ +func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source SourceUnitEnumChunker, report *JobProgress) error { + unitReporter := &mgrUnitReporter{ unitCh: make(chan SourceUnit), + report: report, + } + // Create a function that will save the first error encountered (if + // any) and discard the rest. + fatalErr := make(chan error, 1) + catchFirstFatal := func(err error) { + select { + case fatalErr <- err: + default: + } } // Produce units. go func() { // TODO: Catch panics and add to report. - defer close(reporter.unitCh) - if err := source.Enumerate(ctx, reporter); err != nil { - report.AddError(err) + report.StartEnumerating(time.Now()) + defer func() { report.EndEnumerating(time.Now()) }() + defer close(unitReporter.unitCh) + if err := source.Enumerate(ctx, unitReporter); err != nil { + report.ReportError(Fatal{err}) + catchFirstFatal(Fatal{err}) } }() var wg sync.WaitGroup @@ -311,18 +283,21 @@ func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source // Negative values indicated no limit. unitPool.SetLimit(s.concurrentUnits) } - for unit := range reporter.unitCh { - reporter := &mgrChunkReporter{ - unitID: unit.SourceUnitID(), + for unit := range unitReporter.unitCh { + unit := unit + chunkReporter := &mgrChunkReporter{ + unit: unit, chunkCh: make(chan *Chunk), + report: report, } - unit := unit // Consume units and produce chunks. unitPool.Go(func() error { + report.StartUnitChunking(unit, time.Now()) // TODO: Catch panics and add to report. - defer close(reporter.chunkCh) - if err := source.ChunkUnit(ctx, unit, reporter); err != nil { - report.AddError(err) + defer close(chunkReporter.chunkCh) + if err := source.ChunkUnit(ctx, unit, chunkReporter); err != nil { + report.ReportError(Fatal{err}) + catchFirstFatal(Fatal{err}) } return nil }) @@ -330,16 +305,20 @@ func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source wg.Add(1) go func() { defer wg.Done() - for chunk := range reporter.chunkCh { - // TODO: Introspect on the chunks we got from this unit. - atomic.AddUint64(&report.TotalChunks, 1) + defer func() { report.EndUnitChunking(unit, time.Now()) }() + for chunk := range chunkReporter.chunkCh { + report.ReportChunk(chunkReporter.unit, chunk) _ = common.CancellableWrite(ctx, s.outputChunks, chunk) } }() } wg.Wait() - // TODO: Return fatal errors. - return report, nil + select { + case err := <-fatalErr: + return err + default: + return nil + } } // getInitFunc is a helper method for safe concurrent access to the @@ -368,9 +347,8 @@ func (api *headlessAPI) GetJobID(ctx context.Context, id int64) (int64, error) { // mgrUnitReporter implements the UnitReporter interface. type mgrUnitReporter struct { - unitCh chan SourceUnit - unitErrs []error - unitErrsLock sync.Mutex + unitCh chan SourceUnit + report *JobProgress } func (s *mgrUnitReporter) UnitOk(ctx context.Context, unit SourceUnit) error { @@ -378,18 +356,15 @@ func (s *mgrUnitReporter) UnitOk(ctx context.Context, unit SourceUnit) error { } func (s *mgrUnitReporter) UnitErr(ctx context.Context, err error) error { - s.unitErrsLock.Lock() - defer s.unitErrsLock.Unlock() - s.unitErrs = append(s.unitErrs, err) + s.report.ReportError(err) return nil } // mgrChunkReporter implements the ChunkReporter interface. type mgrChunkReporter struct { - unitID string - chunkCh chan *Chunk - chunkErrs []error - chunkErrsLock sync.Mutex + unit SourceUnit + chunkCh chan *Chunk + report *JobProgress } func (s *mgrChunkReporter) ChunkOk(ctx context.Context, chunk Chunk) error { @@ -397,8 +372,6 @@ func (s *mgrChunkReporter) ChunkOk(ctx context.Context, chunk Chunk) error { } func (s *mgrChunkReporter) ChunkErr(ctx context.Context, err error) error { - s.chunkErrsLock.Lock() - defer s.chunkErrsLock.Unlock() - s.chunkErrs = append(s.chunkErrs, err) + s.report.ReportError(ChunkError{s.unit, err}) return nil } diff --git a/pkg/sources/source_manager_test.go b/pkg/sources/source_manager_test.go index a06a05866608..4f7db5f1ec9d 100644 --- a/pkg/sources/source_manager_test.go +++ b/pkg/sources/source_manager_test.go @@ -102,7 +102,7 @@ func TestSourceManagerRun(t *testing.T) { t.Fatalf("unexpected error enrolling source: %v", err) } for i := 0; i < 3; i++ { - if err := mgr.Run(context.Background(), handle); err != nil { + if _, err := mgr.Run(context.Background(), handle); err != nil { t.Fatalf("unexpected error running source: %v", err) } chunk, err := tryRead(mgr.Chunks()) @@ -127,20 +127,18 @@ func TestSourceManagerWait(t *testing.T) { t.Fatalf("unexpected error enrolling source: %v", err) } // Asynchronously run the source. - if err := mgr.ScheduleRun(context.Background(), handle); err != nil { + if _, err := mgr.ScheduleRun(context.Background(), handle); err != nil { t.Fatalf("unexpected error scheduling run: %v", err) } // Read the 1 chunk we're expecting so Waiting completes. <-mgr.Chunks() // Wait for all resources to complete. - if err := mgr.Wait(); err != nil { - t.Fatalf("unexpected error waiting: %v", err) - } + mgr.Wait() // Enroll and run should return an error now. if _, err := enrollDummy(mgr, &counterChunker{count: 1}); err == nil { t.Fatalf("expected enroll to fail") } - if err := mgr.ScheduleRun(context.Background(), handle); err == nil { + if _, err := mgr.ScheduleRun(context.Background(), handle); err == nil { t.Fatalf("expected scheduling run to fail") } } @@ -152,15 +150,17 @@ func TestSourceManagerError(t *testing.T) { t.Fatalf("unexpected error enrolling source: %v", err) } // A synchronous run should fail. - if err := mgr.Run(context.Background(), handle); err == nil { + if _, err := mgr.Run(context.Background(), handle); err == nil { t.Fatalf("expected run to fail") } // Scheduling a run should not fail, but the error should surface in // Wait(). - if err := mgr.ScheduleRun(context.Background(), handle); err != nil { + ref, err := mgr.ScheduleRun(context.Background(), handle) + if err != nil { t.Fatalf("unexpected error scheduling run: %v", err) } - if err := mgr.Wait(); err == nil { + <-ref.Done() + if err := ref.Snapshot().FatalError(); err == nil { t.Fatalf("expected wait to fail") } } @@ -177,18 +177,15 @@ func TestSourceManagerReport(t *testing.T) { t.Fatalf("unexpected error enrolling source: %v", err) } // Synchronously run the source. - if err := mgr.Run(context.Background(), handle); err != nil { + ref, err := mgr.Run(context.Background(), handle) + if err != nil { t.Fatalf("unexpected error running source: %v", err) } - report := mgr.Report(handle) - if report == nil { - t.Fatalf("expected a report") - } - if err := report.Errors(); err != nil { - t.Fatalf("unexpected error in report: %v", err) + if errs := ref.Snapshot().Errors; len(errs) > 0 { + t.Fatalf("unexpected errors in report: %v", errs) } - if report.TotalChunks != 4 { - t.Fatalf("expected report to have 4 chunks, got: %d", report.TotalChunks) + if count := ref.Snapshot().TotalChunks; count != 4 { + t.Fatalf("expected report to have 4 chunks, got: %d", count) } } } From 32e3f1f0152a92c66e04e9403dab933c3f1f918e Mon Sep 17 00:00:00 2001 From: Miccah Date: Mon, 31 Jul 2023 11:37:25 -0500 Subject: [PATCH 7/8] Fix pubnub regular expression (#1565) One of the sub-groups of the UUIDv4 was missing the characters 0-9. --- pkg/detectors/pubnubpublishkey/pubnubpublishkey.go | 2 +- pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/detectors/pubnubpublishkey/pubnubpublishkey.go b/pkg/detectors/pubnubpublishkey/pubnubpublishkey.go index 43789f52bbbc..04b4a98d0905 100644 --- a/pkg/detectors/pubnubpublishkey/pubnubpublishkey.go +++ b/pkg/detectors/pubnubpublishkey/pubnubpublishkey.go @@ -21,7 +21,7 @@ var ( // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. pubPat = regexp.MustCompile(`\b(pub-c-[0-9a-z]{8}-[0-9a-z]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`) - subPat = regexp.MustCompile(`\b(sub-c-[0-9a-z]{8}-[a-z]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`) + subPat = regexp.MustCompile(`\b(sub-c-[0-9a-z]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`) ) // Keywords are used for efficiently pre-filtering chunks. diff --git a/pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey.go b/pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey.go index ecd61606d15c..2229bb5248f2 100644 --- a/pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey.go +++ b/pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey.go @@ -20,7 +20,7 @@ var ( client = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(`\b(sub-c-[0-9a-z]{8}-[a-z]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`) + keyPat = regexp.MustCompile(`\b(sub-c-[0-9a-z]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`) ) // Keywords are used for efficiently pre-filtering chunks. From b54683acb91a21b6e9aa36027779560e0b90b764 Mon Sep 17 00:00:00 2001 From: Miccah Date: Mon, 31 Jul 2023 11:39:14 -0500 Subject: [PATCH 8/8] gitparse: Use an object for currentDiff (#1573) * gitparse: Use an object for currentDiff instead of a pointer * gitparse: Use an object for currentCommit instead of a pointer * Revert "gitparse: Use an object for currentCommit instead of a pointer" This reverts commit c5f0708b4a69b2f1586b4e2809031d8d1df480b2. --- pkg/gitparse/gitparse.go | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/pkg/gitparse/gitparse.go b/pkg/gitparse/gitparse.go index 8e8925111da4..2f5262cf30e0 100644 --- a/pkg/gitparse/gitparse.go +++ b/pkg/gitparse/gitparse.go @@ -237,7 +237,7 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch outReader := bufio.NewReader(stdOut) var ( currentCommit *Commit - currentDiff *Diff + currentDiff Diff totalLogSize int ) @@ -260,8 +260,8 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch latestState = CommitLine // If there is a currentDiff, add it to currentCommit. - if currentDiff != nil && currentDiff.Content.Len() > 0 { - currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff) + if currentDiff.Content.Len() > 0 { + currentCommit.Diffs = append(currentCommit.Diffs, currentDiff) currentCommit.Size += currentDiff.Content.Len() } // If there is a currentCommit, send it to the channel. @@ -270,7 +270,7 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch totalLogSize += currentCommit.Size } // Create a new currentDiff and currentCommit - currentDiff = &Diff{} + currentDiff = Diff{} currentCommit = &Commit{ Message: strings.Builder{}, } @@ -309,8 +309,8 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch if currentCommit == nil { currentCommit = &Commit{} } - if currentDiff != nil && currentDiff.Content.Len() > 0 { - currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff) + if currentDiff.Content.Len() > 0 { + currentCommit.Diffs = append(currentCommit.Diffs, currentDiff) // If the currentDiff is over 1GB, drop it into the channel so it isn't held in memory waiting for more commits. totalSize := 0 for _, diff := range currentCommit.Diffs { @@ -331,7 +331,7 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch currentCommit.Message.WriteString(oldCommit.Message.String()) } } - currentDiff = &Diff{} + currentDiff = Diff{} case isModeLine(isStaged, latestState, line): latestState = ModeLine // NoOp @@ -354,16 +354,13 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch case isHunkLineNumberLine(isStaged, latestState, line): latestState = HunkLineNumberLine - // TODO: Is it still necessary to check whether the currentDiff is nil? - if currentDiff != nil && currentDiff.Content.Len() > 0 { - currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff) + if currentDiff.Content.Len() > 0 { + currentCommit.Diffs = append(currentCommit.Diffs, currentDiff) } - newDiff := &Diff{ + currentDiff = Diff{ PathB: currentDiff.PathB, } - currentDiff = newDiff - words := bytes.Split(line, []byte(" ")) if len(words) >= 3 { startSlice := bytes.Split(words[2], []byte(",")) @@ -420,14 +417,14 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch latestState = ParseFailure } - if currentDiff != nil && currentDiff.Content.Len() > c.maxDiffSize { + if currentDiff.Content.Len() > c.maxDiffSize { ctx.Logger().V(2).Info(fmt.Sprintf( "Diff for %s exceeded MaxDiffSize(%d)", currentDiff.PathB, c.maxDiffSize, )) break } } - cleanupParse(currentCommit, currentDiff, commitChan, &totalLogSize) + cleanupParse(currentCommit, ¤tDiff, commitChan, &totalLogSize) ctx.Logger().V(2).Info("finished parsing git log.", "total_log_size", totalLogSize) }