Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: integrate otel collector in guardian #173

Merged
merged 22 commits into from
Sep 19, 2024
Merged

feat: integrate otel collector in guardian #173

Show file tree
Hide file tree
Changes from 21 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
9f9ad67
feat: adding otel into guardian
Aug 21, 2024
5e77b12
fix go mod
Aug 22, 2024
6dd1152
fix go mod
Aug 22, 2024
075dcdb
feat: adding otel into guardian
Aug 23, 2024
eb8b126
feat: adding otel into guardian
Aug 26, 2024
732087a
feat: adding otel into guardian
Aug 26, 2024
cfadb8f
fix: load telemetry for cli
Aug 28, 2024
76eb342
fix: load telemetry for cli
Aug 28, 2024
90f1aa4
fix: rpc metric
Sep 2, 2024
da89967
feat: add sql metrics
Sep 4, 2024
d39f53a
feat: add sql metrics
Sep 4, 2024
212cf5c
feat: add sql metrics
Sep 11, 2024
97c2816
fix: load telemetry for cli
Sep 12, 2024
44f9bcc
feat: track metric external provider
Sep 12, 2024
d1efd10
fix test
Sep 12, 2024
b141011
fix: track external services http traces
Sep 13, 2024
226cb6a
fix: track external services http traces test
Sep 13, 2024
d2cfe3f
fix: external metrics
Sep 18, 2024
9bc9768
fix: external metrics
Sep 18, 2024
b83fef8
fix: external metrics
Sep 18, 2024
2c7d3fa
resolve comments
Sep 19, 2024
c28bf38
resolve comments
Sep 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion cli/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,14 @@ import (
guardianv1beta1 "github.com/goto/guardian/api/proto/gotocompany/guardian/v1beta1"
"github.com/spf13/cobra"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"

"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
)

func createConnection(ctx context.Context, host string) (*grpc.ClientConn, error) {
opts := []grpc.DialOption{
grpc.WithInsecure(),
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithBlock(),
grpc.WithUnaryInterceptor(otelgrpc.UnaryClientInterceptor()),
grpc.WithStreamInterceptor(otelgrpc.StreamClientInterceptor()),
Expand Down
6 changes: 3 additions & 3 deletions cli/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,16 @@ import (
"fmt"

"github.com/MakeNowJust/heredoc"
"github.com/goto/guardian/pkg/tracing"
"github.com/goto/guardian/pkg/opentelemetry"
"github.com/goto/salt/cmdx"
"github.com/spf13/cobra"
)

var cliConfig *Config

type Config struct {
Host string `mapstructure:"host"`
Telemetry tracing.Config `mapstructure:"telemetry"`
Host string `mapstructure:"host"`
Telemetry opentelemetry.Config `mapstructure:"telemetry"`
}

func LoadConfig() (*Config, error) {
Expand Down
27 changes: 20 additions & 7 deletions cli/root.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,19 @@
package cli

import (
"context"
"log"

"github.com/MakeNowJust/heredoc"
handlerv1beta1 "github.com/goto/guardian/api/handler/v1beta1"
"github.com/goto/guardian/pkg/tracing"
"github.com/goto/guardian/pkg/opentelemetry"
"github.com/goto/salt/cmdx"
"github.com/spf13/cobra"
)

func New(cfg *Config) *cobra.Command {
cliConfig = cfg
var shutdown func()
var shutdownOtel = func() error { return nil }
var cmd = &cobra.Command{
Use: "guardian <command> <subcommand> [flags]",
Short: "Universal data access control",
Expand All @@ -30,17 +33,27 @@ func New(cfg *Config) *cobra.Command {
},
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
// initialize tracing
var err error
shutdown, err = tracing.InitTracer(cfg.Telemetry)
if err != nil {
return err
ctx := context.Background()

if cliConfig.Telemetry.Enabled {
var err error
shutdownOtel, err = opentelemetry.Init(ctx, cfg.Telemetry)
if err != nil {
return err
}
}

defer func() {
if err := shutdownOtel(); err != nil {
log.Printf("telemetry is disabled: %v", err)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
log.Printf("telemetry is disabled: %v", err)
log.Printf("failed to terminate telemetery: %v\n", err)

}
}()

return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
// shutdown tracing
shutdown()
shutdownOtel()
},
}

Expand Down
119 changes: 74 additions & 45 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,21 @@ module github.com/goto/guardian
go 1.21

require (
cloud.google.com/go/bigquery v1.44.0
cloud.google.com/go/datacatalog v1.8.0
cloud.google.com/go/iam v0.8.0
cloud.google.com/go/storage v1.29.0
github.com/DATA-DOG/go-sqlmock v1.5.0
cloud.google.com/go/bigquery v1.55.0
cloud.google.com/go/datacatalog v1.17.1
cloud.google.com/go/iam v1.1.2
cloud.google.com/go/storage v1.30.1
github.com/DATA-DOG/go-sqlmock v1.5.2
github.com/MakeNowJust/heredoc v1.0.0
github.com/antonmedv/expr v1.15.3
github.com/envoyproxy/protoc-gen-validate v0.6.7
github.com/envoyproxy/protoc-gen-validate v1.0.2
github.com/go-playground/validator/v10 v10.4.1
github.com/golang-migrate/migrate/v4 v4.15.2
github.com/google/go-cmp v0.5.9
github.com/google/uuid v1.3.0
github.com/google/go-cmp v0.6.0
github.com/google/uuid v1.6.0
github.com/goto/salt v0.3.1-0.20230314040737-76516d266233
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0
github.com/imdario/mergo v0.3.12
github.com/jackc/pgx/v5 v5.3.0
github.com/lib/pq v1.10.0
Expand All @@ -26,69 +26,83 @@ require (
github.com/mitchellh/mapstructure v1.4.3
github.com/ory/dockertest/v3 v3.9.1
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/sergi/go-diff v1.0.0
github.com/sergi/go-diff v1.3.1
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cobra v1.2.1
github.com/stretchr/testify v1.8.4
github.com/uptrace/opentelemetry-go-extra/otelgorm v0.1.17
github.com/stretchr/testify v1.9.0
github.com/uptrace/opentelemetry-go-extra/otelgorm v0.1.21
github.com/wI2L/jsondiff v0.5.2
github.com/xanzy/go-gitlab v0.98.0
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.28.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.37.0
go.opentelemetry.io/otel v1.11.2
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.3.0
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.11.2
go.opentelemetry.io/otel/sdk v1.11.2
golang.org/x/net v0.8.0
go.nhat.io/otelsql v0.13.0
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0
go.opentelemetry.io/contrib/instrumentation/host v0.54.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0
go.opentelemetry.io/contrib/instrumentation/runtime v0.54.0
go.opentelemetry.io/otel v1.29.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.45.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0
go.opentelemetry.io/otel/sdk v1.29.0
go.opentelemetry.io/otel/sdk/metric v1.27.0
golang.org/x/net v0.22.0
golang.org/x/oauth2 v0.22.0
golang.org/x/sync v0.1.0
google.golang.org/api v0.106.0
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f
google.golang.org/grpc v1.51.0
google.golang.org/protobuf v1.29.1
golang.org/x/sync v0.4.0
google.golang.org/api v0.128.0
google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97
google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97
google.golang.org/grpc v1.60.1
google.golang.org/protobuf v1.33.0
gopkg.in/yaml.v3 v3.0.1
gorm.io/datatypes v1.0.0
gorm.io/driver/postgres v1.4.7
gorm.io/gorm v1.25.1
)

require (
cloud.google.com/go v0.107.0 // indirect
cloud.google.com/go v0.110.8 // indirect
cloud.google.com/go/compute/metadata v0.3.0 // indirect
cloud.google.com/go/longrunning v0.5.1 // indirect
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
github.com/alecthomas/chroma v0.8.2 // indirect
github.com/andybalholm/brotli v1.0.4 // indirect
github.com/apache/arrow/go/v12 v12.0.0 // indirect
github.com/apache/thrift v0.16.0 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/briandowns/spinner v1.18.0 // indirect
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/charmbracelet/glamour v0.3.0 // indirect
github.com/cli/safeexec v1.0.0 // indirect
github.com/containerd/continuity v0.3.0 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dlclark/regexp2 v1.2.0 // indirect
github.com/dlclark/regexp2 v1.9.0 // indirect
github.com/docker/cli v20.10.14+incompatible // indirect
github.com/docker/docker v20.10.13+incompatible // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/fsnotify/fsnotify v1.4.9 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/go-playground/locales v0.13.0 // indirect
github.com/go-playground/universal-translator v0.17.0 // indirect
github.com/goccy/go-json v0.9.11 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/flatbuffers v2.0.8+incompatible // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/s2a-go v0.1.4 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect
github.com/googleapis/gax-go/v2 v2.7.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.4 // indirect
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
github.com/gorilla/css v1.0.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
Expand All @@ -102,13 +116,19 @@ require (
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/jmoiron/sqlx v1.3.5 // indirect
github.com/klauspost/asmfmt v1.3.2 // indirect
github.com/klauspost/compress v1.15.9 // indirect
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
github.com/leodido/go-urn v1.2.0 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/lufia/plan9stats v0.0.0-20240819163618-b1d8f4d146e7 // indirect
github.com/magiconair/properties v1.8.5 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/microcosm-cc/bluemonday v1.0.6 // indirect
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
github.com/muesli/reflow v0.2.0 // indirect
Expand All @@ -118,45 +138,54 @@ require (
github.com/opencontainers/image-spec v1.0.2 // indirect
github.com/opencontainers/runc v1.1.2 // indirect
github.com/pelletier/go-toml v1.9.3 // indirect
github.com/pierrec/lz4/v4 v4.1.15 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/russross/blackfriday/v2 v2.0.1 // indirect
github.com/schollz/progressbar/v3 v3.8.5 // indirect
github.com/shirou/gopsutil/v4 v4.24.7 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
github.com/spf13/afero v1.9.2 // indirect
github.com/spf13/cast v1.3.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/spf13/viper v1.8.1 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
github.com/tidwall/gjson v1.17.1 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
github.com/tidwall/sjson v1.2.5 // indirect
github.com/uptrace/opentelemetry-go-extra/otelsql v0.1.17 // indirect
github.com/tklauser/go-sysconf v0.3.14 // indirect
github.com/tklauser/numcpus v0.8.0 // indirect
github.com/uptrace/opentelemetry-go-extra/otelsql v0.2.4 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/yuin/goldmark v1.4.13 // indirect
github.com/yuin/goldmark-emoji v1.0.1 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0 // indirect
go.opentelemetry.io/otel/metric v0.34.0 // indirect
go.opentelemetry.io/otel/trace v1.11.2 // indirect
go.opentelemetry.io/proto/otlp v0.11.0 // indirect
go.opentelemetry.io/otel/metric v1.29.0 // indirect
go.opentelemetry.io/otel/trace v1.29.0 // indirect
go.opentelemetry.io/proto/otlp v1.0.0 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/multierr v1.7.0 // indirect
go.uber.org/zap v1.19.0 // indirect
golang.org/x/crypto v0.6.0 // indirect
golang.org/x/sys v0.6.0 // indirect
golang.org/x/term v0.6.0 // indirect
golang.org/x/text v0.8.0 // indirect
golang.org/x/crypto v0.22.0 // indirect
golang.org/x/mod v0.11.0 // indirect
golang.org/x/sys v0.24.0 // indirect
golang.org/x/term v0.19.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.10.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect
gopkg.in/ini.v1 v1.62.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)
Loading
Loading