diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml new file mode 100644 index 0000000..6f2632d --- /dev/null +++ b/.github/workflows/acceptance-tests.yml @@ -0,0 +1,29 @@ +name: Acceptance Tests + +on: [push] + +jobs: + acceptance-tests: + runs-on: ubuntu-latest + steps: + - name: Check out + uses: actions/checkout@v4 + - name: Check out oats + uses: actions/checkout@v4 + with: + repository: grafana/oats + ref: f73c5c5eb1715296d0900eb5fc7c26fd14968b3f + path: oats + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.21' + cache-dependency-path: oats/go.sum + - name: Run acceptance tests + run: ./scripts/run-acceptance-tests.sh + - name: upload log file + uses: actions/upload-artifact@v4 + if: failure() + with: + name: docker-compose.log + path: oats/yaml/build/**/output.log diff --git a/README.md b/README.md index 3426304..9de8a9a 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ The Docker image is available on Docker hub: https://hub.docker.com/r/grafana/ot ## Run the Docker image ```sh -docker run -p 3000:3000 -p 4317:4317 --rm -ti grafana/otel-lgtm +docker run -p 3000:3000 -p 4317:4317 -p 4318:4318 --rm -ti grafana/otel-lgtm ``` ## Send OpenTelemetry Data diff --git a/docker/run-all.sh b/docker/run-all.sh index 4051f67..47dd675 100755 --- a/docker/run-all.sh +++ b/docker/run-all.sh @@ -17,7 +17,8 @@ done echo "The OpenTelemtry collector and the Grafana LGTM stack are up and running!" echo "Open ports:" -echo " - 4317: OpenTelemetry endpoint" +echo " - 4317: OpenTelemetry GRPC endpoint" +echo " - 4318: OpenTelemetry HTTP endpoint" echo " - 3000: Grafana. User: admin, password: admin" sleep infinity diff --git a/examples/go/Dockerfile b/examples/go/Dockerfile new file mode 100644 index 0000000..039c665 --- /dev/null +++ b/examples/go/Dockerfile @@ -0,0 +1,27 @@ +# syntax=docker/dockerfile:1 + +FROM golang:1.21 + +# Set destination for COPY +WORKDIR /app + +# Download Go modules +COPY go.mod go.sum ./ +RUN go mod download + +# Copy the source code. Note the slash at the end, as explained in +# https://docs.docker.com/engine/reference/builder/#copy +COPY *.go ./ + +# Build +RUN CGO_ENABLED=0 GOOS=linux go build -o /rolldice + +# Optional: +# To bind to a TCP port, runtime parameters must be supplied to the docker command. +# But we can document in the Dockerfile what ports +# the application is going to listen on by default. +# https://docs.docker.com/engine/reference/builder/#expose +EXPOSE 8080 + +# Run +CMD ["/rolldice"] diff --git a/examples/go/docker-compose.oats.yml b/examples/go/docker-compose.oats.yml new file mode 100644 index 0000000..728017c --- /dev/null +++ b/examples/go/docker-compose.oats.yml @@ -0,0 +1,12 @@ +version: '3.4' + +services: + go: + build: + context: . + dockerfile: Dockerfile + environment: + OTEL_EXPORTER_OTLP_ENDPOINT: http://collector:4318 + OTEL_METRIC_EXPORT_INTERVAL: "5000" # so we don't have to wait 60s for metrics + ports: + - "8080:8080" diff --git a/examples/go/go.mod b/examples/go/go.mod new file mode 100644 index 0000000..5191ae4 --- /dev/null +++ b/examples/go/go.mod @@ -0,0 +1,33 @@ +module dice + +go 1.21.1 + +require ( + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 + go.opentelemetry.io/contrib/instrumentation/runtime v0.47.0 + go.opentelemetry.io/otel v1.23.0-rc.1 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.45.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0 + go.opentelemetry.io/otel/sdk v1.23.0-rc.1 + go.opentelemetry.io/otel/sdk/metric v1.23.0-rc.1 +) + +require ( + github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect + go.opentelemetry.io/otel/metric v1.23.0-rc.1 // indirect + go.opentelemetry.io/otel/trace v1.23.0-rc.1 // indirect + go.opentelemetry.io/proto/otlp v1.1.0 // indirect + golang.org/x/net v0.20.0 // indirect + golang.org/x/sys v0.16.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240125205218-1f4bbc51befe // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe // indirect + google.golang.org/grpc v1.61.0 // indirect + google.golang.org/protobuf v1.32.0 // indirect +) diff --git a/examples/go/go.sum b/examples/go/go.sum new file mode 100644 index 0000000..5c6a038 --- /dev/null +++ b/examples/go/go.sum @@ -0,0 +1,66 @@ +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 h1:Wqo399gCIufwto+VfwCSvsnfGpF/w5E9CNxSwbpD6No= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0/go.mod h1:qmOFXW2epJhM0qSnUUYpldc7gVz2KMQwJ/QYCDIa7XU= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 h1:sv9kVfal0MK0wBMCOGr+HeJm9v803BkJxGrk2au7j08= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw= +go.opentelemetry.io/contrib/instrumentation/runtime v0.47.0 h1:fQChVLLl1h/42YGVoikLZ8yBrf1VG4DSEJ1zDnMfIog= +go.opentelemetry.io/contrib/instrumentation/runtime v0.47.0/go.mod h1:oEBtteRW7mKJc+yAKRuEu0xk5wyPUKpm41/bDM4ZKeY= +go.opentelemetry.io/otel v1.23.0-rc.1 h1:eIGbuHdW75X7KQSd7WpXYgS1Iwe+18vQxFenkRk5K5E= +go.opentelemetry.io/otel v1.23.0-rc.1/go.mod h1:06VVpzu9fzL3H1BTLZQ1kpC/Y41EQNo5cDMsV6O80jI= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.45.0 h1:+RbSCde0ERway5FwKvXR3aRJIFeDu9rtwC6E7BC6uoM= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.45.0/go.mod h1:zcI8u2EJxbLPyoZ3SkVAAcQPgYb1TDRzW93xLFnsggU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 h1:9M3+rhx7kZCIQQhQRYaZCdNu1V73tm4TvXs2ntl98C4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0/go.mod h1:noq80iT8rrHP1SfybmPiRGc9dc5M8RPmGvtwo7Oo7tc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0 h1:FyjCyI9jVEfqhUh2MoSkmolPjfh5fp2hnV0b0irxH4Q= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0/go.mod h1:hYwym2nDEeZfG/motx0p7L7J1N1vyzIThemQsb4g2qY= +go.opentelemetry.io/otel/metric v1.23.0-rc.1 h1:A2t8VfQCuGL/DjA3pEXoqzJ6mFgxLttCec8BFShlCQE= +go.opentelemetry.io/otel/metric v1.23.0-rc.1/go.mod h1:nq54itv7Gh7nU0zDQ9bMoBFqEry1DC9yjloUUMmnFZI= +go.opentelemetry.io/otel/sdk v1.23.0-rc.1 h1:ih9KV8Pa8RO+yseCz5Zy/gvkTSIWPr4FyjXDotbvUMY= +go.opentelemetry.io/otel/sdk v1.23.0-rc.1/go.mod h1:3LUZujDnr6ab23YqfWWYctiDz0UEhQoXQx8/GPxQOxk= +go.opentelemetry.io/otel/sdk/metric v1.23.0-rc.1 h1:8O4sg2rHQ6Bro444JrEGMtqABzybdbclIs3hn0shTYc= +go.opentelemetry.io/otel/sdk/metric v1.23.0-rc.1/go.mod h1:aBRZyYdmoF5Sj7rd9ZjoU1c0REbgw3niLynBzJJqYGw= +go.opentelemetry.io/otel/trace v1.23.0-rc.1 h1:zA5tq4Mev4r4dvKxFmzIjxibe/UMwjMHMkbrOENwnFQ= +go.opentelemetry.io/otel/trace v1.23.0-rc.1/go.mod h1:vqMYXV//qeU9rT+yWHVSWQ1MhjnVfiERL594PcPcThg= +go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI= +go.opentelemetry.io/proto/otlp v1.1.0/go.mod h1:GpBHCBWiqvVLDqmHZsoMM3C5ySeKTC7ej/RNTae6MdY= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac h1:ZL/Teoy/ZGnzyrqK/Optxxp2pmVh+fmJ97slxSRyzUg= +google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:+Rvu7ElI+aLzyDQhpHMFMMltsD6m7nqpuWDd2CwJw3k= +google.golang.org/genproto/googleapis/api v0.0.0-20240125205218-1f4bbc51befe h1:0poefMBYvYbs7g5UkjS6HcxBPaTRAmznle9jnxYoAI8= +google.golang.org/genproto/googleapis/api v0.0.0-20240125205218-1f4bbc51befe/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe h1:bQnxqljG/wqi4NTXu2+DJ3n7APcEA882QZ1JvhQAq9o= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= +google.golang.org/grpc v1.61.0 h1:TOvOcuXn30kRao+gfcvsebNEa5iZIiLkisYEkf7R7o0= +google.golang.org/grpc v1.61.0/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/examples/go/main.go b/examples/go/main.go new file mode 100644 index 0000000..0f7b28d --- /dev/null +++ b/examples/go/main.go @@ -0,0 +1,83 @@ +package main + +import ( + "context" + "errors" + "log" + "net" + "net/http" + "os" + "os/signal" + "time" + + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" +) + +func main() { + if err := run(); err != nil { + log.Fatalln(err) + } +} + +func run() (err error) { + // Handle SIGINT (CTRL+C) gracefully. + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop() + + // Set up OpenTelemetry. + otelShutdown, err := setupOTelSDK(ctx) + if err != nil { + return + } + // Handle shutdown properly so nothing leaks. + defer func() { + err = errors.Join(err, otelShutdown(context.Background())) + }() + + // Start HTTP server. + srv := &http.Server{ + Addr: ":8080", + BaseContext: func(_ net.Listener) context.Context { return ctx }, + ReadTimeout: time.Second, + WriteTimeout: 10 * time.Second, + Handler: newHTTPHandler(), + } + srvErr := make(chan error, 1) + go func() { + srvErr <- srv.ListenAndServe() + }() + + // Wait for interruption. + select { + case err = <-srvErr: + // Error when starting HTTP server. + return + case <-ctx.Done(): + // Wait for first CTRL+C. + // Stop receiving signal notifications as soon as possible. + stop() + } + + // When Shutdown is called, ListenAndServe immediately returns ErrServerClosed. + err = srv.Shutdown(context.Background()) + return +} + +func newHTTPHandler() http.Handler { + mux := http.NewServeMux() + + // handleFunc is a replacement for mux.HandleFunc + // which enriches the handler's HTTP instrumentation with the pattern as the http.route. + handleFunc := func(pattern string, handlerFunc func(http.ResponseWriter, *http.Request)) { + // Configure the "http.route" for the HTTP instrumentation. + handler := otelhttp.WithRouteTag(pattern, http.HandlerFunc(handlerFunc)) + mux.Handle(pattern, handler) + } + + // Register handlers. + handleFunc("/", rolldice) + + // Add HTTP instrumentation for the whole server. + handler := otelhttp.NewHandler(mux, "/") + return handler +} diff --git a/examples/go/oats.yaml b/examples/go/oats.yaml new file mode 100644 index 0000000..58ad1a6 --- /dev/null +++ b/examples/go/oats.yaml @@ -0,0 +1,16 @@ +docker-compose: + generator: lgtm + files: + - ./docker-compose.oats.yml +input: + - path: /rolldice +expected: + traces: + - traceql: '{ span.http.route = "/rolldice" }' + spans: + - name: '/' # should be "GET /rolldice" + attributes: + otel.library.name: go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp + metrics: + - promql: 'process_runtime_go_goroutines{}' + value: "> 0" diff --git a/examples/go/otel.go b/examples/go/otel.go new file mode 100644 index 0000000..fcdeba8 --- /dev/null +++ b/examples/go/otel.go @@ -0,0 +1,78 @@ +package main + +import ( + "context" + "errors" + "go.opentelemetry.io/contrib/instrumentation/runtime" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/trace" + "log" + "time" +) + +// setupOTelSDK bootstraps the OpenTelemetry pipeline. +// If it does not return an error, make sure to call shutdown for proper cleanup. +func setupOTelSDK(ctx context.Context) (shutdown func(context.Context) error, err error) { + var shutdownFuncs []func(context.Context) error + + // shutdown calls cleanup functions registered via shutdownFuncs. + // The errors from the calls are joined. + // Each registered cleanup will be invoked once. + shutdown = func(ctx context.Context) error { + var err error + for _, fn := range shutdownFuncs { + err = errors.Join(err, fn(ctx)) + } + shutdownFuncs = nil + return err + } + + // handleErr calls shutdown for cleanup and makes sure that all errors are returned. + handleErr := func(inErr error) { + err = errors.Join(inErr, shutdown(ctx)) + } + + prop := propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + ) + otel.SetTextMapPropagator(prop) + + traceExporter, err := otlptrace.New(ctx, otlptracehttp.NewClient()) + if err != nil { + return nil, err + } + + tracerProvider := trace.NewTracerProvider(trace.WithBatcher(traceExporter)) + if err != nil { + handleErr(err) + return + } + shutdownFuncs = append(shutdownFuncs, tracerProvider.Shutdown) + otel.SetTracerProvider(tracerProvider) + + metricExporter, err := otlpmetrichttp.New(ctx) + if err != nil { + return nil, err + } + + meterProvider := metric.NewMeterProvider(metric.WithReader(metric.NewPeriodicReader(metricExporter))) + if err != nil { + handleErr(err) + return + } + shutdownFuncs = append(shutdownFuncs, meterProvider.Shutdown) + otel.SetMeterProvider(meterProvider) + + err = runtime.Start(runtime.WithMinimumReadMemStatsInterval(time.Second)) + if err != nil { + log.Fatal(err) + } + + return +} diff --git a/examples/go/rolldice.go b/examples/go/rolldice.go new file mode 100644 index 0000000..f644e37 --- /dev/null +++ b/examples/go/rolldice.go @@ -0,0 +1,21 @@ +package main + +import ( + "fmt" + "io" + "log" + "math/rand" + "net/http" + "strconv" +) + +func rolldice(w http.ResponseWriter, r *http.Request) { + roll := 1 + rand.Intn(6) + + fmt.Printf("Rolled a %d\n", roll) + + resp := strconv.Itoa(roll) + "\n" + if _, err := io.WriteString(w, resp); err != nil { + log.Printf("Write failed: %v\n", err) + } +} diff --git a/examples/go/run.sh b/examples/go/run.sh new file mode 100755 index 0000000..f09035f --- /dev/null +++ b/examples/go/run.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -euo pipefail + +export OTEL_EXPORTER_OTLP_INSECURE="true" +export OTEL_METRIC_EXPORT_INTERVAL="5000" # so we don't have to wait 60s for metrics +export OTEL_RESOURCE_ATTRIBUTES="service.name=example-app,service.instance.id=localhost:8080" +go run . diff --git a/scripts/run-acceptance-tests.sh b/scripts/run-acceptance-tests.sh new file mode 100755 index 0000000..0a684db --- /dev/null +++ b/scripts/run-acceptance-tests.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -euo pipefail + +cd oats/yaml +go install github.com/onsi/ginkgo/v2/ginkgo +export TESTCASE_TIMEOUT=5m +export TESTCASE_BASE_PATH=../../examples +ginkgo -r