From 1312ac624d06e902daeba1a2e52c5f7cc9fb8afe Mon Sep 17 00:00:00 2001 From: Gert Drapers <1533850+gertd@users.noreply.github.com> Date: Tue, 23 Jul 2024 10:14:30 -0700 Subject: [PATCH] fold on-demand edge sync message (#429) * refactor on-demand sync, adds folding of on-demand edge sync messages * update go-edge-ds * go-directory v0.31.6 (pb-directory v0.31.4) * go-authorizer v0.20.17 (pb-authorizer v0.20.9) * errors v0.0.9 --- builtins/edge/ds/check.go | 2 + cmd/topazd/topaz_run.go | 11 +- go.mod | 35 +++--- go.sum | 70 ++++++----- makefile | 6 +- pkg/app/impl/jwt.go | 11 +- pkg/app/management/command_handler.go | 1 - pkg/cc/config/config.go | 6 +- pkg/cc/config/loader.go | 4 + pkg/cc/config/schema/config.json | 10 +- pkg/cc/config/schema/config.yaml | 4 +- pkg/cli/cmd/directory/test.go | 2 + pkg/debug/debug.go | 71 +++++++----- plugins/edge/plugin.go | 161 +++++++++++++++++++++----- 14 files changed, 252 insertions(+), 142 deletions(-) diff --git a/builtins/edge/ds/check.go b/builtins/edge/ds/check.go index 29b001ab..1100de96 100644 --- a/builtins/edge/ds/check.go +++ b/builtins/edge/ds/check.go @@ -101,6 +101,7 @@ func RegisterCheckRelation(logger *zerolog.Logger, fnName string, dr resolvers.D return nil, errors.Wrapf(err, "get directory client") } + //nolint: staticcheck // SA1019: client.CheckRelation is deprecated resp, err := client.CheckRelation(bctx.Context, &args) if err != nil { traceError(&bctx, fnName, err) @@ -151,6 +152,7 @@ func RegisterCheckPermission(logger *zerolog.Logger, fnName string, dr resolvers return nil, errors.Wrapf(err, "get directory client") } + //nolint: staticcheck // SA1019: client.CheckPermission is deprecated resp, err := client.CheckPermission(bctx.Context, &args) if err != nil { traceError(&bctx, fnName, err) diff --git a/cmd/topazd/topaz_run.go b/cmd/topazd/topaz_run.go index 27d4ef25..97cc0d56 100644 --- a/cmd/topazd/topaz_run.go +++ b/cmd/topazd/topaz_run.go @@ -9,7 +9,6 @@ import ( "github.com/aserto-dev/topaz/pkg/cc/config" "github.com/aserto-dev/topaz/pkg/debug" "github.com/spf13/cobra" - "golang.org/x/sync/errgroup" ) var ( @@ -19,7 +18,6 @@ var ( flagRunIgnorePaths []string flagRunDebug bool debugService *debug.Server - errGroup *errgroup.Group ) var cmdRun = &cobra.Command{ @@ -44,9 +42,7 @@ var cmdRun = &cobra.Command{ } if flagRunDebug { - cfg.Debug.Enabled = true - cfg.Debug.ListenAddress = "localhost:6060" - cfg.Debug.ShutdownTimeout = 5 + cfg.Common.DebugService.Enabled = flagRunDebug } }) defer func() { @@ -63,9 +59,8 @@ var cmdRun = &cobra.Command{ return err } - if topazApp.Configuration.Debug.Enabled { - errGroup = new(errgroup.Group) - debugService = debug.NewServer(&topazApp.Configuration.Debug, topazApp.Logger, errGroup) + if topazApp.Configuration.DebugService.Enabled { + debugService = debug.NewServer(&topazApp.Configuration.DebugService, topazApp.Logger) debugService.Start() } diff --git a/go.mod b/go.mod index 7357e3a6..4e2c9476 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ toolchain go1.22.5 // replace github.com/aserto-dev/go-directory => ../go-directory // replace github.com/aserto-dev/go-directory-cli => ../go-directory-cli // replace github.com/aserto-dev/go-edge-ds => ../go-edge-ds + // replace github.com/aserto-dev/azm => ../azm // replace github.com/aserto-dev/go-topaz-ui => ../go-topaz-ui // replace github.com/aserto-dev/service-host => ../service-host @@ -21,13 +22,13 @@ require ( github.com/aserto-dev/aserto-management v0.9.4 github.com/aserto-dev/azm v0.1.11 github.com/aserto-dev/certs v0.0.6 - github.com/aserto-dev/errors v0.0.8 + github.com/aserto-dev/errors v0.0.9 github.com/aserto-dev/go-aserto v0.31.4 - github.com/aserto-dev/go-authorizer v0.20.6 - github.com/aserto-dev/go-directory v0.31.5 + github.com/aserto-dev/go-authorizer v0.20.7 + github.com/aserto-dev/go-directory v0.31.6 github.com/aserto-dev/go-directory-cli v0.31.3-0.20240711215128-477b23ffcb75 - github.com/aserto-dev/go-edge-ds v0.32.1 - github.com/aserto-dev/go-grpc v0.8.65 + github.com/aserto-dev/go-edge-ds v0.32.3 + github.com/aserto-dev/go-grpc v0.8.67 github.com/aserto-dev/go-topaz-ui v0.1.11 github.com/aserto-dev/header v0.0.7 github.com/aserto-dev/logger v0.0.4 @@ -37,7 +38,7 @@ require ( github.com/aserto-dev/self-decision-logger v0.0.5 github.com/aserto-dev/service-host v0.0.14 github.com/cli/browser v1.3.0 - github.com/docker/docker v27.0.3+incompatible + github.com/docker/docker v27.1.0+incompatible github.com/docker/go-connections v0.5.0 github.com/fatih/color v1.17.0 github.com/fullstorydev/grpcurl v1.9.1 @@ -59,7 +60,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/rivo/tview v0.0.0-20240625185742-b0a7293b8130 github.com/rs/zerolog v1.33.0 - github.com/samber/lo v1.44.0 + github.com/samber/lo v1.46.0 github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 @@ -73,7 +74,7 @@ require ( ) require ( - buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240508200655-46a4cf4ba109.2 // indirect + buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240717164558-a6c49f84cc0f.2 // indirect contrib.go.opencensus.io/exporter/prometheus v0.4.2 // indirect github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect @@ -85,7 +86,7 @@ require ( github.com/aserto-dev/go-decision-logs v0.0.4 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bufbuild/protocompile v0.13.0 // indirect - github.com/bufbuild/protovalidate-go v0.6.2 // indirect + github.com/bufbuild/protovalidate-go v0.6.3 // indirect github.com/bytecodealliance/wasmtime-go/v3 v3.0.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b // indirect @@ -187,16 +188,16 @@ require ( go.opentelemetry.io/otel/sdk v1.27.0 // indirect go.opentelemetry.io/otel/trace v1.27.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.24.0 // indirect - golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect - golang.org/x/mod v0.18.0 // indirect - golang.org/x/net v0.26.0 // indirect - golang.org/x/term v0.21.0 // indirect + golang.org/x/crypto v0.25.0 // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect + golang.org/x/mod v0.19.0 // indirect + golang.org/x/net v0.27.0 // indirect + golang.org/x/term v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.22.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240624140628-dc46fd24d27d // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d // indirect + golang.org/x/tools v0.23.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240722135656-d784300faade // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index bfff773e..2c31feab 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240508200655-46a4cf4ba109.2 h1:cFrEG/pJch6t62+jqndcPXeTNkYcztS4tBRgNkR+drw= -buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240508200655-46a4cf4ba109.2/go.mod h1:ylS4c28ACSI59oJrOdW4pHS4n0Hw4TgSPHn8rpHl4Yw= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240717164558-a6c49f84cc0f.2 h1:SZRVx928rbYZ6hEKUIN+vtGDkl7uotABRWGY4OAg5gM= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240717164558-a6c49f84cc0f.2/go.mod h1:ylS4c28ACSI59oJrOdW4pHS4n0Hw4TgSPHn8rpHl4Yw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -423,24 +423,22 @@ github.com/aserto-dev/azm v0.1.11 h1:cYpIsmeI7DpwnjqKTKUlz4CqhODpx+zncM97G0kSYu8 github.com/aserto-dev/azm v0.1.11/go.mod h1:p0O2nWcmIsGphsST7QrGzOIalafw6SOtNQeyhN2TnUo= github.com/aserto-dev/certs v0.0.6 h1:V79JptXMvgePJ2rgktu8QdgbEQsIXxF1kXqQhPM76NI= github.com/aserto-dev/certs v0.0.6/go.mod h1:ppJ/GVSerB8EsikJeQP71LJBuHiUW2avS2f84oC/75c= -github.com/aserto-dev/errors v0.0.8 h1:lqfCc3n8f2VIrsj0hOEmktnqYUS6tffl+w1yJftBhp8= -github.com/aserto-dev/errors v0.0.8/go.mod h1:wiQ68oKTyAQVAv+SvB5GyuAD6PewalA/D6Fq30gNUbQ= +github.com/aserto-dev/errors v0.0.9 h1:Pi6UT/v+0DYc25AKVFgazkz7Sua3zyU3KWl5VsoRPKk= +github.com/aserto-dev/errors v0.0.9/go.mod h1:mk3xXE6e9cA5NFk2B3KdNuUcSR+dVGbIlvbaT5ZTOtQ= github.com/aserto-dev/go-aserto v0.31.4 h1:i4EYTNwyARhEZrN00CU7OnXJCE5xwo6w67ncwJcxmBM= github.com/aserto-dev/go-aserto v0.31.4/go.mod h1:6mW3wUC62a8TbqYH64lq350MBj+W3WwtlxXNZRybbOY= -github.com/aserto-dev/go-authorizer v0.20.6 h1:fSpnjQwz5k49dl9fBVyzMPbI/dxI/KXCuKpx3ztAv0U= -github.com/aserto-dev/go-authorizer v0.20.6/go.mod h1:UYrV3teiX6KlShTzX7HC8KvAJsRI9QUyxJEWbVFYBFU= +github.com/aserto-dev/go-authorizer v0.20.7 h1:g8N24ttTmOR0MeRi6G6AC5pmTYRwSsB8XRdOl+YROis= +github.com/aserto-dev/go-authorizer v0.20.7/go.mod h1:lncQZDlVYhXMsVBYnPvrFhGjtuvmNBJfUkIamy1dtrM= github.com/aserto-dev/go-decision-logs v0.0.4 h1:beu/mhqZ92ovhSIPOv2f4q0Ci7HWNLla/j/x+ZD5eHw= github.com/aserto-dev/go-decision-logs v0.0.4/go.mod h1:W50DNu4HPCk+iyI39cP3+KBytdrQYVieSPXh9StuRzA= -github.com/aserto-dev/go-directory v0.31.5 h1:DPGE8SGW2Z3EFS7VB6v0ds81meKXtHGpI0YYrZ7Ea80= -github.com/aserto-dev/go-directory v0.31.5/go.mod h1:fgIT515NcK+4m1UyriBCPMwFHl3DHPaa8yBLbEb9r/I= +github.com/aserto-dev/go-directory v0.31.6 h1:gHL/z2Gegc+CmaarUnNGk+Zya/eJ4eS1vGfEiRYQeX0= +github.com/aserto-dev/go-directory v0.31.6/go.mod h1:T125gQBcPXrLUh7bMPFHzsXRHUnZb++DzhPk8o8tc4k= github.com/aserto-dev/go-directory-cli v0.31.3-0.20240711215128-477b23ffcb75 h1:9J/BCcgYnG3+doxp50qaCEhPMgcd+n2c23N537kSf1g= github.com/aserto-dev/go-directory-cli v0.31.3-0.20240711215128-477b23ffcb75/go.mod h1:/y3uE2GDZDDJktgWlOzBLhSiZO7kN3Lri31aOPElbPs= -github.com/aserto-dev/go-edge-ds v0.32.1 h1:4kwC6OvVGD+DYtlRwq4eicYBEqn1YKj5S0RatozHm50= -github.com/aserto-dev/go-edge-ds v0.32.1/go.mod h1:nxGsXHP+JzV6ubKoT9XW2L0MIyzC2Dpc7a9adxjXFR0= -github.com/aserto-dev/go-grpc v0.8.65 h1:qj84Ps9nkjAk7SzS+53jrd9PGgmq3ALq8pad2WzYXDY= -github.com/aserto-dev/go-grpc v0.8.65/go.mod h1:ezNdiKzZZCBwpANK6TYNKwe3qTbNFwZB7OEl+v2d/Gw= -github.com/aserto-dev/go-topaz-ui v0.1.10 h1:EXgWMOoGvoyTHCCDhXGeCw/W0ABxFlsyka+A1dk/OnQ= -github.com/aserto-dev/go-topaz-ui v0.1.10/go.mod h1:BdW4vOhuG0HkjNH4L4omcNa/RJJ4LaxFzbw92qc+pc8= +github.com/aserto-dev/go-edge-ds v0.32.3 h1:/A2HvlZvtKkYIjaAbQ+C4L3uWj5RZAr7+VjzrHUVhQU= +github.com/aserto-dev/go-edge-ds v0.32.3/go.mod h1:H98tGHc/KWTrMczXuao4L4r10B5GPXPhq5K7R2uy9xE= +github.com/aserto-dev/go-grpc v0.8.67 h1:6DGZiIjIWWFMQ81bLswTqbuQLWS5zycbAQIJsyPmftA= +github.com/aserto-dev/go-grpc v0.8.67/go.mod h1:ezNdiKzZZCBwpANK6TYNKwe3qTbNFwZB7OEl+v2d/Gw= github.com/aserto-dev/go-topaz-ui v0.1.11 h1:yfal8RfjBiK7srrhs6u5rUa2U8+HJVlkTo4En7vCg/E= github.com/aserto-dev/go-topaz-ui v0.1.11/go.mod h1:BdW4vOhuG0HkjNH4L4omcNa/RJJ4LaxFzbw92qc+pc8= github.com/aserto-dev/header v0.0.7 h1:hlo5/zYsBOsxzPxtve7LRbXyBbQmKSPAyOfmPhGgirM= @@ -464,8 +462,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bufbuild/protocompile v0.13.0 h1:6cwUB0Y2tSvmNxsbunwzmIto3xOlJOV7ALALuVOs92M= github.com/bufbuild/protocompile v0.13.0/go.mod h1:dr++fGGeMPWHv7jPeT06ZKukm45NJscd7rUxQVzEKRk= -github.com/bufbuild/protovalidate-go v0.6.2 h1:U/V3CGF0kPlR12v41rjO4DrYZtLcS4ZONLmWN+rJVCQ= -github.com/bufbuild/protovalidate-go v0.6.2/go.mod h1:4BR3rKEJiUiTy+sqsusFn2ladOf0kYmA2Reo6BHSBgQ= +github.com/bufbuild/protovalidate-go v0.6.3 h1:wxQyzW035zM16Binbaz/nWAzS12dRIXhZdSUWRY7Fv0= +github.com/bufbuild/protovalidate-go v0.6.3/go.mod h1:J4PtwP9Z2YAGgB0+o+tTWEDtLtXvz/gfhFZD8pbzM/U= github.com/bytecodealliance/wasmtime-go/v3 v3.0.2 h1:3uZCA/BLTIu+DqCfguByNMJa2HVHpXvjfy0Dy7g6fuA= github.com/bytecodealliance/wasmtime-go/v3 v3.0.2/go.mod h1:RnUjnIXxEJcL6BgCvNyzCCRzZcxCgsZCi+RNlvYor5Q= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= @@ -529,8 +527,8 @@ github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+ github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v27.0.3+incompatible h1:aBGI9TeQ4MPlhquTQKq9XbK79rKFVwXNUAYz9aXyEBE= -github.com/docker/docker v27.0.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.1.0+incompatible h1:rEHVQc4GZ0MIQKifQPHSFGV/dVgaZafgRf8fCPtDYBs= +github.com/docker/docker v27.1.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -923,8 +921,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= -github.com/samber/lo v1.44.0 h1:5il56KxRE+GHsm1IR+sZ/6J42NODigFiqCWpSc2dybA= -github.com/samber/lo v1.44.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= +github.com/samber/lo v1.46.0 h1:w8G+oaCPgz1PoCJztqymCFaKwXt+5cCXn51uPxExFfQ= +github.com/samber/lo v1.46.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -1029,8 +1027,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1041,8 +1039,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= -golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1072,8 +1070,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= -golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= +golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1128,8 +1126,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1265,8 +1263,8 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1349,8 +1347,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= -golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= -golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= +golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= +golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1528,10 +1526,10 @@ google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZV google.golang.org/genproto v0.0.0-20230202175211-008b39050e57/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= -google.golang.org/genproto/googleapis/api v0.0.0-20240624140628-dc46fd24d27d h1:Aqf0fiIdUQEj0Gn9mKFFXoQfTTEaNopWpfVyYADxiSg= -google.golang.org/genproto/googleapis/api v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:Od4k8V1LQSizPRUK4OzZ7TBE/20k+jPczUDAEyvn69Y= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d h1:k3zyW3BYYR30e8v3x0bTDdE9vpYFjZHK+HcyqkrppWk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/genproto/googleapis/api v0.0.0-20240722135656-d784300faade h1:WxZOF2yayUHpHSbUE6NMzumUzBxYc3YGwo0YHnbzsJY= +google.golang.org/genproto/googleapis/api v0.0.0-20240722135656-d784300faade/go.mod h1:mw8MG/Qz5wfgYr6VqVCiZcHe/GJEfI+oGGDCohaVgB0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade h1:oCRSWfwGXQsqlVdErcyTt4A93Y8fo0/9D4b1gnI++qo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= diff --git a/makefile b/makefile index 4f188470..b5058771 100644 --- a/makefile +++ b/makefile @@ -22,12 +22,12 @@ GOTESTSUM_VER := 1.11.0 GOLANGCI-LINT_VER := 1.56.2 GORELEASER_VER := 1.24.0 WIRE_VER := 0.6.0 -BUF_VER := 1.30.0 +BUF_VER := 1.34.0 BUF_USER := $(shell ${EXT_BIN_DIR}/vault kv get -field ASERTO_BUF_USER kv/buf.build) BUF_TOKEN := $(shell ${EXT_BIN_DIR}/vault kv get -field ASERTO_BUF_TOKEN kv/buf.build) BUF_REPO := "buf.build/aserto-dev/directory" -BUF_LATEST := $(shell BUF_BETA_SUPPRESS_WARNINGS=1 ${EXT_BIN_DIR}/buf beta registry tag list buf.build/aserto-dev/directory --format json --reverse | jq -r '.results[0].name') +BUF_LATEST := $(shell BUF_BETA_SUPPRESS_WARNINGS=1 ${EXT_BIN_DIR}/buf beta registry label list buf.build/aserto-dev/directory --format json --reverse | jq -r '.results[0].name') BUF_DEV_IMAGE := "../pb-directory/bin/directory.bin" RELEASE_TAG := $$(svu) @@ -114,7 +114,7 @@ buf-build: ${BIN_DIR} .PHONY: buf-push buf-push: @echo -e "$(ATTN_COLOR)==> $@ $(NO_COLOR)" - @${EXT_BIN_DIR}/buf push proto --tag ${RELEASE_TAG} + @${EXT_BIN_DIR}/buf push proto --label ${RELEASE_TAG} .PHONY: buf-mod-update buf-mod-update: diff --git a/pkg/app/impl/jwt.go b/pkg/app/impl/jwt.go index 3aaf08f0..cbe73984 100644 --- a/pkg/app/impl/jwt.go +++ b/pkg/app/impl/jwt.go @@ -12,17 +12,16 @@ import ( "github.com/aserto-dev/go-authorizer/aserto/authorizer/v2/api" "github.com/aserto-dev/go-authorizer/pkg/aerr" - dsc2 "github.com/aserto-dev/go-directory/aserto/directory/common/v2" - "github.com/aserto-dev/go-directory/aserto/directory/common/v3" + dsc3 "github.com/aserto-dev/go-directory/aserto/directory/common/v3" dsr3 "github.com/aserto-dev/go-directory/aserto/directory/reader/v3" "github.com/aserto-dev/go-directory/pkg/pb" + "github.com/aserto-dev/topaz/directory" + "github.com/lestrrat-go/jwx/v2/jwk" "github.com/lestrrat-go/jwx/v2/jwt" "github.com/pkg/errors" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/structpb" - - "github.com/aserto-dev/topaz/directory" ) var ( @@ -36,7 +35,7 @@ var ( // getUserFromJWT. func (s *AuthorizerServer) getUserFromJWT(ctx context.Context, bearerJWT string) (proto.Message, error) { - resp := dsc2.Object{} + resp := dsc3.Object{} ident, err := s.getIdentityFromJWT(ctx, bearerJWT) if err != nil { @@ -262,7 +261,7 @@ func (s *AuthorizerServer) getObject(ctx context.Context, objType, objID string) return addObjectKey(objResp.Result) } -func addObjectKey(in *common.Object) (proto.Message, error) { +func addObjectKey(in *dsc3.Object) (proto.Message, error) { buf := new(bytes.Buffer) if err := pb.ProtoToBuf(buf, in); err != nil { return nil, err diff --git a/pkg/app/management/command_handler.go b/pkg/app/management/command_handler.go index f9a4c2b6..6c5f6a81 100644 --- a/pkg/app/management/command_handler.go +++ b/pkg/app/management/command_handler.go @@ -38,7 +38,6 @@ func HandleCommand(ctx context.Context, cmd *api.Command, r *runtime.Runtime) er if !ok { return errors.Errorf("failed to cast discovery plugin") } - edgePlugin.SyncNow(msg.SyncEdgeDirectory.Mode) default: diff --git a/pkg/cc/config/config.go b/pkg/cc/config/config.go index 3ee9e9dc..6c25b181 100644 --- a/pkg/cc/config/config.go +++ b/pkg/cc/config/config.go @@ -47,9 +47,9 @@ type ServicesConfig struct { // Config holds the configuration for the app. type Common struct { - Version int `json:"version"` - Logging logger.Config `json:"logging"` - Debug debug.Config `json:"debug"` + Version int `json:"version"` + Logging logger.Config `json:"logging"` + DebugService debug.Config `json:"debug_service"` Command struct { Mode CommandMode diff --git a/pkg/cc/config/loader.go b/pkg/cc/config/loader.go index 93d43232..a972df98 100644 --- a/pkg/cc/config/loader.go +++ b/pkg/cc/config/loader.go @@ -47,6 +47,10 @@ func LoadConfiguration(fileName string) (*Loader, error) { v.SetEnvPrefix("TOPAZ") // Set defaults + v.SetDefault("debug_service.enabled", false) + v.SetDefault("debug_service.listen_address", "localhost:6060") + v.SetDefault("debug_service.shutdown_timeout", 0) + v.SetDefault("jwt.acceptable_time_skew_seconds", 5) v.SetDefault("opa.max_plugin_wait_time_seconds", "30") diff --git a/pkg/cc/config/schema/config.json b/pkg/cc/config/schema/config.json index 0f04cd4e..0a98f638 100644 --- a/pkg/cc/config/schema/config.json +++ b/pkg/cc/config/schema/config.json @@ -14,8 +14,8 @@ "logging": { "$ref": "#/definitions/Logging" }, - "debug": { - "$ref": "#/definitions/Debug" + "debug_service": { + "$ref": "#/definitions/DebugService" }, "directory": { "$ref": "#definitions/Directory" @@ -91,8 +91,8 @@ } } }, - "Debug": { - "description": "debug setting", + "DebugService": { + "description": "debug service setting", "type": "object", "additionalProperties": false, "properties": { @@ -104,7 +104,7 @@ "listen_address": { "type": "string", "description": "debug service listen address", - "default": "http://localhost:6060" + "default": "localhost:6060" }, "shutdown_timeout": { "type": "integer", diff --git a/pkg/cc/config/schema/config.yaml b/pkg/cc/config/schema/config.yaml index cfe4b69b..40779880 100644 --- a/pkg/cc/config/schema/config.yaml +++ b/pkg/cc/config/schema/config.yaml @@ -9,8 +9,8 @@ logging: log_level: info grpc_log_level: info -# debug settings -debug: +# debug service settings +debug_service: enabled: false listen_address: "" shutdown_timeout: 0 diff --git a/pkg/cli/cmd/directory/test.go b/pkg/cli/cmd/directory/test.go index c24f06e2..5c1f1e78 100644 --- a/pkg/cli/cmd/directory/test.go +++ b/pkg/cli/cmd/directory/test.go @@ -245,6 +245,7 @@ func checkPermissionV3(ctx context.Context, c *client.Client, msg *structpb.Valu start := time.Now() + //nolint: staticcheck // SA1019: c.Reader.CheckPermission resp, err := c.Reader.CheckPermission(ctx, &req) duration := time.Since(start) @@ -265,6 +266,7 @@ func checkRelationV3(ctx context.Context, c *client.Client, msg *structpb.Value) start := time.Now() + //nolint: staticcheck // SA1019: c.Reader.CheckRelation resp, err := c.Reader.CheckRelation(ctx, &req) duration := time.Since(start) diff --git a/pkg/debug/debug.go b/pkg/debug/debug.go index c72b1e29..55b18c80 100644 --- a/pkg/debug/debug.go +++ b/pkg/debug/debug.go @@ -7,7 +7,6 @@ import ( "time" "github.com/rs/zerolog" - "golang.org/x/sync/errgroup" ) type Config struct { @@ -17,41 +16,45 @@ type Config struct { } type Server struct { - server *http.Server - logger *zerolog.Logger - cfg *Config - errGroup *errgroup.Group + server *http.Server + logger *zerolog.Logger + cfg *Config } -func NewServer(cfg *Config, log *zerolog.Logger, errGroup *errgroup.Group) *Server { +func NewServer(cfg *Config, log *zerolog.Logger) *Server { if cfg.Enabled { - pprofMux := http.NewServeMux() - pprofMux.Handle("/debug/allocs", pprof.Handler("allocs")) - pprofMux.Handle("/debug/block", pprof.Handler("block")) - pprofMux.Handle("/debug/goroutine", pprof.Handler("goroutine")) - pprofMux.Handle("/debug/heap", pprof.Handler("heap")) - pprofMux.Handle("/debug/mutex", pprof.Handler("mutex")) - pprofMux.Handle("/debug/threadcreate", pprof.Handler("threadcreate")) - pprofMux.Handle("/debug/profile", http.HandlerFunc(pprof.Profile)) - pprofMux.Handle("/debug/symbol", http.HandlerFunc(pprof.Symbol)) - pprofMux.Handle("/debug/trace", http.HandlerFunc(pprof.Trace)) + http.DefaultServeMux = http.NewServeMux() + + pprofServeMux := http.NewServeMux() + + pprofServeMux.HandleFunc("/debug/pprof/", pprof.Index) + pprofServeMux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + pprofServeMux.HandleFunc("/debug/pprof/profile", pprof.Profile) + pprofServeMux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + pprofServeMux.HandleFunc("/debug/pprof/trace", pprof.Trace) + + pprofServeMux.Handle("/debug/pprof/allocs", pprof.Handler("allocs")) + pprofServeMux.Handle("/debug/pprof/block", pprof.Handler("block")) + pprofServeMux.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine")) + pprofServeMux.Handle("/debug/pprof/heap", pprof.Handler("heap")) + pprofServeMux.Handle("/debug/pprof/mutex", pprof.Handler("mutex")) + pprofServeMux.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate")) + + debugLogger := log.With().Str("component", "debug").Logger() srv := &http.Server{ Addr: cfg.ListenAddress, - Handler: pprofMux, + Handler: pprofServeMux, ReadTimeout: 5 * time.Second, ReadHeaderTimeout: 5 * time.Second, WriteTimeout: 30 * time.Second, IdleTimeout: 30 * time.Second, } - debugLogger := log.With().Str("component", "debug").Logger() - return &Server{ - server: srv, - logger: &debugLogger, - cfg: cfg, - errGroup: errGroup, + server: srv, + logger: &debugLogger, + cfg: cfg, } } @@ -59,18 +62,25 @@ func NewServer(cfg *Config, log *zerolog.Logger, errGroup *errgroup.Group) *Serv } func (srv *Server) Start() { + if !srv.cfg.Enabled { + return + } + if srv != nil { - srv.errGroup.Go(func() error { - err := srv.server.ListenAndServe() - if err != nil && err != http.ErrServerClosed { - srv.logger.Error().Err(err).Str("address", srv.cfg.ListenAddress).Msg("Profiling endpoint failed to listen") + go func() { + srv.logger.Warn().Str("listen_address", srv.cfg.ListenAddress).Msg("debug-service") + if err := srv.server.ListenAndServe(); err != nil { + srv.logger.Error().Err(err).Msg("debug-service") } - return nil - }) + }() } } func (srv *Server) Stop() { + if !srv.cfg.Enabled { + return + } + if srv != nil { var shutdown context.CancelFunc ctx := context.Background() @@ -79,9 +89,10 @@ func (srv *Server) Stop() { ctx, shutdown = context.WithTimeout(ctx, shutdownTimeout) defer shutdown() } + err := srv.server.Shutdown(ctx) if err != nil { - srv.logger.Info().Err(err).Msg("error shutting down debug server") + srv.logger.Info().Err(err).Str("state", "shutdown").Msg("debug-service") } } } diff --git a/plugins/edge/plugin.go b/plugins/edge/plugin.go index a7543e44..15f0faf0 100644 --- a/plugins/edge/plugin.go +++ b/plugins/edge/plugin.go @@ -3,6 +3,7 @@ package edge import ( "context" "strings" + "sync/atomic" "time" "github.com/aserto-dev/go-aserto/client" @@ -148,47 +149,88 @@ func (p *Plugin) scheduler() { interval := time.NewTicker(1 * time.Second) defer interval.Stop() + var running atomic.Bool + running.Store(false) + cycle := cycles + intervalMode := api.SyncMode_SYNC_MODE_UNKNOWN + onDemandMode := api.SyncMode_SYNC_MODE_UNKNOWN + for { select { case <-p.ctx.Done(): - p.logger.Warn().Time("done", time.Now()).Msg(syncScheduler) + p.logger.Debug().Time("done", time.Now()).Msg(syncScheduler) return case t := <-interval.C: p.logger.Info().Time("dispatch", t).Msg(syncScheduler) interval.Stop() - p.task(api.SyncMode_SYNC_MODE_WATERMARK) // watermark sync + intervalMode = api.SyncMode_SYNC_MODE_WATERMARK if cycle%cycles == 0 { - p.task(api.SyncMode_SYNC_MODE_DIFF) + intervalMode = api.SyncMode_SYNC_MODE_DIFF cycle = 0 } cycle++ + p.logger.Debug().Str("mode", printMode(intervalMode)).Msg("interval handler") case mode := <-p.syncNow: - p.logger.Warn().Time("dispatch", time.Now()).Msg(syncOnDemand) + p.logger.Info().Time("dispatch", time.Now()).Msg(syncOnDemand) interval.Stop() - p.task(mode) + onDemandMode = fold(onDemandMode, mode) + p.logger.Debug().Str("mode", printMode(onDemandMode)).Msg("on-demand handler") } - // calculate the interval in secs - // - // p.config.SyncInterval 1m-60m - // 1m -> 60s -> 15s interval - // 5m -> 300s -> 75s interval - // 60m -> 3600s -> 900s interval - waitInSec := (p.config.SyncInterval * 60) / cycles - - wait := time.Duration(waitInSec) * time.Second - interval.Reset(wait) - p.logger.Info().Str("interval", wait.String()).Time("next-run", time.Now().Add(wait)).Msg(syncScheduler) + if !running.Load() { + // determine the run mode + runMode := api.SyncMode_SYNC_MODE_UNKNOWN + if onDemandMode != api.SyncMode_SYNC_MODE_UNKNOWN { + runMode = onDemandMode + onDemandMode = api.SyncMode_SYNC_MODE_UNKNOWN + } else { + runMode = intervalMode + } + + go func() { + p.logger.Debug().Str("mode", printMode(runMode)).Msg("start task") + + running.Store(true) + defer func() { + p.logger.Debug().Str("mode", printMode(runMode)).Msg("finished task") + running.Store(false) + }() + + p.task(runMode) + + // if on-demand mode is UNKNOWN, meaning no new on-demand requests were received while processing the last run, fall back to interval mode. + if onDemandMode == api.SyncMode_SYNC_MODE_UNKNOWN { + wait := p.calcInterval() + interval.Reset(wait) + p.logger.Info().Str("interval", wait.String()).Time("next-run", time.Now().Add(wait)).Msg(syncScheduler) + } else { + p.logger.Warn().Str("mode", printMode(onDemandMode)).Msg("trigger queued on-demand mode") + p.SyncNow(onDemandMode) + } + }() + } } } +// calcInterval - calculates the next time interval in secs, +// based on the configuration SyncInterval (defined on the EdgeDirectory connection) +// returning a time.Duration. +// +// p.config.SyncInterval 1m-60m +// 1m -> 60s -> 15s interval +// 60m -> 3600s -> 900s interval. +func (p *Plugin) calcInterval() time.Duration { + waitInSec := (p.config.SyncInterval * 60) / cycles + return time.Duration(waitInSec) * time.Second +} + func (p *Plugin) task(mode api.SyncMode) { p.logger.Info().Str(status, started).Msg(syncTask) @@ -215,27 +257,52 @@ func (p *Plugin) task(mode api.SyncMode) { } defer conn.Close() - opts := []datasync.Option{} - switch mode { - case api.SyncMode_SYNC_MODE_UNKNOWN: - opts = append(opts, datasync.WithMode(datasync.Manifest), datasync.WithMode(datasync.Full)) - case api.SyncMode_SYNC_MODE_FULL: - opts = append(opts, datasync.WithMode(datasync.Manifest), datasync.WithMode(datasync.Full)) - case api.SyncMode_SYNC_MODE_DIFF: - opts = append(opts, datasync.WithMode(datasync.Manifest), datasync.WithMode(datasync.Diff)) - case api.SyncMode_SYNC_MODE_WATERMARK: - opts = append(opts, datasync.WithMode(datasync.Manifest), datasync.WithMode(datasync.Watermark)) - case api.SyncMode_SYNC_MODE_MANIFEST: - opts = append(opts, datasync.WithMode(datasync.Manifest)) - default: - } - ds, err := directory.Get() if err != nil { p.logger.Error().Err(err).Msg(syncTask) return } + if mode == api.SyncMode_SYNC_MODE_UNKNOWN { + p.exec(ctx, ds, conn, []datasync.Option{ + datasync.WithMode(datasync.Manifest), + datasync.WithMode(datasync.Full), + }) + return + } + + if has(mode, api.SyncMode_SYNC_MODE_WATERMARK) { + p.exec(ctx, ds, conn, []datasync.Option{ + datasync.WithMode(datasync.Manifest), + datasync.WithMode(datasync.Watermark), + }) + } + + if has(mode, api.SyncMode_SYNC_MODE_DIFF) { + p.exec(ctx, ds, conn, []datasync.Option{ + datasync.WithMode(datasync.Manifest), + datasync.WithMode(datasync.Diff), + }) + return + } + + if has(mode, api.SyncMode_SYNC_MODE_FULL) { + p.exec(ctx, ds, conn, []datasync.Option{ + datasync.WithMode(datasync.Manifest), + datasync.WithMode(datasync.Full), + }) + return + } + + if has(mode, api.SyncMode_SYNC_MODE_MANIFEST) && !has(mode, api.SyncMode_SYNC_MODE_WATERMARK) { + p.exec(ctx, ds, conn, []datasync.Option{ + datasync.WithMode(datasync.Manifest), + }) + return + } +} + +func (p *Plugin) exec(ctx context.Context, ds *directory.Directory, conn *grpc.ClientConn, opts []datasync.Option) { if err := ds.DataSyncClient().Sync(ctx, conn, opts...); err != nil { p.logger.Error().Err(err).Msg(syncTask) } @@ -268,3 +335,35 @@ func (p *Plugin) remoteDirectoryClient(ctx context.Context) (*grpc.ClientConn, e return conn, nil } + +func fold(m ...api.SyncMode) api.SyncMode { + r := api.SyncMode_SYNC_MODE_UNKNOWN + for _, v := range m { + r |= v + } + return r +} + +func printMode(mode api.SyncMode) string { + modes := []string{} + if mode&api.SyncMode_SYNC_MODE_MANIFEST != 0 { + modes = append(modes, "MANIFEST") + } + if mode&api.SyncMode_SYNC_MODE_FULL != 0 { + modes = append(modes, "FULL") + } + if mode&api.SyncMode_SYNC_MODE_DIFF != 0 { + modes = append(modes, "DIFF") + } + if mode&api.SyncMode_SYNC_MODE_WATERMARK != 0 { + modes = append(modes, "WATERMARK") + } + if mode == api.SyncMode_SYNC_MODE_UNKNOWN { + modes = append(modes, "UNKNOWN") + } + return strings.Join(modes, "|") +} + +func has(mode, instance api.SyncMode) bool { + return mode&instance != 0 +}