From 7fbb8983c39a9f7159f71eb6579e0c231d0b41a2 Mon Sep 17 00:00:00 2001 From: Rahmat Hidayat Date: Wed, 26 Jul 2023 11:35:10 +0700 Subject: [PATCH] feat: grant dormancy check --- api/handler/v1beta1/activity.go | 2 +- api/handler/v1beta1/grpc.go | 2 +- api/handler/v1beta1/mocks/activityService.go | 10 +- .../v1beta1/mocks/providerActivityService.go | 10 +- cli/job.go | 6 + core/activity/mocks/providerService.go | 10 +- core/activity/service.go | 4 +- core/grant/mocks/providerService.go | 157 ++++++++++- core/grant/service.go | 88 +++++++ core/provider/errors.go | 1 + core/provider/mocks/activityManager.go | 10 +- core/provider/mocks/dormancyChecker.go | 139 ++++++++++ core/provider/service.go | 29 ++- domain/activity.go | 53 +++- domain/grant.go | 20 +- domain/provider.go | 6 + go.mod | 2 - go.sum | 3 - internal/server/config.go | 1 + jobs/grant_dormancy_check.go | 43 +++ jobs/handler.go | 4 +- jobs/jobs.go | 1 + jobs/mocks/grantService.go | 246 +++++++++++++++++- jobs/mocks/providerService.go | 45 +++- plugins/providers/bigquery/activity.go | 114 +++----- plugins/providers/bigquery/activity_test.go | 53 ++-- plugins/providers/bigquery/client.go | 17 ++ plugins/providers/bigquery/errors.go | 2 + .../bigquery/mocks/BigQueryClient.go | 194 +++++++++++--- .../bigquery/mocks/cloudLoggingClientI.go | 52 ++-- plugins/providers/bigquery/model.go | 52 ++++ plugins/providers/bigquery/provider.go | 171 +++++++++++- plugins/providers/bigquery/provider_test.go | 51 ++-- 33 files changed, 1361 insertions(+), 237 deletions(-) create mode 100644 core/provider/mocks/dormancyChecker.go create mode 100644 jobs/grant_dormancy_check.go diff --git a/api/handler/v1beta1/activity.go b/api/handler/v1beta1/activity.go index 6553e1cca..cd384c081 100644 --- a/api/handler/v1beta1/activity.go +++ b/api/handler/v1beta1/activity.go @@ -66,7 +66,7 @@ func (s *GRPCServer) ListActivities(ctx context.Context, req *guardianv1beta1.Li } func (s *GRPCServer) ImportActivities(ctx context.Context, req *guardianv1beta1.ImportActivitiesRequest) (*guardianv1beta1.ImportActivitiesResponse, error) { - filter := domain.ImportActivitiesFilter{ + filter := domain.ListActivitiesFilter{ ProviderID: req.GetProviderId(), ResourceIDs: req.GetResourceIds(), AccountIDs: req.GetAccountIds(), diff --git a/api/handler/v1beta1/grpc.go b/api/handler/v1beta1/grpc.go index cd0123101..3f603ea23 100644 --- a/api/handler/v1beta1/grpc.go +++ b/api/handler/v1beta1/grpc.go @@ -54,7 +54,7 @@ type resourceService interface { type activityService interface { GetOne(context.Context, string) (*domain.Activity, error) Find(context.Context, domain.ListProviderActivitiesFilter) ([]*domain.Activity, error) - Import(context.Context, domain.ImportActivitiesFilter) ([]*domain.Activity, error) + Import(context.Context, domain.ListActivitiesFilter) ([]*domain.Activity, error) } //go:generate mockery --name=providerService --exported --with-expecter diff --git a/api/handler/v1beta1/mocks/activityService.go b/api/handler/v1beta1/mocks/activityService.go index e3c95b01e..df5b6e7d4 100644 --- a/api/handler/v1beta1/mocks/activityService.go +++ b/api/handler/v1beta1/mocks/activityService.go @@ -117,11 +117,11 @@ func (_c *ActivityService_GetOne_Call) Return(_a0 *domain.Activity, _a1 error) * } // Import provides a mock function with given fields: _a0, _a1 -func (_m *ActivityService) Import(_a0 context.Context, _a1 domain.ImportActivitiesFilter) ([]*domain.Activity, error) { +func (_m *ActivityService) Import(_a0 context.Context, _a1 domain.ListActivitiesFilter) ([]*domain.Activity, error) { ret := _m.Called(_a0, _a1) var r0 []*domain.Activity - if rf, ok := ret.Get(0).(func(context.Context, domain.ImportActivitiesFilter) []*domain.Activity); ok { + if rf, ok := ret.Get(0).(func(context.Context, domain.ListActivitiesFilter) []*domain.Activity); ok { r0 = rf(_a0, _a1) } else { if ret.Get(0) != nil { @@ -130,7 +130,7 @@ func (_m *ActivityService) Import(_a0 context.Context, _a1 domain.ImportActiviti } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, domain.ImportActivitiesFilter) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, domain.ListActivitiesFilter) error); ok { r1 = rf(_a0, _a1) } else { r1 = ret.Error(1) @@ -151,9 +151,9 @@ func (_e *ActivityService_Expecter) Import(_a0 interface{}, _a1 interface{}) *Ac return &ActivityService_Import_Call{Call: _e.mock.On("Import", _a0, _a1)} } -func (_c *ActivityService_Import_Call) Run(run func(_a0 context.Context, _a1 domain.ImportActivitiesFilter)) *ActivityService_Import_Call { +func (_c *ActivityService_Import_Call) Run(run func(_a0 context.Context, _a1 domain.ListActivitiesFilter)) *ActivityService_Import_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(domain.ImportActivitiesFilter)) + run(args[0].(context.Context), args[1].(domain.ListActivitiesFilter)) }) return _c } diff --git a/api/handler/v1beta1/mocks/providerActivityService.go b/api/handler/v1beta1/mocks/providerActivityService.go index 5f4a08368..0be2e5a37 100644 --- a/api/handler/v1beta1/mocks/providerActivityService.go +++ b/api/handler/v1beta1/mocks/providerActivityService.go @@ -117,11 +117,11 @@ func (_c *ProviderActivityService_GetOne_Call) Return(_a0 *domain.Activity, _a1 } // Import provides a mock function with given fields: _a0, _a1 -func (_m *ProviderActivityService) Import(_a0 context.Context, _a1 domain.ImportActivitiesFilter) ([]*domain.Activity, error) { +func (_m *ProviderActivityService) Import(_a0 context.Context, _a1 domain.ListActivitiesFilter) ([]*domain.Activity, error) { ret := _m.Called(_a0, _a1) var r0 []*domain.Activity - if rf, ok := ret.Get(0).(func(context.Context, domain.ImportActivitiesFilter) []*domain.Activity); ok { + if rf, ok := ret.Get(0).(func(context.Context, domain.ListActivitiesFilter) []*domain.Activity); ok { r0 = rf(_a0, _a1) } else { if ret.Get(0) != nil { @@ -130,7 +130,7 @@ func (_m *ProviderActivityService) Import(_a0 context.Context, _a1 domain.Import } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, domain.ImportActivitiesFilter) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, domain.ListActivitiesFilter) error); ok { r1 = rf(_a0, _a1) } else { r1 = ret.Error(1) @@ -151,9 +151,9 @@ func (_e *ProviderActivityService_Expecter) Import(_a0 interface{}, _a1 interfac return &ProviderActivityService_Import_Call{Call: _e.mock.On("Import", _a0, _a1)} } -func (_c *ProviderActivityService_Import_Call) Run(run func(_a0 context.Context, _a1 domain.ImportActivitiesFilter)) *ProviderActivityService_Import_Call { +func (_c *ProviderActivityService_Import_Call) Run(run func(_a0 context.Context, _a1 domain.ListActivitiesFilter)) *ProviderActivityService_Import_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(domain.ImportActivitiesFilter)) + run(args[0].(context.Context), args[1].(domain.ListActivitiesFilter)) }) return _c } diff --git a/cli/job.go b/cli/job.go index cec745564..ce252e916 100644 --- a/cli/job.go +++ b/cli/job.go @@ -43,6 +43,7 @@ func runJobCmd() *cobra.Command { $ guardian job run expiring_grant_notification $ guardian job run revoke_expired_grants $ guardian job run revoke_grants_by_user_criteria + $ guardian job run grant_dormancy_check `), Args: cobra.ExactValidArgs(1), ValidArgs: []string{ @@ -50,6 +51,7 @@ func runJobCmd() *cobra.Command { string(jobs.TypeExpiringGrantNotification), string(jobs.TypeRevokeExpiredGrants), string(jobs.TypeRevokeGrantsByUserCriteria), + string(jobs.TypeGrantDormancyCheck), string(jobs.TypeRevokeExpiredAccess), string(jobs.TypeExpiringAccessNotification), @@ -112,6 +114,10 @@ func runJobCmd() *cobra.Command { handler: handler.RevokeGrantsByUserCriteria, config: config.Jobs.RevokeGrantsByUserCriteria.Config, }, + jobs.TypeGrantDormancyCheck: { + handler: handler.GrantDormancyCheck, + config: config.Jobs.GrantDormancyCheck.Config, + }, // deprecated job names jobs.TypeExpiringAccessNotification: { diff --git a/core/activity/mocks/providerService.go b/core/activity/mocks/providerService.go index cbf0209c2..0bd2c99d9 100644 --- a/core/activity/mocks/providerService.go +++ b/core/activity/mocks/providerService.go @@ -23,11 +23,11 @@ func (_m *ProviderService) EXPECT() *ProviderService_Expecter { } // ImportActivities provides a mock function with given fields: _a0, _a1 -func (_m *ProviderService) ImportActivities(_a0 context.Context, _a1 domain.ImportActivitiesFilter) ([]*domain.Activity, error) { +func (_m *ProviderService) ImportActivities(_a0 context.Context, _a1 domain.ListActivitiesFilter) ([]*domain.Activity, error) { ret := _m.Called(_a0, _a1) var r0 []*domain.Activity - if rf, ok := ret.Get(0).(func(context.Context, domain.ImportActivitiesFilter) []*domain.Activity); ok { + if rf, ok := ret.Get(0).(func(context.Context, domain.ListActivitiesFilter) []*domain.Activity); ok { r0 = rf(_a0, _a1) } else { if ret.Get(0) != nil { @@ -36,7 +36,7 @@ func (_m *ProviderService) ImportActivities(_a0 context.Context, _a1 domain.Impo } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, domain.ImportActivitiesFilter) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, domain.ListActivitiesFilter) error); ok { r1 = rf(_a0, _a1) } else { r1 = ret.Error(1) @@ -57,9 +57,9 @@ func (_e *ProviderService_Expecter) ImportActivities(_a0 interface{}, _a1 interf return &ProviderService_ImportActivities_Call{Call: _e.mock.On("ImportActivities", _a0, _a1)} } -func (_c *ProviderService_ImportActivities_Call) Run(run func(_a0 context.Context, _a1 domain.ImportActivitiesFilter)) *ProviderService_ImportActivities_Call { +func (_c *ProviderService_ImportActivities_Call) Run(run func(_a0 context.Context, _a1 domain.ListActivitiesFilter)) *ProviderService_ImportActivities_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(domain.ImportActivitiesFilter)) + run(args[0].(context.Context), args[1].(domain.ListActivitiesFilter)) }) return _c } diff --git a/core/activity/service.go b/core/activity/service.go index 542e4f199..cc30bcbdb 100644 --- a/core/activity/service.go +++ b/core/activity/service.go @@ -18,7 +18,7 @@ type repository interface { //go:generate mockery --name=providerService --exported --with-expecter type providerService interface { - ImportActivities(context.Context, domain.ImportActivitiesFilter) ([]*domain.Activity, error) + ImportActivities(context.Context, domain.ListActivitiesFilter) ([]*domain.Activity, error) } //go:generate mockery --name=auditLogger --exported --with-expecter @@ -61,7 +61,7 @@ func (s *Service) Find(ctx context.Context, filter domain.ListProviderActivities return s.repo.Find(ctx, filter) } -func (s *Service) Import(ctx context.Context, filter domain.ImportActivitiesFilter) ([]*domain.Activity, error) { +func (s *Service) Import(ctx context.Context, filter domain.ListActivitiesFilter) ([]*domain.Activity, error) { activities, err := s.providerService.ImportActivities(ctx, filter) if err != nil { return nil, err diff --git a/core/grant/mocks/providerService.go b/core/grant/mocks/providerService.go index 668b9bf60..71cda6604 100644 --- a/core/grant/mocks/providerService.go +++ b/core/grant/mocks/providerService.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.10.0. DO NOT EDIT. +// Code generated by mockery v2.20.0. DO NOT EDIT. package mocks @@ -23,11 +23,60 @@ func (_m *ProviderService) EXPECT() *ProviderService_Expecter { return &ProviderService_Expecter{mock: &_m.Mock} } +// CorrelateGrantActivities provides a mock function with given fields: _a0, _a1, _a2, _a3 +func (_m *ProviderService) CorrelateGrantActivities(_a0 context.Context, _a1 domain.Provider, _a2 []*domain.Grant, _a3 []*domain.Activity) error { + ret := _m.Called(_a0, _a1, _a2, _a3) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, domain.Provider, []*domain.Grant, []*domain.Activity) error); ok { + r0 = rf(_a0, _a1, _a2, _a3) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// ProviderService_CorrelateGrantActivities_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CorrelateGrantActivities' +type ProviderService_CorrelateGrantActivities_Call struct { + *mock.Call +} + +// CorrelateGrantActivities is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 domain.Provider +// - _a2 []*domain.Grant +// - _a3 []*domain.Activity +func (_e *ProviderService_Expecter) CorrelateGrantActivities(_a0 interface{}, _a1 interface{}, _a2 interface{}, _a3 interface{}) *ProviderService_CorrelateGrantActivities_Call { + return &ProviderService_CorrelateGrantActivities_Call{Call: _e.mock.On("CorrelateGrantActivities", _a0, _a1, _a2, _a3)} +} + +func (_c *ProviderService_CorrelateGrantActivities_Call) Run(run func(_a0 context.Context, _a1 domain.Provider, _a2 []*domain.Grant, _a3 []*domain.Activity)) *ProviderService_CorrelateGrantActivities_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(domain.Provider), args[2].([]*domain.Grant), args[3].([]*domain.Activity)) + }) + return _c +} + +func (_c *ProviderService_CorrelateGrantActivities_Call) Return(_a0 error) *ProviderService_CorrelateGrantActivities_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *ProviderService_CorrelateGrantActivities_Call) RunAndReturn(run func(context.Context, domain.Provider, []*domain.Grant, []*domain.Activity) error) *ProviderService_CorrelateGrantActivities_Call { + _c.Call.Return(run) + return _c +} + // GetByID provides a mock function with given fields: _a0, _a1 func (_m *ProviderService) GetByID(_a0 context.Context, _a1 string) (*domain.Provider, error) { ret := _m.Called(_a0, _a1) var r0 *domain.Provider + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (*domain.Provider, error)); ok { + return rf(_a0, _a1) + } if rf, ok := ret.Get(0).(func(context.Context, string) *domain.Provider); ok { r0 = rf(_a0, _a1) } else { @@ -36,7 +85,6 @@ func (_m *ProviderService) GetByID(_a0 context.Context, _a1 string) (*domain.Pro } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { r1 = rf(_a0, _a1) } else { @@ -52,8 +100,8 @@ type ProviderService_GetByID_Call struct { } // GetByID is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 string +// - _a0 context.Context +// - _a1 string func (_e *ProviderService_Expecter) GetByID(_a0 interface{}, _a1 interface{}) *ProviderService_GetByID_Call { return &ProviderService_GetByID_Call{Call: _e.mock.On("GetByID", _a0, _a1)} } @@ -70,11 +118,20 @@ func (_c *ProviderService_GetByID_Call) Return(_a0 *domain.Provider, _a1 error) return _c } +func (_c *ProviderService_GetByID_Call) RunAndReturn(run func(context.Context, string) (*domain.Provider, error)) *ProviderService_GetByID_Call { + _c.Call.Return(run) + return _c +} + // ListAccess provides a mock function with given fields: _a0, _a1, _a2 func (_m *ProviderService) ListAccess(_a0 context.Context, _a1 domain.Provider, _a2 []*domain.Resource) (domain.MapResourceAccess, error) { ret := _m.Called(_a0, _a1, _a2) var r0 domain.MapResourceAccess + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, domain.Provider, []*domain.Resource) (domain.MapResourceAccess, error)); ok { + return rf(_a0, _a1, _a2) + } if rf, ok := ret.Get(0).(func(context.Context, domain.Provider, []*domain.Resource) domain.MapResourceAccess); ok { r0 = rf(_a0, _a1, _a2) } else { @@ -83,7 +140,6 @@ func (_m *ProviderService) ListAccess(_a0 context.Context, _a1 domain.Provider, } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, domain.Provider, []*domain.Resource) error); ok { r1 = rf(_a0, _a1, _a2) } else { @@ -99,9 +155,9 @@ type ProviderService_ListAccess_Call struct { } // ListAccess is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 domain.Provider -// - _a2 []*domain.Resource +// - _a0 context.Context +// - _a1 domain.Provider +// - _a2 []*domain.Resource func (_e *ProviderService_Expecter) ListAccess(_a0 interface{}, _a1 interface{}, _a2 interface{}) *ProviderService_ListAccess_Call { return &ProviderService_ListAccess_Call{Call: _e.mock.On("ListAccess", _a0, _a1, _a2)} } @@ -118,6 +174,67 @@ func (_c *ProviderService_ListAccess_Call) Return(_a0 domain.MapResourceAccess, return _c } +func (_c *ProviderService_ListAccess_Call) RunAndReturn(run func(context.Context, domain.Provider, []*domain.Resource) (domain.MapResourceAccess, error)) *ProviderService_ListAccess_Call { + _c.Call.Return(run) + return _c +} + +// ListActivities provides a mock function with given fields: _a0, _a1, _a2 +func (_m *ProviderService) ListActivities(_a0 context.Context, _a1 domain.Provider, _a2 domain.ListActivitiesFilter) ([]*domain.Activity, error) { + ret := _m.Called(_a0, _a1, _a2) + + var r0 []*domain.Activity + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, domain.Provider, domain.ListActivitiesFilter) ([]*domain.Activity, error)); ok { + return rf(_a0, _a1, _a2) + } + if rf, ok := ret.Get(0).(func(context.Context, domain.Provider, domain.ListActivitiesFilter) []*domain.Activity); ok { + r0 = rf(_a0, _a1, _a2) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*domain.Activity) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, domain.Provider, domain.ListActivitiesFilter) error); ok { + r1 = rf(_a0, _a1, _a2) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ProviderService_ListActivities_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListActivities' +type ProviderService_ListActivities_Call struct { + *mock.Call +} + +// ListActivities is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 domain.Provider +// - _a2 domain.ListActivitiesFilter +func (_e *ProviderService_Expecter) ListActivities(_a0 interface{}, _a1 interface{}, _a2 interface{}) *ProviderService_ListActivities_Call { + return &ProviderService_ListActivities_Call{Call: _e.mock.On("ListActivities", _a0, _a1, _a2)} +} + +func (_c *ProviderService_ListActivities_Call) Run(run func(_a0 context.Context, _a1 domain.Provider, _a2 domain.ListActivitiesFilter)) *ProviderService_ListActivities_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(domain.Provider), args[2].(domain.ListActivitiesFilter)) + }) + return _c +} + +func (_c *ProviderService_ListActivities_Call) Return(_a0 []*domain.Activity, _a1 error) *ProviderService_ListActivities_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *ProviderService_ListActivities_Call) RunAndReturn(run func(context.Context, domain.Provider, domain.ListActivitiesFilter) ([]*domain.Activity, error)) *ProviderService_ListActivities_Call { + _c.Call.Return(run) + return _c +} + // RevokeAccess provides a mock function with given fields: _a0, _a1 func (_m *ProviderService) RevokeAccess(_a0 context.Context, _a1 domain.Grant) error { ret := _m.Called(_a0, _a1) @@ -138,8 +255,8 @@ type ProviderService_RevokeAccess_Call struct { } // RevokeAccess is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 domain.Grant +// - _a0 context.Context +// - _a1 domain.Grant func (_e *ProviderService_Expecter) RevokeAccess(_a0 interface{}, _a1 interface{}) *ProviderService_RevokeAccess_Call { return &ProviderService_RevokeAccess_Call{Call: _e.mock.On("RevokeAccess", _a0, _a1)} } @@ -155,3 +272,23 @@ func (_c *ProviderService_RevokeAccess_Call) Return(_a0 error) *ProviderService_ _c.Call.Return(_a0) return _c } + +func (_c *ProviderService_RevokeAccess_Call) RunAndReturn(run func(context.Context, domain.Grant) error) *ProviderService_RevokeAccess_Call { + _c.Call.Return(run) + return _c +} + +type mockConstructorTestingTNewProviderService interface { + mock.TestingT + Cleanup(func()) +} + +// NewProviderService creates a new instance of ProviderService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewProviderService(t mockConstructorTestingTNewProviderService) *ProviderService { + mock := &ProviderService{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/core/grant/service.go b/core/grant/service.go index a941df220..23e988460 100644 --- a/core/grant/service.go +++ b/core/grant/service.go @@ -31,6 +31,8 @@ type providerService interface { GetByID(context.Context, string) (*domain.Provider, error) RevokeAccess(context.Context, domain.Grant) error ListAccess(context.Context, domain.Provider, []*domain.Resource) (domain.MapResourceAccess, error) + ListActivities(context.Context, domain.Provider, domain.ListActivitiesFilter) ([]*domain.Activity, error) + CorrelateGrantActivities(context.Context, domain.Provider, []*domain.Grant, []*domain.Activity) error } //go:generate mockery --name=resourceService --exported --with-expecter @@ -480,6 +482,76 @@ func (s *Service) ImportFromProvider(ctx context.Context, criteria ImportFromPro return newAndUpdatedGrants, nil } +func (s *Service) DormancyCheck(ctx context.Context, criteria domain.DormancyCheckCriteria) error { + provider, err := s.providerService.GetByID(ctx, criteria.ProviderID) + if err != nil { + return fmt.Errorf("getting provider details: %w", err) + } + + s.logger.Info(fmt.Sprintf("getting active grants for provider %q", provider.URN)) + grants, err := s.List(ctx, domain.ListGrantsFilter{ + Statuses: []string{string(domain.GrantStatusActive)}, // TODO: evaluate later to use status_in_provider + ExpirationDateGreaterThan: time.Now().Add(criteria.RetainDuration), + ProviderTypes: []string{provider.Type}, + ProviderURNs: []string{provider.URN}, + }) + if err != nil { + return fmt.Errorf("listing active grants: %w", err) + } + grantIDs := getGrantIDs(grants) + s.logger.Info(fmt.Sprintf("found %d active grants for provider %q", len(grants), provider.URN), "grant_ids", grantIDs) + + var uniqueAccountIDs []string + uniqueAccountIDsMap := map[string]bool{} + for _, g := range grants { + if _, ok := uniqueAccountIDsMap[g.AccountID]; !ok { + uniqueAccountIDs = append(uniqueAccountIDs, g.AccountID) + uniqueAccountIDsMap[g.AccountID] = true + } + } + + activities, err := s.providerService.ListActivities(ctx, *provider, domain.ListActivitiesFilter{ + AccountIDs: uniqueAccountIDs, + TimestampGte: &criteria.TimestampeGte, + TimestampLte: &criteria.TimestampeLte, // TODO: set default value = now + }) + if err != nil { + return fmt.Errorf("listing activities: %w", err) + } + + grantsPointer := make([]*domain.Grant, len(grants)) + for i, g := range grants { + grantsPointer[i] = &g + } + if err := s.providerService.CorrelateGrantActivities(ctx, *provider, grantsPointer, activities); err != nil { + return fmt.Errorf("correlating grant activities: %w", err) + } + + var dormantGrants []*domain.Grant + var dormantGrantsIDs []string + for _, g := range grantsPointer { + if len(g.Activities) == 0 { + g.ExpirationDateReason = fmt.Sprintf("%s: %s", domain.GrantExpirationReasonDormant, criteria.RetainDuration) + newExpDate := time.Now().Add(criteria.RetainDuration) + g.ExpirationDate = &newExpDate + + dormantGrants = append(dormantGrants, g) + dormantGrantsIDs = append(dormantGrantsIDs, g.ID) + } + } + s.logger.Info(fmt.Sprintf("found %d dormant grants for provider %q", len(dormantGrants), provider.URN), "grant_ids", dormantGrantsIDs) + + if criteria.DryRun { + s.logger.Info("dry run mode, skipping updating grants expiration date") + return nil + } + + if err := s.repo.BulkUpsert(ctx, dormantGrants); err != nil { + return fmt.Errorf("updating grants expiration date: %w", err) + } + return nil +} + func getAccountSignature(accountType, accountID string) string { return fmt.Sprintf("%s:%s", accountType, accountID) } @@ -533,3 +605,19 @@ func reduceGrantsByProviderRole(rc domain.ResourceConfig, grants []*domain.Grant return } + +func getGrantIDs(grants []domain.Grant) []string { + var ids []string + for _, g := range grants { + ids = append(ids, g.ID) + } + return ids +} + +func getGrantIDsP(grants []*domain.Grant) []string { + var ids []string + for _, g := range grants { + ids = append(ids, g.ID) + } + return ids +} diff --git a/core/provider/errors.go b/core/provider/errors.go index c910d22ca..860b640db 100644 --- a/core/provider/errors.go +++ b/core/provider/errors.go @@ -20,4 +20,5 @@ var ( ErrUnimplementedMethod = errors.New("method is not yet implemented") ErrImportActivitiesMethodNotSupported = errors.New("import activities is not supported for this provider type") + ErrGetActivityMethodNotSupported = errors.New("get activity is not supported for this provider type") ) diff --git a/core/provider/mocks/activityManager.go b/core/provider/mocks/activityManager.go index 326b8008d..dd89c5184 100644 --- a/core/provider/mocks/activityManager.go +++ b/core/provider/mocks/activityManager.go @@ -23,11 +23,11 @@ func (_m *ActivityManager) EXPECT() *ActivityManager_Expecter { } // GetActivities provides a mock function with given fields: _a0, _a1, _a2 -func (_m *ActivityManager) GetActivities(_a0 context.Context, _a1 domain.Provider, _a2 domain.ImportActivitiesFilter) ([]*domain.Activity, error) { +func (_m *ActivityManager) GetActivities(_a0 context.Context, _a1 domain.Provider, _a2 domain.ListActivitiesFilter) ([]*domain.Activity, error) { ret := _m.Called(_a0, _a1, _a2) var r0 []*domain.Activity - if rf, ok := ret.Get(0).(func(context.Context, domain.Provider, domain.ImportActivitiesFilter) []*domain.Activity); ok { + if rf, ok := ret.Get(0).(func(context.Context, domain.Provider, domain.ListActivitiesFilter) []*domain.Activity); ok { r0 = rf(_a0, _a1, _a2) } else { if ret.Get(0) != nil { @@ -36,7 +36,7 @@ func (_m *ActivityManager) GetActivities(_a0 context.Context, _a1 domain.Provide } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, domain.Provider, domain.ImportActivitiesFilter) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, domain.Provider, domain.ListActivitiesFilter) error); ok { r1 = rf(_a0, _a1, _a2) } else { r1 = ret.Error(1) @@ -58,9 +58,9 @@ func (_e *ActivityManager_Expecter) GetActivities(_a0 interface{}, _a1 interface return &ActivityManager_GetActivities_Call{Call: _e.mock.On("GetActivities", _a0, _a1, _a2)} } -func (_c *ActivityManager_GetActivities_Call) Run(run func(_a0 context.Context, _a1 domain.Provider, _a2 domain.ImportActivitiesFilter)) *ActivityManager_GetActivities_Call { +func (_c *ActivityManager_GetActivities_Call) Run(run func(_a0 context.Context, _a1 domain.Provider, _a2 domain.ListActivitiesFilter)) *ActivityManager_GetActivities_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(domain.Provider), args[2].(domain.ImportActivitiesFilter)) + run(args[0].(context.Context), args[1].(domain.Provider), args[2].(domain.ListActivitiesFilter)) }) return _c } diff --git a/core/provider/mocks/dormancyChecker.go b/core/provider/mocks/dormancyChecker.go new file mode 100644 index 000000000..deaebc4ca --- /dev/null +++ b/core/provider/mocks/dormancyChecker.go @@ -0,0 +1,139 @@ +// Code generated by mockery v2.20.0. DO NOT EDIT. + +package mocks + +import ( + context "context" + + domain "github.com/goto/guardian/domain" + mock "github.com/stretchr/testify/mock" +) + +// DormancyChecker is an autogenerated mock type for the dormancyChecker type +type DormancyChecker struct { + mock.Mock +} + +type DormancyChecker_Expecter struct { + mock *mock.Mock +} + +func (_m *DormancyChecker) EXPECT() *DormancyChecker_Expecter { + return &DormancyChecker_Expecter{mock: &_m.Mock} +} + +// CorrelateGrantActivities provides a mock function with given fields: _a0, _a1, _a2, _a3 +func (_m *DormancyChecker) CorrelateGrantActivities(_a0 context.Context, _a1 domain.Provider, _a2 []*domain.Grant, _a3 []*domain.Activity) error { + ret := _m.Called(_a0, _a1, _a2, _a3) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, domain.Provider, []*domain.Grant, []*domain.Activity) error); ok { + r0 = rf(_a0, _a1, _a2, _a3) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// DormancyChecker_CorrelateGrantActivities_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CorrelateGrantActivities' +type DormancyChecker_CorrelateGrantActivities_Call struct { + *mock.Call +} + +// CorrelateGrantActivities is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 domain.Provider +// - _a2 []*domain.Grant +// - _a3 []*domain.Activity +func (_e *DormancyChecker_Expecter) CorrelateGrantActivities(_a0 interface{}, _a1 interface{}, _a2 interface{}, _a3 interface{}) *DormancyChecker_CorrelateGrantActivities_Call { + return &DormancyChecker_CorrelateGrantActivities_Call{Call: _e.mock.On("CorrelateGrantActivities", _a0, _a1, _a2, _a3)} +} + +func (_c *DormancyChecker_CorrelateGrantActivities_Call) Run(run func(_a0 context.Context, _a1 domain.Provider, _a2 []*domain.Grant, _a3 []*domain.Activity)) *DormancyChecker_CorrelateGrantActivities_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(domain.Provider), args[2].([]*domain.Grant), args[3].([]*domain.Activity)) + }) + return _c +} + +func (_c *DormancyChecker_CorrelateGrantActivities_Call) Return(_a0 error) *DormancyChecker_CorrelateGrantActivities_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *DormancyChecker_CorrelateGrantActivities_Call) RunAndReturn(run func(context.Context, domain.Provider, []*domain.Grant, []*domain.Activity) error) *DormancyChecker_CorrelateGrantActivities_Call { + _c.Call.Return(run) + return _c +} + +// ListActivities provides a mock function with given fields: _a0, _a1, _a2 +func (_m *DormancyChecker) ListActivities(_a0 context.Context, _a1 domain.Provider, _a2 domain.ListActivitiesFilter) ([]*domain.Activity, error) { + ret := _m.Called(_a0, _a1, _a2) + + var r0 []*domain.Activity + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, domain.Provider, domain.ListActivitiesFilter) ([]*domain.Activity, error)); ok { + return rf(_a0, _a1, _a2) + } + if rf, ok := ret.Get(0).(func(context.Context, domain.Provider, domain.ListActivitiesFilter) []*domain.Activity); ok { + r0 = rf(_a0, _a1, _a2) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*domain.Activity) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, domain.Provider, domain.ListActivitiesFilter) error); ok { + r1 = rf(_a0, _a1, _a2) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// DormancyChecker_ListActivities_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListActivities' +type DormancyChecker_ListActivities_Call struct { + *mock.Call +} + +// ListActivities is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 domain.Provider +// - _a2 domain.ListActivitiesFilter +func (_e *DormancyChecker_Expecter) ListActivities(_a0 interface{}, _a1 interface{}, _a2 interface{}) *DormancyChecker_ListActivities_Call { + return &DormancyChecker_ListActivities_Call{Call: _e.mock.On("ListActivities", _a0, _a1, _a2)} +} + +func (_c *DormancyChecker_ListActivities_Call) Run(run func(_a0 context.Context, _a1 domain.Provider, _a2 domain.ListActivitiesFilter)) *DormancyChecker_ListActivities_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(domain.Provider), args[2].(domain.ListActivitiesFilter)) + }) + return _c +} + +func (_c *DormancyChecker_ListActivities_Call) Return(_a0 []*domain.Activity, _a1 error) *DormancyChecker_ListActivities_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *DormancyChecker_ListActivities_Call) RunAndReturn(run func(context.Context, domain.Provider, domain.ListActivitiesFilter) ([]*domain.Activity, error)) *DormancyChecker_ListActivities_Call { + _c.Call.Return(run) + return _c +} + +type mockConstructorTestingTNewDormancyChecker interface { + mock.TestingT + Cleanup(func()) +} + +// NewDormancyChecker creates a new instance of DormancyChecker. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewDormancyChecker(t mockConstructorTestingTNewDormancyChecker) *DormancyChecker { + mock := &DormancyChecker{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/core/provider/service.go b/core/provider/service.go index 0fc5f2a99..f99f3fc14 100644 --- a/core/provider/service.go +++ b/core/provider/service.go @@ -45,7 +45,13 @@ type Client interface { //go:generate mockery --name=activityManager --exported --with-expecter type activityManager interface { - GetActivities(context.Context, domain.Provider, domain.ImportActivitiesFilter) ([]*domain.Activity, error) + GetActivities(context.Context, domain.Provider, domain.ListActivitiesFilter) ([]*domain.Activity, error) +} + +//go:generate mockery --name=dormancyChecker --exported --with-expecter +type dormancyChecker interface { + ListActivities(context.Context, domain.Provider, domain.ListActivitiesFilter) ([]*domain.Activity, error) + CorrelateGrantActivities(context.Context, domain.Provider, []*domain.Grant, []*domain.Activity) error } //go:generate mockery --name=resourceService --exported --with-expecter @@ -468,7 +474,7 @@ func (s *Service) ListAccess(ctx context.Context, p domain.Provider, resources [ return providerAccesses, nil } -func (s *Service) ImportActivities(ctx context.Context, filter domain.ImportActivitiesFilter) ([]*domain.Activity, error) { +func (s *Service) ImportActivities(ctx context.Context, filter domain.ListActivitiesFilter) ([]*domain.Activity, error) { p, err := s.GetByID(ctx, filter.ProviderID) if err != nil { return nil, fmt.Errorf("getting provider details: %w", err) @@ -498,6 +504,25 @@ func (s *Service) ImportActivities(ctx context.Context, filter domain.ImportActi return activities, nil } +func (s *Service) ListActivities(ctx context.Context, p domain.Provider, filter domain.ListActivitiesFilter) ([]*domain.Activity, error) { + c := s.getClient(p.Type) + activityClient, ok := c.(dormancyChecker) + if !ok { + return nil, fmt.Errorf("%w: %s", ErrGetActivityMethodNotSupported, p.Type) + } + + return activityClient.ListActivities(ctx, p, filter) +} + +func (s *Service) CorrelateGrantActivities(ctx context.Context, p domain.Provider, grants []*domain.Grant, activities []*domain.Activity) error { + c := s.getClient(p.Type) + activityClient, ok := c.(dormancyChecker) + if !ok { + return fmt.Errorf("%w: %s", ErrGetActivityMethodNotSupported, p.Type) + } + return activityClient.CorrelateGrantActivities(ctx, p, grants, activities) +} + func (s *Service) getResources(ctx context.Context, p *domain.Provider) ([]*domain.Resource, error) { c := s.getClient(p.Type) if c == nil { diff --git a/domain/activity.go b/domain/activity.go index 2bb15d08d..ccae991b7 100644 --- a/domain/activity.go +++ b/domain/activity.go @@ -32,17 +32,18 @@ type ListProviderActivitiesFilter struct { TimestampLte *time.Time } -type ImportActivitiesFilter struct { - ProviderID string - ResourceIDs []string - AccountIDs []string - TimestampGte *time.Time - TimestampLte *time.Time +type ListActivitiesFilter struct { + ProviderID string + ResourceIDs []string + ResourceIdentifiers []ResourceIdentifier + AccountIDs []string + TimestampGte *time.Time + TimestampLte *time.Time resources map[string]*Resource } -func (f *ImportActivitiesFilter) PopulateResources(resources map[string]*Resource) error { +func (f *ListActivitiesFilter) PopulateResources(resources map[string]*Resource) error { if f.ResourceIDs == nil { return nil } @@ -62,7 +63,7 @@ func (f *ImportActivitiesFilter) PopulateResources(resources map[string]*Resourc return nil } -func (f *ImportActivitiesFilter) GetResources() []*Resource { +func (f *ListActivitiesFilter) GetResources() []*Resource { if f.resources == nil { return nil } @@ -74,3 +75,39 @@ func (f *ImportActivitiesFilter) GetResources() []*Resource { return resources } + +// act exists in domain and used across services +// this should be associatable with grants +type act struct { + AccountID string + ProviderType string + ProviderURN string + ResourceType string + ResourceURN string + Permissions []string + Timestamp time.Time + grant *Grant +} + +// 1. fetch active grants, within a provider +// - get unique account_ids (A) +// 2. fetch acts +// - cluster by provider/project_id +// - filter by account_ids (A) +// 3. associate acts with grants +// - grant has many activities +// - if no activities, set revocation date + +// act.assignGrants([]grant) or grant.assignActs([]act)? +// + +// bigqueryAct exists in plugin +type bigqueryAct struct { + AccountID string + ResourceName string // --> resourceURN, resourceType + Permissions []string // gcp permissions + Timestamp time.Time +} + +type grants []grants +type acts []act diff --git a/domain/grant.go b/domain/grant.go index 56f05d601..6e9e6f22d 100644 --- a/domain/grant.go +++ b/domain/grant.go @@ -8,17 +8,16 @@ import ( ) type GrantStatus string +type GrantSource string const ( GrantStatusActive GrantStatus = "active" GrantStatusInactive GrantStatus = "inactive" -) - -type GrantSource string -const ( GrantSourceAppeal GrantSource = "appeal" GrantSourceImport GrantSource = "import" + + GrantExpirationReasonDormant = "grant/access hasn't been use for a while" ) type Grant struct { @@ -44,8 +43,9 @@ type Grant struct { CreatedAt time.Time `json:"created_at" yaml:"created_at"` UpdatedAt time.Time `json:"updated_at" yaml:"updated_at"` - Resource *Resource `json:"resource,omitempty" yaml:"resource,omitempty"` - Appeal *Appeal `json:"appeal,omitempty" yaml:"appeal,omitempty"` + Resource *Resource `json:"resource,omitempty" yaml:"resource,omitempty"` + Appeal *Appeal `json:"appeal,omitempty" yaml:"appeal,omitempty"` + Activities []*Activity `json:"activities,omitempty" yaml:"activities,omitempty"` } func (g Grant) PermissionsKey() string { @@ -139,3 +139,11 @@ func (ae AccessEntry) ToGrant(resource Resource) Grant { // MapResourceAccess is list of UserAccess grouped by resource urn type MapResourceAccess map[string][]AccessEntry + +type DormancyCheckCriteria struct { + ProviderID string + TimestampeGte time.Time + TimestampeLte time.Time // optional, default is time.Now() + RetainDuration time.Duration + DryRun bool +} diff --git a/domain/provider.go b/domain/provider.go index 1890800e4..eddb32686 100644 --- a/domain/provider.go +++ b/domain/provider.go @@ -75,6 +75,7 @@ type ProviderConfig struct { Appeal *AppealConfig `json:"appeal,omitempty" yaml:"appeal,omitempty" validate:"required"` Resources []*ResourceConfig `json:"resources" yaml:"resources" validate:"required"` Parameters []*ProviderParameter `json:"parameters,omitempty" yaml:"parameters,omitempty"` + Activity *ActivityConfig `json:"activity,omitempty" yaml:"activity,omitempty"` } type ProviderParameter struct { @@ -105,3 +106,8 @@ type ProviderType struct { Name string `json:"name" yaml:"name"` ResourceTypes []string `json:"resource_types" yaml:"resource_types"` } + +type ActivityConfig struct { + Source string + Options map[string]interface{} +} diff --git a/go.mod b/go.mod index 72580d360..d4ef8e0bb 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ require ( cloud.google.com/go/bigquery v1.44.0 cloud.google.com/go/datacatalog v1.8.0 cloud.google.com/go/iam v0.8.0 - cloud.google.com/go/logging v1.6.1 cloud.google.com/go/storage v1.29.0 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/MakeNowJust/heredoc v1.0.0 @@ -55,7 +54,6 @@ require ( cloud.google.com/go v0.107.0 // indirect cloud.google.com/go/compute v1.14.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/longrunning v0.3.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect diff --git a/go.sum b/go.sum index a717718bd..ce5129532 100644 --- a/go.sum +++ b/go.sum @@ -61,10 +61,7 @@ cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqCl cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/iam v0.8.0 h1:E2osAkZzxI/+8pZcxVLcDtAQx/u+hZXVryUaYQ5O0Kk= cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= -cloud.google.com/go/logging v1.6.1 h1:ZBsZK+JG+oCDT+vaxwqF2egKNRjz8soXiS6Xv79benI= -cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw= cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= -cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= diff --git a/internal/server/config.go b/internal/server/config.go index bb4e0293c..5cb167db6 100644 --- a/internal/server/config.go +++ b/internal/server/config.go @@ -27,6 +27,7 @@ type Jobs struct { RevokeExpiredGrants jobs.Job `mapstructure:"revoke_expired_grants"` ExpiringGrantNotification jobs.Job `mapstructure:"expiring_grant_notification"` RevokeGrantsByUserCriteria jobs.Job `mapstructure:"revoke_grants_by_user_criteria"` + GrantDormancyCheck jobs.Job `mapstructure:"grant_dormancy_check"` // Deprecated: use ExpiringGrantNotification instead ExpiringAccessNotification jobs.Job `mapstructure:"expiring_access_notification"` diff --git a/jobs/grant_dormancy_check.go b/jobs/grant_dormancy_check.go new file mode 100644 index 000000000..51dcace3e --- /dev/null +++ b/jobs/grant_dormancy_check.go @@ -0,0 +1,43 @@ +package jobs + +import ( + "context" + "fmt" + "time" + + "github.com/goto/guardian/domain" +) + +type GrantDormancyCheckConfig struct { + DryRun bool `mapstructure:"dry_run"` + StartDate time.Time `mapstructure:"start_date"` + EndDate time.Time `mapstructure:"end_date"` + RetainGrantFor time.Duration `mapstructure:"retain_grant_for"` +} + +func (h *handler) GrantDormancyCheck(ctx context.Context, c Config) error { + var cfg GrantDormancyCheckConfig + if err := c.Decode(&cfg); err != nil { + return fmt.Errorf("invalid config for %s job: %w", TypeRevokeGrantsByUserCriteria, err) + } + + providers, err := h.providerService.Find(ctx) + if err != nil { + return fmt.Errorf("listing providers: %w", err) + } + + for _, p := range providers { + h.logger.Info(fmt.Sprintf("checking dormancy for grants under provider: %q", p.URN)) + if err := h.grantService.DormancyCheck(ctx, domain.DormancyCheckCriteria{ + ProviderID: p.ID, + TimestampeGte: cfg.StartDate, + TimestampeLte: cfg.EndDate, + RetainDuration: cfg.RetainGrantFor, + DryRun: cfg.DryRun, + }); err != nil { + h.logger.Error(fmt.Sprintf("failed to check dormancy for provider %q", p.URN), "error", err) + } + } + + return nil +} diff --git a/jobs/handler.go b/jobs/handler.go index 4e2ac09d3..24016d33b 100644 --- a/jobs/handler.go +++ b/jobs/handler.go @@ -10,17 +10,19 @@ import ( "github.com/goto/salt/log" ) -//go:generate mockery --name=grantService --exported +//go:generate mockery --name=grantService --exported --with-expecter type grantService interface { List(context.Context, domain.ListGrantsFilter) ([]domain.Grant, error) Revoke(ctx context.Context, id, actor, reason string, opts ...grant.Option) (*domain.Grant, error) BulkRevoke(ctx context.Context, filter domain.RevokeGrantsFilter, actor, reason string) ([]*domain.Grant, error) Update(context.Context, *domain.Grant) error + DormancyCheck(context.Context, domain.DormancyCheckCriteria) error } //go:generate mockery --name=providerService --exported type providerService interface { FetchResources(context.Context) error + Find(context.Context) ([]*domain.Provider, error) } type crypto interface { diff --git a/jobs/jobs.go b/jobs/jobs.go index 25b6fd6ac..e3e014b47 100644 --- a/jobs/jobs.go +++ b/jobs/jobs.go @@ -9,6 +9,7 @@ const ( TypeExpiringGrantNotification Type = "expiring_grant_notification" TypeRevokeExpiredGrants Type = "revoke_expired_grants" TypeRevokeGrantsByUserCriteria Type = "revoke_grants_by_user_criteria" + TypeGrantDormancyCheck Type = "grant_dormancy_check" // Deprecated: use RevokeExpiredGrants instead TypeRevokeExpiredAccess Type = "revoke_expired_access" diff --git a/jobs/mocks/grantService.go b/jobs/mocks/grantService.go index 37bbef29b..c1af2e8a1 100644 --- a/jobs/mocks/grantService.go +++ b/jobs/mocks/grantService.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.10.0. DO NOT EDIT. +// Code generated by mockery v2.20.0. DO NOT EDIT. package mocks @@ -16,11 +16,123 @@ type GrantService struct { mock.Mock } +type GrantService_Expecter struct { + mock *mock.Mock +} + +func (_m *GrantService) EXPECT() *GrantService_Expecter { + return &GrantService_Expecter{mock: &_m.Mock} +} + +// BulkRevoke provides a mock function with given fields: ctx, filter, actor, reason +func (_m *GrantService) BulkRevoke(ctx context.Context, filter domain.RevokeGrantsFilter, actor string, reason string) ([]*domain.Grant, error) { + ret := _m.Called(ctx, filter, actor, reason) + + var r0 []*domain.Grant + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, domain.RevokeGrantsFilter, string, string) ([]*domain.Grant, error)); ok { + return rf(ctx, filter, actor, reason) + } + if rf, ok := ret.Get(0).(func(context.Context, domain.RevokeGrantsFilter, string, string) []*domain.Grant); ok { + r0 = rf(ctx, filter, actor, reason) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*domain.Grant) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, domain.RevokeGrantsFilter, string, string) error); ok { + r1 = rf(ctx, filter, actor, reason) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GrantService_BulkRevoke_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'BulkRevoke' +type GrantService_BulkRevoke_Call struct { + *mock.Call +} + +// BulkRevoke is a helper method to define mock.On call +// - ctx context.Context +// - filter domain.RevokeGrantsFilter +// - actor string +// - reason string +func (_e *GrantService_Expecter) BulkRevoke(ctx interface{}, filter interface{}, actor interface{}, reason interface{}) *GrantService_BulkRevoke_Call { + return &GrantService_BulkRevoke_Call{Call: _e.mock.On("BulkRevoke", ctx, filter, actor, reason)} +} + +func (_c *GrantService_BulkRevoke_Call) Run(run func(ctx context.Context, filter domain.RevokeGrantsFilter, actor string, reason string)) *GrantService_BulkRevoke_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(domain.RevokeGrantsFilter), args[2].(string), args[3].(string)) + }) + return _c +} + +func (_c *GrantService_BulkRevoke_Call) Return(_a0 []*domain.Grant, _a1 error) *GrantService_BulkRevoke_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *GrantService_BulkRevoke_Call) RunAndReturn(run func(context.Context, domain.RevokeGrantsFilter, string, string) ([]*domain.Grant, error)) *GrantService_BulkRevoke_Call { + _c.Call.Return(run) + return _c +} + +// DormancyCheck provides a mock function with given fields: _a0, _a1 +func (_m *GrantService) DormancyCheck(_a0 context.Context, _a1 domain.DormancyCheckCriteria) error { + ret := _m.Called(_a0, _a1) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, domain.DormancyCheckCriteria) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// GrantService_DormancyCheck_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DormancyCheck' +type GrantService_DormancyCheck_Call struct { + *mock.Call +} + +// DormancyCheck is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 domain.DormancyCheckCriteria +func (_e *GrantService_Expecter) DormancyCheck(_a0 interface{}, _a1 interface{}) *GrantService_DormancyCheck_Call { + return &GrantService_DormancyCheck_Call{Call: _e.mock.On("DormancyCheck", _a0, _a1)} +} + +func (_c *GrantService_DormancyCheck_Call) Run(run func(_a0 context.Context, _a1 domain.DormancyCheckCriteria)) *GrantService_DormancyCheck_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(domain.DormancyCheckCriteria)) + }) + return _c +} + +func (_c *GrantService_DormancyCheck_Call) Return(_a0 error) *GrantService_DormancyCheck_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *GrantService_DormancyCheck_Call) RunAndReturn(run func(context.Context, domain.DormancyCheckCriteria) error) *GrantService_DormancyCheck_Call { + _c.Call.Return(run) + return _c +} + // List provides a mock function with given fields: _a0, _a1 func (_m *GrantService) List(_a0 context.Context, _a1 domain.ListGrantsFilter) ([]domain.Grant, error) { ret := _m.Called(_a0, _a1) var r0 []domain.Grant + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, domain.ListGrantsFilter) ([]domain.Grant, error)); ok { + return rf(_a0, _a1) + } if rf, ok := ret.Get(0).(func(context.Context, domain.ListGrantsFilter) []domain.Grant); ok { r0 = rf(_a0, _a1) } else { @@ -29,7 +141,6 @@ func (_m *GrantService) List(_a0 context.Context, _a1 domain.ListGrantsFilter) ( } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, domain.ListGrantsFilter) error); ok { r1 = rf(_a0, _a1) } else { @@ -39,6 +150,35 @@ func (_m *GrantService) List(_a0 context.Context, _a1 domain.ListGrantsFilter) ( return r0, r1 } +// GrantService_List_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'List' +type GrantService_List_Call struct { + *mock.Call +} + +// List is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 domain.ListGrantsFilter +func (_e *GrantService_Expecter) List(_a0 interface{}, _a1 interface{}) *GrantService_List_Call { + return &GrantService_List_Call{Call: _e.mock.On("List", _a0, _a1)} +} + +func (_c *GrantService_List_Call) Run(run func(_a0 context.Context, _a1 domain.ListGrantsFilter)) *GrantService_List_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(domain.ListGrantsFilter)) + }) + return _c +} + +func (_c *GrantService_List_Call) Return(_a0 []domain.Grant, _a1 error) *GrantService_List_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *GrantService_List_Call) RunAndReturn(run func(context.Context, domain.ListGrantsFilter) ([]domain.Grant, error)) *GrantService_List_Call { + _c.Call.Return(run) + return _c +} + // Revoke provides a mock function with given fields: ctx, id, actor, reason, opts func (_m *GrantService) Revoke(ctx context.Context, id string, actor string, reason string, opts ...grant.Option) (*domain.Grant, error) { _va := make([]interface{}, len(opts)) @@ -51,6 +191,10 @@ func (_m *GrantService) Revoke(ctx context.Context, id string, actor string, rea ret := _m.Called(_ca...) var r0 *domain.Grant + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, string, ...grant.Option) (*domain.Grant, error)); ok { + return rf(ctx, id, actor, reason, opts...) + } if rf, ok := ret.Get(0).(func(context.Context, string, string, string, ...grant.Option) *domain.Grant); ok { r0 = rf(ctx, id, actor, reason, opts...) } else { @@ -59,7 +203,6 @@ func (_m *GrantService) Revoke(ctx context.Context, id string, actor string, rea } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, string, string, string, ...grant.Option) error); ok { r1 = rf(ctx, id, actor, reason, opts...) } else { @@ -68,3 +211,100 @@ func (_m *GrantService) Revoke(ctx context.Context, id string, actor string, rea return r0, r1 } + +// GrantService_Revoke_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Revoke' +type GrantService_Revoke_Call struct { + *mock.Call +} + +// Revoke is a helper method to define mock.On call +// - ctx context.Context +// - id string +// - actor string +// - reason string +// - opts ...grant.Option +func (_e *GrantService_Expecter) Revoke(ctx interface{}, id interface{}, actor interface{}, reason interface{}, opts ...interface{}) *GrantService_Revoke_Call { + return &GrantService_Revoke_Call{Call: _e.mock.On("Revoke", + append([]interface{}{ctx, id, actor, reason}, opts...)...)} +} + +func (_c *GrantService_Revoke_Call) Run(run func(ctx context.Context, id string, actor string, reason string, opts ...grant.Option)) *GrantService_Revoke_Call { + _c.Call.Run(func(args mock.Arguments) { + variadicArgs := make([]grant.Option, len(args)-4) + for i, a := range args[4:] { + if a != nil { + variadicArgs[i] = a.(grant.Option) + } + } + run(args[0].(context.Context), args[1].(string), args[2].(string), args[3].(string), variadicArgs...) + }) + return _c +} + +func (_c *GrantService_Revoke_Call) Return(_a0 *domain.Grant, _a1 error) *GrantService_Revoke_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *GrantService_Revoke_Call) RunAndReturn(run func(context.Context, string, string, string, ...grant.Option) (*domain.Grant, error)) *GrantService_Revoke_Call { + _c.Call.Return(run) + return _c +} + +// Update provides a mock function with given fields: _a0, _a1 +func (_m *GrantService) Update(_a0 context.Context, _a1 *domain.Grant) error { + ret := _m.Called(_a0, _a1) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, *domain.Grant) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// GrantService_Update_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Update' +type GrantService_Update_Call struct { + *mock.Call +} + +// Update is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *domain.Grant +func (_e *GrantService_Expecter) Update(_a0 interface{}, _a1 interface{}) *GrantService_Update_Call { + return &GrantService_Update_Call{Call: _e.mock.On("Update", _a0, _a1)} +} + +func (_c *GrantService_Update_Call) Run(run func(_a0 context.Context, _a1 *domain.Grant)) *GrantService_Update_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*domain.Grant)) + }) + return _c +} + +func (_c *GrantService_Update_Call) Return(_a0 error) *GrantService_Update_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *GrantService_Update_Call) RunAndReturn(run func(context.Context, *domain.Grant) error) *GrantService_Update_Call { + _c.Call.Return(run) + return _c +} + +type mockConstructorTestingTNewGrantService interface { + mock.TestingT + Cleanup(func()) +} + +// NewGrantService creates a new instance of GrantService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewGrantService(t mockConstructorTestingTNewGrantService) *GrantService { + mock := &GrantService{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/jobs/mocks/providerService.go b/jobs/mocks/providerService.go index 422af48ee..8ceb20461 100644 --- a/jobs/mocks/providerService.go +++ b/jobs/mocks/providerService.go @@ -1,10 +1,12 @@ -// Code generated by mockery v2.10.0. DO NOT EDIT. +// Code generated by mockery v2.20.0. DO NOT EDIT. package mocks import ( context "context" + domain "github.com/goto/guardian/domain" + mock "github.com/stretchr/testify/mock" ) @@ -26,3 +28,44 @@ func (_m *ProviderService) FetchResources(_a0 context.Context) error { return r0 } + +// Find provides a mock function with given fields: _a0 +func (_m *ProviderService) Find(_a0 context.Context) ([]*domain.Provider, error) { + ret := _m.Called(_a0) + + var r0 []*domain.Provider + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) ([]*domain.Provider, error)); ok { + return rf(_a0) + } + if rf, ok := ret.Get(0).(func(context.Context) []*domain.Provider); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*domain.Provider) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewProviderService interface { + mock.TestingT + Cleanup(func()) +} + +// NewProviderService creates a new instance of ProviderService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewProviderService(t mockConstructorTestingTNewProviderService) *ProviderService { + mock := &ProviderService{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/plugins/providers/bigquery/activity.go b/plugins/providers/bigquery/activity.go index 75a9adfe5..8dca566fc 100644 --- a/plugins/providers/bigquery/activity.go +++ b/plugins/providers/bigquery/activity.go @@ -8,10 +8,8 @@ import ( "strings" "time" - "cloud.google.com/go/logging" - "cloud.google.com/go/logging/logadmin" "github.com/goto/guardian/domain" - "google.golang.org/api/iterator" + "google.golang.org/api/logging/v2" "google.golang.org/api/option" "google.golang.org/genproto/googleapis/cloud/audit" ) @@ -51,22 +49,31 @@ func (a auditLog) GetResource(p domain.Provider) *domain.Resource { } type Activity struct { - *logging.Entry + *logging.LogEntry } func (a Activity) getAuditLog() (*auditLog, error) { - l, ok := a.Payload.(*audit.AuditLog) - if !ok { - return nil, fmt.Errorf("%w: %T", ErrInvalidActivityPayloadType, a.Payload) + payload, err := a.ProtoPayload.MarshalJSON() + if err != nil { + return nil, fmt.Errorf("marshalling proto payload: %w", err) + } + var al audit.AuditLog + if err := json.Unmarshal(payload, &al); err != nil { + return nil, fmt.Errorf("unmarshalling proto payload: %w", err) } - return &auditLog{l}, nil + return &auditLog{&al}, nil } func (a Activity) ToDomainActivity(p domain.Provider) (*domain.Activity, error) { + t, err := time.Parse(time.RFC3339Nano, a.Timestamp) + if err != nil { + return nil, fmt.Errorf("parsing timestamp: %w", err) + } + activity := &domain.Activity{ ProviderID: p.ID, - Timestamp: a.Timestamp, - ProviderActivityID: a.InsertID, + Timestamp: t, + ProviderActivityID: a.InsertId, } al, err := a.getAuditLog() @@ -86,7 +93,7 @@ func (a Activity) ToDomainActivity(p domain.Provider) (*domain.Activity, error) loggingEntryMetadata := map[string]interface{}{} loggingEntryMap := map[string]interface{}{ "payload": al, - "insert_id": a.InsertID, + "insert_id": a.InsertId, "severity": a.Severity, "resource": a.Resource, "labels": a.Labels, @@ -94,7 +101,7 @@ func (a Activity) ToDomainActivity(p domain.Provider) (*domain.Activity, error) "trace": a.Trace, "source_location": a.SourceLocation, "timestamp": a.Timestamp, - "span_id": a.SpanID, + "span_id": a.SpanId, "trace_sampled": a.TraceSampled, } if jsonData, err := json.Marshal(loggingEntryMap); err != nil { @@ -118,7 +125,8 @@ func (a Activity) ToDomainActivity(p domain.Provider) (*domain.Activity, error) } type cloudLoggingClient struct { - client *logadmin.Client + client *logging.Service + projectID string } func NewCloudLoggingClient(ctx context.Context, projectID string, credentialsJSON []byte) (*cloudLoggingClient, error) { @@ -126,13 +134,13 @@ func NewCloudLoggingClient(ctx context.Context, projectID string, credentialsJSO if credentialsJSON != nil { options = append(options, option.WithCredentialsJSON(credentialsJSON)) } - client, err := logadmin.NewClient(ctx, projectID, options...) + service, err := logging.NewService(ctx, options...) if err != nil { return nil, err } return &cloudLoggingClient{ - client: client, + client: service, }, nil } @@ -158,77 +166,33 @@ func (r bqResource) fullURN() string { return s } -type importActivitiesFilter struct { - domain.ImportActivitiesFilter - Types []string - Authorizations []string - Limit int -} - -func (f importActivitiesFilter) String() string { - criteria := []string{ - `protoPayload.serviceName="bigquery.googleapis.com"`, - `resource.type="bigquery_dataset"`, // exclude logs for bigquery jobs ("bigquery_project") - } - - if len(f.AccountIDs) > 0 { - criteria = append(criteria, - fmt.Sprintf(`protoPayload.authenticationInfo.principalEmail=("%s")`, strings.Join(f.AccountIDs, `" OR "`)), - ) - } - - resources := f.GetResources() - if len(resources) > 0 { - resourceNames := []string{} - for _, r := range resources { - resourceNames = append(resourceNames, (*bqResource)(r).fullURN()) - } - criteria = append(criteria, fmt.Sprintf(`protoPayload.resourceName:("%s")`, strings.Join(resourceNames, `" OR "`))) - // uses ":" (has/contains) operator instead of "=" (equals) operator for resource name so that the result will also - // include activities on tables under the specified dataset (e.g. "projects/xxx/datasets/yyy" will also include - // activities on "projects/xxx/datasets/yyy/tables/zzz") - } - if len(f.Types) > 0 { - criteria = append(criteria, fmt.Sprintf(`protoPayload.methodName=("%s")`, strings.Join(f.Types, `" OR "`))) - } - // TODO: authorizations - if f.TimestampGte != nil { - criteria = append(criteria, fmt.Sprintf(`timestamp>="%s"`, f.TimestampGte.Format(time.RFC3339))) - } - if f.TimestampLte != nil { - criteria = append(criteria, fmt.Sprintf(`timestamp<="%s"`, f.TimestampLte.Format(time.RFC3339))) - } - - return strings.Join(criteria, " AND ") -} - func (c *cloudLoggingClient) ListLogEntries(ctx context.Context, filter string, limit int) ([]*Activity, error) { var entries []*Activity - options := []logadmin.EntriesOption{logadmin.Filter(filter)} + req := &logging.ListLogEntriesRequest{ + Filter: filter, + ResourceNames: []string{`projects/` + c.projectID}, + } if limit > 0 { - options = append(options, logadmin.NewestFirst()) + req.OrderBy = "timestamp desc" } - it := c.client.Entries(ctx, options...) - for { - if limit > 0 && len(entries) >= limit { - break - } - e, err := it.Next() - if err != nil { - if err == iterator.Done { - break + errLimitReached := errors.New("limit reached") + if err := c.client.Entries.List(req).Pages(ctx, func(page *logging.ListLogEntriesResponse) error { + for _, e := range page.Entries { + if len(entries) >= limit { + return errLimitReached } - return nil, fmt.Errorf("iterating over cloud logging entries: %w", err) - } - if e != nil { entries = append(entries, &Activity{e}) } + return nil + }); err != nil && err != errLimitReached { + return nil, err } + return entries, nil } -func (c *cloudLoggingClient) Close() error { - return c.client.Close() +func (c *cloudLoggingClient) GetLogBucket(ctx context.Context, name string) (*logging.LogBucket, error) { + return c.client.Projects.Locations.Buckets.Get(name).Context(ctx).Do() } diff --git a/plugins/providers/bigquery/activity_test.go b/plugins/providers/bigquery/activity_test.go index ddca810e3..d7fe80711 100644 --- a/plugins/providers/bigquery/activity_test.go +++ b/plugins/providers/bigquery/activity_test.go @@ -1,14 +1,16 @@ package bigquery_test import ( + "encoding/json" "testing" "time" - "cloud.google.com/go/logging" "github.com/goto/guardian/domain" "github.com/goto/guardian/plugins/providers/bigquery" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "google.golang.org/api/googleapi" + "google.golang.org/api/logging/v2" "google.golang.org/genproto/googleapis/cloud/audit" ) @@ -19,28 +21,30 @@ func TestActivity_ToProviderActivity(t *testing.T) { Type: "bigquery", URN: "dummy-provider-urn", } - a := bigquery.Activity{ - &logging.Entry{ - InsertID: "dummy-insert-id", - Timestamp: now, - Payload: &audit.AuditLog{ - MethodName: "test-method-name", - AuthenticationInfo: &audit.AuthenticationInfo{ - PrincipalEmail: "test-principal-email", - }, - AuthorizationInfo: []*audit.AuthorizationInfo{ - { - Permission: "test-permission", - }, - }, - ResourceName: "projects/xxx/datasets/yyy/tables/zzz", + auditLog := &audit.AuditLog{ + MethodName: "test-method-name", + AuthenticationInfo: &audit.AuthenticationInfo{ + PrincipalEmail: "test-principal-email", + }, + AuthorizationInfo: []*audit.AuthorizationInfo{ + { + Permission: "test-permission", }, }, + ResourceName: "projects/xxx/datasets/yyy/tables/zzz", + } + auditLogBytes, err := json.Marshal(auditLog) + if err != nil { + require.NoError(t, err) + } + a := bigquery.Activity{ + &logging.LogEntry{ + InsertId: "dummy-insert-id", + Timestamp: now.Format(time.RFC3339Nano), + ProtoPayload: googleapi.RawMessage(auditLogBytes), + }, } - timestampStr, err := a.Timestamp.MarshalJSON() - require.NoError(t, err) - timestampStr = timestampStr[1 : len(timestampStr)-1] // trim quotes expectedMetadata := map[string]interface{}{ "logging_entry": map[string]interface{}{ "payload": map[string]interface{}{ @@ -55,23 +59,26 @@ func TestActivity_ToProviderActivity(t *testing.T) { }, "resource_name": "projects/xxx/datasets/yyy/tables/zzz", }, - "insert_id": a.InsertID, - "severity": float64(0), + "insert_id": a.InsertId, + "severity": "", "resource": nil, "labels": nil, "operation": nil, "trace": "", "source_location": nil, - "timestamp": string(timestampStr), + "timestamp": a.Timestamp, "span_id": "", "trace_sampled": false, }, } + expectedTimestamp, err := time.Parse(time.RFC3339Nano, a.Timestamp) + require.NoError(t, err) + expectedActivity := &domain.Activity{ ProviderID: dummyProvider.ID, ProviderActivityID: "dummy-insert-id", - Timestamp: a.Timestamp, + Timestamp: expectedTimestamp, Type: "test-method-name", AccountID: "test-principal-email", AccountType: "user", diff --git a/plugins/providers/bigquery/client.go b/plugins/providers/bigquery/client.go index 666be8eec..b7c929ee3 100644 --- a/plugins/providers/bigquery/client.go +++ b/plugins/providers/bigquery/client.go @@ -329,6 +329,23 @@ func (c *bigQueryClient) GetRolePermissions(ctx context.Context, role string) ([ return iamRole.IncludedPermissions, nil } +func (c *bigQueryClient) ListRolePermissions(ctx context.Context, roleIDs []string) (map[string][]string, error) { + permissions := make(map[string][]string) + + iamRoles, err := c.iamService.Roles.List().Context(ctx).Do() + if err != nil { + return nil, err + } + + for _, iamRole := range iamRoles.Roles { + if containsString(roleIDs, iamRole.Name) { + permissions[iamRole.Name] = iamRole.IncludedPermissions + } + } + + return permissions, nil +} + func (c *bigQueryClient) getGrantableRolesForTables() ([]string, error) { var resourceName string ctx := context.Background() diff --git a/plugins/providers/bigquery/errors.go b/plugins/providers/bigquery/errors.go index 4cbf3a399..264544760 100644 --- a/plugins/providers/bigquery/errors.go +++ b/plugins/providers/bigquery/errors.go @@ -27,4 +27,6 @@ var ( ErrInvalidTablePermission = errors.New("provided permission is not supported for table resource") ErrEmptyResource = errors.New("this bigquery project has no resources") ErrCannotVerifyTablePermission = errors.New("cannot verify the table permissions since this bigquery project does not have any tables") + + ErrActivityLogRetentionPeriodExceeded = errors.New("activity log retention period exceeded") ) diff --git a/plugins/providers/bigquery/mocks/BigQueryClient.go b/plugins/providers/bigquery/mocks/BigQueryClient.go index cd5f2718a..d4c33ae3a 100644 --- a/plugins/providers/bigquery/mocks/BigQueryClient.go +++ b/plugins/providers/bigquery/mocks/BigQueryClient.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.10.0. DO NOT EDIT. +// Code generated by mockery v2.20.0. DO NOT EDIT. package mocks @@ -32,6 +32,10 @@ func (_m *BigQueryClient) GetDatasets(_a0 context.Context) ([]*bigquery.Dataset, ret := _m.Called(_a0) var r0 []*bigquery.Dataset + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) ([]*bigquery.Dataset, error)); ok { + return rf(_a0) + } if rf, ok := ret.Get(0).(func(context.Context) []*bigquery.Dataset); ok { r0 = rf(_a0) } else { @@ -40,7 +44,6 @@ func (_m *BigQueryClient) GetDatasets(_a0 context.Context) ([]*bigquery.Dataset, } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context) error); ok { r1 = rf(_a0) } else { @@ -56,7 +59,7 @@ type BigQueryClient_GetDatasets_Call struct { } // GetDatasets is a helper method to define mock.On call -// - _a0 context.Context +// - _a0 context.Context func (_e *BigQueryClient_Expecter) GetDatasets(_a0 interface{}) *BigQueryClient_GetDatasets_Call { return &BigQueryClient_GetDatasets_Call{Call: _e.mock.On("GetDatasets", _a0)} } @@ -73,11 +76,20 @@ func (_c *BigQueryClient_GetDatasets_Call) Return(_a0 []*bigquery.Dataset, _a1 e return _c } +func (_c *BigQueryClient_GetDatasets_Call) RunAndReturn(run func(context.Context) ([]*bigquery.Dataset, error)) *BigQueryClient_GetDatasets_Call { + _c.Call.Return(run) + return _c +} + // GetRolePermissions provides a mock function with given fields: _a0, _a1 func (_m *BigQueryClient) GetRolePermissions(_a0 context.Context, _a1 string) ([]string, error) { ret := _m.Called(_a0, _a1) var r0 []string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) ([]string, error)); ok { + return rf(_a0, _a1) + } if rf, ok := ret.Get(0).(func(context.Context, string) []string); ok { r0 = rf(_a0, _a1) } else { @@ -86,7 +98,6 @@ func (_m *BigQueryClient) GetRolePermissions(_a0 context.Context, _a1 string) ([ } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { r1 = rf(_a0, _a1) } else { @@ -102,8 +113,8 @@ type BigQueryClient_GetRolePermissions_Call struct { } // GetRolePermissions is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 string +// - _a0 context.Context +// - _a1 string func (_e *BigQueryClient_Expecter) GetRolePermissions(_a0 interface{}, _a1 interface{}) *BigQueryClient_GetRolePermissions_Call { return &BigQueryClient_GetRolePermissions_Call{Call: _e.mock.On("GetRolePermissions", _a0, _a1)} } @@ -120,11 +131,20 @@ func (_c *BigQueryClient_GetRolePermissions_Call) Return(_a0 []string, _a1 error return _c } +func (_c *BigQueryClient_GetRolePermissions_Call) RunAndReturn(run func(context.Context, string) ([]string, error)) *BigQueryClient_GetRolePermissions_Call { + _c.Call.Return(run) + return _c +} + // GetTables provides a mock function with given fields: ctx, datasetID func (_m *BigQueryClient) GetTables(ctx context.Context, datasetID string) ([]*bigquery.Table, error) { ret := _m.Called(ctx, datasetID) var r0 []*bigquery.Table + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) ([]*bigquery.Table, error)); ok { + return rf(ctx, datasetID) + } if rf, ok := ret.Get(0).(func(context.Context, string) []*bigquery.Table); ok { r0 = rf(ctx, datasetID) } else { @@ -133,7 +153,6 @@ func (_m *BigQueryClient) GetTables(ctx context.Context, datasetID string) ([]*b } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { r1 = rf(ctx, datasetID) } else { @@ -149,8 +168,8 @@ type BigQueryClient_GetTables_Call struct { } // GetTables is a helper method to define mock.On call -// - ctx context.Context -// - datasetID string +// - ctx context.Context +// - datasetID string func (_e *BigQueryClient_Expecter) GetTables(ctx interface{}, datasetID interface{}) *BigQueryClient_GetTables_Call { return &BigQueryClient_GetTables_Call{Call: _e.mock.On("GetTables", ctx, datasetID)} } @@ -167,6 +186,11 @@ func (_c *BigQueryClient_GetTables_Call) Return(_a0 []*bigquery.Table, _a1 error return _c } +func (_c *BigQueryClient_GetTables_Call) RunAndReturn(run func(context.Context, string) ([]*bigquery.Table, error)) *BigQueryClient_GetTables_Call { + _c.Call.Return(run) + return _c +} + // GrantDatasetAccess provides a mock function with given fields: ctx, d, user, role func (_m *BigQueryClient) GrantDatasetAccess(ctx context.Context, d *bigquery.Dataset, user string, role string) error { ret := _m.Called(ctx, d, user, role) @@ -187,10 +211,10 @@ type BigQueryClient_GrantDatasetAccess_Call struct { } // GrantDatasetAccess is a helper method to define mock.On call -// - ctx context.Context -// - d *bigquery.Dataset -// - user string -// - role string +// - ctx context.Context +// - d *bigquery.Dataset +// - user string +// - role string func (_e *BigQueryClient_Expecter) GrantDatasetAccess(ctx interface{}, d interface{}, user interface{}, role interface{}) *BigQueryClient_GrantDatasetAccess_Call { return &BigQueryClient_GrantDatasetAccess_Call{Call: _e.mock.On("GrantDatasetAccess", ctx, d, user, role)} } @@ -207,6 +231,11 @@ func (_c *BigQueryClient_GrantDatasetAccess_Call) Return(_a0 error) *BigQueryCli return _c } +func (_c *BigQueryClient_GrantDatasetAccess_Call) RunAndReturn(run func(context.Context, *bigquery.Dataset, string, string) error) *BigQueryClient_GrantDatasetAccess_Call { + _c.Call.Return(run) + return _c +} + // GrantTableAccess provides a mock function with given fields: ctx, t, accountType, accountID, role func (_m *BigQueryClient) GrantTableAccess(ctx context.Context, t *bigquery.Table, accountType string, accountID string, role string) error { ret := _m.Called(ctx, t, accountType, accountID, role) @@ -227,11 +256,11 @@ type BigQueryClient_GrantTableAccess_Call struct { } // GrantTableAccess is a helper method to define mock.On call -// - ctx context.Context -// - t *bigquery.Table -// - accountType string -// - accountID string -// - role string +// - ctx context.Context +// - t *bigquery.Table +// - accountType string +// - accountID string +// - role string func (_e *BigQueryClient_Expecter) GrantTableAccess(ctx interface{}, t interface{}, accountType interface{}, accountID interface{}, role interface{}) *BigQueryClient_GrantTableAccess_Call { return &BigQueryClient_GrantTableAccess_Call{Call: _e.mock.On("GrantTableAccess", ctx, t, accountType, accountID, role)} } @@ -248,11 +277,20 @@ func (_c *BigQueryClient_GrantTableAccess_Call) Return(_a0 error) *BigQueryClien return _c } +func (_c *BigQueryClient_GrantTableAccess_Call) RunAndReturn(run func(context.Context, *bigquery.Table, string, string, string) error) *BigQueryClient_GrantTableAccess_Call { + _c.Call.Return(run) + return _c +} + // ListAccess provides a mock function with given fields: ctx, resources func (_m *BigQueryClient) ListAccess(ctx context.Context, resources []*domain.Resource) (domain.MapResourceAccess, error) { ret := _m.Called(ctx, resources) var r0 domain.MapResourceAccess + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, []*domain.Resource) (domain.MapResourceAccess, error)); ok { + return rf(ctx, resources) + } if rf, ok := ret.Get(0).(func(context.Context, []*domain.Resource) domain.MapResourceAccess); ok { r0 = rf(ctx, resources) } else { @@ -261,7 +299,6 @@ func (_m *BigQueryClient) ListAccess(ctx context.Context, resources []*domain.Re } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, []*domain.Resource) error); ok { r1 = rf(ctx, resources) } else { @@ -277,8 +314,8 @@ type BigQueryClient_ListAccess_Call struct { } // ListAccess is a helper method to define mock.On call -// - ctx context.Context -// - resources []*domain.Resource +// - ctx context.Context +// - resources []*domain.Resource func (_e *BigQueryClient_Expecter) ListAccess(ctx interface{}, resources interface{}) *BigQueryClient_ListAccess_Call { return &BigQueryClient_ListAccess_Call{Call: _e.mock.On("ListAccess", ctx, resources)} } @@ -295,18 +332,81 @@ func (_c *BigQueryClient_ListAccess_Call) Return(_a0 domain.MapResourceAccess, _ return _c } +func (_c *BigQueryClient_ListAccess_Call) RunAndReturn(run func(context.Context, []*domain.Resource) (domain.MapResourceAccess, error)) *BigQueryClient_ListAccess_Call { + _c.Call.Return(run) + return _c +} + +// ListRolePermissions provides a mock function with given fields: _a0, _a1 +func (_m *BigQueryClient) ListRolePermissions(_a0 context.Context, _a1 []string) (map[string][]string, error) { + ret := _m.Called(_a0, _a1) + + var r0 map[string][]string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, []string) (map[string][]string, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, []string) map[string][]string); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[string][]string) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, []string) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// BigQueryClient_ListRolePermissions_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListRolePermissions' +type BigQueryClient_ListRolePermissions_Call struct { + *mock.Call +} + +// ListRolePermissions is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 []string +func (_e *BigQueryClient_Expecter) ListRolePermissions(_a0 interface{}, _a1 interface{}) *BigQueryClient_ListRolePermissions_Call { + return &BigQueryClient_ListRolePermissions_Call{Call: _e.mock.On("ListRolePermissions", _a0, _a1)} +} + +func (_c *BigQueryClient_ListRolePermissions_Call) Run(run func(_a0 context.Context, _a1 []string)) *BigQueryClient_ListRolePermissions_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].([]string)) + }) + return _c +} + +func (_c *BigQueryClient_ListRolePermissions_Call) Return(_a0 map[string][]string, _a1 error) *BigQueryClient_ListRolePermissions_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *BigQueryClient_ListRolePermissions_Call) RunAndReturn(run func(context.Context, []string) (map[string][]string, error)) *BigQueryClient_ListRolePermissions_Call { + _c.Call.Return(run) + return _c +} + // ResolveDatasetRole provides a mock function with given fields: role func (_m *BigQueryClient) ResolveDatasetRole(role string) (gobigquery.AccessRole, error) { ret := _m.Called(role) var r0 gobigquery.AccessRole + var r1 error + if rf, ok := ret.Get(0).(func(string) (gobigquery.AccessRole, error)); ok { + return rf(role) + } if rf, ok := ret.Get(0).(func(string) gobigquery.AccessRole); ok { r0 = rf(role) } else { r0 = ret.Get(0).(gobigquery.AccessRole) } - var r1 error if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(role) } else { @@ -322,7 +422,7 @@ type BigQueryClient_ResolveDatasetRole_Call struct { } // ResolveDatasetRole is a helper method to define mock.On call -// - role string +// - role string func (_e *BigQueryClient_Expecter) ResolveDatasetRole(role interface{}) *BigQueryClient_ResolveDatasetRole_Call { return &BigQueryClient_ResolveDatasetRole_Call{Call: _e.mock.On("ResolveDatasetRole", role)} } @@ -339,6 +439,11 @@ func (_c *BigQueryClient_ResolveDatasetRole_Call) Return(_a0 gobigquery.AccessRo return _c } +func (_c *BigQueryClient_ResolveDatasetRole_Call) RunAndReturn(run func(string) (gobigquery.AccessRole, error)) *BigQueryClient_ResolveDatasetRole_Call { + _c.Call.Return(run) + return _c +} + // RevokeDatasetAccess provides a mock function with given fields: ctx, d, user, role func (_m *BigQueryClient) RevokeDatasetAccess(ctx context.Context, d *bigquery.Dataset, user string, role string) error { ret := _m.Called(ctx, d, user, role) @@ -359,10 +464,10 @@ type BigQueryClient_RevokeDatasetAccess_Call struct { } // RevokeDatasetAccess is a helper method to define mock.On call -// - ctx context.Context -// - d *bigquery.Dataset -// - user string -// - role string +// - ctx context.Context +// - d *bigquery.Dataset +// - user string +// - role string func (_e *BigQueryClient_Expecter) RevokeDatasetAccess(ctx interface{}, d interface{}, user interface{}, role interface{}) *BigQueryClient_RevokeDatasetAccess_Call { return &BigQueryClient_RevokeDatasetAccess_Call{Call: _e.mock.On("RevokeDatasetAccess", ctx, d, user, role)} } @@ -379,6 +484,11 @@ func (_c *BigQueryClient_RevokeDatasetAccess_Call) Return(_a0 error) *BigQueryCl return _c } +func (_c *BigQueryClient_RevokeDatasetAccess_Call) RunAndReturn(run func(context.Context, *bigquery.Dataset, string, string) error) *BigQueryClient_RevokeDatasetAccess_Call { + _c.Call.Return(run) + return _c +} + // RevokeTableAccess provides a mock function with given fields: ctx, t, accountType, accountID, role func (_m *BigQueryClient) RevokeTableAccess(ctx context.Context, t *bigquery.Table, accountType string, accountID string, role string) error { ret := _m.Called(ctx, t, accountType, accountID, role) @@ -399,11 +509,11 @@ type BigQueryClient_RevokeTableAccess_Call struct { } // RevokeTableAccess is a helper method to define mock.On call -// - ctx context.Context -// - t *bigquery.Table -// - accountType string -// - accountID string -// - role string +// - ctx context.Context +// - t *bigquery.Table +// - accountType string +// - accountID string +// - role string func (_e *BigQueryClient_Expecter) RevokeTableAccess(ctx interface{}, t interface{}, accountType interface{}, accountID interface{}, role interface{}) *BigQueryClient_RevokeTableAccess_Call { return &BigQueryClient_RevokeTableAccess_Call{Call: _e.mock.On("RevokeTableAccess", ctx, t, accountType, accountID, role)} } @@ -419,3 +529,23 @@ func (_c *BigQueryClient_RevokeTableAccess_Call) Return(_a0 error) *BigQueryClie _c.Call.Return(_a0) return _c } + +func (_c *BigQueryClient_RevokeTableAccess_Call) RunAndReturn(run func(context.Context, *bigquery.Table, string, string, string) error) *BigQueryClient_RevokeTableAccess_Call { + _c.Call.Return(run) + return _c +} + +type mockConstructorTestingTNewBigQueryClient interface { + mock.TestingT + Cleanup(func()) +} + +// NewBigQueryClient creates a new instance of BigQueryClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewBigQueryClient(t mockConstructorTestingTNewBigQueryClient) *BigQueryClient { + mock := &BigQueryClient{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/plugins/providers/bigquery/mocks/cloudLoggingClientI.go b/plugins/providers/bigquery/mocks/cloudLoggingClientI.go index 0b276c35d..17cd4549a 100644 --- a/plugins/providers/bigquery/mocks/cloudLoggingClientI.go +++ b/plugins/providers/bigquery/mocks/cloudLoggingClientI.go @@ -7,6 +7,8 @@ import ( bigquery "github.com/goto/guardian/plugins/providers/bigquery" + logging "google.golang.org/api/logging/v2" + mock "github.com/stretchr/testify/mock" ) @@ -23,43 +25,57 @@ func (_m *CloudLoggingClientI) EXPECT() *CloudLoggingClientI_Expecter { return &CloudLoggingClientI_Expecter{mock: &_m.Mock} } -// Close provides a mock function with given fields: -func (_m *CloudLoggingClientI) Close() error { - ret := _m.Called() +// GetLogBucket provides a mock function with given fields: ctx, name +func (_m *CloudLoggingClientI) GetLogBucket(ctx context.Context, name string) (*logging.LogBucket, error) { + ret := _m.Called(ctx, name) - var r0 error - if rf, ok := ret.Get(0).(func() error); ok { - r0 = rf() + var r0 *logging.LogBucket + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (*logging.LogBucket, error)); ok { + return rf(ctx, name) + } + if rf, ok := ret.Get(0).(func(context.Context, string) *logging.LogBucket); ok { + r0 = rf(ctx, name) } else { - r0 = ret.Error(0) + if ret.Get(0) != nil { + r0 = ret.Get(0).(*logging.LogBucket) + } } - return r0 + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, name) + } else { + r1 = ret.Error(1) + } + + return r0, r1 } -// CloudLoggingClientI_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close' -type CloudLoggingClientI_Close_Call struct { +// CloudLoggingClientI_GetLogBucket_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetLogBucket' +type CloudLoggingClientI_GetLogBucket_Call struct { *mock.Call } -// Close is a helper method to define mock.On call -func (_e *CloudLoggingClientI_Expecter) Close() *CloudLoggingClientI_Close_Call { - return &CloudLoggingClientI_Close_Call{Call: _e.mock.On("Close")} +// GetLogBucket is a helper method to define mock.On call +// - ctx context.Context +// - name string +func (_e *CloudLoggingClientI_Expecter) GetLogBucket(ctx interface{}, name interface{}) *CloudLoggingClientI_GetLogBucket_Call { + return &CloudLoggingClientI_GetLogBucket_Call{Call: _e.mock.On("GetLogBucket", ctx, name)} } -func (_c *CloudLoggingClientI_Close_Call) Run(run func()) *CloudLoggingClientI_Close_Call { +func (_c *CloudLoggingClientI_GetLogBucket_Call) Run(run func(ctx context.Context, name string)) *CloudLoggingClientI_GetLogBucket_Call { _c.Call.Run(func(args mock.Arguments) { - run() + run(args[0].(context.Context), args[1].(string)) }) return _c } -func (_c *CloudLoggingClientI_Close_Call) Return(_a0 error) *CloudLoggingClientI_Close_Call { - _c.Call.Return(_a0) +func (_c *CloudLoggingClientI_GetLogBucket_Call) Return(_a0 *logging.LogBucket, _a1 error) *CloudLoggingClientI_GetLogBucket_Call { + _c.Call.Return(_a0, _a1) return _c } -func (_c *CloudLoggingClientI_Close_Call) RunAndReturn(run func() error) *CloudLoggingClientI_Close_Call { +func (_c *CloudLoggingClientI_GetLogBucket_Call) RunAndReturn(run func(context.Context, string) (*logging.LogBucket, error)) *CloudLoggingClientI_GetLogBucket_Call { _c.Call.Return(run) return _c } diff --git a/plugins/providers/bigquery/model.go b/plugins/providers/bigquery/model.go index 3995bfa27..a8797cbac 100644 --- a/plugins/providers/bigquery/model.go +++ b/plugins/providers/bigquery/model.go @@ -7,6 +7,7 @@ import ( bq "cloud.google.com/go/bigquery" "github.com/goto/guardian/core/resource" "github.com/goto/guardian/domain" + "github.com/mitchellh/mapstructure" ) const ( @@ -14,8 +15,31 @@ const ( ResourceTypeDataset = "dataset" // ResourceTypeTable is the resource type name for BigQuery table ResourceTypeTable = "table" + + resourceTypeUnknown = "unknown" ) +// ResourceURN is Guardian representation of BigQuery resource's relative resource name +// Example: +// - Dataset: "project-id:dataset_name" +// - Table: "project-id:dataset_name.table_name" +type ResourceURN string + +// Type returns "dataset", "table", or "unknown" +func (urn ResourceURN) Type() string { + u := strings.Replace(string(urn), ":", " ", 1) + u = strings.Replace(u, ".", " ", 1) + u = strings.TrimSpace(u) + items := strings.Split(u, " ") + + if len(items) == 2 { + return ResourceTypeDataset + } else if len(items) == 3 { + return ResourceTypeTable + } + return resourceTypeUnknown +} + // Dataset is a reference to a BigQuery dataset type Dataset struct { ProjectID string @@ -146,3 +170,31 @@ func (r BigQueryResourceName) BigQueryResourceID() string { } return urn } + +type cloudLoggingOptions struct { + LogBucket string `json:"log_bucket" yaml:"log_bucket" mapstructure:"log_bucket"` +} + +type activityConfig struct { + *domain.ActivityConfig +} + +func (c activityConfig) Validate() error { + // TODO: + return nil +} + +func (c activityConfig) GetCloudLoggingOptions() (*cloudLoggingOptions, error) { + if c.ActivityConfig == nil || (c.ActivityConfig.Source == "cloud_logging" && c.ActivityConfig.Options == nil) { + return &cloudLoggingOptions{}, nil + } + if c.ActivityConfig.Source != "cloud_logging" { + return nil, fmt.Errorf("invalid source: %q", c.ActivityConfig.Source) + } + + result := &cloudLoggingOptions{} + if err := mapstructure.Decode(c.ActivityConfig.Options, result); err != nil { + return nil, fmt.Errorf("decoding options: %w", err) + } + return result, nil +} diff --git a/plugins/providers/bigquery/provider.go b/plugins/providers/bigquery/provider.go index 809120760..db8db9765 100644 --- a/plugins/providers/bigquery/provider.go +++ b/plugins/providers/bigquery/provider.go @@ -17,6 +17,7 @@ import ( "github.com/mitchellh/mapstructure" "github.com/patrickmn/go-cache" "golang.org/x/sync/errgroup" + "google.golang.org/api/logging/v2" "google.golang.org/api/option" ) @@ -50,12 +51,13 @@ type BigQueryClient interface { ResolveDatasetRole(role string) (bq.AccessRole, error) ListAccess(ctx context.Context, resources []*domain.Resource) (domain.MapResourceAccess, error) GetRolePermissions(context.Context, string) ([]string, error) + ListRolePermissions(context.Context, []string) (map[string][]string, error) } //go:generate mockery --name=cloudLoggingClientI --exported --with-expecter type cloudLoggingClientI interface { - Close() error ListLogEntries(context.Context, string, int) ([]*Activity, error) + GetLogBucket(ctx context.Context, name string) (*logging.LogBucket, error) } //go:generate mockery --name=encryptor --exported --with-expecter @@ -296,17 +298,48 @@ func (p *Provider) ListAccess(ctx context.Context, pc domain.ProviderConfig, res return bqClient.ListAccess(ctx, resources) } -func (p *Provider) GetActivities(ctx context.Context, pd domain.Provider, filter domain.ImportActivitiesFilter) ([]*domain.Activity, error) { +func (p *Provider) GetActivities(ctx context.Context, pd domain.Provider, filter domain.ListActivitiesFilter) ([]*domain.Activity, error) { logClient, err := p.getCloudLoggingClient(ctx, *pd.Config) if err != nil { return nil, fmt.Errorf("initializing cloud logging client: %w", err) } - activityFilter := importActivitiesFilter{ - ImportActivitiesFilter: filter, - Types: BigQueryAuditMetadataMethods, - } - entries, err := logClient.ListLogEntries(ctx, activityFilter.String(), 0) + var resourceNames []string + for _, r := range filter.GetResources() { + resourceNames = append(resourceNames, (*bqResource)(r).fullURN()) + } + filters := []string{ + `protoPayload.serviceName="bigquery.googleapis.com"`, + `resource.type="bigquery_dataset"`, // exclude logs for bigquery jobs ("bigquery_project") + } + if len(filter.AccountIDs) > 0 { + filters = append(filters, + `protoPayload.authenticationInfo.principalEmail=("`+strings.Join(filter.AccountIDs, `" OR "`)+`")`, + ) + } + resources := filter.GetResources() + if len(resources) > 0 { + filters = append(filters, + // uses ":" (has/contains) operator instead of "=" (equals) operator for resource name so that the result will also + // include activities on tables under the specified dataset (e.g. "projects/xxx/datasets/yyy" will also include + // activities on "projects/xxx/datasets/yyy/tables/zzz") + `protoPayload.resourceName:("`+strings.Join(resourceNames, `" OR "`)+`")`, + ) + } + filters = append(filters, + `protoPayload.methodName=("`+strings.Join(BigQueryAuditMetadataMethods, `" OR "`)+`")`, + ) + if filter.TimestampGte != nil { + filters = append(filters, + `timestamp>="`+filter.TimestampGte.Format(time.RFC3339)+`"`, + ) + } + if filter.TimestampLte != nil { + filters = append(filters, + `timestamp<="`+filter.TimestampLte.Format(time.RFC3339)+`"`, + ) + } + entries, err := logClient.ListLogEntries(ctx, strings.Join(filters, " AND "), 0) if err != nil { return nil, fmt.Errorf("listing log entries: %w", err) } @@ -341,6 +374,116 @@ func (p *Provider) GetActivities(ctx context.Context, pd domain.Provider, filter return activities, nil } +// ListActivities returns list of activities +func (p *Provider) ListActivities(ctx context.Context, pd domain.Provider, filter domain.ListActivitiesFilter) ([]*domain.Activity, error) { + if pd.Type != p.typeName { + return nil, ErrProviderTypeMismatch + } + logClient, err := p.getCloudLoggingClient(ctx, *pd.Config) + if err != nil { + return nil, fmt.Errorf("initializing cloud logging client: %w", err) + } + + // check time range against logging retention period + activityConfig := activityConfig{pd.Config.Activity} + clo, err := activityConfig.GetCloudLoggingOptions() + if err != nil { + return nil, fmt.Errorf("getting cloud logging options: %w", err) + } + decryptedCreds, err := ParseCredentials(pd.Config.Credentials, p.encryptor) + if err != nil { + return nil, fmt.Errorf("parsing credentials: %w", err) + } + bucketName := clo.LogBucket + if bucketName == "" { + bucketName = decryptedCreds.ResourceName + "/locations/global/buckets/_Default" + } + logBucket, err := logClient.GetLogBucket(ctx, bucketName) + if err != nil { + return nil, fmt.Errorf("getting log bucket: %w", err) + } + retentionDuration, err := time.ParseDuration(fmt.Sprintf("%dd", logBucket.RetentionDays)) + if err != nil { + return nil, fmt.Errorf("invalid bucket's retention period: %q: %w", logBucket.RetentionDays, err) + } + if filter.TimestampGte != nil && time.Since(*filter.TimestampGte) > retentionDuration { + return nil, fmt.Errorf("%w: log bucket's retention in days: %q", ErrActivityLogRetentionPeriodExceeded, logBucket.RetentionDays) + } else { + t := time.Now().Add(-retentionDuration) + filter.TimestampGte = &t + } + + // TODO: check private log viewer access is granted + + filters := []string{ + `protoPayload.serviceName="bigquery.googleapis.com"`, + `logName:"` + decryptedCreds.ResourceName + `/logs/cloudaudit.googleapis.com%2F"`, // `logName:"projects/{{project_id}}/logs/cloudaudit.googleapis.com%2F"` + `protoPayload.authorizationInfo.granted=true`, + `protoPayload.authorizationInfo.permission!=null`, + `protoPayload.authenticationInfo.principalEmail="` + strings.Join(filter.AccountIDs, `" OR "`) + `"`, + } + if filter.TimestampGte != nil && !filter.TimestampGte.IsZero() { + filters = append(filters, `timestamp>=`+filter.TimestampGte.Format(time.RFC3339)) + } + if filter.TimestampLte != nil && !filter.TimestampLte.IsZero() { + filters = append(filters, `timestamp<=`+filter.TimestampLte.Format(time.RFC3339)) + } + + entries, err := logClient.ListLogEntries(ctx, strings.Join(filters, " AND "), 1) + if err != nil { + return nil, fmt.Errorf("listing log entries: %w", err) + } + + var activities []*domain.Activity + for _, e := range entries { + a, err := e.ToDomainActivity(pd) + if err != nil { + return nil, fmt.Errorf("converting log entry to provider activity: %w", err) + } + activities = append(activities, a) + } + + return activities, nil +} + +func (p *Provider) CorrelateGrantActivities(ctx context.Context, pd domain.Provider, grants []*domain.Grant, activities []*domain.Activity) error { + creds, err := ParseCredentials(pd.Config.Credentials, p.encryptor) + if err != nil { + return fmt.Errorf("parsing credentials: %w", err) + } + + client, err := p.getBigQueryClient(*creds) + if err != nil { + return err + } + + var allRoles []string + for _, g := range grants { + allRoles = append(allRoles, g.Permissions...) // grant.Permissions is slice of gcloud roles + } + uniqueRoles := slices.UniqueStringSlice(allRoles) + permissions, err := client.ListRolePermissions(ctx, uniqueRoles) + if err != nil { + return fmt.Errorf("listing role permissions: %w", err) + } + + for _, g := range grants { + var combinedPermissions []string + for _, role := range g.Permissions { + combinedPermissions = append(combinedPermissions, permissions[role]...) + } + combinedPermissions = slices.UniqueStringSlice(combinedPermissions) + + for _, a := range activities { + if isSubset(a.Authorizations, combinedPermissions) { + g.Activities = append(g.Activities, a) + } + } + } + + return nil +} + func (p *Provider) getBigQueryClient(credentials Credentials) (BigQueryClient, error) { projectID := strings.Replace(credentials.ResourceName, "projects/", "", 1) if p.Clients[projectID] != nil { @@ -498,3 +641,17 @@ func translateDatasetRoleToBigQueryRole(role string) string { return role } } + +// isSubset checks if `subset` is a subset of `superset` +func isSubset(subset, superset []string) bool { + checkset := make(map[string]bool) + for _, element := range subset { + checkset[element] = true + } + for _, element := range superset { + if checkset[element] { + delete(checkset, element) + } + } + return len(checkset) == 0 +} diff --git a/plugins/providers/bigquery/provider_test.go b/plugins/providers/bigquery/provider_test.go index 66a77aaa1..113571d91 100644 --- a/plugins/providers/bigquery/provider_test.go +++ b/plugins/providers/bigquery/provider_test.go @@ -3,13 +3,13 @@ package bigquery_test import ( "context" "encoding/base64" + "encoding/json" "errors" "fmt" "strings" "testing" "time" - "cloud.google.com/go/logging" "github.com/google/go-cmp/cmp" "github.com/goto/guardian/core/provider" "github.com/goto/guardian/domain" @@ -19,7 +19,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" - "google.golang.org/genproto/googleapis/api/monitoredres" + "google.golang.org/api/googleapi" + "google.golang.org/api/logging/v2" "google.golang.org/genproto/googleapis/cloud/audit" ) @@ -1058,24 +1059,30 @@ func (s *BigQueryProviderTestSuite) TestGetActivities_Success() { s.Run("should map bigquery logging entries to guardian activities", func() { now := time.Now() + auditLog := &audit.AuditLog{ + ResourceName: "projects/test-project-id/datasets/test-dataset-id", + ServiceName: "bigquery.googleapis.com", + AuthenticationInfo: &audit.AuthenticationInfo{ + PrincipalEmail: "user@example.com", + }, + AuthorizationInfo: []*audit.AuthorizationInfo{ + { + Permission: "bigquery.datasets.get", + }, + }, + } + auditLogBytes, err := json.Marshal(auditLog) + if err != nil { + s.Require().NoError(err) + } + expectedBigQueryActivities := []*bigquery.Activity{ { - &logging.Entry{ - Timestamp: now, - InsertID: "test-activity-id", - Payload: &audit.AuditLog{ - ResourceName: "projects/test-project-id/datasets/test-dataset-id", - ServiceName: "bigquery.googleapis.com", - AuthenticationInfo: &audit.AuthenticationInfo{ - PrincipalEmail: "user@example.com", - }, - AuthorizationInfo: []*audit.AuthorizationInfo{ - { - Permission: "bigquery.datasets.get", - }, - }, - }, - Resource: &monitoredres.MonitoredResource{ + &logging.LogEntry{ + Timestamp: now.Format(time.RFC3339Nano), + InsertId: "test-activity-id", + ProtoPayload: googleapi.RawMessage(auditLogBytes), + Resource: &logging.MonitoredResource{ Type: "bigquery_dataset", Labels: map[string]string{ "dataset_id": "test-dataset-id", @@ -1137,7 +1144,7 @@ func (s *BigQueryProviderTestSuite) TestGetActivities_Success() { }, "type": "bigquery_dataset", }, - "severity": float64(0), + "severity": "", "source_location": nil, "span_id": "", "timestamp": now.Format(time.RFC3339Nano), @@ -1148,7 +1155,7 @@ func (s *BigQueryProviderTestSuite) TestGetActivities_Success() { }, } - actualActivities, err := s.provider.GetActivities(context.Background(), *s.validProvider, domain.ImportActivitiesFilter{}) + actualActivities, err := s.provider.GetActivities(context.Background(), *s.validProvider, domain.ListActivitiesFilter{}) s.mockCloudLoggingClient.AssertExpectations(s.T()) s.mockBigQueryClient.AssertExpectations(s.T()) @@ -1171,7 +1178,7 @@ func (s *BigQueryProviderTestSuite) TestGetActivities_Success() { }, }, } - _, err := s.provider.GetActivities(context.Background(), *invalidProvider, domain.ImportActivitiesFilter{}) + _, err := s.provider.GetActivities(context.Background(), *invalidProvider, domain.ListActivitiesFilter{}) s.mockEncryptor.AssertExpectations(s.T()) s.ErrorIs(err, expectedError) @@ -1182,7 +1189,7 @@ func (s *BigQueryProviderTestSuite) TestGetActivities_Success() { s.mockCloudLoggingClient.EXPECT(). ListLogEntries(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("string"), 0).Return(nil, expectedError).Once() - _, err := s.provider.GetActivities(context.Background(), *s.validProvider, domain.ImportActivitiesFilter{}) + _, err := s.provider.GetActivities(context.Background(), *s.validProvider, domain.ListActivitiesFilter{}) s.mockCloudLoggingClient.AssertExpectations(s.T()) s.ErrorIs(err, expectedError)