Bloom Client (#10697)

Implemented BloomClient that allows : 1. fetch all meta.json files that satisfy the search criteria 2. upload meta.json 3. delete meta.json 4. fetch bloom blocks in bulk 5. upload bloom blocks in bulk 6. delete bloom blocks in bulk Bloom shipper was extracted to a separate PR #10806 --------- Signed-off-by: Vladyslav Diachenko <[email protected]>
grafana · Oct 9, 2023 · b31ed05 · b31ed05
1 parent a9d7ec6
commit b31ed05
Show file tree

Hide file tree

Showing 6 changed files with 1,001 additions and 6 deletions.
diff --git a/go.mod b/go.mod
@@ -117,6 +117,7 @@ require (
 	github.com/DmitriyVTitov/size v1.5.0
 	github.com/IBM/go-sdk-core/v5 v5.13.1
 	github.com/IBM/ibm-cos-sdk-go v1.10.0
+	github.com/aws/smithy-go v1.11.1
 	github.com/axiomhq/hyperloglog v0.0.0-20230201085229-3ddf4bad03dc
 	github.com/efficientgo/core v1.0.0-rc.2
 	github.com/fsnotify/fsnotify v1.6.0
@@ -176,7 +177,6 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.1 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.11.1 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.16.1 // indirect
-	github.com/aws/smithy-go v1.11.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
 	github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe // indirect

diff --git a/pkg/storage/factory.go b/pkg/storage/factory.go
@@ -287,7 +287,7 @@ func (ns *NamedStores) populateStoreType() error {
 	return nil
 }
 
-func (ns *NamedStores) validate() error {
+func (ns *NamedStores) Validate() error {
 	for name, awsCfg := range ns.AWS {
 		if err := awsCfg.Validate(); err != nil {
 			return errors.Wrap(err, fmt.Sprintf("invalid AWS Storage config with name %s", name))
@@ -391,7 +391,7 @@ func (cfg *Config) Validate() error {
 		return errors.Wrap(err, "invalid tsdb config")
 	}
 
-	return cfg.NamedStores.validate()
+	return cfg.NamedStores.Validate()
 }
 
 // NewIndexClient creates a new index client of the desired type specified in the PeriodConfig

diff --git a/pkg/storage/factory_test.go b/pkg/storage/factory_test.go
@@ -112,7 +112,7 @@ func TestNamedStores(t *testing.T) {
 		},
 		BoltDBShipperConfig: boltdbShipperConfig,
 	}
-	require.NoError(t, cfg.NamedStores.validate())
+	require.NoError(t, cfg.NamedStores.Validate())
 
 	schemaConfig := config.SchemaConfig{
 		Configs: []config.PeriodConfig{

diff --git a/pkg/storage/store_test.go b/pkg/storage/store_test.go
@@ -1021,7 +1021,7 @@ func TestStore_indexPrefixChange(t *testing.T) {
 			},
 		},
 	}
-	require.NoError(t, cfg.NamedStores.validate())
+	require.NoError(t, cfg.NamedStores.Validate())
 
 	firstPeriodDate := parseDate("2019-01-01")
 	secondPeriodDate := parseDate("2019-01-02")
@@ -1198,7 +1198,7 @@ func TestStore_MultiPeriod(t *testing.T) {
 					},
 				},
 			}
-			require.NoError(t, cfg.NamedStores.validate())
+			require.NoError(t, cfg.NamedStores.Validate())
 
 			periodConfigV9 := config.PeriodConfig{
 				From:       config.DayTime{Time: timeToModelTime(firstStoreDate)},