Skip to content

Commit

Permalink
Add path prefix support to hashed prefix snapshots
Browse files Browse the repository at this point in the history
Signed-off-by: Ashish Singh <[email protected]>
  • Loading branch information
ashking94 committed Sep 4, 2024
1 parent 23f5c2f commit 55102e6
Show file tree
Hide file tree
Showing 3 changed files with 19 additions and 0 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- [Remote Publication] Add remote download stats ([#15291](https://github.com/opensearch-project/OpenSearch/pull/15291)))
- Add support for comma-separated list of index names to be used with Snapshot Status API ([#15409](https://github.com/opensearch-project/OpenSearch/pull/15409))
- Add prefix support to hashed prefix & infix path types on remote store ([#15557](https://github.com/opensearch-project/OpenSearch/pull/15557))
- Add path prefix support to hashed prefix snapshots ([#15664](https://github.com/opensearch-project/OpenSearch/pull/15664))
- Optimise snapshot deletion to speed up snapshot deletion and creation ([#15568](https://github.com/opensearch-project/OpenSearch/pull/15568))
- [Remote Publication] Added checksum validation for cluster state behind a cluster setting ([#15218](https://github.com/opensearch-project/OpenSearch/pull/15218))
- Add canRemain method to TargetPoolAllocationDecider to move shards from local to remote pool for hot to warm tiering ([#15010](https://github.com/opensearch-project/OpenSearch/pull/15010))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,16 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
Setting.Property.NodeScope
);

/**
* Controls the fixed prefix for the snapshot shard blob path.
*/
public static final Setting<String> SNAPSHOT_SHARD_PATH_PREFIX = Setting.simpleString(
"cluster.snapshot.shard.path.prefix",
"",
Setting.Property.NodeScope,
Setting.Property.Final
);

protected volatile boolean supportURLRepo;

private volatile int maxShardBlobDeleteBatch;
Expand Down Expand Up @@ -429,6 +439,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp

private final NamedXContentRegistry namedXContentRegistry;

private final String snapshotShardPathPrefix;

/**
* Flag that is set to {@code true} if this instance is started with {@link #metadata} that has a higher value for
* {@link RepositoryMetadata#pendingGeneration()} than for {@link RepositoryMetadata#generation()} indicating a full cluster restart
Expand Down Expand Up @@ -480,6 +492,7 @@ protected BlobStoreRepository(
this.clusterService = clusterService;
this.recoverySettings = recoverySettings;
this.remoteStoreSettings = new RemoteStoreSettings(clusterService.getSettings(), clusterService.getClusterSettings());
this.snapshotShardPathPrefix = SNAPSHOT_SHARD_PATH_PREFIX.get(clusterService.getSettings());
}

@Override
Expand Down Expand Up @@ -2687,6 +2700,7 @@ private BlobPath shardPath(IndexId indexId, int shardId) {
SnapshotShardPathInput shardPathInput = new SnapshotShardPathInput.Builder().basePath(basePath())
.indexUUID(indexId.getId())
.shardId(String.valueOf(shardId))
.fixedPrefix(snapshotShardPathPrefix)
.build();
PathHashAlgorithm pathHashAlgorithm = pathType != PathType.FIXED ? FNV_1A_COMPOSITE_1 : null;
return pathType.path(shardPathInput, pathHashAlgorithm);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -403,6 +403,8 @@ public abstract class OpenSearchIntegTestCase extends OpenSearchTestCase {

private static Boolean segmentsPathFixedPrefix;

private static Boolean snapshotShardPathFixedPrefix;

private Path remoteStoreRepositoryPath;

private ReplicationType randomReplicationType;
Expand All @@ -414,6 +416,7 @@ public static void beforeClass() throws Exception {
prefixModeVerificationEnable = randomBoolean();
translogPathFixedPrefix = randomBoolean();
segmentsPathFixedPrefix = randomBoolean();
snapshotShardPathFixedPrefix = randomBoolean();
testClusterRule.beforeClass();
}

Expand Down Expand Up @@ -2901,6 +2904,7 @@ private static Settings buildRemoteStoreNodeAttributes(
settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), randomBoolean());
settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.getKey(), translogPathFixedPrefix ? "a" : "");
settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.getKey(), segmentsPathFixedPrefix ? "b" : "");
settings.put(BlobStoreRepository.SNAPSHOT_SHARD_PATH_PREFIX.getKey(), segmentsPathFixedPrefix ? "c" : "");
return settings.build();
}

Expand Down

0 comments on commit 55102e6

Please sign in to comment.