Skip to content

Commit

Permalink
Add UTs
Browse files Browse the repository at this point in the history 
Signed-off-by: Ashish Singh <[email protected]>
  • Loading branch information
@ashking94
ashking94 committed Sep 2, 2024
1 parent 8710e95 commit 702c7ab
Show file tree
Hide file tree
Showing 3 changed files with 99 additions and 11 deletions.
8 changes: 4 additions & 4 deletions server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,9 +108,9 @@ boolean requiresHashAlgorithm() {
@Override
public BlobPath generatePath(PathInput pathInput, PathHashAlgorithm hashAlgorithm) {
assert Objects.nonNull(hashAlgorithm) : "hashAlgorithm is expected to be non-null";
BlobPath path = BlobPath.cleanPath();
String fixedPrefix = pathInput.fixedPrefix();
return (Strings.isNullOrEmpty(fixedPrefix) ? path : path.add(fixedPrefix)).add(hashAlgorithm.hash(pathInput))
return BlobPath.cleanPath()
.add(Strings.isNullOrEmpty(fixedPrefix) ? hashAlgorithm.hash(pathInput) : fixedPrefix + hashAlgorithm.hash(pathInput))
.add(pathInput.basePath())
.add(pathInput.fixedSubPath());
}
Expand All @@ -124,9 +124,9 @@ boolean requiresHashAlgorithm() {
@Override
public BlobPath generatePath(PathInput pathInput, PathHashAlgorithm hashAlgorithm) {
assert Objects.nonNull(hashAlgorithm) : "hashAlgorithm is expected to be non-null";
BlobPath path = pathInput.basePath();
String fixedPrefix = pathInput.fixedPrefix();
return (Strings.isNullOrEmpty(fixedPrefix) ? path : path.add(fixedPrefix)).add(hashAlgorithm.hash(pathInput))
return pathInput.basePath()
.add(Strings.isNullOrEmpty(fixedPrefix) ? hashAlgorithm.hash(pathInput) : fixedPrefix + hashAlgorithm.hash(pathInput))
.add(pathInput.fixedSubPath());
}

Expand Down
8 changes: 4 additions & 4 deletions server/src/main/java/org/opensearch/indices/RemoteStoreSettings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ public class RemoteStoreSettings {
/**
* Controls the fixed prefix for the translog path on remote store.
*/
public static final Setting<String> CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX_CHAR = Setting.simpleString(
public static final Setting<String> CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX = Setting.simpleString(
"cluster.remote_store.translog.path.prefix",
"",
Property.NodeScope,
Expand All @@ -177,7 +177,7 @@ public class RemoteStoreSettings {
/**
* Controls the fixed prefix for the segments path on remote store.
*/
public static final Setting<String> CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX_CHAR = Setting.simpleString(
public static final Setting<String> CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX = Setting.simpleString(
"cluster.remote_store.segments.path.prefix",
"",
Property.NodeScope,
Expand Down Expand Up @@ -239,8 +239,8 @@ public RemoteStoreSettings(Settings settings, ClusterSettings clusterSettings) {
pinnedTimestampsLookbackInterval = CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_LOOKBACK_INTERVAL.get(settings);
isPinnedTimestampsEnabled = CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.get(settings);

translogPathFixedPrefix = CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX_CHAR.get(settings);
segmentsPathFixedPrefix = CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX_CHAR.get(settings);
translogPathFixedPrefix = CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.get(settings);
segmentsPathFixedPrefix = CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.get(settings);
}

public TimeValue getClusterRemoteTranslogBufferInterval() {
Expand Down
94 changes: 91 additions & 3 deletions server/src/test/java/org/opensearch/index/remote/RemoteIndexPathTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,16 @@
package org.opensearch.index.remote;

import org.opensearch.common.blobstore.BlobPath;
import org.opensearch.common.settings.ClusterSettings;
import org.opensearch.common.settings.Settings;
import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
import org.opensearch.index.remote.RemoteStoreEnums.PathType;
import org.opensearch.indices.DefaultRemoteStoreSettings;
import org.opensearch.indices.RemoteStoreSettings;
import org.opensearch.test.OpenSearchTestCase;

import java.io.IOException;
Expand Down Expand Up @@ -51,6 +54,28 @@ public void testToXContentWithSegmentRepo() throws IOException {
String expected =
"{\"version\":\"1\",\"index_uuid\":\"djjsid73he8yd7usduh\",\"shard_count\":2,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"segments\":[\"data\",\"metadata\",\"lock_files\"]},\"paths\":[\"9BmBinD5HYs/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/data/\",\"ExCNOD8_5ew/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/data/\",\"z8wtf0yr2l4/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/metadata/\",\"VheHVwFlExE/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/metadata/\",\"IgFKbsDeUpQ/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/lock_files/\",\"pA3gy_GZtns/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/lock_files/\"]}";
assertEquals(expected, xContentBuilder.toString());

// Fixed prefix
Settings settings = Settings.builder().put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.getKey(), ".").build();
ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings);
indexPath = new RemoteIndexPath(
"djjsid73he8yd7usduh",
2,
new BlobPath().add("djsd878ndjh").add("hcs87cj8"),
PathType.HASHED_PREFIX,
PathHashAlgorithm.FNV_1A_BASE64,
RemoteIndexPath.SEGMENT_PATH,
remoteStoreSettings
);
xContentBuilder = MediaTypeRegistry.contentBuilder(MediaTypeRegistry.JSON);
xContentBuilder.startObject();
xContentBuilder = indexPath.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
xContentBuilder.endObject();
expected =
"{\"version\":\"1\",\"index_uuid\":\"djjsid73he8yd7usduh\",\"shard_count\":2,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"segments\":[\"data\",\"metadata\",\"lock_files\"]},\"paths\":[\".9BmBinD5HYs/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/data/\",\".ExCNOD8_5ew/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/data/\",\".z8wtf0yr2l4/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/metadata/\",\".VheHVwFlExE/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/metadata/\",\".IgFKbsDeUpQ/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/lock_files/\",\".pA3gy_GZtns/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/lock_files/\"]}";
assertEquals(expected, xContentBuilder.toString());

}

/**
Expand All @@ -73,6 +98,27 @@ public void testToXContentForTranslogRepoOnly() throws IOException {
String expected =
"{\"version\":\"1\",\"index_uuid\":\"djjsid73he8yd7usduh\",\"shard_count\":2,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"translog\":[\"data\",\"metadata\"]},\"paths\":[\"2EaVODaKBck/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/translog/data/\",\"dTS2VqEOUNo/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/translog/data/\",\"PVNKNGonmZw/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/translog/metadata/\",\"NXmt0Y6NjA8/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/translog/metadata/\"]}";
assertEquals(expected, xContentBuilder.toString());

// Fixed prefix
Settings settings = Settings.builder().put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.getKey(), ".").build();
ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings);
indexPath = new RemoteIndexPath(
"djjsid73he8yd7usduh",
2,
new BlobPath().add("djsd878ndjh").add("hcs87cj8"),
PathType.HASHED_PREFIX,
PathHashAlgorithm.FNV_1A_BASE64,
RemoteIndexPath.TRANSLOG_PATH,
remoteStoreSettings
);
xContentBuilder = MediaTypeRegistry.contentBuilder(MediaTypeRegistry.JSON);
xContentBuilder.startObject();
xContentBuilder = indexPath.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
xContentBuilder.endObject();
expected =
"{\"version\":\"1\",\"index_uuid\":\"djjsid73he8yd7usduh\",\"shard_count\":2,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"translog\":[\"data\",\"metadata\"]},\"paths\":[\".2EaVODaKBck/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/translog/data/\",\".dTS2VqEOUNo/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/translog/data/\",\".PVNKNGonmZw/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/translog/metadata/\",\".NXmt0Y6NjA8/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/translog/metadata/\"]}";
assertEquals(expected, xContentBuilder.toString());
}

/**
Expand All @@ -98,9 +144,42 @@ public void testToXContentForBothRepos() throws IOException {
String expected =
"{\"version\":\"1\",\"index_uuid\":\"csbdqiu8a7sdnjdks\",\"shard_count\":3,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"translog\":[\"data\",\"metadata\"],\"segments\":[\"data\",\"metadata\",\"lock_files\"]},\"paths\":[\"Cjo0F6kNjYk/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/data/\",\"kpayyhxct1I/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/data/\",\"p2RlgnHeIgc/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/data/\",\"gkPIurBtB1w/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/metadata/\",\"Y4YhlbxAB1c/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/metadata/\",\"HYc8fyVPouI/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/metadata/\",\"igzyZCz1ysI/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/lock_files/\",\"uEluEiYmptk/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/lock_files/\",\"TfAD8f06_7A/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/lock_files/\",\"QqKEpasbEGs/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/translog/data/\",\"sNyoimoe1Bw/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/translog/data/\",\"d4YQtONfq50/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/translog/data/\",\"zLr4UXjK8T4/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/translog/metadata/\",\"_s8i7ZmlXGE/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/translog/metadata/\",\"tvtD3-k5ISg/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/translog/metadata/\"]}";
assertEquals(expected, xContentBuilder.toString());

// Fixed prefix
Settings settings = Settings.builder()
.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.getKey(), ".")
.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.getKey(), ".")
.build();
ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings);
indexPath = new RemoteIndexPath(
"csbdqiu8a7sdnjdks",
3,
new BlobPath().add("nxf9yv0").add("c3ejoi"),
PathType.HASHED_PREFIX,
PathHashAlgorithm.FNV_1A_BASE64,
pathCreationMap,
remoteStoreSettings
);
xContentBuilder = MediaTypeRegistry.contentBuilder(MediaTypeRegistry.JSON);
xContentBuilder.startObject();
xContentBuilder = indexPath.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
xContentBuilder.endObject();
expected =
"{\"version\":\"1\",\"index_uuid\":\"csbdqiu8a7sdnjdks\",\"shard_count\":3,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"translog\":[\"data\",\"metadata\"],\"segments\":[\"data\",\"metadata\",\"lock_files\"]},\"paths\":[\".Cjo0F6kNjYk/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/data/\",\".kpayyhxct1I/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/data/\",\".p2RlgnHeIgc/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/data/\",\".gkPIurBtB1w/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/metadata/\",\".Y4YhlbxAB1c/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/metadata/\",\".HYc8fyVPouI/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/metadata/\",\".igzyZCz1ysI/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/lock_files/\",\".uEluEiYmptk/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/lock_files/\",\".TfAD8f06_7A/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/lock_files/\",\".QqKEpasbEGs/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/translog/data/\",\".sNyoimoe1Bw/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/translog/data/\",\".d4YQtONfq50/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/translog/data/\",\".zLr4UXjK8T4/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/translog/metadata/\",\"._s8i7ZmlXGE/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/translog/metadata/\",\".tvtD3-k5ISg/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/translog/metadata/\"]}";
assertEquals(expected, xContentBuilder.toString());
}

public void testRemoteIndexPathWithInvalidPathCreationMap() throws IOException {
public void testRemoteIndexPathWithInvalidPathCreationMap() {
Settings.Builder builder = Settings.builder();
if (randomBoolean()) {
builder.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.getKey(), ".");
}
if (randomBoolean()) {
builder.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.getKey(), ".");
}
ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(builder.build(), clusterSettings);
IllegalArgumentException ex = assertThrows(
IllegalArgumentException.class,
() -> new RemoteIndexPath(
Expand All @@ -110,7 +189,7 @@ public void testRemoteIndexPathWithInvalidPathCreationMap() throws IOException {
PathType.HASHED_PREFIX,
PathHashAlgorithm.FNV_1A_BASE64,
new HashMap<>(),
DefaultRemoteStoreSettings.INSTANCE
remoteStoreSettings
)
);
assertEquals(
Expand All @@ -129,6 +208,15 @@ public void testFromXContent() {
}

public void testInvalidPathCreationMap() {
Settings.Builder builder = Settings.builder();
if (randomBoolean()) {
builder.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.getKey(), ".");
}
if (randomBoolean()) {
builder.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.getKey(), ".");
}
ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(builder.build(), clusterSettings);
IllegalArgumentException ex = assertThrows(
IllegalArgumentException.class,
() -> new RemoteIndexPath(
Expand All @@ -138,7 +226,7 @@ public void testInvalidPathCreationMap() {
PathType.HASHED_PREFIX,
PathHashAlgorithm.FNV_1A_BASE64,
Map.of(TRANSLOG, List.of(LOCK_FILES)),
DefaultRemoteStoreSettings.INSTANCE
remoteStoreSettings
)
);
assertEquals("pathCreationMap={TRANSLOG=[LOCK_FILES]} is having illegal combination of category and type", ex.getMessage());
Expand Down

0 comments on commit 702c7ab

Please sign in to comment.