Skip to content

Commit

Permalink
Add separate test class for remote segment directory with pinned time…
Browse files Browse the repository at this point in the history 
…stamps

Signed-off-by: Sachin Kale <[email protected]>
  • Loading branch information
Sachin Kale committed Aug 26, 2024
1 parent ff823f0 commit 782438e
Show file tree
Hide file tree
Showing 4 changed files with 300 additions and 114 deletions.
4 changes: 2 additions & 2 deletions server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,9 +219,9 @@ private ActionListener<Void> getListenerForWriteCallResponse(

private PinnedTimestamps readExistingPinnedTimestamps(String blobFilename, RemotePinnedTimestamps remotePinnedTimestamps) {
remotePinnedTimestamps.setBlobFileName(blobFilename);
remotePinnedTimestamps.setFullBlobName(pinnedTimestampsBlobStore.getBlobPathForUpload(remotePinnedTimestamps));
remotePinnedTimestamps.setFullBlobName(pinnedTimestampsBlobStore().getBlobPathForUpload(remotePinnedTimestamps));
try {
return pinnedTimestampsBlobStore.read(remotePinnedTimestamps);
return pinnedTimestampsBlobStore().read(remotePinnedTimestamps);
} catch (IOException e) {
throw new RuntimeException("Failed to read existing pinned timestamps", e);
}
Expand Down
45 changes: 6 additions & 39 deletions server/src/test/java/org/opensearch/index/store/BaseRemoteSegmentStoreDirectoryTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,52 +43,19 @@ public class BaseRemoteSegmentStoreDirectoryTests extends IndexShardTestCase {
protected SegmentInfos segmentInfos;
protected ThreadPool threadPool;

protected final String metadataFilename = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(
12,
23,
34,
1,
1,
"node-1",
System.currentTimeMillis() - 300000
);
protected String metadataFilename = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(12, 23, 34, 1, 1, "node-1");

protected final String metadataFilenameDup = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(
protected String metadataFilenameDup = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(
12,
23,
34,
2,
1,
"node-2",
System.currentTimeMillis() - 400000
);
protected final String metadataFilename2 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(
12,
13,
34,
1,
1,
"node-1",
System.currentTimeMillis() - 500000
);
protected final String metadataFilename3 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(
10,
38,
34,
1,
1,
"node-1",
System.currentTimeMillis() - 600000
);
protected final String metadataFilename4 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(
10,
36,
34,
1,
1,
"node-1",
System.currentTimeMillis() - 700000
"node-2"
);
protected String metadataFilename2 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(12, 13, 34, 1, 1, "node-1");
protected String metadataFilename3 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(10, 38, 34, 1, 1, "node-1");
protected String metadataFilename4 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(10, 36, 34, 1, 1, "node-1");

public void setupRemoteSegmentStoreDirectory() throws IOException {
remoteDataDirectory = mock(RemoteDirectory.class);
Expand Down
73 changes: 0 additions & 73 deletions server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,6 @@
import org.opensearch.common.io.VersionedCodecStreamWrapper;
import org.opensearch.common.io.stream.BytesStreamOutput;
import org.opensearch.common.lucene.store.ByteArrayIndexInput;
import org.opensearch.common.settings.ClusterSettings;
import org.opensearch.common.settings.Settings;
import org.opensearch.core.action.ActionListener;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.index.Index;
Expand All @@ -40,7 +38,6 @@
import org.opensearch.index.remote.RemoteStoreUtils;
import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata;
import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadataHandler;
import org.opensearch.indices.RemoteStoreSettings;
import org.opensearch.test.MockLogAppender;
import org.opensearch.test.junit.annotations.TestLogging;
import org.opensearch.threadpool.ThreadPool;
Expand All @@ -64,7 +61,6 @@

import static org.opensearch.index.store.RemoteSegmentStoreDirectory.METADATA_FILES_TO_FETCH;
import static org.opensearch.index.store.RemoteSegmentStoreDirectory.MetadataFilenameUtils.SEPARATOR;
import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED;
import static org.opensearch.test.RemoteStoreTestUtils.createMetadataFileBytes;
import static org.opensearch.test.RemoteStoreTestUtils.getDummyMetadata;
import static org.hamcrest.CoreMatchers.is;
Expand Down Expand Up @@ -1145,75 +1141,6 @@ public void testMetadataFileNameOrder() {
assertEquals(14, count);
}

private void setupRemotePinnedTimestampFeature(boolean enabled) {
RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(
Settings.builder().put(CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), enabled).build(),
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
);
}

public void testInitializeToSpecificTimestampNoMetadataFiles() throws IOException {
setupRemotePinnedTimestampFeature(true);
when(
remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder(
RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX,
Integer.MAX_VALUE
)
).thenReturn(new ArrayList<>());
assertNull(remoteSegmentStoreDirectory.initializeToSpecificTimestamp(1234L));
setupRemotePinnedTimestampFeature(false);
}

public void testInitializeToSpecificTimestampNoMdMatchingTimestamp() throws IOException {
setupRemotePinnedTimestampFeature(true);
String metadataPrefix = "metadata__1__2__3__4__5__";
List<String> metadataFiles = new ArrayList<>();
metadataFiles.add(metadataPrefix + RemoteStoreUtils.invertLong(2000));
metadataFiles.add(metadataPrefix + RemoteStoreUtils.invertLong(3000));
metadataFiles.add(metadataPrefix + RemoteStoreUtils.invertLong(4000));

when(
remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder(
RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX,
Integer.MAX_VALUE
)
).thenReturn(metadataFiles);
assertNull(remoteSegmentStoreDirectory.initializeToSpecificTimestamp(1234L));
setupRemotePinnedTimestampFeature(false);
}

public void testInitializeToSpecificTimestampMatchingMdFile() throws IOException {
setupRemotePinnedTimestampFeature(true);
String metadataPrefix = "metadata__1__2__3__4__5__";
List<String> metadataFiles = new ArrayList<>();
metadataFiles.add(metadataPrefix + RemoteStoreUtils.invertLong(1000));
metadataFiles.add(metadataPrefix + RemoteStoreUtils.invertLong(2000));
metadataFiles.add(metadataPrefix + RemoteStoreUtils.invertLong(3000));

Map<String, String> metadata = new HashMap<>();
metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234::512::" + Version.LATEST.major);
metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345::1024::" + Version.LATEST.major);

when(
remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder(
RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX,
Integer.MAX_VALUE
)
).thenReturn(metadataFiles);
when(remoteMetadataDirectory.getBlobStream(metadataPrefix + RemoteStoreUtils.invertLong(1000))).thenReturn(
createMetadataFileBytes(metadata, indexShard.getLatestReplicationCheckpoint(), segmentInfos)
);

RemoteSegmentMetadata remoteSegmentMetadata = remoteSegmentStoreDirectory.initializeToSpecificTimestamp(1234L);
assertNotNull(remoteSegmentMetadata);
Map<String, RemoteSegmentStoreDirectory.UploadedSegmentMetadata> uploadedSegments = remoteSegmentStoreDirectory
.getSegmentsUploadedToRemoteStore();
assertEquals(2, uploadedSegments.size());
assertTrue(uploadedSegments.containsKey("_0.cfe"));
assertTrue(uploadedSegments.containsKey("_0.cfs"));
setupRemotePinnedTimestampFeature(false);
}

private static class WrapperIndexOutput extends IndexOutput {
public IndexOutput indexOutput;

Expand Down
Loading

0 comments on commit 782438e

Please sign in to comment.