Merge remote-tracking branch 'upstream/main' into prefix-char-for-rem…

…ote-store-path Signed-off-by: Ashish Singh <[email protected]>
ashking94 · Sep 2, 2024 · fac3c9b · fac3c9b
2 parents 3fc0c74 + 758c2aa
commit fac3c9b
Show file tree

Hide file tree

Showing 13 changed files with 1,471 additions and 116 deletions.
diff --git a/...ternalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteSplitIndexIT.java b/...ternalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteSplitIndexIT.java
@@ -66,6 +66,7 @@
 import org.opensearch.index.seqno.SeqNoStats;
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.indices.IndicesService;
+import org.opensearch.indices.RemoteStoreSettings;
 import org.opensearch.indices.replication.common.ReplicationType;
 import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase;
 import org.opensearch.test.OpenSearchIntegTestCase;
@@ -109,13 +110,16 @@ public void cleanUp() throws Exception {
         assertAcked(
             client().admin().indices().prepareDelete("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN).get()
         );
-        assertBusy(() -> {
-            try {
-                assertEquals(0, getFileCount(translogRepoPath));
-            } catch (IOException e) {
-                fail();
-            }
-        }, 30, TimeUnit.SECONDS);
+        // With pinned timestamp, we can have tlog files even after deletion.
+        if (RemoteStoreSettings.isPinnedTimestampsEnabled() == false) {
+            assertBusy(() -> {
+                try {
+                    assertEquals(0, getFileCount(translogRepoPath));
+                } catch (IOException e) {
+                    fail();
+                }
+            }, 30, TimeUnit.SECONDS);
+        }
         super.teardown();
     }
 

diff --git a/...src/internalClusterTest/java/org/opensearch/snapshots/CorruptedBlobStoreRepositoryIT.java b/...src/internalClusterTest/java/org/opensearch/snapshots/CorruptedBlobStoreRepositoryIT.java
@@ -40,13 +40,9 @@
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.metadata.Metadata;
 import org.opensearch.cluster.metadata.RepositoriesMetadata;
-import org.opensearch.common.blobstore.BlobPath;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.util.io.IOUtils;
 import org.opensearch.core.common.unit.ByteSizeUnit;
-import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
-import org.opensearch.index.remote.RemoteStoreEnums.PathType;
-import org.opensearch.index.remote.RemoteStorePathStrategy;
 import org.opensearch.repositories.IndexId;
 import org.opensearch.repositories.RepositoriesService;
 import org.opensearch.repositories.Repository;
@@ -62,7 +58,7 @@
 import java.util.Map;
 import java.util.stream.Stream;
 
-import static org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm.FNV_1A_COMPOSITE_1;
+import static org.opensearch.test.OpenSearchIntegTestCase.resolvePath;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertRequestBuilderThrows;
 import static org.hamcrest.Matchers.containsString;
@@ -683,16 +679,4 @@ private void assertRepositoryBlocked(Client client, String repo, String existing
             containsString("Could not read repository data because the contents of the repository do not match its expected state.")
         );
     }
-
-    private static String resolvePath(IndexId indexId, String shardId) {
-        PathType pathType = PathType.fromCode(indexId.getShardPathType());
-        RemoteStorePathStrategy.SnapshotShardPathInput shardPathInput = new RemoteStorePathStrategy.SnapshotShardPathInput.Builder()
-            .basePath(BlobPath.cleanPath())
-            .indexUUID(indexId.getId())
-            .shardId(shardId)
-            .build();
-        PathHashAlgorithm pathHashAlgorithm = pathType != PathType.FIXED ? FNV_1A_COMPOSITE_1 : null;
-        BlobPath blobPath = pathType.path(shardPathInput, pathHashAlgorithm);
-        return blobPath.buildAsString();
-    }
 }
diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/RepositoriesIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/RepositoriesIT.java
@@ -110,10 +110,17 @@ public void testRepositoryCreation() throws Exception {
         assertThat(findRepository(repositoriesResponse.repositories(), "test-repo-1"), notNullValue());
         assertThat(findRepository(repositoriesResponse.repositories(), "test-repo-2"), notNullValue());
 
+        RepositoryMetadata testRepo1Md = findRepository(repositoriesResponse.repositories(), "test-repo-1");
+
         logger.info("--> check that trying to create a repository with the same settings repeatedly does not update cluster state");
         String beforeStateUuid = clusterStateResponse.getState().stateUUID();
         createRepository("test-repo-1", "fs", Settings.builder().put("location", location));
-        assertEquals(beforeStateUuid, client.admin().cluster().prepareState().clear().get().getState().stateUUID());
+        repositoriesResponse = client.admin().cluster().prepareGetRepositories(randomFrom("_all", "*", "test-repo-*")).get();
+        RepositoryMetadata testRepo1MdAfterUpdate = findRepository(repositoriesResponse.repositories(), "test-repo-1");
+
+        if (testRepo1Md.settings().equals(testRepo1MdAfterUpdate.settings())) {
+            assertEquals(beforeStateUuid, client.admin().cluster().prepareState().clear().get().getState().stateUUID());
+        }
 
         logger.info("--> delete repository test-repo-1");
         client.admin().cluster().prepareDeleteRepository("test-repo-1").get();

diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java
@@ -49,6 +49,7 @@
 import org.opensearch.common.util.io.IOUtils;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.repositories.IndexId;
 import org.opensearch.repositories.blobstore.BlobStoreRepository;
 import org.opensearch.threadpool.ThreadPool;
 
@@ -62,6 +63,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
+import static org.opensearch.test.OpenSearchIntegTestCase.resolvePath;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -200,11 +202,9 @@ public void testExceptionOnMissingShardLevelSnapBlob() throws IOException {
         final SnapshotInfo snapshotInfo = createFullSnapshot("test-repo", "test-snap");
 
         logger.info("--> delete shard-level snap-${uuid}.dat file for one shard in this snapshot to simulate concurrent delete");
-        final String indexRepoId = getRepositoryData("test-repo").resolveIndexId(snapshotInfo.indices().get(0)).getId();
+        IndexId indexId = getRepositoryData("test-repo").resolveIndexId(snapshotInfo.indices().get(0));
         IOUtils.rm(
-            repoPath.resolve("indices")
-                .resolve(indexRepoId)
-                .resolve("0")
+            repoPath.resolve(resolvePath(indexId, "0"))
                 .resolve(BlobStoreRepository.SNAPSHOT_PREFIX + snapshotInfo.snapshotId().getUUID() + ".dat")
         );