[Snapshot V2] Fix for create snapshot build and test failures (opense…

…arch-project#15602) --------- Signed-off-by: Anshu Agarwal <[email protected]> Signed-off-by: Sachin Kale <[email protected]> Co-authored-by: Anshu Agarwal <[email protected]> Co-authored-by: Sachin Kale <[email protected]>
sachinpkale · Sep 3, 2024 · 70ba5a1 · 70ba5a1
1 parent 33be5a9
commit 70ba5a1
Show file tree

Hide file tree

Showing 8 changed files with 127 additions and 3 deletions.
diff --git a/server/src/main/java/org/opensearch/repositories/FilterRepository.java b/server/src/main/java/org/opensearch/repositories/FilterRepository.java
@@ -97,6 +97,27 @@ public void getRepositoryData(ActionListener<RepositoryData> listener) {
         in.getRepositoryData(listener);
     }
 
+    @Override
+    public void finalizeSnapshot(
+        ShardGenerations shardGenerations,
+        long repositoryStateId,
+        Metadata clusterMetadata,
+        SnapshotInfo snapshotInfo,
+        Version repositoryMetaVersion,
+        Function<ClusterState, ClusterState> stateTransformer,
+        ActionListener<RepositoryData> listener
+    ) {
+        in.finalizeSnapshot(
+            shardGenerations,
+            repositoryStateId,
+            clusterMetadata,
+            snapshotInfo,
+            repositoryMetaVersion,
+            stateTransformer,
+            listener
+        );
+    }
+
     @Override
     public void finalizeSnapshot(
         ShardGenerations shardGenerations,

diff --git a/server/src/main/java/org/opensearch/repositories/Repository.java b/server/src/main/java/org/opensearch/repositories/Repository.java
@@ -139,6 +139,30 @@ default Repository create(RepositoryMetadata metadata, Function<String, Reposito
      */
     void getRepositoryData(ActionListener<RepositoryData> listener);
 
+    /**
+     * Finalizes snapshotting process
+     * <p>
+     * This method is called on cluster-manager after all shards are snapshotted.
+     *
+     * @param shardGenerations      updated shard generations
+     * @param repositoryStateId     the unique id identifying the state of the repository when the snapshot began
+     * @param clusterMetadata       cluster metadata
+     * @param snapshotInfo     SnapshotInfo instance to write for this snapshot
+     * @param repositoryMetaVersion version of the updated repository metadata to write
+     * @param stateTransformer      a function that filters the last cluster state update that the snapshot finalization will execute and
+     *                              is used to remove any state tracked for the in-progress snapshot from the cluster state
+     * @param listener              listener to be invoked with the new {@link RepositoryData} after completing the snapshot
+     */
+    void finalizeSnapshot(
+        ShardGenerations shardGenerations,
+        long repositoryStateId,
+        Metadata clusterMetadata,
+        SnapshotInfo snapshotInfo,
+        Version repositoryMetaVersion,
+        Function<ClusterState, ClusterState> stateTransformer,
+        ActionListener<RepositoryData> listener
+    );
+
     /**
      * Finalizes snapshotting process
      * <p>

diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
@@ -2024,6 +2024,28 @@ private DeleteResult cleanUpStaleSnapshotShardPathsFile(List<String> matchingSha
         return new DeleteResult(matchingShardPaths.size(), totalBytes);
     }
 
+    @Override
+    public void finalizeSnapshot(
+        final ShardGenerations shardGenerations,
+        final long repositoryStateId,
+        final Metadata clusterMetadata,
+        SnapshotInfo snapshotInfo,
+        Version repositoryMetaVersion,
+        Function<ClusterState, ClusterState> stateTransformer,
+        final ActionListener<RepositoryData> listener
+    ) {
+        finalizeSnapshot(
+            shardGenerations,
+            repositoryStateId,
+            clusterMetadata,
+            snapshotInfo,
+            repositoryMetaVersion,
+            stateTransformer,
+            Priority.NORMAL,
+            listener
+        );
+    }
+
     @Override
     public void finalizeSnapshot(
         final ShardGenerations shardGenerations,

diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java
@@ -354,6 +354,38 @@ public SnapshotInfo(SnapshotsInProgress.Entry entry) {
         );
     }
 
+    public SnapshotInfo(
+        SnapshotId snapshotId,
+        List<String> indices,
+        List<String> dataStreams,
+        long startTime,
+        String reason,
+        long endTime,
+        int totalShards,
+        List<SnapshotShardFailure> shardFailures,
+        Boolean includeGlobalState,
+        Map<String, Object> userMetadata,
+        Boolean remoteStoreIndexShallowCopy
+    ) {
+        this(
+            snapshotId,
+            indices,
+            dataStreams,
+            snapshotState(reason, shardFailures),
+            reason,
+            Version.CURRENT,
+            startTime,
+            endTime,
+            totalShards,
+            totalShards - shardFailures.size(),
+            shardFailures,
+            includeGlobalState,
+            userMetadata,
+            remoteStoreIndexShallowCopy,
+            0
+        );
+    }
+
     public SnapshotInfo(
         SnapshotId snapshotId,
         List<String> indices,

diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java
@@ -205,6 +205,7 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
         Setting.Property.Dynamic
     );
 
+    private static final String SNAPSHOT_PINNED_TIMESTAMP_DELIMITER = "__";
     /**
      * Setting to specify the maximum number of shards that can be included in the result for the snapshot status
      * API call. Note that it does not apply to V2-shallow snapshots.
@@ -216,8 +217,6 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
         Setting.Property.NodeScope,
         Setting.Property.Dynamic
     );
-
-    private static final String SNAPSHOT_PINNED_TIMESTAMP_DELIMITER = ":";
     private volatile int maxConcurrentOperations;
 
     public SnapshotsService(

diff --git a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java
@@ -681,6 +681,19 @@ public void finalizeSnapshot(
             listener.onResponse(null);
         }
 
+        @Override
+        public void finalizeSnapshot(
+            ShardGenerations shardGenerations,
+            long repositoryStateId,
+            Metadata clusterMetadata,
+            SnapshotInfo snapshotInfo,
+            Version repositoryMetaVersion,
+            Function<ClusterState, ClusterState> stateTransformer,
+            ActionListener<RepositoryData> listener
+        ) {
+            listener.onResponse(null);
+        }
+
         @Override
         public void deleteSnapshots(
             Collection<SnapshotId> snapshotIds,

diff --git a/test/framework/src/main/java/org/opensearch/index/shard/RestoreOnlyRepository.java b/test/framework/src/main/java/org/opensearch/index/shard/RestoreOnlyRepository.java
@@ -118,6 +118,19 @@ public void getRepositoryData(ActionListener<RepositoryData> listener) {
         );
     }
 
+    @Override
+    public void finalizeSnapshot(
+        ShardGenerations shardGenerations,
+        long repositoryStateId,
+        Metadata clusterMetadata,
+        SnapshotInfo snapshotInfo,
+        Version repositoryMetaVersion,
+        Function<ClusterState, ClusterState> stateTransformer,
+        ActionListener<RepositoryData> listener
+    ) {
+        listener.onResponse(null);
+    }
+
     @Override
     public void finalizeSnapshot(
         ShardGenerations shardGenerations,

diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
@@ -2898,7 +2898,7 @@ private static Settings buildRemoteStoreNodeAttributes(
         }
         settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), randomFrom(PathType.values()));
         settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA.getKey(), randomBoolean());
-        settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), false);
+        settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), randomBoolean());
         settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.getKey(), translogPathFixedPrefix ? "a" : "");
         settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.getKey(), segmentsPathFixedPrefix ? "b" : "");
         return settings.build();