From f6d6aa61e5039e4c6143cc25a71c3e448572dd33 Mon Sep 17 00:00:00 2001
From: Sachin Kale <sachinpkale@gmail.com>
Date: Tue, 4 Mar 2025 10:44:36 +0530
Subject: [PATCH] Fix flaky test RemoteIndexRecoveryIT.testRerouteRecovery
 (#17228)

Signed-off-by: Sachin Kale <sachinpkale@gmail.com>
---
 .../indices/recovery/IndexRecoveryIT.java     |  4 ++--
 .../remotestore/RemoteIndexRecoveryIT.java    | 24 +++++++++++++++++++
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java
index d30806b4325ac..9d893cb6f33c7 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java
@@ -250,7 +250,7 @@ private void assertOnGoingRecoveryState(
         assertThat(state.getStage(), not(equalTo(Stage.DONE)));
     }
 
-    private void slowDownRecovery(ByteSizeValue shardSize) {
+    public void slowDownRecovery(ByteSizeValue shardSize) {
         long chunkSize = Math.max(1, shardSize.getBytes() / 10);
         assertTrue(
             client().admin()
@@ -528,7 +528,7 @@ public void testRerouteRecovery() throws Exception {
             assertThat(indicesService.indexServiceSafe(index).getShard(0).recoveryStats().currentAsSource(), equalTo(1));
             indicesService = internalCluster().getInstance(IndicesService.class, nodeB);
             assertThat(indicesService.indexServiceSafe(index).getShard(0).recoveryStats().currentAsTarget(), equalTo(1));
-        }, TimeValue.timeValueSeconds(10), TimeValue.timeValueMillis(500));
+        }, TimeValue.timeValueSeconds(60), TimeValue.timeValueMillis(500));
 
         logger.info("--> request recoveries");
         RecoveryResponse response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java
index 6de61cf203c60..1961b0fa43705 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java
@@ -10,9 +10,12 @@
 
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.index.IndexModule;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.indices.recovery.IndexRecoveryIT;
+import org.opensearch.indices.recovery.RecoverySettings;
 import org.opensearch.indices.replication.common.ReplicationType;
 import org.opensearch.test.OpenSearchIntegTestCase;
 import org.hamcrest.Matcher;
@@ -22,6 +25,7 @@
 
 import java.nio.file.Path;
 
+import static org.opensearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_CHUNK_SIZE_SETTING;
 import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings;
 
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
@@ -54,6 +58,26 @@ public Settings indexSettings() {
             .build();
     }
 
+    @Override
+    public void slowDownRecovery(ByteSizeValue shardSize) {
+        logger.info("--> shardSize: " + shardSize);
+        long chunkSize = Math.max(1, shardSize.getBytes() / 50);
+        assertTrue(
+            client().admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setTransientSettings(
+                    Settings.builder()
+                        // one chunk per sec..
+                        .put(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey(), chunkSize, ByteSizeUnit.BYTES)
+                        // small chunks
+                        .put(INDICES_RECOVERY_CHUNK_SIZE_SETTING.getKey(), new ByteSizeValue(chunkSize, ByteSizeUnit.BYTES))
+                )
+                .get()
+                .isAcknowledged()
+        );
+    }
+
     @After
     public void teardown() {
         clusterAdmin().prepareCleanupRepository(REPOSITORY_NAME).get();