From af7639e23f9ecedf6466915702edc14d70c20bf6 Mon Sep 17 00:00:00 2001
From: Yao You <theyaoyou@gmail.com>
Date: Fri, 29 Sep 2023 13:42:21 -0500
Subject: [PATCH] ci: add retry to elastic search ingest test (#1581)

Occasionally the es test can fail because the index fail to be created
on the first try. Experiments show adding timeout doesn't help but add
retry mitigates the issue. See history of commits in branch:
yao/bump-inference-to-0.6.6
https://github.com/Unstructured-IO/unstructured/pull/1563

---------

Co-authored-by: ryannikolaidis <1208590+ryannikolaidis@users.noreply.github.com>
Co-authored-by: badGarnet <badGarnet@users.noreply.github.com>
---
 CHANGELOG.md                                  |  2 ++
 .../create_and_fill_es.py                     |  6 +++--
 .../jira-diff/JCTP2/10010.json                | 22 +++++++++----------
 unstructured/__version__.py                   |  2 +-
 4 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 653464a23e..f0fd3dd6c3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,5 @@
+## 0.10.19-dev0
+
 ## 0.10.18
 
 ### Enhancements
diff --git a/scripts/elasticsearch-test-helpers/create_and_fill_es.py b/scripts/elasticsearch-test-helpers/create_and_fill_es.py
index a63bd222d2..796e2187a8 100644
--- a/scripts/elasticsearch-test-helpers/create_and_fill_es.py
+++ b/scripts/elasticsearch-test-helpers/create_and_fill_es.py
@@ -10,12 +10,14 @@
 )
 
 print("Connecting to the Elasticsearch cluster.")
-es = Elasticsearch(CLUSTER_URL)
+es = Elasticsearch(CLUSTER_URL, request_timeout=30)
 print(es.info())
 df = pd.read_csv(DATA_PATH).dropna().reset_index()
 
 print("Creating an Elasticsearch index for testing elasticsearch ingest.")
-es.indices.create(index=INDEX_NAME, mappings=MAPPINGS)
+response = es.options(max_retries=5).indices.create(index=INDEX_NAME, mappings=MAPPINGS)
+if response.meta.status != 200:
+    raise RuntimeError("failed to create index")
 
 print("Loading data into the index.")
 bulk_data = []
diff --git a/test_unstructured_ingest/expected-structured-output/jira-diff/JCTP2/10010.json b/test_unstructured_ingest/expected-structured-output/jira-diff/JCTP2/10010.json
index 371718d0b3..e6cb0c425c 100644
--- a/test_unstructured_ingest/expected-structured-output/jira-diff/JCTP2/10010.json
+++ b/test_unstructured_ingest/expected-structured-output/jira-diff/JCTP2/10010.json
@@ -10,7 +10,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -30,7 +30,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -50,7 +50,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -70,7 +70,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -90,7 +90,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -110,7 +110,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -130,7 +130,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -150,7 +150,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -170,7 +170,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -190,7 +190,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -210,7 +210,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
diff --git a/unstructured/__version__.py b/unstructured/__version__.py
index 795e1f92bd..3703d5d96a 100644
--- a/unstructured/__version__.py
+++ b/unstructured/__version__.py
@@ -1 +1 @@
-__version__ = "0.10.18"  # pragma: no cover
+__version__ = "0.10.19-dev0"  # pragma: no cover