ci: add retry to elastic search ingest test (#1581)

Occasionally the es test can fail because the index fail to be created on the first try. Experiments show adding timeout doesn't help but add retry mitigates the issue. See history of commits in branch: yao/bump-inference-to-0.6.6 #1563 --------- Co-authored-by: ryannikolaidis <[email protected]> Co-authored-by: badGarnet <[email protected]>
Unstructured-IO · Sep 29, 2023 · af7639e · af7639e
1 parent 5b994f3
commit af7639e
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 14 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,5 @@
+## 0.10.19-dev0
+
 ## 0.10.18
 
 ### Enhancements

diff --git a/scripts/elasticsearch-test-helpers/create_and_fill_es.py b/scripts/elasticsearch-test-helpers/create_and_fill_es.py
@@ -10,12 +10,14 @@
 )
 
 print("Connecting to the Elasticsearch cluster.")
-es = Elasticsearch(CLUSTER_URL)
+es = Elasticsearch(CLUSTER_URL, request_timeout=30)
 print(es.info())
 df = pd.read_csv(DATA_PATH).dropna().reset_index()
 
 print("Creating an Elasticsearch index for testing elasticsearch ingest.")
-es.indices.create(index=INDEX_NAME, mappings=MAPPINGS)
+response = es.options(max_retries=5).indices.create(index=INDEX_NAME, mappings=MAPPINGS)
+if response.meta.status != 200:
+    raise RuntimeError("failed to create index")
 
 print("Loading data into the index.")
 bulk_data = []

diff --git a/test_unstructured_ingest/expected-structured-output/jira-diff/JCTP2/10010.json b/test_unstructured_ingest/expected-structured-output/jira-diff/JCTP2/10010.json
@@ -10,7 +10,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -30,7 +30,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -50,7 +50,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -70,7 +70,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -90,7 +90,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -110,7 +110,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -130,7 +130,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -150,7 +150,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -170,7 +170,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -190,7 +190,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [
@@ -210,7 +210,7 @@
           "issue_key": "JCTP2-8"
         },
         "date_created": "2023-08-22T11:35:48.407+0000",
-        "date_modified": "2023-08-29T11:46:18.193+0000"
+        "date_modified": "2023-09-29T05:55:11.066+0000"
       },
       "filetype": "text/plain",
       "languages": [

diff --git a/unstructured/__version__.py b/unstructured/__version__.py
@@ -1 +1 @@
-__version__ = "0.10.18"  # pragma: no cover
+__version__ = "0.10.19-dev0"  # pragma: no cover