update elasticsearch docker setup to use docker-compose

.github/workflows/ingest-test-fixtures-update-pr.yml

@@ -9,7 +9,7 @@ env:
 
 jobs:
   setup:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-m
     if: |
       github.event_name == 'workflow_dispatch' ||
       (github.event_name == 'push' && contains(github.event.head_commit.message, 'ingest-test-fixtures-update'))

scripts/elasticsearch-test-helpers/create-and-check-es.sh

@@ -1,37 +1,13 @@
 #!/usr/bin/env bash
 
+set -e
+
 SCRIPT_DIR=$(dirname "$(realpath "$0")")
 
 # Create the Elasticsearch cluster and get the container id
-docker run -d --rm -p 9200:9200 -p 9300:9300 -e "xpack.security.enabled=false" -e "discovery.type=single-node" --name es-test docker.elastic.co/elasticsearch/elasticsearch:8.7.0
-
-# Wait for Elasticsearch container to start
-echo "Waiting for Elasticsearch container to start..."
-sleep 1
-
-url="http://localhost:9200/_cluster/health?wait_for_status=green&timeout=50s"
-status_code=0
-retry_count=0
-max_retries=6
-
-# Check the cluster status repeatedly until it becomes live or maximum retries are reached
-while [ "$status_code" -ne 200 ] && [ "$retry_count" -lt "$max_retries" ]; do
-  # Send a GET request to the cluster health API
-  response=$(curl -s -o /dev/null -w "%{http_code}" "$url")
-  status_code="$response"
+docker-compose -f "$SCRIPT_DIR"/docker-compose.yaml up --wait
+docker-compose -f "$SCRIPT_DIR"/docker-compose.yaml ps
 
-  # Process the files only when the Elasticsearch cluster is live
-  if [ "$status_code" -eq 200 ]; then
-    echo "Cluster is live."
-    python "$SCRIPT_DIR/create_and_fill_es.py"
-  else
-    ((retry_count++))
-    echo "Cluster is not available. Retrying in 5 seconds... (Attempt $retry_count)"
-    sleep 5
-  fi
-done
 
-# If the cluster has not become live, exit after a certain number of tries
-if [ "$status_code" -ne 200 ]; then
-  echo "Cluster took an unusually long time to create (>25 seconds). Expected time is around 10 seconds. Exiting."
-fi
+echo "Cluster is live."
+"$SCRIPT_DIR"/create_and_fill_es.py

scripts/elasticsearch-test-helpers/create_and_fill_es.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import pandas as pd
 from elasticsearch import Elasticsearch
 from elasticsearch.helpers import bulk

scripts/elasticsearch-test-helpers/docker-compose.yaml

@@ -0,0 +1,15 @@
+services:
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.7.0
+    container_name: es-test
+    ports:
+      - 9200:9200
+      - 9300:9300
+    environment:
+      - xpack.security.enabled=false
+      - discovery.type=single-node
+    healthcheck:
+      test: ["CMD-SHELL", "curl --silent --fail localhost:9200/_cluster/health || exit 1"]
+      interval: 30s
+      timeout: 30s
+      retries: 3

test_unstructured_ingest/test-ingest-elasticsearch.sh

@@ -15,10 +15,8 @@ source "$SCRIPT_DIR"/cleanup.sh
 
 function cleanup() {
   # Kill the container so the script can be repeatedly run using the same ports
-  if docker ps --filter "name=es-test"; then
-    echo "Stopping Elasticsearch Docker container"
-    docker stop es-test
-  fi
+  echo "Stopping Elasticsearch Docker container"
+  docker-compose -f scripts/elasticsearch-test-helpers/docker-compose.yaml down --remove-orphans -v
 
   cleanup_dir "$OUTPUT_DIR"
 }