Check CI platform testing (#183)

* blank-commit * Update and test pins * Update default compose for V2 platform support * Fix worker mount * revert requ base branch
OasisLMF · May 22, 2024 · c792c9c · c792c9c
1 parent e2be4b7
commit c792c9c
Show file tree

Hide file tree

Showing 2 changed files with 179 additions and 103 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,101 +1,177 @@
-version: '3.4'
-
-volumes:
-  server-db-OasisData:
-  celery-db-OasisData:
-  filestore-OasisData:
-
-x-oasis-env: &oasis-env
-  OASIS_ADMIN_USER: admin
-  OASIS_ADMIN_PASS: password
-  OASIS_DEBUG: ${DEBUG:-0}
-  OASIS_RABBIT_HOST: rabbit
-  OASIS_RABBIT_PORT: 5672
-  OASIS_RABBIT_USER: rabbit
-  OASIS_RABBIT_PASS: rabbit
-  OASIS_SERVER_DB_HOST: server-db
-  OASIS_SERVER_DB_PASS: oasis
-  OASIS_SERVER_DB_USER: oasis
-  OASIS_SERVER_DB_NAME: oasis
-  OASIS_SERVER_DB_PORT: 5432
-  OASIS_SERVER_DB_ENGINE: django.db.backends.postgresql_psycopg2
-  OASIS_CELERY_DB_ENGINE: db+postgresql+psycopg2
-  OASIS_CELERY_DB_HOST: celery-db
-  OASIS_CELERY_DB_PASS: password
-  OASIS_CELERY_DB_USER: celery
-  OASIS_CELERY_DB_NAME: celery
-  OASIS_CELERY_DB_PORT: 5432
-  STARTUP_RUN_MIGRATIONS: 'true'
-
-services:
-  server:
-   restart: always
-   image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-latest}
-   ports:
-     - 8000:8000
-   links:
-     - server-db
-     - celery-db
-     - rabbit
-   environment:
-     <<: *oasis-env
-   volumes:
-     - filestore-OasisData:/shared-fs:rw
-
-  worker-monitor:
-   restart: always
-   image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-latest}
-   command: [wait-for-server, 'server:8000', celery, -A, src.server.oasisapi, worker, --loglevel=INFO]
-   links:
-     - server-db
-     - celery-db
-     - rabbit
-   environment:
-     <<: *oasis-env
-   volumes:
-     - filestore-OasisData:/shared-fs:rw
-  worker:
-    restart: always
-    image: ${WORKER_IMG:-coreoasis/model_worker}:${WORKER_TAG:-latest}
-    links:
-     - celery-db
-     - rabbit:myrabbit
-    environment:
-     <<: *oasis-env
-     OASIS_MODEL_SUPPLIER_ID: OasisLMF
-     OASIS_MODEL_ID: PiWind
-     OASIS_MODEL_VERSION_ID: 1
-     OASIS_MODEL_DATA_DIRECTORY: /home/worker/model
-    volumes:
-     - ./:/home/worker/model
-     - filestore-OasisData:/shared-fs:rw
-
-  server-db:
-    restart: always
-    image: postgres
-    environment:
-      POSTGRES_DB: oasis
-      POSTGRES_USER: oasis
-      POSTGRES_PASSWORD: oasis
-    volumes:
-      - server-db-OasisData:/var/lib/postgresql/data:rw
-
-  celery-db:
-    restart: always
-    image: postgres
-    environment:
-      POSTGRES_DB: celery
-      POSTGRES_USER: celery
-      POSTGRES_PASSWORD: password
-    volumes:
-      - celery-db-OasisData:/var/lib/postgresql/data:rw
-
-  rabbit:
-    restart: always
-    image: rabbitmq:3.8.14-management
-    environment:
-      RABBITMQ_DEFAULT_USER: rabbit
-      RABBITMQ_DEFAULT_PASS: rabbit
-    ports:
-      - 5672:5672
-      - 15672:15672
+version: '3.4'
+volumes:
+  server-db-OasisData:
+  celery-db-OasisData:
+  filestore-OasisData:
+
+x-server-env: &server-env
+  OASIS_DEBUG: ${DEBUG:-0}
+  OASIS_URL_SUB_PATH: 0
+  OASIS_CELERY_BROKER_URL: "amqp://rabbit:rabbit@broker:5672"
+  OASIS_SERVER_DB_HOST: server-db
+  OASIS_SERVER_DB_PASS: oasis
+  OASIS_SERVER_DB_USER: oasis
+  OASIS_SERVER_DB_NAME: oasis
+  OASIS_SERVER_DB_PORT: 5432
+  OASIS_SERVER_CHANNEL_LAYER_HOST: channel-layer
+  OASIS_SERVER_DB_ENGINE: django.db.backends.postgresql_psycopg2
+  OASIS_CELERY_DB_ENGINE: db+postgresql+psycopg2
+  OASIS_CELERY_DB_HOST: celery-db
+  OASIS_CELERY_DB_PASS: password
+  OASIS_CELERY_DB_USER: celery
+  OASIS_CELERY_DB_NAME: celery
+  OASIS_CELERY_DB_PORT: 5432
+  OASIS_INPUT_GENERATION_CONTROLLER_QUEUE: task-controller  # older setting (before 2.3.1)
+  OASIS_LOSSES_GENERATION_CONTROLLER_QUEUE: task-controller # older setting (before 2.3.1)
+  OASIS_TASK_CONTROLLER_QUEUE: task-controller              # New option v2.3.1+
+
+x-worker-env: &worker-env
+  OASIS_DEBUG: ${DEBUG:-0}
+  OASIS_CELERY_BROKER_URL: "amqp://rabbit:rabbit@broker:5672"
+  OASIS_CELERY_DB_ENGINE: db+postgresql+psycopg2
+  OASIS_CELERY_DB_HOST: celery-db
+  OASIS_CELERY_DB_PASS: password
+  OASIS_CELERY_DB_USER: celery
+  OASIS_CELERY_DB_NAME: celery
+  OASIS_CELERY_DB_PORT: 5432
+  # V1
+  OASIS_RABBIT_HOST: broker
+  OASIS_RABBIT_PORT: 5672
+  OASIS_RABBIT_USER: rabbit
+  OASIS_RABBIT_PASS: rabbit
+
+
+x-volumes: &shared-volumes
+  - filestore-OasisData:/shared-fs:rw
+
+services:
+  server:
+   restart: always
+   image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-latest}
+   command: ["./wsgi/run-wsgi.sh"]
+   ports:
+     - 8000:8000
+   links:
+     - server-db
+     - celery-db
+     - broker
+   environment:
+     <<: *server-env
+     STARTUP_RUN_MIGRATIONS: "true"
+     OASIS_ADMIN_USER: admin
+     OASIS_ADMIN_PASS: password
+   volumes:
+     - filestore-OasisData:/shared-fs:rw
+  websocket:
+   restart: always
+   image: coreoasis/api_server:dev
+   image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-latest}
+   command: ["./asgi/run-asgi.sh"]
+   links:
+     - server-db
+     - celery-db
+     - broker
+   ports:
+     - 8001:8001
+   environment:
+     <<: *server-env
+   volumes:
+     - filestore-OasisData:/shared-fs:rw
+  v1-worker-monitor:
+   restart: always
+   image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-latest}
+   command: [celery, -A, src.server.oasisapi.celery_app_v1, worker, --loglevel=INFO, -Q, celery]
+   links:
+     - server-db
+     - celery-db
+     - broker
+   environment:
+     <<: *server-env
+   volumes:
+     - filestore-OasisData:/shared-fs:rw
+  v2-worker-monitor:
+   restart: always
+   image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-latest}
+   command: [celery, -A, src.server.oasisapi.celery_app_v2, worker, --loglevel=INFO, -Q, celery-v2]
+   links:
+     - server-db
+     - celery-db
+     - broker
+   environment:
+     <<: *server-env
+   volumes:
+     - filestore-OasisData:/shared-fs:rw
+  task-controller:
+   restart: always
+   image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-latest}
+   command: [celery, -A, src.server.oasisapi.celery_app_v2, worker, --loglevel=INFO, -Q, task-controller]
+   links:
+     - server-db
+     - celery-db
+     - broker
+   environment:
+     <<: *server-env
+   volumes:
+     - filestore-OasisData:/shared-fs:rw
+  celery-beat:
+   restart: always
+   image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-latest}
+   command: [celery, -A, src.server.oasisapi.celery_app_v2, beat, --loglevel=INFO]
+   links:
+     - server-db
+     - celery-db
+     - broker
+   environment:
+     <<: *server-env
+   volumes: *shared-volumes
+  worker:
+    restart: always
+    image: ${WORKER_IMG:-coreoasis/model_worker}:${WORKER_TAG:-latest}
+    links:
+     - celery-db
+     - broker:mybroker
+    environment:
+     <<: *worker-env
+     OASIS_MODEL_SUPPLIER_ID: OasisLMF
+     OASIS_MODEL_ID: PiWind
+     OASIS_MODEL_VERSION_ID: 1
+     OASIS_RUN_MODE: ${WORKER_API_VER:-v1}
+    volumes:
+     - ./:/home/worker/model
+     - filestore-OasisData:/shared-fs:rw
+  server-db:
+    restart: always
+    image: postgres
+    environment:
+      - POSTGRES_DB=oasis
+      - POSTGRES_USER=oasis
+      - POSTGRES_PASSWORD=oasis
+    volumes:
+      - server-db-OasisData:/var/lib/postgresql/data:rw
+    ports:
+      - 33307:3306
+  celery-db:
+    restart: always
+    image: postgres
+    environment:
+      - POSTGRES_DB=celery
+      - POSTGRES_USER=celery
+      - POSTGRES_PASSWORD=password
+    volumes:
+      - celery-db-OasisData:/var/lib/postgresql/data:rw
+    ports:
+      - 33306:5432
+  broker:
+    restart: always
+    image: rabbitmq:3.8.14-management
+    environment:
+      - RABBITMQ_DEFAULT_USER=rabbit
+      - RABBITMQ_DEFAULT_PASS=rabbit
+    ports:
+      - 5672:5672
+      - 15672:15672
+  channel-layer:
+    restart: always
+    image: redis:5.0.7
+    ports:
+      - 6379:6379
diff --git a/tests/requirements.in b/tests/requirements.in
@@ -1,8 +1,8 @@
-pytest
+pytest==8.2.0
 pytest-docker-compose
 oasislmf
 pathlib
-requests
+requests==2.31.0
 pandas 
 parametrize
 pyYaml==5.3.1