Update platform s3 storage (#177)

* Update compose file for testing v2 platform S3 storage * Fix
OasisLMF · Jan 18, 2024 · fedfcc2 · fedfcc2
1 parent 40b83bf
commit fedfcc2
Show file tree

Hide file tree

Showing 2 changed files with 172 additions and 1 deletion.
diff --git a/docker/plat2-v2.docker-compose.yml b/docker/plat2-v2.docker-compose.yml
@@ -134,7 +134,7 @@ services:
      OASIS_MODEL_SUPPLIER_ID: OasisLMF
      OASIS_MODEL_ID: PiWind
      OASIS_MODEL_VERSION_ID: 1
-     OASIS_API_VERSION: ${WORKER_API_VER:-v2}
+     OASIS_API_VERSION: ${WORKER_API_VER:-v1}
     volumes:
      - ../:/home/worker/model
      - filestore-OasisData:/shared-fs:rw

diff --git a/docker/plat2-v2.s3.docker-compose.yml b/docker/plat2-v2.s3.docker-compose.yml
@@ -0,0 +1,171 @@
+version: '3.4'
+
+volumes:
+  server-db-OasisData:
+  celery-db-OasisData:
+
+x-oasis-env: &server-env
+  OASIS_DEBUG: 1
+  OASIS_URL_SUB_PATH: 0
+  OASIS_ADMIN_USER: admin
+  OASIS_ADMIN_PASS: password
+  OASIS_CELERY_BROKER_URL: "amqp://rabbit:rabbit@broker:5672"
+  OASIS_SERVER_DB_HOST: server-db
+  OASIS_SERVER_DB_PASS: oasis
+  OASIS_SERVER_DB_USER: oasis
+  OASIS_SERVER_DB_NAME: oasis
+  OASIS_SERVER_DB_PORT: 5432
+  OASIS_SERVER_CHANNEL_LAYER_HOST: channel-layer
+  OASIS_SERVER_DB_ENGINE: django.db.backends.postgresql_psycopg2
+  OASIS_CELERY_DB_ENGINE: db+postgresql+psycopg2
+  OASIS_CELERY_DB_HOST: celery-db
+  OASIS_CELERY_DB_PASS: password
+  OASIS_CELERY_DB_USER: celery
+  OASIS_CELERY_DB_NAME: celery
+  OASIS_CELERY_DB_PORT: 5432
+  STARTUP_RUN_MIGRATIONS: 'true'
+  OASIS_INPUT_GENERATION_CONTROLLER_QUEUE: task-controller
+  OASIS_LOSSES_GENERATION_CONTROLLER_QUEUE: task-controller
+  OASIS_STORAGE_TYPE: S3
+  OASIS_AWS_BUCKET_NAME: example-bucket
+  OASIS_AWS_S3_ENDPOINT_URL: "http://localstack-s3:4572"
+  OASIS_AWS_ACCESS_KEY_ID: foobar
+  OASIS_AWS_SECRET_ACCESS_KEY: foobar
+
+x-worker-env: &worker-env
+  OASIS_DEBUG: 1
+  OASIS_CELERY_BROKER_URL: "amqp://rabbit:rabbit@broker:5672"
+  OASIS_CELERY_DB_ENGINE: db+postgresql+psycopg2
+  OASIS_CELERY_DB_HOST: celery-db
+  OASIS_CELERY_DB_PASS: password
+  OASIS_CELERY_DB_USER: celery
+  OASIS_CELERY_DB_NAME: celery
+  OASIS_CELERY_DB_PORT: 5432
+  # V1
+  OASIS_RABBIT_HOST: broker
+  OASIS_RABBIT_PORT: 5672
+  OASIS_RABBIT_USER: rabbit
+  OASIS_RABBIT_PASS: rabbit
+
+services:
+  localstack-s3:
+    image: localstack-s3:latest
+    build:
+     context: .
+     dockerfile: Dockerfile.localstack-s3
+    ports:
+      - "4567-4599:4567-4599"
+
+  server:
+    restart: always
+    image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-dev}
+    ports:
+      - 8000:8000
+    links:
+      - server-db
+      - celery-db
+      - broker
+    environment:
+      <<: *server-env
+  websocket:
+    restart: always
+    image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-dev}
+    command: ["./asgi/run-asgi.sh"]
+    links:
+      - server-db
+      - celery-db
+      - broker
+    ports:
+      - 8001:8001
+    environment:
+      <<: *server-env
+  v1-worker-monitor:
+    restart: always
+    image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-dev}
+    command: [celery, -A, src.server.oasisapi.celery_app_v1, worker, --loglevel=INFO, -Q, celery]
+    links:
+      - server-db
+      - celery-db
+      - broker
+    environment:
+      <<: *server-env
+  v2-worker-monitor:
+    restart: always
+    image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-dev}
+    command: [celery, -A, src.server.oasisapi.celery_app_v2, worker, --loglevel=INFO, -Q, celery-v2]
+    links:
+      - server-db
+      - celery-db
+      - broker
+    environment:
+      <<: *server-env
+  task-controller:
+    restart: always
+    image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-dev}
+    command: [celery, -A, src.server.oasisapi.celery_app_v2, worker, --loglevel=INFO, -Q, task-controller]
+    links:
+      - server-db
+      - celery-db
+      - broker
+    environment:
+      <<: *server-env
+  celery-beat:
+    restart: always
+    image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-dev}
+    command: [celery, -A, src.server.oasisapi.celery_app_v2, beat, --loglevel=INFO]
+    links:
+      - server-db
+      - celery-db
+      - broker
+    environment:
+      <<: *server-env
+  worker:
+    restart: always
+    image: ${WORKER_IMG:-coreoasis/model_worker}:${WORKER_TAG:-dev}
+    links:
+     - celery-db
+     - broker:mybroker
+    environment:
+     <<: *worker-env
+     OASIS_MODEL_SUPPLIER_ID: OasisLMF
+     OASIS_MODEL_ID: PiWind
+     OASIS_MODEL_VERSION_ID: 1
+     OASIS_API_VERSION: ${WORKER_API_VER:-v1}
+    volumes:
+     - ../:/home/worker/model
+  server-db:
+    restart: always
+    image: postgres
+    environment:
+      - POSTGRES_DB=oasis
+      - POSTGRES_USER=oasis
+      - POSTGRES_PASSWORD=oasis
+    volumes:
+      - server-db-OasisData:/var/lib/postgresql/data:rw
+    ports:
+      - 33307:3306
+  celery-db:
+    restart: always
+    image: postgres
+    environment:
+      - POSTGRES_DB=celery
+      - POSTGRES_USER=celery
+      - POSTGRES_PASSWORD=password
+    volumes:
+      - celery-db-OasisData:/var/lib/postgresql/data:rw
+    ports:
+      - 33306:5432
+  broker:
+    restart: always
+    image: rabbitmq:3.8.14-management
+    environment:
+      - RABBITMQ_DEFAULT_USER=rabbit
+      - RABBITMQ_DEFAULT_PASS=rabbit
+    ports:
+      - 5672:5672
+      - 15672:15672
+  channel-layer:
+    restart: always
+    image: redis:5.0.7
+    ports:
+      - 6379:6379