From fedfcc298482ebe644de48a8825a8f13efe37efb Mon Sep 17 00:00:00 2001 From: sambles Date: Thu, 18 Jan 2024 13:18:42 +0000 Subject: [PATCH] Update platform s3 storage (#177) * Update compose file for testing v2 platform S3 storage * Fix --- docker/plat2-v2.docker-compose.yml | 2 +- docker/plat2-v2.s3.docker-compose.yml | 171 ++++++++++++++++++++++++++ 2 files changed, 172 insertions(+), 1 deletion(-) create mode 100755 docker/plat2-v2.s3.docker-compose.yml diff --git a/docker/plat2-v2.docker-compose.yml b/docker/plat2-v2.docker-compose.yml index 52f66300..1870aff7 100644 --- a/docker/plat2-v2.docker-compose.yml +++ b/docker/plat2-v2.docker-compose.yml @@ -134,7 +134,7 @@ services: OASIS_MODEL_SUPPLIER_ID: OasisLMF OASIS_MODEL_ID: PiWind OASIS_MODEL_VERSION_ID: 1 - OASIS_API_VERSION: ${WORKER_API_VER:-v2} + OASIS_API_VERSION: ${WORKER_API_VER:-v1} volumes: - ../:/home/worker/model - filestore-OasisData:/shared-fs:rw diff --git a/docker/plat2-v2.s3.docker-compose.yml b/docker/plat2-v2.s3.docker-compose.yml new file mode 100755 index 00000000..9ceede23 --- /dev/null +++ b/docker/plat2-v2.s3.docker-compose.yml @@ -0,0 +1,171 @@ +version: '3.4' + +volumes: + server-db-OasisData: + celery-db-OasisData: + +x-oasis-env: &server-env + OASIS_DEBUG: 1 + OASIS_URL_SUB_PATH: 0 + OASIS_ADMIN_USER: admin + OASIS_ADMIN_PASS: password + OASIS_CELERY_BROKER_URL: "amqp://rabbit:rabbit@broker:5672" + OASIS_SERVER_DB_HOST: server-db + OASIS_SERVER_DB_PASS: oasis + OASIS_SERVER_DB_USER: oasis + OASIS_SERVER_DB_NAME: oasis + OASIS_SERVER_DB_PORT: 5432 + OASIS_SERVER_CHANNEL_LAYER_HOST: channel-layer + OASIS_SERVER_DB_ENGINE: django.db.backends.postgresql_psycopg2 + OASIS_CELERY_DB_ENGINE: db+postgresql+psycopg2 + OASIS_CELERY_DB_HOST: celery-db + OASIS_CELERY_DB_PASS: password + OASIS_CELERY_DB_USER: celery + OASIS_CELERY_DB_NAME: celery + OASIS_CELERY_DB_PORT: 5432 + STARTUP_RUN_MIGRATIONS: 'true' + OASIS_INPUT_GENERATION_CONTROLLER_QUEUE: task-controller + OASIS_LOSSES_GENERATION_CONTROLLER_QUEUE: task-controller + OASIS_STORAGE_TYPE: S3 + OASIS_AWS_BUCKET_NAME: example-bucket + OASIS_AWS_S3_ENDPOINT_URL: "http://localstack-s3:4572" + OASIS_AWS_ACCESS_KEY_ID: foobar + OASIS_AWS_SECRET_ACCESS_KEY: foobar + +x-worker-env: &worker-env + OASIS_DEBUG: 1 + OASIS_CELERY_BROKER_URL: "amqp://rabbit:rabbit@broker:5672" + OASIS_CELERY_DB_ENGINE: db+postgresql+psycopg2 + OASIS_CELERY_DB_HOST: celery-db + OASIS_CELERY_DB_PASS: password + OASIS_CELERY_DB_USER: celery + OASIS_CELERY_DB_NAME: celery + OASIS_CELERY_DB_PORT: 5432 + # V1 + OASIS_RABBIT_HOST: broker + OASIS_RABBIT_PORT: 5672 + OASIS_RABBIT_USER: rabbit + OASIS_RABBIT_PASS: rabbit + +services: + localstack-s3: + image: localstack-s3:latest + build: + context: . + dockerfile: Dockerfile.localstack-s3 + ports: + - "4567-4599:4567-4599" + + server: + restart: always + image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-dev} + ports: + - 8000:8000 + links: + - server-db + - celery-db + - broker + environment: + <<: *server-env + websocket: + restart: always + image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-dev} + command: ["./asgi/run-asgi.sh"] + links: + - server-db + - celery-db + - broker + ports: + - 8001:8001 + environment: + <<: *server-env + v1-worker-monitor: + restart: always + image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-dev} + command: [celery, -A, src.server.oasisapi.celery_app_v1, worker, --loglevel=INFO, -Q, celery] + links: + - server-db + - celery-db + - broker + environment: + <<: *server-env + v2-worker-monitor: + restart: always + image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-dev} + command: [celery, -A, src.server.oasisapi.celery_app_v2, worker, --loglevel=INFO, -Q, celery-v2] + links: + - server-db + - celery-db + - broker + environment: + <<: *server-env + task-controller: + restart: always + image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-dev} + command: [celery, -A, src.server.oasisapi.celery_app_v2, worker, --loglevel=INFO, -Q, task-controller] + links: + - server-db + - celery-db + - broker + environment: + <<: *server-env + celery-beat: + restart: always + image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-dev} + command: [celery, -A, src.server.oasisapi.celery_app_v2, beat, --loglevel=INFO] + links: + - server-db + - celery-db + - broker + environment: + <<: *server-env + worker: + restart: always + image: ${WORKER_IMG:-coreoasis/model_worker}:${WORKER_TAG:-dev} + links: + - celery-db + - broker:mybroker + environment: + <<: *worker-env + OASIS_MODEL_SUPPLIER_ID: OasisLMF + OASIS_MODEL_ID: PiWind + OASIS_MODEL_VERSION_ID: 1 + OASIS_API_VERSION: ${WORKER_API_VER:-v1} + volumes: + - ../:/home/worker/model + server-db: + restart: always + image: postgres + environment: + - POSTGRES_DB=oasis + - POSTGRES_USER=oasis + - POSTGRES_PASSWORD=oasis + volumes: + - server-db-OasisData:/var/lib/postgresql/data:rw + ports: + - 33307:3306 + celery-db: + restart: always + image: postgres + environment: + - POSTGRES_DB=celery + - POSTGRES_USER=celery + - POSTGRES_PASSWORD=password + volumes: + - celery-db-OasisData:/var/lib/postgresql/data:rw + ports: + - 33306:5432 + broker: + restart: always + image: rabbitmq:3.8.14-management + environment: + - RABBITMQ_DEFAULT_USER=rabbit + - RABBITMQ_DEFAULT_PASS=rabbit + ports: + - 5672:5672 + - 15672:15672 + channel-layer: + restart: always + image: redis:5.0.7 + ports: + - 6379:6379