Update PiWind testing to merged API versions (#173)

* Test new docker-compose

* fix

.github/workflows/integration.yml

@@ -29,6 +29,10 @@ on:
         description: 'Worker image [semvar]'
         required: false
         default: 'latest'
+      worker_api_ver:
+        description: 'Worker image run mode [v1, v2]'
+        required: false
+        default: 'v2'
       debug_mode:
         description: 'Enable verbose logging for containers [0, 1]'
         required: false
@@ -70,6 +74,10 @@ on:
         description: 'Worker image [semvar]'
         required: false
         type: string
+      worker_api_ver:
+        description: 'Worker image run mode [v1, v2]'
+        required: false
+        type: string
       debug_mode:
         description: 'Enable verbose logging for containers [0, 1]'
         required: false
@@ -119,6 +127,7 @@ jobs:
       SERVER_TAG: 'latest'
       WORKER_IMG: 'coreoasis/model_worker'
       WORKER_TAG: 'latest'
+      WORKER_API_VER: 'v2'
       DEBUG: 1
 
     steps:
@@ -138,6 +147,7 @@ jobs:
        [[ -z "${{ inputs.server_tag }}"   ]] || echo "SERVER_TAG=${{ inputs.server_tag }}" >> $GITHUB_ENV
        [[ -z "${{ inputs.worker_image }}" ]] || echo "WORKER_IMG=${{ inputs.worker_image }}" >> $GITHUB_ENV
        [[ -z "${{ inputs.worker_tag }}"   ]] || echo "WORKER_TAG=${{ inputs.worker_tag }}" >> $GITHUB_ENV
+       [[ -z "${{ inputs.worker_api_ver }}"   ]] || echo "WORKER_API_VER=${{ inputs.worker_api_ver }}" >> $GITHUB_ENV
        [[ -z "${{ inputs.debug_mode }}"   ]] || echo "DEBUG=${{ inputs.debug_mode }}" >> $GITHUB_ENV
 
     - name: clone PiWind repository

docker/plat2-v2.docker-compose.yml

@@ -0,0 +1,174 @@
+version: '3.4'
+volumes:
+  server-db-OasisData:
+  celery-db-OasisData:
+  filestore-OasisData:
+
+x-server-env: &server-env
+  OASIS_DEBUG: 1
+  OASIS_URL_SUB_PATH: 0
+  OASIS_CELERY_BROKER_URL: "amqp://rabbit:rabbit@broker:5672"
+  OASIS_SERVER_DB_HOST: server-db
+  OASIS_SERVER_DB_PASS: oasis
+  OASIS_SERVER_DB_USER: oasis
+  OASIS_SERVER_DB_NAME: oasis
+  OASIS_SERVER_DB_PORT: 5432
+  OASIS_SERVER_CHANNEL_LAYER_HOST: channel-layer
+  OASIS_SERVER_DB_ENGINE: django.db.backends.postgresql_psycopg2
+  OASIS_CELERY_DB_ENGINE: db+postgresql+psycopg2
+  OASIS_CELERY_DB_HOST: celery-db
+  OASIS_CELERY_DB_PASS: password
+  OASIS_CELERY_DB_USER: celery
+  OASIS_CELERY_DB_NAME: celery
+  OASIS_CELERY_DB_PORT: 5432
+  OASIS_INPUT_GENERATION_CONTROLLER_QUEUE: task-controller
+  OASIS_LOSSES_GENERATION_CONTROLLER_QUEUE: task-controller
+
+x-worker-env: &worker-env
+  OASIS_DEBUG: 1
+  OASIS_CELERY_BROKER_URL: "amqp://rabbit:rabbit@broker:5672"
+  OASIS_CELERY_DB_ENGINE: db+postgresql+psycopg2
+  OASIS_CELERY_DB_HOST: celery-db
+  OASIS_CELERY_DB_PASS: password
+  OASIS_CELERY_DB_USER: celery
+  OASIS_CELERY_DB_NAME: celery
+  OASIS_CELERY_DB_PORT: 5432
+  # V1
+  OASIS_RABBIT_HOST: broker
+  OASIS_RABBIT_PORT: 5672
+  OASIS_RABBIT_USER: rabbit
+  OASIS_RABBIT_PASS: rabbit
+
+
+x-volumes: &shared-volumes
+  - filestore-OasisData:/shared-fs:rw
+
+services:
+  server:
+   restart: always
+   image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-2.1.0-dev}
+   command: ["./wsgi/run-wsgi.sh"]
+   ports:
+     - 8000:8000
+   links:
+     - server-db
+     - celery-db
+     - broker
+   environment:
+     <<: *server-env
+     STARTUP_RUN_MIGRATIONS: "true"
+     OASIS_ADMIN_USER: admin
+     OASIS_ADMIN_PASS: password
+   volumes:
+     - filestore-OasisData:/shared-fs:rw
+  websocket:
+   restart: always
+   image: coreoasis/api_server:dev
+   image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-2.1.0-dev}
+   command: ["./asgi/run-asgi.sh"]
+   links:
+     - server-db
+     - celery-db
+     - broker
+   ports:
+     - 8001:8001
+   environment:
+     <<: *server-env
+   volumes:
+     - filestore-OasisData:/shared-fs:rw
+  v1-worker-monitor:
+   restart: always
+   image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-2.1.0-dev}
+   command: [celery, -A, src.server.oasisapi.celery_app_v1, worker, --loglevel=INFO, -Q, celery]
+   links:
+     - server-db
+     - celery-db
+     - broker
+   environment:
+     <<: *server-env
+  v2-worker-monitor:
+   restart: always
+   image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-2.1.0-dev}
+   command: [celery, -A, src.server.oasisapi.celery_app_v2, worker, --loglevel=INFO, -Q, celery-v2]
+   links:
+     - server-db
+     - celery-db
+     - broker
+   environment:
+     <<: *server-env
+   volumes:
+     - filestore-OasisData:/shared-fs:rw
+  task-controller:
+   restart: always
+   image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-2.1.0-dev}
+   command: [celery, -A, src.server.oasisapi.celery_app_v2, worker, --loglevel=INFO, -Q, task-controller]
+   links:
+     - server-db
+     - celery-db
+     - broker
+   environment:
+     <<: *server-env
+   volumes:
+     - filestore-OasisData:/shared-fs:rw
+  celery-beat:
+   restart: always
+   image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-2.1.0-dev}
+   command: [celery, -A, src.server.oasisapi.celery_app_v2, beat, --loglevel=INFO]
+   links:
+     - server-db
+     - celery-db
+     - broker
+   environment:
+     <<: *server-env
+   volumes: *shared-volumes
+  worker:
+    restart: always
+    image: ${WORKER_IMG:-coreoasis/model_worker}:${WORKER_TAG:-2.1.0-dev}
+    links:
+     - celery-db
+     - broker:mybroker
+    environment:
+     <<: *worker-env
+     OASIS_MODEL_SUPPLIER_ID: OasisLMF
+     OASIS_MODEL_ID: PiWind
+     OASIS_MODEL_VERSION_ID: 1
+     OASIS_API_VERSION: ${WORKER_API_VER:-v2}
+    volumes:
+     - ../:/home/worker/model
+     - filestore-OasisData:/shared-fs:rw
+  server-db:
+    restart: always
+    image: postgres
+    environment:
+      - POSTGRES_DB=oasis
+      - POSTGRES_USER=oasis
+      - POSTGRES_PASSWORD=oasis
+    volumes:
+      - server-db-OasisData:/var/lib/postgresql/data:rw
+    ports:
+      - 33307:3306
+  celery-db:
+    restart: always
+    image: postgres
+    environment:
+      - POSTGRES_DB=celery
+      - POSTGRES_USER=celery
+      - POSTGRES_PASSWORD=password
+    volumes:
+      - celery-db-OasisData:/var/lib/postgresql/data:rw
+    ports:
+      - 33306:5432
+  broker:
+    restart: always
+    image: rabbitmq:3.8.14-management
+    environment:
+      - RABBITMQ_DEFAULT_USER=rabbit
+      - RABBITMQ_DEFAULT_PASS=rabbit
+    ports:
+      - 5672:5672
+      - 15672:15672
+  channel-layer:
+    restart: always
+    image: redis:5.0.7
+    ports:
+      - 6379:6379

tests/conftest.py

@@ -1,11 +1,17 @@
 """pytest configuration"""
 
+from os import getenv
 from .modelcheck import wait_for_api, _class_server_conn
 
+
 def pytest_addoption(parser):
     parser.addoption(
         '--generate-expected', action='store_true', default=False,  # dest='generate_expected',
-        help='If True, it generates the expected files instead of running the tests. Default: False.'
+        help='If True, it generates the expected files instead of running the tests. Default: False.',
+    )
+    parser.addoption(
+        '--api-version', type=str, default=getenv('WORKER_API_VER', default='v1'), 
+        help='Set the OasisAPI clients api version [v1, v2]' 
     )  
 
 pytest_plugins = [

tests/modelcheck.py

@@ -51,7 +51,7 @@
 
 
 @pytest.fixture(scope='module')
-def wait_for_api(module_scoped_container_getter):
+def wait_for_api(module_scoped_container_getter, request):
     """
     Fixture that waits for the docker container running the OasisAPI to become responsive.
     see https://github.com/pytest-docker-compose/pytest-docker-compose for details on using
@@ -76,6 +76,7 @@ def wait_for_api(module_scoped_container_getter):
     # Wait for server
     request_session.mount('http://', HTTPAdapter(max_retries=retries))
     server = module_scoped_container_getter.get("server").network_info[0]
+    api_ver = request.config.getoption('--api-version', 'v1')
     api_url = f"http://{server.hostname}:{server.host_port}"
     assert request_session.get(f"{api_url}/healthcheck/")
 
@@ -85,13 +86,12 @@ def wait_for_api(module_scoped_container_getter):
         assert request_session.get(localstack_url)
 
     # Wait for Model
-    oasis_client = APIClient(api_url=api_url)
+    oasis_client = APIClient(api_url=api_url, api_ver=api_ver)
     oasis_client.api.mount('http://', HTTPAdapter(max_retries=retries))
 
     model_headers = {'authorization': f"Bearer {oasis_client.api.tkn_access}"}
-    model_url = f"{api_url}/V1/models/1/"
+    model_url = f"{api_url}/{api_ver}/models/1/"
     assert request_session.get(model_url, headers=model_headers)
-    #assert oasis_client.models.get(1)
     return oasis_client
 
 
@@ -193,7 +193,7 @@ def setUpClass(cls, expected_dir=None, input_tar=None, results_tar=None, params=
 
         # skip run if paramters are missing
         if not cls.params:
-            pytest.skip(f"Skipping TestClass={cls.__name__}, no input files set in params")    
+            pytest.skip(f"Skipping TestClass={cls.__name__}, no input files set in params")
 
         # Find PiWind's model id
         cls.model_id = cls._get_model_id(cls)