Add scicat backend unit test helper.

.github/workflows/integration.yml

@@ -24,8 +24,11 @@ jobs:
           python-version: ${{ inputs.python-version }}
       - run: python -m pip install --upgrade pip
       - run: python -m pip install -r requirements/ci.txt
+      - run: python -m pip install -r requirements/test.txt
       - run: python -m pip install -e .
+      - run: validate_ingestor_config resources/config.sample.json
       - run: docker-compose version
       - run: docker-compose -f tests/docker-compose-file-writer.yml up -d
       - run: python tests/_scicat_ingestor.py -c resources/config.sample.json --verbose
+      - run: python -m pytest --backend-tests
       - run: docker-compose -f tests/docker-compose-file-writer.yml down

README.md

@@ -46,11 +46,21 @@ background_ingestor \\
 You can use a json file to configure options.
 There is a template, ``resources/config.sample.json`` you can copy/paste to make your own configuration file.
 
+In order to update the configurations, you should update it the ``scicat_configuration`` module.
+
+The template file can be synchronized automatically by ``synchronize_config`` command.
+
+### Configuration Validator
+
+You can validate a configuration file with ``validate_ingestor_config`` command.
+
 ```bash
-cp resources/config.sample.json config.20240405.json
+validate_ingestor_config
 ```
 
-Then ``scicat_ingestor`` will automatically use the configuration file.
+It tries building nested configuration dataclasses from the configuration file.
+
+It will throw errors if configuration is invalid.
 
 ## Developer's Guide
 

pyproject.toml

@@ -47,6 +47,8 @@ dynamic = ["version"]
 [project.scripts]
 scicat_ingestor = "scicat_online_ingestor:main"
 background_ingestor = "scicat_offline_ingestor:main"
+validate_ingestor_config = "scicat_configuration:validate_config_file"
+synchronize_config = "scicat_configuration:synchronize_config_file"
 
 [project.entry-points."scicat_ingestor.metadata_extractor"]
 max = "numpy:max"

requirements/basetest.in

@@ -2,3 +2,6 @@
 # Do not make an environment from this file, use test.txt instead!
 
 pytest
+scitacean
+pyyaml  # for scitacean tests
+filelock  # for scitacean tests

requirements/basetest.txt

@@ -1,15 +1,60 @@
-# SHA1:0eaa389e1fdb3a1917c0f987514bd561be5718ee
+# SHA1:cd00eb6bf6784904fa6849347175c3190eb38bc7
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
 #
 #    pip-compile-multi
 #
+annotated-types==0.7.0
+    # via pydantic
+anyio==4.4.0
+    # via httpx
+certifi==2024.7.4
+    # via
+    #   httpcore
+    #   httpx
+dnspython==2.6.1
+    # via email-validator
+email-validator==2.2.0
+    # via scitacean
+filelock==3.15.4
+    # via -r basetest.in
+h11==0.14.0
+    # via httpcore
+httpcore==1.0.5
+    # via httpx
+httpx==0.27.0
+    # via scitacean
+idna==3.7
+    # via
+    #   anyio
+    #   email-validator
+    #   httpx
 iniconfig==2.0.0
     # via pytest
 packaging==24.1
     # via pytest
 pluggy==1.5.0
     # via pytest
+pydantic==2.8.2
+    # via scitacean
+pydantic-core==2.20.1
+    # via pydantic
 pytest==8.3.2
     # via -r basetest.in
+python-dateutil==2.9.0.post0
+    # via scitacean
+pyyaml==6.0.2
+    # via -r basetest.in
+scitacean==24.5.0
+    # via -r basetest.in
+six==1.16.0
+    # via python-dateutil
+sniffio==1.3.1
+    # via
+    #   anyio
+    #   httpx
+typing-extensions==4.12.2
+    # via
+    #   pydantic
+    #   pydantic-core

requirements/ci.txt

@@ -36,7 +36,7 @@ pyproject-api==1.7.1
     # via tox
 smmap==5.0.1
     # via gitdb
-tox==4.17.0
+tox==4.17.1
     # via -r ci.in
 virtualenv==20.26.3
     # via tox

requirements/dev.txt

@@ -10,8 +10,6 @@
 -r mypy.txt
 -r static.txt
 -r test.txt
-annotated-types==0.7.0
-    # via pydantic
 build==1.2.1
     # via pip-tools
 click==8.1.7
@@ -20,12 +18,18 @@ click==8.1.7
     #   pip-tools
 copier==9.3.1
     # via -r dev.in
-dunamai==1.21.2
+dunamai==1.22.0
     # via copier
 funcy==2.0
     # via copier
+jinja2==3.1.4
+    # via
+    #   copier
+    #   jinja2-ansible-filters
 jinja2-ansible-filters==1.3.2
     # via copier
+markupsafe==2.1.5
+    # via jinja2
 pathspec==0.12.1
     # via copier
 pip-compile-multi==2.6.4
@@ -36,10 +40,6 @@ plumbum==1.8.3
     # via copier
 prompt-toolkit==3.0.36
     # via questionary
-pydantic==2.8.2
-    # via copier
-pydantic-core==2.20.1
-    # via pydantic
 pyproject-hooks==1.1.0
     # via
     #   build
@@ -50,7 +50,7 @@ toposort==1.10
     # via pip-compile-multi
 wcwidth==0.2.13
     # via prompt-toolkit
-wheel==0.43.0
+wheel==0.44.0
     # via pip-tools
 
 # The following packages are considered to be unsafe in a requirements file:

requirements/mypy.txt

@@ -10,5 +10,3 @@ mypy==1.11.1
     # via -r mypy.in
 mypy-extensions==1.0.0
     # via mypy
-typing-extensions==4.12.2
-    # via mypy

requirements/nightly.txt

@@ -6,8 +6,6 @@
 #    pip-compile-multi
 #
 -r basetest.txt
-certifi==2024.7.4
-    # via requests
 charset-normalizer==3.3.2
     # via requests
 confluent-kafka==2.5.0
@@ -20,8 +18,6 @@ graypy==2.1.0
     # via -r nightly.in
 h5py==3.11.0
     # via -r nightly.in
-idna==3.7
-    # via requests
 kafka-python==2.0.2
     # via -r nightly.in
 markdown-it-py==3.0.0

resources/config.sample.json

@@ -1,5 +1,5 @@
 {
-  "config_file": "config.json",
+  "config_file": "",
   "id": "",
   "dataset": {
     "check_by_job_id": true,
@@ -15,7 +15,7 @@
   },
   "ingestion": {
     "dry_run": false,
-    "offline_ingestor_executable" : "./scicat_offline_ingestor.py",
+    "offline_ingestor_executable": "background_ingestor",
     "schemas_directory": "schemas",
     "file_handling": {
       "compute_file_stats": true,
@@ -29,14 +29,9 @@
     }
   },
   "kafka": {
-    "topics": [
-      "KAFKA_TOPIC_1",
-      "KAFKA_TOPIC_2"
-    ],
+    "topics": "KAFKA_TOPIC_1,KAFKA_TOPIC_2",
     "group_id": "GROUP_ID",
-    "bootstrap_servers": [
-      "localhost:9093"
-    ],
+    "bootstrap_servers": "localhost:9093",
     "sasl_mechanism": "SCRAM-SHA-256",
     "sasl_username": "USERNAME",
     "sasl_password": "PASSWORD",
@@ -68,4 +63,3 @@
     "verify": false
   }
 }
-