Update base project config files

.github/workflows/build_env_run_tests.yml

@@ -0,0 +1,87 @@
+# Builds the aeon environment; lints formatting and smells via ruff; checks type annotations via pyright;
+# tests via pytest; reports test coverage via pytest-cov and codecov.
+
+name: build_env_run_tests
+on:
+  pull_request:
+    branches: [ main ]
+    types: [opened, reopened, synchronize]
+  workflow_dispatch:  # allows running manually from Github's 'Actions' tab
+
+jobs:
+  build_env_pip_pyproject:  # checks only for building env using pip and pyproject.toml
+    name: Build env using pip and pyproject.toml
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: [3.11]
+      fail-fast: false
+    defaults:
+      run:
+        shell: bash -l {0}  # reset shell for each step
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Create venv and install dependencies
+        run: |
+          python -m venv .venv
+          source .venv/bin/activate
+          pip install -e .[dev]
+          pip list
+          .venv/bin/python -c "import aeon"
+
+  build_env_run_tests:  # checks for building env using mamba and runs codebase checks and tests
+    name: Build env and run tests on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: [3.11]
+      fail-fast: false
+    defaults:
+      run:
+        shell: bash -l {0}  # reset shell for each step
+    steps:
+      - name: checkout repo
+        uses: actions/checkout@v2
+      - name: set up conda env
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          use-mamba: true
+          miniforge-variant: Mambaforge
+          python-version: ${{ matrix.python-version }}
+          environment-file: ./env_config/env.yml
+          activate-environment: aeon
+      - name: Update conda env with dev reqs
+        run: mamba env update -f ./env_config/env_dev.yml
+
+      # Only run codebase checks and tests for ubuntu.
+      - name: ruff
+        if: matrix.os == 'ubuntu-latest'
+        run: python -m ruff check --config ./pyproject.toml .
+      - name: pyright
+        if: matrix.os == 'ubuntu-latest'
+        run: python -m pyright --level error --project ./pyproject.toml . 
+      - name: pytest
+        if: matrix.os == 'ubuntu-latest'
+        run: python -m pytest tests/
+
+      - name: generate test coverage report
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          python -m pytest --cov=aeon ./tests/ --cov-report=xml:./tests/test_coverage/test_coverage_report.xml
+          #python -m pytest --cov=aeon ./tests/ --cov-report=html:./tests/test_coverage/test_coverage_report_html
+      - name: upload test coverage report to codecov
+        if: matrix.os == 'ubuntu-latest'
+        uses: codecov/codecov-action@v2
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          directory: ./tests/test_coverage/
+          files: test_coverage_report.xml
+          fail_ci_if_error: true
+          verbose: true

.github/workflows/gpt_pr_review.yml

@@ -0,0 +1,58 @@
+# Reviews pull requests with OpenAI's GPT API.
+
+name: gpt_pr_review
+permissions:
+  contents: read
+  pull-requests: write
+on:
+  pull_request:
+    branches: [ main ]
+    types: [opened, reopened, synchronize]
+  pull_request_review_comment:
+    branches: [ main ]
+    types: [created]
+concurrency:
+  group: >
+    ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ 
+    github.workflow }}-${{ github.event_name == 'pull_request_review_comment' && 'pr_comment' || 'pr' }}
+  cancel-in-progress: ${{ github.event_name != 'pull_request_review_comment' }}
+jobs:
+  gpt_pr_review:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: coderabbitai/ai-pr-reviewer@latest
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        with:
+          system_message: >
+            Your purpose is to act as a highly experienced 
+            software engineer and provide a thorough review of the code hunks
+            and suggest code snippets to improve key areas such as:
+            - Logic
+            - Security
+            - Performance
+            - Data races
+            - Consistency
+            - Error handling
+            - Maintainability
+            - Modularity
+            - Complexity
+            - Optimization
+            - Best practices: DRY, SOLID, KISS
+            Identify and resolve significant concerns to improve overall code 
+            quality while deliberately disregarding minor issues.
+          summarize: >
+            Provide your final response in markdown with the following content:
+            - **Walkthrough**: A high-level summary of the overall change instead of 
+            specific files within 100 words.
+            - **Changes**: A markdown table of files and their summaries. Group files 
+            with similar changes together into a single row to save space.
+            Avoid additional commentary as this summary will be added as a comment on the 
+            GitHub pull request. Use the titles "Walkthrough" and "Changes" and they must be H2.
+          summarize_release_notes: >
+            Craft concise release notes for the pull request. 
+            Focus on the purpose and user impact, categorizing changes as New Feature, Bug Fix, 
+            Documentation, Refactor, Style, Test, Chore, or Revert. Provide a bullet-point list, 
+            e.g. "- New Feature: Added search functionality to the UI." Limit your response to 50-100 words 
+            and emphasize features visible to the end-user while omitting code-level details.

.github/workflows/squash_merge_to_prod.yml

@@ -0,0 +1,28 @@
+# Squash merges commits from PRs to 'main' into 'prod'. 'prod' thus serves as a  more 
+# readable history of the project, while 'main' retains the full commit history.
+
+name: squash_merge_to_prod
+on:
+  pull_request:
+    branches: [ main ]
+    types: [ closed ]
+jobs:
+  squash_merge_to_prod:
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.merged == true  # only run if PR merged (subset of 'closed')
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0 
+    - name: Configure Git
+      run: |
+        git config user.name "Jai Bhagat"
+        git config user.email "[email protected]"
+    - name: Merge and squash to prod
+      run: |
+        git checkout prod
+        COMMIT_MSG=$(git log main -1 --pretty=format:%s)  # latest commit message from 'main'
+        git merge --squash -X theirs main
+        git commit -m "$COMMIT_MSG"
+        git push origin prod

.pre-commit-config.yaml

@@ -1,35 +1,58 @@
+# For info on running pre-commit manually, see `pre-commit run --help`
+
 default_language_version:
-  python: python3.9
+  python: python3.11
 
-default_stages: [commit, push]
 files: "^(docker|aeon\/dj_pipeline)\/.*$"
 repos:
   - repo: meta
     hooks:
       - id: identity
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.1.0
+    rev: v4.4.0
     hooks:
+      - id: check-json
       - id: check-yaml
+      - id: check-toml
+      - id: check-xml
+      - id: check-merge-conflict
       - id: detect-private-key
       - id: end-of-file-fixer
         exclude: LICENSE
       - id: no-commit-to-branch
+      - id: mixed-line-ending
+      - id: trailing-whitespace
+        args: [--markdown-linebreak-ext=md]
 
   - repo: https://github.com/psf/black
-    rev: 22.1.0
+    rev: 23.7.0
     hooks:
       - id: black
-        args:
-          - "--config"
-          - "./pyproject.toml"
+        args: [--check, --config, ./pyproject.toml]
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.0.286
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix, --config, ./pyproject.toml]
 
-  - repo: https://github.com/pycqa/isort
-    rev: 5.10.1
+  - repo: https://github.com/RobertCraigie/pyright-python
+    rev: v1.1.324
     hooks:
-      - id: isort
-        name: isort (python)
-        args:
-          - "--settings-file"
-          - "./pyproject.toml"
+    - id: pyright
+      args: [--level, error, --project, ./pyproject.toml]
+
+
+  # Pytest is expensive, so we show its set-up but leave it commented out.
+  # - repo: local
+  #   hooks:
+  #     - id: pytest-py
+  #       name: pytest-py
+  #       entry: pytest
+  #       language: system
+  #       pass_filenames: false
+  #       args:
+  #         - "-v"
+  #         - "-n"
+  #         - "auto"

contributing.md

@@ -1 +1,6 @@
-See the [contributing guidelines in 'aeon_blog'](https://github.com/sainsburyWellcomeCentre/aeon_blog#contributing).
+If you wish to contribute to this project, please see and adhere to the [dev practices](https://github.com/sainsburyWellcomeCentre/aeon_blog#dev-practices) outlined in 'aeon_blog'.
+
+Additionally, please ensure the following:
+- Dev dependencies are installed in your Aeon Python environment 
+    - See [./env_config/env_dev.yml](./env_config/env_dev.yml) or equivalently '[project.optional-dependencies]' in [./pyproject.toml](./pyproject.toml).
+- Pre-commit hooks are installed in your Aeon Python environment: run `pre-commit install` in the activated environment

docs/env_setup/local/miniconda_conda_local_setup.md

@@ -1,6 +1,6 @@
 # Local set-up using miniconda and conda
 
-1) Install [miniconda](https://docs.conda.io/en/latest/miniconda.html)
+1. Install [miniconda](https://docs.conda.io/en/latest/miniconda.html)
 	- If prompted with an "Install for", select "Just Me" (instead of "All Users")
 	- Ensure installation is in your home directory:
 		- On Windows: `C:\Users\<your_username>\anaconda3`
@@ -9,16 +9,17 @@
 	- Ensure you add anaconda as a path environment variable (even if it says this option is not recommended)
 	- Ensure you do *not* register anaconda as the default version of Python.
 	- _Note_: These installation settings can always be changed posthoc.
-
-2) Create conda environment and install the code dependencies from the `env.yml` file:
+2. Create conda environment and install the code dependencies from the `env.yml` file:
 ```
 conda update conda
 conda init
-conda env create --file env.yml
+conda env create --file env_config/env.yml
 ```
-
-3) Using the virtual environment:
-
-`conda activate aeon`: activates the virtual environment; any commands now run within this terminal will take place within the virtual environment.
-
-`conda deactivate aeon`: deactivates the virtual environment.
+3. Optionally install development dependencies:
+```
+conda activate aeon
+conda env update --file env_config/env_dev.yml
+```
+4. Using the virtual environment:
+	- `conda activate aeon`: activates the virtual environment; any commands now run within this terminal will take place within the virtual environment.
+	- `conda deactivate aeon`: deactivates the virtual environment.

docs/env_setup/local/pip_venv_local_setup.md

@@ -1,23 +1,22 @@
 # Local set-up using pip and venv
 
-1) Ensure that you have python 3.9 and pip installed on your computer:
+1. Ensure that python >=3.9 and pip are installed (if not, install python >=3.9)
 ```
-python -V         # should return >=3.9
+python -V          # should return >=3.9
 python -m pip -V
 ```
-
-2) Install virtualenv: `python -m pip install virtualenv`
-
-3) Create virtual environment: `python -m venv aeon`
-
-4) Activate the virtual environment and install the code dependencies:
+2. Install virtualenv: `python -m pip install virtualenv`
+3. Create virtual environment: `python -m venv aeon`
+4. Activate the virtual environment and install the code dependencies:
 ```
 source aeon/bin/activate
-python -m pip install -r requirements.txt
+python -m pip install -e .
 ```
-
-5) Using the virtual environment:
-
-`source aeon/bin/activate` activates the virtual environment.
-
-`deactivate` deactivates the virtual environment.
+5. Optionally install the development dependencies:
+```
+source aeon/bin/activate
+python -m pip install -e .[dev]
+```
+6. Using the virtual environment:
+    - `source aeon/bin/activate` activates the virtual environment; any commands now run within this terminal will take place within the virtual environment.
+    - `deactivate` deactivates the virtual environment.