Merge pull request #29 from nens/eli-remove-setuppy-setupcfg

remove setuppy setupcfg, clean up workflows

.github/workflows/lint.yaml

@@ -0,0 +1,24 @@
+name: Lint
+
+on:
+  push:
+    branches:
+      - master
+      - fixes_*
+    tags:
+      - '*'
+  pull_request:
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Run black, flake8, isort
+        uses: pre-commit/[email protected]

.github/workflows/release.yml

@@ -0,0 +1,61 @@
+name: Build and publish
+
+# Run on PR requests. And on master itself.
+on:
+  push:
+    branches:
+      - master  # just build the sdist skip release
+    tags:
+      - "*"
+  pull_request:  # also build on PRs touching some files
+    paths:
+      - ".github/workflows/release.yml"
+      - "MANIFEST.in"
+      - "setup.cfg"
+      - "pyproject.toml"
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Build a source tarball
+        run: |
+          python -m pip install build
+          python -m build
+
+      - uses: actions/upload-artifact@v3
+        with:
+          path: ./dist/*
+          retention-days: 5
+
+  publish:
+    name: Publish on GitHub and PyPI
+    needs: [build]
+    runs-on: ubuntu-latest
+    # release on every tag
+    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/')
+    steps:
+      - uses: actions/download-artifact@v3
+        with:
+          name: artifact
+          path: dist
+
+      - name: Upload Github release
+        id: upload-release-asset
+        uses: softprops/action-gh-release@v2
+
+      - name: Upload Release Assets to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_UPLOAD_TOKEN }}

.github/workflows/tests.yaml

@@ -1,10 +1,11 @@
-name: Tests
+# currently this package has no tests so we just check if it can be installed
+name: Test package installation
 on: [push, workflow_dispatch]
 jobs:
   unittests:
     runs-on: ubuntu-latest
 
-    name: Unit tests, Python ${{ matrix.python-version }}
+    name: Test package installation
 
     steps:
       - name: Check out source repository
@@ -14,6 +15,4 @@ jobs:
         with:
           python-version: "3.10"
       - name: Install packages
-        run: python -m pip install .[test]
-      - name: Run unittests
-        run: python -m pytest
+        run: python -m pip install .

.pre-commit-config.yaml

@@ -0,0 +1,19 @@
+default_language_version:
+  python: python3
+repos:
+- repo: https://github.com/pycqa/isort
+  rev: '5.13.2'
+  hooks:
+  - id: isort
+    exclude: 'settings'
+- repo: https://github.com/psf/black
+  rev: '24.8.0'
+  hooks:
+  - id: black
+    exclude: 'migrations*|urls*|settings*|setup.py'
+- repo: https://github.com/pycqa/flake8
+  rev: '7.1.1'
+  hooks:
+  - id: flake8
+    # NB The "exclude" setting in setup.cfg is ignored by pre-commit
+    exclude: 'migrations*|urls*|settings*'

CHANGES.rst

@@ -7,3 +7,7 @@ Changelog of batch-calculator
 
 - Initial project structure created with cookiecutter and
   https://github.com/nens/cookiecutter-python-template
+
+- Move to pyproject.toml
+
+- Clean up workflows and project structure

batch_calculator/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.dev0"

batch_calculator/process_results.py

@@ -1,25 +1,20 @@
-import click
 import json
 import os
-import pandas
 import shutil
 import zipfile
-
-from batch_calculator.rain_series_simulations import (
-    printProgressBar,
-    api_call,
-)
 from math import floor
 from pathlib import Path
+from typing import Dict, List
+from urllib.request import urlretrieve
+
+import click
+import pandas
 from threedi_api_client import ThreediApi
 from threedi_api_client.openapi.models import SimulationStatus
 from threedi_api_client.versions import V3BetaApi
-from typing import (
-    Dict,
-    List,
-)
 from threedigrid.admin.gridresultadmin import GridH5AggregateResultAdmin
-from urllib.request import urlretrieve
+
+from batch_calculator.rain_series_simulations import api_call, printProgressBar
 
 
 def repetition_time_volumes(weir_results, n, stats=[1, 2, 5, 10]):
@@ -138,9 +133,10 @@ def batch_calculation_statistics(netcdf_dir: Path, gridadmin: str, nr_years: int
 
     if len(nan_results) > 0:
         print(
-            "WARNING: one or more weirs found which have NaN results in their cumulative "
-            "discharge. Please check the nan_rows.json file for more information. "
-            "This file contains weir id and netcdf file where the NaN values are found. "
+            "WARNING: one or more weirs found which have NaN results in their "
+            "cumulative discharge. Please check the nan_rows.json file for more "
+            "information. This file contains weir id and netcdf file where the NaN "
+            "values are found. "
         )
         results_file = netcdf_dir.parent / "nan_rows.json"
         with results_file.open("w") as f:

batch_calculator/rain_series_simulations.py

@@ -1,20 +1,18 @@
-from urllib.request import urlretrieve
-import click
-import numpy as np
 import json
 import os
 import shutil
-import pytz
-import netCDF4 as nc4
 import zipfile
-
-
 from datetime import datetime
 from pathlib import Path
-from sqlalchemy import (
-    create_engine,
-    text,
-)
+from time import sleep
+from typing import Dict, List
+from urllib.request import urlretrieve
+
+import click
+import netCDF4 as nc4
+import numpy as np
+import pytz
+from sqlalchemy import create_engine, text
 from sqlalchemy.orm import Session
 from threedi_api_client import ThreediApi
 from threedi_api_client.files import upload_file
@@ -32,11 +30,6 @@
     UploadEventFile,
 )
 from threedi_api_client.versions import V3BetaApi
-from time import sleep
-from typing import (
-    Dict,
-    List,
-)
 
 RAIN_EVENTS_START_DATE = datetime(1955, 1, 1)
 REQUIRED_AGGREGATION_METHODS = {"cum", "cum_negative", "cum_positive"}
@@ -105,9 +98,17 @@ def validate_sqlite(sqlite_path: Path):
     ).scalar()
 
     if int(database_schema_version) < 222:
-        query = "SELECT timestep, aggregation_method FROM v2_aggregation_settings WHERE flow_variable='discharge'"
+        query = """
+        SELECT timestep, aggregation_method
+        FROM v2_aggregation_settings
+        WHERE flow_variable='discharge';
+        """
     else:
-        query = "SELECT timestep, aggregation_method FROM aggregation_settings WHERE flow_variable='discharge'"
+        query = """
+        SELECT timestep, aggregation_method
+        FROM aggregation_settings
+        WHERE flow_variable='discharge';
+        """
 
     rows = [row for row in session.execute(text(query))]
     timesteps = np.array([row[0] for row in rows])
@@ -292,7 +293,11 @@ def convert_to_netcdf(rain_files_dir: Path) -> List[Dict]:
         #     data=np.array([0]),
         #     dtype=np.int32,
         # )
-        # # one = netcdf.create_dataset("one", np.array([0.0], dtype=np.float64), dtype=np.float64)
+        # # one = netcdf.create_dataset(
+        # #     "one",
+        # #     np.array([0.0], dtype=np.float64),
+        # #     dtype=np.float64
+        # # )
         # time = netcdf.create_dataset("time", data=time, dtype=np.float64)
         # values = netcdf.create_dataset(
         #     "values", data=values_converted, dtype=np.float64
@@ -431,9 +436,9 @@ def create_simulations_from_rain_events(
     rain_files_dir: Path,
 ) -> List[Simulation]:
     """
-    Read start time from rain files filename and create simulations with the corresponding
-    initial state from the DWF runs. Create timeseries rain event from file data.
-    Save created simulations to JSON as fallback.
+    Read start time from rain files filename and create simulations with the
+    corresponding initial state from the DWF runs. Create timeseries rain event
+    from file data. Save created simulations to JSON as fallback.
     """
     rain_event_simulations = []
     warnings = []
@@ -591,7 +596,8 @@ def create_rain_series_simulations(
     Batch rain series calculation consists of 2 parts.
     First part:
         - run simulation in dry state for 3 days
-        - create saved states for every hour in day 3 which will be used as start state for the rain series simulations
+        - create saved states for every hour in day 3 which will be used as
+          start state for the rain series simulations
 
     \b
     Second part:

pyproject.toml

@@ -0,0 +1,43 @@
+[project]
+name = "batch-calculator"
+dynamic = ["version"]
+description = "3Di batch calculations"
+readme = "README.rst"
+license = {text = "MIT"}
+authors = [
+    { name = "Daan van Ingen", email = "[email protected]" },
+]
+keywords = []
+classifiers = [
+    "Framework :: Django",
+    "Programming Language :: Python",
+    "License :: OSI Approved :: MIT License",
+]
+dependencies = [
+    "click",
+    "netCDF4",
+    "pandas",
+    "threedi-api-client>=4.0.0",
+    "threedigrid>=1.0.16",
+]
+
+[project.scripts]
+process-rain-series-results = "batch_calculator.process_results:process_results"
+run-rain-series-simulations = "batch_calculator.rain_series_simulations:create_rain_series_simulations"
+
+[project.urls]
+Repository = "https://github.com/nens/threedi-urban-eia-nl"
+Changelog = "https://github.com/nens/threedi-urban-eia-nl/blob/master/CHANGES.rst"
+
+[tool.setuptools]
+packages = ["batch_calculator"]
+
+[tool.setuptools.dynamic]
+version = {attr = "batch_calculator.__version__"}
+
+[tool.zest-releaser]
+release = false
+
+[tool.isort]
+profile = "black"
+force_alphabetical_sort_within_sections = true

setup.cfg

@@ -1,9 +1,3 @@
-[tool:pytest]
-addopts = --black --flakes --cov --cache-clear batch_calculator
-
-[zest.releaser]
-# Releasing to pypi is done automatically by travis-ci.com (once set up)
-release = no
-
 [flake8]
-max-line-length = 90
+max-line-length = 88
+extend-ignore = E203,E701