Merge pull request #4 from datastax-labs/gbk-harmonise-build

Adds wrapper scripts for tox and poetry, which create a venv, install the respective tool, and run it. This removes any requirement on the caller to mess around with setting up venvs, while not preventing their usage should developers wish to do so.

It also enforces import ordering and removal of redundant imports using autoflake and isort.

.github/workflows/python-app.yml

@@ -16,23 +16,11 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
+
     - name: Set up Python 3.8
       uses: actions/setup-python@v2
       with:
-        python-version: 3.8
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install poetry flake8 pytest
-        poetry install
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-    - name: Lint with flake8
-      run: |
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        # Black declares E203 is not PEP8 compliant: https://github.com/psf/black/issues/315#issuecomment-395457972
-        flake8 . --count --ignore=E203 --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-    - name: Test with pytest
-      run: |
-        poetry run pytest tests
+        python-version: 3.8.x
+
+    - name: Run CI
+      run: ./toxw

CONTRIBUTING.md

@@ -1,46 +1,46 @@
 # Setting up for development
 
-* [Install poetry](https://python-poetry.org/docs/#installation)
+* Ensure that `python3` points to a version of python >= 3.8 (`python3 --version` will tell you).  If it does not, use [pyenv](https://github.com/pyenv/pyenv) to both install a recent python version and make it your current python.
 
-* If you want to maintain your own virtualenv, install pyenv and use pyenv virtualenv to create and manage one.  Poetry will automatically find any active virtualenv and use that.
+* There are two wrappers (`poetryw` and `toxw`) that install and run the correct versions of [poetry](https://python-poetry.org) and [tox](https://tox.wiki) for you.
 
 * Run poetry to install dependencies:
 
 ```
-poetry install
+./poetryw install
 ```
 
 * Run the development version of hunter using poetry:
 
 ```
-poetry run hunter ...
+./poetryw run hunter ...
 ```
 
 See the [poetry docs](https://python-poetry.org/docs) for more.
 
 # Running tests
 
 ```
-poetry run pytest tests
+./poetryw run pytest
 ```
 
 ...or using [tox](https://tox.readthedocs.io/):
 
 ```
-ci-tools/tox-bootstrap
+./toxw
 ```
 
 # Linting and formatting
 
-Code-style is enforced using [black](https://black.readthedocs.io/).  Linting is automatically applied when tox runs tests; if linting fails, you can fix it with:
+Code-style is enforced using [black](https://black.readthedocs.io/) and [flake8](https://flake8.pycqa.org/); import optimisation is handled by [isort](https://pycqa.github.io/isort/) and [autoflake](https://pypi.org/project/autoflake/).  Linting is automatically applied when tox runs tests; if linting fails, you can fix trivial problems with:
 
 ```
-ci-tools/tox-bootstrap -e format
+./toxw -e format
 ```
 
 
 # Build a docker image
 
 ```
-ci-tools/tox-bootstrap -e docker-build
+./toxw -e docker-build
 ```

ci-tools/wrappers.sh

@@ -0,0 +1,65 @@
+# Support functions for bootstrapping python tools
+
+python=${PYTHON:-python3}
+
+# set -x would generate a lot of noise when we activate the venv, so
+# we use this hand-crafted equivalent here:
+run()
+{
+    echo "$@" 1>&2
+    "$@"
+}
+
+build_dir="$thisdir/build"
+venvs_dir="$build_dir/venvs"
+bin_dir="$build_dir/wrappers/bin"
+
+install_tool()
+{
+    tool="$1"
+    shift
+
+    pip_spec="$1"
+    shift
+
+    test -f "$bin_dir/$tool" && return
+
+    run mkdir -p "$venvs_dir" "$bin_dir"
+
+    venv="$venvs_dir/$tool"
+
+    run "$python" -m venv "$venv"
+
+    # Run in a subshell to prevent the activate/deactivate steps
+    # interfering with pyenv
+    (
+        run source "$venv/bin/activate"
+        run "$python" -m pip install -qqq --upgrade pip
+        run "$python" -m pip install -qqq $pip_spec
+        run ln -fs "../../venvs/$tool/bin/$tool" "$bin_dir/$tool"
+        run deactivate
+    )
+}
+
+run_tool()
+{
+    tool="$1"
+    shift
+
+    # Ensure that the tool has access to all the bootstrapped tools
+    PATH="$bin_dir:$PATH"
+
+    exec "$bin_dir/$tool" "$@"
+}
+
+install_and_run_tool()
+{
+    tool="$1"
+    shift
+
+    pip_spec="$1"
+    shift
+
+    install_tool "$tool" "$pip_spec"
+    run_tool "$tool" "$@"
+}

hunter/analysis.py

@@ -1,6 +1,5 @@
 from dataclasses import dataclass
-from typing import Iterable, Reversible
-from typing import List
+from typing import Iterable, List, Reversible
 
 import numpy as np
 from scipy.stats import ttest_ind_from_stats

hunter/config.py

@@ -82,7 +82,7 @@ def load_config_from(config_file: Path) -> Config:
         yaml = YAML(typ="safe")
         config = yaml.load(content)
         """
-        if Grafana configs not explicitly set in yaml file, default to same as Graphite 
+        if Grafana configs not explicitly set in yaml file, default to same as Graphite
         server at port 3000
         """
         graphite_config = None

hunter/csv_options.py

@@ -1,5 +1,4 @@
 import enum
-
 from dataclasses import dataclass
 
 

hunter/graphite.py

@@ -4,7 +4,7 @@
 from dataclasses import dataclass
 from datetime import datetime
 from logging import info
-from typing import Dict, List, Optional, Iterable
+from typing import Dict, Iterable, List, Optional
 
 from hunter.data_selector import DataSelector
 from hunter.util import parse_datetime

hunter/importer.py

@@ -4,24 +4,24 @@
 from dataclasses import dataclass
 from datetime import datetime, timedelta
 from pathlib import Path
-from typing import List, Optional, Dict
+from typing import Dict, List, Optional
 
 from hunter.config import Config
 from hunter.data_selector import DataSelector
 from hunter.graphite import DataPoint, Graphite, GraphiteError
-from hunter.series import Series, Metric
+from hunter.series import Metric, Series
 from hunter.test_config import (
+    CsvMetric,
     CsvTestConfig,
-    TestConfig,
     GraphiteTestConfig,
-    CsvMetric,
     HistoStatTestConfig,
+    TestConfig,
 )
 from hunter.util import (
-    merge_sorted,
-    parse_datetime,
     DateFormatError,
     format_timestamp,
+    merge_sorted,
+    parse_datetime,
     resolution,
     round,
 )

hunter/main.py

@@ -4,27 +4,23 @@
 import sys
 from dataclasses import dataclass
 from datetime import datetime, timedelta
-from typing import Dict, Optional, List
+from typing import Dict, List, Optional
 
 import pytz
 from slack_sdk import WebClient
 
 from hunter import config
 from hunter.attributes import get_back_links
-from hunter.config import ConfigError, Config
+from hunter.config import Config, ConfigError
 from hunter.data_selector import DataSelector
-from hunter.grafana import GrafanaError, Grafana, Annotation
+from hunter.grafana import Annotation, Grafana, GrafanaError
 from hunter.graphite import GraphiteError
 from hunter.importer import DataImportError, Importers
 from hunter.report import Report, ReportType
-from hunter.series import (
-    AnalysisOptions,
-    compare,
-    AnalyzedSeries,
-)
-from hunter.slack import SlackNotifier, NotificationError
-from hunter.test_config import TestConfigError, TestConfig, GraphiteTestConfig
-from hunter.util import parse_datetime, DateFormatError, interpolate
+from hunter.series import AnalysisOptions, AnalyzedSeries, compare
+from hunter.slack import NotificationError, SlackNotifier
+from hunter.test_config import GraphiteTestConfig, TestConfig, TestConfigError
+from hunter.util import DateFormatError, interpolate, parse_datetime
 
 
 @dataclass

hunter/report.py

@@ -4,7 +4,7 @@
 
 from tabulate import tabulate
 
-from hunter.series import Series, ChangePointGroup
+from hunter.series import ChangePointGroup, Series
 from hunter.util import format_timestamp, insert_multiple, remove_common_prefix
 
 

hunter/series.py

@@ -2,15 +2,15 @@
 from dataclasses import dataclass
 from datetime import datetime
 from itertools import groupby
-from typing import Dict, List, Optional, Iterable
+from typing import Dict, Iterable, List, Optional
 
 import numpy as np
 
 from hunter.analysis import (
-    fill_missing,
-    compute_change_points,
     ComparativeStats,
     TTestSignificanceTester,
+    compute_change_points,
+    fill_missing,
 )
 
 

hunter/slack.py

@@ -7,7 +7,7 @@
 from slack_sdk import WebClient
 
 from hunter.data_selector import DataSelector
-from hunter.series import ChangePointGroup, AnalyzedSeries
+from hunter.series import AnalyzedSeries, ChangePointGroup
 
 
 @dataclass