Packaging (#8)

* Partial work porting over and updating warn-scraper conventions
* Initial pass at porting usage docs #6
* Add cli list command
* Update README
* Fix Makefile help text
* Commit lockfile
* Add stories page
* Add throttle and misc cleanups to cli and runner
* Add dependency to setup.py
* Clobber obsolete SD scraper test module
* Log agency slug in runner
* Add customizable throttling
* Port and update cache
* Remove doc tests and disable Python and other downstream actions in CI (for now)
* Partial work on ca_san_diego_pd
* Fix/update usage docs

.github/workflows/continuous-deployment.yml

@@ -0,0 +1,163 @@
+name: Testing and distribution
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  pre-commit:
+    name: Lint and format with pre-commit
+    runs-on: ubuntu-latest
+    steps:
+      - id: checkout
+        name: Checkout
+        uses: actions/checkout@v4
+
+      - id: setup-python
+        name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+
+      - id: pre-commit
+        name: Pre-commit
+        uses: pre-commit/[email protected]
+
+#  test-python:
+#    strategy:
+#      matrix:
+#        python: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12",]
+#    name: Test Python code
+#    runs-on: ubuntu-latest
+#    steps:
+#      - id: checkout
+#        name: Checkout
+#        uses: actions/checkout@v4
+#
+#      - id: setup-python
+#        name: Setup Python
+#        uses: actions/setup-python@v5
+#        with:
+#          python-version: '3.9'
+#          cache: 'pipenv'
+#
+#      - id: install-pipenv
+#        name: Install pipenv
+#        run: curl https://raw.githubusercontent.com/pypa/pipenv/master/get-pipenv.py | python
+#        shell: bash
+#
+#      - id: install-python-dependencies
+#        name: Install Python dependencies
+#        run: pipenv install --dev --python=`which python`
+#        shell: bash
+#
+#      - id: run
+#        name: Run tests
+#        run: make test
+#
+#      - id: coverage
+#        name: Coverage report
+#        run: make coverage
+
+#  test-build:
+#    name: Build Python package
+#    runs-on: ubuntu-latest
+#    needs: [test-python]
+#    steps:
+#      - id: checkout
+#        name: Checkout
+#        uses: actions/checkout@v4
+#
+#      - id: setup-python
+#        name: Setup Python
+#        uses: actions/setup-python@v5
+#        with:
+#          python-version: '3.9'
+#          cache: 'pipenv'
+#
+#      - id: install-pipenv
+#        name: Install pipenv
+#        run: curl https://raw.githubusercontent.com/pypa/pipenv/master/get-pipenv.py | python
+#        shell: bash
+#
+#      - id: install-python-dependencies
+#        name: Install Python dependencies
+#        run: pipenv install --dev --python=`which python`
+#        shell: bash
+#
+#      - id: build
+#        name: Build release
+#        run: make build-release
+#
+#      - id: check
+#        name: Check release
+#        run: make check-release
+#
+#      - id: save
+#        name: Save artifact
+#        uses: actions/upload-artifact@v4
+#        with:
+#          name: test-release-${{ github.run_number }}
+#          path: ./dist
+#          if-no-files-found: error
+#
+#  test-release:
+#    name: Test PyPI release
+#    runs-on: ubuntu-latest
+#    needs: [test-build]
+#    if: startsWith(github.ref, 'refs/tags') == 0
+#    steps:
+#      - id: setup-python
+#        name: Setup Python
+#        uses: actions/setup-python@v5
+#        with:
+#          python-version: '3.9'
+#
+#      - id: fetch
+#        name: Fetch artifact
+#        uses: actions/download-artifact@v4
+#        with:
+#          name: test-release-${{ github.run_number }}
+#          path: ./dist
+#
+#      - id: publish
+#        name: Publish release
+#        uses: pypa/gh-action-pypi-publish@release/v1
+#        env:
+#          PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
+#        if: env.PYPI_API_TOKEN != null
+#        with:
+#          user: __token__
+#          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
+#          repository-url: https://test.pypi.org/legacy/
+#          verbose: true
+#          verify_metadata: false
+#
+#  tag-release:
+#    name: Tagged PyPI release
+#    runs-on: ubuntu-latest
+#    needs: [test-build]
+#    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+#    steps:
+#      - id: setup-python
+#        name: Setup Python
+#        uses: actions/setup-python@v5
+#        with:
+#          python-version: '3.9'
+#
+#      - id: fetch
+#        name: Fetch artifact
+#        uses: actions/download-artifact@v4
+#        with:
+#          name: test-release-${{ github.run_number }}
+#          path: ./dist
+#
+#      - id: publish
+#        name: Publish release
+#        uses: pypa/gh-action-pypi-publish@release/v1
+#        with:
+#          user: __token__
+#          password: ${{ secrets.PYPI_API_TOKEN }}
+#          verbose: true
+#          verify_metadata: false
+#

.gitignore

@@ -1 +1,139 @@
-files/*
+# Our download dir
+.clean-scraper
+
+# Data folders
+logs/
+old_data/
+data/
+archive_data
+process/
+
+# Apple crap
+.DS_Store
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# Swap
+[._]*.s[a-v][a-z]
+!*.svg  # comment out if you don't need vector files
+[._]*.sw[a-p]
+[._]s[a-rt-v][a-z]
+[._]ss[a-gi-z]
+[._]sw[a-p]
+
+# Session
+Session.vim
+Sessionx.vim
+
+# Temporary
+.netrwhist
+*~
+# Auto-generated tag files
+tags
+# Persistent undo
+[._]*.un~
+
+.vscode

.pre-commit-config.yaml

@@ -0,0 +1,55 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: check-added-large-files
+        args: ['--maxkb=10000']
+    -   id: check-byte-order-marker
+    -   id: check-case-conflict
+    -   id: check-json
+    -   id: mixed-line-ending
+
+-   repo: https://github.com/psf/black
+    rev: 24.3.0
+    hooks:
+    -   id: black
+
+-   repo: https://github.com/asottile/blacken-docs
+    rev: 1.16.0
+    hooks:
+    -   id: blacken-docs
+        additional_dependencies: [black]
+
+-   repo: https://github.com/timothycrosley/isort
+    rev: 5.13.2
+    hooks:
+    -   id: isort
+
+-   repo: https://github.com/PyCQA/flake8
+    rev: 7.0.0
+    hooks:
+    - id: flake8
+      additional_dependencies:
+        - flake8-docstrings
+        - flake8-bugbear
+
+-   repo: https://github.com/asottile/pyupgrade
+    rev: v3.15.2
+    hooks:
+    -   id: pyupgrade
+        args: [--py37-plus]
+
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: 'v1.9.0'  # Use the sha / tag you want to point at
+    hooks:
+    -   id: mypy
+        additional_dependencies:
+          - types-requests
+          - types-retry
+          - types-beautifulsoup4
+          - types-openpyxl