diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 00959ef..409bfdb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ^(docs|mcbackend/testdata)/ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.2.0 + rev: v4.5.0 hooks: - id: check-merge-conflict - id: check-toml @@ -10,28 +10,28 @@ repos: - id: end-of-file-fixer - id: requirements-txt-fixer - repo: https://github.com/PyCQA/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort name: isort args: ["--profile", "black"] - repo: https://github.com/asottile/pyupgrade - rev: v2.32.1 + rev: v3.15.0 hooks: - id: pyupgrade args: [--py37-plus] - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 24.1.1 hooks: - id: black - repo: https://github.com/PyCQA/pylint - rev: v2.13.8 + rev: v3.0.3 hooks: - id: pylint args: [--rcfile=.pylintrc] exclude: (test_*|mcbackend/meta.py|mcbackend/npproto/) files: ^mcbackend/ - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.991 + rev: v1.8.0 hooks: - id: mypy diff --git a/mcbackend/__init__.py b/mcbackend/__init__.py index 7ffdf23..40a0877 100644 --- a/mcbackend/__init__.py +++ b/mcbackend/__init__.py @@ -1,6 +1,7 @@ """ A framework agnostic implementation for storage of MCMC draws. """ + from .backends.numpy import NumPyBackend from .core import Backend, Chain, Run from .meta import ChainMeta, Coordinate, DataVariable, ExtendedValue, RunMeta, Variable diff --git a/mcbackend/backends/clickhouse.py b/mcbackend/backends/clickhouse.py index a36d437..31e5169 100644 --- a/mcbackend/backends/clickhouse.py +++ b/mcbackend/backends/clickhouse.py @@ -1,6 +1,7 @@ """ This module implements a backend for sample storage in ClickHouse. """ + import base64 import logging import time @@ -45,6 +46,10 @@ } +class ClickHouseBackendError(Exception): + """Something bad happened in the ClickHouse backend.""" + + def create_runs_table(client: clickhouse_driver.Client): query = """ CREATE TABLE IF NOT EXISTS runs ( @@ -79,7 +84,7 @@ def create_chain_table(client: clickhouse_driver.Client, meta: ChainMeta, rmeta: # Check that it does not already exist cid = chain_id(meta) if client.execute(f"SHOW TABLES LIKE '{cid}';"): - raise Exception(f"A table for {cid} already exists.") + raise ClickHouseBackendError(f"A table for {cid} already exists.") # Create a table with columns corresponding to the model variables columns = [] @@ -235,7 +240,7 @@ def _get_row_at( query = f"SELECT (`{names}`,) FROM {self.cid} WHERE _draw_idx={idx};" data = self._client.execute(query) if not data: - raise Exception(f"No record found for draw index {idx}.") + raise ClickHouseBackendError(f"No record found for draw index {idx}.") result = dict(zip(var_names, data[0][0])) return result @@ -363,7 +368,10 @@ def __init__( raise ValueError("Either a `client` or a `client_fn` must be provided.") if client_fn is None: - client_fn = lambda: client + + def client_fn(): + return client + if client is None: client = client_fn() @@ -381,11 +389,11 @@ def init_run(self, meta: RunMeta) -> ClickHouseRun: else: created_at = datetime.now().astimezone(timezone.utc) query = "INSERT INTO runs (created_at, rid, proto) VALUES" - params = dict( - created_at=created_at, - rid=meta.rid, - proto=base64.encodebytes(bytes(meta)).decode("ascii"), - ) + params = { + "created_at": created_at, + "rid": meta.rid, + "proto": base64.encodebytes(bytes(meta)).decode("ascii"), + } self._client.execute(query, [params]) return ClickHouseRun(meta, client_fn=self._client_fn, created_at=created_at) @@ -407,7 +415,9 @@ def get_run(self, rid: str) -> ClickHouseRun: {"rid": rid}, ) if len(rows) != 1: - raise Exception(f"Unexpected number of {len(rows)} results for rid='{rid}'.") + raise ClickHouseBackendError( + f"Unexpected number of {len(rows)} results for rid='{rid}'." + ) data = base64.decodebytes(rows[0][2].encode("ascii")) meta = RunMeta().parse(data) return ClickHouseRun( diff --git a/mcbackend/backends/numpy.py b/mcbackend/backends/numpy.py index 9c4e55e..cb37f06 100644 --- a/mcbackend/backends/numpy.py +++ b/mcbackend/backends/numpy.py @@ -1,6 +1,7 @@ """ This backend holds draws in memory, managing them via NumPy arrays. """ + import math from typing import Dict, List, Mapping, Optional, Sequence, Tuple @@ -110,7 +111,7 @@ class NumPyRun(Run): """An MCMC run where samples are kept in memory.""" def __init__(self, meta: RunMeta, *, preallocate: int) -> None: - self._settings = dict(preallocate=preallocate) + self._settings = {"preallocate": preallocate} self._chains: List[NumPyChain] = [] super().__init__(meta) @@ -128,9 +129,7 @@ class NumPyBackend(Backend): """An in-memory backend using NumPy.""" def __init__(self, preallocate: int = 1_000) -> None: - self._settings = dict( - preallocate=preallocate, - ) + self._settings = {"preallocate": preallocate} super().__init__() def init_run(self, meta: RunMeta) -> NumPyRun: diff --git a/mcbackend/core.py b/mcbackend/core.py index 85970b8..6f627e7 100644 --- a/mcbackend/core.py +++ b/mcbackend/core.py @@ -1,6 +1,7 @@ """ Module with metadata structures and abstract classes. """ + import collections import logging from typing import Dict, List, Mapping, Optional, Sequence, Sized, TypeVar, Union, cast @@ -26,6 +27,10 @@ __all__ = ("is_rigid", "chain_id", "Chain", "Run", "Backend") +class ChainError(Exception): + """Something is not right in one chain.""" + + def is_rigid(nshape: Optional[Shape]): """Determines wheather the shape is constant. @@ -118,7 +123,7 @@ def __len__(self) -> int: ]: for var in items: return len(method(var.name)) - raise Exception("This chain has no variables or sample stats.") + raise ChainError("This chain has no variables or sample stats.") @property def cid(self) -> str: diff --git a/mcbackend/npproto/utils.py b/mcbackend/npproto/utils.py index 2dd0536..c4d07a7 100644 --- a/mcbackend/npproto/utils.py +++ b/mcbackend/npproto/utils.py @@ -1,6 +1,7 @@ """ Helper functions such as converters between ``ndarray`` and ``Ndarray``. """ + import numpy from . import Ndarray diff --git a/mcbackend/utils.py b/mcbackend/utils.py index 5af8126..b7da676 100644 --- a/mcbackend/utils.py +++ b/mcbackend/utils.py @@ -1,4 +1,5 @@ """Contains helper functions that are independent of McBackend components.""" + from typing import Sequence import numpy as np diff --git a/protobufs/generate.py b/protobufs/generate.py index 8eb4472..817dddf 100644 --- a/protobufs/generate.py +++ b/protobufs/generate.py @@ -2,6 +2,7 @@ This script regenerates the `meta.py` module from protobuf definitions of core metadata structures. """ + import pathlib import shutil import subprocess