From 230790fa0eec252799564fee3f61bd06beb0c3d1 Mon Sep 17 00:00:00 2001 From: Robert Hafner Date: Sat, 10 Feb 2024 14:37:32 -0600 Subject: [PATCH 1/3] cli fixes, ruff, remote setup.cfg/setup.py, setuptools_scm * Add ruff * Remove black and isort * setuptools_scm * Remove versioneer * Fix CLI typing --- .github/workflows/black.yaml | 6 - .github/workflows/dapperdata.yaml | 6 - .github/workflows/isort.yaml | 24 -- .github/workflows/mypy.yaml | 6 - .github/workflows/pypi.yaml | 6 - .github/workflows/pytest.yaml | 9 +- .github/workflows/ruff.yaml | 18 + .gitignore | 4 +- .pre-commit-config.yaml | 23 + .python-version | 2 +- README.md | 5 +- makefile | 27 +- pyproject.toml | 73 ++-- quasiqueue/__init__.py | 12 +- quasiqueue/_version.py | 677 ------------------------------ quasiqueue/builder.py | 4 +- quasiqueue/cli.py | 20 +- quasiqueue/runner.py | 4 +- quasiqueue/settings.py | 19 +- setup.cfg | 47 --- setup.py | 12 - tests/test_quasiqueue.py | 5 +- 22 files changed, 151 insertions(+), 858 deletions(-) delete mode 100644 .github/workflows/isort.yaml create mode 100644 .github/workflows/ruff.yaml create mode 100644 .pre-commit-config.yaml delete mode 100644 quasiqueue/_version.py delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml index a63d18a..724ce77 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/black.yaml @@ -9,13 +9,7 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Get Python Version from File - id: python_version - run: echo "version=$(cat VERSION)" >> $GITHUB_OUTPUT - - uses: actions/setup-python@v5 - with: - python-version: ${{ steps.python_version.outputs.version }} - name: Install Dependencies run: make install diff --git a/.github/workflows/dapperdata.yaml b/.github/workflows/dapperdata.yaml index 587189f..cb6aa7a 100644 --- a/.github/workflows/dapperdata.yaml +++ b/.github/workflows/dapperdata.yaml @@ -9,13 +9,7 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Get Python Version from File - id: python_version - run: echo "version=$(cat VERSION)" >> $GITHUB_OUTPUT - - uses: actions/setup-python@v5 - with: - python-version: ${{ steps.python_version.outputs.version }} - name: Install Dependencies run: make install diff --git a/.github/workflows/isort.yaml b/.github/workflows/isort.yaml deleted file mode 100644 index 17ecadb..0000000 --- a/.github/workflows/isort.yaml +++ /dev/null @@ -1,24 +0,0 @@ -name: Import Sorting - -on: - push: - -jobs: - isort: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Get Python Version from File - id: python_version - run: echo "version=$(cat VERSION)" >> $GITHUB_OUTPUT - - - uses: actions/setup-python@v5 - with: - python-version: ${{ steps.python_version.outputs.version }} - - - name: Install Dependencies - run: make install - - - name: Test Formatting - run: make isort_check diff --git a/.github/workflows/mypy.yaml b/.github/workflows/mypy.yaml index d2cad05..70e6c29 100644 --- a/.github/workflows/mypy.yaml +++ b/.github/workflows/mypy.yaml @@ -9,13 +9,7 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Get Python Version from File - id: python_version - run: echo "version=$(cat VERSION)" >> $GITHUB_OUTPUT - - uses: actions/setup-python@v5 - with: - python-version: ${{ steps.python_version.outputs.version }} - name: Install Dependencies run: make install diff --git a/.github/workflows/pypi.yaml b/.github/workflows/pypi.yaml index 20284d1..a95fc92 100644 --- a/.github/workflows/pypi.yaml +++ b/.github/workflows/pypi.yaml @@ -16,13 +16,7 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Get Python Version from File - id: python_version - run: echo "version=$(cat VERSION)" >> $GITHUB_OUTPUT - - uses: actions/setup-python@v5 - with: - python-version: ${{ steps.python_version.outputs.version }} - name: Install Dependencies run: make install diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index aa803bd..7fa7ad6 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -9,16 +9,15 @@ env: jobs: pytest: runs-on: ubuntu-latest + strategy: + matrix: + python_version: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 - - name: Get Python Version from File - id: python_version - run: echo "version=$(cat VERSION)" >> $GITHUB_OUTPUT - - uses: actions/setup-python@v5 with: - python-version: ${{ steps.python_version.outputs.version }} + python-version: "${{ matrix.python_version }}" - name: Install Dependencies run: make install diff --git a/.github/workflows/ruff.yaml b/.github/workflows/ruff.yaml new file mode 100644 index 0000000..3c5d756 --- /dev/null +++ b/.github/workflows/ruff.yaml @@ -0,0 +1,18 @@ +name: Ruff Validation + +on: + push: + +jobs: + ruff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + + - name: Install Dependencies + run: make install + + - name: Test Formatting + run: make ruff_check diff --git a/.gitignore b/.gitignore index 6769e21..96b0d88 100644 --- a/.gitignore +++ b/.gitignore @@ -157,4 +157,6 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ \ No newline at end of file +#.idea/ + +quasiqueue/_version.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..70c29be --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,23 @@ +repos: + - repo: local + hooks: + - id: pytest + name: pytest + entry: make pytest + language: system + pass_filenames: false + - id: ruff + name: ruff + entry: make ruff_check + language: system + pass_filenames: false + - id: black + name: black + entry: make black_check + language: system + pass_filenames: false + - id: mypy + name: mypy + entry: make mypy_check + language: system + pass_filenames: false diff --git a/.python-version b/.python-version index 371cfe3..e4fba21 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.11.1 +3.12 diff --git a/README.md b/README.md index 9d8d92b..99b135d 100644 --- a/README.md +++ b/README.md @@ -241,13 +241,12 @@ This method is simple, but the downside is that you lose the environment variabl ```python from quasiqueue import Settings, QuasiQueue +from pydantic_settings import SettingsConfigDict class MySettings(Settings) + model_config = SettingsConfigDict(env_prefix="MY_QUEUE_") lookup_block_size: int = 50 - class Config: - prefix="MY_QUEUE_" - QuasiQueue( "MyQueue", reader=reader, diff --git a/makefile b/makefile index b64a5ff..50021cc 100644 --- a/makefile +++ b/makefile @@ -43,21 +43,24 @@ pip: $(PYTHON_VENV) $(PACKAGE_CHECK): $(PYTHON_VENV) $(PYTHON) -m pip install -e .[dev] +.PHONY: pre-commit +pre-commit: + pre-commit install # # Formatting # -.PHONY: pretty -pretty: black_fixes isort_fixes dapperdata_fixes +.PHONY: chores +chores: ruff_fixes black_fixes dapperdata_fixes + +.PHONY: ruff_fixes +ruff_fixes: + $(PYTHON) -m ruff . --fix .PHONY: black_fixes black_fixes: - $(PYTHON) -m black . - -.PHONY: isort_fixes -isort_fixes: - $(PYTHON) -m isort . + $(PYTHON) -m ruff format . .PHONY: dapperdata_fixes dapperdata_fixes: @@ -69,7 +72,7 @@ dapperdata_fixes: # .PHONY: tests -tests: install pytest isort_check black_check mypy_check dapperdata_check +tests: install pytest ruff_check black_check mypy_check dapperdata_check .PHONY: pytest pytest: @@ -79,13 +82,13 @@ pytest: pytest_loud: $(PYTHON) -m pytest -s --cov=./${PACKAGE_SLUG} --cov-report=term-missing tests -.PHONY: isort_check -isort_check: - $(PYTHON) -m isort --check-only . +.PHONY: ruff_check +ruff_check: + $(PYTHON) -m ruff check .PHONY: black_check black_check: - $(PYTHON) -m black . --check + $(PYTHON) -m ruff format . --check .PHONY: mypy_check mypy_check: diff --git a/pyproject.toml b/pyproject.toml index 88ab07b..837364c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,29 +1,52 @@ [build-system] -requires = ["setuptools>=61.0", "versioneer>=0.25"] build-backend = "setuptools.build_meta" +requires = ["setuptools>=67.0", "setuptools_scm[toml]>=7.1"] -[tool.isort] -profile = "black" -skip = [".venv"] -src_paths = ["quasiqueue", "tests"] - -[tool.black] -exclude = ''' -( - /( - \.eggs # exclude a few common directories in the - | \.git # root of the project - | \.hg - | \.mypy_cache - | \.tox - | \.venv - | \.env - | _build - | buck-out - | build - | dist - ) -) -''' +[project] +authors = [{"name" = "Robert Hafner"}] +dependencies = [ + "psutil", + "pydantic~=2.0", + "pydantic-settings", + "typer>=0.9.0", +] +description = "A Simple High Performance Multiprocess Queue" +dynamic = ["version"] +license = {"file" = "LICENSE"} +name = "quasiqueue" +readme = {file = "README.md", content-type = "text/markdown"} + +[project.optional-dependencies] +dev = [ + "build", + "dapperdata", + "glom", + "mypy", + "pytest", + "pytest-cov", + "pytest-pretty", + "ruamel.yaml", + "ruff", + "toml-sort", + "types-psutil", +] + +[project.scripts] +quasiqueue = "quasiqueue.cli:app" + +[tool.ruff] +exclude = [".venv", "./quasiqueue/_version.py"] line-length = 120 -target-version = ['py38', 'py39', 'py310', 'py311'] + +[tool.setuptools.dynamic] +readme = {file = ["README.md"]} + +[tool.setuptools.package-data] +quasiqueue = ["py.typed"] + +[tool.setuptools.packages] +find = {} + +[tool.setuptools_scm] +fallback_version = "0.0.0-dev" +write_to = "quasiqueue/_version.py" diff --git a/quasiqueue/__init__.py b/quasiqueue/__init__.py index 461008a..05781eb 100644 --- a/quasiqueue/__init__.py +++ b/quasiqueue/__init__.py @@ -1,7 +1,9 @@ -from ._version import get_versions +try: + from . import _version -__version__ = get_versions()["version"] -del get_versions + __version__ = _version.__version__ +except: # noqa: E722 + __version__ = "0.0.0-dev" -from .runner import QueueRunner as QuasiQueue -from .settings import Settings +from .runner import QueueRunner as QuasiQueue # noqa: F401 +from .settings import Settings # noqa: F401 diff --git a/quasiqueue/_version.py b/quasiqueue/_version.py deleted file mode 100644 index d2cf58c..0000000 --- a/quasiqueue/_version.py +++ /dev/null @@ -1,677 +0,0 @@ -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. -# Generated by versioneer-0.27 -# https://github.com/python-versioneer/python-versioneer - -"""Git implementation of _version.py.""" - -import errno -import functools -import os -import re -import subprocess -import sys -from typing import Callable, Dict - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "$Format:%d$" - git_full = "$Format:%H$" - git_date = "$Format:%ci$" - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "pep440" - cfg.tag_prefix = "" - cfg.parentdir_prefix = "" - cfg.versionfile_source = "_version.py" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY: Dict[str, str] = {} -HANDLERS: Dict[str, Dict[str, Callable]] = {} - - -def register_vcs_handler(vcs, method): # decorator - """Create decorator to mark a method as the handler of a VCS.""" - - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): - """Call the given command(s).""" - assert isinstance(commands, list) - process = None - - popen_kwargs = {} - if sys.platform == "win32": - # This hides the console window if pythonw.exe is used - startupinfo = subprocess.STARTUPINFO() - startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW - popen_kwargs["startupinfo"] = startupinfo - - for command in commands: - try: - dispcmd = str([command] + args) - # remember shell=False, so use git.cmd on windows, not just git - process = subprocess.Popen( - [command] + args, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr else None), - **popen_kwargs, - ) - break - except OSError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None, None - stdout = process.communicate()[0].strip().decode() - if process.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, process.returncode - return stdout, process.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for _ in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return { - "version": dirname[len(parentdir_prefix) :], - "full-revisionid": None, - "dirty": False, - "error": None, - "date": None, - } - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print("Tried directories %s but none started with prefix %s" % (str(rootdirs), parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - with open(versionfile_abs, "r") as fobj: - for line in fobj: - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - except OSError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if "refnames" not in keywords: - raise NotThisMethod("Short version file found") - date = keywords.get("date") - if date is not None: - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = {r.strip() for r in refnames.strip("()").split(",")} - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)} - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = {r for r in refs if re.search(r"\d", r)} - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix) :] - # Filter out refs that exactly match prefix or that don't start - # with a number once the prefix is stripped (mostly a concern - # when prefix is '') - if not re.match(r"\d", r): - continue - if verbose: - print("picking %s" % r) - return { - "version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": None, - "date": date, - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return { - "version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": "no suitable tags", - "date": None, - } - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - # GIT_DIR can interfere with correct operation of Versioneer. - # It may be intended to be passed to the Versioneer-versioned project, - # but that should not change where we get our version from. - env = os.environ.copy() - env.pop("GIT_DIR", None) - runner = functools.partial(runner, env=env) - - _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=not verbose) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = runner( - GITS, ["describe", "--tags", "--dirty", "--always", "--long", "--match", f"{tag_prefix}[[:digit:]]*"], cwd=root - ) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) - # --abbrev-ref was added in git-1.6.3 - if rc != 0 or branch_name is None: - raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") - branch_name = branch_name.strip() - - if branch_name == "HEAD": - # If we aren't exactly on a branch, pick a branch which represents - # the current commit. If all else fails, we are on a branchless - # commit. - branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) - # --contains was added in git-1.5.4 - if rc != 0 or branches is None: - raise NotThisMethod("'git branch --contains' returned error") - branches = branches.split("\n") - - # Remove the first line if we're running detached - if "(" in branches[0]: - branches.pop(0) - - # Strip off the leading "* " from the list of branches. - branches = [branch[2:] for branch in branches] - if "master" in branches: - branch_name = "master" - elif not branches: - branch_name = None - else: - # Pick the first branch that is returned. Good or bad. - branch_name = branches[0] - - pieces["branch"] = branch_name - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[: git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) - if not mo: - # unparsable. Maybe git-describe is misbehaving? - pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % (full_tag, tag_prefix) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix) :] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) - pieces["distance"] = len(out.split()) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip() - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_branch(pieces): - """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . - - The ".dev0" means not master branch. Note that .dev0 sorts backwards - (a feature branch will appear "older" than the master branch). - - Exceptions: - 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0" - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def pep440_split_post(ver): - """Split pep440 version string at the post-release segment. - - Returns the release segments before the post-release and the - post-release version number (or -1 if no post-release segment is present). - """ - vc = str.split(ver, ".post") - return vc[0], int(vc[1] or 0) if len(vc) == 2 else None - - -def render_pep440_pre(pieces): - """TAG[.postN.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post0.devDISTANCE - """ - if pieces["closest-tag"]: - if pieces["distance"]: - # update the post release segment - tag_version, post_version = pep440_split_post(pieces["closest-tag"]) - rendered = tag_version - if post_version is not None: - rendered += ".post%d.dev%d" % (post_version + 1, pieces["distance"]) - else: - rendered += ".post0.dev%d" % (pieces["distance"]) - else: - # no commits, use the tag as the version - rendered = pieces["closest-tag"] - else: - # exception #1 - rendered = "0.post0.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_post_branch(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . - - The ".dev0" means not master branch. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return { - "version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None, - } - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-branch": - rendered = render_pep440_branch(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-post-branch": - rendered = render_pep440_post_branch(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return { - "version": rendered, - "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], - "error": None, - "date": pieces.get("date"), - } - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, verbose) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for _ in cfg.versionfile_source.split("/"): - root = os.path.dirname(root) - except NameError: - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None, - } - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", - "date": None, - } diff --git a/quasiqueue/builder.py b/quasiqueue/builder.py index 6a9d3d9..2b1943b 100644 --- a/quasiqueue/builder.py +++ b/quasiqueue/builder.py @@ -13,7 +13,6 @@ def __init__(self, queue, settings, writer): self.closed = False async def populate(self, max=50): - self.clean_history() # Writers can be expensive but cheaper when pulling bulk records. @@ -45,13 +44,12 @@ async def populate(self, max=50): successful_adds += 1 if successful_adds >= max: return True - except Full as e: + except Full: logging.debug("Queue has reached max size.") return False def add_to_queue(self, id): if id in self.last_queued: - logging.debug(f"ID {id} is in last_queued") logging.debug(time.time()) logging.debug(self.last_queued[id] + self.settings.prevent_requeuing_time) diff --git a/quasiqueue/cli.py b/quasiqueue/cli.py index f6aaa32..830e394 100644 --- a/quasiqueue/cli.py +++ b/quasiqueue/cli.py @@ -1,11 +1,12 @@ import asyncio from importlib import import_module from logging import getLogger -from typing import Callable +from typing import Annotated, Callable import click import typer +from . import __version__ from .runner import QueueRunner logger = getLogger(__name__) @@ -13,27 +14,33 @@ def get_function_from_string(mod_path: str) -> Callable: - if not ":" in mod_path: + if ":" not in mod_path: raise ValueError("Module paths require variable.") try: module_path, variable = mod_path.split(":") module = import_module(module_path) except: logger.exception(f"Unable to load module from path: {mod_path}") + raise return getattr(module, variable) @app.command() @click.option("--name", default="queue") -def run(reader: str, writer: str, context: str | None = None, name: str = "queue"): +def run( + reader: Annotated[str, typer.Argument()], + writer: Annotated[str, typer.Argument()], + context: Annotated[str, typer.Argument()] = "", + name: Annotated[str, typer.Argument()] = "queue", +): params = { "name": name, "reader": get_function_from_string(reader), "writer": get_function_from_string(writer), } - if context: + if len(context) > 0: params["context"] = get_function_from_string(context) runner = QueueRunner( @@ -45,5 +52,10 @@ def run(reader: str, writer: str, context: str | None = None, name: str = "queue asyncio.run(runner.main()) +@app.command() +def version(): + typer.echo(__version__) + + if __name__ == "__main__": app() diff --git a/quasiqueue/runner.py b/quasiqueue/runner.py index 5e697c8..62758e9 100644 --- a/quasiqueue/runner.py +++ b/quasiqueue/runner.py @@ -45,14 +45,14 @@ async def main(self) -> None: # Inline function to implicitly pass through shutdown_event. def shutdown(a=None, b=None): - if a != None: + if a is not None: logging.debug(f"Signal {a} caught.") # Send shutdown signal to all processes. shutdown_event.set() # Graceful shutdown- wait for children to shut down. - if a == 15 or a == None: + if a == 15 or a is None: logging.debug("Gracefully shutting down child processes.") shutdown_start = time.time() while len(psutil.Process().children()) > 0: diff --git a/quasiqueue/settings.py b/quasiqueue/settings.py index d00c3f0..661d2ba 100644 --- a/quasiqueue/settings.py +++ b/quasiqueue/settings.py @@ -1,12 +1,15 @@ from pydantic import Field -from pydantic_settings import BaseSettings +from pydantic_settings import BaseSettings, SettingsConfigDict class Settings(BaseSettings): + model_config = SettingsConfigDict(env_prefix="QUASIQUEUE_") + num_processes: int = Field(default=2, description="The number of reader processes to run.") max_queue_size: int = Field(default=300, description="The max allowed six of the queue.") prevent_requeuing_time: float = Field( - default=300, description="The time in seconds that an item will be prevented from being readded to the queue." + default=300, + description="The time in seconds that an item will be prevented from being readded to the queue.", ) empty_queue_sleep_time: float = Field( default=1.00, @@ -17,7 +20,8 @@ class Settings(BaseSettings): description="The time in seconds that QuasiQueue will sleep the writer process if the queue is completely full.", ) queue_interaction_timeout: float = Field( - default=0.01, description="The time QuasiQueue will wait for the Queue to be unlocked before throwing an error." + default=0.01, + description="The time QuasiQueue will wait for the Queue to be unlocked before throwing an error.", ) graceful_shutdown_timeout: float = Field( default=30, @@ -28,16 +32,13 @@ class Settings(BaseSettings): description="The default desired_items passed to the writer function. This will be adjusted lower depending on queue dynamics.", ) max_jobs_per_process: int | None = Field( - default=200, description="The number of jobs a reader process will run before it is replaced by a new process." + default=200, + description="The number of jobs a reader process will run before it is replaced by a new process.", ) - class Config: - env_prefix = "QUASIQUEUE_" - def get_named_settings(name: str) -> Settings: class QueueSettings(Settings): - class Config: - env_prefix = f"QUASIQUEUE_{name.upper()}_" + model_config = SettingsConfigDict(env_prefix=f"QUASIQUEUE_{name.upper()}_") return QueueSettings() diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index f862234..0000000 --- a/setup.cfg +++ /dev/null @@ -1,47 +0,0 @@ -[metadata] -name = quasiqueue -description = A Simple High Performance Multiprocess Queue -long_description = file: README.md -long_description_content_type = text/markdown -license = MIT - -[versioneer] -VCS = git -style = pep440 -versionfile_source = quasiqueue/_version.py -versionfile_build = quasiqueue/_version.py -tag_prefix = v - -parentdir_prefix = - -[options] -packages = find: -include_package_data = True -install_requires = - psutil - pydantic - pydantic-settings - typer - -[options.extras_require] -dev = - black - build - dapperdata - glom - isort - mypy - pytest - pytest-cov - pytest-pretty - ruamel.yaml - types-psutil - -[options.package_data] -quasiqueue = py.typed - - -[options.entry_points] -console_scripts = - quasiqueue = quasiqueue.cli:app - diff --git a/setup.py b/setup.py deleted file mode 100644 index 835dc11..0000000 --- a/setup.py +++ /dev/null @@ -1,12 +0,0 @@ -# This file is deprecated by python in favor of setup.cfg -# However, some tools haven't caught up so we leave this here for them. -# All dependency and build configuration comes from setup.cfg. -# Other configuration, such as for linters, is in pyproject.toml. - -import setuptools -import versioneer - -setuptools.setup( - version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), -) diff --git a/tests/test_quasiqueue.py b/tests/test_quasiqueue.py index d35b909..b8f62b2 100644 --- a/tests/test_quasiqueue.py +++ b/tests/test_quasiqueue.py @@ -1,5 +1,2 @@ -import pytest - - def test_quasiqueue_import(): - from quasiqueue import QuasiQueue + from quasiqueue import QuasiQueue # noqa: F401 From b6f5f7022d4c9782a8634f42a5d435abfa806596 Mon Sep 17 00:00:00 2001 From: Robert Hafner Date: Sat, 10 Feb 2024 14:39:24 -0600 Subject: [PATCH 2/3] set minimum python version to 3.10 --- .github/workflows/pytest.yaml | 2 +- pyproject.toml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 7fa7ad6..7c5cd2f 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python_version: ["3.9", "3.10", "3.11", "3.12"] + python_version: ["3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 diff --git a/pyproject.toml b/pyproject.toml index 837364c..00be5e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ description = "A Simple High Performance Multiprocess Queue" dynamic = ["version"] license = {"file" = "LICENSE"} name = "quasiqueue" +python = ">=3.10" readme = {file = "README.md", content-type = "text/markdown"} [project.optional-dependencies] From 8d5f976ad3c847744aa26c350b90812f150bce1d Mon Sep 17 00:00:00 2001 From: Robert Hafner Date: Sat, 10 Feb 2024 14:42:02 -0600 Subject: [PATCH 3/3] fix requires-python argument --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 00be5e6..93ba35f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,8 +14,8 @@ description = "A Simple High Performance Multiprocess Queue" dynamic = ["version"] license = {"file" = "LICENSE"} name = "quasiqueue" -python = ">=3.10" readme = {file = "README.md", content-type = "text/markdown"} +requires-python = ">=3.10" [project.optional-dependencies] dev = [