Merge pull request #114 from Jylpah/dev-1.0

CI update

.github/workflows/codeql.yml

@@ -12,12 +12,9 @@
 name: "CodeQL"
 
 on:
-  push:
-    branches: [ "main", "main-1.0" ]
-  pull_request:
-    branches: [ "main", "main-1.0" ]
+  workflow_dispatch:
   schedule:
-    - cron: '22 9 * * 6'
+    - cron: '22 2 * * 6'
 
 jobs:
   analyze:

.github/workflows/python-package-macos-win.yml

@@ -0,0 +1,50 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Python package MacOS & Windows
+
+on:
+    workflow_dispatch:
+    schedule:
+      - cron: '23 2 5 * *'
+
+#   push:
+#     branches: [ "main", "main-1.0" ]
+#   pull_request:
+#     branches: [ "main", "main-1.0" ]
+
+jobs:
+  build:
+    # runs-on: [ ubuntu-latest ]
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [ "3.11", "3.12"]
+        os: [ windows-latest, macos-latest ]
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install .
+        python -m pip install ".[dev]"
+    - name: Lint with ruff
+      run: | 
+        ruff check src tests
+    - name: Test with mypy
+      run: |
+        mypy src
+    - name: Test with pytest
+      run: |
+        pytest --cov .
+    - name: Upload coverage reports to Codecov
+      uses: codecov/codecov-action@v3
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}  # ignore

.github/workflows/python-package.yml

@@ -11,13 +11,11 @@ on:
 
 jobs:
   build:
-    # runs-on: [ ubuntu-latest ]
+    runs-on: [ ubuntu-latest ]
     strategy:
       fail-fast: false
       matrix:
         python-version: [ "3.11", "3.12"]
-        os: [ ubuntu-latest, windows-latest, macos-latest ]
-    runs-on: ${{ matrix.os }}
 
     steps:
     - uses: actions/checkout@v4
@@ -29,22 +27,20 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install flake8 pytest
         python -m pip install .
         python -m pip install ".[dev]"
-    - name: Lint with flake8
+    - name: Lint with ruff
+      run: | 
+        ruff check src tests
+    - name: Test with mypy
       run: |
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+        mypy src
     - name: Test with pytest
       run: |
         pytest --cov .
-    - name: Test with mypy
-      run: |
-        mypy -m pyutils
     - name: Upload coverage reports to Codecov
       uses: codecov/codecov-action@v3
       env:
         CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        fail_ci_if_error: false
+

codecov.yml

@@ -0,0 +1,9 @@
+coverage:
+  status:
+    project:
+      default:
+        informational: true
+        threshold: 1%
+    patch:
+      default:
+        informational: true

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pyutils"
-version = "1.0.0"
+version = "1.0.1"
 authors = [{ name = "Jylpah", email = "[email protected]" }]
 description = "Misc Python utils and classes"
 readme = "README.md"
@@ -16,29 +16,30 @@ classifiers = [
     "Topic :: Games/Entertainment",
 ]
 dependencies = [
-    "aiofiles~=23.1",
-    "aiohttp~=3.9.1",
-    "aiodns~=3.0",
-    "aioconsole~=0.6",
-    "alive-progress~=3.1",
-    "Brotli~=1.1",
-    "Deprecated~=1.2",
-    "click~=8.1",
-    "typer~=0.9",
+    "aiofiles>=23.1",
+    "aiohttp>=3.9.1",
+    "aiodns>=3.0",
+    "aioconsole>=0.6",
+    "alive-progress>=3.1",
+    "Brotli>=1.1",
+    "Deprecated>=1.2",
+    "click>=8.1",
+    "typer>=0.9",
 ]
 
 [project.optional-dependencies]
 dev = [
-    "build~=0.10",
-    "mypy~=1.7",
-    "pip-chill~=1.0",
-    "pytest~=7.4",
-    "pytest-asyncio~=0.23",
-    "pytest-datafiles~=3.0",
-    "pytest-cov~=4.1",
-    "pytest-timeout~=2.2",
-    "types-aiofiles~=23.1.0.1",
-    "types-Deprecated~=1.2.9.3",
+    "build>=0.10",
+    "mypy>=1.7",
+    "pip-chill>=1.0",
+    "pytest>=7.4",
+    "pytest-asyncio>=0.23",
+    "pytest-datafiles>=3.0",
+    "pytest-cov>=4.1",
+    "pytest-timeout>=2.2",
+    "ruff>=0.1.9",
+    "types-aiofiles>=23.1.0.1",
+    "types-Deprecated>=1.2.9.3",
 ]
 
 
@@ -60,9 +61,11 @@ packages = ["src/pyutils"]
 exclude = ['tmp']
 mypy_path = ['src']
 
-[tool.black]
-# line-length = 100
-include = '\.pyi?$'
+[tool.ruff]
+include = ["pyproject.toml", "src/**/*.py", "tests/**/*.py"]
+indent-width = 4
+extend-include = ["*.ipynb"]
+fixable = ["ALL"]
 
 [tool.pytest.ini_options]
 minversion = "7.0"

src/pyutils/__init__.py

@@ -8,28 +8,11 @@
 from .iterablequeue import IterableQueue as IterableQueue, QueueDone as QueueDone
 from .multilevelformatter import MultilevelFormatter as MultilevelFormatter
 from .throttledclientsession import ThrottledClientSession as ThrottledClientSession
-from .urlqueue import UrlQueue as UrlQueue, UrlQueueItemType, is_url
+from .urlqueue import UrlQueue as UrlQueue
 from .utils import (
     Countable as Countable,
     ClickHelpGen as ClickHelpGen,
     TyperHelpGen as TyperHelpGen,
-    alive_bar_monitor,
-    coro,
-    chunker,
-    epoch_now,
-    # read_config,
-    get_datestr,
-    get_subtype,
-    get_type,
-    get_url,
-    # get_urls,
-    get_url_JSON,
-    # get_urls_JSON,
-    get_temp_filename,
-    is_valid_obj,
-    post_url,
-    set_config,
-    str2path,
 )
 
 

src/pyutils/eventcounter.py

@@ -93,8 +93,9 @@ def get_value(self, category: str) -> int:
         assert category is not None, "param 'category' cannot be None"
         try:
             return self._log[category]
-        except:
-            logger.error(f"invalid categgory: {category}")
+        except Exception as err:
+            error(f"invalid categgory: {category}")
+            debug(f"{err}")
             return 0
 
     def get_values(self) -> dict[str, int]:
@@ -120,7 +121,7 @@ def merge(self, B: "EventCounter") -> bool:
 
         try:
             if not isinstance(B, EventCounter):
-                logger.error(f"input is not type of 'EventCounter' but: {type(B)}")
+                error(f"input is not type of 'EventCounter' but: {type(B)}")
                 return False
             for cat in B.get_categories():
                 value: int = B.get_value(cat)
@@ -130,7 +131,7 @@ def merge(self, B: "EventCounter") -> bool:
                 self._error_status = self._error_status or B.get_error_status()
             return True
         except Exception as err:
-            logger.error(f"{err}")
+            error(f"{err}")
         return False
 
     def merge_child(self, B: "EventCounter") -> bool:
@@ -148,7 +149,7 @@ def merge_child(self, B: "EventCounter") -> bool:
             self._error_status = self._error_status or B.get_error_status()
             return True
         except Exception as err:
-            logger.error(f"{err}")
+            error(f"{err}")
         return False
 
     def get_header(self) -> str:
@@ -171,7 +172,7 @@ def print(self, do_print: bool = True, clean: bool = False) -> Optional[str]:
                     ret += f"\n{self._get_str(cat)}"
                 return ret
         except Exception as err:
-            logger.error(f"{err}")
+            error(f"{err}")
         return None
 
     async def gather_stats(

src/pyutils/filequeue.py

@@ -14,7 +14,7 @@
 from pathlib import Path
 from typing import Optional, Sequence
 
-from .iterablequeue import IterableQueue, QueueDone
+from .iterablequeue import IterableQueue
 from .utils import str2path
 
 logger = logging.getLogger(__name__)

src/pyutils/iterablequeue.py

@@ -1,4 +1,4 @@
-from asyncio import Queue, QueueFull, QueueEmpty, Event, Lock, sleep
+from asyncio import Queue, QueueFull, QueueEmpty, Event, Lock
 from typing import AsyncIterable, TypeVar, Optional
 from .utils import Countable
 import logging
@@ -206,11 +206,11 @@ def task_done(self) -> None:
         self.check_done()
 
     async def join(self) -> None:
-        debug(f"Waiting queue to be filled")
+        debug("Waiting queue to be filled")
         await self._filled.wait()
-        debug(f"Queue filled, waiting when queue is done")
+        debug("Queue filled, waiting when queue is done")
         await self._done.wait()
-        debug(f"queue is done")
+        debug("queue is done")
         return None
 
     def __aiter__(self):

src/pyutils/multilevelformatter.py

@@ -1,4 +1,5 @@
-import logging, sys
+import logging
+import sys
 from typing import Literal, Optional
 from pathlib import Path
 

src/pyutils/throttledclientsession.py

@@ -193,7 +193,7 @@ async def close(self) -> None:
                 await wait_for(self._fillerTask, timeout=0.5)
         except TimeoutError as err:
             debug(f"Timeout while cancelling bucket filler: {err}")
-        except CancelledError as err:
+        except CancelledError:
             debug("Cancelled")
         await super().close()
 

src/pyutils/utils.py

@@ -146,7 +146,7 @@ def is_alphanum(string: str) -> bool:
     """test whether the string is composed of ASCII letters, numbers, hyphens or underscores only"""
     try:
         return not compile(r"[^a-zA-Z0-9_]").search(string)
-    except:
+    except Exception:
         error(f"Illegal characters in the table name: {string}")
     return False
 
@@ -158,7 +158,7 @@ def is_valid_obj(string: str) -> bool:
     """
     try:
         return not compile(r"[^a-zA-Z0-9_]").search(string)
-    except:
+    except Exception:
         error(f"Illegal characters in the table name: {string}")
     return False
 

tests/test_asyncqueue.py

@@ -4,12 +4,19 @@
 from queue import Queue
 
 # from asyncio.queues import QueueEmpty, QueueFull
-from asyncio import Task, create_task, sleep, wait_for, timeout, TimeoutError, QueueEmpty, QueueFull
-import asyncio
+from asyncio import (
+    Task,
+    create_task,
+    sleep,
+    timeout,
+    TimeoutError,
+    QueueEmpty,
+    QueueFull,
+)
 
 sys.path.insert(0, str(Path(__file__).parent.parent.resolve() / "src"))
 
-from pyutils import AsyncQueue
+from pyutils import AsyncQueue  # noqa: E402
 
 QSIZE: int = 10
 N: int = 100  # N >> QSIZE
@@ -86,6 +93,7 @@ async def test_2_put_get_nowait(test_asyncqueue_int: AsyncQueue[int]):
             await Q.join()
     except TimeoutError:
         assert False, "Queue.join() took longer than it should"
+    assert producer.done(), "producer has not finished"
     assert Q.qsize() == 0, "queue not empty"
     assert Q.empty(), "queue not empty"
 

tests/test_bucketmapper.py

@@ -6,8 +6,7 @@
 
 sys.path.insert(0, str(Path(__file__).parent.parent.resolve() / "src"))
 
-from pyutils import BucketMapper
-from pyutils.bucketmapper import T
+from pyutils import BucketMapper  # noqa: E402
 
 ########################################################
 #
@@ -96,7 +95,7 @@ def test_3_get_fails(
     bm_keys: list[int] = _get_keys(bm)
     max_key: int = max(bm_keys)
     for _ in range(LOOPS):
-        if (item := bm.get(max_key + randrange(1, N))) is None:
+        if (_ := bm.get(max_key + randrange(1, N))) is None:
             pass  # OK
         else:
             assert False, "get() should always return None if key outside key range"