Merge branch 'master' into nocopy

.github/workflows/tox.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: ["3.6", "3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4

.pylintrc

@@ -1,22 +1,17 @@
 [MESSAGES CONTROL]
 # C0111 Missing docstring 
 # I0011 Warning locally suppressed using disable-msg
-# I0012 Warning locally suppressed using disable-msg
-# W0704 Except doesn't do anything Used when an except clause does nothing but "pass" and there is no "else" clause
-# W0142 Used * or * magic* Used when a function or method is called using *args or **kwargs to dispatch arguments.
 # W0212 Access to a protected member %s of a client class
-# W0232 Class has no __init__ method Used when a class has no __init__ method, neither its parent classes.
 # W0613 Unused argument %r Used when a function or method argument is not used.
 # W0702 No exception's type specified Used when an except clause doesn't specify exceptions type to catch.
-# R0201 Method could be a function
 # W0614 Unused import XYZ from wildcard import
 # R0914 Too many local variables
 # R0912 Too many branches
 # R0915 Too many statements
 # R0913 Too many arguments
 # R0904 Too many public methods
 # E0211: Method has no argument
-disable=C0103,C0111,I0011,I0012,W0704,W0142,W0212,W0232,W0613,W0702,R0201,W0614,R0914,R0912,R0915,R0913,R0904,R0801
+disable=C0103,C0111,I0011,W0212,W0613,W0702,W0614,R0914,R0912,R0915,R0913,R0904,R0801
 
 [TYPECHECK]
 # Note: This modules are manipulated during the runtime so we can't detect all the properties during

CHANGELOG.rst

@@ -1,6 +1,22 @@
 Changelog
 =========
 
+Unreleased
+----------
+
+Changed
+~~~~~~~
+* Dropped python3.6 support.
+  Contributed by @nzlosh
+* Use pip 24.0 in virtualenv
+  Contributed by @nzlosh
+* Aligned pinned dependencies with St2 core.
+  Contributed by @nzlosh
+
+Fixed
+~~~~~
+
+
 1.6.0
 -----
 
@@ -159,7 +175,7 @@ Added
 ~~~~~
 
 * Add flake8 extension to restrict import alias. (improvement)
-* Add developer docs on getting started, testing, and StackStorm integration. (improvement) 
+* Add developer docs on getting started, testing, and StackStorm integration. (improvement)
 
 Changed
 ~~~~~~~
@@ -189,7 +205,7 @@ Added
 Fixed
 ~~~~~
 
-* Add sleep in while loop for composing execution graph to spread out cpu spike. (improvement) 
+* Add sleep in while loop for composing execution graph to spread out cpu spike. (improvement)
 * Value in quotes in shorthand publish should be evaluated as string type. Fixes #130 (bug fix)
 * Fix interpretation of boolean value in shorthand format of publish. Fixes #119 (bug fix)
 * Update YAQL section in docs on use of "=>" for named parameters in function calls. Closes #124

Makefile

@@ -1,3 +1,6 @@
+# Copyright 2020-2024 StackStorm contributors.
+# Copyright 2019 Extreme Networks, Inc.
+#
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -10,7 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-PY3 := $(shell which python3)
+PY3 := python3
+SYS_PY3 := $(shell which $(PY3))
+PIP_VERSION = 24.0
 
 # Virtual Environment
 VENV_DIR ?= .venv
@@ -21,47 +26,59 @@ TOX_DIR ?= .tox
 # Sphinx Document Options
 SPHINXOPTS    =
 SPHINXBUILD   = sphinx-build
-SPHINXAUTO	  = sphinx-autobuild
+SPHINXAUTO    = sphinx-autobuild
 SPHINXPROJ    = Orquesta
 SOURCEDIR     = docs/source
 BUILDDIR      = docs/build
+EGGDIR        = orquesta.egg-info
 
 # Packaging Options
 PKGDISTDIR    = dist
 PKGBUILDDIR   = build
 
+
+.PHONY: all
+all: clean reqs schemas check package
+
 .PHONY: clean
 clean:
 	rm -rf $(VENV_DIR)
 	rm -rf $(TOX_DIR)
 	rm -rf $(BUILDDIR)
 	rm -rf $(PKGDISTDIR)
 	rm -rf $(PKGBUILDDIR)
+	rm -rf $(EGGDIR)
 
 .PHONY: venv
 venv:
-	test -d $(VENV_DIR) || virtualenv -p $(PY3) $(VENV_DIR)
+	test -d $(VENV_DIR) || $(SYS_PY3) -m venv $(VENV_DIR)
 
 .PHONY: reqs
-reqs: venv
-	$(VENV_DIR)/bin/pip install --upgrade "pip==20.3.3"
+reqs: venv check_virtualenv
+	echo Install pip version $(PIP_VERSION) to match st2 core.
+	$(VENV_DIR)/bin/pip install --upgrade "pip==$(PIP_VERSION)"
 	$(VENV_DIR)/bin/pip install -r requirements.txt
 	$(VENV_DIR)/bin/pip install -r requirements-test.txt
 	$(VENV_DIR)/bin/pip install -r requirements-docs.txt
+	$(VENV_DIR)/bin/pip install -r requirements-ci.txt
 	$(VENV_DIR)/bin/python setup.py develop
 	echo
 
+.PHONY: check_virtualenv
+check_virtualenv:
+	test -d $(VENV_DIR) || exit 1
+
 .PHONY: schemas
-schemas: reqs
-	$(VENV_DIR)/bin/python bin/orquesta-generate-schemas
+schemas: check_virtualenv
+	$(VENV_DIR)/bin/$(PY3) bin/orquesta-generate-schemas
 
 .PHONY: format
-format:
+format: check_virtualenv
 	$(VENV_DIR)/bin/black orquesta bin setup.py -l 100
 
 .PHONY: check
-check:
-	tox
+check: check_virtualenv
+	$(VENV_DIR)/bin/tox
 
 .PHONY: docs
 docs: reqs
@@ -74,14 +91,12 @@ livedocs: reqs
 	. $(VENV_DIR)/bin/activate; $(SPHINXAUTO) -H 0.0.0.0 -b html $(SOURCEDIR) $(BUILDDIR)/html
 
 .PHONY: package
-package:
+package: check_virtualenv
 	rm -rf $(PKGDISTDIR)
 	rm -rf $(PKGBUILDDIR)
-	$(VENV_DIR)/bin/python setup.py sdist bdist_wheel
+	$(VENV_DIR)/bin/$(PY3) setup.py sdist bdist_wheel
 
 .PHONY: publish
 publish: package
-	$(VENV_DIR)/bin/python -m twine upload dist/*
+	$(VENV_DIR)/bin/$(PY3) -m twine upload dist/*
 
-.PHONY: all
-all: clean reqs schemas check package

orquesta/rehearsing.py

@@ -264,6 +264,7 @@ def __init__(self, session, *args, **kwargs):
                 "The session object is not type of WorkflowTestCase or WorkflowRerunTestCase."
             )
 
+        self.maxDiff = None
         self.session = session
         self.inspection_errors = {}
         self.rerun = False

orquesta/specs/native/v1/models.py

@@ -386,6 +386,7 @@ def detect_undefined_tasks(self, parent=None):
         # Identify the undefined task in task transitions.
         result = []
         traversed = []
+        queued_task = []
         q = queue.Queue()
 
         for task in self.get_start_tasks():
@@ -416,8 +417,12 @@ def detect_undefined_tasks(self, parent=None):
                         continue
 
                     if self.has_task(next_task_name):
-                        if next_task_name not in RESERVED_TASK_NAMES + traversed:
+                        if (
+                            next_task_name not in RESERVED_TASK_NAMES + traversed
+                            and next_task_name not in queued_task
+                        ):
                             q.put(next_task_name)
+                            queued_task.append(next_task_name)
                     else:
                         entry = {
                             "message": 'The task "%s" is not defined.' % next_task_name,
@@ -527,6 +532,7 @@ def inspect_context(self, parent=None):
         ctxs = {}
         errors = []
         traversed = []
+        task_ctx_map = {}
         parent_ctx = parent.get("ctx", []) if parent else []
         rolling_ctx = list(set(parent_ctx))
         q = queue.Queue()
@@ -596,7 +602,11 @@ def inspect_context(self, parent=None):
                 next_task_spec = self.get_task(next_task_name)
 
                 if not next_task_spec.has_join():
-                    q.put((next_task_name, branch_ctx))
+                    if next_task_name not in task_ctx_map or task_ctx_map[next_task_name] != set(
+                        branch_ctx
+                    ):
+                        q.put((next_task_name, branch_ctx))
+                        task_ctx_map[next_task_name] = set(branch_ctx)
                 else:
                     next_task_ctx = ctxs.get(next_task_name, [])
                     ctxs[next_task_name] = list(set(next_task_ctx + branch_ctx))

orquesta/tests/unit/conducting/test_workflow_conductor_performance.py

@@ -100,7 +100,7 @@ def test_runtime_function_of_splits_count(self):
         t2 = datetime.datetime.utcnow()
 
         delta = t2 - t1
-        self.assertLess(delta.seconds, 3)
+        self.assertLess(delta.seconds, 2)
 
 
 class WorkflowConductorWithItemsStressTest(test_base.WorkflowConductorWithItemsTest):

requirements-ci.txt

@@ -1,4 +1,3 @@
-# Different versions of tox are required by python version
 tox-gh-actions
-tox==3.28.0 ; python_version == "3.6"
-tox==4.11.3; python_version >= "3.8"
+tox==4.11.3
+wheel

requirements-docs.txt

@@ -1,3 +1,4 @@
-sphinx>=2.0
+# 202404: Align all requirements with st2 test-requirements
+sphinx>=5.0.0,<7.2.0
 sphinx-autobuild
 sphinx_rtd_theme

requirements-test.txt

@@ -1,11 +1,13 @@
-black
-coverage
-flake8<3.9.0,>=3.8.0
-hacking
-mock>=1.0
-pytest
-pytest-cov
-pep8>=1.6.0,<1.7
-pylint>=2.5.2,<2.6
+# 202404: Align all requirements with st2 test-requirements
+coverage==7.2.7
+black==22.3.0
+flake8==7.0.0
+mock==4.0.3
+pytest==6.2.5
+pytest-cov==4.1.0
+pep8==1.7.1
+pylint==3.1.0
 twine
 pytest-benchmark
+# 202404: Use forked version for flake8 v7.0.0 to align requirements with st2 test-requirements
+hacking @ git+https://github.com/nzlosh/hacking@flake8v7

requirements.txt

@@ -1,13 +1,15 @@
-chardet>=3.0.2,<4.0.0
+# 202404: Align all requirements with st2 fixed-requirements
+# 202405: Orquesta is a library, leave version ranges as wide as possible
+#         so pip can handle resolving the final version constraints.
+chardet>=3.0.2
 eventlet
-Jinja2>=2.11 # BSD License (3 clause)
-jsonschema!=2.5.0,>=2.0.0,<=3.2 # MIT
-# networkx v2.6 does not support Python3.6.  Update networkx to match st2
-networkx>=2.5.1,<2.6; python_version < '3.7'
-networkx>=2.6,<3; python_version >= '3.7'
+jinja2>=2.11 # BSD License (3 clause)
+jsonschema>=3,<4 # MIT
+# networkx v3.2 and greater does not support Python3.8.
+networkx>=2.6,<3.2
 python-dateutil
-PyYAML>=3.1.0 # MIT
-six>=1.9.0
+pyyaml>=5.3.1 # MIT
+six>=1.14.0
 stevedore>=1.3.0 # Apache-2.0
 ujson>=1.35 # BSD License
 yaql>=1.1.0 # Apache-2.0

setup.py

@@ -61,10 +61,11 @@ def get_requirements():
         "License :: OSI Approved :: Apache Software License",
         "Operating System :: POSIX :: Linux",
         "Programming Language :: Python",
-        "Programming Language :: Python :: 2",
-        "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
     ],
     entry_points={
         "orquesta.composers": [

tox.ini

@@ -1,11 +1,10 @@
 [tox]
-envlist = py36,py38,py39,py310,py311,pep8,docs
+envlist = py38,py39,py310,py311,pep8,docs
 minversion = 1.6
 skipsdist = True
 
 [gh-actions]
 python =
-    3.6: py36
     3.8: py38,pep8,docs
     3.9: py39
     3.10: py310