split requirements-dev.txt into first and third party dev dependenc…

…ies (spacetelescope#7849)

Co-authored-by: Nadia Dencheva <[email protected]>

.github/workflows/ci.yml

@@ -71,6 +71,7 @@ jobs:
         - linux: py311-xdist
         - macos: py311-xdist
           pytest-results-summary: true
+        - linux: py311-stdevdeps-xdist
         - linux: py311-devdeps-xdist
         - linux: py311-xdist-cov
           coverage: codecov

JenkinsfileRT

@@ -74,15 +74,11 @@ bc0.conda_ver = '22.11.1'
 bc0.conda_packages = [
     "python=${python_version}",
 ]
-bc0.pip_reqs_files = ['requirements-sdp.txt', 'requirements-max.txt']
+bc0.pip_reqs_files = ['requirements-sdp.txt', 'requirements-max.txt', 'requirements-dev-st.txt']
 bc0.build_cmds = [
     "pip install certifi -U --force-reinstall",
     "pip install -e .[test,sdp] --no-cache-dir",
     "pip install pytest-xdist",
-    // these latest-commit installs are overridden by `requirements-sdp.txt` if it is populated
-    "pip install git+https://github.com/spacetelescope/stdatamodels.git@master git+https://github.com/spacetelescope/stcal.git@main",
-    "pip install -r requirements-max.txt",
-    "pip install -r requirements-sdp.txt",
 ] + PipInject(env.OVERRIDE_REQUIREMENTS) + ["pip list"]
 bc0.test_cmds = [
     "pytest --cov-report=xml --cov=./ -r sxf --bigdata --slow \
@@ -107,5 +103,8 @@ bc1.name = 'macos-stable-deps'
 bc1.test_cmds = []
 bc1.test_configs = []
 
-utils.run([jobconfig, bc0, bc1])
+bc2 = utils.copy(bc0)
+bc2.pip_reqs_files = ['requirements-dev-st.txt']
+
+utils.run([jobconfig, bc0, bc1, bc2])
 }  // withCredentials

JenkinsfileRT_dev

@@ -74,12 +74,11 @@ bc0.conda_ver = '22.11.1'
 bc0.conda_packages = [
     "python=${python_version}",
 ]
-bc0.pip_reqs_files = ['requirements-dev.txt']
+bc0.pip_reqs_files = ['requirements-dev-st.txt', 'requirements-dev-thirdparty.txt']
 bc0.build_cmds = [
     "pip install certifi -U --force-reinstall",
     "pip install -e .[test,sdp] --no-cache-dir",
     "pip install pytest-xdist",
-    "pip install -r requirements-dev.txt",
 ] + PipInject(env.OVERRIDE_REQUIREMENTS) + ["pip list"]
 bc0.test_cmds = [
     "pytest -r sxf --bigdata --slow \

requirements-dev-st.txt

@@ -0,0 +1,12 @@
+# JWST upstream packages
+git+https://github.com/spacetelescope/stdatamodels
+
+# shared upstream packages
+git+https://github.com/spacetelescope/stcal
+git+https://github.com/spacetelescope/stpipe
+
+# Other important upstream packages
+git+https://github.com/spacetelescope/crds
+git+https://github.com/spacetelescope/gwcs
+git+https://github.com/spacetelescope/tweakwcs
+git+https://github.com/spacetelescope/wiimatch

requirements-dev.txt → requirements-dev-thirdparty.txt

@@ -1,17 +1,16 @@
+# ASDF upstream packages
 git+https://github.com/asdf-format/asdf-standard
 git+https://github.com/asdf-format/asdf
 git+https://github.com/asdf-format/asdf-transform-schemas
 git+https://github.com/asdf-format/asdf-coordinates-schemas
 git+https://github.com/asdf-format/asdf-wcs-schemas
-git+https://github.com/astropy/astropy
+
+# Use weekly astropy dev build
 git+https://github.com/astropy/asdf-astropy
-git+https://github.com/spacetelescope/crds
-git+https://github.com/spacetelescope/gwcs
-numpy>=0.0.dev0
 git+https://github.com/astropy/photutils
+--extra-index-url https://pypi.anaconda.org/astropy/simple astropy --pre
+
+# Use Bi-weekly numpy/scipy dev builds
+--extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple
+numpy>=0.0.dev0
 scipy>=0.0.dev0
-git+https://github.com/spacetelescope/stdatamodels
-git+https://github.com/spacetelescope/stpipe
-git+https://github.com/spacetelescope/stcal
-git+https://github.com/spacetelescope/tweakwcs
-git+https://github.com/spacetelescope/wiimatch

tox.ini

@@ -1,7 +1,7 @@
 [tox]
 envlist =
     check-{style,security,dependencies}
-    test{,-oldestdeps,-devdeps,-sdpdeps}{,-pyargs,-warnings,-regtests,-cov}-xdist
+    test{,-oldestdeps,-stdevdeps,-devdeps,-sdpdeps}{,-pyargs,-warnings,-regtests,-cov}-xdist
     build-{docs,dist}
 
 # tox environments are constructed with so-called 'factors' (or terms)
@@ -41,7 +41,8 @@ commands =
 [testenv]
 description =
     run tests
-    devdeps: with the latest developer version of key dependencies
+    stdevdeps: with the latest developer version of upstream spacetelescope dependencies
+    devdeps: with the latest developer version of upstream third-party dependencies
     oldestdeps: with the oldest supported version of key dependencies
     sdpdeps: with the recent STScI DMS release pinned dependencies
     pyargs: with --pyargs on installed package
@@ -60,7 +61,6 @@ pass_env =
     VALIDATE_ON_ASSIGNMENT
     TEST_BIGDATA
 set_env =
-    devdeps: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/scientific-python-nightly-wheels/simple
     sdpdeps,regtests: CRDS_CONTEXT = jwst-edit
 extras =
     test
@@ -73,7 +73,8 @@ package =
 commands_pre =
     oldestdeps: minimum_dependencies jwst --filename requirements-min.txt
     oldestdeps: pip install -r requirements-min.txt
-    devdeps: pip install -r requirements-dev.txt -U --upgrade-strategy eager
+    stdevdeps: pip install -r requirements-dev-st.txt -U --upgrade-strategy eager
+    devdeps: pip install -r requirements-dev-thirdparty.txt -U --upgrade-strategy eager
     sdpdeps: pip install -r requirements-sdp.txt
     pip freeze
 commands =