From b95e07ae1769ae97b723bb7735e29485412c6bf6 Mon Sep 17 00:00:00 2001 From: Usama Sadiq Date: Mon, 19 Feb 2024 14:17:18 +0500 Subject: [PATCH] build: update modernisers for python312 (#478) * build: update modernisers for python312 support --- edx_repo_tools/__init__.py | 2 +- edx_repo_tools/codemods/python312/__init__.py | 2 + .../python312/gh_actions_modernizer.py | 86 +++++++++++++ .../codemods/python312/tox_modernizer.py | 118 ++++++++++++++++++ setup.py | 2 + tests/sample_files/sample_ci_file.yml | 8 +- tests/sample_files/sample_ci_file_2.yml | 24 ++-- tests/sample_tox_config.ini | 8 +- tests/sample_tox_config_2.ini | 8 +- tests/test_actions_modernizer.py | 16 +-- tests/test_actions_modernizer_django.py | 23 ++-- tests/test_tox_modernizer.py | 19 +-- 12 files changed, 255 insertions(+), 61 deletions(-) create mode 100644 edx_repo_tools/codemods/python312/__init__.py create mode 100644 edx_repo_tools/codemods/python312/gh_actions_modernizer.py create mode 100644 edx_repo_tools/codemods/python312/tox_modernizer.py diff --git a/edx_repo_tools/__init__.py b/edx_repo_tools/__init__.py index 03ab978d..3e3316ed 100644 --- a/edx_repo_tools/__init__.py +++ b/edx_repo_tools/__init__.py @@ -1,2 +1,2 @@ -__version__ = '0.8.2' +__version__ = '0.8.3' diff --git a/edx_repo_tools/codemods/python312/__init__.py b/edx_repo_tools/codemods/python312/__init__.py new file mode 100644 index 00000000..e73ee6dd --- /dev/null +++ b/edx_repo_tools/codemods/python312/__init__.py @@ -0,0 +1,2 @@ +from .tox_modernizer import ConfigReader, ToxModernizer +from .gh_actions_modernizer import GithubCIModernizer diff --git a/edx_repo_tools/codemods/python312/gh_actions_modernizer.py b/edx_repo_tools/codemods/python312/gh_actions_modernizer.py new file mode 100644 index 00000000..2e4b6a21 --- /dev/null +++ b/edx_repo_tools/codemods/python312/gh_actions_modernizer.py @@ -0,0 +1,86 @@ +""" +Github Actions CI Modernizer to add Python 3.12 and drop Django 3.2 testing +""" +from copy import deepcopy +import click +from edx_repo_tools.utils import YamlLoader + +TO_BE_REMOVED_PYTHON = ['3.5', '3.6', '3.7'] +ALLOWED_PYTHON_VERSIONS = ['3.8', '3.12'] + +ALLOWED_DJANGO_VERSIONS = ['4.2', 'django42'] +DJANGO_ENV_TO_ADD = ['django42'] +DJANGO_ENV_TO_REMOVE = ['django32', 'django40', 'django41'] + + +class GithubCIModernizer(YamlLoader): + def __init__(self, file_path): + super().__init__(file_path) + + def _update_python_and_django_in_matrix(self): + django_versions = list() + python_versions = list() + matrix_elements = dict() + + + for section_key in self.elements['jobs']: + matrix_elements = deepcopy(self.elements['jobs'][section_key]['strategy']['matrix']) + + for key, value in matrix_elements.items(): + if key == 'django-version': + for dj_version in DJANGO_ENV_TO_ADD: + if dj_version not in value: + value.append(dj_version) + django_versions = list(filter(lambda version: version in ALLOWED_DJANGO_VERSIONS, value)) + if django_versions: + self.elements['jobs'][section_key]['strategy']['matrix'][key] = django_versions + + if key in ['tox', 'toxenv', 'tox-env']: + for dj_env in DJANGO_ENV_TO_ADD: + if dj_env not in value: + value.append(dj_env) + tox_envs = list(filter(lambda version: version not in DJANGO_ENV_TO_REMOVE, value)) + if tox_envs: + self.elements['jobs'][section_key]['strategy']['matrix'][key] = tox_envs + + if key == 'python-version': + for version in ALLOWED_PYTHON_VERSIONS: + if version not in value: + value.append(version) + python_versions = list(filter(lambda version: version not in TO_BE_REMOVED_PYTHON, value)) + if python_versions: + self.elements['jobs'][section_key]['strategy']['matrix'][key] = python_versions + else: + del self.elements['jobs'][section_key]['strategy']['matrix'][key] + + elif key in ['include', 'exclude']: + allowed_python_vers = list() + for item in value: + if item['python-version'] not in TO_BE_REMOVED_PYTHON: + allowed_python_vers.append(item) + + if len(allowed_python_vers): + self.elements['jobs'][section_key]['strategy']['matrix'][key] = allowed_python_vers + else: + del self.elements['jobs'][section_key]['strategy']['matrix'][key] + + + def _update_github_actions(self): + self._update_python_and_django_in_matrix() + + def modernize(self): + self._update_github_actions() + self.update_yml_file() + + +@click.command() +@click.option( + '--path', default='.github/workflows/ci.yml', + help="Path to default CI workflow file") +def main(path): + modernizer = GithubCIModernizer(path) + modernizer.modernize() + + +if __name__ == '__main__': + main() diff --git a/edx_repo_tools/codemods/python312/tox_modernizer.py b/edx_repo_tools/codemods/python312/tox_modernizer.py new file mode 100644 index 00000000..884e573a --- /dev/null +++ b/edx_repo_tools/codemods/python312/tox_modernizer.py @@ -0,0 +1,118 @@ +import io +import re +from configparser import ConfigParser, NoSectionError + +import click + +TOX_SECTION = "tox" +ENVLIST = "envlist" +TEST_ENV_SECTION = "testenv" +TEST_ENV_DEPS = "deps" +PYTHON_SUBSTITUTE = "py{38, 312}" +DJANGO_SUBSTITUTE = "django{42}" + +DJANGO_42_DEPENDENCY = "django42: Django>=4.2,<4.3\n" +NEW_DJANGO_DEPENDENCIES = DJANGO_42_DEPENDENCY + +SECTIONS = [TOX_SECTION, TEST_ENV_SECTION] + +PYTHON_PATTERN = "(py{.*?}-?|py[0-9]+,|py[0-9]+-)" + +DJANGO_PATTERN = "(django[0-9]+,|django[0-9]+\n|django{.*}\n|django{.*?}|django[0-9]+-|django{.*}-)" + +DJANGO_DEPENDENCY_PATTERN = "([^\n]*django[0-9]+:.*\n?)" + + +class ConfigReader: + def __init__(self, file_path=None, config_dict=None): + self.config_dict = config_dict + self.file_path = file_path + + def get_modernizer(self): + config_parser = ConfigParser() + if self.config_dict is not None: + config_parser.read_dict(self.config_dict) + else: + config_parser.read(self.file_path) + return ToxModernizer(config_parser, self.file_path) + + +class ToxModernizer: + def __init__(self, config_parser, file_path): + self.file_path = file_path + self.config_parser = config_parser + self._validate_tox_config_sections() + + def _validate_tox_config_sections(self): + if not self.config_parser.sections(): + raise NoSectionError("Bad Config. No sections found.") + + if all(section not in SECTIONS for section in self.config_parser.sections()): + raise NoSectionError("File doesn't contain required sections") + + def _update_env_list(self): + tox_section = self.config_parser[TOX_SECTION] + env_list = tox_section[ENVLIST] + + env_list = ToxModernizer._replace_runners(PYTHON_PATTERN, PYTHON_SUBSTITUTE, env_list) + env_list = ToxModernizer._replace_runners(DJANGO_PATTERN, DJANGO_SUBSTITUTE, env_list) + self.config_parser[TOX_SECTION][ENVLIST] = env_list + + @staticmethod + def _replace_runners(pattern, substitute, env_list): + matches = re.findall(pattern, env_list) + if not matches: + return env_list + substitute = ToxModernizer._get_runner_substitute(matches, substitute) + return ToxModernizer._replace_matches(pattern, substitute, env_list, matches) + + @staticmethod + def _replace_matches(pattern, substitute, target, matches): + if not matches: + return target + occurrences_to_replace = len(matches) - 1 + if occurrences_to_replace > 0: + target = re.sub(pattern, '', target, occurrences_to_replace) + target = re.sub(pattern, substitute, target) + return target + + @staticmethod + def _get_runner_substitute(matches, substitute): + last_match = matches[-1] + has_other_runners = last_match.endswith('-') + return substitute + "-" if has_other_runners else substitute + + def _replace_django_versions(self): + test_environment = self.config_parser[TEST_ENV_SECTION] + dependencies = test_environment[TEST_ENV_DEPS] + matches = re.findall(DJANGO_DEPENDENCY_PATTERN, dependencies) + dependencies = self._replace_matches(DJANGO_DEPENDENCY_PATTERN, NEW_DJANGO_DEPENDENCIES, dependencies, matches) + + self.config_parser[TEST_ENV_SECTION][TEST_ENV_DEPS] = dependencies + + def _update_config_file(self): + # ConfigParser insists on using tabs for output. We want spaces. + with io.StringIO() as configw: + self.config_parser.write(configw) + new_ini = configw.getvalue() + new_ini = new_ini.replace("\t", " ") + with open(self.file_path, 'w') as configfile: + configfile.write(new_ini) + + def modernize(self): + self._update_env_list() + self._replace_django_versions() + self._update_config_file() + + +@click.command() +@click.option( + '--path', default='tox.ini', + help="Path to target tox config file") +def main(path): + modernizer = ConfigReader(path).get_modernizer() + modernizer.modernize() + + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py index 8f486f7f..cef7f21a 100644 --- a/setup.py +++ b/setup.py @@ -113,6 +113,8 @@ def is_requirement(line): 'modernize_tox_django42 = edx_repo_tools.codemods.django42.tox_moderniser_django42:main', 'modernize_github_actions_django42 = edx_repo_tools.codemods.django42.github_actions_modernizer_django42:main', 'remove_providing_args = edx_repo_tools.codemods.django42.remove_providing_args_arg:main', + 'python312_gh_actions_modernizer = edx_repo_tools.codemods.python312.gh_actions_modernizer:main', + 'python312_tox_modernizer = edx_repo_tools.codemods.python312.tox_modernizer:main', ], }, package_data={ diff --git a/tests/sample_files/sample_ci_file.yml b/tests/sample_files/sample_ci_file.yml index 0442a2ac..bee59809 100644 --- a/tests/sample_files/sample_ci_file.yml +++ b/tests/sample_files/sample_ci_file.yml @@ -14,8 +14,12 @@ jobs: matrix: os: [ubuntu-20.04] python-version: ['3.5', '3.8'] - django-version: ['3.2', '4.0'] - toxenv: ['django32', 'quality', 'docs', 'pii_check'] + django-version: [django42] + toxenv: + - 'quality' + - 'docs' + - 'pii_check' + - django42 steps: - uses: actions/checkout@v2 - name: setup python diff --git a/tests/sample_files/sample_ci_file_2.yml b/tests/sample_files/sample_ci_file_2.yml index 338b2cc8..58a92d84 100644 --- a/tests/sample_files/sample_ci_file_2.yml +++ b/tests/sample_files/sample_ci_file_2.yml @@ -13,19 +13,19 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python-version: ['3.5', '3.6', '3.7', '3.8'] + python-version: ['3.5', '3.6', '3.7', '3.8', '3.12'] django-version: ['3.2'] include: - - python-version: "3.5" - toxenv: 'quality' - ubuntu: 20.04 + - python-version: "3.5" + toxenv: 'quality' + ubuntu: 20.04 exclude: - - python-version: "3.5" - toxenv: 'quality' - - python-version: "3.7" - toxenv: 'docs' - - python-version: "3.8" - toxenv: 'pii_check' + - python-version: "3.5" + toxenv: 'quality' + - python-version: "3.7" + toxenv: 'docs' + - python-version: "3.8" + toxenv: 'pii_check' steps: - uses: actions/checkout@v2 - name: setup python @@ -45,8 +45,8 @@ jobs: run: tox - name: Run Coverage - if: matrix.python-version == '3.8' && matrix.toxenv=='django32' - uses: codecov/codecov-action@v1 + if: matrix.python-version == '3.8' && matrix.toxenv=='django42' + uses: codecov/codecov-action@v2 with: flags: unittests fail_ci_if_error: true diff --git a/tests/sample_tox_config.ini b/tests/sample_tox_config.ini index 7639169f..a62f0d77 100644 --- a/tests/sample_tox_config.ini +++ b/tests/sample_tox_config.ini @@ -1,6 +1,6 @@ [tox] envlist = - py{27,35,36,37}-django{111,20,21,22}-drf{39,310,latest}, + py{38}-django{32,40}-drf{39,310,latest}, docs, quality, version_check, @@ -9,10 +9,8 @@ envlist = [testenv] deps = - django111: Django>=1.11,<2.0 - django20: Django>=2.0,<2.1 - django21: Django>=2.1,<2.2 - django22: Django>=2.2,<2.3 + django32: Django>=3.2,<3.3 + django40: Django>=4.0,<4.1 drf39: djangorestframework<3.10.0 drf310: djangorestframework<3.11.0 drflatest: djangorestframework diff --git a/tests/sample_tox_config_2.ini b/tests/sample_tox_config_2.ini index 59432569..173f6763 100644 --- a/tests/sample_tox_config_2.ini +++ b/tests/sample_tox_config_2.ini @@ -1,6 +1,6 @@ [tox] envlist = - py27,py35,py36,py37-django111,django20,django21,django22-drf39,drf310,drflatest, + py37,py38-django32,django40-drf39,drf310,drflatest, docs, quality, version_check, @@ -9,10 +9,8 @@ envlist = [testenv] deps = - django111: Django>=1.11,<2.0 - django20: Django>=2.0,<2.1 - django21: Django>=2.1,<2.2 - django22: Django>=2.2,<2.3 + django32: Django>=3.2,<3.3 + django40: Django>=4.0,<4.1 drf39: djangorestframework<3.10.0 drf310: djangorestframework<3.11.0 drflatest: djangorestframework diff --git a/tests/test_actions_modernizer.py b/tests/test_actions_modernizer.py index b7a2a220..b8f375ed 100644 --- a/tests/test_actions_modernizer.py +++ b/tests/test_actions_modernizer.py @@ -6,7 +6,7 @@ import uuid from unittest import TestCase -from edx_repo_tools.codemods.django3 import GithubCIModernizer +from edx_repo_tools.codemods.python312 import GithubCIModernizer from edx_repo_tools.utils import YamlLoader @@ -37,22 +37,16 @@ def test_python_matrix_items(self): python_versions = ci_elements['jobs']['run_tests']['strategy']['matrix']['python-version'] self.assertIsInstance(python_versions, list) - self.assertNotIn('3.5', python_versions) + self.assertIn('3.8', python_versions) + self.assertIn('3.12', python_versions) def test_python_matrix_items_build_tag(self): ci_elements = TestGithubActionsModernizer._get_updated_yaml_elements(self.test_file3) python_versions = ci_elements['jobs']['build']['strategy']['matrix']['python-version'] self.assertIsInstance(python_versions, list) - self.assertNotIn('3.5', python_versions) - - def test_include_exclude_list(self): - ci_elements = TestGithubActionsModernizer._get_updated_yaml_elements(self.test_file2) - include_list = ci_elements['jobs']['run_tests']['strategy']['matrix'].get('include', {}) - exclude_list = ci_elements['jobs']['run_tests']['strategy']['matrix'].get('exclude', {}) - - for item in list(include_list) + list(exclude_list): - self.assertNotEqual(item['python-version'], '3.5') + self.assertIn('3.8', python_versions) + self.assertIn('3.12', python_versions) def tearDown(self): os.remove(self.test_file1) diff --git a/tests/test_actions_modernizer_django.py b/tests/test_actions_modernizer_django.py index 4cedbdf2..ff0ddd9b 100644 --- a/tests/test_actions_modernizer_django.py +++ b/tests/test_actions_modernizer_django.py @@ -5,7 +5,7 @@ import shutil from os.path import basename, dirname, join -from edx_repo_tools.codemods.django3 import GithubCIDjangoModernizer +from edx_repo_tools.codemods.python312 import GithubCIModernizer from edx_repo_tools.utils import YamlLoader @@ -18,7 +18,7 @@ def setup_local_copy(filepath, tmpdir): def get_updated_yaml_elements(file_path): - modernizer = GithubCIDjangoModernizer(file_path) + modernizer = GithubCIModernizer(file_path) modernizer.modernize() yaml_loader = YamlLoader(file_path) return yaml_loader.elements @@ -32,8 +32,8 @@ def test_matrix_items(tmpdir): ci_elements = get_updated_yaml_elements(test_file) tox_envs = ci_elements['jobs']['run_tests']['strategy']['matrix']['toxenv'] - assert 'django32' in tox_envs - assert 'django40' in tox_envs + assert 'django32' not in tox_envs + assert 'django42' in tox_envs def test_matrix_items_multiple_jobs(tmpdir): @@ -45,18 +45,17 @@ def test_matrix_items_multiple_jobs(tmpdir): # test the case with django env present in one job job1_tox_envs = ci_elements['jobs']['build']['strategy']['matrix']['tox-env'] - assert 'django32' in job1_tox_envs - assert 'django40' in job1_tox_envs + assert 'django32' not in job1_tox_envs + assert 'django42' in job1_tox_envs # test the case with django env present in second job job2_tox_envs = ci_elements['jobs']['django_test']['strategy']['matrix']['django-version'] - assert 'django32' in job2_tox_envs - assert 'django40' in job2_tox_envs + assert 'django32' not in job2_tox_envs + assert 'django42' in job2_tox_envs # test the case with no django env present in third job. job3_tox_envs = ci_elements['jobs']['test']['strategy']['matrix']['tox'] - assert 'django32' not in job3_tox_envs - assert 'django40' not in job3_tox_envs + assert 'django42' in job3_tox_envs def test_include_exclude_list(tmpdir): """ @@ -69,6 +68,6 @@ def test_include_exclude_list(tmpdir): for item in list(include_list) + list(exclude_list): if 'django-version' in item: - assert item['django-version'] != '3.1' + assert item['django-version'] != '3.2' if 'toxenv' in item: - assert item['toxenv'] != 'django30' + assert item['toxenv'] != 'django42' diff --git a/tests/test_tox_modernizer.py b/tests/test_tox_modernizer.py index c340bb56..6d506e5e 100644 --- a/tests/test_tox_modernizer.py +++ b/tests/test_tox_modernizer.py @@ -5,7 +5,7 @@ from unittest import TestCase import shutil import uuid -from edx_repo_tools.codemods.django3 import ConfigReader +from edx_repo_tools.codemods.python312 import ConfigReader class TestToxModernizer(TestCase): @@ -33,27 +33,20 @@ def _assert_django_dependencies_replaced(self, config_file): parser = self._get_parser(config_file) dependencies = parser['testenv']['deps'] - self.assertIn("django32:", dependencies) - self.assertIn("django40:", dependencies) + self.assertNotIn("django32:", dependencies) + self.assertIn("django42:", dependencies) def _assert_replaces_python_interpreters(self, config_file): parser = self._get_parser(config_file) env_list = parser['tox']['envlist'] - self.assertNotRegex("py{27}", env_list) - self.assertNotIn("py{27,35}", env_list) - self.assertNotIn("py{27,35,36}", env_list) - self.assertNotIn("py{27,35,36,37}", env_list) - self.assertIn("py38", env_list) + self.assertIn("py{38, 312}", env_list) def _assert_replaces_django_runners(self, config_file): parser = self._get_parser(config_file) env_list = parser['tox']['envlist'] - self.assertNotIn("django{111}", env_list) - self.assertNotIn("django{111,20}", env_list) - self.assertNotIn("django{111,20,21}", env_list) - self.assertIn("django{32,40}", env_list) + self.assertIn("django{42}", env_list) def _assert_replaces_django_dependencies(self, config_file): self._assert_django_dependencies_replaced(config_file) @@ -63,7 +56,7 @@ def _assert_adds_django_dependencies(self, config_file): parser.read(config_file) dependencies = parser['testenv']['deps'] - dependencies = re.sub("[^\n]*django32.*\n", '', dependencies) + dependencies = re.sub("[^\n]*django42.*\n", '', dependencies) parser['testenv']['deps'] = dependencies with open(config_file, 'w') as configfile: