diff --git a/.cloudbuild/library_generation/library_generation.Dockerfile b/.cloudbuild/library_generation/library_generation.Dockerfile index 4b3b1f4b1c..09d23a106b 100644 --- a/.cloudbuild/library_generation/library_generation.Dockerfile +++ b/.cloudbuild/library_generation/library_generation.Dockerfile @@ -26,7 +26,7 @@ RUN rm $(which python3) RUN ln -s $(which python3.11) /usr/local/bin/python RUN ln -s $(which python3.11) /usr/local/bin/python3 RUN python -m pip install --upgrade pip -RUN cd /src && python -m pip install -r requirements.in +RUN cd /src && python -m pip install -r requirements.txt RUN cd /src && python -m pip install . # set dummy git credentials for empty commit used in postprocessing diff --git a/.github/workflows/create_additional_release_tag.yaml b/.github/workflows/create_additional_release_tag.yaml index 0bc731c127..aa19e89c2d 100644 --- a/.github/workflows/create_additional_release_tag.yaml +++ b/.github/workflows/create_additional_release_tag.yaml @@ -11,7 +11,7 @@ jobs: permissions: write-all steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: token: ${{ secrets.GITHUB_TOKEN }} - name: Set up Git @@ -34,14 +34,3 @@ jobs: git tag $TAG_NAME git push origin $TAG_NAME done - # Generate a tag for unmanaged dependencies check. - # Use fixed tag so that checks in handwritten libraries do not need to - # update the version. - CHECK_LATEST_TAG="unmanaged-dependencies-check-latest" - # delete and create the tag locally. - git tag --delete ${CHECK_LATEST_TAG} - git tag ${CHECK_LATEST_TAG} - # delete the tag in remote repo and push again. - git push --delete origin ${CHECK_LATEST_TAG} - git push origin ${CHECK_LATEST_TAG} - diff --git a/.github/workflows/verify_library_generation.yaml b/.github/workflows/verify_library_generation.yaml index ca308461c0..c51f8ed7b0 100644 --- a/.github/workflows/verify_library_generation.yaml +++ b/.github/workflows/verify_library_generation.yaml @@ -10,17 +10,9 @@ on: name: verify_library_generation jobs: integration_tests: - strategy: - matrix: - java: [ 11 ] runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.java }} - distribution: temurin - cache: maven - uses: actions/setup-python@v5 with: python-version: 3.11 @@ -41,7 +33,7 @@ jobs: run: | set -ex pushd library_generation - pip install -r requirements.in + pip install -r requirements.txt pip install . popd - name: Run integration tests @@ -71,7 +63,7 @@ jobs: run: | set -ex pushd library_generation - pip install -r requirements.in + pip install -r requirements.txt popd - name: install synthtool shell: bash @@ -115,7 +107,7 @@ jobs: run: | set -ex pushd library_generation - pip install -r requirements.in + pip install -r requirements.txt popd - name: Lint shell: bash diff --git a/library_generation/README.md b/library_generation/README.md index 75fc32695d..04da9fda03 100644 --- a/library_generation/README.md +++ b/library_generation/README.md @@ -1,8 +1,8 @@ # Generate a repository containing GAPIC Client Libraries -The script, `generate_repo.py`, allows you to generate a repository containing -GAPIC client libraries (a monorepo, for example, google-cloud-java) from a -configuration file. +The script, `entry_point.py`, allows you to generate a repository containing +GAPIC client libraries with googleapis commit history (a monorepo, for example, +google-cloud-java) from a configuration file. ## Environment @@ -10,6 +10,8 @@ configuration file. - Java runtime environment (8 or above) - Apache Maven (used in formatting source code) - Python (3.11.6 or above) +- Docker +- Git ## Prerequisite @@ -17,30 +19,27 @@ In order to generate a version for each library, a versions.txt has to exist in `repository_path`. Please refer to [Repository path](#repository-path--repositorypath---optional) for more information. -## Parameters to generate a repository using `generate_repo.py` +## Parameters to generate a repository using `entry_point.py` -### Generation configuration yaml (`generation_config_yaml`) +### Baseline generation configuration yaml (`baseline_generation_config`) -A path to a configuration file containing parameters to generate the repository. -Please refer [Configuration to generate a repository](#configuration-to-generate-a-repository) -for more information. - -### Target library API shortname (`target_library_api_shortname`), optional +An absolute or relative path to a generation_config.yaml. +This config file is used for commit history generation, not library +generation. -If specified, the libray whose `api_shortname` equals to `target_library_api_shortname` -will be generated; otherwise all libraries in the configuration file will be -generated. -This can be useful when you just want to generate one library for debugging -purposes. +### Current generation configuration yaml (`current_generation_config`) -The default value is an empty string, which means all libraries will be generated. +An absolute or relative path to a configuration file containing parameters to +generate the repository. +Please refer [Configuration to generate a repository](#configuration-to-generate-a-repository) +for more information. ### Repository path (`repository_path`), optional The path to where the generated repository goes. The default value is the current working directory when running the script. -For example, `cd google-cloud-java && python generate_repo.py ...` without +For example, `cd google-cloud-java && python entry_point.py ...` without specifying the `--repository_path` option will modify the `google-cloud-java` repository the user `cd`'d into. @@ -49,7 +48,9 @@ right version for each library. Please refer [here](go/java-client-releasing#versionstxt-manifest) for more info of versions.txt. -## Output of `generate_repo.py` +## Output of `entry_point.py` + +### GAPIC libraries For each module (e.g. `google-cloud-java/java-asset`), the following files/folders will be created/modified: @@ -73,6 +74,11 @@ will be created/modified: | pom.xml (repo root dir) | Always generated from inputs | | versions.txt | New entries will be added if they don’t exist | +### googleapis commit history + +If both `baseline_generation_config` and `current_generation_config` are +specified, and they contain different googleapis commit, the commit history will +be generated into `pr_description.txt` in the `repository_path`. ## Configuration to generate a repository @@ -184,20 +190,27 @@ libraries: - proto_path: google/cloud/asset/v1p7beta1 ``` -## An example to generate a repository using `generate_repo.py` +# Local Environment Setup before running `entry_point.py` +1. Assuming Python 3 is installed, follow official guide from [Python.org](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#create-and-use-virtual-environments) to create a virtual environment. The virtual environment can be installed to any folder, usually it is recommended to be installed under the root folder of the project(`sdk-platform-java` in this case). +2. Assuming the virtual environment is installed under `sdk-platform-java`. Run the following command under the root folder of `sdk-platform-java` to install the dependencies of `library_generation` ```bash -# install python module (allows the `library_generation` module to be imported from anywhere) -python -m pip install -r library_generation/requirements.in -# generate the repository -python -m library_generation/generate_repo.py generate \ ---generation-config-yaml=/path/to/config-file \ ---repository-path=/path/to/repository +python -m pip install -r library_generation/requirements.txt +``` +3. Run the following command to install `library_generation` as a module, which allows the `library_generation` module to be imported from anywhere +```bash +python -m pip install library_generation/ ``` -## An example of generated repository using `generate_repo.py` +## An example to generate a repository using `entry_point.py` -If you run `generate_repo.py` with the example [configuration](#an-example-of-generation-configuration) +```bash +python library_generation/entry_point.py generate \ +--baseline-generation-config=/path/to/baseline_config_file \ +--current-generation-config=/path/to/current_config_file \ +--repository-path=path/to/repository +``` +If you run `entry_point.py` with the example [configuration](#an-example-of-generation-configuration) shown above, the repository structure is: ``` $repository_path diff --git a/library_generation/cli/entry_point.py b/library_generation/cli/entry_point.py new file mode 100644 index 0000000000..e751a33893 --- /dev/null +++ b/library_generation/cli/entry_point.py @@ -0,0 +1,141 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import click as click +from library_generation.generate_pr_description import generate_pr_descriptions +from library_generation.generate_repo import generate_from_yaml +from library_generation.model.generation_config import from_yaml +from library_generation.utils.generation_config_comparator import compare_config + + +@click.group(invoke_without_command=False) +@click.pass_context +@click.version_option(message="%(version)s") +def main(ctx): + pass + + +@main.command() +@click.option( + "--baseline-generation-config", + required=False, + default=None, + type=str, + help=""" + Absolute or relative path to a generation_config.yaml. + This config file is used for commit history generation, not library + generation. + """, +) +@click.option( + "--current-generation-config", + required=False, + default=None, + type=str, + help=""" + Absolute or relative path to a generation_config.yaml that contains the + metadata about library generation. + """, +) +@click.option( + "--repository-path", + type=str, + default=".", + show_default=True, + help=""" + The repository path to which the generated files + will be sent. + If not specified, the repository will be generated to the current working + directory. + """, +) +def generate( + baseline_generation_config: str, + current_generation_config: str, + repository_path: str, +): + """ + Compare baseline generation config and current generation config and + generate changed libraries based on current generation config with commit + history. + + If baseline generation config is not specified but current generation + config is specified, generate all libraries based on current generation + config without commit history. + + If current generation config is not specified but baseline generation + config is specified, raise FileNotFoundError because current generation + config should be the source of truth of library generation. + + If both baseline generation config and current generation config are not + specified, generate all libraries based on the default generation config, + which is generation_config.yaml in the current working directory. Raise + FileNotFoundError if the default config does not exist. + + The commit history, if generated, will be available in + repository_path/pr_description.txt. + """ + default_generation_config = f"{os.getcwd()}/generation_config.yaml" + + if baseline_generation_config is None and current_generation_config is None: + if not os.path.isfile(default_generation_config): + raise FileNotFoundError( + f"{default_generation_config} does not exist. " + "A valid generation config has to be passed in as " + "current_generation_config or exist in the current working " + "directory." + ) + current_generation_config = default_generation_config + elif current_generation_config is None: + raise FileNotFoundError( + "current_generation_config is not specified when " + "baseline_generation_config is specified. " + "current_generation_config should be the source of truth of " + "library generation." + ) + + current_generation_config = os.path.abspath(current_generation_config) + repository_path = os.path.abspath(repository_path) + if not baseline_generation_config: + # Execute full generation based on current_generation_config if + # baseline_generation_config is not specified. + # Do not generate pull request description. + generate_from_yaml( + config=from_yaml(current_generation_config), + repository_path=repository_path, + ) + return + + # Compare two generation configs and only generate changed libraries. + # Generate pull request description. + baseline_generation_config = os.path.abspath(baseline_generation_config) + config_change = compare_config( + baseline_config=from_yaml(baseline_generation_config), + current_config=from_yaml(current_generation_config), + ) + generate_from_yaml( + config=config_change.current_config, + repository_path=repository_path, + target_library_names=config_change.get_changed_libraries(), + ) + generate_pr_descriptions( + config=config_change.current_config, + baseline_commit=config_change.baseline_config.googleapis_commitish, + description_path=repository_path, + ) + + +if __name__ == "__main__": + main() diff --git a/library_generation/generate_pr_description.py b/library_generation/generate_pr_description.py index 2af636d2fe..03a8d2cfef 100644 --- a/library_generation/generate_pr_description.py +++ b/library_generation/generate_pr_description.py @@ -12,13 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import calendar import os import shutil +import click as click from typing import Dict - -import click from git import Commit, Repo -from library_generation.model.generation_config import from_yaml +from library_generation.model.generation_config import GenerationConfig, from_yaml from library_generation.utils.proto_path_utils import find_versioned_proto_path from library_generation.utils.commit_message_formatter import format_commit_message from library_generation.utils.commit_message_formatter import wrap_override_commit @@ -53,51 +53,61 @@ def main(ctx): This commit should be an ancestor of googleapis commit in configuration. """, ) -@click.option( - "--repo-url", - type=str, - default="https://github.com/googleapis/googleapis.git", - show_default=True, - help=""" - GitHub repository URL. - """, -) def generate( generation_config_yaml: str, - repo_url: str, baseline_commit: str, -) -> str: - description = generate_pr_descriptions( - generation_config_yaml=generation_config_yaml, - repo_url=repo_url, - baseline_commit=baseline_commit, - ) +) -> None: idx = generation_config_yaml.rfind("/") config_path = generation_config_yaml[:idx] - with open(f"{config_path}/pr_description.txt", "w+") as f: - f.write(description) - return description + generate_pr_descriptions( + config=from_yaml(generation_config_yaml), + baseline_commit=baseline_commit, + description_path=config_path, + ) def generate_pr_descriptions( - generation_config_yaml: str, - repo_url: str, + config: GenerationConfig, baseline_commit: str, -) -> str: - config = from_yaml(generation_config_yaml) + description_path: str, + repo_url: str = "https://github.com/googleapis/googleapis.git", +) -> None: + """ + Generate pull request description from baseline_commit (exclusive) to the + googleapis commit (inclusive) in the given generation config. + + The pull request description will be generated into + description_path/pr_description.txt. + + :param config: a GenerationConfig object. The googleapis commit in this + configuration is the latest commit, inclusively, from which the commit + message is considered. + :param baseline_commit: The baseline (oldest) commit, exclusively, from + which the commit message is considered. This commit should be an ancestor + of googleapis commit in configuration. + :param description_path: the path to which the pull request description + file goes. + :param repo_url: the GitHub repository from which retrieves the commit + history. + """ paths = config.get_proto_path_to_library_name() - return __get_commit_messages( + description = get_commit_messages( repo_url=repo_url, - latest_commit=config.googleapis_commitish, + current_commit=config.googleapis_commitish, baseline_commit=baseline_commit, paths=paths, is_monorepo=config.is_monorepo, ) + description_file = f"{description_path}/pr_description.txt" + print(f"Writing pull request description to {description_file}") + with open(description_file, "w+") as f: + f.write(description) + -def __get_commit_messages( +def get_commit_messages( repo_url: str, - latest_commit: str, + current_commit: str, baseline_commit: str, paths: Dict[str, str], is_monorepo: bool, @@ -109,7 +119,7 @@ def __get_commit_messages( Note that baseline_commit should be an ancestor of latest_commit. :param repo_url: the url of the repository. - :param latest_commit: the newest commit to be considered in + :param current_commit: the newest commit to be considered in selecting commit message. :param baseline_commit: the oldest commit to be considered in selecting commit message. This commit should be an ancestor of @@ -121,7 +131,15 @@ def __get_commit_messages( shutil.rmtree(tmp_dir, ignore_errors=True) os.mkdir(tmp_dir) repo = Repo.clone_from(repo_url, tmp_dir) - commit = repo.commit(latest_commit) + commit = repo.commit(current_commit) + current_commit_time = __get_commit_timestamp(commit) + baseline_commit_time = __get_commit_timestamp(repo.commit(baseline_commit)) + if current_commit_time <= baseline_commit_time: + raise ValueError( + f"current_commit ({current_commit[:7]}, committed on " + f"{current_commit_time}) should be newer than baseline_commit " + f"({baseline_commit[:7]}, committed on {baseline_commit_time})." + ) qualified_commits = {} while str(commit.hexsha) != baseline_commit: commit_and_name = __filter_qualified_commit(paths=paths, commit=commit) @@ -133,7 +151,7 @@ def __get_commit_messages( commit = commit_parents[0] shutil.rmtree(tmp_dir, ignore_errors=True) return __combine_commit_messages( - latest_commit=latest_commit, + current_commit=current_commit, baseline_commit=baseline_commit, commits=qualified_commits, is_monorepo=is_monorepo, @@ -159,13 +177,13 @@ def __filter_qualified_commit(paths: Dict[str, str], commit: Commit) -> (Commit, def __combine_commit_messages( - latest_commit: str, + current_commit: str, baseline_commit: str, commits: Dict[Commit, str], is_monorepo: bool, ) -> str: messages = [ - f"This pull request is generated with proto changes between googleapis commit {baseline_commit} (exclusive) and {latest_commit} (inclusive).", + f"This pull request is generated with proto changes between googleapis commit {baseline_commit} (exclusive) and {current_commit} (inclusive).", "Qualified commits are:", ] for commit in commits: @@ -183,5 +201,16 @@ def __combine_commit_messages( return "\n".join(messages) +def __get_commit_timestamp(commit: Commit) -> int: + """ + # Convert datetime to UTC timestamp. For more info: + # https://stackoverflow.com/questions/5067218/get-utc-timestamp-in-python-with-datetime + + :param commit: a Commit object + :return: the timestamp of the commit + """ + return calendar.timegm(commit.committed_datetime.utctimetuple()) + + if __name__ == "__main__": main() diff --git a/library_generation/generate_repo.py b/library_generation/generate_repo.py index 9af3a08c56..a15cfb5f2a 100755 --- a/library_generation/generate_repo.py +++ b/library_generation/generate_repo.py @@ -12,13 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import click as click import library_generation.utils.utilities as util -import click -import os from library_generation.generate_composed_library import generate_composed_library -from library_generation.model.generation_config import GenerationConfig -from library_generation.model.generation_config import from_yaml +from library_generation.model.generation_config import GenerationConfig, from_yaml from library_generation.model.library_config import LibraryConfig from library_generation.utils.monorepo_postprocessor import monorepo_postprocessing @@ -75,8 +72,9 @@ def generate( target_library_names: str, repository_path: str, ): + config = from_yaml(generation_config_yaml) generate_from_yaml( - generation_config_yaml=generation_config_yaml, + config=config, repository_path=repository_path, target_library_names=target_library_names.split(",") if target_library_names is not None @@ -85,30 +83,23 @@ def generate( def generate_from_yaml( - generation_config_yaml: str, + config: GenerationConfig, repository_path: str, target_library_names: list[str] = None, ) -> None: """ - Parses a config yaml and generates libraries via + Based on the generation config, generates libraries via generate_composed_library.py - :param generation_config_yaml: Path to generation_config.yaml that contains - the metadata about library generation - :param repository_path: If specified, the generated files will be sent to - this location. If not specified, the repository will be generated to the - current working directory. + + :param config: a GenerationConfig object. + :param repository_path: The repository path to which the generated files + will be sent. :param target_library_names: a list of libraries to be generated. - If specified, only the library whose library_name is in - target-library-names will be generated. + If specified, only the library whose library_name is in target_library_names + will be generated. If specified with an empty list, then no library will be generated. If not specified, all libraries in the configuration yaml will be generated. """ - # convert paths to absolute paths, so they can be correctly referenced in - # downstream scripts - generation_config_yaml = os.path.abspath(generation_config_yaml) - repository_path = os.path.abspath(repository_path) - - config = from_yaml(generation_config_yaml) target_libraries = get_target_libraries( config=config, target_library_names=target_library_names ) @@ -117,7 +108,7 @@ def generate_from_yaml( ) for library_path, library in repo_config.libraries.items(): - print(f"generating library {library.api_shortname}") + print(f"generating library {library.get_library_name()}") generate_composed_library( config=config, diff --git a/library_generation/model/config_change.py b/library_generation/model/config_change.py index 5b1a97d22a..bfb83a87e6 100644 --- a/library_generation/model/config_change.py +++ b/library_generation/model/config_change.py @@ -47,9 +47,9 @@ def __init__(self, hash_value: int, library: LibraryConfig): class LibraryChange: - def __init__(self, changed_param: str, latest_value: str, library_name: str = ""): + def __init__(self, changed_param: str, current_value: str, library_name: str = ""): self.changed_param = changed_param - self.latest_value = latest_value + self.current_value = current_value self.library_name = library_name @@ -66,17 +66,17 @@ def __init__( self, change_to_libraries: dict[ChangeType, list[LibraryChange]], baseline_config: GenerationConfig, - latest_config: GenerationConfig, + current_config: GenerationConfig, ): self.change_to_libraries = change_to_libraries self.baseline_config = baseline_config - self.latest_config = latest_config + self.current_config = current_config def get_changed_libraries(self) -> Optional[list[str]]: """ Returns a unique, sorted list of library name of changed libraries. None if there is a repository level change, which means all libraries - in the latest_config will be generated. + in the current_config will be generated. :return: library names of change libraries. """ if ChangeType.REPO_LEVEL_CHANGE in self.change_to_libraries: @@ -108,8 +108,8 @@ def get_qualified_commits( os.mkdir(tmp_dir) # we only need commit history, thus shadow clone is enough. repo = Repo.clone_from(url=repo_url, to_path=tmp_dir, filter=["blob:none"]) - commit = repo.commit(self.latest_config.googleapis_commitish) - proto_paths = self.latest_config.get_proto_path_to_library_name() + commit = repo.commit(self.current_config.googleapis_commitish) + proto_paths = self.current_config.get_proto_path_to_library_name() qualified_commits = [] while str(commit.hexsha) != self.baseline_config.googleapis_commitish: qualified_commit = ConfigChange.__create_qualified_commit( diff --git a/library_generation/requirements.in b/library_generation/requirements.txt similarity index 95% rename from library_generation/requirements.in rename to library_generation/requirements.txt index bbbe952c85..0eaf02faa5 100644 --- a/library_generation/requirements.in +++ b/library_generation/requirements.txt @@ -4,7 +4,7 @@ attrs==23.2.0 black==23.12.1 click==8.1.7 gitdb==4.0.11 -GitPython==3.1.40 +GitPython==3.1.41 Jinja2==3.1.2 lxml==5.0.0 MarkupSafe==2.1.3 diff --git a/library_generation/test/cli/__init__.py b/library_generation/test/cli/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/library_generation/test/cli/entry_point_unit_tests.py b/library_generation/test/cli/entry_point_unit_tests.py new file mode 100644 index 0000000000..ec65753609 --- /dev/null +++ b/library_generation/test/cli/entry_point_unit_tests.py @@ -0,0 +1,43 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest +from click.testing import CliRunner +from library_generation.cli.entry_point import generate + + +class EntryPointTest(unittest.TestCase): + def test_entry_point_without_config_raise_file_exception(self): + runner = CliRunner() + # noinspection PyTypeChecker + result = runner.invoke(generate, ["--repository-path=."]) + self.assertEqual(1, result.exit_code) + self.assertEqual(FileNotFoundError, result.exc_info[0]) + self.assertRegex( + result.exception.args[0], "generation_config.yaml does not exist." + ) + + def test_entry_point_with_baseline_without_current_raise_file_exception(self): + runner = CliRunner() + # noinspection PyTypeChecker + result = runner.invoke( + generate, + ["--baseline-generation-config=path/to/config.yaml", "--repository-path=."], + ) + self.assertEqual(1, result.exit_code) + self.assertEqual(FileNotFoundError, result.exc_info[0]) + self.assertRegex( + result.exception.args[0], + "current_generation_config is not specified when " + "baseline_generation_config is specified.", + ) diff --git a/library_generation/test/generate_pr_description_unit_tests.py b/library_generation/test/generate_pr_description_unit_tests.py new file mode 100644 index 0000000000..35dd26f952 --- /dev/null +++ b/library_generation/test/generate_pr_description_unit_tests.py @@ -0,0 +1,49 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +from library_generation.generate_pr_description import get_commit_messages + + +class GeneratePrDescriptionTest(unittest.TestCase): + def test_get_commit_messages_current_is_older_raise_exception(self): + # committed on April 1st, 2024 + current_commit = "36441693dddaf0ed73951ad3a15c215a332756aa" + # committed on April 2nd, 2024 + baseline_commit = "d5020fff4cbe108bdf506074791c56cff7840bef" + self.assertRaisesRegex( + ValueError, + "newer than", + get_commit_messages, + "https://github.com/googleapis/googleapis.git", + current_commit, + baseline_commit, + {}, + True, + ) + + def test_get_commit_messages_current_and_baseline_are_same_raise_exception(self): + # committed on April 1st, 2024 + current_commit = "36441693dddaf0ed73951ad3a15c215a332756aa" + baseline_commit = "36441693dddaf0ed73951ad3a15c215a332756aa" + self.assertRaisesRegex( + ValueError, + "newer than", + get_commit_messages, + "https://github.com/googleapis/googleapis.git", + current_commit, + baseline_commit, + {}, + True, + ) diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index 2acfa7d3b9..2c356d24a1 100644 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -42,15 +42,16 @@ "java-bigtable": "chore/test-hermetic-build", } config_dir = f"{script_dir}/resources/integration" -config_name = "generation_config.yaml" -monorepo_baseline_commit = "a17d4caf184b050d50cacf2b0d579ce72c31ce74" -split_repo_baseline_commit = "679060c64136e85b52838f53cfe612ce51e60d1d" +baseline_config_name = "baseline_generation_config.yaml" +current_config_name = "current_generation_config.yaml" class IntegrationTest(unittest.TestCase): - def test_entry_point_running_in_container(self): - self.__build_image(docker_file=build_file, cwd=repo_root_dir) + @classmethod + def setUpClass(cls) -> None: + IntegrationTest.__build_image(docker_file=build_file, cwd=repo_root_dir) + def test_entry_point_running_in_container(self): shutil.rmtree(f"{golden_dir}", ignore_errors=True) os.makedirs(f"{golden_dir}", exist_ok=True) config_files = self.__get_config_files(config_dir) @@ -74,15 +75,11 @@ def test_entry_point_running_in_container(self): ) repo_volumes = f"-v repo-{repo}:/workspace/{repo} -v config-{repo}:/workspace/config-{repo}" # 4. run entry_point.py in docker container - baseline_commit = ( - monorepo_baseline_commit - if repo == "google-cloud-java" - else split_repo_baseline_commit - ) self.__run_entry_point_in_docker_container( repo=repo, repo_volumes=repo_volumes, - baseline_commit=baseline_commit, + baseline_config=baseline_config_name, + current_config=current_config_name, ) # 5. compare generation result with golden files print( @@ -130,7 +127,7 @@ def test_entry_point_running_in_container(self): self.assertTrue(len(generated_only) == 0) self.assertTrue(len(diff_files) == 0) - print(" No differences found in {library_name}") + print(f" No differences found in {library_name}") # compare .repo-metadata.json self.assertTrue( self.__compare_json_files( @@ -162,7 +159,7 @@ def test_entry_point_running_in_container(self): ) print(" pom.xml comparison succeed.") # compare PR description - description_file = f"{config_dir}/{repo}/pr_description.txt" + description_file = f"{output_dir}/{repo}/pr_description.txt" self.assertTrue( cmp( f"{config_dir}/{repo}/pr-description-golden.txt", @@ -243,7 +240,11 @@ def __bind_device_to_volumes(cls, volume_name: str, device_dir: str): @classmethod def __run_entry_point_in_docker_container( - cls, repo: str, repo_volumes: str, baseline_commit: str + cls, + repo: str, + repo_volumes: str, + baseline_config: str, + current_config: str, ): subprocess.check_call( [ @@ -266,41 +267,14 @@ def __run_entry_point_in_docker_container( "/src", image_tag, "python", - "/src/generate_repo.py", + "/src/cli/entry_point.py", "generate", - f"--generation-config-yaml=/workspace/config-{repo}/{config_name}", + f"--baseline-generation-config=/workspace/config-{repo}/{baseline_config}", + f"--current-generation-config=/workspace/config-{repo}/{current_config}", f"--repository-path=/workspace/{repo}", ] ) - subprocess.check_call( - [ - "docker", - "run", - "--rm", - "-v", - f"repo-{repo}:/workspace/{repo}", - "-v", - f"config-{repo}:/workspace/config-{repo}", - "-v", - "/tmp:/tmp", - "-v", - "/var/run/docker.sock:/var/run/docker.sock", - "-e", - "RUNNING_IN_DOCKER=true", - "-e", - f"REPO_BINDING_VOLUMES={repo_volumes}", - "-w", - "/src", - image_tag, - "python", - "/src/generate_pr_description.py", - "generate", - f"--generation-config-yaml=/workspace/config-{repo}/{config_name}", - f"--baseline-commit={baseline_commit}", - ] - ) - @classmethod def __get_config_files(cls, path: str) -> list[tuple[str, str]]: config_files = [] @@ -310,7 +284,7 @@ def __get_config_files(cls, path: str) -> list[tuple[str, str]]: repo = sub_dir.name if repo in ["golden", "java-bigtable"]: continue - config = f"{sub_dir}/{config_name}" + config = f"{sub_dir}/{current_config_name}" config_files.append((repo, config)) return config_files diff --git a/library_generation/test/model/config_change_unit_tests.py b/library_generation/test/model/config_change_unit_tests.py index d6309c98c1..5df6e8b021 100644 --- a/library_generation/test/model/config_change_unit_tests.py +++ b/library_generation/test/model/config_change_unit_tests.py @@ -33,13 +33,13 @@ def test_get_changed_libraries_with_repo_level_change_returns_all_libraries_chan ChangeType.LIBRARY_LEVEL_CHANGE: [ LibraryChange( changed_param="test", - latest_value="test", + current_value="test", library_name="test-library", ) ], }, baseline_config=ConfigChangeTest.__get_a_gen_config(), - latest_config=ConfigChangeTest.__get_a_gen_config(), + current_config=ConfigChangeTest.__get_a_gen_config(), ) self.assertEqual( ConfigChange.ALL_LIBRARIES_CHANGED, @@ -52,32 +52,32 @@ def test_get_changed_libraries_with_library_level_change_returns_list(self): ChangeType.LIBRARY_LEVEL_CHANGE: [ LibraryChange( changed_param="test", - latest_value="test", + current_value="test", library_name="a-library", ), LibraryChange( changed_param="test", - latest_value="test", + current_value="test", library_name="another-library", ), ], ChangeType.LIBRARIES_ADDITION: [ LibraryChange( changed_param="test", - latest_value="test", + current_value="test", library_name="new-library", ), ], ChangeType.GAPIC_ADDITION: [ LibraryChange( changed_param="test", - latest_value="test", + current_value="test", library_name="library-with-new-version", ), ], }, baseline_config=ConfigChangeTest.__get_a_gen_config(), - latest_config=ConfigChangeTest.__get_a_gen_config(), + current_config=ConfigChangeTest.__get_a_gen_config(), ) self.assertEqual( ["a-library", "another-library", "library-with-new-version", "new-library"], @@ -92,18 +92,18 @@ def test_get_changed_libraries_with_duplicated_library_name_returns_unique_name( ChangeType.LIBRARY_LEVEL_CHANGE: [ LibraryChange( changed_param="a-param", - latest_value="new_test", + current_value="new_test", library_name="a-library", ), LibraryChange( changed_param="another-param", - latest_value="new_value", + current_value="new_value", library_name="a-library", ), ], }, baseline_config=ConfigChangeTest.__get_a_gen_config(), - latest_config=ConfigChangeTest.__get_a_gen_config(), + current_config=ConfigChangeTest.__get_a_gen_config(), ) self.assertEqual( ["a-library"], @@ -119,14 +119,14 @@ def test_get_changed_libraries_with_mix_changes_returns_list(self): ChangeType.LIBRARY_LEVEL_CHANGE: [ LibraryChange( changed_param="test", - latest_value="test", + current_value="test", library_name="a-library", ) ], ChangeType.LIBRARIES_ADDITION: [ LibraryChange( changed_param="test", - latest_value="test", + current_value="test", library_name="new-library", ), ], @@ -134,7 +134,7 @@ def test_get_changed_libraries_with_mix_changes_returns_list(self): baseline_config=ConfigChangeTest.__get_a_gen_config( googleapis_commitish=baseline_commit ), - latest_config=ConfigChangeTest.__get_a_gen_config( + current_config=ConfigChangeTest.__get_a_gen_config( googleapis_commitish=latest_commit, libraries=[ ConfigChangeTest.__get_a_library_config( @@ -163,7 +163,7 @@ def test_get_qualified_commits_success(self): baseline_config=ConfigChangeTest.__get_a_gen_config( googleapis_commitish=baseline_commit ), - latest_config=ConfigChangeTest.__get_a_gen_config( + current_config=ConfigChangeTest.__get_a_gen_config( googleapis_commitish=latest_commit, libraries=[ ConfigChangeTest.__get_a_library_config( @@ -216,7 +216,7 @@ def test_get_qualified_commits_build_only_commit_returns_empty_list(self): baseline_config=ConfigChangeTest.__get_a_gen_config( googleapis_commitish=baseline_commit ), - latest_config=ConfigChangeTest.__get_a_gen_config( + current_config=ConfigChangeTest.__get_a_gen_config( googleapis_commitish=latest_commit, libraries=[ ConfigChangeTest.__get_a_library_config( diff --git a/library_generation/test/resources/integration/google-cloud-java/baseline_generation_config.yaml b/library_generation/test/resources/integration/google-cloud-java/baseline_generation_config.yaml new file mode 100644 index 0000000000..ad9eafb31e --- /dev/null +++ b/library_generation/test/resources/integration/google-cloud-java/baseline_generation_config.yaml @@ -0,0 +1,58 @@ +gapic_generator_version: 2.37.0 +protobuf_version: 25.2 +googleapis_commitish: a17d4caf184b050d50cacf2b0d579ce72c31ce74 +owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 +synthtool_commitish: 5e1fb2032fa44bc170677b38713023b4fec51a4e +template_excludes: + - ".github/*" + - ".kokoro/*" + - "samples/*" + - "CODE_OF_CONDUCT.md" + - "CONTRIBUTING.md" + - "LICENSE" + - "SECURITY.md" + - "java.header" + - "license-checks.xml" + - "renovate.json" + - ".gitignore" +libraries: + - api_shortname: apigeeconnect + name_pretty: Apigee Connect + product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" + api_description: "allows the Apigee hybrid management plane to connect securely to the MART service in the runtime plane without requiring you to expose the MART endpoint on the internet." + release_level: "stable" + library_name: "apigee-connect" + GAPICs: + - proto_path: google/cloud/apigeeconnect/v1 + + - api_shortname: alloydb + name_pretty: AlloyDB + product_documentation: https://cloud.google.com/alloydb/ + api_description: AlloyDB is a fully managed, PostgreSQL-compatible database service + with industry-leading performance, availability, and scale. + rest_documentation: https://cloud.google.com/alloydb/docs/reference/rest + GAPICs: + - proto_path: google/cloud/alloydb/v1 + - proto_path: google/cloud/alloydb/v1alpha + - proto_path: google/cloud/alloydb/v1beta + + - api_shortname: alloydb + name_pretty: AlloyDB connectors + product_documentation: https://cloud.google.com/alloydb/docs + api_description: AlloyDB is a fully-managed, PostgreSQL-compatible database for + demanding transactional workloads. It provides enterprise-grade performance and + availability while maintaining 100% compatibility with open-source PostgreSQL. + library_name: alloydb-connectors + rest_documentation: https://cloud.google.com/alloydb/docs/reference/rest + GAPICs: + - proto_path: google/cloud/alloydb/connectors/v1 + - proto_path: google/cloud/alloydb/connectors/v1alpha + - proto_path: google/cloud/alloydb/connectors/v1beta + + - api_shortname: cloudcontrolspartner + name_pretty: Cloud Controls Partner API + product_documentation: https://cloud.google.com/sovereign-controls-by-partners/docs/sovereign-partners + api_description: Provides insights about your customers and their Assured Workloads based on your Sovereign Controls by Partners offering. + GAPICs: + - proto_path: google/cloud/cloudcontrolspartner/v1 + - proto_path: google/cloud/cloudcontrolspartner/v1beta \ No newline at end of file diff --git a/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml b/library_generation/test/resources/integration/google-cloud-java/current_generation_config.yaml similarity index 100% rename from library_generation/test/resources/integration/google-cloud-java/generation_config.yaml rename to library_generation/test/resources/integration/google-cloud-java/current_generation_config.yaml diff --git a/library_generation/test/utils/generation_config_comparator_unit_tests.py b/library_generation/test/utils/generation_config_comparator_unit_tests.py index e526d6856b..c25c3eff98 100644 --- a/library_generation/test/utils/generation_config_comparator_unit_tests.py +++ b/library_generation/test/utils/generation_config_comparator_unit_tests.py @@ -62,7 +62,7 @@ def setUp(self) -> None: def test_compare_config_not_change(self): result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue(len(result.change_to_libraries) == 0) @@ -75,7 +75,7 @@ def test_compare_config_googleapis_update(self): ) result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertEqual({ChangeType.GOOGLEAPIS_COMMIT: []}, result.change_to_libraries) @@ -84,70 +84,70 @@ def test_compare_config_generator_update(self): self.latest_config.gapic_generator_version = "1.2.4" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] self.assertEqual("gapic_generator_version", config_change.changed_param) - self.assertEqual("1.2.4", config_change.latest_value) + self.assertEqual("1.2.4", config_change.current_value) def test_compare_config_owlbot_cli_update(self): self.baseline_config.owlbot_cli_image = "image_version_123" self.latest_config.owlbot_cli_image = "image_version_456" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] self.assertEqual("owlbot_cli_image", config_change.changed_param) - self.assertEqual("image_version_456", config_change.latest_value) + self.assertEqual("image_version_456", config_change.current_value) def test_compare_config_synthtool_update(self): self.baseline_config.synthtool_commitish = "commit123" self.latest_config.synthtool_commitish = "commit456" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] self.assertEqual("synthtool_commitish", config_change.changed_param) - self.assertEqual("commit456", config_change.latest_value) + self.assertEqual("commit456", config_change.current_value) def test_compare_protobuf_update(self): self.baseline_config.protobuf_version = "3.25.2" self.latest_config.protobuf_version = "3.27.0" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] self.assertEqual("protobuf_version", config_change.changed_param) - self.assertEqual("3.27.0", config_change.latest_value) + self.assertEqual("3.27.0", config_change.current_value) def test_compare_config_grpc_update(self): self.baseline_config.grpc_version = "1.60.0" self.latest_config.grpc_version = "1.61.0" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] self.assertEqual("grpc_version", config_change.changed_param) - self.assertEqual("1.61.0", config_change.latest_value) + self.assertEqual("1.61.0", config_change.current_value) def test_compare_config_template_excludes_update(self): self.baseline_config.template_excludes = [".github/*", ".kokoro/*"] @@ -159,7 +159,7 @@ def test_compare_config_template_excludes_update(self): ] result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 @@ -173,7 +173,7 @@ def test_compare_config_template_excludes_update(self): "samples/*", "CODE_OF_CONDUCT.md", ], - config_change.latest_value, + config_change.current_value, ) def test_compare_config_library_addition(self): @@ -188,7 +188,7 @@ def test_compare_config_library_addition(self): ) result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARIES_ADDITION]) == 1 @@ -200,7 +200,7 @@ def test_compare_config_api_shortname_update_without_library_name(self): self.latest_config.libraries[0].api_shortname = "new_api_shortname" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARIES_ADDITION]) == 1 @@ -224,7 +224,7 @@ def test_compare_config_library_name_update(self): self.latest_config.libraries[0].library_name = "new_library_name" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARIES_ADDITION]) == 1 @@ -236,266 +236,266 @@ def test_compare_config_api_description_update(self): self.latest_config.libraries[0].api_description = "updated description" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("api_description", config_change.changed_param) - self.assertEqual("updated description", config_change.latest_value) + self.assertEqual("updated description", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_name_pretty_update(self): self.latest_config.libraries[0].name_pretty = "new name" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("name_pretty", config_change.changed_param) - self.assertEqual("new name", config_change.latest_value) + self.assertEqual("new name", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_product_docs_update(self): self.latest_config.libraries[0].product_documentation = "new docs" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("product_documentation", config_change.changed_param) - self.assertEqual("new docs", config_change.latest_value) + self.assertEqual("new docs", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_library_type_update(self): self.latest_config.libraries[0].library_type = "GAPIC_COMBO" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("library_type", config_change.changed_param) - self.assertEqual("GAPIC_COMBO", config_change.latest_value) + self.assertEqual("GAPIC_COMBO", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_release_level_update(self): self.latest_config.libraries[0].release_level = "STABLE" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("release_level", config_change.changed_param) - self.assertEqual("STABLE", config_change.latest_value) + self.assertEqual("STABLE", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_api_id_update(self): self.latest_config.libraries[0].api_id = "new_id" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("api_id", config_change.changed_param) - self.assertEqual("new_id", config_change.latest_value) + self.assertEqual("new_id", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_api_reference_update(self): self.latest_config.libraries[0].api_reference = "new api_reference" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("api_reference", config_change.changed_param) - self.assertEqual("new api_reference", config_change.latest_value) + self.assertEqual("new api_reference", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_code_owner_team_update(self): self.latest_config.libraries[0].codeowner_team = "new team" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("codeowner_team", config_change.changed_param) - self.assertEqual("new team", config_change.latest_value) + self.assertEqual("new team", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_excluded_deps_update(self): self.latest_config.libraries[0].excluded_dependencies = "group:artifact" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("excluded_dependencies", config_change.changed_param) - self.assertEqual("group:artifact", config_change.latest_value) + self.assertEqual("group:artifact", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_excluded_poms_update(self): self.latest_config.libraries[0].excluded_poms = "pom.xml" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("excluded_poms", config_change.changed_param) - self.assertEqual("pom.xml", config_change.latest_value) + self.assertEqual("pom.xml", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_client_docs_update(self): self.latest_config.libraries[0].client_documentation = "new client docs" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("client_documentation", config_change.changed_param) - self.assertEqual("new client docs", config_change.latest_value) + self.assertEqual("new client docs", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_distribution_name_update(self): self.latest_config.libraries[0].distribution_name = "new_group:new_artifact" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("distribution_name", config_change.changed_param) - self.assertEqual("new_group:new_artifact", config_change.latest_value) + self.assertEqual("new_group:new_artifact", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_group_id_update(self): self.latest_config.libraries[0].group_id = "new_group" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("group_id", config_change.changed_param) - self.assertEqual("new_group", config_change.latest_value) + self.assertEqual("new_group", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_issue_tracker_update(self): self.latest_config.libraries[0].issue_tracker = "new issue tracker" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("issue_tracker", config_change.changed_param) - self.assertEqual("new issue tracker", config_change.latest_value) + self.assertEqual("new issue tracker", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_rest_docs_update(self): self.latest_config.libraries[0].rest_documentation = "new rest docs" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("rest_documentation", config_change.changed_param) - self.assertEqual("new rest docs", config_change.latest_value) + self.assertEqual("new rest docs", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_rpc_docs_update(self): self.latest_config.libraries[0].rpc_documentation = "new rpc docs" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("rpc_documentation", config_change.changed_param) - self.assertEqual("new rpc docs", config_change.latest_value) + self.assertEqual("new rpc docs", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_cloud_api_update(self): self.latest_config.libraries[0].cloud_api = False result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("cloud_api", config_change.changed_param) - self.assertEqual(False, config_change.latest_value) + self.assertEqual(False, config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_requires_billing_update(self): self.latest_config.libraries[0].requires_billing = False result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("requires_billing", config_change.changed_param) - self.assertEqual(False, config_change.latest_value) + self.assertEqual(False, config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_extra_versioned_mod_update(self): self.latest_config.libraries[0].extra_versioned_modules = "extra module" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("extra_versioned_modules", config_change.changed_param) - self.assertEqual("extra module", config_change.latest_value) + self.assertEqual("extra module", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_version_addition(self): @@ -504,10 +504,10 @@ def test_compare_config_version_addition(self): ] result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue(len(result.change_to_libraries[ChangeType.GAPIC_ADDITION]) == 1) config_change = result.change_to_libraries[ChangeType.GAPIC_ADDITION][0] self.assertEqual("", config_change.changed_param) - self.assertEqual("google/new/library/v1", config_change.latest_value) + self.assertEqual("google/new/library/v1", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) diff --git a/library_generation/utils/generation_config_comparator.py b/library_generation/utils/generation_config_comparator.py index 79a5f3c1aa..09b8a9c514 100644 --- a/library_generation/utils/generation_config_comparator.py +++ b/library_generation/utils/generation_config_comparator.py @@ -25,49 +25,49 @@ def compare_config( - baseline_config: GenerationConfig, latest_config: GenerationConfig + baseline_config: GenerationConfig, current_config: GenerationConfig ) -> ConfigChange: """ Compare two GenerationConfig object and output a mapping from ConfigChange to a list of ConfigChange objects. - All libraries in the latest configuration will be affected if one of the + All libraries in the current configuration will be affected if one of the repository level parameters is changed. :param baseline_config: the baseline GenerationConfig object - :param latest_config: the latest GenerationConfig object + :param current_config: the current GenerationConfig object :return: a ConfigChange objects. """ diff = defaultdict(list[LibraryChange]) baseline_params = __convert_params_to_sorted_list(baseline_config) - latest_params = __convert_params_to_sorted_list(latest_config) - for baseline_param, latest_param in zip(baseline_params, latest_params): - if baseline_param == latest_param: + current_params = __convert_params_to_sorted_list(current_config) + for baseline_param, current_param in zip(baseline_params, current_params): + if baseline_param == current_param: continue if baseline_param[0] == "googleapis_commitish": diff[ChangeType.GOOGLEAPIS_COMMIT] = [] else: config_change = LibraryChange( - changed_param=latest_param[0], - latest_value=latest_param[1], + changed_param=current_param[0], + current_value=current_param[1], ) diff[ChangeType.REPO_LEVEL_CHANGE].append(config_change) __compare_libraries( diff=diff, baseline_library_configs=baseline_config.libraries, - latest_library_configs=latest_config.libraries, + current_library_configs=current_config.libraries, ) return ConfigChange( change_to_libraries=diff, baseline_config=baseline_config, - latest_config=latest_config, + current_config=current_config, ) def __compare_libraries( diff: Dict[ChangeType, list[LibraryChange]], baseline_library_configs: List[LibraryConfig], - latest_library_configs: List[LibraryConfig], + current_library_configs: List[LibraryConfig], ) -> None: """ Compare two lists of LibraryConfig and put the difference into a @@ -75,46 +75,46 @@ def __compare_libraries( :param diff: a mapping from ConfigChange to a list of ConfigChange objects. :param baseline_library_configs: a list of LibraryConfig object. - :param latest_library_configs: a list of LibraryConfig object. + :param current_library_configs: a list of LibraryConfig object. """ baseline_libraries = __convert_to_hashed_library_dict(baseline_library_configs) - latest_libraries = __convert_to_hashed_library_dict(latest_library_configs) + current_libraries = __convert_to_hashed_library_dict(current_library_configs) changed_libraries = [] # 1st round comparison. for library_name, hash_library in baseline_libraries.items(): # 1. find any library removed from baseline_libraries. # a library is removed from baseline_libraries if the library_name - # is not in latest_libraries. + # is not in current_libraries. # please see the reason of comment out these lines of code in the # comment of ChangeType.LIBRARIES_REMOVAL. - # if library_name not in latest_libraries: + # if library_name not in current_libraries: # config_change = ConfigChange( - # changed_param="", latest_value="", library_name=library_name + # changed_param="", current_value="", library_name=library_name # ) # diff[ChangeType.LIBRARIES_REMOVAL].append(config_change) # 2. find any library that exists in both configs but at least one # parameter is changed, which means the hash value is different. if ( - library_name in latest_libraries - and hash_library.hash_value != latest_libraries[library_name].hash_value + library_name in current_libraries + and hash_library.hash_value != current_libraries[library_name].hash_value ): changed_libraries.append(library_name) # 2nd round comparison. - for library_name in latest_libraries: - # find any library added to latest_libraries. - # a library is added to latest_libraries if the library_name + for library_name in current_libraries: + # find any library added to current_libraries. + # a library is added to current_libraries if the library_name # is not in baseline_libraries. if library_name not in baseline_libraries: config_change = LibraryChange( - changed_param="", latest_value="", library_name=library_name + changed_param="", current_value="", library_name=library_name ) diff[ChangeType.LIBRARIES_ADDITION].append(config_change) # 3rd round comparison. __compare_changed_libraries( diff=diff, baseline_libraries=baseline_libraries, - latest_libraries=latest_libraries, + current_libraries=current_libraries, changed_libraries=changed_libraries, ) @@ -139,7 +139,7 @@ def __convert_to_hashed_library_dict( def __compare_changed_libraries( diff: Dict[ChangeType, list[LibraryChange]], baseline_libraries: Dict[str, HashLibrary], - latest_libraries: Dict[str, HashLibrary], + current_libraries: Dict[str, HashLibrary], changed_libraries: List[str], ) -> None: """ @@ -149,18 +149,18 @@ def __compare_changed_libraries( :param diff: a mapping from ConfigChange to a list of ConfigChange objects. :param baseline_libraries: a mapping from library_name to HashLibrary object. - :param latest_libraries: a mapping from library_name to HashLibrary object. + :param current_libraries: a mapping from library_name to HashLibrary object. :param changed_libraries: a list of library_name of changed libraries. :raise ValueError: if api_shortname of a library is changed but library_name remains the same. """ for library_name in changed_libraries: baseline_library = baseline_libraries[library_name].library - latest_library = latest_libraries[library_name].library + current_library = current_libraries[library_name].library baseline_params = __convert_params_to_sorted_list(baseline_library) - latest_params = __convert_params_to_sorted_list(latest_library) - for baseline_param, latest_param in zip(baseline_params, latest_params): - if baseline_param == latest_param: + current_params = __convert_params_to_sorted_list(current_library) + for baseline_param, current_param in zip(baseline_params, current_params): + if baseline_param == current_param: continue if baseline_param[0] == "api_shortname": raise ValueError( @@ -168,20 +168,20 @@ def __compare_changed_libraries( ) else: config_change = LibraryChange( - changed_param=latest_param[0], - latest_value=latest_param[1], + changed_param=current_param[0], + current_value=current_param[1], library_name=library_name, ) diff[ChangeType.LIBRARY_LEVEL_CHANGE].append(config_change) # compare gapic_configs baseline_gapic_configs = baseline_library.gapic_configs - latest_gapic_configs = latest_library.gapic_configs + current_gapic_configs = current_library.gapic_configs __compare_gapic_configs( diff=diff, library_name=library_name, baseline_gapic_configs=baseline_gapic_configs, - latest_gapic_configs=latest_gapic_configs, + current_gapic_configs=current_gapic_configs, ) @@ -189,29 +189,29 @@ def __compare_gapic_configs( diff: Dict[ChangeType, list[LibraryChange]], library_name: str, baseline_gapic_configs: List[GapicConfig], - latest_gapic_configs: List[GapicConfig], + current_gapic_configs: List[GapicConfig], ) -> None: baseline_proto_paths = {config.proto_path for config in baseline_gapic_configs} - latest_proto_paths = {config.proto_path for config in latest_gapic_configs} + current_proto_paths = {config.proto_path for config in current_gapic_configs} # 1st round of comparison, find any versioned proto_path is removed # from baseline gapic configs. # please see the reason of comment out these lines of code in the # comment of ChangeType.GAPIC_REMOVAL. # for proto_path in baseline_proto_paths: - # if proto_path in latest_proto_paths: + # if proto_path in current_proto_paths: # continue # config_change = ConfigChange( - # changed_param="", latest_value=proto_path, library_name=library_name + # changed_param="", current_value=proto_path, library_name=library_name # ) # diff[ChangeType.GAPIC_REMOVAL].append(config_change) # 2nd round of comparison, find any versioned proto_path is added - # to latest gapic configs. - for proto_path in latest_proto_paths: + # to current gapic configs. + for proto_path in current_proto_paths: if proto_path in baseline_proto_paths: continue config_change = LibraryChange( - changed_param="", latest_value=proto_path, library_name=library_name + changed_param="", current_value=proto_path, library_name=library_name ) diff[ChangeType.GAPIC_ADDITION].append(config_change)