From b19fa3373d3fa17531d627d4581d154374a231ff Mon Sep 17 00:00:00 2001 From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> Date: Fri, 5 Apr 2024 22:24:23 +0000 Subject: [PATCH 1/4] feat: add entry point (#2616) In this PR: - Create `entry_point.py` to combine `generate_repo.py` and `generate_pr_description.py`. - Change Integration test. Follow up of #2604. For the goal of series of PRs, please refer to [improvement proposal](https://docs.google.com/document/d/1JiCcG3X7lnxaJErKe0ES_JkyU7ECb40nf2Xez3gWvuo/edit?tab=t.g3vua2kd06gx#bookmark=id.72s3ukwwzevo). --- library_generation/README.md | 53 ++++--- library_generation/cli/entry_point.py | 141 ++++++++++++++++++ library_generation/generate_pr_description.py | 99 +++++++----- library_generation/generate_repo.py | 35 ++--- library_generation/model/config_change.py | 14 +- library_generation/test/cli/__init__.py | 0 .../test/cli/entry_point_unit_tests.py | 43 ++++++ .../generate_pr_description_unit_tests.py | 49 ++++++ library_generation/test/integration_tests.py | 64 +++----- .../test/model/config_change_unit_tests.py | 30 ++-- .../baseline_generation_config.yaml | 58 +++++++ ...ig.yaml => current_generation_config.yaml} | 0 ...generation_config_comparator_unit_tests.py | 114 +++++++------- .../utils/generation_config_comparator.py | 80 +++++----- 14 files changed, 536 insertions(+), 244 deletions(-) create mode 100644 library_generation/cli/entry_point.py create mode 100644 library_generation/test/cli/__init__.py create mode 100644 library_generation/test/cli/entry_point_unit_tests.py create mode 100644 library_generation/test/generate_pr_description_unit_tests.py create mode 100644 library_generation/test/resources/integration/google-cloud-java/baseline_generation_config.yaml rename library_generation/test/resources/integration/google-cloud-java/{generation_config.yaml => current_generation_config.yaml} (100%) diff --git a/library_generation/README.md b/library_generation/README.md index 75fc32695d..364ec6b511 100644 --- a/library_generation/README.md +++ b/library_generation/README.md @@ -1,8 +1,8 @@ # Generate a repository containing GAPIC Client Libraries -The script, `generate_repo.py`, allows you to generate a repository containing -GAPIC client libraries (a monorepo, for example, google-cloud-java) from a -configuration file. +The script, `entry_point.py`, allows you to generate a repository containing +GAPIC client libraries with googleapis commit history (a monorepo, for example, +google-cloud-java) from a configuration file. ## Environment @@ -17,30 +17,27 @@ In order to generate a version for each library, a versions.txt has to exist in `repository_path`. Please refer to [Repository path](#repository-path--repositorypath---optional) for more information. -## Parameters to generate a repository using `generate_repo.py` +## Parameters to generate a repository using `entry_point.py` -### Generation configuration yaml (`generation_config_yaml`) +### Baseline generation configuration yaml (`baseline_generation_config`) -A path to a configuration file containing parameters to generate the repository. -Please refer [Configuration to generate a repository](#configuration-to-generate-a-repository) -for more information. - -### Target library API shortname (`target_library_api_shortname`), optional +An absolute or relative path to a generation_config.yaml. +This config file is used for commit history generation, not library +generation. -If specified, the libray whose `api_shortname` equals to `target_library_api_shortname` -will be generated; otherwise all libraries in the configuration file will be -generated. -This can be useful when you just want to generate one library for debugging -purposes. +### Current generation configuration yaml (`current_generation_config`) -The default value is an empty string, which means all libraries will be generated. +An absolute or relative path to a configuration file containing parameters to +generate the repository. +Please refer [Configuration to generate a repository](#configuration-to-generate-a-repository) +for more information. ### Repository path (`repository_path`), optional The path to where the generated repository goes. The default value is the current working directory when running the script. -For example, `cd google-cloud-java && python generate_repo.py ...` without +For example, `cd google-cloud-java && python entry_point.py ...` without specifying the `--repository_path` option will modify the `google-cloud-java` repository the user `cd`'d into. @@ -49,7 +46,9 @@ right version for each library. Please refer [here](go/java-client-releasing#versionstxt-manifest) for more info of versions.txt. -## Output of `generate_repo.py` +## Output of `entry_point.py` + +### GAPIC libraries For each module (e.g. `google-cloud-java/java-asset`), the following files/folders will be created/modified: @@ -73,6 +72,11 @@ will be created/modified: | pom.xml (repo root dir) | Always generated from inputs | | versions.txt | New entries will be added if they don’t exist | +### googleapis commit history + +If both `baseline_generation_config` and `current_generation_config` are +specified, and they contain different googleapis commit, the commit history will +be generated into `pr_description.txt` in the `repository_path`. ## Configuration to generate a repository @@ -184,20 +188,23 @@ libraries: - proto_path: google/cloud/asset/v1p7beta1 ``` -## An example to generate a repository using `generate_repo.py` +## An example to generate a repository using `entry_point.py` ```bash # install python module (allows the `library_generation` module to be imported from anywhere) python -m pip install -r library_generation/requirements.in +# install library_generation module +python -m pip install library_generation # generate the repository -python -m library_generation/generate_repo.py generate \ ---generation-config-yaml=/path/to/config-file \ +python -m library_generation/entry_point.py generate \ +--baseline-generation-config=/path/to/baseline_config_file \ +--current-generation-config=/path/to/current_config_file \ --repository-path=/path/to/repository ``` -## An example of generated repository using `generate_repo.py` +## An example of generated repository using `entry_point.py` -If you run `generate_repo.py` with the example [configuration](#an-example-of-generation-configuration) +If you run `entry_point.py` with the example [configuration](#an-example-of-generation-configuration) shown above, the repository structure is: ``` $repository_path diff --git a/library_generation/cli/entry_point.py b/library_generation/cli/entry_point.py new file mode 100644 index 0000000000..e751a33893 --- /dev/null +++ b/library_generation/cli/entry_point.py @@ -0,0 +1,141 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import click as click +from library_generation.generate_pr_description import generate_pr_descriptions +from library_generation.generate_repo import generate_from_yaml +from library_generation.model.generation_config import from_yaml +from library_generation.utils.generation_config_comparator import compare_config + + +@click.group(invoke_without_command=False) +@click.pass_context +@click.version_option(message="%(version)s") +def main(ctx): + pass + + +@main.command() +@click.option( + "--baseline-generation-config", + required=False, + default=None, + type=str, + help=""" + Absolute or relative path to a generation_config.yaml. + This config file is used for commit history generation, not library + generation. + """, +) +@click.option( + "--current-generation-config", + required=False, + default=None, + type=str, + help=""" + Absolute or relative path to a generation_config.yaml that contains the + metadata about library generation. + """, +) +@click.option( + "--repository-path", + type=str, + default=".", + show_default=True, + help=""" + The repository path to which the generated files + will be sent. + If not specified, the repository will be generated to the current working + directory. + """, +) +def generate( + baseline_generation_config: str, + current_generation_config: str, + repository_path: str, +): + """ + Compare baseline generation config and current generation config and + generate changed libraries based on current generation config with commit + history. + + If baseline generation config is not specified but current generation + config is specified, generate all libraries based on current generation + config without commit history. + + If current generation config is not specified but baseline generation + config is specified, raise FileNotFoundError because current generation + config should be the source of truth of library generation. + + If both baseline generation config and current generation config are not + specified, generate all libraries based on the default generation config, + which is generation_config.yaml in the current working directory. Raise + FileNotFoundError if the default config does not exist. + + The commit history, if generated, will be available in + repository_path/pr_description.txt. + """ + default_generation_config = f"{os.getcwd()}/generation_config.yaml" + + if baseline_generation_config is None and current_generation_config is None: + if not os.path.isfile(default_generation_config): + raise FileNotFoundError( + f"{default_generation_config} does not exist. " + "A valid generation config has to be passed in as " + "current_generation_config or exist in the current working " + "directory." + ) + current_generation_config = default_generation_config + elif current_generation_config is None: + raise FileNotFoundError( + "current_generation_config is not specified when " + "baseline_generation_config is specified. " + "current_generation_config should be the source of truth of " + "library generation." + ) + + current_generation_config = os.path.abspath(current_generation_config) + repository_path = os.path.abspath(repository_path) + if not baseline_generation_config: + # Execute full generation based on current_generation_config if + # baseline_generation_config is not specified. + # Do not generate pull request description. + generate_from_yaml( + config=from_yaml(current_generation_config), + repository_path=repository_path, + ) + return + + # Compare two generation configs and only generate changed libraries. + # Generate pull request description. + baseline_generation_config = os.path.abspath(baseline_generation_config) + config_change = compare_config( + baseline_config=from_yaml(baseline_generation_config), + current_config=from_yaml(current_generation_config), + ) + generate_from_yaml( + config=config_change.current_config, + repository_path=repository_path, + target_library_names=config_change.get_changed_libraries(), + ) + generate_pr_descriptions( + config=config_change.current_config, + baseline_commit=config_change.baseline_config.googleapis_commitish, + description_path=repository_path, + ) + + +if __name__ == "__main__": + main() diff --git a/library_generation/generate_pr_description.py b/library_generation/generate_pr_description.py index 2af636d2fe..03a8d2cfef 100644 --- a/library_generation/generate_pr_description.py +++ b/library_generation/generate_pr_description.py @@ -12,13 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import calendar import os import shutil +import click as click from typing import Dict - -import click from git import Commit, Repo -from library_generation.model.generation_config import from_yaml +from library_generation.model.generation_config import GenerationConfig, from_yaml from library_generation.utils.proto_path_utils import find_versioned_proto_path from library_generation.utils.commit_message_formatter import format_commit_message from library_generation.utils.commit_message_formatter import wrap_override_commit @@ -53,51 +53,61 @@ def main(ctx): This commit should be an ancestor of googleapis commit in configuration. """, ) -@click.option( - "--repo-url", - type=str, - default="https://github.com/googleapis/googleapis.git", - show_default=True, - help=""" - GitHub repository URL. - """, -) def generate( generation_config_yaml: str, - repo_url: str, baseline_commit: str, -) -> str: - description = generate_pr_descriptions( - generation_config_yaml=generation_config_yaml, - repo_url=repo_url, - baseline_commit=baseline_commit, - ) +) -> None: idx = generation_config_yaml.rfind("/") config_path = generation_config_yaml[:idx] - with open(f"{config_path}/pr_description.txt", "w+") as f: - f.write(description) - return description + generate_pr_descriptions( + config=from_yaml(generation_config_yaml), + baseline_commit=baseline_commit, + description_path=config_path, + ) def generate_pr_descriptions( - generation_config_yaml: str, - repo_url: str, + config: GenerationConfig, baseline_commit: str, -) -> str: - config = from_yaml(generation_config_yaml) + description_path: str, + repo_url: str = "https://github.com/googleapis/googleapis.git", +) -> None: + """ + Generate pull request description from baseline_commit (exclusive) to the + googleapis commit (inclusive) in the given generation config. + + The pull request description will be generated into + description_path/pr_description.txt. + + :param config: a GenerationConfig object. The googleapis commit in this + configuration is the latest commit, inclusively, from which the commit + message is considered. + :param baseline_commit: The baseline (oldest) commit, exclusively, from + which the commit message is considered. This commit should be an ancestor + of googleapis commit in configuration. + :param description_path: the path to which the pull request description + file goes. + :param repo_url: the GitHub repository from which retrieves the commit + history. + """ paths = config.get_proto_path_to_library_name() - return __get_commit_messages( + description = get_commit_messages( repo_url=repo_url, - latest_commit=config.googleapis_commitish, + current_commit=config.googleapis_commitish, baseline_commit=baseline_commit, paths=paths, is_monorepo=config.is_monorepo, ) + description_file = f"{description_path}/pr_description.txt" + print(f"Writing pull request description to {description_file}") + with open(description_file, "w+") as f: + f.write(description) + -def __get_commit_messages( +def get_commit_messages( repo_url: str, - latest_commit: str, + current_commit: str, baseline_commit: str, paths: Dict[str, str], is_monorepo: bool, @@ -109,7 +119,7 @@ def __get_commit_messages( Note that baseline_commit should be an ancestor of latest_commit. :param repo_url: the url of the repository. - :param latest_commit: the newest commit to be considered in + :param current_commit: the newest commit to be considered in selecting commit message. :param baseline_commit: the oldest commit to be considered in selecting commit message. This commit should be an ancestor of @@ -121,7 +131,15 @@ def __get_commit_messages( shutil.rmtree(tmp_dir, ignore_errors=True) os.mkdir(tmp_dir) repo = Repo.clone_from(repo_url, tmp_dir) - commit = repo.commit(latest_commit) + commit = repo.commit(current_commit) + current_commit_time = __get_commit_timestamp(commit) + baseline_commit_time = __get_commit_timestamp(repo.commit(baseline_commit)) + if current_commit_time <= baseline_commit_time: + raise ValueError( + f"current_commit ({current_commit[:7]}, committed on " + f"{current_commit_time}) should be newer than baseline_commit " + f"({baseline_commit[:7]}, committed on {baseline_commit_time})." + ) qualified_commits = {} while str(commit.hexsha) != baseline_commit: commit_and_name = __filter_qualified_commit(paths=paths, commit=commit) @@ -133,7 +151,7 @@ def __get_commit_messages( commit = commit_parents[0] shutil.rmtree(tmp_dir, ignore_errors=True) return __combine_commit_messages( - latest_commit=latest_commit, + current_commit=current_commit, baseline_commit=baseline_commit, commits=qualified_commits, is_monorepo=is_monorepo, @@ -159,13 +177,13 @@ def __filter_qualified_commit(paths: Dict[str, str], commit: Commit) -> (Commit, def __combine_commit_messages( - latest_commit: str, + current_commit: str, baseline_commit: str, commits: Dict[Commit, str], is_monorepo: bool, ) -> str: messages = [ - f"This pull request is generated with proto changes between googleapis commit {baseline_commit} (exclusive) and {latest_commit} (inclusive).", + f"This pull request is generated with proto changes between googleapis commit {baseline_commit} (exclusive) and {current_commit} (inclusive).", "Qualified commits are:", ] for commit in commits: @@ -183,5 +201,16 @@ def __combine_commit_messages( return "\n".join(messages) +def __get_commit_timestamp(commit: Commit) -> int: + """ + # Convert datetime to UTC timestamp. For more info: + # https://stackoverflow.com/questions/5067218/get-utc-timestamp-in-python-with-datetime + + :param commit: a Commit object + :return: the timestamp of the commit + """ + return calendar.timegm(commit.committed_datetime.utctimetuple()) + + if __name__ == "__main__": main() diff --git a/library_generation/generate_repo.py b/library_generation/generate_repo.py index 9af3a08c56..a15cfb5f2a 100755 --- a/library_generation/generate_repo.py +++ b/library_generation/generate_repo.py @@ -12,13 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import click as click import library_generation.utils.utilities as util -import click -import os from library_generation.generate_composed_library import generate_composed_library -from library_generation.model.generation_config import GenerationConfig -from library_generation.model.generation_config import from_yaml +from library_generation.model.generation_config import GenerationConfig, from_yaml from library_generation.model.library_config import LibraryConfig from library_generation.utils.monorepo_postprocessor import monorepo_postprocessing @@ -75,8 +72,9 @@ def generate( target_library_names: str, repository_path: str, ): + config = from_yaml(generation_config_yaml) generate_from_yaml( - generation_config_yaml=generation_config_yaml, + config=config, repository_path=repository_path, target_library_names=target_library_names.split(",") if target_library_names is not None @@ -85,30 +83,23 @@ def generate( def generate_from_yaml( - generation_config_yaml: str, + config: GenerationConfig, repository_path: str, target_library_names: list[str] = None, ) -> None: """ - Parses a config yaml and generates libraries via + Based on the generation config, generates libraries via generate_composed_library.py - :param generation_config_yaml: Path to generation_config.yaml that contains - the metadata about library generation - :param repository_path: If specified, the generated files will be sent to - this location. If not specified, the repository will be generated to the - current working directory. + + :param config: a GenerationConfig object. + :param repository_path: The repository path to which the generated files + will be sent. :param target_library_names: a list of libraries to be generated. - If specified, only the library whose library_name is in - target-library-names will be generated. + If specified, only the library whose library_name is in target_library_names + will be generated. If specified with an empty list, then no library will be generated. If not specified, all libraries in the configuration yaml will be generated. """ - # convert paths to absolute paths, so they can be correctly referenced in - # downstream scripts - generation_config_yaml = os.path.abspath(generation_config_yaml) - repository_path = os.path.abspath(repository_path) - - config = from_yaml(generation_config_yaml) target_libraries = get_target_libraries( config=config, target_library_names=target_library_names ) @@ -117,7 +108,7 @@ def generate_from_yaml( ) for library_path, library in repo_config.libraries.items(): - print(f"generating library {library.api_shortname}") + print(f"generating library {library.get_library_name()}") generate_composed_library( config=config, diff --git a/library_generation/model/config_change.py b/library_generation/model/config_change.py index 5b1a97d22a..bfb83a87e6 100644 --- a/library_generation/model/config_change.py +++ b/library_generation/model/config_change.py @@ -47,9 +47,9 @@ def __init__(self, hash_value: int, library: LibraryConfig): class LibraryChange: - def __init__(self, changed_param: str, latest_value: str, library_name: str = ""): + def __init__(self, changed_param: str, current_value: str, library_name: str = ""): self.changed_param = changed_param - self.latest_value = latest_value + self.current_value = current_value self.library_name = library_name @@ -66,17 +66,17 @@ def __init__( self, change_to_libraries: dict[ChangeType, list[LibraryChange]], baseline_config: GenerationConfig, - latest_config: GenerationConfig, + current_config: GenerationConfig, ): self.change_to_libraries = change_to_libraries self.baseline_config = baseline_config - self.latest_config = latest_config + self.current_config = current_config def get_changed_libraries(self) -> Optional[list[str]]: """ Returns a unique, sorted list of library name of changed libraries. None if there is a repository level change, which means all libraries - in the latest_config will be generated. + in the current_config will be generated. :return: library names of change libraries. """ if ChangeType.REPO_LEVEL_CHANGE in self.change_to_libraries: @@ -108,8 +108,8 @@ def get_qualified_commits( os.mkdir(tmp_dir) # we only need commit history, thus shadow clone is enough. repo = Repo.clone_from(url=repo_url, to_path=tmp_dir, filter=["blob:none"]) - commit = repo.commit(self.latest_config.googleapis_commitish) - proto_paths = self.latest_config.get_proto_path_to_library_name() + commit = repo.commit(self.current_config.googleapis_commitish) + proto_paths = self.current_config.get_proto_path_to_library_name() qualified_commits = [] while str(commit.hexsha) != self.baseline_config.googleapis_commitish: qualified_commit = ConfigChange.__create_qualified_commit( diff --git a/library_generation/test/cli/__init__.py b/library_generation/test/cli/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/library_generation/test/cli/entry_point_unit_tests.py b/library_generation/test/cli/entry_point_unit_tests.py new file mode 100644 index 0000000000..ec65753609 --- /dev/null +++ b/library_generation/test/cli/entry_point_unit_tests.py @@ -0,0 +1,43 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest +from click.testing import CliRunner +from library_generation.cli.entry_point import generate + + +class EntryPointTest(unittest.TestCase): + def test_entry_point_without_config_raise_file_exception(self): + runner = CliRunner() + # noinspection PyTypeChecker + result = runner.invoke(generate, ["--repository-path=."]) + self.assertEqual(1, result.exit_code) + self.assertEqual(FileNotFoundError, result.exc_info[0]) + self.assertRegex( + result.exception.args[0], "generation_config.yaml does not exist." + ) + + def test_entry_point_with_baseline_without_current_raise_file_exception(self): + runner = CliRunner() + # noinspection PyTypeChecker + result = runner.invoke( + generate, + ["--baseline-generation-config=path/to/config.yaml", "--repository-path=."], + ) + self.assertEqual(1, result.exit_code) + self.assertEqual(FileNotFoundError, result.exc_info[0]) + self.assertRegex( + result.exception.args[0], + "current_generation_config is not specified when " + "baseline_generation_config is specified.", + ) diff --git a/library_generation/test/generate_pr_description_unit_tests.py b/library_generation/test/generate_pr_description_unit_tests.py new file mode 100644 index 0000000000..35dd26f952 --- /dev/null +++ b/library_generation/test/generate_pr_description_unit_tests.py @@ -0,0 +1,49 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +from library_generation.generate_pr_description import get_commit_messages + + +class GeneratePrDescriptionTest(unittest.TestCase): + def test_get_commit_messages_current_is_older_raise_exception(self): + # committed on April 1st, 2024 + current_commit = "36441693dddaf0ed73951ad3a15c215a332756aa" + # committed on April 2nd, 2024 + baseline_commit = "d5020fff4cbe108bdf506074791c56cff7840bef" + self.assertRaisesRegex( + ValueError, + "newer than", + get_commit_messages, + "https://github.com/googleapis/googleapis.git", + current_commit, + baseline_commit, + {}, + True, + ) + + def test_get_commit_messages_current_and_baseline_are_same_raise_exception(self): + # committed on April 1st, 2024 + current_commit = "36441693dddaf0ed73951ad3a15c215a332756aa" + baseline_commit = "36441693dddaf0ed73951ad3a15c215a332756aa" + self.assertRaisesRegex( + ValueError, + "newer than", + get_commit_messages, + "https://github.com/googleapis/googleapis.git", + current_commit, + baseline_commit, + {}, + True, + ) diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index 2acfa7d3b9..2c356d24a1 100644 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -42,15 +42,16 @@ "java-bigtable": "chore/test-hermetic-build", } config_dir = f"{script_dir}/resources/integration" -config_name = "generation_config.yaml" -monorepo_baseline_commit = "a17d4caf184b050d50cacf2b0d579ce72c31ce74" -split_repo_baseline_commit = "679060c64136e85b52838f53cfe612ce51e60d1d" +baseline_config_name = "baseline_generation_config.yaml" +current_config_name = "current_generation_config.yaml" class IntegrationTest(unittest.TestCase): - def test_entry_point_running_in_container(self): - self.__build_image(docker_file=build_file, cwd=repo_root_dir) + @classmethod + def setUpClass(cls) -> None: + IntegrationTest.__build_image(docker_file=build_file, cwd=repo_root_dir) + def test_entry_point_running_in_container(self): shutil.rmtree(f"{golden_dir}", ignore_errors=True) os.makedirs(f"{golden_dir}", exist_ok=True) config_files = self.__get_config_files(config_dir) @@ -74,15 +75,11 @@ def test_entry_point_running_in_container(self): ) repo_volumes = f"-v repo-{repo}:/workspace/{repo} -v config-{repo}:/workspace/config-{repo}" # 4. run entry_point.py in docker container - baseline_commit = ( - monorepo_baseline_commit - if repo == "google-cloud-java" - else split_repo_baseline_commit - ) self.__run_entry_point_in_docker_container( repo=repo, repo_volumes=repo_volumes, - baseline_commit=baseline_commit, + baseline_config=baseline_config_name, + current_config=current_config_name, ) # 5. compare generation result with golden files print( @@ -130,7 +127,7 @@ def test_entry_point_running_in_container(self): self.assertTrue(len(generated_only) == 0) self.assertTrue(len(diff_files) == 0) - print(" No differences found in {library_name}") + print(f" No differences found in {library_name}") # compare .repo-metadata.json self.assertTrue( self.__compare_json_files( @@ -162,7 +159,7 @@ def test_entry_point_running_in_container(self): ) print(" pom.xml comparison succeed.") # compare PR description - description_file = f"{config_dir}/{repo}/pr_description.txt" + description_file = f"{output_dir}/{repo}/pr_description.txt" self.assertTrue( cmp( f"{config_dir}/{repo}/pr-description-golden.txt", @@ -243,7 +240,11 @@ def __bind_device_to_volumes(cls, volume_name: str, device_dir: str): @classmethod def __run_entry_point_in_docker_container( - cls, repo: str, repo_volumes: str, baseline_commit: str + cls, + repo: str, + repo_volumes: str, + baseline_config: str, + current_config: str, ): subprocess.check_call( [ @@ -266,41 +267,14 @@ def __run_entry_point_in_docker_container( "/src", image_tag, "python", - "/src/generate_repo.py", + "/src/cli/entry_point.py", "generate", - f"--generation-config-yaml=/workspace/config-{repo}/{config_name}", + f"--baseline-generation-config=/workspace/config-{repo}/{baseline_config}", + f"--current-generation-config=/workspace/config-{repo}/{current_config}", f"--repository-path=/workspace/{repo}", ] ) - subprocess.check_call( - [ - "docker", - "run", - "--rm", - "-v", - f"repo-{repo}:/workspace/{repo}", - "-v", - f"config-{repo}:/workspace/config-{repo}", - "-v", - "/tmp:/tmp", - "-v", - "/var/run/docker.sock:/var/run/docker.sock", - "-e", - "RUNNING_IN_DOCKER=true", - "-e", - f"REPO_BINDING_VOLUMES={repo_volumes}", - "-w", - "/src", - image_tag, - "python", - "/src/generate_pr_description.py", - "generate", - f"--generation-config-yaml=/workspace/config-{repo}/{config_name}", - f"--baseline-commit={baseline_commit}", - ] - ) - @classmethod def __get_config_files(cls, path: str) -> list[tuple[str, str]]: config_files = [] @@ -310,7 +284,7 @@ def __get_config_files(cls, path: str) -> list[tuple[str, str]]: repo = sub_dir.name if repo in ["golden", "java-bigtable"]: continue - config = f"{sub_dir}/{config_name}" + config = f"{sub_dir}/{current_config_name}" config_files.append((repo, config)) return config_files diff --git a/library_generation/test/model/config_change_unit_tests.py b/library_generation/test/model/config_change_unit_tests.py index d6309c98c1..5df6e8b021 100644 --- a/library_generation/test/model/config_change_unit_tests.py +++ b/library_generation/test/model/config_change_unit_tests.py @@ -33,13 +33,13 @@ def test_get_changed_libraries_with_repo_level_change_returns_all_libraries_chan ChangeType.LIBRARY_LEVEL_CHANGE: [ LibraryChange( changed_param="test", - latest_value="test", + current_value="test", library_name="test-library", ) ], }, baseline_config=ConfigChangeTest.__get_a_gen_config(), - latest_config=ConfigChangeTest.__get_a_gen_config(), + current_config=ConfigChangeTest.__get_a_gen_config(), ) self.assertEqual( ConfigChange.ALL_LIBRARIES_CHANGED, @@ -52,32 +52,32 @@ def test_get_changed_libraries_with_library_level_change_returns_list(self): ChangeType.LIBRARY_LEVEL_CHANGE: [ LibraryChange( changed_param="test", - latest_value="test", + current_value="test", library_name="a-library", ), LibraryChange( changed_param="test", - latest_value="test", + current_value="test", library_name="another-library", ), ], ChangeType.LIBRARIES_ADDITION: [ LibraryChange( changed_param="test", - latest_value="test", + current_value="test", library_name="new-library", ), ], ChangeType.GAPIC_ADDITION: [ LibraryChange( changed_param="test", - latest_value="test", + current_value="test", library_name="library-with-new-version", ), ], }, baseline_config=ConfigChangeTest.__get_a_gen_config(), - latest_config=ConfigChangeTest.__get_a_gen_config(), + current_config=ConfigChangeTest.__get_a_gen_config(), ) self.assertEqual( ["a-library", "another-library", "library-with-new-version", "new-library"], @@ -92,18 +92,18 @@ def test_get_changed_libraries_with_duplicated_library_name_returns_unique_name( ChangeType.LIBRARY_LEVEL_CHANGE: [ LibraryChange( changed_param="a-param", - latest_value="new_test", + current_value="new_test", library_name="a-library", ), LibraryChange( changed_param="another-param", - latest_value="new_value", + current_value="new_value", library_name="a-library", ), ], }, baseline_config=ConfigChangeTest.__get_a_gen_config(), - latest_config=ConfigChangeTest.__get_a_gen_config(), + current_config=ConfigChangeTest.__get_a_gen_config(), ) self.assertEqual( ["a-library"], @@ -119,14 +119,14 @@ def test_get_changed_libraries_with_mix_changes_returns_list(self): ChangeType.LIBRARY_LEVEL_CHANGE: [ LibraryChange( changed_param="test", - latest_value="test", + current_value="test", library_name="a-library", ) ], ChangeType.LIBRARIES_ADDITION: [ LibraryChange( changed_param="test", - latest_value="test", + current_value="test", library_name="new-library", ), ], @@ -134,7 +134,7 @@ def test_get_changed_libraries_with_mix_changes_returns_list(self): baseline_config=ConfigChangeTest.__get_a_gen_config( googleapis_commitish=baseline_commit ), - latest_config=ConfigChangeTest.__get_a_gen_config( + current_config=ConfigChangeTest.__get_a_gen_config( googleapis_commitish=latest_commit, libraries=[ ConfigChangeTest.__get_a_library_config( @@ -163,7 +163,7 @@ def test_get_qualified_commits_success(self): baseline_config=ConfigChangeTest.__get_a_gen_config( googleapis_commitish=baseline_commit ), - latest_config=ConfigChangeTest.__get_a_gen_config( + current_config=ConfigChangeTest.__get_a_gen_config( googleapis_commitish=latest_commit, libraries=[ ConfigChangeTest.__get_a_library_config( @@ -216,7 +216,7 @@ def test_get_qualified_commits_build_only_commit_returns_empty_list(self): baseline_config=ConfigChangeTest.__get_a_gen_config( googleapis_commitish=baseline_commit ), - latest_config=ConfigChangeTest.__get_a_gen_config( + current_config=ConfigChangeTest.__get_a_gen_config( googleapis_commitish=latest_commit, libraries=[ ConfigChangeTest.__get_a_library_config( diff --git a/library_generation/test/resources/integration/google-cloud-java/baseline_generation_config.yaml b/library_generation/test/resources/integration/google-cloud-java/baseline_generation_config.yaml new file mode 100644 index 0000000000..ad9eafb31e --- /dev/null +++ b/library_generation/test/resources/integration/google-cloud-java/baseline_generation_config.yaml @@ -0,0 +1,58 @@ +gapic_generator_version: 2.37.0 +protobuf_version: 25.2 +googleapis_commitish: a17d4caf184b050d50cacf2b0d579ce72c31ce74 +owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 +synthtool_commitish: 5e1fb2032fa44bc170677b38713023b4fec51a4e +template_excludes: + - ".github/*" + - ".kokoro/*" + - "samples/*" + - "CODE_OF_CONDUCT.md" + - "CONTRIBUTING.md" + - "LICENSE" + - "SECURITY.md" + - "java.header" + - "license-checks.xml" + - "renovate.json" + - ".gitignore" +libraries: + - api_shortname: apigeeconnect + name_pretty: Apigee Connect + product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" + api_description: "allows the Apigee hybrid management plane to connect securely to the MART service in the runtime plane without requiring you to expose the MART endpoint on the internet." + release_level: "stable" + library_name: "apigee-connect" + GAPICs: + - proto_path: google/cloud/apigeeconnect/v1 + + - api_shortname: alloydb + name_pretty: AlloyDB + product_documentation: https://cloud.google.com/alloydb/ + api_description: AlloyDB is a fully managed, PostgreSQL-compatible database service + with industry-leading performance, availability, and scale. + rest_documentation: https://cloud.google.com/alloydb/docs/reference/rest + GAPICs: + - proto_path: google/cloud/alloydb/v1 + - proto_path: google/cloud/alloydb/v1alpha + - proto_path: google/cloud/alloydb/v1beta + + - api_shortname: alloydb + name_pretty: AlloyDB connectors + product_documentation: https://cloud.google.com/alloydb/docs + api_description: AlloyDB is a fully-managed, PostgreSQL-compatible database for + demanding transactional workloads. It provides enterprise-grade performance and + availability while maintaining 100% compatibility with open-source PostgreSQL. + library_name: alloydb-connectors + rest_documentation: https://cloud.google.com/alloydb/docs/reference/rest + GAPICs: + - proto_path: google/cloud/alloydb/connectors/v1 + - proto_path: google/cloud/alloydb/connectors/v1alpha + - proto_path: google/cloud/alloydb/connectors/v1beta + + - api_shortname: cloudcontrolspartner + name_pretty: Cloud Controls Partner API + product_documentation: https://cloud.google.com/sovereign-controls-by-partners/docs/sovereign-partners + api_description: Provides insights about your customers and their Assured Workloads based on your Sovereign Controls by Partners offering. + GAPICs: + - proto_path: google/cloud/cloudcontrolspartner/v1 + - proto_path: google/cloud/cloudcontrolspartner/v1beta \ No newline at end of file diff --git a/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml b/library_generation/test/resources/integration/google-cloud-java/current_generation_config.yaml similarity index 100% rename from library_generation/test/resources/integration/google-cloud-java/generation_config.yaml rename to library_generation/test/resources/integration/google-cloud-java/current_generation_config.yaml diff --git a/library_generation/test/utils/generation_config_comparator_unit_tests.py b/library_generation/test/utils/generation_config_comparator_unit_tests.py index e526d6856b..c25c3eff98 100644 --- a/library_generation/test/utils/generation_config_comparator_unit_tests.py +++ b/library_generation/test/utils/generation_config_comparator_unit_tests.py @@ -62,7 +62,7 @@ def setUp(self) -> None: def test_compare_config_not_change(self): result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue(len(result.change_to_libraries) == 0) @@ -75,7 +75,7 @@ def test_compare_config_googleapis_update(self): ) result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertEqual({ChangeType.GOOGLEAPIS_COMMIT: []}, result.change_to_libraries) @@ -84,70 +84,70 @@ def test_compare_config_generator_update(self): self.latest_config.gapic_generator_version = "1.2.4" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] self.assertEqual("gapic_generator_version", config_change.changed_param) - self.assertEqual("1.2.4", config_change.latest_value) + self.assertEqual("1.2.4", config_change.current_value) def test_compare_config_owlbot_cli_update(self): self.baseline_config.owlbot_cli_image = "image_version_123" self.latest_config.owlbot_cli_image = "image_version_456" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] self.assertEqual("owlbot_cli_image", config_change.changed_param) - self.assertEqual("image_version_456", config_change.latest_value) + self.assertEqual("image_version_456", config_change.current_value) def test_compare_config_synthtool_update(self): self.baseline_config.synthtool_commitish = "commit123" self.latest_config.synthtool_commitish = "commit456" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] self.assertEqual("synthtool_commitish", config_change.changed_param) - self.assertEqual("commit456", config_change.latest_value) + self.assertEqual("commit456", config_change.current_value) def test_compare_protobuf_update(self): self.baseline_config.protobuf_version = "3.25.2" self.latest_config.protobuf_version = "3.27.0" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] self.assertEqual("protobuf_version", config_change.changed_param) - self.assertEqual("3.27.0", config_change.latest_value) + self.assertEqual("3.27.0", config_change.current_value) def test_compare_config_grpc_update(self): self.baseline_config.grpc_version = "1.60.0" self.latest_config.grpc_version = "1.61.0" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] self.assertEqual("grpc_version", config_change.changed_param) - self.assertEqual("1.61.0", config_change.latest_value) + self.assertEqual("1.61.0", config_change.current_value) def test_compare_config_template_excludes_update(self): self.baseline_config.template_excludes = [".github/*", ".kokoro/*"] @@ -159,7 +159,7 @@ def test_compare_config_template_excludes_update(self): ] result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 @@ -173,7 +173,7 @@ def test_compare_config_template_excludes_update(self): "samples/*", "CODE_OF_CONDUCT.md", ], - config_change.latest_value, + config_change.current_value, ) def test_compare_config_library_addition(self): @@ -188,7 +188,7 @@ def test_compare_config_library_addition(self): ) result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARIES_ADDITION]) == 1 @@ -200,7 +200,7 @@ def test_compare_config_api_shortname_update_without_library_name(self): self.latest_config.libraries[0].api_shortname = "new_api_shortname" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARIES_ADDITION]) == 1 @@ -224,7 +224,7 @@ def test_compare_config_library_name_update(self): self.latest_config.libraries[0].library_name = "new_library_name" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARIES_ADDITION]) == 1 @@ -236,266 +236,266 @@ def test_compare_config_api_description_update(self): self.latest_config.libraries[0].api_description = "updated description" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("api_description", config_change.changed_param) - self.assertEqual("updated description", config_change.latest_value) + self.assertEqual("updated description", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_name_pretty_update(self): self.latest_config.libraries[0].name_pretty = "new name" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("name_pretty", config_change.changed_param) - self.assertEqual("new name", config_change.latest_value) + self.assertEqual("new name", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_product_docs_update(self): self.latest_config.libraries[0].product_documentation = "new docs" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("product_documentation", config_change.changed_param) - self.assertEqual("new docs", config_change.latest_value) + self.assertEqual("new docs", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_library_type_update(self): self.latest_config.libraries[0].library_type = "GAPIC_COMBO" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("library_type", config_change.changed_param) - self.assertEqual("GAPIC_COMBO", config_change.latest_value) + self.assertEqual("GAPIC_COMBO", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_release_level_update(self): self.latest_config.libraries[0].release_level = "STABLE" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("release_level", config_change.changed_param) - self.assertEqual("STABLE", config_change.latest_value) + self.assertEqual("STABLE", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_api_id_update(self): self.latest_config.libraries[0].api_id = "new_id" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("api_id", config_change.changed_param) - self.assertEqual("new_id", config_change.latest_value) + self.assertEqual("new_id", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_api_reference_update(self): self.latest_config.libraries[0].api_reference = "new api_reference" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("api_reference", config_change.changed_param) - self.assertEqual("new api_reference", config_change.latest_value) + self.assertEqual("new api_reference", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_code_owner_team_update(self): self.latest_config.libraries[0].codeowner_team = "new team" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("codeowner_team", config_change.changed_param) - self.assertEqual("new team", config_change.latest_value) + self.assertEqual("new team", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_excluded_deps_update(self): self.latest_config.libraries[0].excluded_dependencies = "group:artifact" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("excluded_dependencies", config_change.changed_param) - self.assertEqual("group:artifact", config_change.latest_value) + self.assertEqual("group:artifact", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_excluded_poms_update(self): self.latest_config.libraries[0].excluded_poms = "pom.xml" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("excluded_poms", config_change.changed_param) - self.assertEqual("pom.xml", config_change.latest_value) + self.assertEqual("pom.xml", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_client_docs_update(self): self.latest_config.libraries[0].client_documentation = "new client docs" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("client_documentation", config_change.changed_param) - self.assertEqual("new client docs", config_change.latest_value) + self.assertEqual("new client docs", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_distribution_name_update(self): self.latest_config.libraries[0].distribution_name = "new_group:new_artifact" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("distribution_name", config_change.changed_param) - self.assertEqual("new_group:new_artifact", config_change.latest_value) + self.assertEqual("new_group:new_artifact", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_group_id_update(self): self.latest_config.libraries[0].group_id = "new_group" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("group_id", config_change.changed_param) - self.assertEqual("new_group", config_change.latest_value) + self.assertEqual("new_group", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_issue_tracker_update(self): self.latest_config.libraries[0].issue_tracker = "new issue tracker" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("issue_tracker", config_change.changed_param) - self.assertEqual("new issue tracker", config_change.latest_value) + self.assertEqual("new issue tracker", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_rest_docs_update(self): self.latest_config.libraries[0].rest_documentation = "new rest docs" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("rest_documentation", config_change.changed_param) - self.assertEqual("new rest docs", config_change.latest_value) + self.assertEqual("new rest docs", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_rpc_docs_update(self): self.latest_config.libraries[0].rpc_documentation = "new rpc docs" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("rpc_documentation", config_change.changed_param) - self.assertEqual("new rpc docs", config_change.latest_value) + self.assertEqual("new rpc docs", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_cloud_api_update(self): self.latest_config.libraries[0].cloud_api = False result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("cloud_api", config_change.changed_param) - self.assertEqual(False, config_change.latest_value) + self.assertEqual(False, config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_requires_billing_update(self): self.latest_config.libraries[0].requires_billing = False result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("requires_billing", config_change.changed_param) - self.assertEqual(False, config_change.latest_value) + self.assertEqual(False, config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_extra_versioned_mod_update(self): self.latest_config.libraries[0].extra_versioned_modules = "extra module" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("extra_versioned_modules", config_change.changed_param) - self.assertEqual("extra module", config_change.latest_value) + self.assertEqual("extra module", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_version_addition(self): @@ -504,10 +504,10 @@ def test_compare_config_version_addition(self): ] result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue(len(result.change_to_libraries[ChangeType.GAPIC_ADDITION]) == 1) config_change = result.change_to_libraries[ChangeType.GAPIC_ADDITION][0] self.assertEqual("", config_change.changed_param) - self.assertEqual("google/new/library/v1", config_change.latest_value) + self.assertEqual("google/new/library/v1", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) diff --git a/library_generation/utils/generation_config_comparator.py b/library_generation/utils/generation_config_comparator.py index 79a5f3c1aa..09b8a9c514 100644 --- a/library_generation/utils/generation_config_comparator.py +++ b/library_generation/utils/generation_config_comparator.py @@ -25,49 +25,49 @@ def compare_config( - baseline_config: GenerationConfig, latest_config: GenerationConfig + baseline_config: GenerationConfig, current_config: GenerationConfig ) -> ConfigChange: """ Compare two GenerationConfig object and output a mapping from ConfigChange to a list of ConfigChange objects. - All libraries in the latest configuration will be affected if one of the + All libraries in the current configuration will be affected if one of the repository level parameters is changed. :param baseline_config: the baseline GenerationConfig object - :param latest_config: the latest GenerationConfig object + :param current_config: the current GenerationConfig object :return: a ConfigChange objects. """ diff = defaultdict(list[LibraryChange]) baseline_params = __convert_params_to_sorted_list(baseline_config) - latest_params = __convert_params_to_sorted_list(latest_config) - for baseline_param, latest_param in zip(baseline_params, latest_params): - if baseline_param == latest_param: + current_params = __convert_params_to_sorted_list(current_config) + for baseline_param, current_param in zip(baseline_params, current_params): + if baseline_param == current_param: continue if baseline_param[0] == "googleapis_commitish": diff[ChangeType.GOOGLEAPIS_COMMIT] = [] else: config_change = LibraryChange( - changed_param=latest_param[0], - latest_value=latest_param[1], + changed_param=current_param[0], + current_value=current_param[1], ) diff[ChangeType.REPO_LEVEL_CHANGE].append(config_change) __compare_libraries( diff=diff, baseline_library_configs=baseline_config.libraries, - latest_library_configs=latest_config.libraries, + current_library_configs=current_config.libraries, ) return ConfigChange( change_to_libraries=diff, baseline_config=baseline_config, - latest_config=latest_config, + current_config=current_config, ) def __compare_libraries( diff: Dict[ChangeType, list[LibraryChange]], baseline_library_configs: List[LibraryConfig], - latest_library_configs: List[LibraryConfig], + current_library_configs: List[LibraryConfig], ) -> None: """ Compare two lists of LibraryConfig and put the difference into a @@ -75,46 +75,46 @@ def __compare_libraries( :param diff: a mapping from ConfigChange to a list of ConfigChange objects. :param baseline_library_configs: a list of LibraryConfig object. - :param latest_library_configs: a list of LibraryConfig object. + :param current_library_configs: a list of LibraryConfig object. """ baseline_libraries = __convert_to_hashed_library_dict(baseline_library_configs) - latest_libraries = __convert_to_hashed_library_dict(latest_library_configs) + current_libraries = __convert_to_hashed_library_dict(current_library_configs) changed_libraries = [] # 1st round comparison. for library_name, hash_library in baseline_libraries.items(): # 1. find any library removed from baseline_libraries. # a library is removed from baseline_libraries if the library_name - # is not in latest_libraries. + # is not in current_libraries. # please see the reason of comment out these lines of code in the # comment of ChangeType.LIBRARIES_REMOVAL. - # if library_name not in latest_libraries: + # if library_name not in current_libraries: # config_change = ConfigChange( - # changed_param="", latest_value="", library_name=library_name + # changed_param="", current_value="", library_name=library_name # ) # diff[ChangeType.LIBRARIES_REMOVAL].append(config_change) # 2. find any library that exists in both configs but at least one # parameter is changed, which means the hash value is different. if ( - library_name in latest_libraries - and hash_library.hash_value != latest_libraries[library_name].hash_value + library_name in current_libraries + and hash_library.hash_value != current_libraries[library_name].hash_value ): changed_libraries.append(library_name) # 2nd round comparison. - for library_name in latest_libraries: - # find any library added to latest_libraries. - # a library is added to latest_libraries if the library_name + for library_name in current_libraries: + # find any library added to current_libraries. + # a library is added to current_libraries if the library_name # is not in baseline_libraries. if library_name not in baseline_libraries: config_change = LibraryChange( - changed_param="", latest_value="", library_name=library_name + changed_param="", current_value="", library_name=library_name ) diff[ChangeType.LIBRARIES_ADDITION].append(config_change) # 3rd round comparison. __compare_changed_libraries( diff=diff, baseline_libraries=baseline_libraries, - latest_libraries=latest_libraries, + current_libraries=current_libraries, changed_libraries=changed_libraries, ) @@ -139,7 +139,7 @@ def __convert_to_hashed_library_dict( def __compare_changed_libraries( diff: Dict[ChangeType, list[LibraryChange]], baseline_libraries: Dict[str, HashLibrary], - latest_libraries: Dict[str, HashLibrary], + current_libraries: Dict[str, HashLibrary], changed_libraries: List[str], ) -> None: """ @@ -149,18 +149,18 @@ def __compare_changed_libraries( :param diff: a mapping from ConfigChange to a list of ConfigChange objects. :param baseline_libraries: a mapping from library_name to HashLibrary object. - :param latest_libraries: a mapping from library_name to HashLibrary object. + :param current_libraries: a mapping from library_name to HashLibrary object. :param changed_libraries: a list of library_name of changed libraries. :raise ValueError: if api_shortname of a library is changed but library_name remains the same. """ for library_name in changed_libraries: baseline_library = baseline_libraries[library_name].library - latest_library = latest_libraries[library_name].library + current_library = current_libraries[library_name].library baseline_params = __convert_params_to_sorted_list(baseline_library) - latest_params = __convert_params_to_sorted_list(latest_library) - for baseline_param, latest_param in zip(baseline_params, latest_params): - if baseline_param == latest_param: + current_params = __convert_params_to_sorted_list(current_library) + for baseline_param, current_param in zip(baseline_params, current_params): + if baseline_param == current_param: continue if baseline_param[0] == "api_shortname": raise ValueError( @@ -168,20 +168,20 @@ def __compare_changed_libraries( ) else: config_change = LibraryChange( - changed_param=latest_param[0], - latest_value=latest_param[1], + changed_param=current_param[0], + current_value=current_param[1], library_name=library_name, ) diff[ChangeType.LIBRARY_LEVEL_CHANGE].append(config_change) # compare gapic_configs baseline_gapic_configs = baseline_library.gapic_configs - latest_gapic_configs = latest_library.gapic_configs + current_gapic_configs = current_library.gapic_configs __compare_gapic_configs( diff=diff, library_name=library_name, baseline_gapic_configs=baseline_gapic_configs, - latest_gapic_configs=latest_gapic_configs, + current_gapic_configs=current_gapic_configs, ) @@ -189,29 +189,29 @@ def __compare_gapic_configs( diff: Dict[ChangeType, list[LibraryChange]], library_name: str, baseline_gapic_configs: List[GapicConfig], - latest_gapic_configs: List[GapicConfig], + current_gapic_configs: List[GapicConfig], ) -> None: baseline_proto_paths = {config.proto_path for config in baseline_gapic_configs} - latest_proto_paths = {config.proto_path for config in latest_gapic_configs} + current_proto_paths = {config.proto_path for config in current_gapic_configs} # 1st round of comparison, find any versioned proto_path is removed # from baseline gapic configs. # please see the reason of comment out these lines of code in the # comment of ChangeType.GAPIC_REMOVAL. # for proto_path in baseline_proto_paths: - # if proto_path in latest_proto_paths: + # if proto_path in current_proto_paths: # continue # config_change = ConfigChange( - # changed_param="", latest_value=proto_path, library_name=library_name + # changed_param="", current_value=proto_path, library_name=library_name # ) # diff[ChangeType.GAPIC_REMOVAL].append(config_change) # 2nd round of comparison, find any versioned proto_path is added - # to latest gapic configs. - for proto_path in latest_proto_paths: + # to current gapic configs. + for proto_path in current_proto_paths: if proto_path in baseline_proto_paths: continue config_change = LibraryChange( - changed_param="", latest_value=proto_path, library_name=library_name + changed_param="", current_value=proto_path, library_name=library_name ) diff[ChangeType.GAPIC_ADDITION].append(config_change) From ad5596774372349542fef276c30bec189b2aaaec Mon Sep 17 00:00:00 2001 From: Blake Li Date: Fri, 5 Apr 2024 19:22:17 -0400 Subject: [PATCH 2/4] refactor: Rename requirements.in to requirements.txt. (#2619) Rename requirements.in to requirements.txt. It is a more standard way to manage Python dependencies, it can be recognized automatically by some IDEs. Add environment setup section to README for running hermetic build scripts. --- .../library_generation.Dockerfile | 2 +- .../workflows/verify_library_generation.yaml | 14 +++------- library_generation/README.md | 26 ++++++++++++------- .../{requirements.in => requirements.txt} | 0 4 files changed, 20 insertions(+), 22 deletions(-) rename library_generation/{requirements.in => requirements.txt} (100%) diff --git a/.cloudbuild/library_generation/library_generation.Dockerfile b/.cloudbuild/library_generation/library_generation.Dockerfile index 4b3b1f4b1c..09d23a106b 100644 --- a/.cloudbuild/library_generation/library_generation.Dockerfile +++ b/.cloudbuild/library_generation/library_generation.Dockerfile @@ -26,7 +26,7 @@ RUN rm $(which python3) RUN ln -s $(which python3.11) /usr/local/bin/python RUN ln -s $(which python3.11) /usr/local/bin/python3 RUN python -m pip install --upgrade pip -RUN cd /src && python -m pip install -r requirements.in +RUN cd /src && python -m pip install -r requirements.txt RUN cd /src && python -m pip install . # set dummy git credentials for empty commit used in postprocessing diff --git a/.github/workflows/verify_library_generation.yaml b/.github/workflows/verify_library_generation.yaml index ca308461c0..c51f8ed7b0 100644 --- a/.github/workflows/verify_library_generation.yaml +++ b/.github/workflows/verify_library_generation.yaml @@ -10,17 +10,9 @@ on: name: verify_library_generation jobs: integration_tests: - strategy: - matrix: - java: [ 11 ] runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.java }} - distribution: temurin - cache: maven - uses: actions/setup-python@v5 with: python-version: 3.11 @@ -41,7 +33,7 @@ jobs: run: | set -ex pushd library_generation - pip install -r requirements.in + pip install -r requirements.txt pip install . popd - name: Run integration tests @@ -71,7 +63,7 @@ jobs: run: | set -ex pushd library_generation - pip install -r requirements.in + pip install -r requirements.txt popd - name: install synthtool shell: bash @@ -115,7 +107,7 @@ jobs: run: | set -ex pushd library_generation - pip install -r requirements.in + pip install -r requirements.txt popd - name: Lint shell: bash diff --git a/library_generation/README.md b/library_generation/README.md index 364ec6b511..04da9fda03 100644 --- a/library_generation/README.md +++ b/library_generation/README.md @@ -10,6 +10,8 @@ google-cloud-java) from a configuration file. - Java runtime environment (8 or above) - Apache Maven (used in formatting source code) - Python (3.11.6 or above) +- Docker +- Git ## Prerequisite @@ -188,22 +190,26 @@ libraries: - proto_path: google/cloud/asset/v1p7beta1 ``` +# Local Environment Setup before running `entry_point.py` + +1. Assuming Python 3 is installed, follow official guide from [Python.org](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#create-and-use-virtual-environments) to create a virtual environment. The virtual environment can be installed to any folder, usually it is recommended to be installed under the root folder of the project(`sdk-platform-java` in this case). +2. Assuming the virtual environment is installed under `sdk-platform-java`. Run the following command under the root folder of `sdk-platform-java` to install the dependencies of `library_generation` +```bash +python -m pip install -r library_generation/requirements.txt +``` +3. Run the following command to install `library_generation` as a module, which allows the `library_generation` module to be imported from anywhere +```bash +python -m pip install library_generation/ +``` + ## An example to generate a repository using `entry_point.py` ```bash -# install python module (allows the `library_generation` module to be imported from anywhere) -python -m pip install -r library_generation/requirements.in -# install library_generation module -python -m pip install library_generation -# generate the repository -python -m library_generation/entry_point.py generate \ +python library_generation/entry_point.py generate \ --baseline-generation-config=/path/to/baseline_config_file \ --current-generation-config=/path/to/current_config_file \ ---repository-path=/path/to/repository +--repository-path=path/to/repository ``` - -## An example of generated repository using `entry_point.py` - If you run `entry_point.py` with the example [configuration](#an-example-of-generation-configuration) shown above, the repository structure is: ``` diff --git a/library_generation/requirements.in b/library_generation/requirements.txt similarity index 100% rename from library_generation/requirements.in rename to library_generation/requirements.txt From e41bd8f00ffcb271b2a008364f334693d9dfdc9d Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Sat, 6 Apr 2024 02:58:56 +0200 Subject: [PATCH 3/4] deps: update dependency gitpython to v3.1.41 [security] (#2625) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [![Mend Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Change | Age | Adoption | Passing | Confidence | |---|---|---|---|---|---| | [GitPython](https://togithub.com/gitpython-developers/GitPython) | `==3.1.40` -> `==3.1.41` | [![age](https://developer.mend.io/api/mc/badges/age/pypi/GitPython/3.1.41?slim=true)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://developer.mend.io/api/mc/badges/adoption/pypi/GitPython/3.1.41?slim=true)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://developer.mend.io/api/mc/badges/compatibility/pypi/GitPython/3.1.40/3.1.41?slim=true)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://developer.mend.io/api/mc/badges/confidence/pypi/GitPython/3.1.40/3.1.41?slim=true)](https://docs.renovatebot.com/merge-confidence/) | --- > [!WARNING] > Some dependencies could not be looked up. Check the Dependency Dashboard for more information. ### GitHub Vulnerability Alerts #### [CVE-2024-22190](https://togithub.com/gitpython-developers/GitPython/security/advisories/GHSA-2mqj-m65w-jghx) ### Summary This issue exists because of an incomplete fix for CVE-2023-40590. On Windows, GitPython uses an untrusted search path if it uses a shell to run `git`, as well as when it runs `bash.exe` to interpret hooks. If either of those features are used on Windows, a malicious `git.exe` or `bash.exe` may be run from an untrusted repository. ### Details Although GitPython often avoids executing programs found in an untrusted search path since 3.1.33, two situations remain where this still occurs. Either can allow arbitrary code execution under some circumstances. #### When a shell is used GitPython can be told to run `git` commands through a shell rather than as direct subprocesses, by passing `shell=True` to any method that accepts it, or by both setting `Git.USE_SHELL = True` and not passing `shell=False`. Then the Windows `cmd.exe` shell process performs the path search, and GitPython does not prevent that shell from finding and running `git` in the current directory. When GitPython runs `git` directly rather than through a shell, the GitPython process performs the path search, and currently omits the current directory by setting `NoDefaultCurrentDirectoryInExePath` in its own environment during the `Popen` call. Although the `cmd.exe` shell will honor this environment variable when present, GitPython does not currently pass it into the shell subprocess's environment. Furthermore, because GitPython sets the subprocess CWD to the root of a repository's working tree, using a shell will run a malicious `git.exe` in an untrusted repository even if GitPython itself is run from a trusted location. This also applies if `Git.execute` is called directly with `shell=True` (or after `Git.USE_SHELL = True`) to run any command. #### When hook scripts are run On Windows, GitPython uses `bash.exe` to run hooks that appear to be scripts. However, unlike when running `git`, no steps are taken to avoid finding and running `bash.exe` in the current directory. This allows the author of an untrusted fork or branch to cause a malicious `bash.exe` to be run in some otherwise safe workflows. An example of such a scenario is if the user installs a trusted hook while on a trusted branch, then switches to an untrusted feature branch (possibly from a fork) to review proposed changes. If the untrusted feature branch contains a malicious `bash.exe` and the user's current working directory is the working tree, and the user performs an action that runs the hook, then although the hook itself is uncorrupted, it runs with the malicious `bash.exe`. Note that, while `bash.exe` is a shell, this is a separate scenario from when `git` is run using the unrelated Windows `cmd.exe` shell. ### PoC On Windows, create a `git.exe` file in a repository. Then create a `Repo` object, and call any method through it (directly or indirectly) that supports the `shell` keyword argument with `shell=True`: ```powershell mkdir testrepo git init testrepo cp ... testrepo git.exe # Replace "..." with any executable of choice. python -c "import git; print(git.Repo('testrepo').git.version(shell=True))" ``` The `git.exe` executable in the repository directory will be run. Or use no `Repo` object, but do it from the location with the `git.exe`: ```powershell cd testrepo python -c "import git; print(git.Git().version(shell=True))" ``` The `git.exe` executable in the current directory will be run. For the scenario with hooks, install a hook in a repository, create a `bash.exe` file in the current directory, and perform an operation that causes GitPython to attempt to run the hook: ```powershell mkdir testrepo cd testrepo git init mv .git/hooks/pre-commit.sample .git/hooks/pre-commit cp ... bash.exe # Replace "..." with any executable of choice. echo "Some text" >file.txt git add file.txt python -c "import git; git.Repo().index.commit('Some message')" ``` The `bash.exe` executable in the current directory will be run. ### Impact The greatest impact is probably in applications that set `Git.USE_SHELL = True` for historical reasons. (Undesired console windows had, in the past, been created in some kinds of applications, when it was not used.) Such an application may be vulnerable to arbitrary code execution from a malicious repository, even with no other exacerbating conditions. This is to say that, if a shell is used to run `git`, the full effect of CVE-2023-40590 is still present. Furthermore, as noted above, running the application itself from a trusted directory is not a sufficient mitigation. An application that does not direct GitPython to use a shell to run `git` subprocesses thus avoids most of the risk. However, there is no such straightforward way to prevent GitPython from running `bash.exe` to interpret hooks. So while the conditions needed for that to be exploited are more involved, it may be harder to mitigate decisively prior to patching. ### Possible solutions A straightforward approach would be to address each bug directly: - When a shell is used, pass `NoDefaultCurrentDirectoryInExePath` into the subprocess environment, because in that scenario the subprocess is the `cmd.exe` shell that itself performs the path search. - Set `NoDefaultCurrentDirectoryInExePath` in the GitPython process environment during the `Popen` call made to run hooks with a `bash.exe` subprocess. These need only be done on Windows. --- ### Release Notes
gitpython-developers/GitPython (GitPython) ### [`v3.1.41`](https://togithub.com/gitpython-developers/GitPython/releases/tag/3.1.41): - fix Windows security issue [Compare Source](https://togithub.com/gitpython-developers/GitPython/compare/3.1.40...3.1.41) The details about the Windows security issue [can be found in this advisory](https://togithub.com/gitpython-developers/GitPython/security/advisories/GHSA-2mqj-m65w-jghx). Special thanks go to [@​EliahKagan](https://togithub.com/EliahKagan) who reported the issue and fixed it in a single stroke, while being responsible for an incredible amount of improvements that he contributed over the last couple of months ❀️. #### What's Changed - Add `__all__` in git.exc by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1719](https://togithub.com/gitpython-developers/GitPython/pull/1719) - Set submodule update cadence to weekly by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1721](https://togithub.com/gitpython-developers/GitPython/pull/1721) - Never modify sys.path by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1720](https://togithub.com/gitpython-developers/GitPython/pull/1720) - Bump git/ext/gitdb from `8ec2390` to `ec58b7e` by [@​dependabot](https://togithub.com/dependabot) in [https://github.com/gitpython-developers/GitPython/pull/1722](https://togithub.com/gitpython-developers/GitPython/pull/1722) - Revise comments, docstrings, some messages, and a bit of code by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1725](https://togithub.com/gitpython-developers/GitPython/pull/1725) - Use zero-argument super() by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1726](https://togithub.com/gitpython-developers/GitPython/pull/1726) - Remove obsolete note in \_iter_packed_refs by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1727](https://togithub.com/gitpython-developers/GitPython/pull/1727) - Reorganize test_util and make xfail marks precise by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1729](https://togithub.com/gitpython-developers/GitPython/pull/1729) - Clarify license and make module top comments more consistent by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1730](https://togithub.com/gitpython-developers/GitPython/pull/1730) - Deprecate compat.is\_, rewriting all uses by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1732](https://togithub.com/gitpython-developers/GitPython/pull/1732) - Revise and restore some module docstrings by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1735](https://togithub.com/gitpython-developers/GitPython/pull/1735) - Make the rmtree callback Windows-only by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1739](https://togithub.com/gitpython-developers/GitPython/pull/1739) - List all non-passing tests in test summaries by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1740](https://togithub.com/gitpython-developers/GitPython/pull/1740) - Document some minor subtleties in test_util.py by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1749](https://togithub.com/gitpython-developers/GitPython/pull/1749) - Always read metadata files as UTF-8 in setup.py by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1748](https://togithub.com/gitpython-developers/GitPython/pull/1748) - Test native Windows on CI by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1745](https://togithub.com/gitpython-developers/GitPython/pull/1745) - Test macOS on CI by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1752](https://togithub.com/gitpython-developers/GitPython/pull/1752) - Let close_fds be True on all platforms by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1753](https://togithub.com/gitpython-developers/GitPython/pull/1753) - Fix IndexFile.from_tree on Windows by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1751](https://togithub.com/gitpython-developers/GitPython/pull/1751) - Remove unused TASKKILL fallback in AutoInterrupt by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1754](https://togithub.com/gitpython-developers/GitPython/pull/1754) - Don't return with operand when conceptually void by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1755](https://togithub.com/gitpython-developers/GitPython/pull/1755) - Group .gitignore entries by purpose by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1758](https://togithub.com/gitpython-developers/GitPython/pull/1758) - Adding dubious ownership handling by [@​marioaag](https://togithub.com/marioaag) in [https://github.com/gitpython-developers/GitPython/pull/1746](https://togithub.com/gitpython-developers/GitPython/pull/1746) - Avoid brittle assumptions about preexisting temporary files in tests by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1759](https://togithub.com/gitpython-developers/GitPython/pull/1759) - Overhaul noqa directives by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1760](https://togithub.com/gitpython-developers/GitPython/pull/1760) - Clarify some Git.execute kill_after_timeout limitations by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1761](https://togithub.com/gitpython-developers/GitPython/pull/1761) - Bump actions/setup-python from 4 to 5 by [@​dependabot](https://togithub.com/dependabot) in [https://github.com/gitpython-developers/GitPython/pull/1763](https://togithub.com/gitpython-developers/GitPython/pull/1763) - Don't install black on Cygwin by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1766](https://togithub.com/gitpython-developers/GitPython/pull/1766) - Extract all "import gc" to module level by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1765](https://togithub.com/gitpython-developers/GitPython/pull/1765) - Extract remaining local "import gc" to module level by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1768](https://togithub.com/gitpython-developers/GitPython/pull/1768) - Replace xfail with gc.collect in TestSubmodule.test_rename by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1767](https://togithub.com/gitpython-developers/GitPython/pull/1767) - Enable CodeQL by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1769](https://togithub.com/gitpython-developers/GitPython/pull/1769) - Replace some uses of the deprecated mktemp function by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1770](https://togithub.com/gitpython-developers/GitPython/pull/1770) - Bump github/codeql-action from 2 to 3 by [@​dependabot](https://togithub.com/dependabot) in [https://github.com/gitpython-developers/GitPython/pull/1773](https://togithub.com/gitpython-developers/GitPython/pull/1773) - Run some Windows environment variable tests only on Windows by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1774](https://togithub.com/gitpython-developers/GitPython/pull/1774) - Fix TemporaryFileSwap regression where file_path could not be Path by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1776](https://togithub.com/gitpython-developers/GitPython/pull/1776) - Improve hooks tests by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1777](https://togithub.com/gitpython-developers/GitPython/pull/1777) - Fix if items of Index is of type PathLike by [@​stegm](https://togithub.com/stegm) in [https://github.com/gitpython-developers/GitPython/pull/1778](https://togithub.com/gitpython-developers/GitPython/pull/1778) - Better document IterableObj.iter_items and improve some subclasses by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1780](https://togithub.com/gitpython-developers/GitPython/pull/1780) - Revert "Don't install black on Cygwin" by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1783](https://togithub.com/gitpython-developers/GitPython/pull/1783) - Add missing pip in $PATH on Cygwin CI by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1784](https://togithub.com/gitpython-developers/GitPython/pull/1784) - Shorten Iterable docstrings and put IterableObj first by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1785](https://togithub.com/gitpython-developers/GitPython/pull/1785) - Fix incompletely revised Iterable/IterableObj docstrings by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1786](https://togithub.com/gitpython-developers/GitPython/pull/1786) - Pre-deprecate setting Git.USE_SHELL by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1782](https://togithub.com/gitpython-developers/GitPython/pull/1782) - Deprecate Git.USE_SHELL by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1787](https://togithub.com/gitpython-developers/GitPython/pull/1787) - In handle_process_output don't forward finalizer result by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1788](https://togithub.com/gitpython-developers/GitPython/pull/1788) - Fix mypy warning "Missing return statement" by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1789](https://togithub.com/gitpython-developers/GitPython/pull/1789) - Fix two remaining Windows untrusted search path cases by [@​EliahKagan](https://togithub.com/EliahKagan) in [https://github.com/gitpython-developers/GitPython/pull/1792](https://togithub.com/gitpython-developers/GitPython/pull/1792) #### New Contributors - [@​marioaag](https://togithub.com/marioaag) made their first contribution in [https://github.com/gitpython-developers/GitPython/pull/1746](https://togithub.com/gitpython-developers/GitPython/pull/1746) - [@​stegm](https://togithub.com/stegm) made their first contribution in [https://github.com/gitpython-developers/GitPython/pull/1778](https://togithub.com/gitpython-developers/GitPython/pull/1778) **Full Changelog**: https://github.com/gitpython-developers/GitPython/compare/3.1.40...3.1.41
--- ### Configuration πŸ“… **Schedule**: Branch creation - "" (UTC), Automerge - At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. β™» **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. πŸ”• **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [Mend Renovate](https://www.mend.io/free-developer-tools/renovate/). View repository job log [here](https://developer.mend.io/github/googleapis/sdk-platform-java). --- library_generation/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library_generation/requirements.txt b/library_generation/requirements.txt index bbbe952c85..0eaf02faa5 100644 --- a/library_generation/requirements.txt +++ b/library_generation/requirements.txt @@ -4,7 +4,7 @@ attrs==23.2.0 black==23.12.1 click==8.1.7 gitdb==4.0.11 -GitPython==3.1.40 +GitPython==3.1.41 Jinja2==3.1.2 lxml==5.0.0 MarkupSafe==2.1.3 From ba648ceac869d4fab1a9cbeb40f2e04f69dad4e3 Mon Sep 17 00:00:00 2001 From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> Date: Mon, 8 Apr 2024 13:44:33 +0000 Subject: [PATCH 4/4] chore: remove tag of unmanaged dependency check (#2626) After b/328660418 is resolved, we don't need the `unmanaged-dependencies-check-latest` tag. --- .../workflows/create_additional_release_tag.yaml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/.github/workflows/create_additional_release_tag.yaml b/.github/workflows/create_additional_release_tag.yaml index 0bc731c127..aa19e89c2d 100644 --- a/.github/workflows/create_additional_release_tag.yaml +++ b/.github/workflows/create_additional_release_tag.yaml @@ -11,7 +11,7 @@ jobs: permissions: write-all steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: token: ${{ secrets.GITHUB_TOKEN }} - name: Set up Git @@ -34,14 +34,3 @@ jobs: git tag $TAG_NAME git push origin $TAG_NAME done - # Generate a tag for unmanaged dependencies check. - # Use fixed tag so that checks in handwritten libraries do not need to - # update the version. - CHECK_LATEST_TAG="unmanaged-dependencies-check-latest" - # delete and create the tag locally. - git tag --delete ${CHECK_LATEST_TAG} - git tag ${CHECK_LATEST_TAG} - # delete the tag in remote repo and push again. - git push --delete origin ${CHECK_LATEST_TAG} - git push origin ${CHECK_LATEST_TAG} -