Merge branch 'main' into renovate/platformdirs-4.x

.cloudbuild/library_generation/library_generation.Dockerfile

@@ -15,26 +15,68 @@
 # build from the root of this repo:
 FROM gcr.io/cloud-devrel-public-resources/python
 
-# install tools
+ARG SYNTHTOOL_COMMITTISH=63cc541da2c45fcfca2136c43e638da1fbae174d
+ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550
+ENV HOME=/home
+
+# install OS tools
 RUN apt-get update && apt-get install -y \
 	unzip openjdk-17-jdk rsync maven jq \
 	&& apt-get clean
 
-COPY library_generation /src
-
+# use python 3.11 (the base image has several python versions; here we define the default one)
 RUN rm $(which python3)
 RUN ln -s $(which python3.11) /usr/local/bin/python
 RUN ln -s $(which python3.11) /usr/local/bin/python3
 RUN python -m pip install --upgrade pip
-RUN cd /src && python -m pip install -r requirements.txt
-RUN cd /src && python -m pip install .
 
-# set dummy git credentials for empty commit used in postprocessing
-RUN git config --global user.email "[email protected]"
-RUN git config --global user.name "Cloud Java Bot"
+# copy source code
+COPY library_generation /src
 
-WORKDIR /workspace
-RUN chmod 750 /workspace
-RUN chmod 750 /src/generate_repo.py
+# install scripts as a python package
+WORKDIR /src
+RUN python -m pip install -r requirements.txt
+RUN python -m pip install .
+
+# install synthtool
+WORKDIR /tools
+RUN git clone https://github.com/googleapis/synthtool
+WORKDIR /tools/synthtool
+RUN git checkout "${SYNTHTOOL_COMMITTISH}"
+RUN python3 -m pip install --no-deps -e .
+RUN python3 -m pip install -r requirements.in
 
-CMD [ "/src/generate_repo.py" ]
+# Install nvm with node and npm
+ENV NODE_VERSION 20.12.0
+WORKDIR /home
+RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash
+RUN chmod o+rx /home/.nvm
+ENV NODE_PATH=/home/.nvm/versions/node/v${NODE_VERSION}/bin
+ENV PATH=${PATH}:${NODE_PATH}
+RUN node --version
+RUN npm --version
+
+# install the owl-bot CLI
+WORKDIR /tools
+RUN git clone https://github.com/googleapis/repo-automation-bots
+WORKDIR /tools/repo-automation-bots/packages/owl-bot
+RUN git checkout "${OWLBOT_CLI_COMMITTISH}"
+RUN npm i && npm run compile && npm link
+RUN owl-bot copy-code --version
+RUN chmod -R o+rx ${NODE_PATH}
+RUN ln -sf ${NODE_PATH}/* /usr/local/bin
+
+# allow users to access the script folders
+RUN chmod -R o+rx /src
+
+# set dummy git credentials for the empty commit used in postprocessing
+# we use system so all users using the container will use this configuration
+RUN git config --system user.email "[email protected]"
+RUN git config --system user.name "Cloud Java Bot"
+
+# allow read-write for /home and execution for binaries in /home/.nvm
+RUN chmod -R a+rw /home
+RUN chmod -R a+rx /home/.nvm
+
+WORKDIR /workspace
+ENTRYPOINT [ "python", "/src/cli/entry_point.py", "generate" ]

.github/workflows/anaylze_dependency.yaml → .github/workflows/analyze_dependency.yaml

@@ -25,7 +25,7 @@ jobs:
     - uses: actions/setup-java@v4
       with:
         distribution: temurin
-        java-version: 11
+        java-version: 17
         cache: maven
     - name: Set up Maven
       uses: stCarolas/[email protected]

.github/workflows/nightly.yaml

@@ -23,7 +23,7 @@ jobs:
           distribution: temurin
           cache: maven
       - run: mvn -version
-      - name: Unit Tests
+      - name: Install
         run: mvn install --errors --batch-mode --no-transfer-progress -Dcheckstyle.skip -T 1C
       - name: Create issue if previous step fails
         if: ${{ failure() }}

.github/workflows/verify_library_generation.yaml

@@ -3,14 +3,38 @@ on:
     branches:
     - main
   pull_request:
-    paths:
-    - library_generation/**
 
   workflow_dispatch:
 name: verify_library_generation
 jobs:
-  integration_tests:
+  should-run-library-generation-tests:
     runs-on: ubuntu-22.04
+    outputs:
+      should_run: ${{ steps.get_changed_directories.outputs.should_run }}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: get changed directories in the pull request
+      id: get_changed_directories
+      shell: bash
+      run: |
+        set -ex
+        git checkout "${base_ref}"
+        git checkout "${head_ref}"
+        changed_directories="$(git diff --name-only ${base_ref} ${head_ref})"
+        if [[ ${changed_directories} =~ "library_generation/" ]]; then
+          echo "should_run=true" >> $GITHUB_OUTPUT
+        else
+          echo "should_run=false" >> $GITHUB_OUTPUT
+        fi
+      env:
+        base_ref: ${{ github.event.pull_request.base.ref }}
+        head_ref: ${{ github.event.pull_request.head.ref }}
+  library-generation-integration-tests:
+    runs-on: ubuntu-22.04
+    needs: should-run-library-generation-tests
+    if: needs.should-run-library-generation-tests.outputs.should_run == 'true'
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
@@ -41,12 +65,14 @@ jobs:
       run: |
         set -x
         python -m unittest library_generation/test/integration_tests.py
-  unit_tests:
+  library-generation-unit-tests:
+    runs-on: ${{ matrix.os }}
+    needs: should-run-library-generation-tests
+    if: needs.should-run-library-generation-tests.outputs.should_run == 'true'
     strategy:
       matrix:
         java: [ 8 ]
         os: [ ubuntu-22.04, macos-12 ]
-    runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v4
     - name: install utils (macos)
@@ -88,8 +114,10 @@ jobs:
       run: |
         set -x
         python -m unittest discover -s library_generation/test/ -p "*unit_tests.py"
-  lint-shell:
+  library-generation-lint-shell:
     runs-on: ubuntu-22.04
+    needs: should-run-library-generation-tests
+    if: needs.should-run-library-generation-tests.outputs.should_run == 'true'
     steps:
     - uses: actions/checkout@v4
     - name: Run ShellCheck
@@ -98,8 +126,10 @@ jobs:
         scandir: 'library_generation'
         format: tty
         severity: error
-  lint-python:
+  library-generation-lint-python:
     runs-on: ubuntu-22.04
+    needs: should-run-library-generation-tests
+    if: needs.should-run-library-generation-tests.outputs.should_run == 'true'
     steps:
     - uses: actions/checkout@v4
     - name: install python dependencies

gapic-generator-java/pom.xml

@@ -351,7 +351,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
-        <version>3.5.2</version>
+        <version>3.5.3</version>
         <executions>
           <execution>
             <phase>package</phase>

gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/Composer.java

@@ -207,7 +207,7 @@ static List<GapicClass> prepareExecutableSamples(List<GapicClass> clazzes) {
                   sample ->
                       samples.add(
                           addRegionTagAndHeaderToSample(
-                              sample, gapicClass.apiShortName(), gapicClass.apiVersion())));
+                              sample, gapicClass.apiShortName(), gapicClass.packageVersion())));
           clazzesWithSamples.add(gapicClass.withSamples(samples));
         });
     return clazzesWithSamples;

gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/common/AbstractServiceClientClassComposer.java

@@ -173,7 +173,7 @@ public GapicClass generate(GapicContext context, Service service) {
     updateGapicMetadata(context, service, className, grpcRpcsToJavaMethodNames);
     return GapicClass.create(kind, classDef, SampleComposerUtil.handleDuplicateSamples(samples))
         .withApiShortName(service.apiShortName())
-        .withApiVersion(service.apiVersion());
+        .withPackageVersion(service.packageVersion());
   }
 
   private static List<AnnotationNode> createClassAnnotations(Service service, TypeStore typeStore) {

gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/common/AbstractServiceSettingsClassComposer.java

@@ -128,7 +128,7 @@ public GapicClass generate(GapicContext context, Service service) {
             .build();
     return GapicClass.create(kind, classDef, SampleComposerUtil.handleDuplicateSamples(samples))
         .withApiShortName(service.apiShortName())
-        .withApiVersion(service.apiVersion());
+        .withPackageVersion(service.packageVersion());
   }
 
   private static List<CommentStatement> createClassHeaderComments(

gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/common/AbstractServiceStubSettingsClassComposer.java

@@ -204,7 +204,7 @@ public GapicClass generate(GapicContext context, Service service) {
     return GapicClass.create(
             GapicClass.Kind.STUB, classDef, SampleComposerUtil.handleDuplicateSamples(samples))
         .withApiShortName(service.apiShortName())
-        .withApiVersion(service.apiVersion());
+        .withPackageVersion(service.packageVersion());
   }
 
   protected MethodDefinition createDefaultCredentialsProviderBuilderMethod() {
@@ -370,6 +370,16 @@ protected MethodDefinition createApiClientHeaderProviderBuilderMethod(
             .setReturnType(returnType)
             .build();
 
+    if (service.hasApiVersion()) {
+
+      returnExpr =
+          MethodInvocationExpr.builder()
+              .setExprReferenceExpr(returnExpr)
+              .setMethodName("setApiVersionToken")
+              .setArguments(ValueExpr.withValue(StringObjectValue.withValue(service.apiVersion())))
+              .setReturnType(returnType)
+              .build();
+    }
     return MethodDefinition.builder()
         .setScope(ScopeNode.PUBLIC)
         .setIsStatic(true)

gapic-generator-java/src/main/java/com/google/api/generator/gapic/model/GapicClass.java

@@ -43,7 +43,7 @@ public enum Kind {
   public abstract String apiShortName();
 
   // Only used for generating the region tag for samples; therefore only used in select Composers.
-  public abstract String apiVersion();
+  public abstract String packageVersion();
 
   /**
    * Create a GapicClass with minimal information. This is intended to be used for GapicClasses that
@@ -76,7 +76,7 @@ static Builder builder() {
     return new AutoValue_GapicClass.Builder()
         .setSamples(Collections.emptyList())
         .setApiShortName("")
-        .setApiVersion("");
+        .setPackageVersion("");
   }
 
   abstract Builder toBuilder();
@@ -89,8 +89,8 @@ public final GapicClass withApiShortName(String apiShortName) {
     return toBuilder().setApiShortName(apiShortName).build();
   }
 
-  public final GapicClass withApiVersion(String apiVersion) {
-    return toBuilder().setApiVersion(apiVersion).build();
+  public final GapicClass withPackageVersion(String packageVersion) {
+    return toBuilder().setPackageVersion(packageVersion).build();
   }
 
   @AutoValue.Builder
@@ -103,7 +103,7 @@ abstract static class Builder {
 
     abstract Builder setApiShortName(String apiShortName);
 
-    abstract Builder setApiVersion(String apiVersion);
+    abstract Builder setPackageVersion(String packageVersion);
 
     abstract GapicClass build();
   }

gapic-generator-java/src/main/java/com/google/api/generator/gapic/model/Service.java

@@ -27,6 +27,9 @@
 public abstract class Service {
   public abstract String name();
 
+  @Nullable
+  public abstract String apiVersion();
+
   public abstract String defaultHost();
 
   public abstract ImmutableList<String> oauthScopes();
@@ -52,6 +55,10 @@ public boolean hasDescription() {
     return !Strings.isNullOrEmpty(description());
   }
 
+  public boolean hasApiVersion() {
+    return !Strings.isNullOrEmpty(apiVersion());
+  }
+
   public String hostServiceName() {
     // Host Service Name is guaranteed to exist and be non-null and non-empty
     // Parser will fail if the default host is not supplied
@@ -65,9 +72,9 @@ public String apiShortName() {
     return "";
   }
 
-  public String apiVersion() {
+  public String packageVersion() {
     if (!Strings.isNullOrEmpty(protoPakkage())) {
-      return parseApiVersion(protoPakkage());
+      return parsePackageVersion(protoPakkage());
     }
     return "";
   }
@@ -158,6 +165,8 @@ public abstract static class Builder {
 
     public abstract Builder setOverriddenName(String overriddenName);
 
+    public abstract Builder setApiVersion(String apiVersion);
+
     public abstract Builder setDefaultHost(String defaultHost);
 
     public abstract Builder setOauthScopes(List<String> oauthScopes);
@@ -177,17 +186,17 @@ public abstract static class Builder {
     public abstract Service build();
   }
 
-  private static String parseApiVersion(String protoPackage) {
-    //  parse protoPackage for apiVersion
+  private static String parsePackageVersion(String protoPackage) {
+    //  parse protoPackage for packageVersion
     String[] pakkage = protoPackage.split("\\.");
-    String apiVersion;
+    String packageVersion;
     //  e.g. v1, v2, v1beta1
     if (pakkage[pakkage.length - 1].matches("v[0-9].*")) {
-      apiVersion = pakkage[pakkage.length - 1];
+      packageVersion = pakkage[pakkage.length - 1];
     } else {
-      apiVersion = "";
+      packageVersion = "";
     }
-    return apiVersion;
+    return packageVersion;
   }
 
   // Parse the service name from the default host configured in the protos

gapic-generator-java/src/main/java/com/google/api/generator/gapic/protoparser/Parser.java

@@ -470,6 +470,11 @@ public static List<Service> parseService(
                 }
               }
 
+              if (serviceOptions.hasExtension(ClientProto.apiVersion)) {
+                String apiVersion = serviceOptions.getExtension(ClientProto.apiVersion);
+                serviceBuilder.setApiVersion(apiVersion);
+              }
+
               String serviceName = s.getName();
               String overriddenServiceName = serviceName;
               String pakkage = TypeParser.getPackage(fileDescriptor);

gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/ComposerTest.java

@@ -45,7 +45,7 @@ public class ComposerTest {
           GrpcServiceCallableFactoryClassComposer.instance()
               .generate(context, echoProtoService)
               .withApiShortName(echoProtoService.apiShortName())
-              .withApiVersion(echoProtoService.apiVersion()));
+              .withPackageVersion(echoProtoService.packageVersion()));
   private final Sample sample =
       Sample.builder()
           .setRegionTag(
@@ -160,7 +160,7 @@ private List<GapicClass> getTestClassListFromService(Service testService) {
             .generate(context, testService)
             .withSamples(ListofSamples)
             .withApiShortName(testService.apiShortName())
-            .withApiVersion(testService.apiVersion());
+            .withPackageVersion(testService.packageVersion());
     List<GapicClass> testClassList = Arrays.asList(new GapicClass[] {testClass});
     return testClassList;
   }

gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/ServiceClientClassComposerTest.java

@@ -72,7 +72,7 @@ public static Collection<Object[]> data() {
   public String apiShortNameExpected;
 
   @Parameterized.Parameter(3)
-  public String apiVersionExpected;
+  public String packageVersionExpected;
 
   @Test
   public void generateServiceClientClasses() {
@@ -83,6 +83,6 @@ public void generateServiceClientClasses() {
     Assert.assertGoldenSamples(
         this.getClass(), name, clazz.classDefinition().packageString(), clazz.samples());
     Assert.assertCodeEquals(clazz.apiShortName(), apiShortNameExpected);
-    Assert.assertCodeEquals(clazz.apiVersion(), apiVersionExpected);
+    Assert.assertCodeEquals(clazz.packageVersion(), packageVersionExpected);
   }
 }