Managed BigQueryIO (#31486)

* managed bigqueryio

* spotless

* move managed dependency to test only

* cleanup after merging snake_case PR

* choose write method based on boundedness and pipeline options

* rename bigquery write config class

* spotless

* change read output tag to 'output'

* spotless

* revert logic that depends on DataflowServiceOptions. switching BQ methods can instead be done in Dataflow service side

* spotless

* fix typo

* separate BQ write config to a new class

* fix doc

* resolve after syncing to HEAD

* spotless

* fork on batch/streaming

* cleanup

* spotless

* move forking logic to BQ schematransform side

* add file loads translation and tests; add test checks that the correct transform is chosen

* set top-level wrapper to be the underlying managed BQ transform urn; change tests to verify underlying transform name

* move unit tests to respectvie schematransform test classes

* expose to Python SDK as well

.github/trigger_files/beam_PostCommit_Java_DataflowV2.json

@@ -1,3 +1,4 @@
 {
-    "comment": "Modify this file in a trivial way to cause this test suite to run"
+    "comment": "Modify this file in a trivial way to cause this test suite to run",
+    "modification": 1
 }

.github/trigger_files/beam_PostCommit_Python_Xlang_Gcp_Direct.json

@@ -1,4 +1,4 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "modification": 2
+  "modification": 1
 }

model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/external_transforms.proto

@@ -70,6 +70,10 @@ message ManagedTransforms {
       "beam:schematransform:org.apache.beam:kafka_read:v1"];
     KAFKA_WRITE = 3 [(org.apache.beam.model.pipeline.v1.beam_urn) =
       "beam:schematransform:org.apache.beam:kafka_write:v1"];
+    BIGQUERY_READ = 4 [(org.apache.beam.model.pipeline.v1.beam_urn) =
+      "beam:schematransform:org.apache.beam:bigquery_storage_read:v1"];
+    BIGQUERY_WRITE = 5 [(org.apache.beam.model.pipeline.v1.beam_urn) =
+      "beam:schematransform:org.apache.beam:bigquery_write:v1"];
   }
 }
 

sdks/java/io/google-cloud-platform/build.gradle

@@ -159,6 +159,7 @@ dependencies {
   testImplementation project(path: ":sdks:java:extensions:google-cloud-platform-core", configuration: "testRuntimeMigration")
   testImplementation project(path: ":sdks:java:extensions:protobuf", configuration: "testRuntimeMigration")
   testImplementation project(path: ":runners:direct-java", configuration: "shadow")
+  testImplementation project(":sdks:java:managed")
   testImplementation project(path: ":sdks:java:io:common")
   testImplementation project(path: ":sdks:java:testing:test-utils")
   testImplementation library.java.commons_math3

sdks/java/io/google-cloud-platform/expansion-service/build.gradle

@@ -36,6 +36,9 @@ dependencies {
     permitUnusedDeclared project(":sdks:java:io:google-cloud-platform") // BEAM-11761
     implementation project(":sdks:java:extensions:schemaio-expansion-service")
     permitUnusedDeclared project(":sdks:java:extensions:schemaio-expansion-service") // BEAM-11761
+    implementation project(":sdks:java:managed")
+    permitUnusedDeclared project(":sdks:java:managed") // BEAM-11761
+
     runtimeOnly library.java.slf4j_jdk14
 }