Release 3.3.0 (2024-03-11)

### Features * create BuyerCodeFetchManager to handle buyer udf fetching logic * enable consented debugging in AWS deployment * integrate buyer code fetch management * Output raw metric for consented request Bug: N/A GitOrigin-RevId: eb602840a4338d507de5a88628a4f8fed5904373 Change-Id: I50083bf52399de4c706c86e9fb5e7d6f546a40ee
privacysandbox · Mar 12, 2024 · fe7b8ba · fe7b8ba
1 parent ef249b3
commit fe7b8ba
Show file tree

Hide file tree

Showing 30 changed files with 963 additions and 172 deletions.
diff --git a/.gitignore b/.gitignore
@@ -60,3 +60,5 @@ terraform.rc
 
 # Docker
 docker-buildx-*.log
+
+core
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,16 @@
 
 All notable changes to this project will be documented in this file. See [commit-and-tag-version](https://github.com/absolute-version/commit-and-tag-version) for commit guidelines.
 
+## 3.3.0 (2024-03-11)
+
+
+### Features
+
+* create BuyerCodeFetchManager to handle buyer udf fetching logic
+* enable consented debugging in AWS deployment
+* integrate buyer code fetch management
+* Output raw metric for consented request
+
 ## 3.2.0 (2024-03-06)
 
 

diff --git a/production/deploy/aws/terraform/environment/demo/buyer/buyer.tf b/production/deploy/aws/terraform/environment/demo/buyer/buyer.tf
@@ -69,12 +69,15 @@ module "buyer" {
     ENABLE_BIDDING_COMPRESSION                    = "" # Example: "true"
     PROTECTED_APP_SIGNALS_GENERATE_BID_TIMEOUT_MS = "" # Example: "60000"
     TELEMETRY_CONFIG                              = "" # Example: "mode: EXPERIMENT"
+    ENABLE_OTEL_BASED_LOGGING                     = "" # Example: "true"
+    CONSENTED_DEBUG_TOKEN                         = "" # Example: "123456"
     TEST_MODE                                     = "" # Example: "false"
     BUYER_CODE_FETCH_CONFIG                       = "" # Example:
     ENABLE_PROTECTED_APP_SIGNALS                  = "" # Example: "false"
     ENABLE_PROTECTED_AUDIENCE                     = "" # Example: "true"
     PS_VERBOSITY                                  = "" # Example: "10"
     # "{
+    #    "fetchMode": 0,
     #    "biddingJsPath": "",
     #    "biddingJsUrl": "https://example.com/generateBid.js",
     #    "protectedAppSignalsBiddingJsUrl": "placeholder",

diff --git a/production/deploy/aws/terraform/environment/demo/seller/seller.tf b/production/deploy/aws/terraform/environment/demo/seller/seller.tf
@@ -71,6 +71,8 @@ module "seller" {
     PS_VERBOSITY                           = "" # Example: "10"
     CREATE_NEW_EVENT_ENGINE                = "" # Example: "false"
     TELEMETRY_CONFIG                       = "" # Example: "mode: EXPERIMENT"
+    ENABLE_OTEL_BASED_LOGGING              = "" # Example: "true"
+    CONSENTED_DEBUG_TOKEN                  = "" # Example: "123456"
     TEST_MODE                              = "" # Example: "false"
     SELLER_CODE_FETCH_CONFIG               = "" # Example:
     # "{

diff --git a/production/deploy/gcp/terraform/environment/demo/buyer/buyer.tf b/production/deploy/gcp/terraform/environment/demo/buyer/buyer.tf
@@ -70,6 +70,7 @@ module "buyer" {
     # and additional latency for parsing the logs.
     BUYER_CODE_FETCH_CONFIG = "" # Example:
     # "{
+    #    "fetchMode": 0,
     #    "biddingJsPath": "",
     #    "biddingJsUrl": "https://example.com/generateBid.js",
     #    "protectedAppSignalsBiddingJsUrl": "placeholder",

diff --git a/production/packaging/aws/common/ami/otel_collector_config.yaml b/production/packaging/aws/common/ami/otel_collector_config.yaml
@@ -31,6 +31,8 @@ processors:
     send_batch_size: 50
   batch/metrics:
     timeout: 60s
+  batch/logs:
+    timeout: 60s
 
 exporters:
   awsxray:
@@ -39,6 +41,9 @@ exporters:
     namespace: '$SERVICE'
     resource_to_telemetry_conversion:
         enabled: true
+  awscloudwatchlogs:
+    log_group_name: "bidding-auction"
+    log_stream_name: "consented-log-stream"
 
 service:
   pipelines:
@@ -50,5 +55,9 @@ service:
       receivers: [otlp]
       processors: [batch/metrics]
       exporters: [awsemf]
+    logs:
+      receivers: [otlp]
+      processors: [batch/logs]
+      exporters: [awscloudwatchlogs]
 
   extensions: [health_check]
diff --git a/services/auction_service/score_ads_reactor.cc b/services/auction_service/score_ads_reactor.cc
@@ -201,6 +201,9 @@ ScoreAdsReactor::ScoreAdsReactor(
   CHECK_OK([this]() {
     PS_ASSIGN_OR_RETURN(metric_context_,
                         metric::AuctionContextMap()->Remove(request_));
+    if (log_context_.is_consented()) {
+      metric_context_->SetConsented(raw_request_.log_context().generation_id());
+    }
     return absl::OkStatus();
   }()) << "AuctionContextMap()->Get(request) should have been called";
 }

diff --git a/services/bidding_service/BUILD b/services/bidding_service/BUILD
@@ -28,6 +28,56 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "buyer_code_fetch_manager",
+    srcs = [
+        "buyer_code_fetch_manager.cc",
+    ],
+    hdrs = [
+        "buyer_code_fetch_manager.h",
+    ],
+    deps = [
+        ":bidding_code_fetch_config_cc_proto",
+        "//services/bidding_service/code_wrapper:buyer_code_wrapper",
+        "//services/common/clients/code_dispatcher:v8_dispatcher",
+        "//services/common/code_fetch:periodic_bucket_fetcher",
+        "//services/common/code_fetch:periodic_code_fetcher",
+        "//services/common/util:file_util",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/time",
+        "@google_privacysandbox_servers_common//scp/cc/public/core/interface:errors",
+        "@google_privacysandbox_servers_common//scp/cc/public/core/interface:execution_result",
+        "@google_privacysandbox_servers_common//scp/cc/public/cpio/interface:cpio",
+        "@google_privacysandbox_servers_common//scp/cc/public/cpio/interface/blob_storage_client",
+        "@google_privacysandbox_servers_common//src/cpp/concurrent:executor",
+        "@google_privacysandbox_servers_common//src/cpp/util/status_macro:status_macros",
+    ],
+)
+
+cc_test(
+    name = "buyer_code_fetch_manager_test",
+    size = "small",
+    srcs = ["buyer_code_fetch_manager_test.cc"],
+    deps = [
+        ":buyer_code_fetch_manager",
+        "//services/bidding_service/code_wrapper:buyer_code_wrapper",
+        "//services/common/clients/code_dispatcher:v8_dispatcher",
+        "//services/common/clients/http:http_fetcher_async",
+        "//services/common/code_fetch:periodic_bucket_fetcher",
+        "//services/common/code_fetch:periodic_code_fetcher",
+        "//services/common/test:mocks",
+        "@com_google_absl//absl/functional:any_invocable",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/synchronization",
+        "@com_google_absl//absl/time",
+        "@com_google_googletest//:gtest",
+        "@com_google_googletest//:gtest_main",
+        "@google_privacysandbox_servers_common//scp/cc/public/cpio/mock/blob_storage_client:blob_storage_client_mock",
+    ],
+)
+
 cc_library(
     name = "generate_bids_reactor",
     srcs = [
@@ -191,6 +241,7 @@ cc_binary(
     deps = [
         ":bidding_code_fetch_config_cc_proto",
         ":bidding_service",
+        ":buyer_code_fetch_manager",
         "//api:bidding_auction_servers_cc_grpc_proto",
         "//api:bidding_auction_servers_cc_proto",
         "//services/bidding_service:bidding_constants",
@@ -213,6 +264,7 @@ cc_binary(
         "@com_google_absl//absl/flags:parse",
         "@com_google_absl//absl/log:check",
         "@com_google_absl//absl/strings",
+        "@google_privacysandbox_servers_common//scp/cc/public/cpio/interface/blob_storage_client",
         "@google_privacysandbox_servers_common//src/cpp/concurrent:executor",
         "@google_privacysandbox_servers_common//src/cpp/encryption/key_fetcher/src:key_fetcher_manager",
         "@google_privacysandbox_servers_common//src/cpp/telemetry",

diff --git a/services/bidding_service/bidding_code_fetch_config.proto b/services/bidding_service/bidding_code_fetch_config.proto
@@ -18,6 +18,17 @@ syntax = "proto3";
 
 package privacy_sandbox.bidding_auction_servers.bidding_service;
 
+// Specify how to fetch the code blob.
+enum FetchMode {
+   // Fetch a single blob from an arbitrary url.
+   FETCH_MODE_URL = 0;
+   // Fetch all blobs from a specified bucket
+   // and rely on a default blob specification.
+   FETCH_MODE_BUCKET = 1;
+   // Fetch a blob from a local path. Dev-only.
+   FETCH_MODE_LOCAL = 2;
+}
+
 message BuyerCodeFetchConfig {
 
    // The javascript generateBid script.
@@ -56,4 +67,35 @@ message BuyerCodeFetchConfig {
    // URL endpoint to the wasm file. AdTechs can choose to include wasm file
    // which will be combined with the js file before loading into Roma.
    string prepare_data_for_ads_retrieval_wasm_helper_url = 12;
+
+   // The name of a bucket from which to fetch protected auction code blobs.
+   // All blobs will be fetched from this bucket.
+   // TODO: the concept of a separate wasm_helper blob is not yet supported if using
+   // bucket-based fetching.
+   string protected_auction_bidding_js_bucket = 14;
+
+   // The name of the bucket's default protected auction code blob to use.
+   // The default will be used if the bid request does not specify a version.
+   string protected_auction_bidding_js_bucket_default_blob = 15;
+
+   // The name of a bucket from which to fetch protected app signal code blobs.
+   // TODO: the concept of a separate wasm_helper blob is not yet supported if using
+   // bucket-based fetching.
+   string protected_app_signals_bidding_js_bucket = 16;
+
+   // The name of the bucket's default protected app signal code blob to use.
+   // The default will be used if the bid request does not specify a version.
+   string protected_app_signals_bidding_js_bucket_default_blob = 17;
+
+   // The name of a bucket from which to fetch prepareDataForAdsRetrieval js.
+   // TODO: the concept of a separate wasm_helper blob is not yet supported if using
+   // bucket-based fetching.
+   string ads_retrieval_js_bucket = 18;
+
+   // The name of the bucket's default prepareDataForAdsRetrieval code blob to use.
+   // The default will be used if the bid request does not specify a version.
+   string ads_retrieval_bucket_default_blob = 19;
+
+   FetchMode fetch_mode = 20;
+
 }