diff --git a/.gitignore b/.gitignore index a821ad4e..22cf53b1 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,5 @@ terraform.rc # Docker docker-buildx-*.log + +core diff --git a/CHANGELOG.md b/CHANGELOG.md index 01f0c7f4..45c2907e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ All notable changes to this project will be documented in this file. See [commit-and-tag-version](https://github.com/absolute-version/commit-and-tag-version) for commit guidelines. +## 3.3.0 (2024-03-11) + + +### Features + +* create BuyerCodeFetchManager to handle buyer udf fetching logic +* enable consented debugging in AWS deployment +* integrate buyer code fetch management +* Output raw metric for consented request + ## 3.2.0 (2024-03-06) diff --git a/production/deploy/aws/terraform/environment/demo/buyer/buyer.tf b/production/deploy/aws/terraform/environment/demo/buyer/buyer.tf index 6c2ae446..a1f70462 100644 --- a/production/deploy/aws/terraform/environment/demo/buyer/buyer.tf +++ b/production/deploy/aws/terraform/environment/demo/buyer/buyer.tf @@ -69,12 +69,15 @@ module "buyer" { ENABLE_BIDDING_COMPRESSION = "" # Example: "true" PROTECTED_APP_SIGNALS_GENERATE_BID_TIMEOUT_MS = "" # Example: "60000" TELEMETRY_CONFIG = "" # Example: "mode: EXPERIMENT" + ENABLE_OTEL_BASED_LOGGING = "" # Example: "true" + CONSENTED_DEBUG_TOKEN = "" # Example: "123456" TEST_MODE = "" # Example: "false" BUYER_CODE_FETCH_CONFIG = "" # Example: ENABLE_PROTECTED_APP_SIGNALS = "" # Example: "false" ENABLE_PROTECTED_AUDIENCE = "" # Example: "true" PS_VERBOSITY = "" # Example: "10" # "{ + # "fetchMode": 0, # "biddingJsPath": "", # "biddingJsUrl": "https://example.com/generateBid.js", # "protectedAppSignalsBiddingJsUrl": "placeholder", diff --git a/production/deploy/aws/terraform/environment/demo/seller/seller.tf b/production/deploy/aws/terraform/environment/demo/seller/seller.tf index 158d752e..a23a1eef 100644 --- a/production/deploy/aws/terraform/environment/demo/seller/seller.tf +++ b/production/deploy/aws/terraform/environment/demo/seller/seller.tf @@ -71,6 +71,8 @@ module "seller" { PS_VERBOSITY = "" # Example: "10" CREATE_NEW_EVENT_ENGINE = "" # Example: "false" TELEMETRY_CONFIG = "" # Example: "mode: EXPERIMENT" + ENABLE_OTEL_BASED_LOGGING = "" # Example: "true" + CONSENTED_DEBUG_TOKEN = "" # Example: "123456" TEST_MODE = "" # Example: "false" SELLER_CODE_FETCH_CONFIG = "" # Example: # "{ diff --git a/production/deploy/gcp/terraform/environment/demo/buyer/buyer.tf b/production/deploy/gcp/terraform/environment/demo/buyer/buyer.tf index 8b2ecbe3..559e4a0b 100644 --- a/production/deploy/gcp/terraform/environment/demo/buyer/buyer.tf +++ b/production/deploy/gcp/terraform/environment/demo/buyer/buyer.tf @@ -70,6 +70,7 @@ module "buyer" { # and additional latency for parsing the logs. BUYER_CODE_FETCH_CONFIG = "" # Example: # "{ + # "fetchMode": 0, # "biddingJsPath": "", # "biddingJsUrl": "https://example.com/generateBid.js", # "protectedAppSignalsBiddingJsUrl": "placeholder", diff --git a/production/packaging/aws/common/ami/otel_collector_config.yaml b/production/packaging/aws/common/ami/otel_collector_config.yaml index b5e34aa6..1476dc7d 100644 --- a/production/packaging/aws/common/ami/otel_collector_config.yaml +++ b/production/packaging/aws/common/ami/otel_collector_config.yaml @@ -31,6 +31,8 @@ processors: send_batch_size: 50 batch/metrics: timeout: 60s + batch/logs: + timeout: 60s exporters: awsxray: @@ -39,6 +41,9 @@ exporters: namespace: '$SERVICE' resource_to_telemetry_conversion: enabled: true + awscloudwatchlogs: + log_group_name: "bidding-auction" + log_stream_name: "consented-log-stream" service: pipelines: @@ -50,5 +55,9 @@ service: receivers: [otlp] processors: [batch/metrics] exporters: [awsemf] + logs: + receivers: [otlp] + processors: [batch/logs] + exporters: [awscloudwatchlogs] extensions: [health_check] diff --git a/services/auction_service/score_ads_reactor.cc b/services/auction_service/score_ads_reactor.cc index a1566ac4..bd281311 100644 --- a/services/auction_service/score_ads_reactor.cc +++ b/services/auction_service/score_ads_reactor.cc @@ -201,6 +201,9 @@ ScoreAdsReactor::ScoreAdsReactor( CHECK_OK([this]() { PS_ASSIGN_OR_RETURN(metric_context_, metric::AuctionContextMap()->Remove(request_)); + if (log_context_.is_consented()) { + metric_context_->SetConsented(raw_request_.log_context().generation_id()); + } return absl::OkStatus(); }()) << "AuctionContextMap()->Get(request) should have been called"; } diff --git a/services/bidding_service/BUILD b/services/bidding_service/BUILD index f9117d49..5ce45fa1 100644 --- a/services/bidding_service/BUILD +++ b/services/bidding_service/BUILD @@ -28,6 +28,56 @@ cc_library( ], ) +cc_library( + name = "buyer_code_fetch_manager", + srcs = [ + "buyer_code_fetch_manager.cc", + ], + hdrs = [ + "buyer_code_fetch_manager.h", + ], + deps = [ + ":bidding_code_fetch_config_cc_proto", + "//services/bidding_service/code_wrapper:buyer_code_wrapper", + "//services/common/clients/code_dispatcher:v8_dispatcher", + "//services/common/code_fetch:periodic_bucket_fetcher", + "//services/common/code_fetch:periodic_code_fetcher", + "//services/common/util:file_util", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/time", + "@google_privacysandbox_servers_common//scp/cc/public/core/interface:errors", + "@google_privacysandbox_servers_common//scp/cc/public/core/interface:execution_result", + "@google_privacysandbox_servers_common//scp/cc/public/cpio/interface:cpio", + "@google_privacysandbox_servers_common//scp/cc/public/cpio/interface/blob_storage_client", + "@google_privacysandbox_servers_common//src/cpp/concurrent:executor", + "@google_privacysandbox_servers_common//src/cpp/util/status_macro:status_macros", + ], +) + +cc_test( + name = "buyer_code_fetch_manager_test", + size = "small", + srcs = ["buyer_code_fetch_manager_test.cc"], + deps = [ + ":buyer_code_fetch_manager", + "//services/bidding_service/code_wrapper:buyer_code_wrapper", + "//services/common/clients/code_dispatcher:v8_dispatcher", + "//services/common/clients/http:http_fetcher_async", + "//services/common/code_fetch:periodic_bucket_fetcher", + "//services/common/code_fetch:periodic_code_fetcher", + "//services/common/test:mocks", + "@com_google_absl//absl/functional:any_invocable", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/synchronization", + "@com_google_absl//absl/time", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + "@google_privacysandbox_servers_common//scp/cc/public/cpio/mock/blob_storage_client:blob_storage_client_mock", + ], +) + cc_library( name = "generate_bids_reactor", srcs = [ @@ -191,6 +241,7 @@ cc_binary( deps = [ ":bidding_code_fetch_config_cc_proto", ":bidding_service", + ":buyer_code_fetch_manager", "//api:bidding_auction_servers_cc_grpc_proto", "//api:bidding_auction_servers_cc_proto", "//services/bidding_service:bidding_constants", @@ -213,6 +264,7 @@ cc_binary( "@com_google_absl//absl/flags:parse", "@com_google_absl//absl/log:check", "@com_google_absl//absl/strings", + "@google_privacysandbox_servers_common//scp/cc/public/cpio/interface/blob_storage_client", "@google_privacysandbox_servers_common//src/cpp/concurrent:executor", "@google_privacysandbox_servers_common//src/cpp/encryption/key_fetcher/src:key_fetcher_manager", "@google_privacysandbox_servers_common//src/cpp/telemetry", diff --git a/services/bidding_service/bidding_code_fetch_config.proto b/services/bidding_service/bidding_code_fetch_config.proto index c7e0a637..fdbfe340 100644 --- a/services/bidding_service/bidding_code_fetch_config.proto +++ b/services/bidding_service/bidding_code_fetch_config.proto @@ -18,6 +18,17 @@ syntax = "proto3"; package privacy_sandbox.bidding_auction_servers.bidding_service; +// Specify how to fetch the code blob. +enum FetchMode { + // Fetch a single blob from an arbitrary url. + FETCH_MODE_URL = 0; + // Fetch all blobs from a specified bucket + // and rely on a default blob specification. + FETCH_MODE_BUCKET = 1; + // Fetch a blob from a local path. Dev-only. + FETCH_MODE_LOCAL = 2; +} + message BuyerCodeFetchConfig { // The javascript generateBid script. @@ -56,4 +67,35 @@ message BuyerCodeFetchConfig { // URL endpoint to the wasm file. AdTechs can choose to include wasm file // which will be combined with the js file before loading into Roma. string prepare_data_for_ads_retrieval_wasm_helper_url = 12; + + // The name of a bucket from which to fetch protected auction code blobs. + // All blobs will be fetched from this bucket. + // TODO: the concept of a separate wasm_helper blob is not yet supported if using + // bucket-based fetching. + string protected_auction_bidding_js_bucket = 14; + + // The name of the bucket's default protected auction code blob to use. + // The default will be used if the bid request does not specify a version. + string protected_auction_bidding_js_bucket_default_blob = 15; + + // The name of a bucket from which to fetch protected app signal code blobs. + // TODO: the concept of a separate wasm_helper blob is not yet supported if using + // bucket-based fetching. + string protected_app_signals_bidding_js_bucket = 16; + + // The name of the bucket's default protected app signal code blob to use. + // The default will be used if the bid request does not specify a version. + string protected_app_signals_bidding_js_bucket_default_blob = 17; + + // The name of a bucket from which to fetch prepareDataForAdsRetrieval js. + // TODO: the concept of a separate wasm_helper blob is not yet supported if using + // bucket-based fetching. + string ads_retrieval_js_bucket = 18; + + // The name of the bucket's default prepareDataForAdsRetrieval code blob to use. + // The default will be used if the bid request does not specify a version. + string ads_retrieval_bucket_default_blob = 19; + + FetchMode fetch_mode = 20; + } diff --git a/services/bidding_service/bidding_main.cc b/services/bidding_service/bidding_main.cc index 3aec4b93..fa975886 100644 --- a/services/bidding_service/bidding_main.cc +++ b/services/bidding_service/bidding_main.cc @@ -31,10 +31,12 @@ #include "grpcpp/grpcpp.h" #include "grpcpp/health_check_service_interface.h" #include "public/cpio/interface/cpio.h" +#include "scp/cc/public/cpio/proto/blob_storage_service/v1/blob_storage_service.pb.h" #include "services/bidding_service/benchmarking/bidding_benchmarking_logger.h" #include "services/bidding_service/benchmarking/bidding_no_op_logger.h" #include "services/bidding_service/bidding_code_fetch_config.pb.h" #include "services/bidding_service/bidding_service.h" +#include "services/bidding_service/buyer_code_fetch_manager.h" #include "services/bidding_service/code_wrapper/buyer_code_wrapper.h" #include "services/bidding_service/constants.h" #include "services/bidding_service/data/runtime_config.h" @@ -87,15 +89,13 @@ ABSL_FLAG(std::optional, kv_server_egress_tls, std::nullopt, namespace privacy_sandbox::bidding_auction_servers { using bidding_service::BuyerCodeFetchConfig; +using ::google::scp::cpio::BlobStorageClientFactory; using ::google::scp::cpio::Cpio; using ::google::scp::cpio::CpioOptions; using ::google::scp::cpio::LogOption; using ::grpc::Server; using ::grpc::ServerBuilder; -constexpr int kMinNumCodeBlobs = 1; -constexpr int kMaxNumCodeBlobs = 2; - absl::StatusOr GetConfigClient( std::string config_param_prefix) { TrustedServersConfigClient config_client(GetServiceFlags()); @@ -180,128 +180,6 @@ absl::StatusOr GetConfigClient( return config_client; } -absl::StatusOr> StartFetchingCodeBlob( - const std::string& js_url, const std::string& wasm_helper_url, - const std::string& roma_version, absl::string_view script_logging_name, - absl::Duration url_fetch_period_ms, absl::Duration url_fetch_timeout_ms, - absl::AnyInvocable&)> wrap_code, - V8Dispatcher& dispatcher, server_common::Executor* executor, - HttpFetcherAsync* curl_http_fetcher) { - if (js_url.empty()) { - return absl::InvalidArgumentError( - absl::StrCat("JS URL for ", script_logging_name, " is missing")); - } - - std::vector endpoints = {js_url}; - if (!wasm_helper_url.empty()) { - endpoints.emplace_back(wasm_helper_url); - } - auto code_fetcher = std::make_unique( - std::move(endpoints), url_fetch_period_ms, curl_http_fetcher, &dispatcher, - executor, url_fetch_timeout_ms, std::move(wrap_code), roma_version); - code_fetcher->Start(); - return code_fetcher; -} - -absl::Status StartProtectedAppSignalsUdfFetching( - const BuyerCodeFetchConfig& code_fetch_proto, V8Dispatcher& dispatcher, - server_common::Executor* executor, - std::vector>& code_fetchers, - HttpFetcherAsync* curl_http_fetcher) { - absl::Duration url_fetch_period_ms = - absl::Milliseconds(code_fetch_proto.url_fetch_period_ms()); - absl::Duration url_fetch_timeout_ms = - absl::Milliseconds(code_fetch_proto.url_fetch_timeout_ms()); - PS_ASSIGN_OR_RETURN( - auto bidding_code_fetcher, - StartFetchingCodeBlob( - code_fetch_proto.protected_app_signals_bidding_js_url(), - code_fetch_proto.protected_app_signals_bidding_wasm_helper_url(), - kProtectedAppSignalsGenerateBidBlobVersion, - "protected_app_signals_bidding_js_url", url_fetch_period_ms, - url_fetch_timeout_ms, - [](const std::vector& ad_tech_code_blobs) { - DCHECK_GE(ad_tech_code_blobs.size(), kMinNumCodeBlobs); - DCHECK_LE(ad_tech_code_blobs.size(), kMaxNumCodeBlobs); - return GetBuyerWrappedCode( - /*ad_tech_js=*/ad_tech_code_blobs[0], - /*ad_tech_wasm=*/ - ad_tech_code_blobs.size() == 2 ? ad_tech_code_blobs[1] : "", - AuctionType::kProtectedAppSignals, - /*auction_specific_setup=*/"// No additional setup"); - }, - dispatcher, executor, curl_http_fetcher)); - - code_fetchers.emplace_back(std::move(bidding_code_fetcher)); - PS_ASSIGN_OR_RETURN( - auto prepare_data_for_ads_code_fetcher, - StartFetchingCodeBlob( - code_fetch_proto.prepare_data_for_ads_retrieval_js_url(), - code_fetch_proto.prepare_data_for_ads_retrieval_wasm_helper_url(), - kPrepareDataForAdRetrievalBlobVersion, - "prepare_data_for_ads_retrieval_js_url", url_fetch_period_ms, - url_fetch_timeout_ms, - [](const std::vector& ad_tech_code_blobs) { - DCHECK_GE(ad_tech_code_blobs.size(), kMinNumCodeBlobs); - DCHECK_LE(ad_tech_code_blobs.size(), kMaxNumCodeBlobs); - return GetProtectedAppSignalsGenericBuyerWrappedCode( - /*ad_tech_js=*/ad_tech_code_blobs[0], - /*ad_tech_wasm=*/ - ad_tech_code_blobs.size() == 2 ? ad_tech_code_blobs[1] : "", - kPrepareDataForAdRetrievalHandler, - kPrepareDataForAdRetrievalArgs); - }, - dispatcher, executor, curl_http_fetcher)); - code_fetchers.emplace_back(std::move(prepare_data_for_ads_code_fetcher)); - return absl::OkStatus(); -} - -absl::Status StartProtectedAudienceUdfFetching( - const BuyerCodeFetchConfig& code_fetch_proto, V8Dispatcher& dispatcher, - server_common::Executor* executor, - std::vector>& code_fetchers, - HttpFetcherAsync* curl_http_fetcher) { - std::string js_url = code_fetch_proto.bidding_js_url(); - // Starts periodic code blob fetching from an arbitrary url only if js_url is - // specified - if (!js_url.empty()) { - auto wrap_code = [](const std::vector& adtech_code_blobs) { - return GetBuyerWrappedCode( - /*ad_tech_js=*/adtech_code_blobs.at(0), - adtech_code_blobs.size() == 2 ? adtech_code_blobs.at(1) : /*wasm=*/"", - AuctionType::kProtectedAudience); - }; - - std::vector endpoints = {js_url}; - if (!code_fetch_proto.bidding_wasm_helper_url().empty()) { - endpoints = {std::move(js_url), - code_fetch_proto.bidding_wasm_helper_url()}; - } - - auto code_fetcher = std::make_unique( - std::move(endpoints), - absl::Milliseconds(code_fetch_proto.url_fetch_period_ms()), - curl_http_fetcher, &dispatcher, executor, - absl::Milliseconds(code_fetch_proto.url_fetch_timeout_ms()), wrap_code, - kProtectedAudienceGenerateBidBlobVersion); - - code_fetcher->Start(); - code_fetchers.emplace_back(std::move(code_fetcher)); - } else if (!code_fetch_proto.bidding_js_path().empty()) { - PS_ASSIGN_OR_RETURN(auto adtech_code_blob, - GetFileContent(code_fetch_proto.bidding_js_path(), - /*log_on_error=*/true)); - adtech_code_blob = GetBuyerWrappedCode(adtech_code_blob, ""); - PS_RETURN_IF_ERROR(dispatcher.LoadSync( - kProtectedAudienceGenerateBidBlobVersion, adtech_code_blob)) - << "Could not load Adtech untrusted code for bidding."; - } else { - return absl::UnavailableError( - "Code fetching config requires either a path or url."); - } - return absl::OkStatus(); -} - // Brings up the gRPC BiddingService on FLAGS_port. absl::Status RunServer() { TrustedServerConfigUtil config_util(absl::GetFlag(FLAGS_init_config_client)); @@ -330,40 +208,36 @@ absl::Status RunServer() { std::make_unique( grpc_event_engine::experimental::CreateEventEngine()); - std::vector> code_fetchers; - // Convert Json string into a BiddingCodeBlobFetcherConfig proto - BuyerCodeFetchConfig code_fetch_proto; + BuyerCodeFetchConfig udf_config; absl::Status result = google::protobuf::util::JsonStringToMessage( config_client.GetStringParameter(BUYER_CODE_FETCH_CONFIG).data(), - &code_fetch_proto); - CHECK(result.ok()) << "Could not parse BUYER_CODE_FETCH_CONFIG JsonString to " - "a proto message."; + &udf_config); + PS_RETURN_IF_ERROR(result) + << "Could not parse BUYER_CODE_FETCH_CONFIG JsonString to " + "a proto message: " + << result.message(); auto curl_http_fetcher = std::make_unique(executor.get()); bool enable_buyer_debug_url_generation = - code_fetch_proto.enable_buyer_debug_url_generation(); - bool enable_adtech_code_logging = - code_fetch_proto.enable_adtech_code_logging(); + udf_config.enable_buyer_debug_url_generation(); + bool enable_adtech_code_logging = udf_config.enable_adtech_code_logging(); const bool enable_protected_audience = config_client.HasParameter(ENABLE_PROTECTED_AUDIENCE) && config_client.GetBooleanParameter(ENABLE_PROTECTED_AUDIENCE); - if (enable_protected_audience) { - PS_RETURN_IF_ERROR(StartProtectedAudienceUdfFetching( - code_fetch_proto, dispatcher, executor.get(), code_fetchers, - curl_http_fetcher.get())); - } - const bool enable_protected_app_signals = config_client.HasParameter(ENABLE_PROTECTED_APP_SIGNALS) && config_client.GetBooleanParameter(ENABLE_PROTECTED_APP_SIGNALS); - if (enable_protected_app_signals) { - PS_RETURN_IF_ERROR(StartProtectedAppSignalsUdfFetching( - code_fetch_proto, dispatcher, executor.get(), code_fetchers, - curl_http_fetcher.get())); - } + + auto http_fetcher_async = + std::make_unique(executor.get()); + BuyerCodeFetchManager udf_fetcher( + executor.get(), http_fetcher_async.get(), &dispatcher, + BlobStorageClientFactory::Create(), udf_config, enable_protected_audience, + enable_protected_app_signals); + PS_RETURN_IF_ERROR(udf_fetcher.Init()) << "Failed to initialize UDF fetch."; bool enable_bidding_service_benchmark = config_client.GetBooleanParameter(ENABLE_BIDDING_SERVICE_BENCHMARK); @@ -404,7 +278,7 @@ absl::Status RunServer() { "least one."); } - const BiddingServiceRuntimeConfig runtime_config = { + BiddingServiceRuntimeConfig runtime_config = { .tee_ad_retrieval_kv_server_addr = std::move(tee_ad_retrieval_kv_server_addr), .tee_kv_server_addr = std::move(tee_kv_server_addr), @@ -425,6 +299,19 @@ absl::Status RunServer() { .kv_server_egress_tls = config_client.GetBooleanParameter(KV_SERVER_EGRESS_TLS)}; + if (udf_config.fetch_mode() == bidding_service::FETCH_MODE_BUCKET) { + if (enable_protected_audience) { + runtime_config.default_protected_auction_generate_bid_version = + udf_config.protected_auction_bidding_js_bucket_default_blob(); + } + if (enable_protected_app_signals) { + runtime_config.default_protected_app_signals_generate_bid_version = + udf_config.protected_app_signals_bidding_js_bucket_default_blob(); + runtime_config.default_ad_retrieval_version = + udf_config.ads_retrieval_bucket_default_blob(); + } + } + auto protected_app_signals_generate_bids_reactor_factory = [&client](const grpc::CallbackServerContext* context, const GenerateProtectedAppSignalsBidsRequest* request, @@ -483,12 +370,7 @@ absl::Status RunServer() { // responsible for shutting down the server for this call to ever return. PS_VLOG(1) << "Server listening on " << server_address; server->Wait(); - // Ends periodic code blob fetching from an arbitrary url. - for (const auto& code_fetcher : code_fetchers) { - if (code_fetcher) { - code_fetcher->End(); - } - } + PS_RETURN_IF_ERROR(udf_fetcher.End()) << "Error shutting down UDF fetcher."; PS_RETURN_IF_ERROR(dispatcher.Stop()) << "Error shutting down code dispatcher."; return absl::OkStatus(); diff --git a/services/bidding_service/bidding_service_integration_test.cc b/services/bidding_service/bidding_service_integration_test.cc index 9d8b420e..789d8237 100644 --- a/services/bidding_service/bidding_service_integration_test.cc +++ b/services/bidding_service/bidding_service_integration_test.cc @@ -438,7 +438,10 @@ void SetupV8Dispatcher(V8Dispatcher* dispatcher, absl::string_view adtech_js, absl::string_view adtech_wasm = "") { ASSERT_TRUE(dispatcher->Init().ok()); std::string wrapper_blob = GetBuyerWrappedCode(adtech_js, adtech_wasm); - ASSERT_TRUE(dispatcher->LoadSync("v1", wrapper_blob).ok()); + ASSERT_TRUE( + dispatcher + ->LoadSync(kProtectedAudienceGenerateBidBlobVersion, wrapper_blob) + .ok()); } absl::StatusOr diff --git a/services/bidding_service/buyer_code_fetch_manager.cc b/services/bidding_service/buyer_code_fetch_manager.cc new file mode 100644 index 00000000..852a1b5e --- /dev/null +++ b/services/bidding_service/buyer_code_fetch_manager.cc @@ -0,0 +1,304 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "services/bidding_service/buyer_code_fetch_manager.h" + +#include +#include +#include +#include +#include + +#include "services/bidding_service/bidding_code_fetch_config.pb.h" +#include "services/bidding_service/code_wrapper/buyer_code_wrapper.h" +#include "services/bidding_service/constants.h" +#include "services/common/clients/code_dispatcher/v8_dispatcher.h" +#include "services/common/code_fetch/periodic_bucket_fetcher.h" +#include "services/common/code_fetch/periodic_code_fetcher.h" +#include "services/common/util/file_util.h" +#include "src/cpp/concurrent/event_engine_executor.h" +#include "src/cpp/util/status_macro/status_macros.h" + +#include "buyer_code_fetch_manager.h" + +namespace privacy_sandbox::bidding_auction_servers { +namespace { + +constexpr int kJsBlobIndex = 0; +constexpr int kWasmBlobIndex = 1; +constexpr int kMinNumCodeBlobs = 1; +constexpr int kMaxNumCodeBlobs = 2; +constexpr char kUnusedWasmBlob[] = ""; + +} // namespace + +absl::Status BuyerCodeFetchManager::Init() { + switch (udf_config_.fetch_mode()) { + case bidding_service::FETCH_MODE_LOCAL: + return InitializeLocalCodeFetch(); + case bidding_service::FETCH_MODE_BUCKET: + return InitializeBucketCodeFetch(); + case bidding_service::FETCH_MODE_URL: + return InitializeUrlCodeFetch(); + default: + return absl::InvalidArgumentError(kFetchModeInvalid); + } +} + +absl::Status BuyerCodeFetchManager::End() { + if (enable_protected_audience_) { + if (pa_udf_fetcher_ != nullptr) { + pa_udf_fetcher_->End(); + } + } + if (enable_protected_app_signals_) { + if (pas_bidding_udf_fetcher_ != nullptr) { + pas_bidding_udf_fetcher_->End(); + } + if (pas_ads_retrieval_udf_fetcher_ != nullptr) { + pas_ads_retrieval_udf_fetcher_->End(); + } + } + return absl::OkStatus(); +} + +// PA only; PAS not supported. +absl::Status BuyerCodeFetchManager::InitializeLocalCodeFetch() { + if (udf_config_.bidding_js_path().empty()) { + return absl::UnavailableError(kLocalFetchNeedsPath); + } + PS_ASSIGN_OR_RETURN(auto adtech_code_blob, + GetFileContent(udf_config_.bidding_js_path(), + /*log_on_error=*/true)); + adtech_code_blob = GetBuyerWrappedCode(adtech_code_blob, ""); + return dispatcher_.LoadSync(kProtectedAudienceGenerateBidBlobVersion, + adtech_code_blob); +} + +absl::Status BuyerCodeFetchManager::InitializeBucketCodeFetch() { + if (enable_protected_audience_ || enable_protected_app_signals_) { + PS_RETURN_IF_ERROR(InitBucketClient()) << kBlobStorageClientInitFailed; + } + if (enable_protected_audience_) { + PS_RETURN_IF_ERROR(InitializeBucketCodeFetchForPA()); + } + if (enable_protected_app_signals_) { + PS_RETURN_IF_ERROR(InitializeBucketCodeFetchForPAS()); + } + return absl::OkStatus(); +} + +absl::Status BuyerCodeFetchManager::InitializeBucketCodeFetchForPA() { + auto wrap_code = [](const std::vector& adtech_code_blobs) { + return GetBuyerWrappedCode(adtech_code_blobs[kJsBlobIndex], kUnusedWasmBlob, + AuctionType::kProtectedAudience); + }; + + PS_ASSIGN_OR_RETURN( + pa_udf_fetcher_, + StartBucketFetch( + udf_config_.protected_auction_bidding_js_bucket(), + udf_config_.protected_auction_bidding_js_bucket_default_blob(), + kProtectedAuctionJsId, + absl::Milliseconds(udf_config_.url_fetch_period_ms()), + std::move(wrap_code))); + + return absl::OkStatus(); +} + +absl::Status BuyerCodeFetchManager::InitializeBucketCodeFetchForPAS() { + auto wrap_bidding_code = + [](const std::vector& ad_tech_code_blobs) { + return GetBuyerWrappedCode( + ad_tech_code_blobs[kJsBlobIndex], kUnusedWasmBlob, + AuctionType::kProtectedAppSignals, + /*auction_specific_setup=*/"// No additional setup"); + }; + + auto wrap_ads_retrieval_code = + [](const std::vector& ad_tech_code_blobs) { + return GetProtectedAppSignalsGenericBuyerWrappedCode( + ad_tech_code_blobs[kJsBlobIndex], kUnusedWasmBlob, + kPrepareDataForAdRetrievalHandler, kPrepareDataForAdRetrievalArgs); + }; + + absl::Duration url_fetch_period_ms = + absl::Milliseconds(udf_config_.url_fetch_period_ms()); + + PS_ASSIGN_OR_RETURN( + pas_bidding_udf_fetcher_, + StartBucketFetch( + udf_config_.protected_app_signals_bidding_js_bucket(), + udf_config_.protected_app_signals_bidding_js_bucket_default_blob(), + kProtectedAppSignalsJsId, url_fetch_period_ms, + std::move(wrap_bidding_code))); + + PS_ASSIGN_OR_RETURN( + pas_ads_retrieval_udf_fetcher_, + StartBucketFetch(udf_config_.ads_retrieval_js_bucket(), + udf_config_.ads_retrieval_bucket_default_blob(), + kAdsRetrievalJsId, url_fetch_period_ms, + std::move(wrap_ads_retrieval_code))); + + return absl::OkStatus(); +} + +absl::Status BuyerCodeFetchManager::InitializeUrlCodeFetch() { + if (enable_protected_audience_) { + PS_RETURN_IF_ERROR(InitializeUrlCodeFetchForPA()); + } + if (enable_protected_app_signals_) { + PS_RETURN_IF_ERROR(InitializeUrlCodeFetchForPAS()); + } + + return absl::OkStatus(); +} + +absl::Status BuyerCodeFetchManager::InitializeUrlCodeFetchForPA() { + auto wrap_code = [](const std::vector& adtech_code_blobs) { + return GetBuyerWrappedCode(adtech_code_blobs[kJsBlobIndex], + adtech_code_blobs.size() == kMaxNumCodeBlobs + ? adtech_code_blobs[kWasmBlobIndex] + : kUnusedWasmBlob, + AuctionType::kProtectedAudience); + }; + + PS_ASSIGN_OR_RETURN( + pa_udf_fetcher_, + StartUrlFetch(udf_config_.bidding_js_url(), + udf_config_.bidding_wasm_helper_url(), + kProtectedAudienceGenerateBidBlobVersion, "bidding_js_url", + absl::Milliseconds(udf_config_.url_fetch_period_ms()), + absl::Milliseconds(udf_config_.url_fetch_timeout_ms()), + std::move(wrap_code))); + return absl::OkStatus(); +} + +absl::Status BuyerCodeFetchManager::InitializeUrlCodeFetchForPAS() { + auto wrap_bidding_code = + [](const std::vector& ad_tech_code_blobs) { + DCHECK_GE(ad_tech_code_blobs.size(), kMinNumCodeBlobs); + DCHECK_LE(ad_tech_code_blobs.size(), kMaxNumCodeBlobs); + return GetBuyerWrappedCode( + ad_tech_code_blobs[kJsBlobIndex], + ad_tech_code_blobs.size() == kMaxNumCodeBlobs + ? ad_tech_code_blobs[kWasmBlobIndex] + : kUnusedWasmBlob, + AuctionType::kProtectedAppSignals, + /*auction_specific_setup=*/"// No additional setup"); + }; + + auto wrap_ads_retrieval_code = + [](const std::vector& ad_tech_code_blobs) { + DCHECK_GE(ad_tech_code_blobs.size(), kMinNumCodeBlobs); + DCHECK_LE(ad_tech_code_blobs.size(), kMaxNumCodeBlobs); + return GetProtectedAppSignalsGenericBuyerWrappedCode( + ad_tech_code_blobs[kJsBlobIndex], + ad_tech_code_blobs.size() == kMaxNumCodeBlobs + ? ad_tech_code_blobs[kWasmBlobIndex] + : kUnusedWasmBlob, + kPrepareDataForAdRetrievalHandler, kPrepareDataForAdRetrievalArgs); + }; + + absl::Duration url_fetch_period_ms = + absl::Milliseconds(udf_config_.url_fetch_period_ms()); + absl::Duration url_fetch_timeout_ms = + absl::Milliseconds(udf_config_.url_fetch_timeout_ms()); + + PS_ASSIGN_OR_RETURN( + pas_bidding_udf_fetcher_, + StartUrlFetch(udf_config_.protected_app_signals_bidding_js_url(), + udf_config_.protected_app_signals_bidding_wasm_helper_url(), + kProtectedAppSignalsGenerateBidBlobVersion, + "protected_app_signals_bidding_js_url", url_fetch_period_ms, + url_fetch_timeout_ms, std::move(wrap_bidding_code))); + + PS_ASSIGN_OR_RETURN( + pas_ads_retrieval_udf_fetcher_, + StartUrlFetch( + udf_config_.prepare_data_for_ads_retrieval_js_url(), + udf_config_.prepare_data_for_ads_retrieval_wasm_helper_url(), + kPrepareDataForAdRetrievalBlobVersion, + "prepare_data_for_ads_retrieval_js_url", url_fetch_period_ms, + url_fetch_timeout_ms, std::move(wrap_ads_retrieval_code))); + + return absl::OkStatus(); +} + +absl::StatusOr> +BuyerCodeFetchManager::StartUrlFetch( + const std::string& js_url, const std::string& wasm_helper_url, + const std::string& roma_version, absl::string_view script_logging_name, + absl::Duration url_fetch_period_ms, absl::Duration url_fetch_timeout_ms, + absl::AnyInvocable&)> + wrap_code) { + if (js_url.empty()) { + return absl::InvalidArgumentError( + absl::StrCat("JS URL for ", script_logging_name, " is missing")); + } + + std::vector endpoints = {js_url}; + if (!wasm_helper_url.empty()) { + endpoints.emplace_back(wasm_helper_url); + } + auto code_fetcher = std::make_unique( + std::move(endpoints), url_fetch_period_ms, &http_fetcher_, &dispatcher_, + &executor_, url_fetch_timeout_ms, std::move(wrap_code), roma_version); + + PS_RETURN_IF_ERROR(code_fetcher->Start()) + << absl::StrCat("Failed url fetcher startup for ", script_logging_name); + return code_fetcher; +} + +absl::StatusOr> +BuyerCodeFetchManager::StartBucketFetch( + const std::string& bucket_name, const std::string& default_version, + absl::string_view script_logging_name, absl::Duration url_fetch_period_ms, + absl::AnyInvocable&)> + wrap_code) { + if (bucket_name.empty()) { + return absl::InvalidArgumentError( + absl::StrCat(kEmptyBucketName, script_logging_name)); + } else if (default_version.empty()) { + return absl::InvalidArgumentError( + absl::StrCat(kEmptyBucketDefault, script_logging_name)); + } + + auto bucket_fetcher = std::make_unique( + bucket_name, url_fetch_period_ms, &dispatcher_, &executor_, + std::move(wrap_code), blob_storage_client_.get()); + PS_RETURN_IF_ERROR(bucket_fetcher->Start()) + << absl::StrCat("Failed bucket fetch startup for ", script_logging_name, + " ", bucket_name); + return bucket_fetcher; +} + +absl::Status BuyerCodeFetchManager::InitBucketClient() { + auto result = blob_storage_client_->Init(); + if (!result.Successful()) { + return absl::UnavailableError( + absl::StrFormat("Failed to init BlobStorageClient (status_code: %s)\n", + GetErrorMessage(result.status_code))); + } + + result = blob_storage_client_->Run(); + if (!result.Successful()) { + return absl::UnavailableError( + absl::StrFormat("Failed to run BlobStorageClient (status_code: %s)\n", + GetErrorMessage(result.status_code))); + } + return absl::OkStatus(); +} + +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/bidding_service/buyer_code_fetch_manager.h b/services/bidding_service/buyer_code_fetch_manager.h new file mode 100644 index 00000000..3c2139d3 --- /dev/null +++ b/services/bidding_service/buyer_code_fetch_manager.h @@ -0,0 +1,126 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SERVICES_BUYER_CODE_FETCH_MANAGER_H_ +#define SERVICES_BUYER_CODE_FETCH_MANAGER_H_ + +#include +#include +#include +#include +#include + +#include "services/bidding_service/bidding_code_fetch_config.pb.h" +#include "services/common/clients/code_dispatcher/v8_dispatcher.h" +#include "services/common/clients/http/http_fetcher_async.h" +#include "services/common/code_fetch/periodic_bucket_fetcher.h" +#include "services/common/code_fetch/periodic_code_fetcher.h" +#include "src/cpp/concurrent/event_engine_executor.h" +#include "src/cpp/util/status_macro/status_macros.h" + +namespace privacy_sandbox::bidding_auction_servers { + +constexpr char kProtectedAuctionJsId[] = "bidding_js"; +constexpr char kProtectedAppSignalsJsId[] = "protected_app_signals_bidding_js"; +constexpr char kAdsRetrievalJsId[] = "prepare_data_for_ads_retrieval_js"; +constexpr char kFetchModeInvalid[] = "Fetch mode invalid."; +constexpr char kLocalFetchNeedsPath[] = + "Local fetch mode requires a non-empty path."; +constexpr char kBlobStorageClientInitFailed[] = + "Failed to init cloud blob storage client."; +constexpr char kEmptyBucketName[] = "Empty bucket name for "; +constexpr char kEmptyBucketDefault[] = + "Bucket fetch mode requires a non-empty bucket default version object for "; +constexpr char kFailedBucketFetchStartup[] = "Failed bucket fetch startup for "; + +// BuyerCodeFetchManager acts as a wrapper for all logic related to fetching +// bidding service UDFs. This class consumes a BuyerCodeFetchConfig and uses it, +// along with various other dependencies, to create and own all instances of +// CodeFetcherInterface in the bidding service. +class BuyerCodeFetchManager { + public: + // All raw pointers indicate that we are borrowing a reference and MUST + // outlive BuyerCodeFetchManager. + explicit BuyerCodeFetchManager( + server_common::Executor* executor, HttpFetcherAsync* http_fetcher, + V8Dispatcher* dispatcher, + std::unique_ptr + blob_storage_client, + const bidding_service::BuyerCodeFetchConfig udf_config, + bool enable_protected_audience, bool enable_protected_app_signals) + : executor_(*executor), + http_fetcher_(*http_fetcher), + dispatcher_(*dispatcher), + blob_storage_client_(std::move(blob_storage_client)), + udf_config_(udf_config), + enable_protected_audience_(enable_protected_audience), + enable_protected_app_signals_(enable_protected_app_signals) {} + + // Not copyable or movable. + BuyerCodeFetchManager(const BuyerCodeFetchManager&) = delete; + BuyerCodeFetchManager& operator=(const BuyerCodeFetchManager&) = delete; + + // Must be called exactly once. Failure to Init means that the bidding service + // has not successfully loaded any UDFs and is unable to serve any requests. + // A successful Init means that Roma has succeeded in loading a UDF. + absl::Status Init(); + + // Must be called exactly once. This should only be called on server shutdown, + // and only after Init has returned (either a success or error is fine). + // Failure to End means there was an issue releasing resources and should + // be investigated to ensure that requests are being terminated gracefully. + absl::Status End(); + + private: + absl::Status InitializeLocalCodeFetch(); + + absl::Status InitBucketClient(); + absl::Status InitializeBucketCodeFetch(); + absl::Status InitializeBucketCodeFetchForPA(); + absl::Status InitializeBucketCodeFetchForPAS(); + + absl::StatusOr> StartBucketFetch( + const std::string& bucket_name, const std::string& default_version, + absl::string_view script_logging_name, absl::Duration url_fetch_period_ms, + absl::AnyInvocable&)> + wrap_code); + + absl::Status InitializeUrlCodeFetch(); + absl::Status InitializeUrlCodeFetchForPA(); + absl::Status InitializeUrlCodeFetchForPAS(); + + absl::StatusOr> StartUrlFetch( + const std::string& js_url, const std::string& wasm_helper_url, + const std::string& roma_version, absl::string_view script_logging_name, + absl::Duration url_fetch_period_ms, absl::Duration url_fetch_timeout_ms, + absl::AnyInvocable&)> + wrap_code); + + server_common::Executor& executor_; + HttpFetcherAsync& http_fetcher_; + V8Dispatcher& dispatcher_; + std::unique_ptr + blob_storage_client_; + const bidding_service::BuyerCodeFetchConfig udf_config_; + const bool enable_protected_audience_; + const bool enable_protected_app_signals_; + + std::unique_ptr pa_udf_fetcher_; + std::unique_ptr pas_bidding_udf_fetcher_; + std::unique_ptr pas_ads_retrieval_udf_fetcher_; +}; + +} // namespace privacy_sandbox::bidding_auction_servers + +#endif // SERVICES_BUYER_CODE_FETCH_MANAGER_H_ diff --git a/services/bidding_service/buyer_code_fetch_manager_test.cc b/services/bidding_service/buyer_code_fetch_manager_test.cc new file mode 100644 index 00000000..44cdc18d --- /dev/null +++ b/services/bidding_service/buyer_code_fetch_manager_test.cc @@ -0,0 +1,301 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "services/bidding_service/buyer_code_fetch_manager.h" + +#include +#include + +#include "absl/strings/str_cat.h" +#include "absl/synchronization/blocking_counter.h" +#include "absl/synchronization/notification.h" +#include "absl/time/time.h" +#include "gtest/gtest.h" +#include "scp/cc/core/interface/async_context.h" +#include "scp/cc/public/cpio/interface/blob_storage_client/blob_storage_client_interface.h" +#include "scp/cc/public/cpio/interface/cpio.h" +#include "scp/cc/public/cpio/interface/error_codes.h" +#include "scp/cc/public/cpio/mock/blob_storage_client/mock_blob_storage_client.h" +#include "services/bidding_service/code_wrapper/buyer_code_wrapper.h" +#include "services/common/test/mocks.h" +#include "services/common/util/file_util.h" + +namespace privacy_sandbox::bidding_auction_servers { +namespace { + +using bidding_service::BuyerCodeFetchConfig; +using bidding_service::FetchMode; + +using ::google::cmrt::sdk::blob_storage_service::v1::BlobMetadata; +using ::google::cmrt::sdk::blob_storage_service::v1::GetBlobRequest; +using ::google::cmrt::sdk::blob_storage_service::v1::GetBlobResponse; +using ::google::cmrt::sdk::blob_storage_service::v1::ListBlobsMetadataRequest; +using ::google::cmrt::sdk::blob_storage_service::v1::ListBlobsMetadataResponse; +using ::google::scp::core::AsyncContext; +using ::google::scp::core::ExecutionResult; +using ::google::scp::core::FailureExecutionResult; +using ::google::scp::core::SuccessExecutionResult; +using ::google::scp::cpio::MockBlobStorageClient; +using ::testing::Return; + +class BuyerCodeFetchManagerTest : public testing::Test { + protected: + void SetUp() override { + executor_ = std::make_unique(); + http_fetcher_ = std::make_unique(); + dispatcher_ = std::make_unique(); + blob_storage_client_ = std::make_unique(); + } + + std::unique_ptr executor_; + std::unique_ptr http_fetcher_; + std::unique_ptr dispatcher_; + std::unique_ptr blob_storage_client_; +}; + +TEST_F(BuyerCodeFetchManagerTest, FetchModeLocalTriesFileLoad) { + EXPECT_CALL(*blob_storage_client_, Init).Times(0); + EXPECT_CALL(*blob_storage_client_, Run).Times(0); + + BuyerCodeFetchConfig udf_config; + udf_config.set_fetch_mode(FetchMode::FETCH_MODE_LOCAL); + const std::string bad_path = "error"; + udf_config.set_bidding_js_path(bad_path); + BuyerCodeFetchManager udf_fetcher(executor_.get(), http_fetcher_.get(), + dispatcher_.get(), + std::move(blob_storage_client_), udf_config, + true /*enable_protected_audience*/, + false /*enable_protected_app_signals*/); + absl::Status load_status = udf_fetcher.Init(); + EXPECT_FALSE(load_status.ok()); + EXPECT_EQ(load_status.message(), absl::StrCat(kPathFailed, bad_path)); +} + +TEST_F(BuyerCodeFetchManagerTest, FetchModeBucketTriesBucketLoad) { + EXPECT_CALL(*blob_storage_client_, Init) + .WillOnce(Return(SuccessExecutionResult())); + EXPECT_CALL(*blob_storage_client_, Run) + .WillOnce(Return(SuccessExecutionResult())); + BuyerCodeFetchConfig udf_config; + udf_config.set_fetch_mode(FetchMode::FETCH_MODE_BUCKET); + BuyerCodeFetchManager udf_fetcher(executor_.get(), http_fetcher_.get(), + dispatcher_.get(), + std::move(blob_storage_client_), udf_config, + true /*enable_protected_audience*/, + true /*enable_protected_app_signals*/); + absl::Status load_status = udf_fetcher.Init(); + EXPECT_FALSE(load_status.ok()); + EXPECT_EQ(load_status.message(), + absl::StrCat(kEmptyBucketName, kProtectedAuctionJsId)); +} + +TEST_F(BuyerCodeFetchManagerTest, TriesBucketFetchForProtectedAuction) { + BuyerCodeFetchConfig udf_config; + udf_config.set_fetch_mode(FetchMode::FETCH_MODE_BUCKET); + udf_config.set_url_fetch_period_ms(1); + udf_config.set_protected_auction_bidding_js_bucket("pa"); + udf_config.set_protected_auction_bidding_js_bucket_default_blob("pa_test"); + + EXPECT_CALL(*blob_storage_client_, Init) + .WillOnce(Return(SuccessExecutionResult())); + EXPECT_CALL(*blob_storage_client_, Run) + .WillOnce(Return(SuccessExecutionResult())); + + // EXPECT_CALL(*blob_storage_client_) + BuyerCodeFetchManager udf_fetcher(executor_.get(), http_fetcher_.get(), + dispatcher_.get(), + std::move(blob_storage_client_), udf_config, + true /*enable_protected_audience*/, + false /*enable_protected_app_signals*/); + absl::Status load_status = udf_fetcher.Init(); + EXPECT_FALSE(load_status.ok()); + EXPECT_TRUE(absl::StrContains( + load_status.message(), + absl::StrCat(kFailedBucketFetchStartup, kProtectedAuctionJsId, " pa"))); +} + +TEST_F(BuyerCodeFetchManagerTest, TriesBucketFetchForProtectedAppSignals) { + BuyerCodeFetchConfig udf_config; + udf_config.set_fetch_mode(FetchMode::FETCH_MODE_BUCKET); + udf_config.set_url_fetch_period_ms(1); + udf_config.set_protected_app_signals_bidding_js_bucket("pas"); + udf_config.set_protected_app_signals_bidding_js_bucket_default_blob( + "pas_test"); + + EXPECT_CALL(*blob_storage_client_, Init) + .WillOnce(Return(SuccessExecutionResult())); + EXPECT_CALL(*blob_storage_client_, Run) + .WillOnce(Return(SuccessExecutionResult())); + + BuyerCodeFetchManager udf_fetcher(executor_.get(), http_fetcher_.get(), + dispatcher_.get(), + std::move(blob_storage_client_), udf_config, + false /*enable_protected_audience*/, + true /*enable_protected_app_signals*/); + absl::Status load_status = udf_fetcher.Init(); + EXPECT_FALSE(load_status.ok()); + EXPECT_TRUE(absl::StrContains( + load_status.message(), absl::StrCat(kFailedBucketFetchStartup, + kProtectedAppSignalsJsId, " pas"))); +} + +TEST_F(BuyerCodeFetchManagerTest, + TriesBucketFetchForAdsRetrievalAfterProtectedAppSignalsFetchSuccess) { + const std::string pas_bucket = "pas"; + const std::string pas_object = "pas_test"; + const std::string pas_data = "sample_data"; + const std::string ads_bucket = "ads_retrieval"; + const std::string ads_object = "ads_test"; + + BuyerCodeFetchConfig udf_config; + udf_config.set_fetch_mode(FetchMode::FETCH_MODE_BUCKET); + udf_config.set_url_fetch_period_ms(1); + udf_config.set_protected_app_signals_bidding_js_bucket(pas_bucket); + udf_config.set_protected_app_signals_bidding_js_bucket_default_blob( + pas_object); + udf_config.set_ads_retrieval_js_bucket(ads_bucket); + udf_config.set_ads_retrieval_bucket_default_blob(ads_object); + + EXPECT_CALL(*blob_storage_client_, Init) + .WillOnce(Return(SuccessExecutionResult())); + EXPECT_CALL(*blob_storage_client_, Run) + .WillOnce(Return(SuccessExecutionResult())); + EXPECT_CALL(*blob_storage_client_, ListBlobsMetadata) + .WillOnce( + [&](AsyncContext + context) { + EXPECT_EQ(context.request->blob_metadata().bucket_name(), + pas_bucket); + BlobMetadata md; + md.set_bucket_name(pas_bucket); + md.set_blob_name(pas_object); + + context.response = std::make_shared(); + context.response->mutable_blob_metadatas()->Add(std::move(md)); + context.Finish(SuccessExecutionResult()); + return SuccessExecutionResult(); + }) + .WillOnce( + [&](AsyncContext + context) { + EXPECT_EQ(context.request->blob_metadata().bucket_name(), + ads_bucket); + return FailureExecutionResult(SC_UNKNOWN); + }); + EXPECT_CALL(*blob_storage_client_, GetBlob) + .WillOnce( + [&](AsyncContext async_context) { + auto async_bucket_name = + async_context.request->blob_metadata().bucket_name(); + auto async_blob_name = + async_context.request->blob_metadata().blob_name(); + EXPECT_EQ(async_bucket_name, pas_bucket); + EXPECT_EQ(async_blob_name, pas_object); + + async_context.response = std::make_shared(); + async_context.response->mutable_blob()->set_data(pas_data); + async_context.result = SuccessExecutionResult(); + async_context.Finish(); + + return SuccessExecutionResult(); + }); + + EXPECT_CALL(*dispatcher_, LoadSync) + .WillOnce([&](std::string_view version, absl::string_view blob_data) { + EXPECT_EQ(version, pas_object); + EXPECT_EQ( + blob_data, + GetBuyerWrappedCode(pas_data, "", AuctionType::kProtectedAppSignals, + "// No additional setup")); + return absl::OkStatus(); + }); + + BuyerCodeFetchManager udf_fetcher(executor_.get(), http_fetcher_.get(), + dispatcher_.get(), + std::move(blob_storage_client_), udf_config, + false /*enable_protected_audience*/, + true /*enable_protected_app_signals*/); + absl::Status load_status = udf_fetcher.Init(); + + EXPECT_TRUE(absl::StrContains( + load_status.message(), absl::StrCat(kFailedBucketFetchStartup, + kAdsRetrievalJsId, " ", ads_bucket))); +} + +TEST_F(BuyerCodeFetchManagerTest, FetchModeUrlTriesUrlLoad) { + EXPECT_CALL(*blob_storage_client_, Init).Times(0); + EXPECT_CALL(*blob_storage_client_, Run).Times(0); + + const std::string pa_url = "a"; + const std::string pa_wasm_url = "b"; + const std::string pas_url = "c"; + const std::string pas_wasm_url = "d"; + const std::string ads_retrieval_url = "e"; + const std::string ads_retrieval_wasm_url = "f"; + + BuyerCodeFetchConfig udf_config; + udf_config.set_fetch_mode(FetchMode::FETCH_MODE_URL); + + udf_config.set_bidding_js_url(pa_url); + udf_config.set_bidding_wasm_helper_url(pa_wasm_url); + + udf_config.set_protected_app_signals_bidding_js_url(pas_url); + udf_config.set_protected_app_signals_bidding_wasm_helper_url(pas_wasm_url); + + udf_config.set_prepare_data_for_ads_retrieval_js_url(ads_retrieval_url); + udf_config.set_prepare_data_for_ads_retrieval_wasm_helper_url( + ads_retrieval_wasm_url); + + udf_config.set_url_fetch_period_ms(200000); + udf_config.set_url_fetch_timeout_ms(100000); + + EXPECT_CALL(*dispatcher_, LoadSync) + .WillRepeatedly( + [&](std::string_view version, absl::string_view blob_data) { + return absl::OkStatus(); + }); + + EXPECT_CALL(*http_fetcher_, FetchUrls) + .WillOnce([&](const std::vector& requests, + absl::Duration timeout, OnDoneFetchUrls done_callback) { + EXPECT_EQ(requests[0].url, pa_url); + EXPECT_EQ(requests[1].url, pa_wasm_url); + std::move(done_callback)({""}); + }) + .WillOnce([&](const std::vector& requests, + absl::Duration timeout, OnDoneFetchUrls done_callback) { + EXPECT_EQ(requests[0].url, pas_url); + EXPECT_EQ(requests[1].url, pas_wasm_url); + std::move(done_callback)({""}); + }) + .WillOnce([&](const std::vector& requests, + absl::Duration timeout, OnDoneFetchUrls done_callback) { + EXPECT_EQ(requests[0].url, ads_retrieval_url); + EXPECT_EQ(requests[1].url, ads_retrieval_wasm_url); + std::move(done_callback)({""}); + }); + + BuyerCodeFetchManager udf_fetcher(executor_.get(), http_fetcher_.get(), + dispatcher_.get(), + std::move(blob_storage_client_), udf_config, + true /*enable_protected_audience*/, + true /*enable_protected_app_signals*/); + absl::Status load_status = udf_fetcher.Init(); + EXPECT_TRUE(load_status.ok()); +} + +} // namespace +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/bidding_service/data/BUILD b/services/bidding_service/data/BUILD index 2a2baf41..6d517ebd 100644 --- a/services/bidding_service/data/BUILD +++ b/services/bidding_service/data/BUILD @@ -21,4 +21,7 @@ cc_library( hdrs = [ "runtime_config.h", ], + deps = [ + "//services/bidding_service:bidding_constants", + ], ) diff --git a/services/bidding_service/data/runtime_config.h b/services/bidding_service/data/runtime_config.h index f2da1090..da4d838d 100644 --- a/services/bidding_service/data/runtime_config.h +++ b/services/bidding_service/data/runtime_config.h @@ -19,6 +19,8 @@ #include +#include "services/bidding_service/constants.h" + namespace privacy_sandbox::bidding_auction_servers { struct BiddingServiceRuntimeConfig { @@ -54,6 +56,13 @@ struct BiddingServiceRuntimeConfig { bool ad_retrieval_kv_server_egress_tls = true; // Whether GRPC client to KV server should use TLS. bool kv_server_egress_tls = true; + // Default UDF versions: + std::string default_protected_auction_generate_bid_version = + kProtectedAudienceGenerateBidBlobVersion; + std::string default_protected_app_signals_generate_bid_version = + kProtectedAppSignalsGenerateBidBlobVersion; + std::string default_ad_retrieval_version = + kPrepareDataForAdRetrievalBlobVersion; }; } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/bidding_service/generate_bids_reactor.cc b/services/bidding_service/generate_bids_reactor.cc index 7e15de10..7cf281c3 100644 --- a/services/bidding_service/generate_bids_reactor.cc +++ b/services/bidding_service/generate_bids_reactor.cc @@ -277,7 +277,7 @@ absl::StatusOr BuildGenerateBidRequest( const TrustedBiddingSignalsByIg& ig_trusted_signals_map, const bool enable_buyer_debug_url_generation, server_common::log::ContextImpl& log_context, - const bool enable_adtech_code_logging, absl::string_view version) { + const bool enable_adtech_code_logging, const std::string& version) { // Construct the wrapper struct for our V8 Dispatch Request. DispatchRequest generate_bid_request; generate_bid_request.id = interest_group.name(); @@ -419,10 +419,15 @@ GenerateBidsReactor::GenerateBidsReactor( benchmarking_logger_(std::move(benchmarking_logger)), auction_scope_(raw_request_.top_level_seller().empty() ? AuctionScope::kSingleSeller - : AuctionScope::kDeviceComponentSeller) { + : AuctionScope::kDeviceComponentSeller), + protected_auction_generate_bid_version_( + runtime_config.default_protected_auction_generate_bid_version) { CHECK_OK([this]() { PS_ASSIGN_OR_RETURN(metric_context_, metric::BiddingContextMap()->Remove(request_)); + if (log_context_.is_consented()) { + metric_context_->SetConsented(raw_request_.log_context().generation_id()); + } return absl::OkStatus(); }()) << "BiddingContextMap()->Get(request) should have been called"; } @@ -458,7 +463,7 @@ void GenerateBidsReactor::Execute() { ig_trusted_signals_map.value(), enable_buyer_debug_url_generation_, log_context_, enable_adtech_code_logging_, - kProtectedAudienceGenerateBidBlobVersion); + protected_auction_generate_bid_version_); if (!generate_bid_request.ok()) { PS_VLOG(3, log_context_) << "Unable to build GenerateBidRequest: " diff --git a/services/bidding_service/generate_bids_reactor.h b/services/bidding_service/generate_bids_reactor.h index 685e358d..0a998411 100644 --- a/services/bidding_service/generate_bids_reactor.h +++ b/services/bidding_service/generate_bids_reactor.h @@ -92,6 +92,9 @@ class GenerateBidsReactor // Impacts the creation of generateBid input params and // parsing of generateBid output. AuctionScope auction_scope_; + + // UDF version to use for this request. + const std::string& protected_auction_generate_bid_version_; }; } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/bidding_service/protected_app_signals_generate_bids_reactor.cc b/services/bidding_service/protected_app_signals_generate_bids_reactor.cc index 6d9dc859..612ee59b 100644 --- a/services/bidding_service/protected_app_signals_generate_bids_reactor.cc +++ b/services/bidding_service/protected_app_signals_generate_bids_reactor.cc @@ -86,7 +86,10 @@ ProtectedAppSignalsGenerateBidsReactor::ProtectedAppSignalsGenerateBidsReactor( kv_async_client_(kv_async_client), ad_bids_retrieval_timeout_ms_(runtime_config.ad_retrieval_timeout_ms), metadata_(GrpcMetadataToRequestMetadata(context->client_metadata(), - kBuyerMetadataKeysMap)) { + kBuyerMetadataKeysMap)), + protected_app_signals_generate_bid_version_( + runtime_config.default_protected_app_signals_generate_bid_version), + ad_retrieval_version_(runtime_config.default_ad_retrieval_version) { DCHECK(ad_retrieval_async_client_) << "Missing: KV server Async GRPC client"; } @@ -190,7 +193,7 @@ ProtectedAppSignalsGenerateBidsReactor::CreateGenerateBidsRequest( ArgIndex(GenerateBidsUdfArgs::kFeatureFlags), input); DispatchRequest request = { .id = raw_request_.log_context().generation_id(), - .version_string = kProtectedAppSignalsGenerateBidBlobVersion, + .version_string = protected_app_signals_generate_bid_version_, .handler_name = kDispatchHandlerFunctionNameWithCodeWrapper, .input = std::move(input), }; @@ -275,7 +278,7 @@ DispatchRequest ProtectedAppSignalsGenerateBidsReactor:: ArgIndex(PrepareDataForRetrievalUdfArgs::kFeatureFlags), input); DispatchRequest request = { .id = raw_request_.log_context().generation_id(), - .version_string = kPrepareDataForAdRetrievalBlobVersion, + .version_string = ad_retrieval_version_, .handler_name = kPrepareDataForAdRetrievalEntryFunctionName, .input = std::move(input), }; diff --git a/services/bidding_service/protected_app_signals_generate_bids_reactor.h b/services/bidding_service/protected_app_signals_generate_bids_reactor.h index c82d3127..a33d58af 100644 --- a/services/bidding_service/protected_app_signals_generate_bids_reactor.h +++ b/services/bidding_service/protected_app_signals_generate_bids_reactor.h @@ -151,6 +151,10 @@ class ProtectedAppSignalsGenerateBidsReactor RequestMetadata metadata_; std::vector embeddings_requests_; absl::optional is_contextual_retrieval_request_; + + // UDF versions to use for this request. + const std::string& protected_app_signals_generate_bid_version_; + const std::string& ad_retrieval_version_; }; } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/buyer_frontend_service/get_bids_unary_reactor.cc b/services/buyer_frontend_service/get_bids_unary_reactor.cc index ba9213d5..ec8dacc0 100644 --- a/services/buyer_frontend_service/get_bids_unary_reactor.cc +++ b/services/buyer_frontend_service/get_bids_unary_reactor.cc @@ -122,6 +122,9 @@ GetBidsUnaryReactor::GetBidsUnaryReactor( CHECK_OK([this]() { PS_ASSIGN_OR_RETURN(metric_context_, metric::BfeContextMap()->Remove(request_)); + if (log_context_.is_consented()) { + metric_context_->SetConsented(raw_request_.log_context().generation_id()); + } return absl::OkStatus(); }()) << "BfeContextMap()->Get(request) should have been called"; diff --git a/services/common/metric/server_definition.h b/services/common/metric/server_definition.h index fa32e6c5..852419a5 100644 --- a/services/common/metric/server_definition.h +++ b/services/common/metric/server_definition.h @@ -778,16 +778,18 @@ inline void AddErrorTypePartition( template void LogCommonMetric(const RequestT* request, const ResponseT* response) { auto& metric_context = metric::MetricContextMap()->Get(request); - LogIfError(metric_context.template LogUpDownCounter< - server_common::metrics::kTotalRequestCount>(1)); + LogIfError(metric_context.template LogUpDownCounterDeferred< + server_common::metrics::kTotalRequestCount>( + []() -> int { return 1; })); LogIfError(metric_context.template LogHistogramDeferred< server_common::metrics::kServerTotalTimeMs>( [start = absl::Now()]() -> int { return (absl::Now() - start) / absl::Milliseconds(1); })); - LogIfError(metric_context - .template LogHistogram( - (int)request->ByteSizeLong())); + LogIfError( + metric_context + .template LogHistogramDeferred( + [request]() -> int { return request->ByteSizeLong(); })); LogIfError( metric_context .template LogHistogramDeferred( diff --git a/services/common/telemetry/configure_telemetry.h b/services/common/telemetry/configure_telemetry.h index 2111b7bb..16698e52 100644 --- a/services/common/telemetry/configure_telemetry.h +++ b/services/common/telemetry/configure_telemetry.h @@ -37,7 +37,7 @@ namespace privacy_sandbox::bidding_auction_servers { // TODO (b/278899152): get version dynamically inline constexpr std::string_view kOpenTelemetryVersion = "1.9.1"; -inline constexpr std::string_view kBuildVersion = "3.2.0"; +inline constexpr std::string_view kBuildVersion = "3.3.0"; inline constexpr std::string_view kOperator = "operator"; inline constexpr std::string_view kZone = "zone"; inline constexpr std::string_view kRegion = "region"; diff --git a/services/common/test/mocks.h b/services/common/test/mocks.h index d7a6c557..394d1239 100644 --- a/services/common/test/mocks.h +++ b/services/common/test/mocks.h @@ -334,6 +334,7 @@ class KVServiceMock : public kv_server::v2::KeyValueService::CallbackService { class MockV8Dispatcher : public V8Dispatcher { public: + virtual ~MockV8Dispatcher() = default; MOCK_METHOD(absl::Status, Init, ()); MOCK_METHOD(absl::Status, Stop, ()); MOCK_METHOD(absl::Status, LoadSync, diff --git a/services/common/util/file_util.cc b/services/common/util/file_util.cc index a8380d80..09d8362d 100644 --- a/services/common/util/file_util.cc +++ b/services/common/util/file_util.cc @@ -27,7 +27,7 @@ absl::StatusOr GetFileContent(absl::string_view path, bool log_on_error) { std::ifstream ifs(path.data()); if (!ifs.good()) { - std::string err_str = absl::StrCat("Failed to load file from path: ", path); + std::string err_str = absl::StrCat(kPathFailed, path); ABSL_LOG_IF(ERROR, log_on_error) << err_str; return absl::InvalidArgumentError(std::move(err_str)); } diff --git a/services/common/util/file_util.h b/services/common/util/file_util.h index 2a99892d..7ffb71a9 100644 --- a/services/common/util/file_util.h +++ b/services/common/util/file_util.h @@ -24,6 +24,8 @@ namespace privacy_sandbox::bidding_auction_servers { +constexpr char kPathFailed[] = "Failed to load file from path: "; + // Gets the contents of the provided path. absl::StatusOr GetFileContent(absl::string_view path, bool log_on_error = false); diff --git a/services/common/util/hpke_utils_test.cc b/services/common/util/hpke_utils_test.cc index 3c4044fc..1dcc3097 100644 --- a/services/common/util/hpke_utils_test.cc +++ b/services/common/util/hpke_utils_test.cc @@ -53,7 +53,7 @@ MockKeyFetcherReturningPublicKey( EXPECT_CALL(*key_fetcher_manager, GetPublicKey) .Times(1) .WillOnce( - [&expected_cp](const server_common::CloudPlatform& cloud_platform) { + [expected_cp](const server_common::CloudPlatform& cloud_platform) { // Verify cloud platform is sent as is. EXPECT_EQ(expected_cp, cloud_platform); return MockPublicKey(); diff --git a/services/seller_frontend_service/select_ad_reactor.cc b/services/seller_frontend_service/select_ad_reactor.cc index 2d3eaf2f..72c89f1f 100644 --- a/services/seller_frontend_service/select_ad_reactor.cc +++ b/services/seller_frontend_service/select_ad_reactor.cc @@ -158,10 +158,6 @@ grpc::Status SelectAdReactor::DecryptRequest() { } else { encapsulated_req = request_->protected_audience_ciphertext(); } - LogIfError(metric_context_->LogHistogram( - (int)encapsulated_req.size())); - LogIfError(metric_context_->LogHistogram( - (int)request_->auction_config().ByteSizeLong())); PS_VLOG(5) << "Protected " << (is_protected_auction_request_ ? "auction" : "audience") @@ -304,6 +300,25 @@ void SelectAdReactor::Execute() { }, protected_auction_input_)); + if (log_context_.is_consented()) { + std::string generation_id = std::visit( + [](const auto& protected_input) -> std::string { + return {protected_input.generation_id()}; + }, + protected_auction_input_); + metric_context_->SetConsented(generation_id); + } + + if (is_protected_auction_request_) { + LogIfError(metric_context_->LogHistogram( + (int)request_->protected_auction_ciphertext().size())); + } else { + LogIfError(metric_context_->LogHistogram( + (int)request_->protected_audience_ciphertext().size())); + } + LogIfError(metric_context_->LogHistogram( + (int)request_->auction_config().ByteSizeLong())); + PS_VLOG(kEncrypted, log_context_) << "Encrypted SelectAdRequest:\n" << request_->ShortDebugString(); PS_VLOG(2, log_context_) << "Headers:\n" diff --git a/tools/debug/start_bidding b/tools/debug/start_bidding index c28d215a..d19e3c93 100755 --- a/tools/debug/start_bidding +++ b/tools/debug/start_bidding @@ -33,6 +33,7 @@ export SERVER_START_CMD=$(cat << END --telemetry_config="mode: EXPERIMENT" \ --roma_timeout_ms="120000" \ --buyer_code_fetch_config='{ + "fetchMode": 0, "biddingJsPath": "", "biddingJsUrl": "https://td.doubleclick.net/td/bjs", "protectedAppSignalsBiddingJsUrl": "https://td.doubleclick.net/td/bjs", diff --git a/version.txt b/version.txt index a4f52a5d..0fa4ae48 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -3.2.0 \ No newline at end of file +3.3.0 \ No newline at end of file