From fa6b006192c31ee82af5f663558223d8983e2ce6 Mon Sep 17 00:00:00 2001 From: Roopal Nahar Date: Tue, 14 May 2024 21:55:06 +0000 Subject: [PATCH] Release 3.7.0 (2024-05-13) ### Features * Add the CPU affinity lib for inference ### Bug Fixes * Cap debug reporting URL * Fix the recency value passed to generateBid and scoreAds * make aws-image-tag optional * remove unitOverride in gcp seller dashboard terraform * replace ABSL_LOG with PS_LOG in reactors * update empty device signals to be an object, not string Bug: 340281159 GitOrigin-RevId: 64cb08b38dcb7b2f18000e626140c2376296ab40 Change-Id: I9ac2d2fa772cbb761afaf1f01cc5b8b86fe1554e --- CHANGELOG.md | 17 +++ WORKSPACE | 8 +- api/bidding_auction_servers.proto | 27 ++--- .../dashboards/seller_dashboard/main.tf | 3 +- .../packaging/aws/auction_service/BUILD | 47 ++------ .../packaging/aws/bidding_service/BUILD | 42 ++----- .../aws/buyer_frontend_service/BUILD | 42 ++----- .../aws/seller_frontend_service/BUILD | 42 ++----- .../packaging/build_and_test_all_in_docker | 4 +- services/auction_service/BUILD | 1 + services/auction_service/score_ads_reactor.cc | 24 ++-- .../auction_service/score_ads_reactor_test.cc | 58 +++++++++ services/auction_service/utils/proto_utils.cc | 2 +- services/bidding_service/bidding_main.cc | 21 ++-- .../bidding_service/generate_bids_reactor.cc | 9 +- .../generate_bids_reactor_test.cc | 95 ++++++++++++++- .../inference/inference_flags.cc | 2 - .../inference/inference_flags.h | 7 +- .../get_bids_unary_reactor_test.cc | 10 +- .../util/proto_factory.cc | 2 +- .../util/proto_factory.h | 2 +- .../util/proto_factory_test.cc | 40 +++--- services/common/test/random.cc | 3 +- services/common/test/utils/cbor_test_utils.cc | 13 +- .../common/util/request_response_constants.h | 11 +- .../inference_sidecar/common/benchmark/BUILD | 16 ++- .../common/benchmark/module_benchmark.cc | 6 +- .../common/benchmark/request_utils.cc | 28 +++++ .../common/benchmark/request_utils.h | 11 ++ .../common/benchmark/request_utils_test.cc | 64 ++++++++++ .../common/benchmark/sandbox_benchmark.cc | 24 +++- .../common/inference_sidecar_main.cc | 9 +- .../common/tools/debug/start_inference | 8 +- services/inference_sidecar/common/utils/BUILD | 24 ++++ .../inference_sidecar/common/utils/cpu.cc | 60 +++++++++ services/inference_sidecar/common/utils/cpu.h | 29 +++++ .../common/utils/cpu_test.cc | 64 ++++++++++ .../modules/pytorch_v2_1_1/benchmark/BUILD | 25 ++++ .../schemas/interest_group.json | 4 + .../select_ad_reactor.cc | 13 +- .../select_ad_reactor_test.cc | 4 +- .../select_ad_reactor_web.cc | 5 +- .../select_ad_reactor_web_test.cc | 10 +- .../util/select_ad_reactor_test_utils.h | 5 +- .../seller_frontend_service/util/web_utils.cc | 10 ++ .../seller_frontend_service/util/web_utils.h | 2 + .../util/web_utils_test.cc | 54 +++++++-- tools/secure_invoke/README.md | 114 +++++++++++++++++- version.txt | 2 +- 49 files changed, 865 insertions(+), 258 deletions(-) create mode 100644 services/inference_sidecar/common/benchmark/request_utils_test.cc create mode 100644 services/inference_sidecar/common/utils/cpu.cc create mode 100644 services/inference_sidecar/common/utils/cpu.h create mode 100644 services/inference_sidecar/common/utils/cpu_test.cc diff --git a/CHANGELOG.md b/CHANGELOG.md index bbf7eae3..a11407b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,23 @@ All notable changes to this project will be documented in this file. See [commit-and-tag-version](https://github.com/absolute-version/commit-and-tag-version) for commit guidelines. +## 3.7.0 (2024-05-13) + + +### Features + +* Add the CPU affinity lib for inference + + +### Bug Fixes + +* Cap debug reporting URL +* Fix the recency value passed to generateBid and scoreAds +* make aws-image-tag optional +* remove unitOverride in gcp seller dashboard terraform +* replace ABSL_LOG with PS_LOG in reactors +* update empty device signals to be an object, not string + ## 3.6.0 (2024-05-01) diff --git a/WORKSPACE b/WORKSPACE index ff8a21e8..ea223048 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -23,11 +23,11 @@ http_archive( http_archive( name = "google_privacysandbox_servers_common", - # 2024-04-19 - sha256 = "ac5dd014d81cdc76e86cdaf8b584d5ee8494453315dc986860eb1e60f424a668", - strip_prefix = "data-plane-shared-libraries-4b1e6744cf3e46efee0cf29d886b22fe93274c79", + # 2024-05-06 + sha256 = "1b0d52c6218fa4db7f4760a7e197c2e656a1408549fee40cdea2cb7496a3f836", + strip_prefix = "data-plane-shared-libraries-10be592dd582044a79172d4c0530ce8b7ac39ae4", urls = [ - "https://github.com/privacysandbox/data-plane-shared-libraries/archive/4b1e6744cf3e46efee0cf29d886b22fe93274c79.zip", + "https://github.com/privacysandbox/data-plane-shared-libraries/archive/10be592dd582044a79172d4c0530ce8b7ac39ae4.zip", ], ) diff --git a/api/bidding_auction_servers.proto b/api/bidding_auction_servers.proto index 834d1464..049ca11e 100644 --- a/api/bidding_auction_servers.proto +++ b/api/bidding_auction_servers.proto @@ -179,6 +179,14 @@ message BrowserSignals { // that occurred in the last 30 days. The time is specified in seconds // before the containing auctionBlob was requested. string prev_wins = 4; + + // The most recent join time for this group expressed in milli seconds + // before the containing auctionBlob was requested. This field will be used + // by newer client versions. For older devices, the precison will be in seconds. + // If recency_ms is present, this value will be used to offer higher precision. + // If not, recency will be used. + // Only one of the recency or recency_ms is expected to present in the request. + optional int64 recency_ms = 5; } // Information passed by Android. @@ -606,17 +614,6 @@ message SelectAdResponse { string key_id = 3; } -// Context useful for logging and debugging requests. -message LogContext { - // UUID for the request (as originating from client). - string generation_id = 1; - - // Adtech debug id that can be used for correlating the request with the - // adtech. This will contain `buyer_debug_id` when used in context of buyer - // services and `seller_debug_id` when used in context of seller services. - string adtech_debug_id = 2; -} - // Buyer's FrontEnd service. service BuyerFrontEnd { // Returns bids for each Interest Group / Custom Audience. @@ -679,7 +676,7 @@ message GetBidsRequest { bool enable_debug_reporting = 7; // Helpful context for logging and tracing the request. - LogContext log_context = 8; + server_common.LogContext log_context = 8; // Consented debugging field. server_common.ConsentedDebugConfiguration consented_debug_config = 9; @@ -890,7 +887,7 @@ message GenerateBidsRequest { string publisher_name = 7; // Helpful context for logging and tracing the request. - LogContext log_context = 8; + server_common.LogContext log_context = 8; // Consented debugging field. server_common.ConsentedDebugConfiguration consented_debug_config = 9; @@ -963,7 +960,7 @@ message GenerateProtectedAppSignalsBidsRequest { string publisher_name = 5; // Helpful context for logging and tracing the request. - LogContext log_context = 6; + server_common.LogContext log_context = 6; // Consented debugging field. server_common.ConsentedDebugConfiguration consented_debug_config = 7; @@ -1167,7 +1164,7 @@ message ScoreAdsRequest { bool enable_debug_reporting = 6; // Helpful context for logging and tracing the request. - LogContext log_context = 7; + server_common.LogContext log_context = 7; // Map of buyer_origin to buyer_signals required as input for event level // reporting url generation for the winning buyer. diff --git a/production/deploy/gcp/terraform/services/dashboards/seller_dashboard/main.tf b/production/deploy/gcp/terraform/services/dashboards/seller_dashboard/main.tf index cff6d42c..d2b60f3c 100644 --- a/production/deploy/gcp/terraform/services/dashboards/seller_dashboard/main.tf +++ b/production/deploy/gcp/terraform/services/dashboards/seller_dashboard/main.tf @@ -1199,8 +1199,7 @@ resource "google_monitoring_dashboard" "environment_dashboard" { "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { - "timeSeriesQueryLanguage": "fetch generic_task\n| { metric 'workload.googleapis.com/sfe.initiated_request.to_bfe.duration_ms'\n ; metric 'workload.googleapis.com/sfe.initiated_request.to_bfe.count' }\n| filter (metric.deployment_environment == '${var.environment}')\n| group_by\n [metric.buyer, metric.service_name, metric.deployment_environment,\n metric.operator, metric.Noise, resource.task_id, metric.service_version]\n| align rate(1m)\n| outer_join 0\n| div", - "unitOverride": "" + "timeSeriesQueryLanguage": "fetch generic_task\n| { metric 'workload.googleapis.com/sfe.initiated_request.to_bfe.duration_ms'\n ; metric 'workload.googleapis.com/sfe.initiated_request.to_bfe.count' }\n| filter (metric.deployment_environment == '${var.environment}')\n| group_by\n [metric.buyer, metric.service_name, metric.deployment_environment,\n metric.operator, metric.Noise, resource.task_id, metric.service_version]\n| align rate(1m)\n| outer_join 0\n| div" } } ], diff --git a/production/packaging/aws/auction_service/BUILD b/production/packaging/aws/auction_service/BUILD index 4264e89e..a6a58203 100644 --- a/production/packaging/aws/auction_service/BUILD +++ b/production/packaging/aws/auction_service/BUILD @@ -22,6 +22,7 @@ load( "@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_files", + "pkg_mklink", ) load("@rules_pkg//pkg:tar.bzl", "pkg_tar") load("@rules_pkg//pkg:zip.bzl", "pkg_zip") @@ -36,17 +37,16 @@ pkg_files( prefix = "/", ) -pkg_files( - name = "kmstool_enclave_executable", - srcs = [ - "@google_privacysandbox_servers_common//src/cpio/client_providers/kms_client_provider/aws:kms_cli", - ], - attributes = pkg_attributes(mode = "0555"), - prefix = "/", +# Create a symlink between where kmstool_enclave_cli expects shell to be +# (/bin/sh) and where it actually is on our image (/busybox/sh). +pkg_mklink( + name = "busybox_sh_symlink", + link_name = "/bin/sh", + target = "/busybox/sh", ) server_binaries = [ - ":kmstool_enclave_executable", + ":busybox_sh_symlink", ":server_executables", ] @@ -68,35 +68,6 @@ container_layer( ], ) -# Ensure libnsm ends up in the load path. -pkg_tar( - name = "libnsm-tar", - srcs = [ - "@google_privacysandbox_servers_common//src/cpio/client_providers/kms_client_provider/aws:libnsm_so", - ], - mode = "0644", - # TODO(b/294113069): Remove arch-specific path. - package_dir = "/lib/x86_64-linux-gnu/", - visibility = ["//visibility:public"], -) - -# Create a symlink between where kmstool_enclave_cli expects shell to be -# (/bin/sh) and where it actually is on our image (/busybox/sh). -container_layer( - name = "kmstool_enclave_cli_layer", - env = { - "GLOG_logtostderr": "1", - "GLOG_stderrthreshold": "0", - "GRPC_DNS_RESOLVER": "native", - }, - symlinks = { - "/bin/sh": "/busybox/sh", - }, - tars = [ - ":libnsm-tar", - ], -) - # This image target is meant for testing running the server in an enclave using # aws proxy to abstract vsock communication. container_image( @@ -117,7 +88,6 @@ container_image( env = LOG_ENV_VARS, layers = [ ":server_binary_layer", - ":kmstool_enclave_cli_layer", ] + select({ "//:e2e_build": [ ], @@ -125,6 +95,7 @@ container_image( }), tars = [ "@google_privacysandbox_servers_common//src/aws/proxy:libnsm_and_proxify_tar", + "@google_privacysandbox_servers_common//src/cpio/client_providers/kms_client_provider/aws:kms_binaries", ], ) diff --git a/production/packaging/aws/bidding_service/BUILD b/production/packaging/aws/bidding_service/BUILD index f0c9ce37..1026f3c2 100644 --- a/production/packaging/aws/bidding_service/BUILD +++ b/production/packaging/aws/bidding_service/BUILD @@ -22,6 +22,7 @@ load( "@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_files", + "pkg_mklink", ) load("@rules_pkg//pkg:tar.bzl", "pkg_tar") load("@rules_pkg//pkg:zip.bzl", "pkg_zip") @@ -47,17 +48,16 @@ pkg_files( prefix = "/", ) -pkg_files( - name = "kmstool_enclave_executable", - srcs = [ - "@google_privacysandbox_servers_common//src/cpio/client_providers/kms_client_provider/aws:kms_cli", - ], - attributes = pkg_attributes(mode = "0555"), - prefix = "/", +# Create a symlink between where kmstool_enclave_cli expects shell to be +# (/bin/sh) and where it actually is on our image (/busybox/sh). +pkg_mklink( + name = "busybox_sh_symlink", + link_name = "/bin/sh", + target = "/busybox/sh", ) server_binaries = [ - ":kmstool_enclave_executable", + ":busybox_sh_symlink", ":server_executables", ] @@ -71,30 +71,6 @@ pkg_tar( srcs = server_binaries, ) -# Ensure libnsm ends up in the load path. -pkg_tar( - name = "libnsm-tar", - srcs = [ - "@google_privacysandbox_servers_common//src/cpio/client_providers/kms_client_provider/aws:libnsm_so", - ], - mode = "0644", - # TODO(b/294113069): Remove arch-specific path. - package_dir = "/lib/x86_64-linux-gnu/", - visibility = ["//visibility:public"], -) - -# Create a symlink between where kmstool_enclave_cli expects shell to be -# (/bin/sh) and where it actually is on our image (/busybox/sh). -container_layer( - name = "kmstool_enclave_cli_layer", - symlinks = { - "/bin/sh": "/busybox/sh", - }, - tars = [ - ":libnsm-tar", - ], -) - container_layer( name = "server_binary_layer", directory = "/", @@ -128,7 +104,6 @@ container_image( env = LOG_ENV_VARS, layers = [ ":server_binary_layer", - ":kmstool_enclave_cli_layer", ] + select({ "//:e2e_build": [ ], @@ -136,6 +111,7 @@ container_image( }), tars = [ "@google_privacysandbox_servers_common//src/aws/proxy:libnsm_and_proxify_tar", + "@google_privacysandbox_servers_common//src/cpio/client_providers/kms_client_provider/aws:kms_binaries", ], ) diff --git a/production/packaging/aws/buyer_frontend_service/BUILD b/production/packaging/aws/buyer_frontend_service/BUILD index cd2ba167..4aef7e20 100644 --- a/production/packaging/aws/buyer_frontend_service/BUILD +++ b/production/packaging/aws/buyer_frontend_service/BUILD @@ -22,6 +22,7 @@ load( "@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_files", + "pkg_mklink", ) load("@rules_pkg//pkg:tar.bzl", "pkg_tar") load("@rules_pkg//pkg:zip.bzl", "pkg_zip") @@ -36,17 +37,16 @@ pkg_files( prefix = "/", ) -pkg_files( - name = "kmstool_enclave_executable", - srcs = [ - "@google_privacysandbox_servers_common//src/cpio/client_providers/kms_client_provider/aws:kms_cli", - ], - attributes = pkg_attributes(mode = "0555"), - prefix = "/", +# Create a symlink between where kmstool_enclave_cli expects shell to be +# (/bin/sh) and where it actually is on our image (/busybox/sh). +pkg_mklink( + name = "busybox_sh_symlink", + link_name = "/bin/sh", + target = "/busybox/sh", ) server_binaries = [ - ":kmstool_enclave_executable", + ":busybox_sh_symlink", ":server_executables", ] @@ -60,30 +60,6 @@ pkg_tar( srcs = server_binaries, ) -# Ensure libnsm ends up in the load path. -pkg_tar( - name = "libnsm-tar", - srcs = [ - "@google_privacysandbox_servers_common//src/cpio/client_providers/kms_client_provider/aws:libnsm_so", - ], - mode = "0644", - # TODO(b/294113069): Remove arch-specific path. - package_dir = "/lib/x86_64-linux-gnu/", - visibility = ["//visibility:public"], -) - -# Create a symlink between where kmstool_enclave_cli expects shell to be -# (/bin/sh) and where it actually is on our image (/busybox/sh). -container_layer( - name = "kmstool_enclave_cli_layer", - symlinks = { - "/bin/sh": "/busybox/sh", - }, - tars = [ - ":libnsm-tar", - ], -) - container_layer( name = "server_binary_layer", directory = "/", @@ -117,7 +93,6 @@ container_image( env = LOG_ENV_VARS, layers = [ ":server_binary_layer", - ":kmstool_enclave_cli_layer", ] + select({ "//:e2e_build": [ ], @@ -125,6 +100,7 @@ container_image( }), tars = [ "@google_privacysandbox_servers_common//src/aws/proxy:libnsm_and_proxify_tar", + "@google_privacysandbox_servers_common//src/cpio/client_providers/kms_client_provider/aws:kms_binaries", ], ) diff --git a/production/packaging/aws/seller_frontend_service/BUILD b/production/packaging/aws/seller_frontend_service/BUILD index 92d24873..4cafa1f8 100644 --- a/production/packaging/aws/seller_frontend_service/BUILD +++ b/production/packaging/aws/seller_frontend_service/BUILD @@ -22,6 +22,7 @@ load( "@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_files", + "pkg_mklink", ) load("@rules_pkg//pkg:tar.bzl", "pkg_tar") load("@rules_pkg//pkg:zip.bzl", "pkg_zip") @@ -36,17 +37,16 @@ pkg_files( prefix = "/", ) -pkg_files( - name = "kmstool_enclave_executable", - srcs = [ - "@google_privacysandbox_servers_common//src/cpio/client_providers/kms_client_provider/aws:kms_cli", - ], - attributes = pkg_attributes(mode = "0555"), - prefix = "/", +# Create a symlink between where kmstool_enclave_cli expects shell to be +# (/bin/sh) and where it actually is on our image (/busybox/sh). +pkg_mklink( + name = "busybox_sh_symlink", + link_name = "/bin/sh", + target = "/busybox/sh", ) server_binaries = [ - ":kmstool_enclave_executable", + ":busybox_sh_symlink", ":server_executables", ] @@ -60,30 +60,6 @@ pkg_tar( srcs = server_binaries, ) -# Ensure libnsm ends up in the load path. -pkg_tar( - name = "libnsm-tar", - srcs = [ - "@google_privacysandbox_servers_common//src/cpio/client_providers/kms_client_provider/aws:libnsm_so", - ], - mode = "0644", - # TODO(b/294113069): Remove arch-specific path. - package_dir = "/lib/x86_64-linux-gnu/", - visibility = ["//visibility:public"], -) - -# Create a symlink between where kmstool_enclave_cli expects shell to be -# (/bin/sh) and where it actually is on our image (/busybox/sh). -container_layer( - name = "kmstool_enclave_cli_layer", - symlinks = { - "/bin/sh": "/busybox/sh", - }, - tars = [ - ":libnsm-tar", - ], -) - container_layer( name = "server_binary_layer", directory = "/", @@ -117,7 +93,6 @@ container_image( env = LOG_ENV_VARS, layers = [ ":server_binary_layer", - ":kmstool_enclave_cli_layer", ] + select({ "//:e2e_build": [ ], @@ -125,6 +100,7 @@ container_image( }), tars = [ "@google_privacysandbox_servers_common//src/aws/proxy:libnsm_and_proxify_tar", + "@google_privacysandbox_servers_common//src/cpio/client_providers/kms_client_provider/aws:kms_binaries", ], ) diff --git a/production/packaging/build_and_test_all_in_docker b/production/packaging/build_and_test_all_in_docker index e0b67b89..0e617070 100755 --- a/production/packaging/build_and_test_all_in_docker +++ b/production/packaging/build_and_test_all_in_docker @@ -318,7 +318,9 @@ if [[ ${NO_PLATFORM_BUILD} -eq 0 ]]; then for region in "${AMI_REGIONS[@]}"; do EXTRA_ARGS+=(--with-ami "${region}") done - EXTRA_ARGS+=(--aws-image-tag ${AWS_IMAGE_TAG}) + if [[ -n "${AWS_IMAGE_TAG}" ]]; then + EXTRA_ARGS+=(--aws-image-tag ${AWS_IMAGE_TAG}) + fi if [[ ${VERBOSE} -eq 1 ]]; then EXTRA_ARGS+=("--verbose") fi diff --git a/services/auction_service/BUILD b/services/auction_service/BUILD index bae60e26..42f7db09 100644 --- a/services/auction_service/BUILD +++ b/services/auction_service/BUILD @@ -134,6 +134,7 @@ cc_test( "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", "@google_privacysandbox_servers_common//src/encryption/key_fetcher/mock:mock_key_fetcher_manager", + "@google_privacysandbox_servers_common//src/logger:request_context_logger", ], ) diff --git a/services/auction_service/score_ads_reactor.cc b/services/auction_service/score_ads_reactor.cc index 9dd90a3f..8c155c73 100644 --- a/services/auction_service/score_ads_reactor.cc +++ b/services/auction_service/score_ads_reactor.cc @@ -73,11 +73,11 @@ inline void LogWarningForBadResponse( PS_LOG(ERROR, log_context) << "Failed to parse response from Roma ", status.ToString(absl::StatusToStringMode::kWithEverything); if (ad_with_bid_metadata) { - ABSL_LOG(WARNING) + PS_LOG(WARNING, log_context) << "Invalid json output from code execution for interest group " << ad_with_bid_metadata->interest_group_name() << ": " << response.resp; } else { - ABSL_LOG(WARNING) + PS_LOG(WARNING, log_context) << "Invalid json output from code execution for protected app signals " "ad: " << response.resp; @@ -677,12 +677,14 @@ void ScoreAdsReactor::FindScoredAdType( ScoringData ScoreAdsReactor::FindWinningAd( const std::vector>& responses) { ScoringData scoring_data; + int64_t current_all_debug_urls_chars = 0; for (int index = 0; index < responses.size(); ++index) { const auto& response = responses[index]; if (!response.ok()) { - ABSL_LOG(WARNING) << "Invalid execution (possibly invalid input): " - << responses[index].status().ToString( - absl::StatusToStringMode::kWithEverything); + PS_LOG(WARNING, log_context_) + << "Invalid execution (possibly invalid input): " + << responses[index].status().ToString( + absl::StatusToStringMode::kWithEverything); continue; } @@ -698,10 +700,11 @@ ScoringData ScoreAdsReactor::FindWinningAd( if (!ad && !protected_app_signals_ad_with_bid) { // This should never happen but we log here in case there is a bug in // our implementation. - ABSL_LOG(ERROR) << "Scored ad is neither a protected audience ad, nor a " - "protected app " - "signals ad: " - << response->resp; + PS_LOG(ERROR, log_context_) + << "Scored ad is neither a protected audience ad, nor a " + "protected app " + "signals ad: " + << response->resp; continue; } @@ -711,7 +714,6 @@ ScoringData ScoreAdsReactor::FindWinningAd( continue; } - long current_all_debug_urls_chars = 0; auto ad_score = ParseScoreAdResponse( *response_json, max_allowed_size_debug_url_chars_, max_allowed_size_all_debug_urls_chars_, @@ -850,7 +852,7 @@ void ScoreAdsReactor::ScoreAdsCallback( LogIfError(metric_context_ ->AccumulateMetric( 1, metric::kAuctionScoreAdsNoAdSelected)); - ABSL_LOG(WARNING) << "No ad was selected as most desirable"; + PS_LOG(WARNING, log_context_) << "No ad was selected as most desirable"; if (enable_debug_reporting) { PerformDebugReporting(winning_ad); } diff --git a/services/auction_service/score_ads_reactor_test.cc b/services/auction_service/score_ads_reactor_test.cc index c9360cd6..f50d30f5 100644 --- a/services/auction_service/score_ads_reactor_test.cc +++ b/services/auction_service/score_ads_reactor_test.cc @@ -45,6 +45,7 @@ #include "services/common/util/proto_util.h" #include "src/encryption/key_fetcher/interface/key_fetcher_manager_interface.h" #include "src/encryption/key_fetcher/mock/mock_key_fetcher_manager.h" +#include "src/logger/request_context_logger.h" namespace privacy_sandbox::bidding_auction_servers { namespace { @@ -3067,5 +3068,62 @@ TEST_F(ScoreAdsReactorProtectedAppSignalsTest, kTestInteractionUrl); } +TEST_F(ScoreAdsReactorTest, RespectConfiguredDebugUrlLimits) { + server_common::log::PS_VLOG_IS_ON(0, 10); + MockCodeDispatchClient dispatcher; + RawRequest raw_request; + AdWithBidMetadata foo, bar; + GetTestAdWithBidFoo(foo); + GetTestAdWithBidBar(bar); + BuildRawRequest({foo, bar}, kTestSellerSignals, kTestAuctionSignals, + kTestScoringSignals, kTestPublisherHostname, raw_request, + /*enable_debug_reporting=*/true); + std::string long_win_url(1024, 'A'); + std::string long_loss_url(1025, 'B'); + + EXPECT_CALL(dispatcher, BatchExecute) + .WillRepeatedly([long_win_url, long_loss_url]( + std::vector& batch, + BatchDispatchDoneCallback done_callback) { + std::vector score_logic; + score_logic.reserve(batch.size()); + for (int i = 0; i < batch.size(); ++i) { + score_logic.emplace_back(absl::Substitute( + R"JSON( + { + "response" : { + "desirability" : $0, + "bid" : 1, + "allowComponentAuction" : false, + "debugReportUrls": { + "auctionDebugLossUrl": "$1", + "auctionDebugWinUrl": "$2" + } + }, + "logs":[] + } + )JSON", + batch.size() - i, long_loss_url, long_win_url)); + } + return FakeExecute(batch, std::move(done_callback), + std::move(score_logic)); + }); + AuctionServiceRuntimeConfig runtime_config = { + .enable_seller_debug_url_generation = true, + .max_allowed_size_all_debug_urls_kb = 1}; + auto response = ExecuteScoreAds(raw_request, dispatcher, runtime_config); + + ScoreAdsResponse::ScoreAdsRawResponse raw_response; + ASSERT_TRUE(raw_response.ParseFromString(response.response_ciphertext())); + const auto& scored_ad = raw_response.ad_score(); + PS_VLOG(5) << "Response:\n" << scored_ad.DebugString(); + // Desirability must be present but was determined by the scoring code. + EXPECT_GT(scored_ad.desirability(), std::numeric_limits::min()); + EXPECT_TRUE(scored_ad.has_debug_report_urls()); + EXPECT_FALSE(scored_ad.debug_report_urls().auction_debug_win_url().empty()); + EXPECT_EQ(scored_ad.debug_report_urls().auction_debug_win_url().size(), 1024); + EXPECT_TRUE(scored_ad.debug_report_urls().auction_debug_loss_url().empty()); +} + } // namespace } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/utils/proto_utils.cc b/services/auction_service/utils/proto_utils.cc index 8c7b8880..f19c4c6d 100644 --- a/services/auction_service/utils/proto_utils.cc +++ b/services/auction_service/utils/proto_utils.cc @@ -218,7 +218,7 @@ inline absl::string_view GetDebugUrlIfInLimit( PS_VLOG(8) << "Skipping " << url_type << " of length: " << url_length << ", since it will cause the new running total to become: " << current_all_debug_urls_chars + url_length - << "(which surpasses the limit: " + << " (which surpasses the limit: " << max_allowed_size_all_debug_urls_chars << ")"; } return ""; diff --git a/services/bidding_service/bidding_main.cc b/services/bidding_service/bidding_main.cc index 6c0c65b6..f1ffd4d3 100644 --- a/services/bidding_service/bidding_main.cc +++ b/services/bidding_service/bidding_main.cc @@ -173,8 +173,6 @@ absl::StatusOr GetConfigClient( config_client.SetFlag(FLAGS_kv_server_egress_tls, KV_SERVER_EGRESS_TLS); config_client.SetFlag(FLAGS_inference_sidecar_binary_path, INFERENCE_SIDECAR_BINARY_PATH); - config_client.SetFlag(FLAGS_inference_model_local_paths, - INFERENCE_MODEL_LOCAL_PATHS); config_client.SetFlag(FLAGS_inference_model_bucket_name, INFERENCE_MODEL_BUCKET_NAME); config_client.SetFlag(FLAGS_inference_model_bucket_paths, @@ -337,14 +335,17 @@ absl::Status RunServer() { "required arguements."; } } - PS_LOG(INFO) << "Register models from local."; - std::string_view local_paths = - GetStringParameterSafe(config_client, INFERENCE_MODEL_LOCAL_PATHS); - if (absl::Status status = inference::RegisterModelsFromLocal( - absl::StrSplit(local_paths, ',')); - !status.ok()) { - PS_LOG(INFO) << "Skip registering models from local: " - << status.message(); + + if (std::optional local_paths = + absl::GetFlag(FLAGS_inference_model_local_paths); + local_paths.has_value()) { + PS_LOG(INFO) << "Register models from local for testing."; + if (absl::Status status = inference::RegisterModelsFromLocal( + absl::StrSplit(local_paths.value(), ',')); + !status.ok()) { + PS_LOG(INFO) << "Skip registering models from local: " + << status.message(); + } } } diff --git a/services/bidding_service/generate_bids_reactor.cc b/services/bidding_service/generate_bids_reactor.cc index 8b934d84..371bcd0b 100644 --- a/services/bidding_service/generate_bids_reactor.cc +++ b/services/bidding_service/generate_bids_reactor.cc @@ -128,11 +128,18 @@ std::string MakeBrowserSignalsForScript(absl::string_view publisher_name, absl::StrAppend(&device_signals_str, kJsonStringEnd, kTopLevelSeller, kJsonStringValueStart, top_level_seller); } + int64_t recency_ms; + if (browser_signals.has_recency_ms()) { + recency_ms = browser_signals.recency_ms(); + } else { + recency_ms = browser_signals.recency() * 1000; + } absl::StrAppend( &device_signals_str, kJsonStringEnd, kJoinCount, kJsonValueStart, browser_signals.join_count(), kJsonValueEnd, kBidCount, kJsonValueStart, browser_signals.bid_count(), kJsonValueEnd, kRecency, kJsonValueStart, - browser_signals.recency(), kJsonValueEnd, kPrevWins, kJsonValueStart, + /*recency is expected to be in milli seconds.*/ + recency_ms, kJsonValueEnd, kPrevWins, kJsonValueStart, browser_signals.prev_wins().empty() ? kJsonEmptyString : browser_signals.prev_wins(), "}"); diff --git a/services/bidding_service/generate_bids_reactor_test.cc b/services/bidding_service/generate_bids_reactor_test.cc index c62ee78f..0c21ca66 100644 --- a/services/bidding_service/generate_bids_reactor_test.cc +++ b/services/bidding_service/generate_bids_reactor_test.cc @@ -209,8 +209,10 @@ IGForBidding GetIGForBiddingFoo() { return interest_group; } -constexpr char kComponentBrowserSignals[] = - R"json({"topWindowHostname":"www.example-publisher.com","seller":"https://www.example-ssp.com","topLevelSeller":"https://www.example-top-ssp.com","joinCount":5,"bidCount":25,"recency":1684134092,"prevWins":[[1,"1689"],[1,"1776"]]})json"; +absl::string_view kComponentBrowserSignals = + R"json({"topWindowHostname":"www.example-publisher.com","seller":"https://www.example-ssp.com","topLevelSeller":"https://www.example-top-ssp.com","joinCount":5,"bidCount":25,"recency":1684134092000,"prevWins":[[1,"1689"],[1,"1776"]]})json"; +absl::string_view kComponentBrowserSignalsWithRecencyMs = + R"json({"topWindowHostname":"www.example-publisher.com","seller":"https://www.example-ssp.com","topLevelSeller":"https://www.example-top-ssp.com","joinCount":5,"bidCount":25,"recency":123456000,"prevWins":[[1,"1689"],[1,"1776"]]})json"; IGForBidding GetIGForBiddingBar(bool make_browser_signals = true) { IGForBidding interest_group; @@ -232,7 +234,9 @@ IGForBidding GetIGForBiddingBar(bool make_browser_signals = true) { } constexpr char bar_browser_signals[] = - R"json({"topWindowHostname":"www.example-publisher.com","seller":"https://www.example-ssp.com","joinCount":5,"bidCount":25,"recency":1684134093,"prevWins":[[1,"1868"],[1,"1954"]]})json"; + R"json({"topWindowHostname":"www.example-publisher.com","seller":"https://www.example-ssp.com","joinCount":5,"bidCount":25,"recency":1684134093000,"prevWins":[[1,"1868"],[1,"1954"]]})json"; +constexpr char kExpectedBrowserSignalsWithRecencyMs[] = + R"json({"topWindowHostname":"www.example-publisher.com","seller":"https://www.example-ssp.com","joinCount":5,"bidCount":25,"recency":123456000,"prevWins":[[1,"1868"],[1,"1954"]]})json"; // Allows re-serialization. void CheckForAndReplaceUBSWithEmptyString( @@ -611,6 +615,45 @@ TEST_F(GenerateBidsReactorTest, CreatesGenerateBidInputsInCorrectOrder) { CheckGenerateBids(raw_request, ads); } +TEST_F(GenerateBidsReactorTest, + CreatesGenerateBidInputsInCorrectOrderWithRecencyMs) { + std::string response_json = GetTestResponse(kTestRenderUrl, 1); + + AdWithBid bid; + bid.set_render(kTestRenderUrl); + bid.set_bid(1); + bid.set_interest_group_name("ig_name_Bar"); + Response ads; + GenerateBidsResponse::GenerateBidsRawResponse raw_response; + *raw_response.add_bids() = std::move(bid); + *ads.mutable_response_ciphertext() = raw_response.SerializeAsString(); + std::vector igs; + auto ig = GetIGForBiddingBar(); + ig.mutable_browser_signals()->set_recency_ms(123456000); + igs.push_back(std::move(ig)); + + EXPECT_CALL(dispatcher_, BatchExecute) + .WillOnce([response_json](std::vector& batch, + BatchDispatchDoneCallback batch_callback) { + auto input = batch.at(0).input; + EXPECT_EQ(input.size(), 6); + if (input.size() == 6) { + CheckCorrectnessOfIg(*input[0], GetIGForBiddingBar()); + EXPECT_EQ(*input[1], R"JSON({"auction_signal": "test 1"})JSON"); + EXPECT_EQ(*input[2], R"JSON({"buyer_signal": "test 2"})JSON"); + EXPECT_EQ( + *input[3], + R"JSON({"trusted_bidding_signal_key":"some_trusted_bidding_signal_value"})JSON"); + EXPECT_EQ(*input[4], kExpectedBrowserSignalsWithRecencyMs); + } + return FakeExecute(batch, std::move(batch_callback), response_json); + }); + RawRequest raw_request; + BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, + kTestBiddingSignals, raw_request); + CheckGenerateBids(raw_request, ads); +} + TEST_F(GenerateBidsReactorTest, CreatesGenerateBidInputsCorrectlyForComponentAuction) { std::string json = GetComponentAuctionResponse( @@ -656,6 +699,52 @@ TEST_F(GenerateBidsReactorTest, CheckGenerateBids(raw_request, ads); } +TEST_F(GenerateBidsReactorTest, + CreatesGenerateBidInputsCorrectlyForComponentAuctionWithRecencyMs) { + std::string json = GetComponentAuctionResponse( + kTestRenderUrl, /*bid=*/1, /*allow_component_auction=*/true); + auto ig = GetIGForBiddingFoo(); + AdWithBid bid; + bid.set_render(kTestRenderUrl); + bid.set_bid(1); + bid.set_interest_group_name(ig.name()); + bid.set_allow_component_auction(true); + ig.mutable_browser_signals()->set_recency_ms(123456000); + Response ads; + GenerateBidsResponse::GenerateBidsRawResponse raw_response; + *raw_response.add_bids() = bid; + *ads.mutable_response_ciphertext() = raw_response.SerializeAsString(); + std::vector igs{ig}; + + EXPECT_CALL(dispatcher_, BatchExecute) + .WillOnce([&json, &ig](std::vector& batch, + BatchDispatchDoneCallback batch_callback) { + // Test setup check. + CHECK_EQ(batch.size(), 1) + << absl::InternalError("Test setup error. Batch size must be 1."); + auto input = batch.at(0).input; + CHECK_EQ(batch.size(), 1) + << absl::InternalError("Test setup error. Input size must be 6."); + CheckCorrectnessOfIg(*input[ArgIndex(GenerateBidArgs::kInterestGroup)], + ig); + EXPECT_EQ(*input[ArgIndex(GenerateBidArgs::kAuctionSignals)], + R"JSON({"auction_signal": "test 1"})JSON"); + EXPECT_EQ(*input[ArgIndex(GenerateBidArgs::kBuyerSignals)], + R"JSON({"buyer_signal": "test 2"})JSON"); + EXPECT_EQ( + *input[ArgIndex(GenerateBidArgs::kTrustedBiddingSignals)], + R"JSON({"trusted_bidding_signal_key":"some_trusted_bidding_signal_value"})JSON"); + EXPECT_EQ(*input[ArgIndex(GenerateBidArgs::kDeviceSignals)], + kComponentBrowserSignalsWithRecencyMs); + return FakeExecute(batch, std::move(batch_callback), json); + }); + RawRequest raw_request; + BuildRawRequestForComponentAuction(igs, kTestAuctionSignals, + kTestBuyerSignals, kTestBiddingSignals, + raw_request); + CheckGenerateBids(raw_request, ads); +} + TEST_F(GenerateBidsReactorTest, ParsesAllowComponentAuctionFieldForComponentAuction) { std::string json = GetComponentAuctionResponse( diff --git a/services/bidding_service/inference/inference_flags.cc b/services/bidding_service/inference/inference_flags.cc index 60a3797f..b0f47366 100644 --- a/services/bidding_service/inference/inference_flags.cc +++ b/services/bidding_service/inference/inference_flags.cc @@ -23,8 +23,6 @@ ABSL_FLAG(std::optional, inference_sidecar_binary_path, std::nullopt, "The binary path of the inference sidecar."); // Used to load local models for debugging purposes. In the prod setting, this // flag is not used, and we fetch models from cloud buckets. -// TODO(b/330817239): Bidding server should throw an error if this flag is set -// in the prod mode. ABSL_FLAG(std::optional, inference_model_local_paths, std::nullopt, "Comma separated list of inference model paths to read from the " "local disk. It's mainly used for testing."); diff --git a/services/bidding_service/inference/inference_flags.h b/services/bidding_service/inference/inference_flags.h index 4dac6e59..16151486 100644 --- a/services/bidding_service/inference/inference_flags.h +++ b/services/bidding_service/inference/inference_flags.h @@ -31,8 +31,6 @@ namespace privacy_sandbox::bidding_auction_servers { inline constexpr char INFERENCE_SIDECAR_BINARY_PATH[] = "INFERENCE_SIDECAR_BINARY_PATH"; -inline constexpr char INFERENCE_MODEL_LOCAL_PATHS[] = - "INFERENCE_MODEL_LOCAL_PATHS"; inline constexpr char INFERENCE_MODEL_BUCKET_NAME[] = "INFERENCE_MODEL_BUCKET_NAME"; inline constexpr char INFERENCE_MODEL_BUCKET_PATHS[] = @@ -40,9 +38,8 @@ inline constexpr char INFERENCE_MODEL_BUCKET_PATHS[] = inline constexpr char INFERENCE_SIDECAR_RUNTIME_CONFIG[] = "INFERENCE_SIDECAR_RUNTIME_CONFIG"; inline constexpr absl::string_view kInferenceFlags[] = { - INFERENCE_SIDECAR_BINARY_PATH, INFERENCE_MODEL_LOCAL_PATHS, - INFERENCE_MODEL_BUCKET_NAME, INFERENCE_MODEL_BUCKET_PATHS, - INFERENCE_SIDECAR_RUNTIME_CONFIG}; + INFERENCE_SIDECAR_BINARY_PATH, INFERENCE_MODEL_BUCKET_NAME, + INFERENCE_MODEL_BUCKET_PATHS, INFERENCE_SIDECAR_RUNTIME_CONFIG}; } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/buyer_frontend_service/get_bids_unary_reactor_test.cc b/services/buyer_frontend_service/get_bids_unary_reactor_test.cc index e8f87f60..858212ed 100644 --- a/services/buyer_frontend_service/get_bids_unary_reactor_test.cc +++ b/services/buyer_frontend_service/get_bids_unary_reactor_test.cc @@ -240,11 +240,11 @@ TEST_F(GetBidUnaryReactorTest, EXPECT_FALSE(response_.response_ciphertext().empty()); } -auto EqLogContext(const LogContext& log_context) { - return AllOf( - Property(&LogContext::generation_id, Eq(log_context.generation_id())), - Property(&LogContext::adtech_debug_id, - Eq(log_context.adtech_debug_id()))); +auto EqLogContext(const server_common::LogContext& log_context) { + return AllOf(Property(&server_common::LogContext::generation_id, + Eq(log_context.generation_id())), + Property(&server_common::LogContext::adtech_debug_id, + Eq(log_context.adtech_debug_id()))); } auto EqGenerateBidsRawRequestWithLogContext( diff --git a/services/buyer_frontend_service/util/proto_factory.cc b/services/buyer_frontend_service/util/proto_factory.cc index a992bd0d..08933f7a 100644 --- a/services/buyer_frontend_service/util/proto_factory.cc +++ b/services/buyer_frontend_service/util/proto_factory.cc @@ -78,7 +78,7 @@ std::unique_ptr CreateGenerateBidsRawRequest( const GetBidsRawRequest& get_bids_raw_request, const BuyerInput& buyer_input, std::unique_ptr bidding_signals, - const LogContext& log_context) { + const server_common::LogContext& log_context) { auto generate_bids_raw_request = std::make_unique(); // 1. Set Interest Group for bidding diff --git a/services/buyer_frontend_service/util/proto_factory.h b/services/buyer_frontend_service/util/proto_factory.h index 7254632a..bcb963f7 100644 --- a/services/buyer_frontend_service/util/proto_factory.h +++ b/services/buyer_frontend_service/util/proto_factory.h @@ -38,7 +38,7 @@ CreateGenerateBidsRawRequest( const GetBidsRequest::GetBidsRawRequest& get_bid_raw_request, const BuyerInput& buyer_input, std::unique_ptr bidding_signals, - const LogContext& log_context); + const server_common::LogContext& log_context); // Creates a request to generate bid for protected app signals. std::unique_ptrtrusted_signals = std::make_unique(expected_raw_output.bidding_signals()); - auto raw_output = CreateGenerateBidsRawRequest( - input, input.buyer_input(), std::move(bidding_signals), LogContext{}); + auto raw_output = CreateGenerateBidsRawRequest(input, input.buyer_input(), + std::move(bidding_signals), + server_common::LogContext{}); std::string difference; MessageDifferencer differencer; @@ -195,8 +196,9 @@ TEST(CreateGenerateBidsRequestTest, SetsAllFieldsFromInputParamsForTestIG) { input.set_seller(expected_raw_output.seller()); input.set_publisher_name(expected_raw_output.publisher_name()); - auto raw_output = CreateGenerateBidsRawRequest( - input, input.buyer_input(), std::move(bidding_signals), LogContext{}); + auto raw_output = CreateGenerateBidsRawRequest(input, input.buyer_input(), + std::move(bidding_signals), + server_common::LogContext{}); ASSERT_GT(expected_raw_output.interest_group_for_bidding().size(), 0); ASSERT_GT(raw_output->interest_group_for_bidding().size(), 0); @@ -228,7 +230,7 @@ TEST(CreateGenerateBidsRequestTest, SetsTopLevelSellerForComponentAuction) { auto raw_output = CreateGenerateBidsRawRequest( input, input.buyer_input(), std::make_unique(), - LogContext{}); + server_common::LogContext{}); EXPECT_EQ(input.top_level_seller(), raw_output->top_level_seller()); } @@ -251,7 +253,7 @@ TEST(CreateGenerateBidsRequestTest, SetsEmptyBiddingSignalKeysForBrowserIG) { auto raw_output = CreateGenerateBidsRawRequest( input, input.buyer_input(), std::make_unique(), - LogContext{}); + server_common::LogContext{}); ASSERT_EQ(raw_output->interest_group_for_bidding().size(), 1); // Expect no bidding signal keys in output. @@ -281,7 +283,7 @@ TEST(CreateGenerateBidsRequestTest, SetsEmptyBiddingSignalKeysForAndroidIG) { auto raw_output = CreateGenerateBidsRawRequest( get_bids_raw_request, get_bids_raw_request.buyer_input(), - std::make_unique(), LogContext{}); + std::make_unique(), server_common::LogContext{}); ASSERT_EQ(raw_output->interest_group_for_bidding().size(), 1); // Expect no bidding signal keys in output. @@ -361,8 +363,9 @@ TEST(CreateGenerateBidsRequestTest, SetsAllFieldsFromInputParamsForBrowser) { input.set_seller(expected_raw_output.seller()); input.set_publisher_name(expected_raw_output.publisher_name()); - auto raw_output = CreateGenerateBidsRawRequest( - input, input.buyer_input(), std::move(bidding_signals), LogContext{}); + auto raw_output = CreateGenerateBidsRawRequest(input, input.buyer_input(), + std::move(bidding_signals), + server_common::LogContext{}); EXPECT_TRUE(MessageDifferencer::Equals(expected_raw_output, *raw_output)); @@ -396,9 +399,10 @@ TEST(CreateGenerateBidsRequestTest, auto bidding_signals = std::make_unique(); GetBidsRequest::GetBidsRawRequest input; - LogContext log_context; - auto raw_output = CreateGenerateBidsRawRequest( - input, input.buyer_input(), std::move(bidding_signals), LogContext{}); + server_common::LogContext log_context; + auto raw_output = CreateGenerateBidsRawRequest(input, input.buyer_input(), + std::move(bidding_signals), + server_common::LogContext{}); EXPECT_TRUE(raw_output->bidding_signals().empty()); } @@ -408,8 +412,9 @@ TEST(CreateGenerateBidsRequestTest, SetsEnableEventLevelDebugReporting) { GetBidsRequest::GetBidsRawRequest input; input.set_enable_debug_reporting(true); - auto raw_output = CreateGenerateBidsRawRequest( - input, input.buyer_input(), std::move(bidding_signals), LogContext{}); + auto raw_output = CreateGenerateBidsRawRequest(input, input.buyer_input(), + std::move(bidding_signals), + server_common::LogContext{}); EXPECT_TRUE(raw_output->enable_debug_reporting()); } @@ -418,7 +423,7 @@ TEST(CreateGenerateBidsRequestTest, SetsLogContext) { auto bidding_signals = std::make_unique(); GetBidsRequest::GetBidsRawRequest input; - LogContext log_context; + server_common::LogContext log_context; log_context.set_generation_id(kSampleGenerationId); log_context.set_adtech_debug_id(kSampleAdtechDebugId); auto raw_output = CreateGenerateBidsRawRequest( @@ -438,8 +443,9 @@ TEST(CreateGenerateBidsRequestTest, SetsConsentedDebugConfig) { consented_debug_config->set_is_consented(kIsConsentedDebug); consented_debug_config->set_token(kConsentedDebugToken); - auto raw_output = CreateGenerateBidsRawRequest( - input, input.buyer_input(), std::move(bidding_signals), LogContext{}); + auto raw_output = CreateGenerateBidsRawRequest(input, input.buyer_input(), + std::move(bidding_signals), + server_common::LogContext{}); EXPECT_EQ(raw_output->consented_debug_config().is_consented(), kIsConsentedDebug); diff --git a/services/common/test/random.cc b/services/common/test/random.cc index fbce4c06..5c0fc81f 100644 --- a/services/common/test/random.cc +++ b/services/common/test/random.cc @@ -142,7 +142,8 @@ BrowserSignals MakeRandomBrowserSignalsForIG( BrowserSignals browser_signals; browser_signals.set_join_count(MakeARandomInt(0, 10)); browser_signals.set_bid_count(MakeARandomInt(0, 50)); - browser_signals.set_recency((unsigned long)std::time(NULL)); + browser_signals.set_recency(60); // secs + browser_signals.set_recency_ms(60000); browser_signals.set_prev_wins(MakeRandomPreviousWins(ad_render_ids)); return browser_signals; } diff --git a/services/common/test/utils/cbor_test_utils.cc b/services/common/test/utils/cbor_test_utils.cc index e8b32ecb..dc781598 100644 --- a/services/common/test/utils/cbor_test_utils.cc +++ b/services/common/test/utils/cbor_test_utils.cc @@ -140,9 +140,16 @@ absl::Status CborSerializeBrowserSignals(absl::string_view key, PS_RETURN_IF_ERROR(CborSerializeKeyValue(kBidCount, &cbor_build_uint64, **browser_signals_map, browser_signals.bid_count())); - PS_RETURN_IF_ERROR(CborSerializeKeyValue(kRecency, &cbor_build_uint64, - **browser_signals_map, - browser_signals.recency())); + if (browser_signals.recency()) { + PS_RETURN_IF_ERROR(CborSerializeKeyValue(kRecency, &cbor_build_uint64, + **browser_signals_map, + browser_signals.recency())); + } + if (browser_signals.recency_ms()) { + PS_RETURN_IF_ERROR(CborSerializeKeyValue(kRecencyMs, &cbor_build_uint64, + **browser_signals_map, + browser_signals.recency_ms())); + } std::vector prev_wins_to_encode; if (!browser_signals.prev_wins().empty()) { PS_ASSIGN_OR_RETURN(prev_wins_to_encode, diff --git a/services/common/util/request_response_constants.h b/services/common/util/request_response_constants.h index 42671962..edd168b9 100644 --- a/services/common/util/request_response_constants.h +++ b/services/common/util/request_response_constants.h @@ -66,6 +66,7 @@ inline constexpr char kBrowserSignals[] = "browserSignals"; inline constexpr char kBidCount[] = "bidCount"; inline constexpr char kJoinCount[] = "joinCount"; inline constexpr char kRecency[] = "recency"; +inline constexpr char kRecencyMs[] = "recencyMs"; inline constexpr char kPrevWins[] = "prevWins"; inline constexpr char kConsentedDebugConfig[] = "consentedDebugConfig"; inline constexpr char kIsConsented[] = "isConsented"; @@ -99,14 +100,10 @@ inline constexpr std::array kInterestGroupKeys = {kName, kBiddingSignalsKeys, kUserBiddingSignals, kAds, kAdComponents, kBrowserSignals}; -inline constexpr int kNumBrowserSignalKeys = 4; +inline constexpr int kNumBrowserSignalKeys = 5; inline constexpr std::array - kBrowserSignalKeys = { - kBidCount, - kJoinCount, - kRecency, - kPrevWins, -}; + kBrowserSignalKeys = {kBidCount, kJoinCount, kRecency, kPrevWins, + kRecencyMs}; inline constexpr std::array kConsentedDebugConfigKeys = { kIsConsented, diff --git a/services/inference_sidecar/common/benchmark/BUILD b/services/inference_sidecar/common/benchmark/BUILD index a4b322ce..6cf74dc8 100644 --- a/services/inference_sidecar/common/benchmark/BUILD +++ b/services/inference_sidecar/common/benchmark/BUILD @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library") +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") package(default_visibility = ["//visibility:public"]) @@ -98,6 +98,20 @@ cc_library( ], ) +cc_test( + name = "request_utils_test", + size = "small", + srcs = ["request_utils_test.cc"], + data = [ + ], + deps = [ + ":request_utils", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + # Builds the IPC inference sidecar for testing. cc_binary( name = "ipc_inference_sidecar_test_bin", diff --git a/services/inference_sidecar/common/benchmark/module_benchmark.cc b/services/inference_sidecar/common/benchmark/module_benchmark.cc index 9b0c1a2a..67319b11 100644 --- a/services/inference_sidecar/common/benchmark/module_benchmark.cc +++ b/services/inference_sidecar/common/benchmark/module_benchmark.cc @@ -103,8 +103,12 @@ class ModuleFixture : public benchmark::Fixture { BENCHMARK_DEFINE_F(ModuleFixture, BM_Predict)(benchmark::State& state) { for (auto _ : state) { + state.PauseTiming(); + std::string input = StringFormat(kJsonString); + state.ResumeTiming(); + PredictRequest predict_request; - predict_request.set_input(kJsonString); + predict_request.set_input(input); absl::StatusOr response = module_->Predict(predict_request); CHECK(response.ok()) << response.status().message(); diff --git a/services/inference_sidecar/common/benchmark/request_utils.cc b/services/inference_sidecar/common/benchmark/request_utils.cc index 844e63df..2547854b 100644 --- a/services/inference_sidecar/common/benchmark/request_utils.cc +++ b/services/inference_sidecar/common/benchmark/request_utils.cc @@ -14,6 +14,8 @@ #include "benchmark/request_utils.h" +#include +#include #include #include @@ -46,4 +48,30 @@ RegisterModelRequest CreateRegisterModelRequest( return new_register_request; } +// Returns an array of random floating numbers. +std::string GenerateRandomFloats(int n) { + if (n < 1) { + return ""; + } + std::string result; + absl::StrAppend(&result, "\"", GenerateRandomFloat(), "\""); + for (int i = 1; i < n; i++) { + absl::StrAppend(&result, ", \"", GenerateRandomFloat(), "\""); + } + return result; +} + +std::string StringFormat(const std::string& s) { + std::regex pattern("GenerateRandomFloats\\((\\d+)\\)"); + std::smatch match; + std::string result = s; + + while (std::regex_search(result, match, pattern)) { + int num = std::stoi(match.str(1)); + result = std::regex_replace(result, pattern, GenerateRandomFloats(num), + std::regex_constants::format_first_only); + } + return result; +} + } // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/inference_sidecar/common/benchmark/request_utils.h b/services/inference_sidecar/common/benchmark/request_utils.h index a6a1e66c..95341d13 100644 --- a/services/inference_sidecar/common/benchmark/request_utils.h +++ b/services/inference_sidecar/common/benchmark/request_utils.h @@ -15,6 +15,7 @@ #ifndef SERVICES_INFERENCE_SIDECAR_COMMON_BENCHMARK_REQUEST_UTILS_H_ #define SERVICES_INFERENCE_SIDECAR_COMMON_BENCHMARK_REQUEST_UTILS_H_ +#include #include #include "proto/inference_sidecar.pb.h" @@ -26,6 +27,16 @@ RegisterModelRequest CreateRegisterModelRequest( const RegisterModelRequest& register_request, const std::string& new_model_path); +inline std::string GenerateRandomFloat() { + return std::to_string(static_cast(rand()) / // NOLINT + static_cast(RAND_MAX)); +} + +// Returns an array of random floating numbers. +std::string GenerateRandomFloats(int n); + +std::string StringFormat(const std::string& s); + } // namespace privacy_sandbox::bidding_auction_servers::inference #endif // SERVICES_INFERENCE_SIDECAR_COMMON_BENCHMARK_REQUEST_UTILS_H_ diff --git a/services/inference_sidecar/common/benchmark/request_utils_test.cc b/services/inference_sidecar/common/benchmark/request_utils_test.cc new file mode 100644 index 00000000..9c3f5c0d --- /dev/null +++ b/services/inference_sidecar/common/benchmark/request_utils_test.cc @@ -0,0 +1,64 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "benchmark/request_utils.h" + +#include +#include + +#include "absl/strings/str_cat.h" +#include "gtest/gtest.h" + +namespace privacy_sandbox::bidding_auction_servers::inference { +namespace { + +constexpr int kSeed = 123; + +TEST(RequestUtilsTest, GenerateRandomFloat) { + srand(kSeed); + std::string expect = std::to_string(static_cast(rand()) / // NOLINT + static_cast(RAND_MAX)); + srand(kSeed); + ASSERT_EQ(GenerateRandomFloat(), expect); +} + +TEST(RequestUtilsTest, GenerateRandomFloats) { + const int kNums = 5; + srand(kSeed); + std::string expect; + for (int i = 0; i < kNums; i++) { + absl::StrAppend(&expect, "\"", GenerateRandomFloat(), "\"", + (i == kNums - 1) ? "" : ", "); + } + srand(kSeed); + std::string actual = GenerateRandomFloats(5); + ASSERT_EQ(actual, expect); +} + +TEST(RequestUtilsTest, StringFormat) { + ASSERT_EQ(StringFormat("{}"), "{}"); + ASSERT_EQ(StringFormat("{GenerateRandomFloats}"), "{GenerateRandomFloats}"); + ASSERT_EQ(StringFormat("{GenerateRandomFloats(0)}"), "{}"); + + srand(kSeed); + std::string expect = absl::StrCat("{", GenerateRandomFloats(3), "}"); + srand(kSeed); + ASSERT_EQ(StringFormat("{GenerateRandomFloats(3)}"), expect); + srand(kSeed); + ASSERT_EQ(StringFormat("{GenerateRandomFloats(2), GenerateRandomFloats(1)}"), + expect); +} + +} // namespace +} // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/inference_sidecar/common/benchmark/sandbox_benchmark.cc b/services/inference_sidecar/common/benchmark/sandbox_benchmark.cc index 8eae5a9d..5e49fdca 100644 --- a/services/inference_sidecar/common/benchmark/sandbox_benchmark.cc +++ b/services/inference_sidecar/common/benchmark/sandbox_benchmark.cc @@ -112,8 +112,12 @@ static void BM_Multiworker_Predict_GRPC(benchmark::State& state) { CHECK(status.ok()) << status.error_message(); for (auto _ : state) { + state.PauseTiming(); + std::string input = StringFormat(kJsonString); + state.ResumeTiming(); + PredictRequest predict_request; - predict_request.set_input(kJsonString); + predict_request.set_input(input); PredictResponse predict_response; grpc::ClientContext context; @@ -157,8 +161,12 @@ static void BM_Predict_GRPC(benchmark::State& state) { } for (auto _ : state) { + state.PauseTiming(); + std::string input = StringFormat(kJsonString); + state.ResumeTiming(); + PredictRequest predict_request; - predict_request.set_input(kJsonString); + predict_request.set_input(input); PredictResponse predict_response; grpc::ClientContext context; @@ -189,8 +197,12 @@ static void BM_Multiworker_Predict_IPC(benchmark::State& state) { CHECK(comms.SendProtoBuf(register_request)); for (auto _ : state) { + state.PauseTiming(); + std::string input = StringFormat(kJsonString); + state.ResumeTiming(); + PredictRequest predict_request; - predict_request.set_input(kJsonString); + predict_request.set_input(input); CHECK(comms.SendProtoBuf(predict_request)); PredictResponse predict_response; @@ -231,8 +243,12 @@ static void BM_Predict_IPC(benchmark::State& state) { } for (auto _ : state) { + state.PauseTiming(); + std::string input = StringFormat(kJsonString); + state.ResumeTiming(); + PredictRequest predict_request; - predict_request.set_input(kJsonString); + predict_request.set_input(input); absl::MutexLock lock(&mutex); diff --git a/services/inference_sidecar/common/inference_sidecar_main.cc b/services/inference_sidecar/common/inference_sidecar_main.cc index b19dcf7a..c3b0985a 100644 --- a/services/inference_sidecar/common/inference_sidecar_main.cc +++ b/services/inference_sidecar/common/inference_sidecar_main.cc @@ -16,6 +16,7 @@ #include +#include "absl/log/absl_log.h" #include "absl/log/check.h" #include "absl/status/status.h" #include "proto/inference_sidecar.pb.h" @@ -28,7 +29,11 @@ int main(int argc, char** argv) { CHECK(google::protobuf::util::JsonStringToMessage(argv[0], &config).ok()) << "Could not parse inference sidecar runtime config JsonString to a " "proto message."; - CHECK(privacy_sandbox::bidding_auction_servers::inference::Run(config).ok()) - << "Unsuccessful run of the inference sidecar."; + if (absl::Status run_status = + privacy_sandbox::bidding_auction_servers::inference::Run(config); + !run_status.ok()) { + ABSL_LOG(FATAL) << "Unsuccessful run of the inference sidecar due to " + << run_status; + } return 0; } diff --git a/services/inference_sidecar/common/tools/debug/start_inference b/services/inference_sidecar/common/tools/debug/start_inference index 287e56d5..c7cbf445 100755 --- a/services/inference_sidecar/common/tools/debug/start_inference +++ b/services/inference_sidecar/common/tools/debug/start_inference @@ -34,7 +34,7 @@ INFERENCE_SIDECAR_MOUNT=${PROJECT_ROOT}/services/inference_sidecar/modules/pytor INFERENCE_MODEL_MOUNT=${PROJECT_ROOT}/services/inference_sidecar/common/testdata/models/pytorch_e2e_model1.pt:/test_model # Default client JS code: -BIDDING_JS_PATH="/src/workspace/testing/functional/suts/inference/data/buyer-bidding-service/generateBidRunInference.js" +BIDDING_JS_PATH="/src/workspace/testing/functional/suts/inference_pytorch/data/buyer-bidding-service/generateBidRunInference.js" print_usage() { cat << USAGE @@ -114,8 +114,8 @@ export SERVER_START_CMD=$(cat << END --inference_model_local_paths="/test_model" \ --inference_sidecar_runtime_config='{ "num_interop_threads": 4, - "num_intraop_threads": 4 - "module_name": "tensorflow_v2_14_0" + "num_intraop_threads": 4, + "module_name": "pytorch_v2_1_1" }' \ --enable_bidding_service_benchmark="true" \ --init_config_client="false" --port=50057 \ @@ -141,6 +141,8 @@ export SERVER_START_CMD=$(cat << END --consented_debug_token="" \ --enable_protected_app_signals="false" \ --ps_verbosity=2 \ +--bidding_tcmalloc_background_release_rate_bytes_per_second=4096 \ +--bidding_tcmalloc_max_total_thread_cache_bytes=10737418240 \ --ad_retrieval_timeout_ms="60000" && exit END ) diff --git a/services/inference_sidecar/common/utils/BUILD b/services/inference_sidecar/common/utils/BUILD index bcd40b1e..48b48918 100644 --- a/services/inference_sidecar/common/utils/BUILD +++ b/services/inference_sidecar/common/utils/BUILD @@ -68,3 +68,27 @@ cc_test( "@com_google_googletest//:gtest_main", ], ) + +cc_library( + name = "cpu", + srcs = ["cpu.cc"], + hdrs = ["cpu.h"], + deps = [ + "@com_google_absl//absl/base:log_severity", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + ], +) + +cc_test( + name = "cpu_test", + size = "small", + srcs = ["cpu_test.cc"], + deps = [ + ":cpu", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/services/inference_sidecar/common/utils/cpu.cc b/services/inference_sidecar/common/utils/cpu.cc new file mode 100644 index 00000000..b8376ed6 --- /dev/null +++ b/services/inference_sidecar/common/utils/cpu.cc @@ -0,0 +1,60 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "utils/cpu.h" + +#include +#include +#include + +#include + +#include "absl/log/absl_log.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" + +namespace privacy_sandbox::bidding_auction_servers::inference { + +absl::Status SetCpuAffinity(const std::vector& cpus) { + if (cpus.size() == 0) { + return absl::InvalidArgumentError("cpus should have at least one element."); + } + cpu_set_t allowed_cpus; + if (sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus) != 0) { + // Failed to get CPU affinity of the current process. + return absl::ErrnoToStatus( + errno, absl::StrCat("sched_getaffinity() failed: ", errno)); + } + + const int num_virtual_cpus = CPU_COUNT(&allowed_cpus); + CPU_ZERO(&allowed_cpus); + ABSL_LOG(INFO) << "Num of the available CPUs: " << num_virtual_cpus; + + for (int cpu : cpus) { + if (cpu >= num_virtual_cpus) { + ABSL_LOG(WARNING) << "CPU (" << cpu << ") might not be available."; + } + CPU_SET(cpu, &allowed_cpus); + } + + if (sched_setaffinity(0, sizeof(allowed_cpus), &allowed_cpus) != 0) { + return absl::ErrnoToStatus( + errno, absl::StrCat("sched_setaffinity() failed: ", errno)); + } + const int num_pinned_cpus = CPU_COUNT(&allowed_cpus); + ABSL_LOG(INFO) << "Num of the pinned CPUs: " << num_pinned_cpus; + return absl::OkStatus(); +} + +} // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/inference_sidecar/common/utils/cpu.h b/services/inference_sidecar/common/utils/cpu.h new file mode 100644 index 00000000..0d144917 --- /dev/null +++ b/services/inference_sidecar/common/utils/cpu.h @@ -0,0 +1,29 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SERVICES_INFERENCE_SIDECAR_COMMON_UTILS_CPU_H_ +#define SERVICES_INFERENCE_SIDECAR_COMMON_UTILS_CPU_H_ + +#include + +#include "absl/status/status.h" + +namespace privacy_sandbox::bidding_auction_servers::inference { + +// Set the CPU affinity to the current process given a list of CPU IDs. +absl::Status SetCpuAffinity(const std::vector& cpus); + +} // namespace privacy_sandbox::bidding_auction_servers::inference + +#endif // SERVICES_INFERENCE_SIDECAR_COMMON_UTILS_CPU_H_ diff --git a/services/inference_sidecar/common/utils/cpu_test.cc b/services/inference_sidecar/common/utils/cpu_test.cc new file mode 100644 index 00000000..7c02ae29 --- /dev/null +++ b/services/inference_sidecar/common/utils/cpu_test.cc @@ -0,0 +1,64 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "utils/cpu.h" + +#include +#include + +#include "absl/log/absl_log.h" +#include "absl/status/status.h" +#include "googletest/include/gtest/gtest.h" + +namespace privacy_sandbox::bidding_auction_servers::inference { +namespace { + +constexpr int kMinCpuId = 0; +constexpr int kNonExistentCpuId = 99999; + +TEST(SetCpuAffinity, Success) { + EXPECT_EQ(SetCpuAffinity({kMinCpuId}).code(), absl::StatusCode::kOk); +} + +TEST(SetCpuAffinity, EmptyCpuSet) { + auto status = SetCpuAffinity({}); + ABSL_LOG(INFO) << status.message(); + EXPECT_EQ(status.code(), absl::StatusCode::kInvalidArgument); +} + +TEST(SetCpuAffinity, NoCpuExists) { + auto status = SetCpuAffinity({kNonExistentCpuId}); + ABSL_LOG(INFO) << status.message(); + EXPECT_EQ(status.code(), absl::StatusCode::kInvalidArgument); +} + +TEST(SetCpuAffinity, CpuExistsAfterFiltering) { + auto status = SetCpuAffinity({kMinCpuId, kNonExistentCpuId}); + ABSL_LOG(INFO) << status.message(); + EXPECT_EQ(status.code(), absl::StatusCode::kOk); +} + +TEST(SetCpuAffinity, SetAllCpus) { + const int num_cpus = sysconf(_SC_NPROCESSORS_ONLN); + std::vector all_cpus; + for (int i = 0; i < num_cpus; i++) { + all_cpus.push_back(i); + } + auto status = SetCpuAffinity(all_cpus); + ABSL_LOG(INFO) << status.message(); + EXPECT_EQ(status.code(), absl::StatusCode::kOk); +} + +} // namespace +} // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/inference_sidecar/modules/pytorch_v2_1_1/benchmark/BUILD b/services/inference_sidecar/modules/pytorch_v2_1_1/benchmark/BUILD index f92810a1..895042e7 100644 --- a/services/inference_sidecar/modules/pytorch_v2_1_1/benchmark/BUILD +++ b/services/inference_sidecar/modules/pytorch_v2_1_1/benchmark/BUILD @@ -33,3 +33,28 @@ cc_binary( "@inference_common//utils:file_util", ], ) + +cc_binary( + name = "sandbox_benchmark", + srcs = ["@inference_common//benchmark:sandbox_benchmark.cc"], + data = [ + "//:inference_sidecar_test_target", + "//:test_model_target", + ], + deps = [ + "//:pytorch", + "@com_github_grpc_grpc//:grpc++", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/synchronization", + "@com_google_benchmark//:benchmark", + "@com_google_benchmark//:benchmark_main", + "@inference_common//benchmark:request_utils", + "@inference_common//modules:module_interface", + "@inference_common//proto:inference_sidecar_cc_grpc_proto", + "@inference_common//proto:inference_sidecar_cc_proto", + "@inference_common//sandbox:sandbox_executor", + "@inference_common//utils:file_util", + ], +) diff --git a/services/seller_frontend_service/schemas/interest_group.json b/services/seller_frontend_service/schemas/interest_group.json index f3ad5575..aaad1b4f 100644 --- a/services/seller_frontend_service/schemas/interest_group.json +++ b/services/seller_frontend_service/schemas/interest_group.json @@ -56,6 +56,10 @@ { "$ref": "#/$defs/adRenderId" } ] } + }, + "recencyMs": { + "type": "number", + "description": "The most recent join time for this group expressed in milli seconds before the containing auctionBlob was requested. This field will be used by newer client versions. For older devices, the precison will be in seconds. If recencyMs is present, this value will be used to offer higher precision. If not, recency will be used. Only one of the recency or recencyMs is expected to present in the request." } } } diff --git a/services/seller_frontend_service/select_ad_reactor.cc b/services/seller_frontend_service/select_ad_reactor.cc index 9f326064..ade3435d 100644 --- a/services/seller_frontend_service/select_ad_reactor.cc +++ b/services/seller_frontend_service/select_ad_reactor.cc @@ -43,6 +43,8 @@ namespace privacy_sandbox::bidding_auction_servers { namespace { +constexpr int kMillisInMinute = 60000; +constexpr int kSecsInMinute = 60; using ScoreAdsRawRequest = ScoreAdsRequest::ScoreAdsRawRequest; using AdScore = ScoreAdsResponse::AdScore; using AdWithBidMetadata = @@ -124,8 +126,15 @@ AdWithBidMetadata SelectAdReactor::BuildAdWithBidMetadata( PS_VLOG(kNoisyInfo, log_context_) << "BrowserSignal: Recency:" << interest_group.browser_signals().recency(); - - result.set_recency(interest_group.browser_signals().recency()); + if (interest_group.browser_signals().has_recency_ms()) { + // ScoreAdsRequest requires recency to be in minutes. + result.set_recency(static_cast( + interest_group.browser_signals().recency_ms() / kMillisInMinute)); + } else { + // ScoreAdsRequest requires recency to be in minutes. + result.set_recency(static_cast( + interest_group.browser_signals().recency() / kSecsInMinute)); + } } // May as well skip further iterations. break; diff --git a/services/seller_frontend_service/select_ad_reactor_test.cc b/services/seller_frontend_service/select_ad_reactor_test.cc index 547ce4ac..52ca080d 100644 --- a/services/seller_frontend_service/select_ad_reactor_test.cc +++ b/services/seller_frontend_service/select_ad_reactor_test.cc @@ -793,7 +793,9 @@ TYPED_TEST(SellerFrontEndServiceTest, ReturnsWinningAdAfterScoring) { EXPECT_EQ(bid.join_count(), interest_group.browser_signals().join_count()); EXPECT_EQ(bid.recency(), - interest_group.browser_signals().recency()); + static_cast( + interest_group.browser_signals().recency_ms() / + 60000)); // recency in minutes } } diff --git a/services/seller_frontend_service/select_ad_reactor_web.cc b/services/seller_frontend_service/select_ad_reactor_web.cc index ca6465a5..78aea27c 100644 --- a/services/seller_frontend_service/select_ad_reactor_web.cc +++ b/services/seller_frontend_service/select_ad_reactor_web.cc @@ -122,8 +122,9 @@ absl::StatusOr SelectAdReactorForWeb::GetNonEncryptedResponse( absl::StatusOr compressed_data = GzipCompress(data_to_compress); if (!compressed_data.ok()) { - ABSL_LOG(ERROR) << "Failed to compress the CBOR serialized data: " - << compressed_data.status().message(); + PS_LOG(ERROR, log_context_) + << "Failed to compress the CBOR serialized data: " + << compressed_data.status().message(); FinishWithStatus( grpc::Status(grpc::INTERNAL, "Failed to compress CBOR data")); return absl::InternalError(""); diff --git a/services/seller_frontend_service/select_ad_reactor_web_test.cc b/services/seller_frontend_service/select_ad_reactor_web_test.cc index 87543de6..564eba03 100644 --- a/services/seller_frontend_service/select_ad_reactor_web_test.cc +++ b/services/seller_frontend_service/select_ad_reactor_web_test.cc @@ -215,11 +215,11 @@ TYPED_TEST(SelectAdReactorForWebTest, VerifyChaffedResponse) { EXPECT_TRUE(deserialized_auction_result->is_chaff()); } -auto EqLogContext(const LogContext& log_context) { - return AllOf( - Property(&LogContext::generation_id, Eq(log_context.generation_id())), - Property(&LogContext::adtech_debug_id, - Eq(log_context.adtech_debug_id()))); +auto EqLogContext(const server_common::LogContext& log_context) { + return AllOf(Property(&server_common::LogContext::generation_id, + Eq(log_context.generation_id())), + Property(&server_common::LogContext::adtech_debug_id, + Eq(log_context.adtech_debug_id()))); } auto EqGetBidsRawRequestWithLogContext( diff --git a/services/seller_frontend_service/util/select_ad_reactor_test_utils.h b/services/seller_frontend_service/util/select_ad_reactor_test_utils.h index 59212cee..41de44a2 100644 --- a/services/seller_frontend_service/util/select_ad_reactor_test_utils.h +++ b/services/seller_frontend_service/util/select_ad_reactor_test_utils.h @@ -224,7 +224,10 @@ EncryptedSelectAdRequestWithContext GetSampleSelectAdRequest( auto* interest_group = buyer_input.mutable_interest_groups()->Add(); interest_group->set_name(kSampleInterestGroupName); *interest_group->mutable_bidding_signals_keys()->Add() = "[]"; - + google::protobuf::RepeatedPtrField ad_render_ids; + ad_render_ids.Add(MakeARandomString()); + interest_group->mutable_browser_signals()->CopyFrom( + MakeRandomBrowserSignalsForIG(ad_render_ids)); google::protobuf::Map decoded_buyer_inputs; decoded_buyer_inputs.emplace(kSampleBuyer, buyer_input); google::protobuf::Map encoded_buyer_inputs; diff --git a/services/seller_frontend_service/util/web_utils.cc b/services/seller_frontend_service/util/web_utils.cc index 29838f4f..706618e1 100644 --- a/services/seller_frontend_service/util/web_utils.cc +++ b/services/seller_frontend_service/util/web_utils.cc @@ -227,6 +227,16 @@ BrowserSignals DecodeBrowserSignals(const cbor_item_t* root, } break; } + case 4: { // RecencyMs. + bool is_recency_valid_type = + IsTypeValid(&cbor_is_int, signal.value, kBrowserSignalsRecencyMs, + kInt, error_accumulator); + RETURN_IF_PREV_ERRORS(error_accumulator, fail_fast, signals); + if (is_recency_valid_type) { + signals.set_recency_ms(cbor_get_int(signal.value)); + } + break; + } } } diff --git a/services/seller_frontend_service/util/web_utils.h b/services/seller_frontend_service/util/web_utils.h index b468c843..5138d825 100644 --- a/services/seller_frontend_service/util/web_utils.h +++ b/services/seller_frontend_service/util/web_utils.h @@ -73,6 +73,8 @@ inline constexpr char kBrowserSignalsBidCount[] = "browserSignals[x].bidCount"; inline constexpr char kBrowserSignalsJoinCount[] = "browserSignals[x].joinCount"; inline constexpr char kBrowserSignalsRecency[] = "browserSignals[x].recency"; +inline constexpr char kBrowserSignalsRecencyMs[] = + "browserSignals[x].recencyMs"; inline constexpr char kBrowserSignalsPrevWins[] = "browserSignals[x].prevWins"; inline constexpr char kPrevWinsEntry[] = "browserSignals[x].prevWins[y]"; inline constexpr char kPrevWinsTimeEntry[] = "browserSignals[x].prevWins[y][0]"; diff --git a/services/seller_frontend_service/util/web_utils_test.cc b/services/seller_frontend_service/util/web_utils_test.cc index 8623fa03..5b60e639 100644 --- a/services/seller_frontend_service/util/web_utils_test.cc +++ b/services/seller_frontend_service/util/web_utils_test.cc @@ -53,6 +53,7 @@ inline constexpr char kSampleUserBiddingSignals[] = inline constexpr int kSampleJoinCount = 1; inline constexpr int kSampleBidCount = 2; inline constexpr int kSampleRecency = 3; +inline constexpr int kSampleRecencyMs = 3000; inline constexpr char kSampleIgOwner[] = "foo_owner"; inline constexpr char kSampleGenerationId[] = "6fa459ea-ee8a-3ca4-894e-db77e160355e"; @@ -79,6 +80,11 @@ using InteractionUrlMap = ::google::protobuf::Map; using EncodedBuyerInputs = ::google::protobuf::Map; using DecodedBuyerInputs = absl::flat_hash_map; +struct CborInterestGroupConfig { + std::optional recency; + std::optional recency_ms; +}; + cbor_pair BuildStringMapPair(absl::string_view key, absl::string_view value) { return {cbor_move(cbor_build_stringn(key.data(), key.size())), cbor_move(cbor_build_stringn(value.data(), value.size()))}; @@ -114,7 +120,8 @@ cbor_pair BuildStringArrayMapPair( cbor_move(array)}; } -struct cbor_item_t* BuildSampleCborInterestGroup() { +struct cbor_item_t* BuildSampleCborInterestGroup( + CborInterestGroupConfig cbor_interest_group_config = {}) { cbor_item_t* interest_group = cbor_new_definite_map(kNumInterestGroupKeys); EXPECT_TRUE( cbor_map_add(interest_group, BuildStringMapPair(kName, kSampleIgName))); @@ -138,9 +145,17 @@ struct cbor_item_t* BuildSampleCborInterestGroup() { BuildIntMapPair(kJoinCount, kSampleJoinCount))); EXPECT_TRUE(cbor_map_add(browser_signals, BuildIntMapPair(kBidCount, kSampleBidCount))); - EXPECT_TRUE( - cbor_map_add(browser_signals, BuildIntMapPair(kRecency, kSampleRecency))); - + if (cbor_interest_group_config.recency.has_value()) { + EXPECT_TRUE(cbor_map_add( + browser_signals, + BuildIntMapPair(kRecency, cbor_interest_group_config.recency.value()))); + } + if (cbor_interest_group_config.recency_ms.has_value()) { + EXPECT_TRUE(cbor_map_add( + browser_signals, + BuildIntMapPair(kRecencyMs, + cbor_interest_group_config.recency_ms.value()))); + } // Build prevWins arrays. cbor_item_t* child_arr_1 = cbor_new_definite_array(2); EXPECT_TRUE(cbor_array_push(child_arr_1, cbor_move(cbor_build_uint64(-20)))); @@ -236,7 +251,7 @@ ScoreAdsResponse::AdScore MakeARandomComponentAdScore() { return winner; } -TEST(ChromeRequestUtils, Decode_Success) { +void TestDecode(const CborInterestGroupConfig& cbor_interest_group_config) { ScopedCbor protected_auction_input( cbor_new_definite_map(kNumRequestRootKeys)); EXPECT_TRUE(cbor_map_add(*protected_auction_input, @@ -248,7 +263,8 @@ TEST(ChromeRequestUtils, Decode_Success) { BuildBoolMapPair(kDebugReporting, true))); ScopedCbor ig_array(cbor_new_definite_array(1)); - EXPECT_TRUE(cbor_array_push(*ig_array, BuildSampleCborInterestGroup())); + EXPECT_TRUE(cbor_array_push( + *ig_array, BuildSampleCborInterestGroup(cbor_interest_group_config))); cbor_item_t* ig_bytestring = CompressInterestGroups(ig_array); cbor_item_t* interest_group_data_map = cbor_new_definite_map(1); @@ -300,7 +316,12 @@ TEST(ChromeRequestUtils, Decode_Success) { BrowserSignals* signals = expected_ig.mutable_browser_signals(); signals->set_join_count(kSampleJoinCount); signals->set_bid_count(kSampleBidCount); - signals->set_recency(kSampleRecency); + if (cbor_interest_group_config.recency.has_value()) { + signals->set_recency(cbor_interest_group_config.recency.value()); + } + if (cbor_interest_group_config.recency_ms.has_value()) { + signals->set_recency_ms(cbor_interest_group_config.recency_ms.value()); + } std::string prev_wins_json_str = absl::StrFormat( R"([[-20,"%s"],[-100,"%s"]])", kSampleAdRenderId1, kSampleAdRenderId2); signals->set_prev_wins(prev_wins_json_str); @@ -362,6 +383,25 @@ TEST(ChromeRequestUtils, Decode_Success) { } } +TEST(ChromeRequestUtils, DecodeSuccessWithRecencyInBrowserSignals) { + CborInterestGroupConfig cbor_interest_group_config = { + .recency = kSampleRecency, + }; + TestDecode(cbor_interest_group_config); +} + +TEST(ChromeRequestUtils, DecodeSuccessWithRecencyMsInBrowserSignals) { + CborInterestGroupConfig cbor_interest_group_config = {.recency_ms = + kSampleRecencyMs}; + TestDecode(cbor_interest_group_config); +} + +TEST(ChromeRequestUtils, DecodeSuccessWithRecencyAndRecencyMsInBrowserSignals) { + CborInterestGroupConfig cbor_interest_group_config = { + .recency = kSampleRecency, .recency_ms = kSampleRecencyMs}; + TestDecode(cbor_interest_group_config); +} + TEST(ChromeRequestUtils, Decode_FailOnWrongType) { ScopedCbor root(cbor_build_stringn("string", 6)); std::string serialized_cbor = SerializeCbor(*root); diff --git a/tools/secure_invoke/README.md b/tools/secure_invoke/README.md index 32b14b9a..913dfa08 100644 --- a/tools/secure_invoke/README.md +++ b/tools/secure_invoke/README.md @@ -47,6 +47,70 @@ This method expects a plaintext json with the following fields. } ``` +A sample request is as follows: + +```json +{ + "auction_config": { + "auction_signals": "{}", + "buyer_list": ["Placeholder-Should-Match-With-The-Keys-In-BUYER_SERVER_HOSTS-In-SFE"], + "buyer_timeout_ms": 100000, + "per_buyer_config": { + "Placeholder-Should-Match-With-The-Keys-In-BUYER_SERVER_HOSTS-In-SFE": { + "buyer_debug_id": "Buyer-Debug-ID-Useful-For-Debugging", + "buyer_signals": "{}" + } + }, + "seller": "Placeholder-Should-Match-With-seller_origin_domain-In-SFE-Config", + "seller_debug_id": "Seller-Debug-ID-Useful-For-Debugging", + "seller_signals": "[]" + }, + "client_type": "CLIENT_TYPE_BROWSER", + "raw_protected_audience_input": { + "generation_id": "A-UUID", + "publisher_name": "example.com", + "raw_buyer_input": { + "Placeholder-Should-Match-With-Keys-In-BUYER_SERVER_HOSTS-In-SFE": { + "interest_groups": [ + { + "ad_render_ids": [ + "test_id-That-Will-Be-Used-To-Generated-Ad-Render-Id-In-GenerateBid" + ], + "bidding_signals_keys": [ + "Sample-key-To-Be-Used-To-Lookup-Real-Time-Signals-From-Buyer-KV" + ], + "browser_signals": { + "bid_count": "1", + "join_count": "2", + "prev_wins": "[]" + }, + "name": "Test-Interest-Group-Name", + "user_bidding_signals": "[]" + } + ] + } + } + } +} +``` + +Notes: + +- Hosts in `buyer_list` must be a subset of keys in `BUYER_SERVER_HOST` + [map](https://github.com/privacysandbox/bidding-auction-servers/blob/290329503f5f5b57acb3ddc3b0fe79502cd7da05/production/deploy/gcp/terraform/environment/demo/seller/seller.tf#L66) + configured in SFE terraform config (when running on cloud) OR key in `buyer_server_hosts` + [map](https://github.com/privacysandbox/bidding-auction-servers/blob/290329503f5f5b57acb3ddc3b0fe79502cd7da05/tools/debug/start_sfe#L31) + (when running locally). +- The key in `per_buyer_config` and `raw_buyer_input` should also match with the buyer domain name + as mentioned in the last point (about `buyer_list`) +- The `seller` field must match with + [`SELLER_ORIGIN_DOMAIN`](https://github.com/privacysandbox/bidding-auction-servers/blob/290329503f5f5b57acb3ddc3b0fe79502cd7da05/production/deploy/gcp/terraform/environment/demo/seller/seller.tf#L64) + (when running on cloud) OR + [`seller_origin_domain`](https://github.com/privacysandbox/bidding-auction-servers/blob/290329503f5f5b57acb3ddc3b0fe79502cd7da05/tools/debug/start_sfe#L29) + (when running locally). +- When sending requests to a locally running service, a `-insecure` param also needs to be added + to the secure_invoke CLI. + ### Top Level Auction [SelectAdRequest] to SFE This method expects a plaintext json with the following fields. @@ -85,6 +149,7 @@ This method expects a plaintext json with the following fields. # Setup arguments. INPUT_PATH=select_ad_request.json # Needs to be a valid plaintext in the root of the B&A project (i.e. the path is .../bidding-auction-server/select_ad_request.json) SFE_HOST_ADDRESS=seller.domain.com # DNS name of SFE (e.g. dns:///seller.domain.com) +(For local services, use: SFE_HOST_ADDRESS=localhost:50053) CLIENT_IP= # Run the tool with desired arguments. @@ -124,9 +189,10 @@ gRPC request to BFE: ```bash # Setup arguments. -INPUT_PATH=/tmp/get_bids_request.txt # Needs to be a valid GetBidsRawRequest +INPUT_PATH=/tmp/get_bids_request.json # Needs to be a valid GetBidsRawRequest INPUT_FORMAT=PROTO BFE_HOST_ADDRESS=buyer.domain.com # DNS name of BFE service (Example: dns:///buyer.domain.com) +(For local runs services, use: BFE_HOST_ADDRESS=localhost:50051) CLIENT_IP= # Run the tool with desired arguments. @@ -147,6 +213,7 @@ gRPC request to BFE: INPUT_PATH=/tmp/get_bids_request.json # Needs to be a valid GetBidsRawRequest INPUT_FORMAT=JSON BFE_HOST_ADDRESS=buyer.domain.com # DNS name of BFE service (Example: dns:///buyer.domain.com) +(For local runs services, use: BFE_HOST_ADDRESS=localhost:50051) CLIENT_IP= # Run the tool with desired arguments. @@ -163,6 +230,45 @@ Notes: - `target_service` flag must be set to `bfe` when sending a `GetBidsRawRequest` to a BFE. - The input request can be specified either as JSON or proto. +- When sending requests to a locally running service, a `-insecure` param also needs to be added + to the secure_invoke CLI. + +A sample request is as follows: + +```json +{ + "client_type": "CLIENT_TYPE_BROWSER", + "buyer_input": { + "interest_groups": [ + { + "name": "Test-Interest-Group-Name", + "bidding_signals_keys": "Sample-key-To-Be-Used-To-Lookup-Real-Time-Signals-From-Buyer-KV", + "ad_render_ids": [ + "Sample-key-To-Be-Used-To-Lookup-Real-Time-Signals-From-Buyer-KV" + ], + "user_bidding_signals": "[]", + "browser_signals": { + "join_count": "1", + "bid_count": "2", + "prev_wins": "[]" + } + } + ] + }, + "auction_signals": "{}", + "buyer_signals": "{}", + "seller": "Placeholder-Should-Match-With-seller_origin_domain-In-SFE-Config", + "publisher_name": "example.com", + "log_context": { + "generation_id": "A-UUID", + "adtech_debug_id": "A-Debug-ID-Useful-For-Debugging" + }, + "consented_debug_config": { + "is_consented": true, + "token": "123456" + } +} +``` [selectadrequest]: https://github.com/privacysandbox/bidding-auction-servers/blob/332e46b216bfa51873ca410a5a47f8bec9615948/api/bidding_auction_servers.proto#L225 @@ -209,3 +315,9 @@ If you want to send a request to servers running on localhost, make sure to spec `DOCKER_NETWORK=host` environment variable. See [here](https://github.com/privacysandbox/bidding-auction-servers/blob/main/tools/debug/README.md#test-buyer-stack) for more detail. + +Also, note when sending requests to services running on localhost, `-insecure` flag needs to be +added to the commands. + +Finally, when sending requests to local services, use `localhost:50053` and `localhost:50051` for +SFE and BFE host addresses (i.e. `-host_addr` param) respectively. diff --git a/version.txt b/version.txt index 084e244c..240bba90 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -3.6.0 \ No newline at end of file +3.7.0 \ No newline at end of file