diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ac2dd613..83fd8f4e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,7 +22,8 @@ exclude: (?x)^( services/inference_sidecar/common/builders/.*| services/inference_sidecar/modules/pytorch_v2_1_1/builders/.*| services/inference_sidecar/modules/tensorflow_v2_14_0/builders/.*| - testing/functionaltest-system/.* + testing/functionaltest-system/.*| + google_internal/kokoro/key/.* )$ fail_fast: false @@ -118,6 +119,18 @@ repos: types: [ "dockerfile" ] entry: builders/tools/hadolint + - id: ps_vlog_123 + name: Don't use PS_VLOG(0/1/2/3) + entry: sh -c '! grep -RE "PS_VLOG[(][01][,)]" services/' + language: system + types_or: + - c++ + - c + pass_filenames: false + always_run: false + fail_fast: false + verbose: true + - repo: https://github.com/pre-commit/mirrors-prettier rev: v3.0.3 hooks: @@ -127,8 +140,8 @@ repos: - markdown exclude: (?x)^( testing/functional/suts/inference/data/seller-auction-service/scoreAd\.js| - testing/functional/suts/basic/data/seller-auction-service/scoreAd\.js| - testing/functional/suts/basic/data/buyer-bidding-service/generateBid\.js| + testing/functional/suts/.*/data/seller-auction-service/scoreAd\.js| + testing/functional/suts/.*/data/buyer-bidding-service/generateBid\.js| testing/functional/suts/.*/.*\.reply\.json )$ diff --git a/BUILD b/BUILD index 4f953eae..a23dc523 100644 --- a/BUILD +++ b/BUILD @@ -182,14 +182,6 @@ string_flag( ], ) -config_setting( - name = "inference", - flag_values = { - ":inference_build": "yes", - }, - visibility = ["//visibility:public"], -) - string_flag( name = "inference_runtime", build_setting_default = "noop", @@ -203,7 +195,6 @@ string_flag( config_setting( name = "inference_noop", flag_values = { - ":inference_build": "yes", ":inference_runtime": "noop", }, visibility = ["//visibility:public"], diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a7bac74..13f71458 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,37 @@ All notable changes to this project will be documented in this file. See [commit-and-tag-version](https://github.com/absolute-version/commit-and-tag-version) for commit guidelines. +## 3.5.0 (2024-04-24) + + +### Features + +* Add runtime check for correct inference backend version +* Adds debug info to secure invoke response +* Adds implementation of GetComponentAuctionCiphertexts API +* Adds optional flag to configure cloud platforms for component sellers +* allow AWS Session Manager instance connection +* enable bucket fetching for score ad +* Enables and adds tests for GetComponentAuctionsCiphertexts API +* Enables Top Level Auctions feature +* Implement Support for Bid Currency in Reporting +* Set Seller Rejection Reason on AdWithBid for Mismatch with Expected Buyer Currency +* Set Seller Rejection Reasons on AdScore for currency-related rejectons +* Support inference runtime configurations +* Support Interest Group Origin in InterestGroup and AuctionResponse +* Top Level auction does not accept multiple results from same seller +* upload aws/gcp hashes as artifacts on release + + +### Bug Fixes + +* Correct check condition in inference main +* Increase the gRPC recv message limit for the inference sidecar +* Pass complete ad object to scoreAd +* Remove redundant clang-tidy repo, move config to .clang-tidy +* RPC is finished properly in top level auction with reporting +* Temporarily add bazel_clang_tidy repo + ## 3.4.0 (2024-04-02) diff --git a/WORKSPACE b/WORKSPACE index 9102a7b0..7f7d1772 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -11,13 +11,23 @@ http_archive( python_deps("//builders/bazel") +# TODO: Remove bazel_clang_tidy once we sync to the common repo commit 9edb0c3 (4/3/2024) or later +http_archive( + name = "bazel_clang_tidy", + sha256 = "352aeb57ad7ed53ff6e02344885de426421fb6fd7a3890b04d14768d759eb598", + strip_prefix = "bazel_clang_tidy-4884c32e09c1ea9ac96b3f08c3004f3ac4c3fe39", + urls = [ + "https://github.com/erenon/bazel_clang_tidy/archive/4884c32e09c1ea9ac96b3f08c3004f3ac4c3fe39.zip", + ], +) + http_archive( name = "google_privacysandbox_servers_common", - # 2024-04-01 - sha256 = "e36cc26c917ec4b1066a32777b48ac8728ba13c276cdda2e91c36ad2037d9bcd", - strip_prefix = "data-plane-shared-libraries-1fbac466b6b88e00a4ca037f7359ee1942ade13e", + # 2024-04-19 + sha256 = "ac5dd014d81cdc76e86cdaf8b584d5ee8494453315dc986860eb1e60f424a668", + strip_prefix = "data-plane-shared-libraries-4b1e6744cf3e46efee0cf29d886b22fe93274c79", urls = [ - "https://github.com/privacysandbox/data-plane-shared-libraries/archive/1fbac466b6b88e00a4ca037f7359ee1942ade13e.zip", + "https://github.com/privacysandbox/data-plane-shared-libraries/archive/4b1e6744cf3e46efee0cf29d886b22fe93274c79.zip", ], ) @@ -75,11 +85,11 @@ http_archive( http_archive( name = "service_value_key_fledge_privacysandbox", - # commit af184d649be5d9f0a62738db41ed1496de427bcd 2024-03-15 - sha256 = "", - strip_prefix = "protected-auction-key-value-service-af184d649be5d9f0a62738db41ed1496de427bcd", + # commit 89f678982d5cc1bcfde354980ed9380239f54b96 2024-03-22 + sha256 = "dfa9da5e5b2b71aa706781e1edc0c6f4e752d4ebf30f82624f711e776553fbe2", + strip_prefix = "protected-auction-key-value-service-89f678982d5cc1bcfde354980ed9380239f54b96", urls = [ - "https://github.com/privacysandbox/protected-auction-key-value-service/archive/af184d649be5d9f0a62738db41ed1496de427bcd.zip", + "https://github.com/privacysandbox/protected-auction-key-value-service/archive/89f678982d5cc1bcfde354980ed9380239f54b96.zip", ], ) @@ -116,16 +126,3 @@ local_repository( name = "tensorflow_v2_14_0", path = "services/inference_sidecar/modules/tensorflow_v2_14_0", ) - -# TODO: Remove this once we sync to the common repo version that includes this -# as a dependecy. -http_archive( - name = "bazel_clang_tidy", - patch_args = ["-p1"], - patches = ["//third_party:bazel_clang_tidy.patch"], - sha256 = "352aeb57ad7ed53ff6e02344885de426421fb6fd7a3890b04d14768d759eb598", - strip_prefix = "bazel_clang_tidy-4884c32e09c1ea9ac96b3f08c3004f3ac4c3fe39", - urls = [ - "https://github.com/erenon/bazel_clang_tidy/archive/4884c32e09c1ea9ac96b3f08c3004f3ac4c3fe39.zip", - ], -) diff --git a/api/bidding_auction_servers.proto b/api/bidding_auction_servers.proto index 5e914e69..d7b4ca36 100644 --- a/api/bidding_auction_servers.proto +++ b/api/bidding_auction_servers.proto @@ -318,6 +318,16 @@ message AuctionResult { // This is only populated for component auctions. AuctionParams auction_params = 18; + + // BuyerReportingId of the winning Ad. + // This will be verified with the buyerReportingId in the Ad properties on + // the browser. + string buyer_reporting_id = 19; + + // BuyerAndSellerReportingId of the winning Ad. + // This will be verified with the buyerAndSellerReportingId in the Ad + // properties on the browser. + string buyer_and_seller_reporting_id = 20; } message GetComponentAuctionCiphertextsRequest { @@ -713,8 +723,8 @@ message GetBidsResponse { // Bid for an ad candidate. message AdWithBid { // Metadata of the ad, this will be passed to Seller's scoring function. - // Represents a serialized string that is deserialized to a JSON object - // before passing to Adtech script. + // Represents an opaque object that is eventually passed to seller Adtech + // script. // Note: API will be updated separately for Component Ads. google.protobuf.Value ad = 1; @@ -752,6 +762,9 @@ message AdWithBid { // Indicates the currency used for the bid price (expressed as ISO 4217 alpha code). string bid_currency = 10; + + // Optional. buyerReportingId set in buyerReportingMetadata of reportWin() + string buyer_reporting_id = 11; } // Bidding service operated by buyer. @@ -989,12 +1002,17 @@ message ProtectedAppSignalsAdWithBid { // Indicates the currency used for the bid price. string bid_currency = 6; - // Holds schema version as well as features related information that needs - // to be send back to the buyer who has the winning bid. - bytes egress_features = 7; + reserved 7; // Optional field for debug report URLs. DebugReportUrls debug_report_urls = 8; + + // Holds schema version as well as features related information that needs + // to be send back to the buyer who has the winning bid. + string egress_payload = 9; + + // Temporary unlimited size egress features. Meant for experimentation only. + string temporary_unlimited_egress_payload = 10; } // Encrypted response to GenerateProtectedAppSignalsBidsRequest with bid prices @@ -1032,8 +1050,7 @@ message ScoreAdsRequest { // Bid for an ad along with other information required to score the ad. message AdWithBidMetadata { // Metadata of the ad, this will be passed to Seller's scoring function. - // Represents a serialized string that is deserialized to a JSON object - // before passing to Adtech script. + // Represents an opaque ad metadata object. google.protobuf.Value ad = 1; // Bid price corresponding to an ad. @@ -1081,6 +1098,13 @@ message ScoreAdsRequest { // Indicates the currency used for the bid price (expressed as ISO 4217 alpha code). string bid_currency = 12; + + // Creator of the IG to which the winning ad belongs (applicable for + // Protected Audience only). + string interest_group_origin = 13; + + // Optional. buyerReportingId set in buyerReportingMetadata of reportWin() + string buyer_reporting_id = 14; } // Ad with bid. repeated AdWithBidMetadata ad_bids = 1; @@ -1161,15 +1185,21 @@ message ScoreAdsRequest { // performed stochastically. double ad_cost = 5; - // Features to be sent back to the buyer of winning app install ad to help - // train their models. - bytes egress_features = 6; + reserved 6; // Domain of Buyer who owns app signals used for this ad. string owner = 7; // Indicates the currency used for the bid price. string bid_currency = 8; + + // Features to be sent back to the buyer of winning app install ad to help + // train their models. + string egress_payload = 9; + + // Temporary unlimited size egress features. Meant for experimentation + // only. + string temporary_unlimited_egress_payload = 10; } // Ad with bid for protected app signals. @@ -1326,6 +1356,24 @@ message ScoreAdsResponse { // If the seller specified a seller_currency in the AuctionConfig, // this value, if specified, must match it. If not, the bid is rejected. string bid_currency = 16; + + // Currency of the of the original buyer bid if present + // (expressed as ISO 4217 alpha code). + // Not expected to be set in output from scoreAd(); + // B&A will set this interally from the AdWithBid. + string buyer_bid_currency = 17; + // BuyerReportingId of the winning Ad. + // This will be verified with the buyerReportingId in the Ad properties on + // the browser. + string buyer_reporting_id = 18; + // BuyerAnd SellerReportingId of the winning Ad. + // This will be verified with the buyerAndSellerReportingId in the Ad + // properties on the browser. + string buyer_and_seller_reporting_id = 19; + + // Owner of the IG to which the winning ad belongs (applicable for + // Protected Audience only). + string interest_group_origin = 20; } // The response includes the top scored ad along with other related data. diff --git a/production/deploy/aws/terraform/environment/demo/seller/seller.tf b/production/deploy/aws/terraform/environment/demo/seller/seller.tf index a23a1eef..135d7347 100644 --- a/production/deploy/aws/terraform/environment/demo/seller/seller.tf +++ b/production/deploy/aws/terraform/environment/demo/seller/seller.tf @@ -63,6 +63,7 @@ module "seller" { AUCTION_SERVER_HOST = "" # Example: "dns:///auction.seller-frontend.com:443" KEY_VALUE_SIGNALS_HOST = "" # Example: "https://pubads.g.doubleclick.net/td/sts" BUYER_SERVER_HOSTS = "" # Example: "{ \"https://bid1.com\": { \"url\": \"dns:///bidding1.com:443\", \"cloudPlatform\": \"AWS\" } }" + SELLER_CLOUD_PLATFORMS_MAP = "" # Example: "{ \"https://partner-seller1.com\": "GCP", \"https://partner-seller2.com\": "AWS"}" ENABLE_SELLER_FRONTEND_BENCHMARKING = "" # Example: "false" ENABLE_AUCTION_COMPRESSION = "" # Example: "false" ENABLE_BUYER_COMPRESSION = "" # Example: "false" @@ -76,6 +77,7 @@ module "seller" { TEST_MODE = "" # Example: "false" SELLER_CODE_FETCH_CONFIG = "" # Example: # "{ + # "fetchMode": 0, # "auctionJsPath": "", # "auctionJsUrl": "https://example.com/scoreAd.js", # "urlFetchPeriodMs": 13000000, diff --git a/production/deploy/aws/terraform/modules/buyer/service_vars.tf b/production/deploy/aws/terraform/modules/buyer/service_vars.tf index f0389365..e69e0f82 100644 --- a/production/deploy/aws/terraform/modules/buyer/service_vars.tf +++ b/production/deploy/aws/terraform/modules/buyer/service_vars.tf @@ -145,6 +145,8 @@ variable "vpc_interface_endpoint_services" { default = [ "ec2", "ssm", + "ec2messages", + "ssmmessages", "autoscaling", "monitoring", "xray", diff --git a/production/deploy/aws/terraform/modules/seller/service_vars.tf b/production/deploy/aws/terraform/modules/seller/service_vars.tf index a13c0000..5bce1340 100644 --- a/production/deploy/aws/terraform/modules/seller/service_vars.tf +++ b/production/deploy/aws/terraform/modules/seller/service_vars.tf @@ -157,6 +157,8 @@ variable "vpc_interface_endpoint_services" { default = [ "ec2", "ssm", + "ec2messages", + "ssmmessages", "autoscaling", "monitoring", "xray", diff --git a/production/deploy/aws/terraform/services/iam_role_policies/main.tf b/production/deploy/aws/terraform/services/iam_role_policies/main.tf index ae81a141..6d36829a 100644 --- a/production/deploy/aws/terraform/services/iam_role_policies/main.tf +++ b/production/deploy/aws/terraform/services/iam_role_policies/main.tf @@ -94,6 +94,11 @@ resource "aws_iam_role_policy_attachment" "instance_role_policy_attachment" { role = var.server_instance_role_name } +resource "aws_iam_role_policy_attachment" "ssm_instance_role_attachment" { + role = var.server_instance_role_name + policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" +} + # Set up policies for using EC2 instance connect. data "aws_iam_policy_document" "ssh_instance_policy_doc" { statement { diff --git a/production/deploy/gcp/terraform/environment/demo/README.md b/production/deploy/gcp/terraform/environment/demo/README.md index eaacdd1e..ce500470 100644 --- a/production/deploy/gcp/terraform/environment/demo/README.md +++ b/production/deploy/gcp/terraform/environment/demo/README.md @@ -27,8 +27,81 @@ This document will be continually updated with all GCP concerns. 1. Run: - gcloud auth login gcloud beta auth application-default login gcloud config set project + ```bash + gcloud auth login + gcloud beta auth application-default login + gcloud config set project + ``` + +1. Create two GCP secrets via the + [Secret Manager](https://cloud.google.com/secret-manager/docs/create-secret-quickstart). These + should match with the names found in `modules/secrets/secrets.tf`. You may use the following + examples directly. + + 1. `envoy-tls-termination-key`. Example: + + ```bash + -----BEGIN PRIVATE KEY----- + MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQC8Xer29Ko4WlI8 + zyGMyUdPgSrcSZn8sreNRpZ3shAwJG96XrDdmPtCaPDPw+YN0nplrYmQ5Sxceitm + raBUl1PpEgPje/JxfLMtRsk1S7gfKMW24gqsleraV9ZBqin9EeroefmIaDv/otAK + GjV6Ty/j5rZl49vMMLeRDs2rN9Oi9ukdaoMWNXyPNTpm1yt4b8PB+ZVVNYKHLb2r + Hf1+Fa1NBLMixtBLd/UquNmlJzSNBoulqbBlmyObbGEwMaxI7KHNbP88YmGhp5KM + cWo/2PC13fSM71OiuaLUoHRG9JfEWqya9NtmNNhnf1KTQXwA2u5Fe7Wc+4mRhdjP + BOP26NSTAgMBAAECggEBALrgCg166a0CnnfJnqVHwsFzigwF0QlMXKGCGCEjvL+m + RhqG+ry92vglmFLnLMMlv1xEcCgZ1IrigVBajKefghXGU6lJ/FrutewDP/bp6f6v + uocXdjOGf/qiDeQTZ5i0P/Lnn9HeZzfUVMTQ/6EaEo7tAqPPDO5knpkAsLZeqk4P + JzUaqoZdMRXuq9qmJZl9vyRj1rRqS99+JV6Oody9+SO2a6hKQqH8w7EfLTTcULqp + ZYHXdx6CnOMudf8F7fqBLvWL0piCeyfGX3JXxmujgu4XUMf6Y/jnV2O97ZGmALYh + S2N2huP3dloxKUacubILlruqDBtL0s/o9x4MMSyNtqECgYEA5ptbRQXT8svxJ3hB + DkT0JMTV2NXZ4EfA5RRTMRzr0vgoryIFWq77Gu056kRtM1/ymWSJiPj18lWAraqi + wv6ywHx8W3Pdp2vrnMsaqnCI8LkrF5iKbJF0GSSrLjckND+WFfwGGbbfqxa21peo + 8dRl2+tIwBzqaa7dtQ7vESNS9RsCgYEA0RvYi8KQXQ+w9Xelcflgs+NKc73jKC6k + 3xc+toGHgfRZK+C6vRbfF1c+Kf1JRXpOFbmpE4RjgPVdWk+Pue/6Gs6J//SxCDFF + ZqMINB07vGpeh3AP90C6P4zc8YUsSRsw9F/hQgSDLhStWZxFqChbfdBjU5kf5asf + /z81vjEWLekCgYBufPD12Rz7r4sThiJlW9Q96bEr+wow0zAwkdRqK5kxs4SKpJo8 + IKpe9FpTTAWmH8p0hB8BaYctXJoSmzbwhmfOodZTWuhQVvzEWuujzddOvulOnN91 + tRsTEOaTdgf6oJygW+fwWhZAOtnPZ0qi00kaXVi18yS9DfNb1JPmei49EQKBgQDP + MpJNWcqGC8hCUf2jg4Cofm0FZoAxDpbbX0MKwCovQJki+xjNyF3h2NaF8K2rpFa+ + /CpmZmXaIEYR+Ifnq7vc2A6xihnojjnAS4cTbGwGdDeaaBXJ318tHTzILDcHcWP+ + oQqoyaPaAy8JfekfiG2vqs7gxPdwMTIRTubHwAfEEQKBgQC3XlPy/r5r7q2O5iwW + d6UJdh0V+h4zMyenMWRixF1aokNxc9V6GZfK3Lj2tSzjCCHsUkiil8DukB4NVRGW + KX6inN4S29QarQSdfDW2PvEko0JyfejK5VY2dl8GvGldDhNIt0i9UtXdk6bqcRpy + lq0bHiQyycFfivMIGsQMP+qVjg== + -----END PRIVATE KEY----- + ``` + + 1. `envoy-tls-termination-cert` must be the corresponding cert. May be self-signed. Example: + + ```bash + -----BEGIN CERTIFICATE----- + MIIEaDCCAtCgAwIBAgIRAN1DWyoIZOr/6En495nTwsowDQYJKoZIhvcNAQELBQAw + gZsxHjAcBgNVBAoTFW1rY2VydCBkZXZlbG9wbWVudCBDQTE4MDYGA1UECwwvZGFu + a29jb2pAZGFua29jb2ouYy5nb29nbGVycy5jb20gKERhbmllbCBLb2NvaikxPzA9 + BgNVBAMMNm1rY2VydCBkYW5rb2NvakBkYW5rb2Nvai5jLmdvb2dsZXJzLmNvbSAo + RGFuaWVsIEtvY29qKTAeFw0yMjExMjIyMTMxMDhaFw0yNTAyMjIyMTMxMDhaMGMx + JzAlBgNVBAoTHm1rY2VydCBkZXZlbG9wbWVudCBjZXJ0aWZpY2F0ZTE4MDYGA1UE + CwwvZGFua29jb2pAZGFua29jb2ouYy5nb29nbGVycy5jb20gKERhbmllbCBLb2Nv + aikwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC8Xer29Ko4WlI8zyGM + yUdPgSrcSZn8sreNRpZ3shAwJG96XrDdmPtCaPDPw+YN0nplrYmQ5SxceitmraBU + l1PpEgPje/JxfLMtRsk1S7gfKMW24gqsleraV9ZBqin9EeroefmIaDv/otAKGjV6 + Ty/j5rZl49vMMLeRDs2rN9Oi9ukdaoMWNXyPNTpm1yt4b8PB+ZVVNYKHLb2rHf1+ + Fa1NBLMixtBLd/UquNmlJzSNBoulqbBlmyObbGEwMaxI7KHNbP88YmGhp5KMcWo/ + 2PC13fSM71OiuaLUoHRG9JfEWqya9NtmNNhnf1KTQXwA2u5Fe7Wc+4mRhdjPBOP2 + 6NSTAgMBAAGjXjBcMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggrBgEFBQcD + ATAfBgNVHSMEGDAWgBQkq9fd2gnVl5uTLLvccD36dkSHGDAUBgNVHREEDTALggls + b2NhbGhvc3QwDQYJKoZIhvcNAQELBQADggGBAJxVLCW415Lq/29QRWfidcBtGfGF + Egf9s9j/M9YLknpRGe4OTMWMES0MFnOyxmLHKdBAxXhV0tDtmSN3TZXNtI/f0A7D + dUJAoAJsiEwkkBIyh6Q4xLe1MR8XUVQi18DDz74VZa6ZMkffWhhoKhLA8LG35Agr + wnWQFeBw5giO9JWTnAC5jiqtz+wMD+avspewdZlvBF0M6cmsRpVX1gkTi4Rod06O + wMI6FHlR9P7zIEYzIIbN0129/bR1pVjOZSX+PISKhTPDYU/AvFX/L7s4Zzb9RhD2 + kSx2XwQRXQIeL7jE7uCriM6nlaiyWi86c8EhzDdvpLUxhgmgK7V2Oq7CUly6recx + Vy1bllpso+ZrW5h0bifMRI9ShPZkBdYOfr6GPtxPHBJieTMzaQWtp4G34/e4/71+ + QJu4D31p6AhQJitOaNwng0U31E+HLJNh/hb4YEO2R6FxXnqd05AU1kGp7u6Me76N + vhZzX/nWZUgSmC+c7FqxyP1rjnosG1NEpR7HAQ== + -----END CERTIFICATE----- + + ``` ## Configuration @@ -65,12 +138,14 @@ descriptions, please refer to `../modules/buyer/service_vars.tf` and anything, although naming it after your environment may be convenient. Example: `my_env` 1. Copy either ./buyer or ./seller to your directory. Example: +```bash |-- environment | |-- demo | | |-- buyer | | `-- seller | `-- my_env | `-- seller +``` 1. Set the copied buyer or seller directory as your new working directory. 1. Modify all of the variables in buyer.tf or seller.tf. diff --git a/production/deploy/gcp/terraform/environment/demo/buyer/buyer.tf b/production/deploy/gcp/terraform/environment/demo/buyer/buyer.tf index bf43e450..f4cd5a34 100644 --- a/production/deploy/gcp/terraform/environment/demo/buyer/buyer.tf +++ b/production/deploy/gcp/terraform/environment/demo/buyer/buyer.tf @@ -33,6 +33,11 @@ resource "google_compute_project_metadata" "default" { } } +# See README.md for instructions on how to use the secrets module. +module "secrets" { + source = "../../../modules/secrets" +} + module "buyer" { source = "../../../modules/buyer" environment = local.environment @@ -96,23 +101,23 @@ module "buyer" { # Reach out to the Privacy Sandbox B&A team to enroll with Coordinators. # More information on enrollment can be found here: https://github.com/privacysandbox/fledge-docs/blob/main/bidding_auction_services_api.md#enroll-with-coordinators PUBLIC_KEY_ENDPOINT = "https://publickeyservice.pa.gcp.privacysandboxservices.com/.well-known/protected-auction/v1/public-keys" - PRIMARY_COORDINATOR_PRIVATE_KEY_ENDPOINT = "https://privatekeyservice-a.pa-1.gcp.privacysandboxservices.com/v1alpha/encryptionKeys" - SECONDARY_COORDINATOR_PRIVATE_KEY_ENDPOINT = "https://privatekeyservice-b.pa-2.gcp.privacysandboxservices.com/v1alpha/encryptionKeys" - PRIMARY_COORDINATOR_ACCOUNT_IDENTITY = "a-opverifiedusr@ps-pa-coord-prd-gg-wif.iam.gserviceaccount.com" - SECONDARY_COORDINATOR_ACCOUNT_IDENTITY = "b-opverifiedusr@ps-pa-coord-prd-gg-wif.iam.gserviceaccount.com" + PRIMARY_COORDINATOR_PRIVATE_KEY_ENDPOINT = "https://privatekeyservice-a.pa-3.gcp.privacysandboxservices.com/v1alpha/encryptionKeys" + SECONDARY_COORDINATOR_PRIVATE_KEY_ENDPOINT = "https://privatekeyservice-b.pa-4.gcp.privacysandboxservices.com/v1alpha/encryptionKeys" + PRIMARY_COORDINATOR_ACCOUNT_IDENTITY = "a-opverifiedusr@ps-pa-coord-prd-g3p-wif.iam.gserviceaccount.com" + SECONDARY_COORDINATOR_ACCOUNT_IDENTITY = "b-opverifiedusr@ps-prod-pa-type2-fe82.iam.gserviceaccount.com" PRIMARY_COORDINATOR_REGION = "us-central1" SECONDARY_COORDINATOR_REGION = "us-central1" - GCP_PRIMARY_WORKLOAD_IDENTITY_POOL_PROVIDER = "projects/787276892073/locations/global/workloadIdentityPools/a-opwip/providers/a-opwip-pvdr" - GCP_SECONDARY_WORKLOAD_IDENTITY_POOL_PROVIDER = "projects/787276892073/locations/global/workloadIdentityPools/b-opwip/providers/b-opwip-pvdr" - GCP_PRIMARY_KEY_SERVICE_CLOUD_FUNCTION_URL = "https://a-us-central1-encryption-key-service-cloudfunctio-mik44m5f7q-uc.a.run.app" - GCP_SECONDARY_KEY_SERVICE_CLOUD_FUNCTION_URL = "https://b-us-central1-encryption-key-service-cloudfunctio-amv3tcudsq-uc.a.run.app" + GCP_PRIMARY_WORKLOAD_IDENTITY_POOL_PROVIDER = "projects/732552956908/locations/global/workloadIdentityPools/a-opwip/providers/a-opwip-pvdr" + GCP_SECONDARY_WORKLOAD_IDENTITY_POOL_PROVIDER = "projects/99438709206/locations/global/workloadIdentityPools/b-opwip/providers/b-opwip-pvdr" + GCP_PRIMARY_KEY_SERVICE_CLOUD_FUNCTION_URL = "https://a-us-central1-encryption-key-service-cloudfunctio-j27wiaaz5q-uc.a.run.app" + GCP_SECONDARY_KEY_SERVICE_CLOUD_FUNCTION_URL = "https://b-us-central1-encryption-key-service-cloudfunctio-wdqaqbifva-uc.a.run.app" PRIVATE_KEY_CACHE_TTL_SECONDS = "3974400" KEY_REFRESH_FLOW_RUN_FREQUENCY_SECONDS = "20000" - BFE_TLS_KEY = "" # You can either set this here or via a secrets.auto.tfvars. - BFE_TLS_CERT = "" # You can either set this here or via a secrets.auto.tfvars. - MAX_ALLOWED_SIZE_DEBUG_URL_BYTES = "" # Example: "65536" - MAX_ALLOWED_SIZE_ALL_DEBUG_URLS_KB = "" # Example: "3000" + BFE_TLS_KEY = module.secrets.tls_key # You may remove the secrets module and instead either inline or use an auto.tfvars for this variable. + BFE_TLS_CERT = module.secrets.tls_cert # You may remove the secrets module and instead either inline or use an auto.tfvars for this variable. + MAX_ALLOWED_SIZE_DEBUG_URL_BYTES = "" # Example: "65536" + MAX_ALLOWED_SIZE_ALL_DEBUG_URLS_KB = "" # Example: "3000" INFERENCE_SIDECAR_BINARY_PATH = "" # Example: "/server/bin/inference_sidecar" INFERENCE_MODEL_BUCKET_NAME = "" # Example: "" @@ -129,7 +134,7 @@ module "buyer" { vm_startup_delay_seconds = 200 # Example: 200 cpu_utilization_percent = 0.6 # Example: 0.6 use_confidential_space_debug_image = false # Example: false - tee_impersonate_service_accounts = "a-opallowedusr@ps-pa-coord-prd-gg-svcacc.iam.gserviceaccount.com,b-opallowedusr@ps-pa-coord-prd-gg-svcacc.iam.gserviceaccount.com" + tee_impersonate_service_accounts = "a-opallowedusr@ps-pa-coord-prd-g3p-svcacc.iam.gserviceaccount.com,b-opallowedusr@ps-prod-pa-type2-fe82.iam.gserviceaccount.com" collector_service_port = 4317 collector_startup_script = templatefile("../../../services/autoscaling/collector_startup.tftpl", { collector_port = 4317 diff --git a/production/deploy/gcp/terraform/environment/demo/seller/seller.tf b/production/deploy/gcp/terraform/environment/demo/seller/seller.tf index 7ccff76d..fab9eb2c 100644 --- a/production/deploy/gcp/terraform/environment/demo/seller/seller.tf +++ b/production/deploy/gcp/terraform/environment/demo/seller/seller.tf @@ -34,6 +34,11 @@ resource "google_compute_project_metadata" "default" { } } +# See README.md for instructions on how to use the secrets module. +module "secrets" { + source = "../../../modules/secrets" +} + module "seller" { source = "../../../modules/seller" environment = local.environment @@ -59,6 +64,7 @@ module "seller" { SELLER_ORIGIN_DOMAIN = "" # Example: "https://securepubads.g.doubleclick.net" KEY_VALUE_SIGNALS_HOST = "" # Example: "https://pubads.g.doubleclick.net/td/sts" BUYER_SERVER_HOSTS = "" # Example: "{ \"https://example-bidder.com\": { \"url\": \"dns:///bidding-service-host:443\", \"cloudPlatform\": \"GCP\" } }" + SELLER_CLOUD_PLATFORMS_MAP = "" # Example: "{ \"https://partner-seller1.com\": "GCP", \"https://partner-seller2.com\": "AWS"}" ENABLE_SELLER_FRONTEND_BENCHMARKING = "" # Example: "false" ENABLE_AUCTION_COMPRESSION = "" # Example: "false" ENABLE_BUYER_COMPRESSION = "" # Example: "false" @@ -68,6 +74,7 @@ module "seller" { CREATE_NEW_EVENT_ENGINE = "" # Example: "false" SELLER_CODE_FETCH_CONFIG = "" # Example: # "{ + # "fetchMode": 0, # "auctionJsPath": "", # "auctionJsUrl": "https://example.com/scoreAd.js", # "urlFetchPeriodMs": 13000000, @@ -100,23 +107,23 @@ module "seller" { }" EOF PUBLIC_KEY_ENDPOINT = "https://publickeyservice.pa.gcp.privacysandboxservices.com/.well-known/protected-auction/v1/public-keys" - PRIMARY_COORDINATOR_PRIVATE_KEY_ENDPOINT = "https://privatekeyservice-a.pa-1.gcp.privacysandboxservices.com/v1alpha/encryptionKeys" - SECONDARY_COORDINATOR_PRIVATE_KEY_ENDPOINT = "https://privatekeyservice-b.pa-2.gcp.privacysandboxservices.com/v1alpha/encryptionKeys" - PRIMARY_COORDINATOR_ACCOUNT_IDENTITY = "a-opverifiedusr@ps-pa-coord-prd-gg-wif.iam.gserviceaccount.com" - SECONDARY_COORDINATOR_ACCOUNT_IDENTITY = "b-opverifiedusr@ps-pa-coord-prd-gg-wif.iam.gserviceaccount.com" + PRIMARY_COORDINATOR_PRIVATE_KEY_ENDPOINT = "https://privatekeyservice-a.pa-3.gcp.privacysandboxservices.com/v1alpha/encryptionKeys" + SECONDARY_COORDINATOR_PRIVATE_KEY_ENDPOINT = "https://privatekeyservice-b.pa-4.gcp.privacysandboxservices.com/v1alpha/encryptionKeys" + PRIMARY_COORDINATOR_ACCOUNT_IDENTITY = "a-opverifiedusr@ps-pa-coord-prd-g3p-wif.iam.gserviceaccount.com" + SECONDARY_COORDINATOR_ACCOUNT_IDENTITY = "b-opverifiedusr@ps-prod-pa-type2-fe82.iam.gserviceaccount.com" PRIMARY_COORDINATOR_REGION = "us-central1" SECONDARY_COORDINATOR_REGION = "us-central1" - GCP_PRIMARY_WORKLOAD_IDENTITY_POOL_PROVIDER = "projects/787276892073/locations/global/workloadIdentityPools/a-opwip/providers/a-opwip-pvdr" - GCP_SECONDARY_WORKLOAD_IDENTITY_POOL_PROVIDER = "projects/787276892073/locations/global/workloadIdentityPools/b-opwip/providers/b-opwip-pvdr" - GCP_PRIMARY_KEY_SERVICE_CLOUD_FUNCTION_URL = "https://a-us-central1-encryption-key-service-cloudfunctio-mik44m5f7q-uc.a.run.app" - GCP_SECONDARY_KEY_SERVICE_CLOUD_FUNCTION_URL = "https://b-us-central1-encryption-key-service-cloudfunctio-amv3tcudsq-uc.a.run.app" + GCP_PRIMARY_WORKLOAD_IDENTITY_POOL_PROVIDER = "projects/732552956908/locations/global/workloadIdentityPools/a-opwip/providers/a-opwip-pvdr" + GCP_SECONDARY_WORKLOAD_IDENTITY_POOL_PROVIDER = "projects/99438709206/locations/global/workloadIdentityPools/b-opwip/providers/b-opwip-pvdr" + GCP_PRIMARY_KEY_SERVICE_CLOUD_FUNCTION_URL = "https://a-us-central1-encryption-key-service-cloudfunctio-j27wiaaz5q-uc.a.run.app" + GCP_SECONDARY_KEY_SERVICE_CLOUD_FUNCTION_URL = "https://b-us-central1-encryption-key-service-cloudfunctio-wdqaqbifva-uc.a.run.app" PRIVATE_KEY_CACHE_TTL_SECONDS = "3974400" KEY_REFRESH_FLOW_RUN_FREQUENCY_SECONDS = "20000" - SFE_TLS_KEY = "" # You can either set this here or via a secrets.auto.tfvars. - SFE_TLS_CERT = "" # You can either set this here or via a secrets.auto.tfvars. - MAX_ALLOWED_SIZE_DEBUG_URL_BYTES = "" # Example: "65536" - MAX_ALLOWED_SIZE_ALL_DEBUG_URLS_KB = "" # Example: "3000" + SFE_TLS_KEY = module.secrets.tls_key # You may remove the secrets module and instead either inline or use an auto.tfvars for this variable. + SFE_TLS_CERT = module.secrets.tls_cert # You may remove the secrets module and instead either inline or use an auto.tfvars for this variable. + MAX_ALLOWED_SIZE_DEBUG_URL_BYTES = "" # Example: "65536" + MAX_ALLOWED_SIZE_ALL_DEBUG_URLS_KB = "" # Example: "3000" } # Please manually create a Google Cloud domain name, dns zone, and SSL certificate. @@ -128,7 +135,7 @@ module "seller" { vm_startup_delay_seconds = 200 # Example: 200 cpu_utilization_percent = 0.6 # Example: 0.6 use_confidential_space_debug_image = false # Example: false - tee_impersonate_service_accounts = "a-opallowedusr@ps-pa-coord-prd-gg-svcacc.iam.gserviceaccount.com,b-opallowedusr@ps-pa-coord-prd-gg-svcacc.iam.gserviceaccount.com" + tee_impersonate_service_accounts = "a-opallowedusr@ps-pa-coord-prd-g3p-svcacc.iam.gserviceaccount.com,b-opallowedusr@ps-prod-pa-type2-fe82.iam.gserviceaccount.com" collector_service_port = 4317 collector_startup_script = templatefile("../../../services/autoscaling/collector_startup.tftpl", { collector_port = 4317 diff --git a/production/deploy/gcp/terraform/modules/secrets/secrets.tf b/production/deploy/gcp/terraform/modules/secrets/secrets.tf new file mode 100644 index 00000000..586583e6 --- /dev/null +++ b/production/deploy/gcp/terraform/modules/secrets/secrets.tf @@ -0,0 +1,35 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +data "google_secret_manager_secret_version" "tls_key" { + provider = google-beta + secret = "envoy-tls-termination-key" +} + +data "google_secret_manager_secret_version" "tls_cert" { + provider = google-beta + secret = "envoy-tls-termination-cert" +} + +output "tls_key" { + value = data.google_secret_manager_secret_version.tls_key.secret_data + description = "The TLS Private Key used by BFE/SFE to terminate connections from the load balancer." + sensitive = true +} + +output "tls_cert" { + value = data.google_secret_manager_secret_version.tls_cert.secret_data + description = "The TLS Private Certificate used by BFE/SFE. May be self-signed, the parent chain is not validated." + sensitive = true +} diff --git a/production/packaging/aws/auction_service/BUILD b/production/packaging/aws/auction_service/BUILD index 461f6624..4264e89e 100644 --- a/production/packaging/aws/auction_service/BUILD +++ b/production/packaging/aws/auction_service/BUILD @@ -118,7 +118,11 @@ container_image( layers = [ ":server_binary_layer", ":kmstool_enclave_cli_layer", - ], + ] + select({ + "//:e2e_build": [ + ], + "//conditions:default": [], + }), tars = [ "@google_privacysandbox_servers_common//src/aws/proxy:libnsm_and_proxify_tar", ], diff --git a/production/packaging/aws/bidding_service/BUILD b/production/packaging/aws/bidding_service/BUILD index dbabb7bc..f0c9ce37 100644 --- a/production/packaging/aws/bidding_service/BUILD +++ b/production/packaging/aws/bidding_service/BUILD @@ -129,7 +129,11 @@ container_image( layers = [ ":server_binary_layer", ":kmstool_enclave_cli_layer", - ], + ] + select({ + "//:e2e_build": [ + ], + "//conditions:default": [], + }), tars = [ "@google_privacysandbox_servers_common//src/aws/proxy:libnsm_and_proxify_tar", ], diff --git a/production/packaging/aws/build_and_test b/production/packaging/aws/build_and_test index 037395a6..a0c38ca9 100755 --- a/production/packaging/aws/build_and_test +++ b/production/packaging/aws/build_and_test @@ -151,6 +151,9 @@ bazel ${BAZEL_STARTUP_ARGS} run ${BAZEL_EXTRA_ARGS} //production/packaging/aws/c "${WORKSPACE}"/builders/tools/normalize-dist if [[ -n ${AMI_REGIONS[0]} ]]; then + if [[ -v KOKORO_ARTIFACTS_DIR ]]; then + sudo apt-get install -y unzip + fi unzip -o -d "${DIST}"/aws "${DIST}"/aws/nonenclave_artifacts.zip unzip -o -d "${DIST}"/aws "${DIST}"/aws/artifacts_proto.zip printf "==== build AWS AMI (using packer) =====\n" diff --git a/production/packaging/aws/buyer_frontend_service/BUILD b/production/packaging/aws/buyer_frontend_service/BUILD index 6884dfaa..cd2ba167 100644 --- a/production/packaging/aws/buyer_frontend_service/BUILD +++ b/production/packaging/aws/buyer_frontend_service/BUILD @@ -118,7 +118,11 @@ container_image( layers = [ ":server_binary_layer", ":kmstool_enclave_cli_layer", - ], + ] + select({ + "//:e2e_build": [ + ], + "//conditions:default": [], + }), tars = [ "@google_privacysandbox_servers_common//src/aws/proxy:libnsm_and_proxify_tar", ], diff --git a/production/packaging/aws/seller_frontend_service/BUILD b/production/packaging/aws/seller_frontend_service/BUILD index 8ababfc3..92d24873 100644 --- a/production/packaging/aws/seller_frontend_service/BUILD +++ b/production/packaging/aws/seller_frontend_service/BUILD @@ -118,7 +118,11 @@ container_image( layers = [ ":server_binary_layer", ":kmstool_enclave_cli_layer", - ], + ] + select({ + "//:e2e_build": [ + ], + "//conditions:default": [], + }), tars = [ "@google_privacysandbox_servers_common//src/aws/proxy:libnsm_and_proxify_tar", ], diff --git a/services/auction_service/BUILD b/services/auction_service/BUILD index 49347cce..d58f45a6 100644 --- a/services/auction_service/BUILD +++ b/services/auction_service/BUILD @@ -33,10 +33,7 @@ cc_library( hdrs = [ "auction_constants.h", ], - visibility = [ - "//services/auction_service:__pkg__", - "//services/auction_service/utils:__pkg__", - ], + visibility = ["//visibility:public"], ) cc_library( @@ -284,16 +281,16 @@ cc_binary( visibility = ["//visibility:public"], deps = [ ":auction_code_fetch_config_cc_proto", + ":auction_constants", ":auction_service", + ":seller_code_fetch_manager", "//api:bidding_auction_servers_cc_grpc_proto", "//api:bidding_auction_servers_cc_proto", "//services/auction_service/benchmarking:score_ads_benchmarking_logger", "//services/auction_service/benchmarking:score_ads_no_op_logger", - "//services/auction_service/code_wrapper:seller_code_wrapper", "//services/auction_service/data:runtime_config", "//services/common/clients/config:config_client_util", "//services/common/clients/http:multi_curl_http_fetcher_async", - "//services/common/code_fetch:periodic_code_fetcher", "//services/common/encryption:crypto_client_factory", "//services/common/encryption:key_fetcher_factory", "//services/common/telemetry:configure_telemetry", @@ -306,6 +303,7 @@ cc_binary( "@com_google_absl//absl/strings", "@google_privacysandbox_servers_common//src/concurrent:executor", "@google_privacysandbox_servers_common//src/encryption/key_fetcher:key_fetcher_manager", + "@google_privacysandbox_servers_common//src/public/cpio/interface/blob_storage_client", "@google_privacysandbox_servers_common//src/util:rlimit_core_config", "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", ], @@ -350,6 +348,7 @@ cc_library( "//services/common/encryption:mock_crypto_client_wrapper", "//services/common/test:mocks", "//services/common/test:random", + "//services/common/util:proto_util", "@com_google_absl//absl/strings", ], ) diff --git a/services/auction_service/auction_constants.h b/services/auction_service/auction_constants.h index 42758080..6f311a95 100644 --- a/services/auction_service/auction_constants.h +++ b/services/auction_service/auction_constants.h @@ -44,6 +44,9 @@ constexpr char kScoreAdBlobVersion[] = "v1"; constexpr char kIGOwnerPropertyForScoreAd[] = "interestGroupOwner"; constexpr char kTopWindowHostnamePropertyForScoreAd[] = "topWindowHostname"; +// TODO(b/306257710): Update to differentiate from kScoreAdBlobVersion. +constexpr char kReportingBlobVersion[] = "v1"; + constexpr int kArgSizeWithWrapper = 7; // Acceptable error margin for float currency comparisons. // Arbitrarily specified to be one one-hundred-thousandth, diff --git a/services/auction_service/auction_main.cc b/services/auction_service/auction_main.cc index 50186696..51527c2a 100644 --- a/services/auction_service/auction_main.cc +++ b/services/auction_service/auction_main.cc @@ -33,22 +33,24 @@ #include "grpcpp/grpcpp.h" #include "grpcpp/health_check_service_interface.h" #include "services/auction_service/auction_code_fetch_config.pb.h" +#include "services/auction_service/auction_constants.h" #include "services/auction_service/auction_service.h" #include "services/auction_service/benchmarking/score_ads_benchmarking_logger.h" #include "services/auction_service/benchmarking/score_ads_no_op_logger.h" #include "services/auction_service/code_wrapper/seller_code_wrapper.h" #include "services/auction_service/data/runtime_config.h" #include "services/auction_service/runtime_flags.h" +#include "services/auction_service/seller_code_fetch_manager.h" #include "services/common/clients/config/trusted_server_config_client.h" #include "services/common/clients/config/trusted_server_config_client_util.h" #include "services/common/clients/http/multi_curl_http_fetcher_async.h" -#include "services/common/code_fetch/periodic_code_fetcher.h" #include "services/common/encryption/crypto_client_factory.h" #include "services/common/encryption/key_fetcher_factory.h" #include "services/common/telemetry/configure_telemetry.h" #include "src/concurrent/event_engine_executor.h" #include "src/core/lib/event_engine/default_event_engine.h" #include "src/encryption/key_fetcher/key_fetcher_manager.h" +#include "src/public/cpio/interface/blob_storage_client/blob_storage_client_interface.h" #include "src/public/cpio/interface/cpio.h" #include "src/util/rlimit_core_config.h" #include "src/util/status_macro/status_macros.h" @@ -76,6 +78,7 @@ ABSL_FLAG(std::optional, enable_report_win_input_noising, std::nullopt, namespace privacy_sandbox::bidding_auction_servers { +using ::google::scp::cpio::BlobStorageClientFactory; using ::google::scp::cpio::Cpio; using ::google::scp::cpio::CpioOptions; using ::google::scp::cpio::LogOption; @@ -143,45 +146,13 @@ absl::StatusOr GetConfigClient( server_common::log::PS_VLOG_IS_ON( 0, config_client.GetIntParameter(PS_VERBOSITY)); - PS_VLOG(1) << "Protected App Signals support enabled on the service: " - << config_client.GetBooleanParameter(ENABLE_PROTECTED_APP_SIGNALS); - PS_VLOG(1) << "Successfully constructed the config client."; + PS_LOG(INFO) << "Protected App Signals support enabled on the service: " + << config_client.GetBooleanParameter( + ENABLE_PROTECTED_APP_SIGNALS); + PS_LOG(INFO) << "Successfully constructed the config client."; return config_client; } -inline void CollectBuyerOriginEndpoints( - const google::protobuf::Map& - buyer_reporting_code_url, - std::vector& buyer_origins, - std::vector& endpoints) { - for (const auto& [buyer_origin, report_win_url] : buyer_reporting_code_url) { - buyer_origins.push_back(buyer_origin); - endpoints.push_back(report_win_url); - } -} - -// Gets code blob mapping from buyer origin => code blob fetched using the code -// fetcher. Note: Caller needs to ensure that the `buyer_origins` and -// `ad_tech_code_blobs` are in sync with each other and the `first` and `last` -// indices are in bound. -absl::flat_hash_map GetCodeBlobMap( - int first, int last, const std::vector& buyer_origins, - const std::vector& ad_tech_code_blobs, - absl::string_view data_type) { - absl::flat_hash_map code_blob_map; - for (unsigned int ind = first; ind < last; ind++) { - if (auto [unused_it, inserted] = code_blob_map.try_emplace( - buyer_origins[ind], ad_tech_code_blobs[ind]); - !inserted) { - PS_VLOG(0) << "Malformed config for '" << data_type - << "': Possible duplicate entries for buyer: " - << buyer_origins[ind] - << ", existing map size: " << code_blob_map.size(); - } - } - return code_blob_map; -} - // Brings up the gRPC AuctionService on FLAGS_port. absl::Status RunServer() { TrustedServerConfigUtil config_util(absl::GetFlag(FLAGS_init_config_client)); @@ -209,10 +180,6 @@ absl::Status RunServer() { std::unique_ptr executor = std::make_unique( grpc_event_engine::experimental::CreateEventEngine()); - std::unique_ptr http_fetcher = - std::make_unique(executor.get()); - - std::unique_ptr code_fetcher; // Convert Json string into a AuctionCodeBlobFetcherConfig proto auction_service::SellerCodeFetchConfig code_fetch_proto; @@ -231,79 +198,27 @@ absl::Status RunServer() { code_fetch_proto.enable_report_result_url_generation(); bool enable_report_win_url_generation = code_fetch_proto.enable_report_win_url_generation(); - std::vector endpoints; - std::vector buyer_origins; - CollectBuyerOriginEndpoints(code_fetch_proto.buyer_report_win_js_urls(), - buyer_origins, endpoints); - const int num_protected_audience_endpoints = buyer_origins.size(); const bool enable_protected_app_signals = config_client.GetBooleanParameter(ENABLE_PROTECTED_APP_SIGNALS); - if (enable_protected_app_signals) { - CollectBuyerOriginEndpoints( - code_fetch_proto.protected_app_signals_buyer_report_win_js_urls(), - buyer_origins, endpoints); - } - // Starts periodic code blob fetching from an arbitrary url only if js_url is - // specified - std::string js_url = code_fetch_proto.auction_js_url(); - if (!js_url.empty()) { - endpoints.push_back(js_url); - auto wrap_code = - [num_protected_audience_endpoints, enable_protected_app_signals, - enable_report_result_url_generation, enable_report_win_url_generation, - buyer_origins](const std::vector& ad_tech_code_blobs) { - CHECK(buyer_origins.size() == ad_tech_code_blobs.size() - 1) - << "Error fetching code blobs from buyer. Buyer size:" - << buyer_origins.size() - << " and blobs count:" << ad_tech_code_blobs.size(); - // Collect code blobs for protected audience. - auto buyer_origin_code_map = GetCodeBlobMap( - /*first=*/0, num_protected_audience_endpoints, buyer_origins, - ad_tech_code_blobs, "buyer_report_win_js_urls"); - // Collect code blobs for protected app signals. - absl::flat_hash_map - protected_app_signals_buyer_origin_code_map; - if (enable_protected_app_signals) { - protected_app_signals_buyer_origin_code_map = GetCodeBlobMap( - num_protected_audience_endpoints, buyer_origins.size(), - buyer_origins, ad_tech_code_blobs, - "protected_app_signals_buyer_report_win_js_urls"); - } - return GetSellerWrappedCode( - ad_tech_code_blobs.at(ad_tech_code_blobs.size() - 1), - enable_report_result_url_generation, enable_protected_app_signals, - enable_report_win_url_generation, buyer_origin_code_map, - protected_app_signals_buyer_origin_code_map); - }; - - code_fetcher = std::make_unique( - endpoints, absl::Milliseconds(code_fetch_proto.url_fetch_period_ms()), - http_fetcher.get(), &dispatcher, executor.get(), - absl::Milliseconds(code_fetch_proto.url_fetch_timeout_ms()), - std::move(wrap_code), "v1"); - - code_fetcher->Start(); - } else if (!code_fetch_proto.auction_js_path().empty()) { - std::ifstream ifs(code_fetch_proto.auction_js_path().data()); - std::string adtech_code_blob((std::istreambuf_iterator(ifs)), - (std::istreambuf_iterator())); - - adtech_code_blob = GetSellerWrappedCode( - adtech_code_blob, enable_report_result_url_generation, false, {}); - - PS_RETURN_IF_ERROR(dispatcher.LoadSync("v1", adtech_code_blob)) - << "Could not load Adtech untrusted code for scoring."; - } else { - return absl::UnavailableError( - "Code fetching config requires either a path or url."); - } + MultiCurlHttpFetcherAsync http_fetcher = + MultiCurlHttpFetcherAsync(executor.get()); + HttpFetcherAsync* seller_udf_fetcher = &http_fetcher; + HttpFetcherAsync* buyer_reporting_udf_fetcher = &http_fetcher; + SellerCodeFetchManager code_fetch_manager( + BlobStorageClientFactory::Create(), executor.get(), seller_udf_fetcher, + buyer_reporting_udf_fetcher, &dispatcher, code_fetch_proto, + enable_protected_app_signals); + PS_RETURN_IF_ERROR(code_fetch_manager.Init()) + << "Failed to initialize UDF fetch."; bool enable_auction_service_benchmark = config_client.GetBooleanParameter(ENABLE_AUCTION_SERVICE_BENCHMARK); InitTelemetry(config_util, config_client, metric::kAs); + // TODO(b/334909636) : AsyncReporter should not own HttpFetcher, + // this needs to be decoupled so we can test different configurations. std::unique_ptr async_reporter = std::make_unique( std::make_unique(executor.get())); @@ -326,6 +241,11 @@ absl::Status RunServer() { runtime_config); }; + std::string default_code_version = + code_fetch_proto.fetch_mode() == auction_service::FETCH_MODE_BUCKET + ? code_fetch_proto.auction_js_bucket_default_blob() + : kScoreAdBlobVersion; + AuctionServiceRuntimeConfig runtime_config = { .enable_seller_debug_url_generation = enable_seller_debug_url_generation, .roma_timeout_ms = @@ -341,7 +261,8 @@ absl::Status RunServer() { .max_allowed_size_debug_url_bytes = config_client.GetIntParameter(MAX_ALLOWED_SIZE_DEBUG_URL_BYTES), .max_allowed_size_all_debug_urls_kb = - config_client.GetIntParameter(MAX_ALLOWED_SIZE_ALL_DEBUG_URLS_KB)}; + config_client.GetIntParameter(MAX_ALLOWED_SIZE_ALL_DEBUG_URLS_KB), + .default_code_version = default_code_version}; AuctionService auction_service( std::move(score_ads_reactor_factory), CreateKeyFetcherManager(config_client, /* public_key_fetcher= */ nullptr), @@ -376,12 +297,11 @@ absl::Status RunServer() { } // Wait for the server to shutdown. Note that some other thread must be // responsible for shutting down the server for this call to ever return. - PS_VLOG(1) << "Server listening on " << server_address; + PS_LOG(INFO) << "Server listening on " << server_address; server->Wait(); // Ends periodic code blob fetching from an arbitrary url. - if (code_fetcher) { - code_fetcher->End(); - } + PS_RETURN_IF_ERROR(code_fetch_manager.End()) + << "Error shutting down UDF fetcher."; return absl::OkStatus(); } } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/auction_service_integration_test.cc b/services/auction_service/auction_service_integration_test.cc index 3b0d9ef2..8bb57185 100644 --- a/services/auction_service/auction_service_integration_test.cc +++ b/services/auction_service/auction_service_integration_test.cc @@ -20,6 +20,7 @@ #include "google/protobuf/text_format.h" #include "google/protobuf/util/message_differencer.h" #include "gtest/gtest.h" +#include "services/auction_service/auction_constants.h" #include "services/auction_service/auction_service.h" #include "services/auction_service/benchmarking/score_ads_benchmarking_logger.h" #include "services/auction_service/benchmarking/score_ads_no_op_logger.h" @@ -51,17 +52,28 @@ constexpr int kTestJoinCount = 5; constexpr char kTestBuyerSignals[] = R"([1,"test",[2]])"; constexpr char kTestAuctionSignals[] = R"([[3,"test",[4]]])"; constexpr char kTestConsentToken[] = "testConsentToken"; +constexpr char kEurosIsoCode[] = "EUR"; +constexpr char kPublisherHostname[] = "fenceStreetJournal.com"; +constexpr char kDefaultBarInterestGroupOwner[] = "barStandardAds.com"; constexpr char kTestReportWinUrlWithSignals[] = "http://test.com?seller=http://" "seller.com&interestGroupName=testInterestGroupName&adCost=2&" - "modelingSignals=4&recency=3&madeHighestScoringOtherBid=false&joinCount=5&" + "modelingSignals=4&recency=3&madeHighestScoringOtherBid=true&joinCount=5&" "signalsForWinner=testSignalsForWinner&perBuyerSignals=1,test,2&" "auctionSignals=3,test,4&desirability=undefined"; +constexpr char kTestReportWinUrlWithBuyerReportingId[] = + "http://test.com?seller=http://" + "seller.com&interestGroupName=undefined&adCost=2&" + "modelingSignals=4&recency=3&madeHighestScoringOtherBid=true&joinCount=5&" + "signalsForWinner=testSignalsForWinner&perBuyerSignals=1,test,2&" + "auctionSignals=3,test,4&desirability=undefined&buyerReportingId=" + "testBuyerReportingId"; + constexpr char kTestReportWinUrlWithNoising[] = "http://test.com?seller=http://" "seller.com&interestGroupName=testInterestGroupName&adCost=2&" - "madeHighestScoringOtherBid=false&" + "madeHighestScoringOtherBid=true&" "signalsForWinner=testSignalsForWinner&perBuyerSignals=1,test,2&" "auctionSignals=3,test,4&desirability=undefined"; @@ -289,7 +301,11 @@ void BuildScoreAdsRequest( std::string trusted_scoring_signals = R"json({"renderUrls":{"placeholder_url":[123])json"; for (int i = 0; i < desired_ad_count; i++) { - auto ad = MakeARandomAdWithBidMetadata(1, 10); + auto ad = MakeARandomAdWithBidMetadata(/*min_bid=*/1, /*max_bid=*/1); + ad.set_bid_currency(kEurosIsoCode); + ad.set_interest_group_owner(kDefaultBarInterestGroupOwner); + ad.set_render( + absl::StrFormat("%s/ads?id=%d", kDefaultBarInterestGroupOwner, i)); *raw_request.mutable_ad_bids()->Add() = ad; raw_request.mutable_per_buyer_signals()->try_emplace( ad.interest_group_owner(), MakeARandomString()); @@ -307,6 +323,7 @@ void BuildScoreAdsRequest( raw_request.set_enable_debug_reporting(enable_debug_reporting); } raw_request.set_seller(seller); + raw_request.set_publisher_hostname(kPublisherHostname); if (component_auction) { raw_request.set_top_level_seller(kTestTopLevelSeller); } @@ -328,17 +345,26 @@ void BuildScoreAdsRequestForReporting( absl::flat_hash_map* interest_group_to_ad, const TestBuyerReportingSignals& test_buyer_reporting_signals, bool enable_debug_reporting = false, int desired_ad_count = 90, - absl::string_view top_level_seller = "", const bool is_consented = false) { + absl::string_view top_level_seller = "", const bool is_consented = false, + absl::string_view buyer_reporting_id = "") { ScoreAdsRequest::ScoreAdsRawRequest raw_request; std::string trusted_scoring_signals = R"json({"renderUrls":{"placeholder_url":[123])json"; for (int i = 0; i < desired_ad_count; i++) { - auto ad = MakeARandomAdWithBidMetadata(1, 1); // fix bid value to 1 + ScoreAdsRequest::ScoreAdsRawRequest::AdWithBidMetadata ad; + ad = MakeARandomAdWithBidMetadata(/*min_bid=*/1, /*max_bid=*/1); + if (!buyer_reporting_id.empty()) { + ad.set_buyer_reporting_id(buyer_reporting_id); + } + ad.set_bid_currency(kEurosIsoCode); ad.set_ad_cost(test_buyer_reporting_signals.ad_cost); ad.set_modeling_signals(test_buyer_reporting_signals.modeling_signals); ad.set_join_count(test_buyer_reporting_signals.join_count); ad.set_interest_group_name( test_buyer_reporting_signals.interest_group_name); + ad.set_interest_group_owner(kDefaultBarInterestGroupOwner); + ad.set_render( + absl::StrFormat("%s/ads?id=%d", kDefaultBarInterestGroupOwner, i)); ad.set_recency(test_buyer_reporting_signals.recency); *raw_request.mutable_ad_bids()->Add() = ad; raw_request.mutable_per_buyer_signals()->try_emplace( @@ -357,6 +383,7 @@ void BuildScoreAdsRequestForReporting( raw_request.set_enable_debug_reporting(enable_debug_reporting); } raw_request.set_seller("http://seller.com"); + raw_request.set_publisher_hostname(kPublisherHostname); raw_request.set_auction_signals(test_buyer_reporting_signals.auction_signals); raw_request.mutable_consented_debug_config()->set_is_consented(is_consented); raw_request.mutable_consented_debug_config()->set_token(kTestConsentToken); @@ -412,13 +439,13 @@ TEST_F(AuctionServiceIntegrationTest, ScoresAdsWithCustomScoringLogic) { V8Dispatcher dispatcher; CodeDispatchClient client(dispatcher); ASSERT_TRUE(dispatcher.Init().ok()); - ASSERT_TRUE( - dispatcher - .LoadSync("v1", GetSellerWrappedCode( - std::string(js_code), - kEnableReportResultUrlGenerationFalse, - kEnableReportResultWinGenerationFalse, {})) - .ok()); + ASSERT_TRUE(dispatcher + .LoadSync(kScoreAdBlobVersion, + GetSellerWrappedCode( + std::string(js_code), + kEnableReportResultUrlGenerationFalse, + kEnableReportResultWinGenerationFalse, {})) + .ok()); std::unique_ptr async_reporter_ = std::make_unique( std::make_unique()); @@ -450,6 +477,7 @@ TEST_F(AuctionServiceIntegrationTest, ScoresAdsWithCustomScoringLogic) { auto key_fetcher_manager = CreateKeyFetcherManager(config_client, /* public_key_fetcher= */ nullptr); AuctionServiceRuntimeConfig auction_service_runtime_config; + auction_service_runtime_config.default_code_version = kScoreAdBlobVersion; AuctionService service( std::move(score_ads_reactor_factory), std::move(key_fetcher_manager), std::move(crypto_client), auction_service_runtime_config); @@ -510,7 +538,7 @@ void SellerCodeWrappingTestHelper( std::string wrapper_js_blob = GetSellerWrappedCode( adtech_code_blob, enable_report_result_url_generation, enable_report_win_url_generation, buyer_origin_code_map); - ASSERT_TRUE(dispatcher.LoadSync("v1", wrapper_js_blob).ok()); + ASSERT_TRUE(dispatcher.LoadSync(kScoreAdBlobVersion, wrapper_js_blob).ok()); auto score_ads_reactor_factory = [&client, async_reporter_ = std::move(async_reporter_)]( @@ -547,7 +575,8 @@ void SellerCodeWrappingTestHelper( enable_report_result_url_generation, .enable_report_win_url_generation = enable_report_win_url_generation, .enable_protected_app_signals = enable_protected_app_signals, - .enable_report_win_input_noising = enable_report_win_input_noising}; + .enable_report_win_input_noising = enable_report_win_input_noising, + .default_code_version = kScoreAdBlobVersion}; AuctionService service( std::move(score_ads_reactor_factory), std::move(key_fetcher_manager), std::move(crypto_client), auction_service_runtime_config); @@ -764,6 +793,53 @@ TEST_F(AuctionServiceIntegrationTest, kTestInteractionReportingUrl); } +TEST_F(AuctionServiceIntegrationTest, + AdScoreIncludesWinReportingUrlsAndBuyerReportingId) { + bool enable_seller_debug_url_generation = false; + bool enable_debug_reporting = false; + bool enable_adtech_code_logging = true; + bool enable_report_result_url_generation = true; + bool enable_report_win_url_generation = true; + int desired_ad_count = 90; + bool enable_report_win_input_noising = false; + bool is_consented = false; + TestBuyerReportingSignals test_buyer_reporting_signals; + ScoreAdsRequest request; + absl::flat_hash_map interest_group_to_ad; + BuildScoreAdsRequestForReporting(&request, &interest_group_to_ad, + test_buyer_reporting_signals, + enable_debug_reporting, desired_ad_count, "", + is_consented, kTestBuyerReportingId); + ScoreAdsResponse response; + SellerCodeWrappingTestHelper( + &request, &response, kSellerBaseCode, enable_seller_debug_url_generation, + enable_debug_reporting, enable_adtech_code_logging, + enable_report_result_url_generation, enable_report_win_url_generation, + enable_report_win_input_noising); + ScoreAdsResponse::ScoreAdsRawResponse raw_response; + raw_response.ParseFromString(response.response_ciphertext()); + const auto& scoredAd = raw_response.ad_score(); + EXPECT_EQ(scoredAd.buyer_reporting_id(), kTestBuyerReportingId); + EXPECT_GT(scoredAd.desirability(), 0); + EXPECT_EQ(scoredAd.win_reporting_urls() + .top_level_seller_reporting_urls() + .reporting_url(), + kTestReportResultUrl); + EXPECT_EQ(scoredAd.win_reporting_urls() + .top_level_seller_reporting_urls() + .interaction_reporting_urls() + .at(kTestInteractionEvent), + kTestInteractionReportingUrl); + EXPECT_EQ( + scoredAd.win_reporting_urls().buyer_reporting_urls().reporting_url(), + kTestReportWinUrlWithBuyerReportingId); + EXPECT_EQ(scoredAd.win_reporting_urls() + .buyer_reporting_urls() + .interaction_reporting_urls() + .at(kTestInteractionEvent), + kTestInteractionReportingUrl); +} + TEST_F(AuctionServiceIntegrationTest, ReportingSuccessfulWithNoising) { bool enable_seller_debug_url_generation = false; bool enable_debug_reporting = false; @@ -866,6 +942,9 @@ TEST_F(AuctionServiceIntegrationTest, test_buyer_reporting_signals, false, 90, kTestTopLevelSeller); ScoreAdsResponse response; + ScoreAdsRequest::ScoreAdsRawRequest score_ads_raw_request; + score_ads_raw_request.ParseFromString(request.request_ciphertext()); + ASSERT_FALSE(score_ads_raw_request.top_level_seller().empty()); SellerCodeWrappingTestHelper( &request, &response, kComponentAuctionCode, enable_seller_debug_url_generation, enable_debug_reporting, @@ -874,11 +953,19 @@ TEST_F(AuctionServiceIntegrationTest, ScoreAdsResponse::ScoreAdsRawResponse raw_response; raw_response.ParseFromString(response.response_ciphertext()); const auto& scoredAd = raw_response.ad_score(); - EXPECT_GT(scoredAd.desirability(), 0); + // Make sure that this the AdScore we expect before checking its reporting + // urls. + EXPECT_FLOAT_EQ(scoredAd.desirability(), 1); + EXPECT_FLOAT_EQ(scoredAd.buyer_bid(), 1); + EXPECT_EQ(scoredAd.buyer_bid_currency(), "EUR"); + EXPECT_FLOAT_EQ(scoredAd.bid(), 2); + EXPECT_FLOAT_EQ(scoredAd.incoming_bid_in_seller_currency(), 1.868); + EXPECT_EQ(scoredAd.bid_currency(), "USD"); + EXPECT_EQ(scoredAd.allow_component_auction(), true); EXPECT_EQ(scoredAd.win_reporting_urls() .component_seller_reporting_urls() .reporting_url(), - kTestComponentReportResultUrl); + kTestComponentReportResultUrlWithEverything); EXPECT_EQ(scoredAd.win_reporting_urls() .component_seller_reporting_urls() .interaction_reporting_urls() diff --git a/services/auction_service/code_wrapper/BUILD b/services/auction_service/code_wrapper/BUILD index 80a727c8..e6b11193 100644 --- a/services/auction_service/code_wrapper/BUILD +++ b/services/auction_service/code_wrapper/BUILD @@ -25,6 +25,7 @@ cc_library( "seller_code_wrapper.h", ], deps = [ + "//services/common/util:reporting_util", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/functional:any_invocable", "@com_google_absl//absl/log:absl_log", @@ -42,6 +43,7 @@ cc_test( deps = [ ":seller_code_wrapper", "//services/common/util:json_util", + "//services/common/util:reporting_util", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/strings", "@com_google_googletest//:gtest", diff --git a/services/auction_service/code_wrapper/buyer_reporting_fetcher.cc b/services/auction_service/code_wrapper/buyer_reporting_fetcher.cc index 87070169..761033d9 100644 --- a/services/auction_service/code_wrapper/buyer_reporting_fetcher.cc +++ b/services/auction_service/code_wrapper/buyer_reporting_fetcher.cc @@ -77,7 +77,7 @@ void BuyerReportingFetcher::PeriodicBuyerReportingFetchSync() { } if (requests.empty()) { - PS_VLOG(1) << "No buyer reporting UDFs to fetch in config."; + PS_LOG(ERROR) << "No buyer reporting UDFs to fetch in config."; return; } @@ -89,9 +89,9 @@ void BuyerReportingFetcher::PeriodicBuyerReportingFetchSync() { for (int i = 0; i < results.size(); i++) { auto& result = results[i]; if (!result.ok()) { - PS_VLOG(1) << "Failed origin " << buyer_origins[i] << " fetch at " - << requests[i].url - << " with status: " << result.status(); + PS_LOG(ERROR) << "Failed origin " << buyer_origins[i] + << " fetch at " << requests[i].url + << " with status: " << result.status(); } else { if (i < config_.buyer_report_win_js_urls().size()) { protected_auction_code_blob_per_origin_[buyer_origins[i]] = diff --git a/services/auction_service/code_wrapper/seller_code_wrapper.cc b/services/auction_service/code_wrapper/seller_code_wrapper.cc index 39b904a5..e0a6ac00 100644 --- a/services/auction_service/code_wrapper/seller_code_wrapper.cc +++ b/services/auction_service/code_wrapper/seller_code_wrapper.cc @@ -24,6 +24,7 @@ #include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" +#include "services/common/util/reporting_util.h" namespace privacy_sandbox::bidding_auction_servers { @@ -33,17 +34,6 @@ using ProcessReportWinBlob = absl::AnyInvocable; using TagToValue = std::vector>; -void AppendFeatureFlagValue(std::string& feature_flags, - absl::string_view feature_name, - bool is_feature_enabled) { - absl::string_view enable_feature = kFeatureDisabled; - if (is_feature_enabled) { - enable_feature = kFeatureEnabled; - } - feature_flags.append( - absl::StrCat("\"", feature_name, "\": ", enable_feature)); -} - // Returns the reportWin wrapper function that should be called from // reportingEntryFunction. The function name is // _reportWinWrapper buyer_origin is sanitized by @@ -124,7 +114,7 @@ std::string GetSellerWrappedCode( wrap_code.append(GetSellerWrappedCodeHelper( enable_report_result_url_generation, enable_report_win_url_generation, protected_app_signals_buyer_origin_code_map, - {{kSuffix, kProtectedAppSignalsTag}, {kExtraArgs, kEgressFeaturesTag}}, + {{kSuffix, kProtectedAppSignalsTag}, {kExtraArgs, kEgressPayloadTag}}, [](absl::string_view buyer_origin, absl::string_view buyer_code, std::string& report_win_blob) { ReplacePlaceholders( @@ -132,7 +122,7 @@ std::string GetSellerWrappedCode( {{kReportWinWrapperNamePlaceholder, GetProtectedAppSignalsReportWinFunctionName(buyer_origin)}, {kReportWinCodePlaceholder, buyer_code}, - {kExtraArgs, kEgressFeaturesTag}, + {kExtraArgs, kEgressPayloadTag}, {"reportWin(", "reportWinProtectedAppSignals("}, {"reportWin =", "reportWinProtectedAppSignals ="}}); })); @@ -152,14 +142,4 @@ std::string GetSellerWrappedCode( return wrap_code; } -std::string GetFeatureFlagJson(bool enable_logging, - bool enable_debug_url_generation) { - std::string feature_flags = "{"; - AppendFeatureFlagValue(feature_flags, kFeatureLogging, enable_logging); - feature_flags.append(","); - AppendFeatureFlagValue(feature_flags, kFeatureDebugUrlGeneration, - enable_debug_url_generation); - feature_flags.append("}"); - return feature_flags; -} } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/code_wrapper/seller_code_wrapper.h b/services/auction_service/code_wrapper/seller_code_wrapper.h index 20c6a76d..dc9ebf9e 100644 --- a/services/auction_service/code_wrapper/seller_code_wrapper.h +++ b/services/auction_service/code_wrapper/seller_code_wrapper.h @@ -23,18 +23,13 @@ namespace privacy_sandbox::bidding_auction_servers { -constexpr char kFeatureLogging[] = "enable_logging"; -constexpr char kFeatureDebugUrlGeneration[] = "enable_debug_url_generation"; - -constexpr char kFeatureDisabled[] = "false"; -constexpr char kFeatureEnabled[] = "true"; - inline constexpr char kReportWinWrapperNamePlaceholder[] = "$reportWinWrapperName"; inline constexpr char kExtraArgs[] = "$extraArgs"; inline constexpr char kSuffix[] = "$suffix"; inline constexpr char kProtectedAppSignalsTag[] = "ProtectedAppSignals"; -inline constexpr char kEgressFeaturesTag[] = "egressFeatures"; +inline constexpr char kEgressPayloadTag[] = + "egressPayload, temporaryUnlimitedEgressPayload"; inline constexpr char kReportWinCodePlaceholder[] = "$reportWinCode"; inline constexpr char kReportWinWrapperFunctionName[] = "reportWinWrapper"; @@ -152,7 +147,9 @@ inline constexpr absl::string_view kReportingEntryFunction = var auctionSignals = auctionConfig.auctionSignals var buyerReportingSignals = sellerReportingSignals delete buyerReportingSignals.desirability - buyerReportingSignals.interestGroupName = buyerReportingMetadata.interestGroupName + if(buyerReportingMetadata.hasOwnProperty("interestGroupName")){ + buyerReportingSignals.interestGroupName = buyerReportingMetadata.interestGroupName + } buyerReportingSignals.madeHighestScoringOtherBid = buyerReportingMetadata.madeHighestScoringOtherBid buyerReportingSignals.joinCount = buyerReportingMetadata.joinCount buyerReportingSignals.recency = buyerReportingMetadata.recency @@ -160,6 +157,9 @@ inline constexpr absl::string_view kReportingEntryFunction = perBuyerSignals = buyerReportingMetadata.perBuyerSignals buyerReportingSignals.seller = buyerReportingMetadata.seller buyerReportingSignals.adCost = buyerReportingMetadata.adCost + if(buyerReportingMetadata.hasOwnProperty("buyerReportingId")){ + buyerReportingSignals.buyerReportingId = buyerReportingMetadata.buyerReportingId + } // Absence of interest group indicates that this is a protected app // signals ad. if (buyerReportingMetadata.enableProtectedAppSignals && @@ -268,10 +268,6 @@ std::string GetSellerWrappedCode( const absl::flat_hash_map& protected_app_signals_buyer_origin_code_map); -// Returns a JSON string for feature flags to be used by the wrapper script. -std::string GetFeatureFlagJson(bool enable_logging, - bool enable_debug_url_generation); - } // namespace privacy_sandbox::bidding_auction_servers #endif // FLEDGE_SERVICES_SELLER_CODE_WRAPPER_H_ diff --git a/services/auction_service/code_wrapper/seller_code_wrapper_test.cc b/services/auction_service/code_wrapper/seller_code_wrapper_test.cc index c9f3cf97..b3c124e8 100644 --- a/services/auction_service/code_wrapper/seller_code_wrapper_test.cc +++ b/services/auction_service/code_wrapper/seller_code_wrapper_test.cc @@ -20,6 +20,7 @@ #include "rapidjson/document.h" #include "services/auction_service/code_wrapper/seller_code_wrapper_test_constants.h" #include "services/common/util/json_util.h" +#include "services/common/util/reporting_util.h" namespace privacy_sandbox::bidding_auction_servers { diff --git a/services/auction_service/code_wrapper/seller_code_wrapper_test_constants.h b/services/auction_service/code_wrapper/seller_code_wrapper_test_constants.h index 5da329a5..55110ef3 100644 --- a/services/auction_service/code_wrapper/seller_code_wrapper_test_constants.h +++ b/services/auction_service/code_wrapper/seller_code_wrapper_test_constants.h @@ -18,10 +18,17 @@ namespace privacy_sandbox::bidding_auction_servers { constexpr char kBuyerOrigin[] = "http://buyer1.com"; -constexpr char kTestReportResultUrl[] = "http://test.com"; -constexpr char kTestComponentReportResultUrl[] = +constexpr char kTestReportResultUrl[] = + "http://test.com&bid=1&bidCurrency=EUR&" + "highestScoringOtherBid=1&highestScoringOtherBidCurrency=???&" + "topWindowHostname=fenceStreetJournal.com&interestGroupOwner=" + "barStandardAds.com"; +constexpr char kTestComponentReportResultUrlWithEverything[] = "http://test.com&topLevelSeller=topLevelSeller&componentSeller=http://" - "seller.com&bid=1&modifiedBid=2"; + "seller.com&bid=1&bidCurrency=EUR&modifiedBid=2&modifiedBidCurrency=USD&" + "highestScoringOtherBid=1&highestScoringOtherBidCurrency=???&" + "topWindowHostname=fenceStreetJournal.com&interestGroupOwner=" + "barStandardAds.com"; constexpr char kTestComponentReportResultUrlWithNoModifiedBid[] = "http://test.com&topLevelSeller=topLevelSeller&componentSeller=http://" "seller.com&bid=1&modifiedBid=1"; @@ -31,7 +38,7 @@ constexpr char kTestReportWinUrl[] = "http://test.com?seller=http://" "seller.com&interestGroupName=testInterestGroupName&adCost=2&" "modelingSignals=4&recency=3&joinCount=5"; - +constexpr char kTestBuyerReportingId[] = "testBuyerReportingId"; constexpr absl::string_view kBuyerBaseCodeSimple = R"JS_CODE(reportWin = function(auctionSignals, perBuyerSignals, signalsForWinner, buyerReportingSignals, directFromSellerSignals){ @@ -53,11 +60,6 @@ constexpr absl::string_view kBuyerBaseCode = console.error("Missing seller in input to reportWin") return } - if(buyerReportingSignals.interestGroupName == "" || buyerReportingSignals.interestGroupName == undefined - || buyerReportingSignals.interestGroupName == null){ - console.error("Missing interestGroupName in input to reportWin") - return - } if(buyerReportingSignals.adCost == 0 || buyerReportingSignals.adCost == -1 || buyerReportingSignals.adCost == undefined || buyerReportingSignals.adCost == null){ @@ -70,8 +72,10 @@ constexpr absl::string_view kBuyerBaseCode = buyerReportingSignals.modelingSignals+"&recency="+buyerReportingSignals.recency+ "&madeHighestScoringOtherBid="+buyerReportingSignals.madeHighestScoringOtherBid+ "&joinCount="+buyerReportingSignals.joinCount+"&signalsForWinner="+signalsForWinner+ - "&perBuyerSignals="+perBuyerSignals+"&auctionSignals="+auctionSignals+"&desirability="+buyerReportingSignals.desirability; - + "&perBuyerSignals="+perBuyerSignals+"&auctionSignals="+auctionSignals+"&desirability="+buyerReportingSignals.desirability + if(buyerReportingSignals.hasOwnProperty("buyerReportingId")){ + reportWinUrl = reportWinUrl+"&buyerReportingId="+buyerReportingSignals.buyerReportingId + } console.log("Logging from ReportWin"); console.error("Logging error from ReportWin") console.warn("Logging warning from ReportWin") @@ -126,7 +130,7 @@ constexpr absl::string_view kBuyerBaseCodeWithValidation = constexpr absl::string_view kProtectedAppSignalsBuyerBaseCode = R"JS_CODE(reportWin = function(auctionSignals, perBuyerSignals, signalsForWinner, buyerReportingSignals, - directFromSellerSignals, egressFeatures){ + directFromSellerSignals, egressPayload, temporaryUnlimitedEgressPayload){ console.log("Testing Protected App Signals"); sendReportTo("http://test.com"); registerAdBeacon({"clickEvent":"http://click.com"}); @@ -154,7 +158,7 @@ constexpr absl::string_view kSellerBaseCode = R"JS_CODE( function reportResult(auctionConfig, sellerReportingSignals, directFromSellerSignals){ console.log("Logging from ReportResult"); if(sellerReportingSignals.topLevelSeller === undefined || sellerReportingSignals.topLevelSeller.length === 0){ - sendReportTo("http://test.com") + sendReportTo("http://test.com"+"&bid="+sellerReportingSignals.bid+"&bidCurrency="+sellerReportingSignals.bidCurrency+"&highestScoringOtherBid="+sellerReportingSignals.highestScoringOtherBid+"&highestScoringOtherBidCurrency="+sellerReportingSignals.highestScoringOtherBidCurrency+"&topWindowHostname="+sellerReportingSignals.topWindowHostname+"&interestGroupOwner="+sellerReportingSignals.interestGroupOwner) } else { sendReportTo("http://test.com&topLevelSeller="+sellerReportingSignals.topLevelSeller+"&componentSeller="+sellerReportingSignals.componentSeller) } @@ -177,9 +181,9 @@ constexpr absl::string_view kComponentAuctionCode = R"JS_CODE( function reportResult(auctionConfig, sellerReportingSignals, directFromSellerSignals){ console.log("Logging from ReportResult"); if(sellerReportingSignals.topLevelSeller === undefined || sellerReportingSignals.topLevelSeller.length === 0){ - sendReportTo("http://test.com") + sendReportTo("http://test.com"+"&bid="+sellerReportingSignals.bid+"&bidCurrency="+sellerReportingSignals.bidCurrency+"&highestScoringOtherBid="+sellerReportingSignals.highestScoringOtherBid+"&highestScoringOtherBidCurrency="+sellerReportingSignals.highestScoringOtherBidCurrency+"&topWindowHostname="+sellerReportingSignals.topWindowHostname+"&interestGroupOwner="+sellerReportingSignals.interestGroupOwner) } else { - sendReportTo("http://test.com&topLevelSeller="+sellerReportingSignals.topLevelSeller+"&componentSeller="+sellerReportingSignals.componentSeller+"&bid="+sellerReportingSignals.bid+"&modifiedBid="+sellerReportingSignals.modifiedBid) + sendReportTo("http://test.com&topLevelSeller="+sellerReportingSignals.topLevelSeller+"&componentSeller="+sellerReportingSignals.componentSeller+"&bid="+sellerReportingSignals.bid+"&bidCurrency="+sellerReportingSignals.bidCurrency+"&modifiedBid="+sellerReportingSignals.modifiedBid+"&modifiedBidCurrency="+sellerReportingSignals.modifiedBidCurrency+"&highestScoringOtherBid="+sellerReportingSignals.highestScoringOtherBid+"&highestScoringOtherBidCurrency="+sellerReportingSignals.highestScoringOtherBidCurrency+"&topWindowHostname="+sellerReportingSignals.topWindowHostname+"&interestGroupOwner="+sellerReportingSignals.interestGroupOwner) } registerAdBeacon({"clickEvent":"http://click.com"}) return "testSignalsForWinner" @@ -337,7 +341,9 @@ constexpr absl::string_view kExpectedFinalCode = R"JS_CODE( var auctionSignals = auctionConfig.auctionSignals var buyerReportingSignals = sellerReportingSignals delete buyerReportingSignals.desirability - buyerReportingSignals.interestGroupName = buyerReportingMetadata.interestGroupName + if(buyerReportingMetadata.hasOwnProperty("interestGroupName")){ + buyerReportingSignals.interestGroupName = buyerReportingMetadata.interestGroupName + } buyerReportingSignals.madeHighestScoringOtherBid = buyerReportingMetadata.madeHighestScoringOtherBid buyerReportingSignals.joinCount = buyerReportingMetadata.joinCount buyerReportingSignals.recency = buyerReportingMetadata.recency @@ -345,6 +351,9 @@ constexpr absl::string_view kExpectedFinalCode = R"JS_CODE( perBuyerSignals = buyerReportingMetadata.perBuyerSignals buyerReportingSignals.seller = buyerReportingMetadata.seller buyerReportingSignals.adCost = buyerReportingMetadata.adCost + if(buyerReportingMetadata.hasOwnProperty("buyerReportingId")){ + buyerReportingSignals.buyerReportingId = buyerReportingMetadata.buyerReportingId + } // Absence of interest group indicates that this is a protected app // signals ad. if (buyerReportingMetadata.enableProtectedAppSignals && @@ -424,11 +433,6 @@ constexpr absl::string_view kExpectedFinalCode = R"JS_CODE( console.error("Missing seller in input to reportWin") return } - if(buyerReportingSignals.interestGroupName == "" || buyerReportingSignals.interestGroupName == undefined - || buyerReportingSignals.interestGroupName == null){ - console.error("Missing interestGroupName in input to reportWin") - return - } if(buyerReportingSignals.adCost == 0 || buyerReportingSignals.adCost == -1 || buyerReportingSignals.adCost == undefined || buyerReportingSignals.adCost == null){ @@ -441,8 +445,10 @@ constexpr absl::string_view kExpectedFinalCode = R"JS_CODE( buyerReportingSignals.modelingSignals+"&recency="+buyerReportingSignals.recency+ "&madeHighestScoringOtherBid="+buyerReportingSignals.madeHighestScoringOtherBid+ "&joinCount="+buyerReportingSignals.joinCount+"&signalsForWinner="+signalsForWinner+ - "&perBuyerSignals="+perBuyerSignals+"&auctionSignals="+auctionSignals+"&desirability="+buyerReportingSignals.desirability; - + "&perBuyerSignals="+perBuyerSignals+"&auctionSignals="+auctionSignals+"&desirability="+buyerReportingSignals.desirability + if(buyerReportingSignals.hasOwnProperty("buyerReportingId")){ + reportWinUrl = reportWinUrl+"&buyerReportingId="+buyerReportingSignals.buyerReportingId + } console.log("Logging from ReportWin"); console.error("Logging error from ReportWin") console.warn("Logging warning from ReportWin") @@ -484,7 +490,7 @@ constexpr absl::string_view kExpectedFinalCode = R"JS_CODE( function reportResult(auctionConfig, sellerReportingSignals, directFromSellerSignals){ console.log("Logging from ReportResult"); if(sellerReportingSignals.topLevelSeller === undefined || sellerReportingSignals.topLevelSeller.length === 0){ - sendReportTo("http://test.com") + sendReportTo("http://test.com"+"&bid="+sellerReportingSignals.bid+"&bidCurrency="+sellerReportingSignals.bidCurrency+"&highestScoringOtherBid="+sellerReportingSignals.highestScoringOtherBid+"&highestScoringOtherBidCurrency="+sellerReportingSignals.highestScoringOtherBidCurrency+"&topWindowHostname="+sellerReportingSignals.topWindowHostname+"&interestGroupOwner="+sellerReportingSignals.interestGroupOwner) } else { sendReportTo("http://test.com&topLevelSeller="+sellerReportingSignals.topLevelSeller+"&componentSeller="+sellerReportingSignals.componentSeller) } @@ -548,7 +554,7 @@ constexpr absl::string_view kExpectedProtectedAppSignalsFinalCode = R"JS_CODE( //Handler method to call adTech provided reportResult method and wrap the // response with reportResult url and interaction reporting urls. - function reportingEntryFunctionProtectedAppSignals(auctionConfig, sellerReportingSignals, directFromSellerSignals, enable_logging, buyerReportingMetadata, egressFeatures) { + function reportingEntryFunctionProtectedAppSignals(auctionConfig, sellerReportingSignals, directFromSellerSignals, enable_logging, buyerReportingMetadata, egressPayload, temporaryUnlimitedEgressPayload) { ps_signalsForWinner = "" var ps_report_result_response = { reportResultUrl : "", @@ -592,7 +598,9 @@ constexpr absl::string_view kExpectedProtectedAppSignalsFinalCode = R"JS_CODE( var auctionSignals = auctionConfig.auctionSignals var buyerReportingSignals = sellerReportingSignals delete buyerReportingSignals.desirability - buyerReportingSignals.interestGroupName = buyerReportingMetadata.interestGroupName + if(buyerReportingMetadata.hasOwnProperty("interestGroupName")){ + buyerReportingSignals.interestGroupName = buyerReportingMetadata.interestGroupName + } buyerReportingSignals.madeHighestScoringOtherBid = buyerReportingMetadata.madeHighestScoringOtherBid buyerReportingSignals.joinCount = buyerReportingMetadata.joinCount buyerReportingSignals.recency = buyerReportingMetadata.recency @@ -600,6 +608,9 @@ constexpr absl::string_view kExpectedProtectedAppSignalsFinalCode = R"JS_CODE( perBuyerSignals = buyerReportingMetadata.perBuyerSignals buyerReportingSignals.seller = buyerReportingMetadata.seller buyerReportingSignals.adCost = buyerReportingMetadata.adCost + if(buyerReportingMetadata.hasOwnProperty("buyerReportingId")){ + buyerReportingSignals.buyerReportingId = buyerReportingMetadata.buyerReportingId + } // Absence of interest group indicates that this is a protected app // signals ad. if (buyerReportingMetadata.enableProtectedAppSignals && @@ -608,7 +619,7 @@ constexpr absl::string_view kExpectedProtectedAppSignalsFinalCode = R"JS_CODE( functionSuffix += "ProtectedAppSignals"; } var reportWinFunction = "reportWinWrapper"+functionSuffix+"(auctionSignals, perBuyerSignals, ps_signalsForWinner, buyerReportingSignals,"+ - "directFromSellerSignals, enable_logging, egressFeatures)" + "directFromSellerSignals, enable_logging, egressPayload, temporaryUnlimitedEgressPayload)" var reportWinResponse = eval(reportWinFunction) return { reportResultResponse: ps_report_result_response, @@ -635,7 +646,7 @@ constexpr absl::string_view kExpectedProtectedAppSignalsFinalCode = R"JS_CODE( // Handler method to call adTech provided reportWin method and wrap the // response with reportWin url and interaction reporting urls. function reportWinWrapperhttpbuyer1comProtectedAppSignals(auctionSignals, perBuyerSignals, signalsForWinner, buyerReportingSignals, - directFromSellerSignals, enable_logging, egressFeatures) { + directFromSellerSignals, enable_logging, egressPayload, temporaryUnlimitedEgressPayload) { var ps_report_win_response = { reportWinUrl : "", interactionReportingUrls : {}, @@ -672,7 +683,7 @@ constexpr absl::string_view kExpectedProtectedAppSignalsFinalCode = R"JS_CODE( } { reportWinProtectedAppSignals = function(auctionSignals, perBuyerSignals, signalsForWinner, buyerReportingSignals, - directFromSellerSignals, egressFeatures){ + directFromSellerSignals, egressPayload, temporaryUnlimitedEgressPayload){ console.log("Testing Protected App Signals"); sendReportTo("http://test.com"); registerAdBeacon({"clickEvent":"http://click.com"}); @@ -682,7 +693,7 @@ constexpr absl::string_view kExpectedProtectedAppSignalsFinalCode = R"JS_CODE( } try{ reportWinProtectedAppSignals(auctionSignals, perBuyerSignals, signalsForWinner, buyerReportingSignals, - directFromSellerSignals, egressFeatures) + directFromSellerSignals, egressPayload, temporaryUnlimitedEgressPayload) } catch(ex){ console.error(ex.message) } @@ -740,7 +751,9 @@ constexpr absl::string_view kExpectedProtectedAppSignalsFinalCode = R"JS_CODE( var auctionSignals = auctionConfig.auctionSignals var buyerReportingSignals = sellerReportingSignals delete buyerReportingSignals.desirability - buyerReportingSignals.interestGroupName = buyerReportingMetadata.interestGroupName + if(buyerReportingMetadata.hasOwnProperty("interestGroupName")){ + buyerReportingSignals.interestGroupName = buyerReportingMetadata.interestGroupName + } buyerReportingSignals.madeHighestScoringOtherBid = buyerReportingMetadata.madeHighestScoringOtherBid buyerReportingSignals.joinCount = buyerReportingMetadata.joinCount buyerReportingSignals.recency = buyerReportingMetadata.recency @@ -748,6 +761,9 @@ constexpr absl::string_view kExpectedProtectedAppSignalsFinalCode = R"JS_CODE( perBuyerSignals = buyerReportingMetadata.perBuyerSignals buyerReportingSignals.seller = buyerReportingMetadata.seller buyerReportingSignals.adCost = buyerReportingMetadata.adCost + if(buyerReportingMetadata.hasOwnProperty("buyerReportingId")){ + buyerReportingSignals.buyerReportingId = buyerReportingMetadata.buyerReportingId + } // Absence of interest group indicates that this is a protected app // signals ad. if (buyerReportingMetadata.enableProtectedAppSignals && @@ -856,7 +872,7 @@ constexpr absl::string_view kExpectedProtectedAppSignalsFinalCode = R"JS_CODE( function reportResult(auctionConfig, sellerReportingSignals, directFromSellerSignals){ console.log("Logging from ReportResult"); if(sellerReportingSignals.topLevelSeller === undefined || sellerReportingSignals.topLevelSeller.length === 0){ - sendReportTo("http://test.com") + sendReportTo("http://test.com"+"&bid="+sellerReportingSignals.bid+"&bidCurrency="+sellerReportingSignals.bidCurrency+"&highestScoringOtherBid="+sellerReportingSignals.highestScoringOtherBid+"&highestScoringOtherBidCurrency="+sellerReportingSignals.highestScoringOtherBidCurrency+"&topWindowHostname="+sellerReportingSignals.topWindowHostname+"&interestGroupOwner="+sellerReportingSignals.interestGroupOwner) } else { sendReportTo("http://test.com&topLevelSeller="+sellerReportingSignals.topLevelSeller+"&componentSeller="+sellerReportingSignals.componentSeller) } @@ -964,7 +980,9 @@ constexpr absl::string_view kExpectedCodeWithReportWinDisabled = R"JS_CODE( var auctionSignals = auctionConfig.auctionSignals var buyerReportingSignals = sellerReportingSignals delete buyerReportingSignals.desirability - buyerReportingSignals.interestGroupName = buyerReportingMetadata.interestGroupName + if(buyerReportingMetadata.hasOwnProperty("interestGroupName")){ + buyerReportingSignals.interestGroupName = buyerReportingMetadata.interestGroupName + } buyerReportingSignals.madeHighestScoringOtherBid = buyerReportingMetadata.madeHighestScoringOtherBid buyerReportingSignals.joinCount = buyerReportingMetadata.joinCount buyerReportingSignals.recency = buyerReportingMetadata.recency @@ -972,6 +990,9 @@ constexpr absl::string_view kExpectedCodeWithReportWinDisabled = R"JS_CODE( perBuyerSignals = buyerReportingMetadata.perBuyerSignals buyerReportingSignals.seller = buyerReportingMetadata.seller buyerReportingSignals.adCost = buyerReportingMetadata.adCost + if(buyerReportingMetadata.hasOwnProperty("buyerReportingId")){ + buyerReportingSignals.buyerReportingId = buyerReportingMetadata.buyerReportingId + } // Absence of interest group indicates that this is a protected app // signals ad. if (buyerReportingMetadata.enableProtectedAppSignals && @@ -1023,7 +1044,7 @@ constexpr absl::string_view kExpectedCodeWithReportWinDisabled = R"JS_CODE( function reportResult(auctionConfig, sellerReportingSignals, directFromSellerSignals){ console.log("Logging from ReportResult"); if(sellerReportingSignals.topLevelSeller === undefined || sellerReportingSignals.topLevelSeller.length === 0){ - sendReportTo("http://test.com") + sendReportTo("http://test.com"+"&bid="+sellerReportingSignals.bid+"&bidCurrency="+sellerReportingSignals.bidCurrency+"&highestScoringOtherBid="+sellerReportingSignals.highestScoringOtherBid+"&highestScoringOtherBidCurrency="+sellerReportingSignals.highestScoringOtherBidCurrency+"&topWindowHostname="+sellerReportingSignals.topWindowHostname+"&interestGroupOwner="+sellerReportingSignals.interestGroupOwner) } else { sendReportTo("http://test.com&topLevelSeller="+sellerReportingSignals.topLevelSeller+"&componentSeller="+sellerReportingSignals.componentSeller) } @@ -1104,7 +1125,7 @@ constexpr absl::string_view kExpectedCodeWithReportingDisabled = R"JS_CODE( function reportResult(auctionConfig, sellerReportingSignals, directFromSellerSignals){ console.log("Logging from ReportResult"); if(sellerReportingSignals.topLevelSeller === undefined || sellerReportingSignals.topLevelSeller.length === 0){ - sendReportTo("http://test.com") + sendReportTo("http://test.com"+"&bid="+sellerReportingSignals.bid+"&bidCurrency="+sellerReportingSignals.bidCurrency+"&highestScoringOtherBid="+sellerReportingSignals.highestScoringOtherBid+"&highestScoringOtherBidCurrency="+sellerReportingSignals.highestScoringOtherBidCurrency+"&topWindowHostname="+sellerReportingSignals.topWindowHostname+"&interestGroupOwner="+sellerReportingSignals.interestGroupOwner) } else { sendReportTo("http://test.com&topLevelSeller="+sellerReportingSignals.topLevelSeller+"&componentSeller="+sellerReportingSignals.componentSeller) } diff --git a/services/auction_service/data/BUILD b/services/auction_service/data/BUILD index 2a2baf41..e8322bdf 100644 --- a/services/auction_service/data/BUILD +++ b/services/auction_service/data/BUILD @@ -21,4 +21,7 @@ cc_library( hdrs = [ "runtime_config.h", ], + deps = [ + "//services/auction_service:auction_constants", + ], ) diff --git a/services/auction_service/data/runtime_config.h b/services/auction_service/data/runtime_config.h index 3423163f..9cf7108a 100644 --- a/services/auction_service/data/runtime_config.h +++ b/services/auction_service/data/runtime_config.h @@ -19,6 +19,8 @@ #include +#include "services/auction_service/auction_constants.h" + namespace privacy_sandbox::bidding_auction_servers { struct AuctionServiceRuntimeConfig { @@ -50,6 +52,9 @@ struct AuctionServiceRuntimeConfig { // The max allowed size of all debug win or loss URLs for an auction. // Default value is 3000 kilobytes. int max_allowed_size_all_debug_urls_kb = 3000; + + // Default code version to pass to Roma. + std::string default_code_version = kScoreAdBlobVersion; }; } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/reporting/BUILD b/services/auction_service/reporting/BUILD index 78dae8b3..7e009d28 100644 --- a/services/auction_service/reporting/BUILD +++ b/services/auction_service/reporting/BUILD @@ -34,6 +34,8 @@ cc_library( deps = [ ":noiser_and_bucketer", ":reporting_response", + "//services/auction_service:auction_constants", + "//services/common:feature_flags", "//services/common/clients/code_dispatcher:code_dispatch_client", "//services/common/code_dispatch:code_dispatch_reactor", "//services/common/util:json_util", diff --git a/services/auction_service/reporting/reporting_helper.cc b/services/auction_service/reporting/reporting_helper.cc index 76797487..0d43362d 100644 --- a/services/auction_service/reporting/reporting_helper.cc +++ b/services/auction_service/reporting/reporting_helper.cc @@ -17,13 +17,16 @@ #include #include +#include "absl/flags/flag.h" #include "absl/status/statusor.h" #include "absl/strings/str_format.h" #include "api/bidding_auction_servers.pb.h" #include "rapidjson/document.h" +#include "services/auction_service/auction_constants.h" #include "services/auction_service/reporting/noiser_and_bucketer.h" #include "services/auction_service/reporting/reporting_response.h" #include "services/common/clients/code_dispatcher/v8_dispatcher.h" +#include "services/common/feature_flags.h" #include "services/common/util/json_util.h" #include "services/common/util/post_auction_signals.h" #include "services/common/util/reporting_util.h" @@ -157,6 +160,26 @@ absl::StatusOr GetSellerReportingSignals( if (bid > -1) { document.AddMember(kBid, bid, document.GetAllocator()); } + if (!dispatch_request_data.post_auction_signals.winning_bid_currency + .empty()) { + rapidjson::Value winning_bid_currency_value; + winning_bid_currency_value.SetString( + dispatch_request_data.post_auction_signals.winning_bid_currency.c_str(), + document.GetAllocator()); + document.AddMember(kWinningBidCurrencyTag, winning_bid_currency_value, + document.GetAllocator()); + } + if (!dispatch_request_data.post_auction_signals + .highest_scoring_other_bid_currency.empty()) { + rapidjson::Value highest_scoring_other_bid_currency_value; + highest_scoring_other_bid_currency_value.SetString( + dispatch_request_data.post_auction_signals + .highest_scoring_other_bid_currency.c_str(), + document.GetAllocator()); + document.AddMember(kHighestScoringOtherBidCurrencyTag, + highest_scoring_other_bid_currency_value, + document.GetAllocator()); + } double desirability = GetEightBitRoundedValue( dispatch_request_config.enable_report_win_input_noising, dispatch_request_data.post_auction_signals.winning_score); @@ -188,6 +211,16 @@ absl::StatusOr GetSellerReportingSignals( if (modified_bid > -1) { document.AddMember(kModifiedBid, modified_bid, document.GetAllocator()); } + if (!dispatch_request_data.component_reporting_metadata + .modified_bid_currency.empty()) { + rapidjson::Value modified_bid_currency; + modified_bid_currency.SetString( + dispatch_request_data.component_reporting_metadata + .modified_bid_currency.c_str(), + document.GetAllocator()); + document.AddMember(kModifiedBidCurrencyTag, modified_bid_currency, + document.GetAllocator()); + } document.AddMember(kComponentSeller, component_seller, document.GetAllocator()); } @@ -216,7 +249,7 @@ std::string GetBuyerMetadataJson( buyer_signals_obj = ParseJsonString( dispatch_request_data.buyer_reporting_metadata.buyer_signals); if (!buyer_signals_obj.ok()) { - PS_VLOG(1, dispatch_request_data.log_context) + PS_LOG(ERROR, dispatch_request_data.log_context) << "Error parsing buyer signals to Json object"; } else { buyer_reporting_signals_obj.AddMember( @@ -249,7 +282,7 @@ std::string GetBuyerMetadataJson( kJoinCount, join_count, buyer_reporting_signals_obj.GetAllocator()); } if (dispatch_request_data.buyer_reporting_metadata.recency.has_value()) { - PS_VLOG(1, dispatch_request_data.log_context) + PS_VLOG(kInfoMsg, dispatch_request_data.log_context) << "BuyerReportingMetadata: Recency:" << dispatch_request_data.buyer_reporting_metadata.recency.value(); long recency = @@ -291,13 +324,6 @@ std::string GetBuyerMetadataJson( kSellerTag, std::move(seller), buyer_reporting_signals_obj.GetAllocator()); } - rapidjson::Value interest_group_name; - interest_group_name.SetString(dispatch_request_data.buyer_reporting_metadata - .interest_group_name.c_str(), - buyer_reporting_signals_obj.GetAllocator()); - buyer_reporting_signals_obj.AddMember( - kInterestGroupName, std::move(interest_group_name), - buyer_reporting_signals_obj.GetAllocator()); double ad_cost = GetEightBitRoundedValue( dispatch_request_config.enable_report_win_input_noising, dispatch_request_data.buyer_reporting_metadata.ad_cost); @@ -305,6 +331,25 @@ std::string GetBuyerMetadataJson( buyer_reporting_signals_obj.AddMember( kAdCostTag, ad_cost, buyer_reporting_signals_obj.GetAllocator()); } + // if buyer_reporting_id is present, interestGroupName will not be set. + rapidjson::Value buyer_reporting_id; + rapidjson::Value interest_group_name; + if (!dispatch_request_data.buyer_reporting_metadata.buyer_reporting_id + .empty()) { + buyer_reporting_id.SetString(dispatch_request_data.buyer_reporting_metadata + .buyer_reporting_id.c_str(), + buyer_reporting_signals_obj.GetAllocator()); + buyer_reporting_signals_obj.AddMember( + kBuyerReportingIdTag, std::move(buyer_reporting_id), + buyer_reporting_signals_obj.GetAllocator()); + } else { + interest_group_name.SetString(dispatch_request_data.buyer_reporting_metadata + .interest_group_name.c_str(), + buyer_reporting_signals_obj.GetAllocator()); + buyer_reporting_signals_obj.AddMember( + kInterestGroupName, std::move(interest_group_name), + buyer_reporting_signals_obj.GetAllocator()); + } absl::StatusOr buyer_reporting_metadata_json = SerializeJsonDoc(buyer_reporting_signals_obj); if (!buyer_reporting_metadata_json.ok()) { @@ -347,8 +392,16 @@ std::vector> GetReportingInput( input[ReportingArgIndex(ReportingArgs::kBuyerReportingMetadata)] = std::make_shared(buyer_reporting_metadata_json); if (dispatch_request_config.enable_protected_app_signals) { - input[ReportingArgIndex(ReportingArgs::kEgressFeatures)] = - std::make_shared(dispatch_request_data.egress_features); + input[ReportingArgIndex(ReportingArgs::kEgressPayload)] = + std::make_shared(dispatch_request_data.egress_payload); + if (absl::GetFlag(FLAGS_enable_temporary_unlimited_egress)) { + input[ReportingArgIndex(ReportingArgs::kTemporaryEgressPayload)] = + std::make_shared( + dispatch_request_data.temporary_egress_payload); + } else { + input[ReportingArgIndex(ReportingArgs::kTemporaryEgressPayload)] = + std::make_shared(""); + } } PS_VLOG(2, dispatch_request_data.log_context) @@ -372,7 +425,7 @@ DispatchRequest GetReportingDispatchRequest( // Construct the wrapper struct for our V8 Dispatch Request. return { .id = dispatch_request_data.post_auction_signals.winning_ad_render_url, - .version_string = kDispatchRequestVersion, + .version_string = kReportingBlobVersion, .handler_name = dispatch_request_data.handler_name, .input = GetReportingInput(dispatch_request_config, dispatch_request_data), diff --git a/services/auction_service/reporting/reporting_helper.h b/services/auction_service/reporting/reporting_helper.h index b53de6aa..b27ccc1d 100644 --- a/services/auction_service/reporting/reporting_helper.h +++ b/services/auction_service/reporting/reporting_helper.h @@ -47,13 +47,16 @@ inline constexpr char kBuyerLogs[] = "buyerLogs"; inline constexpr char kBuyerErrors[] = "buyerErrors"; inline constexpr char kBuyerWarnings[] = "buyerWarnings"; inline constexpr int kReportingArgSize = 5; -inline constexpr int kReportingArgSizeWithProtectedAppSignals = 6; +inline constexpr int kReportingArgSizeWithProtectedAppSignals = 7; inline constexpr char kReportingDispatchHandlerFunctionName[] = "reportingEntryFunction"; inline constexpr char kReportingProtectedAppSignalsFunctionName[] = "reportingEntryFunctionProtectedAppSignals"; -inline constexpr char kDispatchRequestVersion[] = "v1"; inline constexpr char kTopLevelSellerTag[] = "topLevelSeller"; +inline constexpr char kWinningBidCurrencyTag[] = "bidCurrency"; +inline constexpr char kHighestScoringOtherBidCurrencyTag[] = + "highestScoringOtherBidCurrency"; +inline constexpr char kModifiedBidCurrencyTag[] = "modifiedBidCurrency"; inline constexpr char kComponentSeller[] = "componentSeller"; inline constexpr char kEnableReportWinUrlGeneration[] = "enableReportWinUrlGeneration"; @@ -67,6 +70,7 @@ inline constexpr char kBuyerSignals[] = "perBuyerSignals"; inline constexpr char kBuyerOriginTag[] = "buyerOrigin"; inline constexpr char kSellerTag[] = "seller"; inline constexpr char kAdCostTag[] = "adCost"; +inline constexpr char kBuyerReportingIdTag[] = "buyerReportingId"; inline constexpr char kMadeHighestScoringOtherBid[] = "madeHighestScoringOtherBid"; inline constexpr char kInteractionReportingUrlsWrapperResponse[] = @@ -78,7 +82,8 @@ enum class ReportingArgs : int { kDirectFromSellerSignals, kEnableAdTechCodeLogging, kBuyerReportingMetadata, - kEgressFeatures + kEgressPayload, + kTemporaryEgressPayload, }; inline constexpr int ReportingArgIndex(const ReportingArgs& arg) { @@ -93,11 +98,13 @@ struct BuyerReportingMetadata { std::string seller; std::string interest_group_name; double ad_cost; + std::string buyer_reporting_id; }; struct ComponentReportingMetadata { std::string top_level_seller; std::string component_seller; + std::string modified_bid_currency; float modified_bid; }; @@ -120,7 +127,8 @@ struct ReportingDispatchRequestData { server_common::log::ContextImpl& log_context; BuyerReportingMetadata buyer_reporting_metadata; ComponentReportingMetadata component_reporting_metadata; - absl::string_view egress_features; + absl::string_view egress_payload; + absl::string_view temporary_egress_payload; }; // Bid metadata passed as input to reportResult and reportWin @@ -131,9 +139,12 @@ struct SellerReportingMetadata { std::string interest_group_owner; std::string render_url; float bid; + std::string bid_currency; float desirability; float modified_bid; + std::string modified_bid_currency; float highest_scoring_other_bid; + std::string highest_scoring_other_bid_currency; }; inline const std::string kDefaultBuyerReportingMetadata = absl::StrFormat( diff --git a/services/auction_service/reporting/reporting_helper_test.cc b/services/auction_service/reporting/reporting_helper_test.cc index 6f24fe29..0810f002 100644 --- a/services/auction_service/reporting/reporting_helper_test.cc +++ b/services/auction_service/reporting/reporting_helper_test.cc @@ -21,6 +21,7 @@ #include "gtest/gtest.h" #include "rapidjson/document.h" #include "rapidjson/stringbuffer.h" +#include "services/auction_service/auction_constants.h" #include "services/auction_service/reporting/reporting_helper_test_constants.h" #include "services/auction_service/reporting/reporting_response.h" #include "services/common/clients/code_dispatcher/v8_dispatcher.h" @@ -33,7 +34,7 @@ namespace privacy_sandbox::bidding_auction_servers { namespace { -constexpr char kTestEgressFeatures[] = "testEgressFeatures"; +constexpr char kTestEgressPayload[] = "testEgressPayload"; constexpr int kMinNoisedJoinCount = 0; constexpr int kMaxNoisedJoinCount = 17; constexpr int kMinNoisedRecency = -1; @@ -182,7 +183,7 @@ BuyerReportingMetadata GetTestBuyerReportingMetadata() { ReportingDispatchRequestData GetTestDispatchRequestData( const ScoreAdsResponse::AdScore& winning_ad_score, const ReportingDispatchRequestConfig& dispatch_request_config, - const std::string& handler_name) { + const std::string& handler_name, absl::string_view seller_currency) { server_common::log::ContextImpl log_context( {}, server_common::ConsentedDebugConfiguration()); std::shared_ptr auction_config = @@ -190,7 +191,8 @@ ReportingDispatchRequestData GetTestDispatchRequestData( ReportingDispatchRequestData reporting_dispatch_request_data = { .handler_name = handler_name, .auction_config = auction_config, - .post_auction_signals = GeneratePostAuctionSignals(winning_ad_score), + .post_auction_signals = + GeneratePostAuctionSignals(winning_ad_score, seller_currency), .publisher_hostname = kTestPublisherHostName, .log_context = log_context}; if (dispatch_request_config.enable_report_win_url_generation) { @@ -209,24 +211,27 @@ ReportingDispatchRequestData GetTestDispatchRequestData( ReportingDispatchRequestData GetTestComponentDispatchRequestData( const ScoreAdsResponse::AdScore& winning_ad_score, const ReportingDispatchRequestConfig& dispatch_request_config, - const std::string& handler_name) { + const std::string& handler_name, absl::string_view seller_currency) { ReportingDispatchRequestData reporting_dispatch_request_data = GetTestDispatchRequestData(winning_ad_score, dispatch_request_config, - handler_name); + handler_name, seller_currency); reporting_dispatch_request_data.component_reporting_metadata = { .top_level_seller = kTestTopLevelSeller, .component_seller = kTestSeller, - .modified_bid = kTestModifiedBid}; + .modified_bid_currency = winning_ad_score.bid_currency(), + .modified_bid = winning_ad_score.bid()}; return reporting_dispatch_request_data; } ReportingDispatchRequestData GetTestComponentDispatchRequestDataForPAS( const ScoreAdsResponse::AdScore& winning_ad_score, - const ReportingDispatchRequestConfig& dispatch_request_config) { + const ReportingDispatchRequestConfig& dispatch_request_config, + absl::string_view seller_currency) { ReportingDispatchRequestData reporting_dispatch_request_data = GetTestDispatchRequestData(winning_ad_score, dispatch_request_config, - kReportingProtectedAppSignalsFunctionName); - reporting_dispatch_request_data.egress_features = kTestEgressFeatures; + kReportingProtectedAppSignalsFunctionName, + seller_currency); + reporting_dispatch_request_data.egress_payload = kTestEgressPayload; return reporting_dispatch_request_data; } @@ -259,7 +264,7 @@ void TestArgs(std::vector> response_vector, absl::string_view expected_seller_reporting_signals, absl::string_view expected_unnoised_buyer_metadata_json = kTestBuyerMetadata, - absl::string_view expected_egress_features = "") { + absl::string_view expected_egress_payload = "") { EXPECT_EQ( *(response_vector[ReportingArgIndex(ReportingArgs::kAuctionConfig)]), kTestAuctionConfig); @@ -284,8 +289,8 @@ void TestArgs(std::vector> response_vector, } if (dispatch_request_config.enable_protected_app_signals) { EXPECT_EQ( - *(response_vector[ReportingArgIndex(ReportingArgs::kEgressFeatures)]), - expected_egress_features); + *(response_vector[ReportingArgIndex(ReportingArgs::kEgressPayload)]), + expected_egress_payload); } } @@ -423,6 +428,10 @@ TEST(ParseAndGetReportingResponseJson, HandlesEmptyResponse) { TEST(GetReportingInput, ReturnsTheInputArgsForReportResultForComponentAuction) { ScoreAdsResponse::AdScore winning_ad_score; winning_ad_score.set_buyer_bid(kTestBuyerBid); + winning_ad_score.set_buyer_bid_currency(kEurosIsoCode); + winning_ad_score.set_bid(kTestModifiedBid); + // This is the currency of the modified bid. + winning_ad_score.set_bid_currency(kUsdIsoCode); winning_ad_score.set_interest_group_owner(kTestInterestGroupOwner); winning_ad_score.set_interest_group_name(kTestInterestGroupName); winning_ad_score.mutable_ig_owner_highest_scoring_other_bids_map() @@ -438,7 +447,7 @@ TEST(GetReportingInput, ReturnsTheInputArgsForReportResultForComponentAuction) { ReportingDispatchRequestData dispatch_request_data = GetTestComponentDispatchRequestData( winning_ad_score, dispatch_request_config, - kReportingDispatchHandlerFunctionName); + kReportingDispatchHandlerFunctionName, ""); std::vector> response_vector = GetReportingInput(dispatch_request_config, dispatch_request_data); TestArgs(response_vector, dispatch_request_config, @@ -448,6 +457,7 @@ TEST(GetReportingInput, ReturnsTheInputArgsForReportResultForComponentAuction) { TEST(GetReportingDispatchRequest, ReturnsTheDispatchRequestForReportResult) { ScoreAdsResponse::AdScore winning_ad_score; winning_ad_score.set_buyer_bid(kTestBuyerBid); + winning_ad_score.set_buyer_bid_currency(kEurosIsoCode); winning_ad_score.set_interest_group_owner(kTestInterestGroupOwner); winning_ad_score.set_interest_group_name(kTestInterestGroupName); winning_ad_score.mutable_ig_owner_highest_scoring_other_bids_map() @@ -462,13 +472,15 @@ TEST(GetReportingDispatchRequest, ReturnsTheDispatchRequestForReportResult) { .enable_adtech_code_logging = true}; ReportingDispatchRequestData dispatch_request_data = GetTestDispatchRequestData(winning_ad_score, dispatch_request_config, - kReportingDispatchHandlerFunctionName); + kReportingDispatchHandlerFunctionName, + /*seller_currency=*/kUsdIsoCode); DispatchRequest request = GetReportingDispatchRequest(dispatch_request_config, dispatch_request_data); - TestArgs(request.input, dispatch_request_config, kTestSellerReportingSignals); + TestArgs(request.input, dispatch_request_config, + kTestSellerReportingSignalsWithOtherBidCurrency); EXPECT_EQ(request.id, kTestRender); EXPECT_EQ(request.handler_name, kReportingDispatchHandlerFunctionName); - EXPECT_EQ(request.version_string, kDispatchRequestVersion); + EXPECT_EQ(request.version_string, kReportingBlobVersion); } TEST(GetReportingDispatchRequest, ReturnsDispatchRequestWithReportWin) { @@ -490,13 +502,14 @@ TEST(GetReportingDispatchRequest, ReturnsDispatchRequestWithReportWin) { .enable_adtech_code_logging = true}; ReportingDispatchRequestData dispatch_request_data = GetTestDispatchRequestData(winning_ad_score, dispatch_request_config, - kReportingDispatchHandlerFunctionName); + kReportingDispatchHandlerFunctionName, + /*seller_currency=*/""); DispatchRequest request = GetReportingDispatchRequest(dispatch_request_config, dispatch_request_data); TestArgs(request.input, dispatch_request_config, kTestSellerReportingSignals); EXPECT_EQ(request.id, kTestRender); EXPECT_EQ(request.handler_name, kReportingDispatchHandlerFunctionName); - EXPECT_EQ(request.version_string, kDispatchRequestVersion); + EXPECT_EQ(request.version_string, kReportingBlobVersion); } TEST(GetReportingDispatchRequest, @@ -523,7 +536,8 @@ TEST(GetReportingDispatchRequest, .enable_adtech_code_logging = true}; ReportingDispatchRequestData reporting_dispatch_request_data = GetTestDispatchRequestData(winning_ad_score, dispatch_request_config, - kReportingDispatchHandlerFunctionName); + kReportingDispatchHandlerFunctionName, + /*seller_currency=*/""); DispatchRequest request = GetReportingDispatchRequest( dispatch_request_config, reporting_dispatch_request_data); TestArgs(request.input, dispatch_request_config, kTestSellerReportingSignals, @@ -534,7 +548,7 @@ TEST(GetReportingDispatchRequest, GetTestBuyerReportingMetadata()); EXPECT_EQ(request.id, kTestRender); EXPECT_EQ(request.handler_name, kReportingDispatchHandlerFunctionName); - EXPECT_EQ(request.version_string, kDispatchRequestVersion); + EXPECT_EQ(request.version_string, kReportingBlobVersion); } TEST(GetReportingDispatchRequest, @@ -557,19 +571,19 @@ TEST(GetReportingDispatchRequest, .enable_report_win_input_noising = true}; ReportingDispatchRequestData dispatch_request_data = GetTestComponentDispatchRequestDataForPAS(winning_ad_score, - dispatch_request_config); + dispatch_request_config, ""); DispatchRequest request = GetReportingDispatchRequest(dispatch_request_config, dispatch_request_data); TestArgs(request.input, dispatch_request_config, kTestSellerReportingSignals, /*expected_unnoised_buyer_metadata_json=*/"", - dispatch_request_data.egress_features); + dispatch_request_data.egress_payload); VerifyBuyerReportingMetadata( *(request .input[ReportingArgIndex(ReportingArgs::kBuyerReportingMetadata)]), GetTestBuyerReportingMetadata()); EXPECT_EQ(request.id, kTestRender); EXPECT_EQ(request.handler_name, kReportingProtectedAppSignalsFunctionName); - EXPECT_EQ(request.version_string, kDispatchRequestVersion); + EXPECT_EQ(request.version_string, kReportingBlobVersion); } } // namespace } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/reporting/reporting_helper_test_constants.h b/services/auction_service/reporting/reporting_helper_test_constants.h index a7560f3c..a94ed76d 100644 --- a/services/auction_service/reporting/reporting_helper_test_constants.h +++ b/services/auction_service/reporting/reporting_helper_test_constants.h @@ -18,6 +18,9 @@ namespace privacy_sandbox::bidding_auction_servers { constexpr char kTestReportResultUrl[] = "http://reportResultUrl.com"; constexpr char kTestReportWinUrl[] = "http://reportWinUrl.com"; +constexpr char kTestBuyerReportingId[] = "testBuyerReportingId"; +constexpr char kTestReportWinUrlWithBuyerReportingId[] = + "http://reportWinUrl.com&buyerReportingId=testId"; constexpr char kTestLog[] = "testLog"; constexpr bool kSendReportToInvokedTrue = true; constexpr bool kRegisterAdBeaconInvokedTrue = true; @@ -26,9 +29,11 @@ constexpr char kTestInteractionUrl[] = "http://event.com"; constexpr char kTestPublisherHostName[] = "publisherName"; constexpr char kTestAuctionConfig[] = "testAuctionConfig"; constexpr char kTestSellerReportingSignalsForComponentSeller[] = - R"({"topWindowHostname":"publisherName","interestGroupOwner":"testOwner","renderURL":"http://testurl.com","renderUrl":"http://testurl.com","bid":1.0,"desirability":2.0,"highestScoringOtherBid":0.5,"topLevelSeller":"testTopLevelSeller","modifiedBid":1.0,"componentSeller":"http://seller.com"})"; + R"({"topWindowHostname":"publisherName","interestGroupOwner":"testOwner","renderURL":"http://testurl.com","renderUrl":"http://testurl.com","bid":1.0,"bidCurrency":"EUR","highestScoringOtherBidCurrency":"???","desirability":2.0,"highestScoringOtherBid":0.5,"topLevelSeller":"testTopLevelSeller","modifiedBid":1.0,"modifiedBidCurrency":"USD","componentSeller":"http://seller.com"})"; +constexpr char kTestSellerReportingSignalsWithOtherBidCurrency[] = + R"({"topWindowHostname":"publisherName","interestGroupOwner":"testOwner","renderURL":"http://testurl.com","renderUrl":"http://testurl.com","bid":1.0,"bidCurrency":"EUR","highestScoringOtherBidCurrency":"USD","desirability":2.0,"highestScoringOtherBid":0.5})"; constexpr char kTestSellerReportingSignals[] = - R"({"topWindowHostname":"publisherName","interestGroupOwner":"testOwner","renderURL":"http://testurl.com","renderUrl":"http://testurl.com","bid":1.0,"desirability":2.0,"highestScoringOtherBid":0.5})"; + R"({"topWindowHostname":"publisherName","interestGroupOwner":"testOwner","renderURL":"http://testurl.com","renderUrl":"http://testurl.com","bid":1.0,"bidCurrency":"???","highestScoringOtherBidCurrency":"???","desirability":2.0,"highestScoringOtherBid":0.5})"; constexpr char kTestInterestGroupOwner[] = "testOwner"; constexpr char kTestInterestGroupName[] = "testInterestGroupName"; constexpr char kTestRender[] = "http://testurl.com"; @@ -40,7 +45,7 @@ constexpr int kTestJoinCount = 1; constexpr float kTestRecency = 2.1; constexpr int kTestModelingSignals = 3; constexpr char kTestBuyerMetadata[] = - R"({"enableReportWinUrlGeneration":true,"enableProtectedAppSignals":false,"perBuyerSignals":{"testkey":"testvalue"},"buyerOrigin":"testOwner","madeHighestScoringOtherBid":true,"joinCount":1,"recency":2,"modelingSignals":3,"seller":"http://seller.com","interestGroupName":"testInterestGroupName","adCost":5.0})"; + R"({"enableReportWinUrlGeneration":true,"enableProtectedAppSignals":false,"perBuyerSignals":{"testkey":"testvalue"},"buyerOrigin":"testOwner","madeHighestScoringOtherBid":true,"joinCount":1,"recency":2,"modelingSignals":3,"seller":"http://seller.com","adCost":5.0,"interestGroupName":"testInterestGroupName"})"; constexpr char kTestBuyerMetadataWithProtectedAppSignals[] = R"({"enableReportWinUrlGeneration":true,"enableProtectedAppSignals":true,"perBuyerSignals":{"testkey":"testvalue"},"buyerOrigin":"testOwner","madeHighestScoringOtherBid":true,"joinCount":1,"recency":2,"modelingSignals":3,"seller":"http://seller.com","interestGroupName":"testInterestGroupName","adCost":5.0})"; constexpr char kTestBuyerSignals[] = "{\"testkey\":\"testvalue\"}"; @@ -48,6 +53,8 @@ constexpr char kTestSeller[] = "http://seller.com"; constexpr double kTestAdCost = 5.0; constexpr char kTestTopLevelSeller[] = "testTopLevelSeller"; constexpr float kTestModifiedBid = 1.0; +constexpr char kUsdIsoCode[] = "USD"; +constexpr char kEurosIsoCode[] = "EUR"; constexpr bool kTestEnableReportWinInputNoisingTrue = true; } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/score_ads_reactor.cc b/services/auction_service/score_ads_reactor.cc index f9a51f4d..aef1ecf1 100644 --- a/services/auction_service/score_ads_reactor.cc +++ b/services/auction_service/score_ads_reactor.cc @@ -70,7 +70,7 @@ inline void MayVlogRomaResponses( inline void LogWarningForBadResponse( const absl::Status& status, const DispatchResponse& response, const AdWithBidMetadata* ad_with_bid_metadata, ContextImpl& log_context) { - PS_VLOG(0, log_context) << "Failed to parse response from Roma ", + PS_LOG(ERROR, log_context) << "Failed to parse response from Roma ", status.ToString(absl::StatusToStringMode::kWithEverything); if (ad_with_bid_metadata) { ABSL_LOG(WARNING) @@ -197,7 +197,8 @@ ScoreAdsReactor::ScoreAdsReactor( runtime_config.max_allowed_size_debug_url_bytes), max_allowed_size_all_debug_urls_chars_( kBytesMultiplyer * runtime_config.max_allowed_size_all_debug_urls_kb), - auction_scope_(GetAuctionScope(raw_request_)) { + auction_scope_(GetAuctionScope(raw_request_)), + code_version_(runtime_config.default_code_version) { CHECK_OK([this]() { PS_ASSIGN_OR_RETURN(metric_context_, metric::AuctionContextMap()->Remove(request_)); @@ -248,7 +249,8 @@ absl::Status ScoreAdsReactor::PopulateTopLevelAuctionDispatchRequests( auction_result.ad_component_render_urls(), auction_result.auction_params().component_seller(), auction_result.bid_currency()), - log_context_, enable_adtech_code_logging_, enable_debug_reporting); + log_context_, enable_adtech_code_logging_, enable_debug_reporting, + code_version_); if (!dispatch_request.ok()) { PS_VLOG(2, log_context_) << "Failed to create scoring request for protected audience: " @@ -291,7 +293,8 @@ void ScoreAdsReactor::PopulateProtectedAudienceDispatchRequests( MakeBidMetadata(raw_request_.publisher_hostname(), ad->interest_group_owner(), ad->render(), ad->ad_components(), raw_request_.top_level_seller(), - ad->bid_currency())); + ad->bid_currency()), + code_version_); if (!dispatch_request.ok()) { PS_VLOG(2, log_context_) << "Failed to create scoring request for protected audience: " @@ -338,7 +341,8 @@ void ScoreAdsReactor::MayPopulateProtectedAppSignalsDispatchRequests( MakeBidMetadata( raw_request_.publisher_hostname(), pas_ad_with_bid->owner(), pas_ad_with_bid->render(), GetEmptyAdComponentRenderUrls(), - raw_request_.top_level_seller(), pas_ad_with_bid->bid_currency())); + raw_request_.top_level_seller(), pas_ad_with_bid->bid_currency()), + code_version_); if (!dispatch_request.ok()) { PS_VLOG(2, log_context_) << "Failed to create scoring request for protected app signals ad: " @@ -364,7 +368,7 @@ void ScoreAdsReactor::Execute() { PS_VLOG(kEncrypted, log_context_) << "Encrypted ScoreAdsRequest:\n" << request_->ShortDebugString(); PS_VLOG(kPlain, log_context_) << "ScoreAdsRawRequest:\n" - << raw_request_.DebugString(); + << raw_request_.ShortDebugString(); DCHECK(raw_request_.protected_app_signals_ad_bids().empty() || enable_protected_app_signals_) @@ -376,7 +380,7 @@ void ScoreAdsReactor::Execute() { !raw_request_.protected_app_signals_ad_bids().empty()) { // This path should be unreachable from SFE. // Component PA and PAS auctions cannot be done together for now. - PS_VLOG(1, log_context_) + PS_LOG(ERROR, log_context_) << "Finishing RPC: " << kDeviceComponentAuctionWithPAS; FinishWithStatus(::grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, kDeviceComponentAuctionWithPAS)); @@ -413,8 +417,8 @@ void ScoreAdsReactor::Execute() { BuildTrustedScoringSignals(raw_request_, log_context_); if (!scoring_signals.ok()) { - PS_VLOG(1, log_context_) << "No scoring signals found, finishing RPC: " - << scoring_signals.status(); + PS_LOG(ERROR, log_context_) << "No scoring signals found, finishing RPC: " + << scoring_signals.status(); FinishWithStatus(server_common::FromAbslStatus(scoring_signals.status())); return; } @@ -449,7 +453,7 @@ void ScoreAdsReactor::Execute() { LogIfError(metric_context_ ->AccumulateMetric( 1, metric::kAuctionScoreAdsFailedToDispatchCode)); - PS_VLOG(1, log_context_) + PS_LOG(ERROR, log_context_) << "Execution request failed for batch: " << raw_request_.DebugString() << status.ToString(absl::StatusToStringMode::kWithEverything); LogIfError( @@ -469,6 +473,8 @@ void ScoreAdsReactor::PerformReporting( // ad->recency() // ad->modeling_signals(), // ad->ad_cost() + benchmarking_logger_->HandleResponseEnd(); + EncryptAndFinishOK(); return; } if (auto ad_it = ad_data_.find(id); ad_it != ad_data_.end()) { @@ -484,6 +490,11 @@ void ScoreAdsReactor::PerformReporting( .seller = raw_request_.seller(), .interest_group_name = winning_ad_score.interest_group_name(), .ad_cost = ad->ad_cost()}; + if (!ad->buyer_reporting_id().empty()) { + raw_response_.mutable_ad_score()->set_buyer_reporting_id( + ad->buyer_reporting_id()); + buyer_reporting_metadata.buyer_reporting_id = ad->buyer_reporting_id(); + } } DispatchReportingRequestForPA(winning_ad_score, BuildAuctionConfig(raw_request_), @@ -506,9 +517,10 @@ void ScoreAdsReactor::PerformReporting( } DispatchReportingRequestForPAS( winning_ad_score, BuildAuctionConfig(raw_request_), - buyer_reporting_metadata, ad->egress_features()); + buyer_reporting_metadata, ad->egress_payload(), + ad->temporary_unlimited_egress_payload()); } else { - PS_VLOG(1, log_context_) + PS_LOG(ERROR, log_context_) << "Following id didn't map to any ProtectedAudience or " "ProtectedAppSignals Ad: " << id; @@ -521,10 +533,12 @@ void ScoreAdsReactor::HandleScoredAd(int index, float buyer_bid, absl::string_view ad_with_bid_currency, absl::string_view interest_group_name, absl::string_view interest_group_owner, + absl::string_view interest_group_origin, const rapidjson::Document& response_json, AdType ad_type, ScoreAdsResponse::AdScore& ad_score, - ScoringData& scoring_data) { + ScoringData& scoring_data, + const std::string& dispatch_response_id) { // Get ad rejection reason before updating the scoring data. std::optional ad_rejection_reason; @@ -536,16 +550,19 @@ void ScoreAdsReactor::HandleScoredAd(int index, float buyer_bid, ad_score.set_interest_group_name(interest_group_name); ad_score.set_interest_group_owner(interest_group_owner); + ad_score.set_interest_group_origin(interest_group_origin); ad_score.set_ad_type(ad_type); ad_score.set_buyer_bid(buyer_bid); + ad_score.set_buyer_bid_currency(ad_with_bid_currency); // Used for debug reporting. // Should include bids rejected by bid currency mismatch // and those not allowed in component auctions. - ad_scores_.push_back(std::make_unique(ad_score)); + ad_scores_.emplace(dispatch_response_id, + std::make_unique(ad_score)); if (CheckAndUpdateModifiedBid(auction_scope_, buyer_bid, ad_with_bid_currency, &ad_score)) { - PS_VLOG(1, log_context_) + PS_VLOG(kInfoMsg, log_context_) << "Setting modified bid value to original buyer bid value (and " "currency) as the " "modified bid is not set. For interest group: " @@ -554,25 +571,33 @@ void ScoreAdsReactor::HandleScoredAd(int index, float buyer_bid, if (IsBidCurrencyMismatched(auction_scope_, raw_request_.seller_currency(), ad_score.bid(), ad_score.bid_currency())) { - // Ignore currency-mismatched winner. - // TODO(b/311234165): Add metric for ads rejected for mismatched - // currency. - PS_VLOG(1, log_context_) + ad_rejection_reason = ScoreAdsResponse::AdScore::AdRejectionReason{}; + ad_rejection_reason->set_interest_group_name( + ad_score.interest_group_name()); + ad_rejection_reason->set_interest_group_owner( + ad_score.interest_group_owner()); + ad_rejection_reason->set_rejection_reason( + SellerRejectionReason::BID_FROM_SCORE_AD_FAILED_CURRENCY_CHECK); + PS_VLOG(kInfoMsg, log_context_) << "Skipping component bid as it does not match seller_currency: " << interest_group_name << ": " << ad_score.DebugString(); - return; } if (IsIncomingBidInSellerCurrencyIllegallyModified( raw_request_.seller_currency(), ad_with_bid_currency, ad_score.incoming_bid_in_seller_currency(), buyer_bid)) { - // Ignore illegally modified AdScore. - // TODO(b/311234165): Add metric for ads rejected for mismatched currency. - PS_VLOG(1, log_context_) - << "Skipping ad_score as its incomingBidInSellerCurrency was modified " + ad_rejection_reason = ScoreAdsResponse::AdScore::AdRejectionReason{}; + ad_rejection_reason->set_interest_group_name( + ad_score.interest_group_name()); + ad_rejection_reason->set_interest_group_owner( + ad_score.interest_group_owner()); + ad_rejection_reason->set_rejection_reason( + SellerRejectionReason::INVALID_BID); + PS_VLOG(kInfoMsg, log_context_) + << "Skipping ad_score as its incomingBidInSellerCurrency was " + "modified " "when it should not have been: " << interest_group_name << ": " << ad_score.DebugString(); - return; } const bool is_valid_ad = @@ -589,7 +614,7 @@ void ScoreAdsReactor::HandleScoredAd(int index, float buyer_bid, // Ignore component level winner if it is not allowed to // participate in the top level auction. // TODO(b/311234165): Add metric for rejected component ads. - PS_VLOG(1, log_context_) + PS_VLOG(kInfoMsg, log_context_) << "Skipping component bid as it is not allowed for " << interest_group_name << ": " << ad_score.DebugString(); return; @@ -598,11 +623,13 @@ void ScoreAdsReactor::HandleScoredAd(int index, float buyer_bid, } if (is_valid_ad && ad_score.desirability() > 0) { - // Consider scored ad as valid (i.e. not rejected) when it has a positive - // desirability and either: + // Consider scored ad as valid (i.e. not rejected) when it has a + // positive desirability and either: // 1. scoreAd returned a number. - // 2. scoreAd returned an object but the reject reason was not populated. - // 3. scoreAd returned an object and the reject reason was explicitly set to + // 2. scoreAd returned an object but the reject reason was not + // populated. + // 3. scoreAd returned an object and the reject reason was explicitly + // set to // "not-available". // Only consider valid bids for populating other highest bids. scoring_data.score_ad_map[ad_score.desirability()].push_back(index); @@ -613,7 +640,8 @@ void ScoreAdsReactor::HandleScoredAd(int index, float buyer_bid, // desirability. if (ad_score.desirability() <= 0 && !ad_rejection_reason.has_value()) { PS_VLOG(5, log_context_) - << "Non-positive desirability for ad and no rejection reason populated " + << "Non-positive desirability for ad and no rejection reason " + "populated " "by seller, providing a default rejection reason"; ad_rejection_reason = ScoreAdsResponse::AdScore::AdRejectionReason{}; ad_rejection_reason->set_interest_group_owner(interest_group_owner); @@ -666,8 +694,8 @@ ScoringData ScoreAdsReactor::FindWinningAd( nullptr; FindScoredAdType(response->id, &ad, &protected_app_signals_ad_with_bid); if (!ad && !protected_app_signals_ad_with_bid) { - // This should never happen but we log here in case there is a bug in our - // implementation. + // This should never happen but we log here in case there is a bug in + // our implementation. ABSL_LOG(ERROR) << "Scored ad is neither a protected audience ad, nor a " "protected app " "signals ad: " @@ -698,14 +726,16 @@ ScoringData ScoreAdsReactor::FindWinningAd( if (ad) { HandleScoredAd(index, ad->bid(), ad->bid_currency(), ad->interest_group_name(), ad->interest_group_owner(), - *response_json, AdType::AD_TYPE_PROTECTED_AUDIENCE_AD, - *ad_score, scoring_data); + ad->interest_group_origin(), *response_json, + AdType::AD_TYPE_PROTECTED_AUDIENCE_AD, *ad_score, + scoring_data, response->id); } else { HandleScoredAd(index, protected_app_signals_ad_with_bid->bid(), /*ad_with_bid_currency=*/"", /*interest_group_name=*/"", - protected_app_signals_ad_with_bid->owner(), *response_json, + protected_app_signals_ad_with_bid->owner(), + /*interest_group_origin=*/"", *response_json, AdType::AD_TYPE_PROTECTED_APP_SIGNALS_AD, *ad_score, - scoring_data); + scoring_data, response->id); } } return scoring_data; @@ -774,17 +804,23 @@ void ScoreAdsReactor::PopulateHighestScoringOtherBidsData( auto* highest_scoring_other_bids_map = winning_ad_score.mutable_ig_owner_highest_scoring_other_bids_map(); - if (ad_with_bid_metadata_from_buyer) { - UpdateHighestScoringOtherBidMap( - ad_with_bid_metadata_from_buyer->bid(), - ad_with_bid_metadata_from_buyer->interest_group_owner(), - *highest_scoring_other_bids_map); + std::string owner; + float bid = 0.0; + if (ad_with_bid_metadata_from_buyer != nullptr) { + bid = ad_with_bid_metadata_from_buyer->bid(); + owner = ad_with_bid_metadata_from_buyer->interest_group_owner(); } else { - UpdateHighestScoringOtherBidMap( - protected_app_signals_ad_with_bid->bid(), - protected_app_signals_ad_with_bid->owner(), - *highest_scoring_other_bids_map); + bid = protected_app_signals_ad_with_bid->bid(); + owner = protected_app_signals_ad_with_bid->owner(); + } + if (!raw_request_.seller_currency().empty()) { + auto ad_score_it = ad_scores_.find(responses[current_index]->id); + if (ad_score_it != ad_scores_.end()) { + bid = ad_score_it->second->incoming_bid_in_seller_currency(); + } } + UpdateHighestScoringOtherBidMap(bid, owner, + *highest_scoring_other_bids_map); } } } @@ -866,41 +902,41 @@ void ScoreAdsReactor::ReportingCallback( ParseAndGetReportingResponse(enable_adtech_code_logging_, response.value().resp); if (!reporting_response.ok()) { - PS_VLOG(0, log_context_) << "Failed to parse response from Roma ", + PS_LOG(ERROR, log_context_) << "Failed to parse response from Roma ", reporting_response.status().ToString( absl::StatusToStringMode::kWithEverything); continue; } if (PS_VLOG_IS_ON(1) && enable_adtech_code_logging_) { for (std::string& log : reporting_response.value().seller_logs) { - PS_VLOG(1, log_context_) + PS_VLOG(kInfoMsg, log_context_) << "Log from Seller's execution script:" << log; } for (std::string& log : reporting_response.value().seller_error_logs) { - PS_VLOG(1, log_context_) + PS_LOG(ERROR, log_context_) << "Error Log from Seller's execution script:" << log; } for (std::string& log : reporting_response.value().seller_warning_logs) { - PS_VLOG(1, log_context_) + PS_LOG(ERROR, log_context_) << "Warning Log from Seller's execution script:" << log; } for (std::string& log : reporting_response.value().buyer_logs) { - PS_VLOG(1, log_context_) + PS_VLOG(kInfoMsg, log_context_) << "Log from Buyer's execution script:" << log; } for (std::string& log : reporting_response.value().buyer_error_logs) { - PS_VLOG(1, log_context_) + PS_LOG(ERROR, log_context_) << "Error Log from Buyer's execution script:" << log; } for (std::string& log : reporting_response.value().buyer_warning_logs) { - PS_VLOG(1, log_context_) + PS_LOG(ERROR, log_context_) << "Warning Log from Buyer's execution script:" << log; } } - // For component auctions, the reporting urls for seller are set in the - // component_seller_reporting_urls field. For single seller auctions and - // top level auctions, the reporting urls are set in the + // For component auctions, the reporting urls for seller are set in + // the component_seller_reporting_urls field. For single seller + // auctions and top level auctions, the reporting urls are set in the // top_level_seller_reporting_urls field. if (auction_scope_ == AuctionScope::AUCTION_SCOPE_DEVICE_COMPONENT_MULTI_SELLER) { @@ -969,9 +1005,9 @@ void ScoreAdsReactor::PerformDebugReporting( if (auction_scope_ == AuctionScope::AUCTION_SCOPE_SERVER_TOP_LEVEL_SELLER) { return; } - PostAuctionSignals post_auction_signals = - GeneratePostAuctionSignals(winning_ad_score); - for (const auto& ad_score : ad_scores_) { + PostAuctionSignals post_auction_signals = GeneratePostAuctionSignals( + winning_ad_score, raw_request_.seller_currency()); + for (const auto& [id, ad_score] : ad_scores_) { if (ad_score->has_debug_report_urls()) { absl::string_view debug_url; bool is_win_debug_url = false; @@ -1018,7 +1054,7 @@ void ScoreAdsReactor::PerformDebugReporting( void ScoreAdsReactor::EncryptAndFinishOK() { PS_VLOG(kPlain, log_context_) << "ScoreAdsRawResponse:\n" - << raw_response_.DebugString(); + << raw_response_.ShortDebugString(); EncryptResponse(); PS_VLOG(kEncrypted, log_context_) << "Encrypted ScoreAdsResponse\n" << response_->ShortDebugString(); @@ -1033,4 +1069,86 @@ void ScoreAdsReactor::FinishWithStatus(const grpc::Status& status) { Finish(status); } +void ScoreAdsReactor::DispatchReportingRequestForPA( + const ScoreAdsResponse::AdScore& winning_ad_score, + const std::shared_ptr& auction_config, + const BuyerReportingMetadata& buyer_reporting_metadata) { + ReportingDispatchRequestData dispatch_request_data = { + .handler_name = kReportingDispatchHandlerFunctionName, + .auction_config = auction_config, + .post_auction_signals = GeneratePostAuctionSignals( + winning_ad_score, raw_request_.seller_currency()), + .publisher_hostname = raw_request_.publisher_hostname(), + .log_context = log_context_, + .buyer_reporting_metadata = buyer_reporting_metadata}; + if (auction_scope_ == + AuctionScope::AUCTION_SCOPE_DEVICE_COMPONENT_MULTI_SELLER || + auction_scope_ == + AuctionScope::AUCTION_SCOPE_SERVER_COMPONENT_MULTI_SELLER) { + dispatch_request_data.component_reporting_metadata = { + .top_level_seller = raw_request_.top_level_seller(), + .component_seller = raw_request_.seller()}; + } + if (winning_ad_score.bid() > 0) { + dispatch_request_data.component_reporting_metadata.modified_bid = + winning_ad_score.bid(); + } else { + dispatch_request_data.component_reporting_metadata.modified_bid = + winning_ad_score.buyer_bid(); + } + // By this point in the logic, the bid currency has already been set + // to refer to the modified bid if present or the buyer_bid if not (by + // CheckAndUpdateModifiedBid() in HandleScoredAd()), so no logic is needed + // to find the right currency. + dispatch_request_data.component_reporting_metadata.modified_bid_currency = + winning_ad_score.bid_currency(); + DispatchReportingRequest(dispatch_request_data); +} + +void ScoreAdsReactor::DispatchReportingRequestForPAS( + const ScoreAdsResponse::AdScore& winning_ad_score, + const std::shared_ptr& auction_config, + const BuyerReportingMetadata& buyer_reporting_metadata, + std::string_view egress_payload, + absl::string_view temporary_egress_payload) { + DispatchReportingRequest( + {.handler_name = kReportingProtectedAppSignalsFunctionName, + .auction_config = auction_config, + .post_auction_signals = GeneratePostAuctionSignals( + winning_ad_score, raw_request_.seller_currency()), + .publisher_hostname = raw_request_.publisher_hostname(), + .log_context = log_context_, + .buyer_reporting_metadata = buyer_reporting_metadata, + .egress_payload = egress_payload, + .temporary_egress_payload = temporary_egress_payload}); +} + +void ScoreAdsReactor::DispatchReportingRequest( + const ReportingDispatchRequestData& dispatch_request_data) { + ReportingDispatchRequestConfig dispatch_request_config = { + .enable_report_win_url_generation = enable_report_win_url_generation_, + .enable_protected_app_signals = enable_protected_app_signals_, + .enable_report_win_input_noising = enable_report_win_input_noising_, + .enable_adtech_code_logging = enable_adtech_code_logging_}; + DispatchRequest dispatch_request = GetReportingDispatchRequest( + dispatch_request_config, dispatch_request_data); + dispatch_request.tags[kRomaTimeoutMs] = roma_timeout_ms_; + + std::vector dispatch_requests = {dispatch_request}; + auto status = dispatcher_.BatchExecute( + dispatch_requests, + [this](const std::vector>& result) { + ReportingCallback(result); + }); + + if (!status.ok()) { + std::string original_request; + google::protobuf::TextFormat::PrintToString(raw_request_, + &original_request); + PS_LOG(ERROR, log_context_) + << "Reporting execution request failed for batch: " << original_request + << status.ToString(absl::StatusToStringMode::kWithEverything); + EncryptAndFinishOK(); + } +} } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/score_ads_reactor.h b/services/auction_service/score_ads_reactor.h index 5c40757a..c0738b4f 100644 --- a/services/auction_service/score_ads_reactor.h +++ b/services/auction_service/score_ads_reactor.h @@ -51,7 +51,6 @@ inline constexpr char kNoAdsWithValidScoringSignals[] = "No ads with valid scoring signals."; inline constexpr char kNoValidComponentAuctions[] = "No component auction results in request."; - // An aggregate of the data we track when scoring all the ads. struct ScoringData { // Index of the most desirable ad. This helps us to set the overall response @@ -147,74 +146,17 @@ class ScoreAdsReactor void DispatchReportingRequestForPA( const ScoreAdsResponse::AdScore& winning_ad_score, const std::shared_ptr& auction_config, - const BuyerReportingMetadata& buyer_reporting_metadata) { - ReportingDispatchRequestData dispatch_request_data = { - .handler_name = kReportingDispatchHandlerFunctionName, - .auction_config = auction_config, - .post_auction_signals = GeneratePostAuctionSignals(winning_ad_score), - .publisher_hostname = raw_request_.publisher_hostname(), - .log_context = log_context_, - .buyer_reporting_metadata = buyer_reporting_metadata}; - if (auction_scope_ == - AuctionScope::AUCTION_SCOPE_DEVICE_COMPONENT_MULTI_SELLER) { - dispatch_request_data.component_reporting_metadata = { - .top_level_seller = raw_request_.top_level_seller(), - .component_seller = raw_request_.seller()}; - } - if (winning_ad_score.bid() > 0) { - dispatch_request_data.component_reporting_metadata.modified_bid = - winning_ad_score.bid(); - } else { - dispatch_request_data.component_reporting_metadata.modified_bid = - winning_ad_score.buyer_bid(); - } - DispatchReportingRequest(dispatch_request_data); - } + const BuyerReportingMetadata& buyer_reporting_metadata); void DispatchReportingRequestForPAS( const ScoreAdsResponse::AdScore& winning_ad_score, const std::shared_ptr& auction_config, const BuyerReportingMetadata& buyer_reporting_metadata, - std::string_view egress_features) { - DispatchReportingRequest( - {.handler_name = kReportingProtectedAppSignalsFunctionName, - .auction_config = auction_config, - .post_auction_signals = GeneratePostAuctionSignals(winning_ad_score), - .publisher_hostname = raw_request_.publisher_hostname(), - .log_context = log_context_, - .buyer_reporting_metadata = buyer_reporting_metadata, - .egress_features = egress_features}); - } + std::string_view egress_payload, + absl::string_view temporary_egress_payload); void DispatchReportingRequest( - const ReportingDispatchRequestData& dispatch_request_data) { - ReportingDispatchRequestConfig dispatch_request_config = { - .enable_report_win_url_generation = enable_report_win_url_generation_, - .enable_protected_app_signals = enable_protected_app_signals_, - .enable_report_win_input_noising = enable_report_win_input_noising_, - .enable_adtech_code_logging = enable_adtech_code_logging_}; - DispatchRequest dispatch_request = GetReportingDispatchRequest( - dispatch_request_config, dispatch_request_data); - dispatch_request.tags[kRomaTimeoutMs] = roma_timeout_ms_; - - std::vector dispatch_requests = {dispatch_request}; - auto status = dispatcher_.BatchExecute( - dispatch_requests, - [this](const std::vector>& result) { - ReportingCallback(result); - }); - - if (!status.ok()) { - std::string original_request; - google::protobuf::TextFormat::PrintToString(raw_request_, - &original_request); - PS_VLOG(1, log_context_) - << "Reporting execution request failed for batch: " - << original_request - << status.ToString(absl::StatusToStringMode::kWithEverything); - EncryptAndFinishOK(); - } - } + const ReportingDispatchRequestData& dispatch_request_data); void PerformReporting(const ScoreAdsResponse::AdScore& winning_ad_score, absl::string_view id); @@ -264,9 +206,11 @@ class ScoreAdsReactor absl::string_view ad_with_bid_currency, absl::string_view interest_group_name, absl::string_view interest_group_owner, + absl::string_view interest_group_origin, const rapidjson::Document& response_json, AdType ad_type, ScoreAdsResponse::AdScore& score_ads_response, - ScoringData& scoring_data); + ScoringData& scoring_data, + const std::string& dispatch_response_id); // The key is the id of the DispatchRequest, and the value is the ad // used to create the dispatch request. This map is used to amend each ad's @@ -288,7 +232,9 @@ class ScoreAdsReactor // Used to log metric, same life time as reactor. std::unique_ptr metric_context_; - std::vector> ad_scores_; + // Used for debug reporting. Keyed on Roma dispatch ID. + absl::flat_hash_map> + ad_scores_; // Flags needed to be passed as input to the code which wraps AdTech provided // code. @@ -306,6 +252,9 @@ class ScoreAdsReactor // parsing of scoreAd output. AuctionScope auction_scope_; + // Specifies which verison of scoreAd to use for this request. + absl::string_view code_version_; + google::protobuf::RepeatedPtrField GetEmptyAdComponentRenderUrls() { static google::protobuf::RepeatedPtrField diff --git a/services/auction_service/score_ads_reactor_test.cc b/services/auction_service/score_ads_reactor_test.cc index e1d42a75..c9360cd6 100644 --- a/services/auction_service/score_ads_reactor_test.cc +++ b/services/auction_service/score_ads_reactor_test.cc @@ -21,6 +21,7 @@ #include #include "absl/container/flat_hash_set.h" +#include "absl/flags/flag.h" #include "absl/status/status.h" #include "absl/strings/str_format.h" #include "gmock/gmock.h" @@ -37,9 +38,11 @@ #include "services/common/constants/common_service_flags.h" #include "services/common/encryption/key_fetcher_factory.h" #include "services/common/encryption/mock_crypto_client_wrapper.h" +#include "services/common/feature_flags.h" #include "services/common/metric/server_definition.h" #include "services/common/test/mocks.h" #include "services/common/test/random.h" +#include "services/common/util/proto_util.h" #include "src/encryption/key_fetcher/interface/key_fetcher_manager_interface.h" #include "src/encryption/key_fetcher/mock/mock_key_fetcher_manager.h" @@ -55,14 +58,17 @@ constexpr char kTestReportingWinResponseJson[] = R"({"reportResultResponse":{"reportResultUrl":"http://reportResultUrl.com","signalsForWinner":"{testKey:testValue}","sendReportToInvoked":true,"registerAdBeaconInvoked":true,"interactionReportingUrls":{"click":"http://event.com"}},"sellerLogs":["testLog"], "sellerErrors":["testLog"], "sellerWarnings":["testLog"], "reportWinResponse":{"reportWinUrl":"http://reportWinUrl.com","sendReportToInvoked":true,"registerAdBeaconInvoked":true,"interactionReportingUrls":{"click":"http://event.com"}},"buyerLogs":["testLog"], "buyerErrors":["testLog"], "buyerWarnings":["testLog"]})"; constexpr int kTestDesirability = 5; +constexpr float kNotAnyOriginalBid = 1689.7; constexpr char kTestConsentToken[] = "testConsentedToken"; -constexpr char kTestEgressFeatures[] = "testEgressFeatures"; +constexpr char kTestEgressPayload[] = "testEgressPayload"; +constexpr char kTestTemporaryEgressPayload[] = "testTemporaryEgressPayload"; constexpr char kTestComponentReportResultUrl[] = "http://reportResultUrl.com&bid=2.10&modifiedBid=1.0"; constexpr char kTestComponentReportWinUrl[] = "http://reportWinUrl.com&bid=2.10&modifiedBid=1.0"; constexpr char kUsdIsoCode[] = "USD"; constexpr char kEuroIsoCode[] = "EUR"; +constexpr char kSterlingIsoCode[] = "GBP"; constexpr char kFooComponentsRenderUrl[] = "fooMeOnceAds.com/render_ad?id=hasFooComp"; @@ -130,6 +136,20 @@ constexpr char kComponentWithCurrencyAdScore[] = R"( } )"; +constexpr char kComponentWithCurrencyAndIncomingAndRejectReasAdScore[] = R"( + { + "response" : { + "desirability" : $0, + "allowComponentAuction" : $1, + "incomingBidInSellerCurrency": $2, + "bid" : $3, + "bidCurrency": "$4", + "rejectReason" : "$5" + }, + "logs":[] + } +)"; + using ::google::protobuf::TextFormat; using ::testing::AnyNumber; using ::testing::HasSubstr; @@ -140,43 +160,55 @@ using AdWithBidMetadata = ScoreAdsRequest::ScoreAdsRawRequest::AdWithBidMetadata; using ProtectedAppSignalsAdWithBidMetadata = ScoreAdsRequest::ScoreAdsRawRequest::ProtectedAppSignalsAdWithBidMetadata; +using ::google::protobuf::util::MessageDifferencer; constexpr char kInterestGroupOwnerOfBarBidder[] = "barStandardAds.com"; +constexpr char kInterestGroupOrigin[] = "candy_smush"; -void GetTestAdWithBidFoo(AdWithBidMetadata& foo) { +void GetTestAdWithBidFoo(AdWithBidMetadata& foo, + absl::string_view buyer_reporting_id = "") { int number_of_component_ads = 3; foo.mutable_ad()->mutable_struct_value()->MergeFrom( MakeAnAd("https://googleads.g.doubleclick.net/td/adfetch/" "gda?adg_id=142601302539&cr_id=628073386727&cv_id=0", "arbitraryMetadataKey", 2)); + + // This must have an entry in kTestScoringSignals. foo.set_render( "https://googleads.g.doubleclick.net/td/adfetch/" "gda?adg_id=142601302539&cr_id=628073386727&cv_id=0"); foo.set_bid(2.10); foo.set_interest_group_name("foo"); foo.set_interest_group_owner("https://fooAds.com"); + foo.set_interest_group_origin(kInterestGroupOrigin); for (int i = 0; i < number_of_component_ads; i++) { foo.add_ad_components( absl::StrCat("adComponent.com/foo_components/id=", i)); } + if (!buyer_reporting_id.empty()) { + foo.set_buyer_reporting_id(kTestBuyerReportingId); + } } void GetTestAdWithBidSameComponentAsFoo(AdWithBidMetadata& foo) { int number_of_component_ads = 3; foo.mutable_ad()->mutable_struct_value()->MergeFrom( MakeAnAd(kFooComponentsRenderUrl, "arbitraryMetadataKey", 2)); + // This must have an entry in kTestScoringSignals. foo.set_render(kFooComponentsRenderUrl); foo.set_bid(2.10); foo.set_bid_currency(kUsdIsoCode); foo.set_interest_group_name("foo"); foo.set_interest_group_owner("https://fooAds.com"); + foo.set_interest_group_origin(kInterestGroupOrigin); for (int i = 0; i < number_of_component_ads; i++) { foo.add_ad_components( absl::StrCat("adComponent.com/foo_components/id=", i)); } } -void GetTestAdWithBidBar(AdWithBidMetadata& bar) { +void GetTestAdWithBidBar(AdWithBidMetadata& bar, + absl::string_view buyer_reporting_id = "") { int number_of_component_ads = 3; std::string render_url = "barStandardAds.com/render_ad?id=bar"; auto* bar_ad_map = bar.mutable_ad()->mutable_struct_value()->mutable_fields(); @@ -187,16 +219,20 @@ void GetTestAdWithBidBar(AdWithBidMetadata& bar) { MakeANullValue(), MakeAStringValue("18281019067"), })); - + // This must have an entry in kTestScoringSignals. bar.set_render(render_url); bar.set_bid(2); bar.set_bid_currency(kUsdIsoCode); bar.set_interest_group_name("ig_bar"); bar.set_interest_group_owner(kInterestGroupOwnerOfBarBidder); + bar.set_interest_group_origin(kInterestGroupOrigin); for (int i = 0; i < number_of_component_ads; i++) { bar.add_ad_components( absl::StrCat("adComponent.com/bar_components/id=", i)); } + if (!buyer_reporting_id.empty()) { + bar.set_buyer_reporting_id(buyer_reporting_id); + } } void GetTestAdWithBidBarbecue(AdWithBidMetadata& ad_with_bid) { @@ -210,17 +246,40 @@ void GetTestAdWithBidBarbecue(AdWithBidMetadata& ad_with_bid) { MakeAStringValue("pulled_pork"), MakeAStringValue("smoked_chicken"), })); - + // This must have an entry in kTestScoringSignals. ad_with_bid.set_render(render_url); ad_with_bid.set_bid(17.76); ad_with_bid.set_interest_group_name("barbecue_lovers"); ad_with_bid.set_interest_group_owner("barStandardAds.com"); + ad_with_bid.set_interest_group_origin(kInterestGroupOrigin); for (int i = 0; i < number_of_component_ads; i++) { ad_with_bid.add_ad_components( absl::StrCat("barStandardAds.com/ad_components/id=", i)); } } +void GetTestAdWithBidBoots(AdWithBidMetadata& ad_with_bid) { + int number_of_component_ads = 0; + std::string render_url = "bootScootin.com/render_ad?id=cowboy_boots"; + auto* ad_map = + ad_with_bid.mutable_ad()->mutable_struct_value()->mutable_fields(); + ad_map->try_emplace("renderUrl", MakeAStringValue(render_url)); + ad_map->try_emplace("metadata", MakeAListValue({ + MakeAStringValue("french_toe"), + MakeAStringValue("cherry_goat"), + MakeAStringValue("lucchese"), + })); + // This must have an entry in kTestScoringSignals. + ad_with_bid.set_render(render_url); + ad_with_bid.set_bid(12.15); + ad_with_bid.set_interest_group_name("western_boot_lovers"); + ad_with_bid.set_interest_group_owner("bootScootin.com"); + for (int i = 0; i < number_of_component_ads; i++) { + ad_with_bid.add_ad_components( + absl::StrCat("bootScootin.com/ad_components/id=", i)); + } +} + void GetTestAdWithBidBarbecueWithComponents(AdWithBidMetadata& ad_with_bid) { int number_of_component_ads = 1; std::string render_url = "barStandardAds.com/render_ad?id=barbecue2"; @@ -233,10 +292,12 @@ void GetTestAdWithBidBarbecueWithComponents(AdWithBidMetadata& ad_with_bid) { MakeAStringValue("smoked_chicken"), })); + // This must have an entry in kTestScoringSignals. ad_with_bid.set_render(render_url); ad_with_bid.set_bid(17.76); ad_with_bid.set_interest_group_name("barbecue_lovers"); ad_with_bid.set_interest_group_owner("barStandardAds.com"); + ad_with_bid.set_interest_group_origin(kInterestGroupOrigin); for (int i = 0; i < number_of_component_ads; i++) { ad_with_bid.add_ad_components( absl::StrCat("barStandardAds.com/ad_components/id=", i)); @@ -267,6 +328,10 @@ constexpr char kTestScoringSignals[] = R"json( "fooMeOnceAds.com/render_ad?id=hasFooComp": [ 1689, 1868 + ], + "bootScootin.com/render_ad?id=cowboy_boots": [ + 1689, + 1868 ] }, "adComponentRenderUrls": { @@ -337,8 +402,20 @@ void BuildRawRequestForComponentAuction( } // namespace -void CheckInputCorrectForFoo(std::vector> input) { - EXPECT_EQ(*input[0], R"JSON({"arbitraryMetadataKey":2})JSON"); +void CheckInputCorrectForFoo(std::vector> input, + const google::protobuf::Value& expected_ad) { + auto observed_ad = + JsonStringToValue(*input[static_cast(ScoreAdArgs::kAdMetadata)]); + CHECK_OK(observed_ad) << "Malformed observed ad JSON: " + << *input[static_cast(ScoreAdArgs::kAdMetadata)]; + std::string difference; + google::protobuf::util::MessageDifferencer differencer; + differencer.ReportDifferencesToString(&difference); + EXPECT_TRUE(differencer.Compare(*observed_ad, expected_ad)) + << "\n Observed:\n" + << observed_ad->DebugString() << "\n\nExpected:\n" + << expected_ad.DebugString() << "\n\nDifference:\n" + << difference; EXPECT_EQ(*input[1], R"JSON(2.100000)JSON"); EXPECT_EQ( *input[2], @@ -348,13 +425,24 @@ void CheckInputCorrectForFoo(std::vector> input) { R"JSON({"adComponentRenderUrls":{"adComponent.com/foo_components/id=0":["foo0"],"adComponent.com/foo_components/id=1":["foo1"],"adComponent.com/foo_components/id=2":["foo2"]},"renderUrl":{"https://googleads.g.doubleclick.net/td/adfetch/gda?adg_id=142601302539&cr_id=628073386727&cv_id=0":[123]}})JSON"); EXPECT_EQ( *input[4], - R"JSON({"interestGroupOwner":"https://fooAds.com","topWindowHostname":"publisher_hostname","adComponents":["adComponent.com/foo_components/id=0","adComponent.com/foo_components/id=1","adComponent.com/foo_components/id=2"],"renderUrl":"https://googleads.g.doubleclick.net/td/adfetch/gda?adg_id=142601302539&cr_id=628073386727&cv_id=0"})JSON"); + R"JSON({"interestGroupOwner":"https://fooAds.com","topWindowHostname":"publisher_hostname","adComponents":["adComponent.com/foo_components/id=0","adComponent.com/foo_components/id=1","adComponent.com/foo_components/id=2"],"bidCurrency":"???","renderUrl":"https://googleads.g.doubleclick.net/td/adfetch/gda?adg_id=142601302539&cr_id=628073386727&cv_id=0"})JSON"); EXPECT_EQ(*input[5], "{}"); } -void CheckInputCorrectForBar(std::vector> input) { - EXPECT_EQ(*input[0], - R"JSON(["140583167746","627640802621",null,"18281019067"])JSON"); +void CheckInputCorrectForBar(std::vector> input, + const google::protobuf::Value& expected_ad) { + auto observed_ad = + JsonStringToValue(*input[static_cast(ScoreAdArgs::kAdMetadata)]); + CHECK_OK(observed_ad) << "Malformed observed ad JSON: " + << *input[static_cast(ScoreAdArgs::kAdMetadata)]; + std::string difference; + google::protobuf::util::MessageDifferencer differencer; + differencer.ReportDifferencesToString(&difference); + EXPECT_TRUE(differencer.Compare(*observed_ad, expected_ad)) + << "\n Observed:\n" + << observed_ad->DebugString() << "\n\nExpected:\n" + << expected_ad.DebugString() << "\n\nDifference:\n" + << difference; EXPECT_EQ(*input[1], R"JSON(2.000000)JSON"); EXPECT_EQ( *input[2], @@ -369,8 +457,15 @@ void CheckInputCorrectForBar(std::vector> input) { } void CheckInputCorrectForAdWithFooComp( - std::vector> input) { - EXPECT_EQ(*input[0], R"JSON({"arbitraryMetadataKey":2})JSON"); + std::vector> input, + const google::protobuf::Value& expected_ad) { + auto observed_ad = + JsonStringToValue(*input[static_cast(ScoreAdArgs::kAdMetadata)]); + CHECK_OK(observed_ad) << "Malformed observed ad JSON: " + << *input[static_cast(ScoreAdArgs::kAdMetadata)]; + std::string difference; + google::protobuf::util::MessageDifferencer differencer; + differencer.ReportDifferencesToString(&difference); EXPECT_EQ(*input[1], R"JSON(2.100000)JSON"); EXPECT_EQ( *input[2], @@ -448,22 +543,46 @@ TEST_F( ScoreAdsReactorTest, CreatesScoreAdInputsWithParsedScoringSignalsForTwoAdsWithSameComponents) { MockCodeDispatchClient dispatcher; - + auto expected_foo_ad = JsonStringToValue( + R"JSON( + { + "metadata": {"arbitraryMetadataKey": 2}, + "renderUrl": "https://googleads.g.doubleclick.net/td/adfetch/gda?adg_id=142601302539&cr_id=628073386727&cv_id=0" + })JSON"); + CHECK_OK(expected_foo_ad) << "Malformed ad JSON"; + auto expected_foo_comp_ad_1 = JsonStringToValue( + R"JSON( + { + "renderUrl": "https://googleads.g.doubleclick.net/td/adfetch/gda?adg_id=142601302539&cr_id=628073386727&cv_id=0", + "metadata": {"arbitraryMetadataKey": 2} + })JSON"); + CHECK_OK(expected_foo_comp_ad_1) << "Malformed ad JSON"; + auto expected_foo_comp_ad_2 = JsonStringToValue( + R"JSON( + { + "metadata": {"arbitraryMetadataKey": 2}, + "renderUrl": "fooMeOnceAds.com/render_ad?id=hasFooComp" + })JSON"); + CHECK_OK(expected_foo_comp_ad_2) << "Malformed ad JSON"; EXPECT_CALL(dispatcher, BatchExecute) - .WillOnce([](std::vector& batch, - BatchDispatchDoneCallback done_callback) { - EXPECT_EQ(batch.size(), 2); - if (batch.size() == 2) { - if (batch.at(0).id == kFooComponentsRenderUrl) { - CheckInputCorrectForAdWithFooComp(batch.at(0).input); - CheckInputCorrectForFoo(batch.at(1).input); - } else { - CheckInputCorrectForAdWithFooComp(batch.at(1).input); - CheckInputCorrectForFoo(batch.at(0).input); - } - } - return absl::OkStatus(); - }); + .WillOnce( + [expected_foo_ad, expected_foo_comp_ad_1, expected_foo_comp_ad_2]( + std::vector& batch, + BatchDispatchDoneCallback done_callback) { + EXPECT_EQ(batch.size(), 2); + if (batch.size() == 2) { + if (batch.at(0).id == kFooComponentsRenderUrl) { + CheckInputCorrectForAdWithFooComp(batch.at(0).input, + *expected_foo_comp_ad_1); + CheckInputCorrectForFoo(batch.at(1).input, *expected_foo_ad); + } else { + CheckInputCorrectForAdWithFooComp(batch.at(1).input, + *expected_foo_comp_ad_2); + CheckInputCorrectForFoo(batch.at(0).input, *expected_foo_ad); + } + } + return absl::OkStatus(); + }); RawRequest raw_request; AdWithBidMetadata foo, has_foo_components; GetTestAdWithBidFoo(foo); @@ -479,17 +598,52 @@ TEST_F(ScoreAdsReactorTest, CreatesScoreAdInputsInCorrectOrderWithParsedScoringSignalsForTwoAds) { MockCodeDispatchClient dispatcher; + auto expected_foo_ad_1 = JsonStringToValue( + R"JSON( + { + "metadata": { + "arbitraryMetadataKey": 2 + }, + "renderUrl": "fooMeOnceAds.com/render_ad?id=hasFooComp" + })JSON"); + CHECK_OK(expected_foo_ad_1) << "Malformed ad JSON"; + auto expected_foo_ad_2 = JsonStringToValue( + R"JSON( + { + "metadata": { + "arbitraryMetadataKey": 2 + }, + "renderUrl": "https://googleads.g.doubleclick.net/td/adfetch/gda?adg_id=142601302539&cr_id=628073386727&cv_id=0" + })JSON"); + CHECK_OK(expected_foo_ad_2) << "Malformed ad JSON"; + auto expected_bar_ad_1 = JsonStringToValue( + R"JSON( + { + "metadata": { + "arbitraryMetadataKey":2 + }, + "renderUrl": "https://googleads.g.doubleclick.net/td/adfetch/gda?adg_id=142601302539&cr_id=628073386727&cv_id=0" + })JSON"); + CHECK_OK(expected_bar_ad_1) << "Malformed ad JSON"; + auto expected_bar_ad_2 = JsonStringToValue( + R"JSON( + { + "metadata": ["140583167746", "627640802621", null, "18281019067"], + "renderUrl": "barStandardAds.com/render_ad?id=bar" + })JSON"); + CHECK_OK(expected_bar_ad_2) << "Malformed ad JSON"; EXPECT_CALL(dispatcher, BatchExecute) - .WillOnce([](std::vector& batch, - BatchDispatchDoneCallback done_callback) { + .WillOnce([expected_bar_ad_1, expected_bar_ad_2, expected_foo_ad_1, + expected_foo_ad_2](std::vector& batch, + BatchDispatchDoneCallback done_callback) { EXPECT_EQ(batch.size(), 2); if (batch.size() == 2) { if (batch.at(0).id == "barStandardAds.com/render_ad?id=bar") { - CheckInputCorrectForFoo(batch.at(1).input); - CheckInputCorrectForBar(batch.at(0).input); + CheckInputCorrectForFoo(batch.at(1).input, *expected_foo_ad_2); + CheckInputCorrectForBar(batch.at(0).input, *expected_bar_ad_2); } else { - CheckInputCorrectForFoo(batch.at(0).input); - CheckInputCorrectForBar(batch.at(1).input); + CheckInputCorrectForFoo(batch.at(0).input, *expected_foo_ad_1); + CheckInputCorrectForBar(batch.at(1).input, *expected_bar_ad_1); } } return absl::OkStatus(); @@ -507,12 +661,35 @@ TEST_F(ScoreAdsReactorTest, TEST_F(ScoreAdsReactorTest, CreatesScoreAdInputsInCorrectOrderWithParsedScoringSignals) { MockCodeDispatchClient dispatcher; + auto expected_foo_ad_1 = JsonStringToValue( + R"JSON( + { + "metadata": { + "arbitraryMetadataKey": 2 + }, + "renderUrl": "fooMeOnceAds.com/render_ad?id=hasFooComp" + })JSON"); + CHECK_OK(expected_foo_ad_1) << "Malformed ad JSON"; + auto expected_foo_ad_2 = JsonStringToValue( + R"JSON( + { + "metadata": { + "arbitraryMetadataKey": 2 + }, + "renderUrl": "https://googleads.g.doubleclick.net/td/adfetch/gda?adg_id=142601302539&cr_id=628073386727&cv_id=0" + })JSON"); + CHECK_OK(expected_foo_ad_2) << "Malformed ad JSON"; EXPECT_CALL(dispatcher, BatchExecute) - .WillOnce([](std::vector& batch, - BatchDispatchDoneCallback done_callback) { + .WillOnce([expected_foo_ad_1, expected_foo_ad_2]( + std::vector& batch, + BatchDispatchDoneCallback done_callback) { for (const auto& request : batch) { - CheckInputCorrectForFoo(request.input); + if (request.id == "fooMeOnceAds.com/render_ad?id=hasFooComp") { + CheckInputCorrectForFoo(request.input, *expected_foo_ad_1); + } else { + CheckInputCorrectForFoo(request.input, *expected_foo_ad_2); + } } return absl::OkStatus(); }); @@ -527,12 +704,32 @@ TEST_F(ScoreAdsReactorTest, TEST_F(ScoreAdsReactorTest, CreatesScoreAdInputsInCorrectOrderWithParsedScoringSignalsForBar) { MockCodeDispatchClient dispatcher; - + auto expected_bar_ad_1 = JsonStringToValue( + R"JSON( + { + "metadata": { + "arbitraryMetadataKey":2 + }, + "renderUrl": "https://googleads.g.doubleclick.net/td/adfetch/gda?adg_id=142601302539&cr_id=628073386727&cv_id=0" + })JSON"); + CHECK_OK(expected_bar_ad_1) << "Malformed ad JSON"; + auto expected_bar_ad_2 = JsonStringToValue( + R"JSON( + { + "metadata": ["140583167746", "627640802621", null, "18281019067"], + "renderUrl": "barStandardAds.com/render_ad?id=bar" + })JSON"); + CHECK_OK(expected_bar_ad_2) << "Malformed ad JSON"; EXPECT_CALL(dispatcher, BatchExecute) - .WillOnce([](std::vector& batch, - BatchDispatchDoneCallback done_callback) { + .WillOnce([expected_bar_ad_1, expected_bar_ad_2]( + std::vector& batch, + BatchDispatchDoneCallback done_callback) { for (const auto& request : batch) { - CheckInputCorrectForBar(request.input); + if (batch.at(0).id == "barStandardAds.com/render_ad?id=bar") { + CheckInputCorrectForBar(request.input, *expected_bar_ad_2); + } else { + CheckInputCorrectForBar(request.input, *expected_bar_ad_1); + } } return absl::OkStatus(); }); @@ -547,14 +744,28 @@ TEST_F(ScoreAdsReactorTest, TEST_F(ScoreAdsReactorTest, CreatesCorrectScoreAdInputsWithParsedScoringSignalsForNoComponentAds) { MockCodeDispatchClient dispatcher; - + auto expected_ad = JsonStringToValue( + R"JSON( + { + "metadata":["brisket","pulled_pork","smoked_chicken"], + "renderUrl":"barStandardAds.com/render_ad?id=barbecue" + })JSON"); + CHECK_OK(expected_ad) << "Malformed ad JSON"; EXPECT_CALL(dispatcher, BatchExecute) - .WillOnce([](std::vector& batch, - BatchDispatchDoneCallback done_callback) { + .WillOnce([expected_ad](std::vector& batch, + BatchDispatchDoneCallback done_callback) { for (const auto& request : batch) { auto input = request.input; - EXPECT_EQ(*input[0], - R"JSON(["brisket","pulled_pork","smoked_chicken"])JSON"); + auto observed_ad = JsonStringToValue( + *input[static_cast(ScoreAdArgs::kAdMetadata)]); + CHECK_OK(observed_ad) + << "Malformed observed ad JSON: " + << *input[static_cast(ScoreAdArgs::kAdMetadata)]; + std::string difference; + google::protobuf::util::MessageDifferencer differencer; + differencer.ReportDifferencesToString(&difference); + EXPECT_TRUE(differencer.Compare(*observed_ad, *expected_ad)) + << difference; EXPECT_EQ(*input[1], R"JSON(17.760000)JSON"); EXPECT_EQ( *input[2], @@ -564,7 +775,7 @@ TEST_F(ScoreAdsReactorTest, R"JSON({"adComponentRenderUrls":{},"renderUrl":{"barStandardAds.com/render_ad?id=barbecue":[1689,1868]}})JSON"); EXPECT_EQ( *input[4], - R"JSON({"interestGroupOwner":"barStandardAds.com","topWindowHostname":"publisher_hostname","renderUrl":"barStandardAds.com/render_ad?id=barbecue"})JSON"); + R"JSON({"interestGroupOwner":"barStandardAds.com","topWindowHostname":"publisher_hostname","bidCurrency":"???","renderUrl":"barStandardAds.com/render_ad?id=barbecue"})JSON"); } return absl::OkStatus(); }); @@ -580,13 +791,28 @@ TEST_F(ScoreAdsReactorTest, CreatesScoreAdInputsWellWithParsedScoringSignalsButNotForComponentAds) { MockCodeDispatchClient dispatcher; + auto expected_ad = JsonStringToValue( + R"JSON( + { + "renderUrl": "barStandardAds.com/render_ad?id=barbecue2", + "metadata":["brisket","pulled_pork","smoked_chicken"] + })JSON"); + CHECK_OK(expected_ad) << "Malformed ad JSON"; EXPECT_CALL(dispatcher, BatchExecute) - .WillOnce([](std::vector& batch, - BatchDispatchDoneCallback done_callback) { + .WillOnce([expected_ad](std::vector& batch, + BatchDispatchDoneCallback done_callback) { for (const auto& request : batch) { auto input = request.input; - EXPECT_EQ(*input[0], - R"JSON(["brisket","pulled_pork","smoked_chicken"])JSON"); + auto observed_ad = JsonStringToValue( + *input[static_cast(ScoreAdArgs::kAdMetadata)]); + CHECK_OK(observed_ad) + << "Malformed observed ad JSON: " + << *input[static_cast(ScoreAdArgs::kAdMetadata)]; + std::string difference; + google::protobuf::util::MessageDifferencer differencer; + differencer.ReportDifferencesToString(&difference); + EXPECT_TRUE(differencer.Compare(*observed_ad, *expected_ad)) + << difference; EXPECT_EQ(*input[1], R"JSON(17.760000)JSON"); EXPECT_EQ( *input[2], @@ -596,7 +822,7 @@ TEST_F(ScoreAdsReactorTest, R"JSON({"adComponentRenderUrls":{},"renderUrl":{"barStandardAds.com/render_ad?id=barbecue2":[1689,1868]}})JSON"); EXPECT_EQ( *input[4], - R"JSON({"interestGroupOwner":"barStandardAds.com","topWindowHostname":"publisher_hostname","adComponents":["barStandardAds.com/ad_components/id=0"],"renderUrl":"barStandardAds.com/render_ad?id=barbecue2"})JSON"); + R"JSON({"interestGroupOwner":"barStandardAds.com","topWindowHostname":"publisher_hostname","adComponents":["barStandardAds.com/ad_components/id=0"],"bidCurrency":"???","renderUrl":"barStandardAds.com/render_ad?id=barbecue2"})JSON"); } return absl::OkStatus(); }); @@ -739,6 +965,9 @@ TEST_F(ScoreAdsReactorTest, original_ad_with_bid.interest_group_name()); EXPECT_EQ(scored_ad.interest_group_owner(), original_ad_with_bid.interest_group_owner()); + EXPECT_EQ(scored_ad.interest_group_origin(), + original_ad_with_bid.interest_group_origin()); + EXPECT_EQ(scored_ad.interest_group_origin(), kInterestGroupOrigin); EXPECT_EQ(scored_ad.buyer_bid(), original_ad_with_bid.bid()); // Not a component auction, bid should not be set. EXPECT_FLOAT_EQ(scored_ad.bid(), 0.0f); @@ -763,16 +992,17 @@ TEST_F(ScoreAdsReactorTest, TEST_F(ScoreAdsReactorTest, RejectsBidsForFailureToMatchIncomingBidInSellerCurrency) { MockCodeDispatchClient dispatcher; - const int low_score = 1, high_score = 10; bool allowComponentAuction = false; RawRequest raw_request; // foo_two and bar are already in USD. - AdWithBidMetadata foo_two, bar; + AdWithBidMetadata foo_two, bar, barbecue; GetTestAdWithBidSameComponentAsFoo(foo_two); GetTestAdWithBidBar(bar); + GetTestAdWithBidBarbecue(barbecue); // Setting seller_currency to USD requires the converting of all bids to USD. - BuildRawRequest({foo_two, bar}, kTestSellerSignals, kTestAuctionSignals, - kTestScoringSignals, kTestPublisherHostname, raw_request, + BuildRawRequest({bar, foo_two, barbecue}, kTestSellerSignals, + kTestAuctionSignals, kTestScoringSignals, + kTestPublisherHostname, raw_request, /*enable_debug_reporting=*/false, /*enable_adtech_code_logging=*/false, /*top_level_seller=*/"", @@ -788,34 +1018,41 @@ TEST_F(ScoreAdsReactorTest, EXPECT_CALL(dispatcher, BatchExecute) .WillRepeatedly([&allowComponentAuction, &score_to_ad, &id_to_ad, - &foo_two, - &bar](std::vector& batch, - BatchDispatchDoneCallback done_callback) { + &foo_two](std::vector& batch, + BatchDispatchDoneCallback done_callback) { ABSL_LOG(INFO) << "Batch executing"; // Each original ad request (AdWithBidMetadata) is stored by its // expected score and later compared to the output AdScore with the // matching score. std::vector json_ad_scores; + int desirability_score = 1; for (const auto& request : batch) { ABSL_LOG(INFO) << "Accessing id ad mapping for " << request.id; - score_to_ad.insert_or_assign( - (request.id == foo_two.render()) ? low_score : high_score, - id_to_ad.at(request.id)); + score_to_ad.insert_or_assign(desirability_score, + id_to_ad.at(request.id)); ABSL_LOG(INFO) << "Successfully accessed id ad mapping for " << request.id; json_ad_scores.push_back(absl::Substitute( kOneSellerSimpleAdScoreTemplate, - // Explicitly setting foo_two to the lower score and bar to the - // higher should make bar win.... but see below! - (request.id == foo_two.render()) ? low_score : high_score, + // AdWithBids are fed into the script in the reverse order they + // are added to the request, so the scores should be: { bar: 3, + // foo_two: 2, barbecue: 1}. Thus bar is the most desirable and + // should win, except that bar will be messed with below. + desirability_score++, // Both foo_two and bar have a bid_currency of USD, and - // sellerCurrency is USD, so incomingBidInSellerCurrency must be - // either A) unset or B) unchanged, else the bid will be rejected. - // Setting to bar.bid plus 1689 will thus cause rejection (as it - // does not match bar.bid) whereas setting to 0 is acceptable. - // foo_two has the lower bid but will win - // because bar will be rejected for modified bid currency. - (request.id == foo_two.render()) ? 0.0f : bar.bid() + 1689, + // sellerCurrency is USD, so for these two + // incomingBidInSellerCurrency must be either A) unset or B) + // unchanged, else the bid will be rejected (barbecue can have its + // incomingBidInSellerCurrency set to anything, as it has no bid + // currency). Setting incomingBidInSellerCurrency on bar and + // barbecue to kNotAnyOriginalBid will thus cause rejection for + // bar, (as bar.incomingBidInSellerCurrency does not match + // bar.bid) whereas setting to 0 is acceptable. foo_two has a + // lower score thann bar but will win because bar will be rejected + // for modified bid currency. barbecue will be regarded as valid + // but lose the auction. barbecue.incomingBidInSellerCurrency will + // be recored as the highestScoringOtherBid. + (request.id == foo_two.render()) ? 0.0f : kNotAnyOriginalBid, (allowComponentAuction) ? "true" : "false")); } return FakeExecute(batch, std::move(done_callback), @@ -837,10 +1074,13 @@ TEST_F(ScoreAdsReactorTest, // We expect foo_two to win the auction even though it is explicitly set to // the lower bid. This is because we modified the incomingBidInSellerCurrency - // on bar, but just didn't set it. + // on bar. EXPECT_EQ(scored_ad.render(), kFooComponentsRenderUrl); - EXPECT_EQ(scored_ad.buyer_bid(), foo_two.bid()); + // Desirability should be 2 (bar should have been 3 and should have been + // rejected). + EXPECT_FLOAT_EQ(scored_ad.desirability(), 2); + EXPECT_FLOAT_EQ(scored_ad.buyer_bid(), foo_two.bid()); EXPECT_EQ(scored_ad.component_renders_size(), 3); EXPECT_EQ(scored_ad.component_renders().size(), original_ad_with_bid.ad_components().size()); @@ -848,17 +1088,45 @@ TEST_F(ScoreAdsReactorTest, original_ad_with_bid.interest_group_name()); EXPECT_EQ(scored_ad.interest_group_owner(), original_ad_with_bid.interest_group_owner()); + EXPECT_EQ(scored_ad.interest_group_origin(), + original_ad_with_bid.interest_group_origin()); EXPECT_EQ(scored_ad.buyer_bid(), original_ad_with_bid.bid()); // Not a component auction, bid should not be set. EXPECT_FLOAT_EQ(scored_ad.bid(), 0.0f); - // Desirability must be present but was determined by the scoring code. - EXPECT_GT(scored_ad.desirability(), std::numeric_limits::min()); // Since in the above test we are assuming not component auctions, verify. EXPECT_FALSE(scored_ad.allow_component_auction()); - // The other bid should have been straighr-up rejected for illegally modifying - // the incomingBidInSellerCurrency. - ASSERT_EQ(scored_ad.ig_owner_highest_scoring_other_bids_map().size(), 0); EXPECT_EQ(scored_ad.ad_type(), AdType::AD_TYPE_PROTECTED_AUDIENCE_AD); + + // bar should have been invalidated and thus should not be recorded. barbecue + // should have simply lost so should be recorded here. + ASSERT_EQ(scored_ad.ig_owner_highest_scoring_other_bids_map_size(), 1); + const auto& highest_scoring_other_bids_in_bar_standard_ads_itr = + scored_ad.ig_owner_highest_scoring_other_bids_map().find( + bar.interest_group_owner()); + ASSERT_NE(highest_scoring_other_bids_in_bar_standard_ads_itr, + scored_ad.ig_owner_highest_scoring_other_bids_map().end()); + ASSERT_EQ(highest_scoring_other_bids_in_bar_standard_ads_itr->second.values() + .size(), + 1); + ASSERT_TRUE( + highest_scoring_other_bids_in_bar_standard_ads_itr->second.values()[0] + .has_number_value()); + // Since seller_currency is set, the incomingBidInSellerCurrency is recorded + // as the bid in highestScoringOtherBid. + ASSERT_FLOAT_EQ( + highest_scoring_other_bids_in_bar_standard_ads_itr->second.values()[0] + .number_value(), + kNotAnyOriginalBid); + + // Bar should be marked as invalid. + ASSERT_EQ(scored_ad.ad_rejection_reasons_size(), 1); + const auto& bar_ad_rejection_reason = scored_ad.ad_rejection_reasons()[0]; + EXPECT_EQ(bar_ad_rejection_reason.interest_group_name(), + bar.interest_group_name()); + EXPECT_EQ(bar_ad_rejection_reason.interest_group_owner(), + bar.interest_group_owner()); + EXPECT_EQ(bar_ad_rejection_reason.rejection_reason(), + SellerRejectionReason::INVALID_BID); } TEST_F(ScoreAdsReactorTest, @@ -924,6 +1192,8 @@ TEST_F(ScoreAdsReactorTest, original_ad_with_bid.interest_group_name()); EXPECT_EQ(scored_ad.interest_group_owner(), original_ad_with_bid.interest_group_owner()); + EXPECT_EQ(scored_ad.interest_group_origin(), + original_ad_with_bid.interest_group_origin()); EXPECT_EQ(scored_ad.buyer_bid(), original_ad_with_bid.bid()); // Desirability must be present but was determined by the scoring code. EXPECT_GT(scored_ad.desirability(), std::numeric_limits::min()); @@ -938,6 +1208,101 @@ TEST_F(ScoreAdsReactorTest, 2); } +TEST_F(ScoreAdsReactorTest, + IncomingBidInSellerCurrencyUsedAsHighestScoringOtherBid) { + MockCodeDispatchClient dispatcher; + int current_score = 1; + const float kDefaultIncomingBidInSellerCurrency = 0.92f; + bool allowComponentAuction = false; + RawRequest raw_request; + AdWithBidMetadata foo, bar; + GetTestAdWithBidFoo(foo); + GetTestAdWithBidBar(bar); + // Setting a seller currency has no effect on the auction, + // as this is not a component auction. + BuildRawRequest({foo, bar}, kTestSellerSignals, kTestAuctionSignals, + kTestScoringSignals, kTestPublisherHostname, raw_request, + /*enable_debug_reporting=*/false, + /*enable_adtech_code_logging=*/false, + /*top_level_seller=*/"", + /*seller_currency=*/kEurosIsoCode); + RawRequest raw_request_copy = raw_request; + + absl::flat_hash_map id_to_ad; + for (auto& ad : raw_request_copy.ad_bids()) { + // Make sure id is tracked to render urls to stay unique. + id_to_ad.insert_or_assign(ad.render(), ad); + } + absl::flat_hash_map score_to_ad; + + EXPECT_CALL(dispatcher, BatchExecute) + .WillRepeatedly([¤t_score, &allowComponentAuction, &score_to_ad, + &id_to_ad, &kDefaultIncomingBidInSellerCurrency]( + std::vector& batch, + BatchDispatchDoneCallback done_callback) { + ABSL_LOG(INFO) << "Batch executing"; + // Each original ad request (AdWithBidMetadata) is stored by its + // expected score and later compared to the output AdScore with the + // matching score. + std::vector score_logic; + for (auto& request : batch) { + ABSL_LOG(INFO) << "Accessing id ad mapping for " << request.id; + ++current_score; + score_to_ad.insert_or_assign(current_score, id_to_ad.at(request.id)); + ABSL_LOG(INFO) << "Successfully accessed id ad mapping for " + << request.id; + score_logic.push_back( + absl::Substitute(kOneSellerSimpleAdScoreTemplate, current_score, + // AwB.currency is USD but sellerCurrency is EUR, + // so incomingBidInSellerCurrency is updated. + kDefaultIncomingBidInSellerCurrency, + (allowComponentAuction) ? "true" : "false")); + } + return FakeExecute(batch, std::move(done_callback), + std::move(score_logic)); + }); + auto response = + ExecuteScoreAds(raw_request, dispatcher, AuctionServiceRuntimeConfig()); + + ScoreAdsResponse::ScoreAdsRawResponse raw_response; + raw_response.ParseFromString(response.response_ciphertext()); + const auto& scored_ad = raw_response.ad_score(); + ABSL_LOG(INFO) << "Accessing score to ad mapping for " + << scored_ad.desirability(); + auto original_ad_with_bid = score_to_ad.at(scored_ad.desirability()); + ABSL_LOG(INFO) << "Accessed score to ad mapping for " + << scored_ad.desirability(); + // All scored ads must have these next four fields present and set. + // The next three of these are all taken from the ad_with_bid. + EXPECT_EQ(scored_ad.render(), original_ad_with_bid.render()); + EXPECT_EQ(scored_ad.component_renders_size(), 3); + EXPECT_EQ(scored_ad.component_renders().size(), + original_ad_with_bid.ad_components().size()); + EXPECT_EQ(scored_ad.interest_group_name(), + original_ad_with_bid.interest_group_name()); + EXPECT_EQ(scored_ad.interest_group_owner(), + original_ad_with_bid.interest_group_owner()); + EXPECT_EQ(scored_ad.buyer_bid(), original_ad_with_bid.bid()); + EXPECT_EQ(scored_ad.buyer_bid_currency(), + original_ad_with_bid.bid_currency()); + // Desirability must be present but was determined by the scoring code. + EXPECT_GT(scored_ad.desirability(), std::numeric_limits::min()); + // Since in the above test we are assuming not component auctions, verify. + EXPECT_FALSE(scored_ad.allow_component_auction()); + ASSERT_EQ(scored_ad.ig_owner_highest_scoring_other_bids_map().size(), 1); + EXPECT_EQ(scored_ad.ad_type(), AdType::AD_TYPE_PROTECTED_AUDIENCE_AD); + ABSL_LOG(INFO) + << "Accessing mapping for ig_owner_highest_scoring_other_bids_map"; + EXPECT_EQ(scored_ad.ig_owner_highest_scoring_other_bids_map() + .at(kInterestGroupOwnerOfBarBidder) + .values() + .at(0) + .number_value(), + kDefaultIncomingBidInSellerCurrency); + ABSL_LOG(INFO) + << "Accessed mapping for ig_owner_highest_scoring_other_bids_map"; +} + TEST_F(ScoreAdsReactorTest, PassesTopLevelSellerToComponentAuction) { MockCodeDispatchClient dispatcher; EXPECT_CALL(dispatcher, BatchExecute) @@ -947,7 +1312,7 @@ TEST_F(ScoreAdsReactorTest, PassesTopLevelSellerToComponentAuction) { const auto& input = request.input; EXPECT_EQ( *input[4], - R"JSON({"interestGroupOwner":"barStandardAds.com","topWindowHostname":"publisher_hostname","adComponents":["barStandardAds.com/ad_components/id=0"],"topLevelSeller":"testTopLevelSeller","renderUrl":"barStandardAds.com/render_ad?id=barbecue2"})JSON"); + R"JSON({"interestGroupOwner":"barStandardAds.com","topWindowHostname":"publisher_hostname","adComponents":["barStandardAds.com/ad_components/id=0"],"bidCurrency":"???","topLevelSeller":"testTopLevelSeller","renderUrl":"barStandardAds.com/render_ad?id=barbecue2"})JSON"); } return absl::OkStatus(); }); @@ -1831,6 +2196,7 @@ TEST_F(ScoreAdsReactorTest, } TEST_F(ScoreAdsReactorTest, SuccessfullyExecutesReportResultAndReportWin) { + absl::SetFlag(&FLAGS_enable_temporary_unlimited_egress, true); MockCodeDispatchClient dispatcher; int current_score = 0; bool allowComponentAuction = false; @@ -1936,6 +2302,114 @@ TEST_F(ScoreAdsReactorTest, SuccessfullyExecutesReportResultAndReportWin) { kTestInteractionUrl); } +TEST_F(ScoreAdsReactorTest, ReportingUrlsAndBuyerReportingIdSetInAdScore) { + absl::SetFlag(&FLAGS_enable_temporary_unlimited_egress, true); + MockCodeDispatchClient dispatcher; + int current_score = 0; + bool allowComponentAuction = false; + bool enable_adtech_code_logging = true; + bool enable_report_result_url_generation = true; + bool enable_report_result_win_generation = true; + bool enable_debug_reporting = false; + RawRequest raw_request; + AdWithBidMetadata foo, bar; + GetTestAdWithBidFoo(foo, kTestBuyerReportingId); + GetTestAdWithBidBar(bar, kTestBuyerReportingId); + BuildRawRequest({foo, bar}, kTestSellerSignals, kTestAuctionSignals, + kTestScoringSignals, kTestPublisherHostname, raw_request, + enable_debug_reporting, enable_adtech_code_logging); + RawRequest raw_request_copy = raw_request; + absl::flat_hash_map id_to_ad; + for (const auto& ad : raw_request_copy.ad_bids()) { + // Make sure id is tracked to render urls to stay unique. + id_to_ad.insert_or_assign(ad.render(), ad); + } + absl::flat_hash_map score_to_ad; + + EXPECT_CALL(dispatcher, BatchExecute) + .WillRepeatedly([¤t_score, &allowComponentAuction, &score_to_ad, + &id_to_ad, enable_adtech_code_logging, + enable_debug_reporting, + enable_report_result_win_generation]( + std::vector& batch, + BatchDispatchDoneCallback done_callback) { + // Each original ad request (AdWithBidMetadata) is stored by its + // expected score and later compared to the output AdScore with the + // matching score. + std::vector response; + for (const auto& request : batch) { + if (std::strcmp(request.handler_name.c_str(), + kReportingDispatchHandlerFunctionName) == 0) { + if (enable_report_result_win_generation) { + response.emplace_back(kTestReportingWinResponseJson); + } else { + response.emplace_back(kTestReportingResponseJson); + } + } else { + score_to_ad.insert_or_assign(current_score, + id_to_ad.at(request.id)); + response.push_back(absl::StrFormat( + R"JSON( + { + "response": { + "desirability":%d, + "bid":%f, + "allowComponentAuction":%s + }, + "logs":["test log"] + } + )JSON", + current_score++, 1 + (std::rand() % 20), + ((allowComponentAuction) ? "true" : "false"))); + } + } + return FakeExecute(batch, std::move(done_callback), std::move(response), + enable_debug_reporting, enable_adtech_code_logging); + }); + AuctionServiceRuntimeConfig runtime_config = { + .enable_seller_debug_url_generation = enable_debug_reporting, + .enable_adtech_code_logging = enable_adtech_code_logging, + .enable_report_result_url_generation = + enable_report_result_url_generation, + .enable_report_win_url_generation = enable_report_result_win_generation}; + const auto& response = ExecuteScoreAds( + raw_request, dispatcher, runtime_config, enable_debug_reporting); + + ScoreAdsResponse::ScoreAdsRawResponse raw_response; + raw_response.ParseFromString(response.response_ciphertext()); + const auto& scored_ad = raw_response.ad_score(); + // Desirability must be present but was determined by the scoring code. + EXPECT_GT(scored_ad.desirability(), std::numeric_limits::min()); + EXPECT_EQ(scored_ad.buyer_reporting_id(), kTestBuyerReportingId); + EXPECT_EQ(scored_ad.win_reporting_urls() + .top_level_seller_reporting_urls() + .reporting_url(), + kTestReportResultUrl); + EXPECT_EQ(scored_ad.win_reporting_urls() + .top_level_seller_reporting_urls() + .interaction_reporting_urls() + .size(), + 1); + EXPECT_EQ(scored_ad.win_reporting_urls() + .top_level_seller_reporting_urls() + .interaction_reporting_urls() + .at(kTestInteractionEvent), + kTestInteractionUrl); + EXPECT_EQ( + scored_ad.win_reporting_urls().buyer_reporting_urls().reporting_url(), + kTestReportWinUrl); + EXPECT_EQ(scored_ad.win_reporting_urls() + .buyer_reporting_urls() + .interaction_reporting_urls() + .size(), + 1); + EXPECT_EQ(scored_ad.win_reporting_urls() + .buyer_reporting_urls() + .interaction_reporting_urls() + .at(kTestInteractionEvent), + kTestInteractionUrl); +} + TEST_F(ScoreAdsReactorTest, ReportResultFailsReturnsOkayResponse) { MockCodeDispatchClient dispatcher; int current_score = 0; @@ -2133,26 +2607,50 @@ TEST_F(ScoreAdsReactorTest, VerifyDecryptionEncryptionSuccessful) { // ScoreAds for ads where it was of any valid value other than 'not-available'. TEST_F(ScoreAdsReactorTest, CaptureRejectionReasonsForRejectedAds) { MockCodeDispatchClient dispatcher; - bool allowComponentAuction = false; + bool allowComponentAuction = true; RawRequest raw_request; - AdWithBidMetadata foo, bar; + // Each ad must have its own unique render url, lest the IDs clash during roma + // batch execution and one gets clobbered. + AdWithBidMetadata foo, bar, barbecue, boots; + // AdWithBid Foo is to be rejected by the scoreAd() code, which will delcare + // it invalid. GetTestAdWithBidFoo(foo); + // AdWithBid Bar has bid currency USD. Since this matches the USD set as the + // seller currency below, the AdScore for bar must have an + // incomingBidInSellerCurrency which is unmodified (that is, which matches + // bar.bid()). This test will deliberately violate this requirement below to + // test the seller rejectionn reason setting. GetTestAdWithBidBar(bar); + // This AdWithBid is intended to be the sole non-rejected AwB. + GetTestAdWithBidBarbecue(barbecue); + // Boots's modified bid currency of Sterling should clash with the stated + // auction currency of USD below, leading to a seller rejection reason of + // currency mismatch. + GetTestAdWithBidBoots(boots); - BuildRawRequest({foo, bar}, kTestSellerSignals, kTestAuctionSignals, - kTestScoringSignals, kTestPublisherHostname, raw_request, - true); + BuildRawRequestForComponentAuction( + {foo, bar, barbecue, boots}, kTestSellerSignals, kTestAuctionSignals, + kTestScoringSignals, kTestPublisherHostname, raw_request, + /*enable_debug_reporting=*/true, + /*seller_currency=*/kUsdIsoCode); + + ASSERT_EQ(raw_request.ad_bids_size(), 4); RawRequest raw_request_copy = raw_request; absl::flat_hash_map id_to_rejection_reason; // Make sure id is tracked to render urls to stay unique. + // Bar is to have its rejection reason assigned by the reactor code for + // incomingBidInSellerCurrency illegal modification. id_to_rejection_reason.insert_or_assign(bar.render(), "not-available"); id_to_rejection_reason.insert_or_assign(foo.render(), "invalid-bid"); + // barbecue is to survive unscathed and should win the auction. + id_to_rejection_reason.insert_or_assign(barbecue.render(), "not-available"); + id_to_rejection_reason.insert_or_assign(boots.render(), "not-available"); EXPECT_CALL(dispatcher, BatchExecute) - .WillRepeatedly([&allowComponentAuction, &id_to_rejection_reason]( - std::vector& batch, - BatchDispatchDoneCallback done_callback) { + .WillRepeatedly([&allowComponentAuction, &id_to_rejection_reason, &bar, + &boots](std::vector& batch, + BatchDispatchDoneCallback done_callback) { // Each original ad request (AdWithBidMetadata) is stored by its // expected score and later compared to the output AdScore with the // matching score. @@ -2160,19 +2658,28 @@ TEST_F(ScoreAdsReactorTest, CaptureRejectionReasonsForRejectedAds) { for (const auto& request : batch) { std::string rejection_reason = id_to_rejection_reason.at(request.id); score_logic.push_back(absl::Substitute( - R"( - { - "response" : { - "desirability" : $0, - "bid" : $1, - "allowComponentAuction" : $2, - "rejectReason" : "$3" - }, - "logs":[] - } -)", - 1, 1 + (std::rand() % 20), - (allowComponentAuction) ? "true" : "false", rejection_reason)); + kComponentWithCurrencyAndIncomingAndRejectReasAdScore, + // Disable this check because all args to + // absl::Substitute() are named generically, so explicitly calling + // out "desirability" and "bid" runs afoul of these checks. + // NOLINTNEXTLINE + /*desirability=*/1, (allowComponentAuction) ? "true" : "false", + // IG Bar's bid currency is USD. As Seller currency is USD, bar's + // incomingBidInSellerCurrency must equal the original bid or the + // bid should be rejected for invalid incomingBidInSellerCurrency. + // So here, we deliberately set + // barAdScore.incomingBidInSellerCurrency to NOT bar.bid(), so + // that bar is rejected and we can test its reject reason. The + // other AdScores just don't have incomingBidInSellerCurrency set + // on them (0 is default). + (request.id == bar.render()) ? bar.bid() + 1689 : 0.0f, + // NOLINTNEXTLINE + /*bid=*/1 + (std::rand() % 20), + // Only the boots AwB shall be rejected for currency mismatch. + // NOLINTNEXTLINE + /*bidCurrency=*/ + (request.id == boots.render()) ? kSterlingIsoCode : "", + rejection_reason)); } return FakeExecute(batch, std::move(done_callback), std::move(score_logic), true); @@ -2185,13 +2692,33 @@ TEST_F(ScoreAdsReactorTest, CaptureRejectionReasonsForRejectedAds) { auto scored_ad = raw_response.ad_score(); // Desirability must be present but was determined by the scoring code. EXPECT_GT(scored_ad.desirability(), std::numeric_limits::min()); - EXPECT_EQ(scored_ad.ad_rejection_reasons().size(), 1); - const auto& ad_rejection_reason = scored_ad.ad_rejection_reasons().at(0); - EXPECT_EQ(ad_rejection_reason.interest_group_name(), + // Assertion avoids out-of-bounds array accesses below. + ASSERT_EQ(scored_ad.ad_rejection_reasons().size(), 3); + + // Note that the batch responses are in reverse order. + + const auto& boots_ad_rejection_reason = scored_ad.ad_rejection_reasons()[0]; + EXPECT_EQ(boots_ad_rejection_reason.interest_group_name(), + boots.interest_group_name()); + EXPECT_EQ(boots_ad_rejection_reason.interest_group_owner(), + boots.interest_group_owner()); + EXPECT_EQ(boots_ad_rejection_reason.rejection_reason(), + SellerRejectionReason::BID_FROM_SCORE_AD_FAILED_CURRENCY_CHECK); + + const auto& bar_ad_rejection_reason = scored_ad.ad_rejection_reasons()[1]; + EXPECT_EQ(bar_ad_rejection_reason.interest_group_name(), + bar.interest_group_name()); + EXPECT_EQ(bar_ad_rejection_reason.interest_group_owner(), + bar.interest_group_owner()); + EXPECT_EQ(bar_ad_rejection_reason.rejection_reason(), + SellerRejectionReason::INVALID_BID); + + const auto& foo_ad_rejection_reason = scored_ad.ad_rejection_reasons()[2]; + EXPECT_EQ(foo_ad_rejection_reason.interest_group_name(), foo.interest_group_name()); - EXPECT_EQ(ad_rejection_reason.interest_group_owner(), + EXPECT_EQ(foo_ad_rejection_reason.interest_group_owner(), foo.interest_group_owner()); - EXPECT_EQ(ad_rejection_reason.rejection_reason(), + EXPECT_EQ(foo_ad_rejection_reason.rejection_reason(), SellerRejectionReason::INVALID_BID); } @@ -2328,13 +2855,17 @@ ProtectedAppSignalsAdWithBidMetadata GetProtectedAppSignalsAdWithBidMetadata( ad.set_bid(bid); ad.set_bid_currency(kUsdIsoCode); ad.set_owner(kTestProtectedAppSignalsAdOwner); - ad.set_egress_features(kTestEgressFeatures); + ad.set_egress_payload(kTestEgressPayload); + ad.set_temporary_unlimited_egress_payload(kTestTemporaryEgressPayload); return ad; } class ScoreAdsReactorProtectedAppSignalsTest : public ScoreAdsReactorTest { protected: - void SetUp() override { runtime_config_.enable_protected_app_signals = true; } + void SetUp() override { + runtime_config_.enable_protected_app_signals = true; + server_common::log::PS_VLOG_IS_ON(/*verbose_level=*/0, /*max_v=*/10); + } ScoreAdsResponse ExecuteScoreAds(const RawRequest& raw_request, MockCodeDispatchClient& dispatcher) { @@ -2348,17 +2879,29 @@ class ScoreAdsReactorProtectedAppSignalsTest : public ScoreAdsReactorTest { TEST_F(ScoreAdsReactorProtectedAppSignalsTest, AdDispatchedForScoringWhenSignalsPresent) { MockCodeDispatchClient dispatcher; - + auto expected_ad = JsonStringToValue( + R"JSON( + { + "metadata": {"arbitraryMetadataKey":2}, + "renderUrl": "testAppAds.com/render_ad?id=bar" + })JSON"); + CHECK_OK(expected_ad) << "Malformed ad JSON"; EXPECT_CALL(dispatcher, BatchExecute) - .WillOnce([](std::vector& batch, - BatchDispatchDoneCallback done_callback) { + .WillOnce([expected_ad](std::vector& batch, + BatchDispatchDoneCallback done_callback) { EXPECT_EQ(batch.size(), 1); const auto& req = batch[0]; EXPECT_EQ(req.handler_name, "scoreAdEntryFunction"); EXPECT_EQ(req.input.size(), 7); - EXPECT_EQ(*req.input[static_cast(ScoreAdArgs::kAdMetadata)], - "{\"arbitraryMetadataKey\":2}"); + auto observed_ad = JsonStringToValue( + *req.input[static_cast(ScoreAdArgs::kAdMetadata)]); + CHECK_OK(observed_ad); + std::string difference; + google::protobuf::util::MessageDifferencer differencer; + differencer.ReportDifferencesToString(&difference); + EXPECT_TRUE(differencer.Compare(*observed_ad, *expected_ad)) + << difference; EXPECT_THAT(*req.input[static_cast(ScoreAdArgs::kBid)], HasSubstr(absl::StrCat(kTestBid))); EXPECT_EQ(*req.input[static_cast(ScoreAdArgs::kAuctionConfig)], @@ -2426,7 +2969,7 @@ TEST_F(ScoreAdsReactorProtectedAppSignalsTest, BatchDispatchDoneCallback done_callback) { EXPECT_EQ(batch.size(), 1); const auto& request = batch[0]; - PS_VLOG(0) << "UDF handler: " << request.handler_name; + PS_LOG(INFO) << "UDF handler: " << request.handler_name; EXPECT_EQ(request.handler_name, "scoreAdEntryFunction"); auto incoming_bid = request.input[static_cast(ScoreAdArgs::kBid)]; @@ -2455,12 +2998,15 @@ TEST_F(ScoreAdsReactorProtectedAppSignalsTest, BatchDispatchDoneCallback done_callback) { EXPECT_EQ(batch.size(), 1); const auto& request = batch[0]; - PS_VLOG(0) << "UDF handler: " << request.handler_name; + PS_LOG(INFO) << "UDF handler: " << request.handler_name; EXPECT_EQ(request.handler_name, kReportingProtectedAppSignalsFunctionName); EXPECT_EQ( - *request.input[ReportingArgIndex(ReportingArgs::kEgressFeatures)], - kTestEgressFeatures); + *request.input[ReportingArgIndex(ReportingArgs::kEgressPayload)], + kTestEgressPayload); + EXPECT_EQ(*request.input[ReportingArgIndex( + ReportingArgs::kTemporaryEgressPayload)], + kTestTemporaryEgressPayload); DispatchResponse response; if (enable_report_result_win_generation) { response.resp = kTestReportingWinResponseJson; diff --git a/services/auction_service/seller_code_fetch_manager.cc b/services/auction_service/seller_code_fetch_manager.cc index e1c48836..5e449b7c 100644 --- a/services/auction_service/seller_code_fetch_manager.cc +++ b/services/auction_service/seller_code_fetch_manager.cc @@ -151,19 +151,10 @@ SellerCodeFetchManager::InitializeUrlCodeFetch() { } absl::Status SellerCodeFetchManager::InitBucketClient() { - auto result = blob_storage_client_->Init(); - if (!result.Successful()) { - return absl::UnavailableError( - absl::StrFormat("Failed to init BlobStorageClient (status_code: %s)\n", - GetErrorMessage(result.status_code))); - } - - result = blob_storage_client_->Run(); - if (!result.Successful()) { - return absl::UnavailableError( - absl::StrFormat("Failed to run BlobStorageClient (status_code: %s)\n", - GetErrorMessage(result.status_code))); - } + PS_RETURN_IF_ERROR(blob_storage_client_->Init()).SetPrepend() + << "Failed to init BlobStorageClient: "; + PS_RETURN_IF_ERROR(blob_storage_client_->Run()).SetPrepend() + << "Failed to run BlobStorageClient: "; return absl::OkStatus(); } diff --git a/services/auction_service/seller_code_fetch_manager_test.cc b/services/auction_service/seller_code_fetch_manager_test.cc index 0e304956..416618b1 100644 --- a/services/auction_service/seller_code_fetch_manager_test.cc +++ b/services/auction_service/seller_code_fetch_manager_test.cc @@ -100,14 +100,14 @@ TEST_F(SellerCodeFetchManagerTest, udf_config.set_auction_js_bucket("js"); udf_config.set_auction_js_bucket_default_blob("default"); - EXPECT_CALL(*blob_storage_client_, Init) - .WillOnce(Return(SuccessExecutionResult())); - EXPECT_CALL(*blob_storage_client_, Run) - .WillOnce(Return(SuccessExecutionResult())); + EXPECT_CALL(*blob_storage_client_, Init).WillOnce(Return(absl::OkStatus())); + EXPECT_CALL(*blob_storage_client_, Run).WillOnce(Return(absl::OkStatus())); EXPECT_CALL(*executor_, RunAfter).Times(2); EXPECT_CALL(*http_fetcher_, FetchUrls) .Times(1) - .WillOnce([&](const std::vector& requests, + .WillOnce([&udf_config, &pa_buyer_origin, &pas_buyer_origin, + &pa_reporting_udf_data, &pas_reporting_udf_data]( + const std::vector& requests, absl::Duration timeout, OnDoneFetchUrls done_callback) { EXPECT_EQ(requests[0].url, udf_config.buyer_report_win_js_urls().at(pa_buyer_origin)); @@ -122,7 +122,8 @@ TEST_F(SellerCodeFetchManagerTest, EXPECT_CALL(*blob_storage_client_, ListBlobsMetadata) .WillOnce( - [&](AsyncContext + [&udf_config]( + AsyncContext context) { EXPECT_EQ(context.request->blob_metadata().bucket_name(), udf_config.auction_js_bucket()); @@ -133,12 +134,13 @@ TEST_F(SellerCodeFetchManagerTest, context.response = std::make_shared(); context.response->mutable_blob_metadatas()->Add(std::move(md)); context.Finish(SuccessExecutionResult()); - return SuccessExecutionResult(); + return absl::OkStatus(); }); EXPECT_CALL(*blob_storage_client_, GetBlob) .WillOnce( - [&](AsyncContext async_context) { + [&udf_config, &fake_udf]( + AsyncContext async_context) { auto async_bucket_name = async_context.request->blob_metadata().bucket_name(); auto async_blob_name = @@ -152,11 +154,13 @@ TEST_F(SellerCodeFetchManagerTest, async_context.result = SuccessExecutionResult(); async_context.Finish(); - return SuccessExecutionResult(); + return absl::OkStatus(); }); EXPECT_CALL(*dispatcher_, LoadSync) - .WillOnce([&](std::string_view version, absl::string_view blob_data) { + .WillOnce([&udf_config, &fake_udf, &pa_buyer_origin, &pas_buyer_origin, + &pa_reporting_udf_data, &pas_reporting_udf_data]( + std::string_view version, absl::string_view blob_data) { EXPECT_EQ(version, udf_config.auction_js_bucket_default_blob()); absl::flat_hash_map pa_reporting_udfs; pa_reporting_udfs.try_emplace(pa_buyer_origin, pa_reporting_udf_data); @@ -206,7 +210,9 @@ TEST_F(SellerCodeFetchManagerTest, FetchModeUrlSucceedsLoadingWrappedUrlBlobs) { EXPECT_CALL(*executor_, RunAfter).Times(2); EXPECT_CALL(*http_fetcher_, FetchUrls) .Times(2) - .WillOnce([&](const std::vector& requests, + .WillOnce([&udf_config, &pa_buyer_origin, &pas_buyer_origin, + &pa_reporting_udf_data, &pas_reporting_udf_data]( + const std::vector& requests, absl::Duration timeout, OnDoneFetchUrls done_callback) { EXPECT_EQ(requests[0].url, udf_config.buyer_report_win_js_urls().at(pa_buyer_origin)); @@ -218,14 +224,17 @@ TEST_F(SellerCodeFetchManagerTest, FetchModeUrlSucceedsLoadingWrappedUrlBlobs) { std::move(done_callback)( {pa_reporting_udf_data, pas_reporting_udf_data}); }) - .WillOnce([&](const std::vector& requests, + .WillOnce([&udf_config, &fake_udf]( + const std::vector& requests, absl::Duration timeout, OnDoneFetchUrls done_callback) { EXPECT_EQ(requests[0].url, udf_config.auction_js_url()); std::move(done_callback)({fake_udf}); }); EXPECT_CALL(*dispatcher_, LoadSync) - .WillOnce([&](std::string_view version, absl::string_view blob_data) { + .WillOnce([&udf_config, &fake_udf, &pa_buyer_origin, &pas_buyer_origin, + &pa_reporting_udf_data, &pas_reporting_udf_data]( + std::string_view version, absl::string_view blob_data) { EXPECT_EQ(version, kScoreAdBlobVersion); absl::flat_hash_map pa_reporting_udfs; pa_reporting_udfs.try_emplace(pa_buyer_origin, pa_reporting_udf_data); diff --git a/services/auction_service/utils/BUILD b/services/auction_service/utils/BUILD index bddbe20c..fe1bcda7 100644 --- a/services/auction_service/utils/BUILD +++ b/services/auction_service/utils/BUILD @@ -48,6 +48,7 @@ cc_test( ], deps = [ ":proto_utils", + "//services/auction_service:auction_constants", "//services/auction_service:score_ads_reactor_test_util", "//services/common/test:random", "@com_google_googletest//:gtest_main", diff --git a/services/auction_service/utils/proto_utils.cc b/services/auction_service/utils/proto_utils.cc index f01466e1..b13e8e35 100644 --- a/services/auction_service/utils/proto_utils.cc +++ b/services/auction_service/utils/proto_utils.cc @@ -177,11 +177,14 @@ std::string MakeOpenBidMetadataJson( } absl::StrAppend(&bid_metadata, R"(],)"); } - // Only add bid currency to bid metadata if it's non-empty. if (!bid_currency.empty()) { absl::StrAppend(&bid_metadata, R"JSON(")JSON", kBidCurrencyPropertyForScoreAd, R"JSON(":")JSON", bid_currency, R"JSON(",)JSON"); + } else { + absl::StrAppend(&bid_metadata, R"JSON(")JSON", + kBidCurrencyPropertyForScoreAd, R"JSON(":")JSON", + kEmptyBidCurrencyCode, R"JSON(",)JSON"); } return bid_metadata; } @@ -375,8 +378,8 @@ void MayPopulateScoringSignalsForProtectedAppSignals( combined_signals.try_emplace(protected_app_signals_ad_bid.render(), std::move(combined_signals_for_this_bid)); if (!succeeded) { - PS_VLOG(1, log_context) << "Render URL overlaps between bids: " - << protected_app_signals_ad_bid.render(); + PS_LOG(ERROR, log_context) << "Render URL overlaps between bids: " + << protected_app_signals_ad_bid.render(); } } } @@ -548,13 +551,14 @@ absl::StatusOr BuildScoreAdRequest( const std::shared_ptr& auction_config, absl::string_view bid_metadata, server_common::log::ContextImpl& log_context, - const bool enable_adtech_code_logging, const bool enable_debug_reporting) { + const bool enable_adtech_code_logging, const bool enable_debug_reporting, + absl::string_view code_version) { // Construct the wrapper struct for our V8 Dispatch Request. DispatchRequest score_ad_request; // TODO(b/250893468) Revisit dispatch id. score_ad_request.id = ad_render_url; // TODO(b/258790164) Update after code is fetched periodically. - score_ad_request.version_string = "v1"; + score_ad_request.version_string = code_version; score_ad_request.handler_name = DispatchHandlerFunctionWithSellerWrapper; score_ad_request.input = std::vector>( diff --git a/services/auction_service/utils/proto_utils.h b/services/auction_service/utils/proto_utils.h index dad6b471..52936aee 100644 --- a/services/auction_service/utils/proto_utils.h +++ b/services/auction_service/utils/proto_utils.h @@ -118,7 +118,8 @@ absl::StatusOr BuildScoreAdRequest( const std::shared_ptr& auction_config, absl::string_view bid_metadata, server_common::log::ContextImpl& log_context, - const bool enable_adtech_code_logging, const bool enable_debug_reporting); + const bool enable_adtech_code_logging, const bool enable_debug_reporting, + absl::string_view code_version); /** * Builds ScoreAdInput with AdWithBid or ProtectedAppSignalsAdWithBid. @@ -130,18 +131,15 @@ absl::StatusOr BuildScoreAdRequest( scoring_signals, const bool enable_debug_reporting, server_common::log::ContextImpl& log_context, - const bool enable_adtech_code_logging, absl::string_view bid_metadata) { - std::string ad_metadata_as_json; - const auto& it = ad.ad().struct_value().fields().find("metadata"); - if (it != ad.ad().struct_value().fields().end()) { - PS_RETURN_IF_ERROR(google::protobuf::util::MessageToJsonString( - it->second, &ad_metadata_as_json)); - } - return BuildScoreAdRequest(ad.render(), ad_metadata_as_json, - scoring_signals.at(ad.render()).GetString(), - ad.bid(), auction_config, bid_metadata, - log_context, enable_adtech_code_logging, - enable_debug_reporting); + const bool enable_adtech_code_logging, absl::string_view bid_metadata, + absl::string_view code_version) { + std::string ad_object_json; + PS_RETURN_IF_ERROR( + google::protobuf::util::MessageToJsonString(ad.ad(), &ad_object_json)); + return BuildScoreAdRequest( + ad.render(), ad_object_json, scoring_signals.at(ad.render()).GetString(), + ad.bid(), auction_config, bid_metadata, log_context, + enable_adtech_code_logging, enable_debug_reporting, code_version); } } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/utils/proto_utils_test.cc b/services/auction_service/utils/proto_utils_test.cc index a12d3ef6..d394bede 100644 --- a/services/auction_service/utils/proto_utils_test.cc +++ b/services/auction_service/utils/proto_utils_test.cc @@ -17,12 +17,15 @@ #include #include "gmock/gmock.h" +#include "google/protobuf/util/message_differencer.h" #include "gtest/gtest.h" #include "rapidjson/document.h" #include "rapidjson/pointer.h" #include "rapidjson/writer.h" +#include "services/auction_service/auction_constants.h" #include "services/auction_service/score_ads_reactor_test_util.h" #include "services/common/test/random.h" +#include "services/common/util/proto_util.h" #include "src/core/test/utils/proto_test_utils.h" namespace privacy_sandbox::bidding_auction_servers { @@ -37,6 +40,8 @@ constexpr char kTestAdComponentUrl_1[] = "https://compoennt_1"; constexpr char kTestAdComponentUrl_2[] = "https://component_2"; constexpr char kTestBidCurrency[] = "ABC"; +using ::google::protobuf::util::MessageDifferencer; + google::protobuf::RepeatedPtrField MakeMockAdComponentUrls() { google::protobuf::RepeatedPtrField component_urls; component_urls.Add(kTestAdComponentUrl_1); @@ -126,13 +131,25 @@ TEST(MapAuctionResultToAdWithBidMetadataTest, PopulatesExpectedValues) { } } -constexpr char kTestAdMetadataJson[] = R"JSON({"arbitraryMetadataKey":2})JSON"; +constexpr absl::string_view kTestAdMetadataJson = R"JSON( + { + "metadata": { + "arbitraryMetadataKey": 2 + }, + "renderUrl": "https://render_url" + })JSON"; constexpr char kTestScoringSignals[] = R"JSON({"RandomScoringSignals":{}})JSON"; constexpr char kTestAuctionConfig[] = R"JSON({"RandomAuctionConfig":{}})JSON"; constexpr char kTestBidMetadata[] = R"JSON({"RandomBidMetadata":{}})JSON"; server_common::log::ContextImpl log_context{ {}, server_common::ConsentedDebugConfiguration()}; +google::protobuf::Value GetTestAdMetadata() { + auto ads_metadata = JsonStringToValue(kTestAdMetadataJson); + CHECK_OK(ads_metadata) << "Malformed JSON: " << kTestAdMetadataJson; + return *ads_metadata; +} + TEST(BuildScoreAdRequestTest, PopulatesExpectedValuesInDispatchRequest) { float test_bid = MakeARandomNumber(0.1, 10.1); auto output = BuildScoreAdRequest( @@ -140,11 +157,12 @@ TEST(BuildScoreAdRequestTest, PopulatesExpectedValuesInDispatchRequest) { std::make_shared(kTestAuctionConfig), kTestBidMetadata, log_context, /*enable_adtech_code_logging = */ false, - /*enable_debug_reporting = */ false); + /*enable_debug_reporting = */ false, kScoreAdBlobVersion); ASSERT_TRUE(output.ok()); EXPECT_EQ(output->id, kTestRenderUrl); - EXPECT_EQ(output->version_string, "v1"); + EXPECT_EQ(output->version_string, kScoreAdBlobVersion); EXPECT_EQ(output->handler_name, DispatchHandlerFunctionWithSellerWrapper); + EXPECT_EQ(*output->input[ScoreArgIndex(ScoreAdArgs::kAdMetadata)], kTestAdMetadataJson); EXPECT_EQ(*output->input[ScoreArgIndex(ScoreAdArgs::kBid)], @@ -160,6 +178,32 @@ TEST(BuildScoreAdRequestTest, PopulatesExpectedValuesInDispatchRequest) { "{}"); } +TEST(BuildScoreAdRequestTest, HandlesAdsMetadataString) { + float test_bid = MakeARandomNumber(0.1, 10.1); + std::string ad_metadata_json = R"JSON("test_ads_data")JSON"; + auto output = BuildScoreAdRequest( + kTestRenderUrl, ad_metadata_json, kTestScoringSignals, test_bid, + std::make_shared(kTestAuctionConfig), kTestBidMetadata, + log_context, + /*enable_adtech_code_logging = */ false, + /*enable_debug_reporting = */ false, kScoreAdBlobVersion); + auto observed_ad = JsonStringToValue( + *output->input[ScoreArgIndex(ScoreAdArgs::kAdMetadata)]); + CHECK_OK(observed_ad) + << "Malformed observed ad JSON: " + << *output->input[static_cast(ScoreAdArgs::kAdMetadata)]; + std::string difference; + google::protobuf::util::MessageDifferencer differencer; + differencer.ReportDifferencesToString(&difference); + auto expected_ad = JsonStringToValue(ad_metadata_json); + CHECK_OK(expected_ad) << "Malformed JSON: " << ad_metadata_json; + EXPECT_TRUE(differencer.Compare(*observed_ad, *expected_ad)) + << "\n Observed:\n" + << observed_ad->DebugString() << "\n\nExpected:\n" + << expected_ad->DebugString() << "\n\nDifference:\n" + << difference; +} + AdWithBidMetadata MakeAnAdWithMetadata(float bid) { AdWithBidMetadata ad; ad.set_render(kTestRenderUrl); @@ -188,13 +232,26 @@ TEST(BuildScoreAdRequestTest, PopulatesExpectedValuesForAdWithBidMetadata) { MakeAnAdWithMetadata(test_bid), std::make_shared(kTestAuctionConfig), MakeScoringSignalsForAd(), /*enable_debug_reporting = */ false, - log_context, /*enable_adtech_code_logging = */ false, kTestBidMetadata); + log_context, /*enable_adtech_code_logging = */ false, kTestBidMetadata, + kScoreAdBlobVersion); ASSERT_TRUE(output.ok()); EXPECT_EQ(output->id, kTestRenderUrl); - EXPECT_EQ(output->version_string, "v1"); + EXPECT_EQ(output->version_string, kScoreAdBlobVersion); EXPECT_EQ(output->handler_name, DispatchHandlerFunctionWithSellerWrapper); - EXPECT_EQ(*output->input[ScoreArgIndex(ScoreAdArgs::kAdMetadata)], - kTestAdMetadataJson); + auto observed_ad = JsonStringToValue( + *output->input[ScoreArgIndex(ScoreAdArgs::kAdMetadata)]); + CHECK_OK(observed_ad) + << "Malformed observed ad JSON: " + << *output->input[static_cast(ScoreAdArgs::kAdMetadata)]; + std::string difference; + google::protobuf::util::MessageDifferencer differencer; + differencer.ReportDifferencesToString(&difference); + auto expected_ad = GetTestAdMetadata(); + EXPECT_TRUE(differencer.Compare(*observed_ad, expected_ad)) + << "\n Observed:\n" + << observed_ad->DebugString() << "\n\nExpected:\n" + << expected_ad.DebugString() << "\n\nDifference:\n" + << difference; EXPECT_EQ(*output->input[ScoreArgIndex(ScoreAdArgs::kBid)], std::to_string(test_bid)); EXPECT_EQ(*output->input[ScoreArgIndex(ScoreAdArgs::kAuctionConfig)], @@ -215,13 +272,26 @@ TEST(BuildScoreAdRequestTest, GetProtectedAppSignalsAdWithBidMetadata(kTestRenderUrl, test_bid), std::make_shared(kTestAuctionConfig), MakeScoringSignalsForAd(), /*enable_debug_reporting = */ false, - log_context, /*enable_adtech_code_logging = */ false, kTestBidMetadata); + log_context, /*enable_adtech_code_logging = */ false, kTestBidMetadata, + kScoreAdBlobVersion); ASSERT_TRUE(output.ok()); EXPECT_EQ(output->id, kTestRenderUrl); EXPECT_EQ(output->version_string, "v1"); EXPECT_EQ(output->handler_name, DispatchHandlerFunctionWithSellerWrapper); - EXPECT_EQ(*output->input[ScoreArgIndex(ScoreAdArgs::kAdMetadata)], - kTestAdMetadataJson); + auto observed_ad = JsonStringToValue( + *output->input[ScoreArgIndex(ScoreAdArgs::kAdMetadata)]); + CHECK_OK(observed_ad) + << "Malformed observed ad JSON: " + << *output->input[static_cast(ScoreAdArgs::kAdMetadata)]; + std::string difference; + google::protobuf::util::MessageDifferencer differencer; + differencer.ReportDifferencesToString(&difference); + auto expected_ad = GetTestAdMetadata(); + EXPECT_TRUE(differencer.Compare(*observed_ad, expected_ad)) + << "\n Observed:\n" + << observed_ad->DebugString() << "\n\nExpected:\n" + << expected_ad.DebugString() << "\n\nDifference:\n" + << difference; EXPECT_EQ(*output->input[ScoreArgIndex(ScoreAdArgs::kBid)], std::to_string(test_bid)); EXPECT_EQ(*output->input[ScoreArgIndex(ScoreAdArgs::kAuctionConfig)], @@ -253,7 +323,7 @@ TEST(BuildScoreAdRequestTest, PopulatesFeatureFlagsInDispatchRequest) { std::make_shared(kTestAuctionConfig), kTestBidMetadata, log_context, /*enable_adtech_code_logging = */ flag_1, - /*enable_debug_reporting = */ flag_2); + /*enable_debug_reporting = */ flag_2, kScoreAdBlobVersion); ASSERT_TRUE(output.ok()); EXPECT_EQ(*output->input[ScoreArgIndex(ScoreAdArgs::kFeatureFlags)], MakeFeatureFlagJson(flag_1, flag_2)); @@ -270,7 +340,7 @@ TEST(BuildScoreAdRequestTest, PopulatesFeatureFlagsForAdWithBidMetadata) { std::make_shared(kTestAuctionConfig), MakeScoringSignalsForAd(), /*enable_debug_reporting = */ flag_2, log_context, /*enable_adtech_code_logging = */ flag_1, - kTestBidMetadata); + kTestBidMetadata, kScoreAdBlobVersion); ASSERT_TRUE(output.ok()); EXPECT_EQ(*output->input[ScoreArgIndex(ScoreAdArgs::kFeatureFlags)], MakeFeatureFlagJson(flag_1, flag_2)); @@ -287,7 +357,7 @@ TEST(BuildScoreAdRequestTest, PopulatesFeatureFlagsForPASAdWithBidMetadata) { std::make_shared(kTestAuctionConfig), MakeScoringSignalsForAd(), /*enable_debug_reporting = */ flag_2, log_context, /*enable_adtech_code_logging = */ flag_1, - kTestBidMetadata); + kTestBidMetadata, kScoreAdBlobVersion); ASSERT_TRUE(output.ok()); EXPECT_EQ(*output->input[ScoreArgIndex(ScoreAdArgs::kFeatureFlags)], MakeFeatureFlagJson(flag_1, flag_2)); diff --git a/services/bidding_service/BUILD b/services/bidding_service/BUILD index f36c64e3..6dcc044d 100644 --- a/services/bidding_service/BUILD +++ b/services/bidding_service/BUILD @@ -99,6 +99,7 @@ cc_library( "//services/bidding_service/benchmarking:bidding_no_op_logger", "//services/bidding_service/code_wrapper:buyer_code_wrapper", "//services/bidding_service/data:runtime_config", + "//services/common:feature_flags", "//services/common/clients/code_dispatcher:code_dispatch_client", "//services/common/clients/kv_server:kv_async_client", "//services/common/code_dispatch:code_dispatch_reactor", @@ -232,10 +233,6 @@ cc_proto_library( cc_binary( name = "server", srcs = ["bidding_main.cc"], - defines = select({ - "//:inference": ["PS_INFERENCE_NON_PROD=1"], - "//conditions:default": [], - }), linkopts = [ "-Wl,-rpath,\\$$ORIGIN/../lib", ], @@ -295,6 +292,7 @@ cc_test( "//services/bidding_service:generate_bids_reactor", "//services/bidding_service:generate_bids_reactor_test_utils", "//services/bidding_service/code_wrapper:buyer_code_wrapper", + "//services/common:feature_flags", "//services/common/constants:common_service_flags", "//services/common/encryption:key_fetcher_factory", "//services/common/encryption:mock_crypto_client_wrapper", diff --git a/services/bidding_service/bidding_main.cc b/services/bidding_service/bidding_main.cc index 7911206f..a56cd733 100644 --- a/services/bidding_service/bidding_main.cc +++ b/services/bidding_service/bidding_main.cc @@ -30,6 +30,8 @@ #include "grpcpp/ext/proto_server_reflection_plugin.h" #include "grpcpp/grpcpp.h" #include "grpcpp/health_check_service_interface.h" +#include "sandbox/sandbox_executor.h" +#include "sandboxed_api/sandbox2/comms.h" #include "services/bidding_service/benchmarking/bidding_benchmarking_logger.h" #include "services/bidding_service/benchmarking/bidding_no_op_logger.h" #include "services/bidding_service/bidding_code_fetch_config.pb.h" @@ -38,8 +40,10 @@ #include "services/bidding_service/code_wrapper/buyer_code_wrapper.h" #include "services/bidding_service/constants.h" #include "services/bidding_service/data/runtime_config.h" +#include "services/bidding_service/inference/inference_utils.h" #include "services/bidding_service/protected_app_signals_generate_bids_reactor.h" #include "services/bidding_service/runtime_flags.h" +#include "services/common/blob_fetch/blob_fetcher.h" #include "services/common/clients/code_dispatcher/code_dispatch_client.h" #include "services/common/clients/config/trusted_server_config_client.h" #include "services/common/clients/config/trusted_server_config_client_util.h" @@ -58,13 +62,6 @@ #include "src/util/rlimit_core_config.h" #include "src/util/status_macro/status_macros.h" -#ifdef PS_INFERENCE_NON_PROD -#include "sandbox/sandbox_executor.h" -#include "sandboxed_api/sandbox2/comms.h" -#include "services/bidding_service/inference/inference_utils.h" -#include "services/common/blob_fetch/blob_fetcher.h" -#endif - ABSL_FLAG(std::optional, port, std::nullopt, "Port the server is listening on."); ABSL_FLAG(std::optional, healthcheck_port, std::nullopt, @@ -171,6 +168,8 @@ absl::StatusOr GetConfigClient( INFERENCE_MODEL_BUCKET_NAME); config_client.SetFlag(FLAGS_inference_model_bucket_paths, INFERENCE_MODEL_BUCKET_PATHS); + config_client.SetFlag(FLAGS_inference_sidecar_runtime_config, + INFERENCE_SIDECAR_RUNTIME_CONFIG); if (absl::GetFlag(FLAGS_init_config_client)) { PS_RETURN_IF_ERROR(config_client.Init(config_param_prefix)).LogError() @@ -189,16 +188,16 @@ absl::StatusOr GetConfigClient( "support enabled"; } - PS_VLOG(1) << "Protected App Signals support enabled on the service: " - << enable_protected_app_signals; - PS_VLOG(1) << "Protected Audience support enabled on the service: " - << enable_protected_audience; - PS_VLOG(1) << "Successfully constructed the config client."; + PS_LOG(INFO) << "Protected App Signals support enabled on the service: " + << enable_protected_app_signals; + PS_LOG(INFO) << "Protected Audience support enabled on the service: " + << enable_protected_audience; + PS_LOG(INFO) << "Successfully constructed the config client."; return config_client; } -absl::string_view GetStringParameter(const TrustedServersConfigClient& client, - absl::string_view name) { +absl::string_view GetStringParameterSafe( + const TrustedServersConfigClient& client, absl::string_view name) { if (client.HasParameter(name)) { return client.GetStringParameter(name); } @@ -218,7 +217,7 @@ absl::Status RunServer() { << "BUYER_CODE_FETCH_CONFIG is a mandatory flag."; std::string_view inference_sidecar_binary_path = - GetStringParameter(config_client, INFERENCE_SIDECAR_BINARY_PATH); + GetStringParameterSafe(config_client, INFERENCE_SIDECAR_BINARY_PATH); const bool enable_inference = !inference_sidecar_binary_path.empty(); auto dispatcher = V8Dispatcher([&config_client, &enable_inference]() { @@ -228,28 +227,28 @@ absl::Status RunServer() { config.number_of_workers = config_client.GetIntParameter(JS_NUM_WORKERS); if (enable_inference) { -#ifdef PS_INFERENCE_NON_PROD - PS_VLOG(1) << "Register runInference API."; + PS_LOG(INFO) << "Register runInference API."; auto run_inference_function_object = std::make_unique>(); run_inference_function_object->function_name = std::string(inference::kInferenceFunctionName); run_inference_function_object->function = inference::RunInference; config.RegisterFunctionBinding(std::move(run_inference_function_object)); - PS_VLOG(1) << "RunInference registered."; + PS_LOG(INFO) << "RunInference registered."; - PS_VLOG(1) << "Start the inference sidecar."; - // This usage of flag is not consistent with rest of the codebase, where - // we use the parameter from the config client directly instead of passing - // it back to the absl flag. + PS_LOG(INFO) << "Start the inference sidecar."; + // This usage of the following two flags is not consistent with rest of + // the codebase, where we use the parameter from the config client + // directly instead of passing it back to the absl flag. absl::SetFlag( &FLAGS_inference_sidecar_binary_path, - config_client.GetStringParameter(INFERENCE_SIDECAR_BINARY_PATH) - .data()); + GetStringParameterSafe(config_client, INFERENCE_SIDECAR_BINARY_PATH)); + absl::SetFlag(&FLAGS_inference_sidecar_runtime_config, + GetStringParameterSafe(config_client, + INFERENCE_SIDECAR_RUNTIME_CONFIG)); inference::SandboxExecutor& inference_executor = inference::Executor(); CHECK_EQ(inference_executor.StartSandboxee().code(), absl::StatusCode::kOk); -#endif } return config; }()); @@ -289,41 +288,46 @@ absl::Status RunServer() { enable_protected_app_signals); PS_RETURN_IF_ERROR(udf_fetcher.Init()) << "Failed to initialize UDF fetch."; -#ifdef PS_INFERENCE_NON_PROD bool init_config_client = absl::GetFlag(FLAGS_init_config_client); if (enable_inference) { if (init_config_client) { - PS_VLOG(1) << "Start blob fetcher to read from a cloud bucket."; - std::unique_ptr blob_storage_client = - BlobStorageClientFactory::Create(); + PS_LOG(INFO) << "Start blob fetcher to read from a cloud bucket."; std::string_view bucket_name = - GetStringParameter(config_client, INFERENCE_MODEL_BUCKET_NAME); + GetStringParameterSafe(config_client, INFERENCE_MODEL_BUCKET_NAME); std::string_view bucket_paths = - GetStringParameter(config_client, INFERENCE_MODEL_BUCKET_PATHS); + GetStringParameterSafe(config_client, INFERENCE_MODEL_BUCKET_PATHS); std::vector models = absl::StrSplit(bucket_paths, ','); - auto blob_fetcher = std::make_unique( - bucket_name, executor.get(), std::move(blob_storage_client)); - CHECK(blob_fetcher->FetchSync().ok()) << "FetchSync() failed."; - const std::vector& files = blob_fetcher->snapshot(); - PS_VLOG(1) << "Register models from bucket."; - if (absl::Status status = - inference::RegisterModelsFromBucket(bucket_name, models, files); - !status.ok()) { - PS_VLOG(1) << "Skip registering models from bucket: " - << status.message(); + if (!bucket_name.empty() && !bucket_paths.empty()) { + std::unique_ptr blob_storage_client = + BlobStorageClientFactory::Create(); + auto blob_fetcher = std::make_unique( + bucket_name, executor.get(), std::move(blob_storage_client)); + CHECK(blob_fetcher->FetchSync().ok()) << "FetchSync() failed."; + const std::vector& files = blob_fetcher->snapshot(); + PS_LOG(INFO) << "Register models from bucket."; + if (absl::Status status = + inference::RegisterModelsFromBucket(bucket_name, models, files); + !status.ok()) { + PS_LOG(INFO) << "Skip registering models from bucket: " + << status.message(); + } + } else { + PS_LOG(INFO) + << "Skip blob fetcher read from a cloud bucket due to empty " + "required arguements."; } } - PS_VLOG(1) << "Register models from local."; + PS_LOG(INFO) << "Register models from local."; std::string_view local_paths = - GetStringParameter(config_client, INFERENCE_MODEL_LOCAL_PATHS); + GetStringParameterSafe(config_client, INFERENCE_MODEL_LOCAL_PATHS); if (absl::Status status = inference::RegisterModelsFromLocal( absl::StrSplit(local_paths, ',')); !status.ok()) { - PS_VLOG(1) << "Skip registering models from local: " << status.message(); + PS_LOG(INFO) << "Skip registering models from local: " + << status.message(); } } -#endif bool enable_bidding_service_benchmark = config_client.GetBooleanParameter(ENABLE_BIDDING_SERVICE_BENCHMARK); @@ -454,7 +458,7 @@ absl::Status RunServer() { } // Wait for the server to shutdown. Note that some other thread must be // responsible for shutting down the server for this call to ever return. - PS_VLOG(1) << "Server listening on " << server_address; + PS_LOG(INFO) << "Server listening on " << server_address; server->Wait(); return absl::OkStatus(); } diff --git a/services/bidding_service/bidding_service_integration_test.cc b/services/bidding_service/bidding_service_integration_test.cc index 6d1512f7..bc9eb2fd 100644 --- a/services/bidding_service/bidding_service_integration_test.cc +++ b/services/bidding_service/bidding_service_integration_test.cc @@ -80,6 +80,37 @@ constexpr absl::string_view js_code_template = R"JS_CODE( } )JS_CODE"; +constexpr absl::string_view js_code_with_buyer_reporting_id_template = + R"JS_CODE( + function fibonacci(num) { + if (num <= 1) return 1; + return fibonacci(num - 1) + fibonacci(num - 2); + } + + function generateBid(interest_group, + auction_signals, + buyer_signals, + trusted_bidding_signals, + device_signals) { + // Do a random amount of work to generate the price: + const bid = fibonacci(Math.floor(Math.random() * 10 + 1)); + + // Reshaped into an AdWithBid. + return { + render: "%s" + interest_group.adRenderIds[0], + ad: {"arbitraryMetadataField": 1}, + bid: bid, + bidCurrency: "USD", + adComponents: [ + "adComponent.com/comp?id=1", + "adComponent.com/comp?id=2" + ], + allowComponentAuction: false, + buyerReportingId: "%s" + }; + } + )JS_CODE"; + constexpr absl::string_view js_code_requiring_user_bidding_signals_template = R"JS_CODE( function fibonacci(num) { @@ -576,6 +607,80 @@ TEST_F(GenerateBidsReactorIntegrationTest, GeneratesBidsByInterestGroupCode) { } } +TEST_F(GenerateBidsReactorIntegrationTest, BuyerReportingIdSetInResponse) { + grpc::CallbackServerContext context; + V8Dispatcher dispatcher; + CodeDispatchClient client(dispatcher); + std::string test_buyer_reporting_id = "testBuyerReportingId"; + SetupV8Dispatcher( + &dispatcher, + absl::StrFormat(js_code_with_buyer_reporting_id_template, + kAdRenderUrlPrefixForTest, test_buyer_reporting_id)); + + GenerateBidsRequest request; + request.set_key_id(kKeyId); + absl::flat_hash_map> + interest_group_to_ad; + auto req = BuildGenerateBidsRequestFromBrowser(&interest_group_to_ad); + ASSERT_TRUE(req.ok()) << req.status(); + request.set_request_ciphertext(req->SerializeAsString()); + GenerateBidsResponse response; + + auto generate_bids_reactor_factory = + [&client](const GenerateBidsRequest* request, + GenerateBidsResponse* response, + server_common::KeyFetcherManagerInterface* key_fetcher_manager, + CryptoClientWrapperInterface* crypto_client, + const BiddingServiceRuntimeConfig& runtime_config) { + // You can manually flip this flag to turn benchmarking logging on or + // off + bool enable_benchmarking = true; + std::unique_ptr benchmarking_logger; + if (enable_benchmarking) { + benchmarking_logger = std::make_unique( + FormatTime(absl::Now())); + } else { + benchmarking_logger = std::make_unique(); + } + return new GenerateBidsReactor( + client, request, response, std::move(benchmarking_logger), + key_fetcher_manager, crypto_client, runtime_config); + }; + + auto protected_app_signals_generate_bids_reactor_factory = + [&client](const grpc::CallbackServerContext* context, + const GenerateProtectedAppSignalsBidsRequest* request, + const BiddingServiceRuntimeConfig& runtime_config, + GenerateProtectedAppSignalsBidsResponse* response, + server_common::KeyFetcherManagerInterface* key_fetcher_manager, + CryptoClientWrapperInterface* crypto_client, + KVAsyncClient* ad_retrieval_async_client, + KVAsyncClient* kv_async_client) { + return new ProtectedAppSignalsGenerateBidsReactor( + context, client, runtime_config, request, response, + key_fetcher_manager, crypto_client, ad_retrieval_async_client, + kv_async_client); + }; + + BiddingService service( + std::move(generate_bids_reactor_factory), std::move(key_fetcher_manager_), + std::move(crypto_client_), bidding_service_runtime_config_, + std::move(protected_app_signals_generate_bids_reactor_factory)); + service.GenerateBids(&context, &request, &response); + + // Todo(b/336330082) Redesign the tests to avoid waiting with thread sleep. + std::this_thread::sleep_for( + absl::ToChronoSeconds(absl::Seconds(kGenerateBidExecutionTimeSeconds))); + + GenerateBidsResponse::GenerateBidsRawResponse raw_response; + raw_response.ParseFromString(response.response_ciphertext()); + EXPECT_GT(raw_response.bids_size(), 0); + for (const auto& ad_with_bid : raw_response.bids()) { + EXPECT_GT(ad_with_bid.bid(), 0); + EXPECT_EQ(ad_with_bid.buyer_reporting_id(), test_buyer_reporting_id); + } +} + TEST_F(GenerateBidsReactorIntegrationTest, GeneratesBidsWithParsedUserBiddingSignals) { grpc::CallbackServerContext context; diff --git a/services/bidding_service/buyer_code_fetch_manager.cc b/services/bidding_service/buyer_code_fetch_manager.cc index 62eb5792..a3a8c619 100644 --- a/services/bidding_service/buyer_code_fetch_manager.cc +++ b/services/bidding_service/buyer_code_fetch_manager.cc @@ -48,7 +48,7 @@ using ::google::scp::core::errors::GetErrorMessage; BuyerCodeFetchManager::~BuyerCodeFetchManager() { if (absl::Status shutdown = End(); !shutdown.ok()) { - PS_VLOG(1) << "BuyerCodeFetchManager shutdown failed. " << shutdown; + PS_LOG(ERROR) << "BuyerCodeFetchManager shutdown failed. " << shutdown; } } @@ -132,7 +132,7 @@ absl::Status BuyerCodeFetchManager::InitializeBucketCodeFetchForPAS() { return GetBuyerWrappedCode( ad_tech_code_blobs[kJsBlobIndex], kUnusedWasmBlob, AuctionType::kProtectedAppSignals, - /*auction_specific_setup=*/"// No additional setup"); + /*auction_specific_setup=*/kEncodedProtectedAppSignalsHandler); }; auto wrap_ads_retrieval_code = @@ -205,7 +205,7 @@ absl::Status BuyerCodeFetchManager::InitializeUrlCodeFetchForPAS() { ? ad_tech_code_blobs[kWasmBlobIndex] : kUnusedWasmBlob, AuctionType::kProtectedAppSignals, - /*auction_specific_setup=*/"// No additional setup"); + /*auction_specific_setup=*/kEncodedProtectedAppSignalsHandler); }; auto wrap_ads_retrieval_code = @@ -294,19 +294,10 @@ BuyerCodeFetchManager::StartBucketFetch( } absl::Status BuyerCodeFetchManager::InitBucketClient() { - auto result = blob_storage_client_->Init(); - if (!result.Successful()) { - return absl::UnavailableError( - absl::StrFormat("Failed to init BlobStorageClient (status_code: %s)\n", - GetErrorMessage(result.status_code))); - } - - result = blob_storage_client_->Run(); - if (!result.Successful()) { - return absl::UnavailableError( - absl::StrFormat("Failed to run BlobStorageClient (status_code: %s)\n", - GetErrorMessage(result.status_code))); - } + PS_RETURN_IF_ERROR(blob_storage_client_->Init()).SetPrepend() + << "Failed to init BlobStorageClient: "; + PS_RETURN_IF_ERROR(blob_storage_client_->Run()).SetPrepend() + << "Failed to run BlobStorageClient: "; return absl::OkStatus(); } diff --git a/services/bidding_service/buyer_code_fetch_manager_test.cc b/services/bidding_service/buyer_code_fetch_manager_test.cc index dfe9ac85..ed24f43b 100644 --- a/services/bidding_service/buyer_code_fetch_manager_test.cc +++ b/services/bidding_service/buyer_code_fetch_manager_test.cc @@ -85,10 +85,8 @@ TEST_F(BuyerCodeFetchManagerTest, FetchModeLocalTriesFileLoad) { } TEST_F(BuyerCodeFetchManagerTest, FetchModeBucketTriesBucketLoad) { - EXPECT_CALL(*blob_storage_client_, Init) - .WillOnce(Return(SuccessExecutionResult())); - EXPECT_CALL(*blob_storage_client_, Run) - .WillOnce(Return(SuccessExecutionResult())); + EXPECT_CALL(*blob_storage_client_, Init).WillOnce(Return(absl::OkStatus())); + EXPECT_CALL(*blob_storage_client_, Run).WillOnce(Return(absl::OkStatus())); BuyerCodeFetchConfig udf_config; udf_config.set_fetch_mode(FetchMode::FETCH_MODE_BUCKET); BuyerCodeFetchManager udf_fetcher(executor_.get(), http_fetcher_.get(), @@ -109,12 +107,17 @@ TEST_F(BuyerCodeFetchManagerTest, TriesBucketFetchForProtectedAuction) { udf_config.set_protected_auction_bidding_js_bucket("pa"); udf_config.set_protected_auction_bidding_js_bucket_default_blob("pa_test"); - EXPECT_CALL(*blob_storage_client_, Init) - .WillOnce(Return(SuccessExecutionResult())); - EXPECT_CALL(*blob_storage_client_, Run) - .WillOnce(Return(SuccessExecutionResult())); + EXPECT_CALL(*blob_storage_client_, Init).WillOnce(Return(absl::OkStatus())); + EXPECT_CALL(*blob_storage_client_, Run).WillOnce(Return(absl::OkStatus())); + EXPECT_CALL(*blob_storage_client_, ListBlobsMetadata) + .WillOnce( + [](AsyncContext + context) { + context.response = std::make_shared(); + context.Finish(SuccessExecutionResult()); + return absl::OkStatus(); + }); - // EXPECT_CALL(*blob_storage_client_) BuyerCodeFetchManager udf_fetcher(executor_.get(), http_fetcher_.get(), dispatcher_.get(), std::move(blob_storage_client_), udf_config, @@ -135,10 +138,16 @@ TEST_F(BuyerCodeFetchManagerTest, TriesBucketFetchForProtectedAppSignals) { udf_config.set_protected_app_signals_bidding_js_bucket_default_blob( "pas_test"); - EXPECT_CALL(*blob_storage_client_, Init) - .WillOnce(Return(SuccessExecutionResult())); - EXPECT_CALL(*blob_storage_client_, Run) - .WillOnce(Return(SuccessExecutionResult())); + EXPECT_CALL(*blob_storage_client_, Init).WillOnce(Return(absl::OkStatus())); + EXPECT_CALL(*blob_storage_client_, Run).WillOnce(Return(absl::OkStatus())); + EXPECT_CALL(*blob_storage_client_, ListBlobsMetadata) + .WillOnce( + [](AsyncContext + context) { + context.response = std::make_shared(); + context.Finish(SuccessExecutionResult()); + return absl::OkStatus(); + }); BuyerCodeFetchManager udf_fetcher(executor_.get(), http_fetcher_.get(), dispatcher_.get(), @@ -169,13 +178,12 @@ TEST_F(BuyerCodeFetchManagerTest, udf_config.set_ads_retrieval_js_bucket(ads_bucket); udf_config.set_ads_retrieval_bucket_default_blob(ads_object); - EXPECT_CALL(*blob_storage_client_, Init) - .WillOnce(Return(SuccessExecutionResult())); - EXPECT_CALL(*blob_storage_client_, Run) - .WillOnce(Return(SuccessExecutionResult())); + EXPECT_CALL(*blob_storage_client_, Init).WillOnce(Return(absl::OkStatus())); + EXPECT_CALL(*blob_storage_client_, Run).WillOnce(Return(absl::OkStatus())); EXPECT_CALL(*blob_storage_client_, ListBlobsMetadata) .WillOnce( - [&](AsyncContext + [&pas_bucket, &pas_object]( + AsyncContext context) { EXPECT_EQ(context.request->blob_metadata().bucket_name(), pas_bucket); @@ -186,13 +194,13 @@ TEST_F(BuyerCodeFetchManagerTest, context.response = std::make_shared(); context.response->mutable_blob_metadatas()->Add(std::move(md)); context.Finish(SuccessExecutionResult()); - return SuccessExecutionResult(); + return absl::OkStatus(); }) .WillOnce([&ads_bucket]( const AsyncContext& context) { EXPECT_EQ(context.request->blob_metadata().bucket_name(), ads_bucket); - return FailureExecutionResult(SC_UNKNOWN); + return absl::UnknownError(""); }); EXPECT_CALL(*blob_storage_client_, GetBlob) .WillOnce( @@ -210,7 +218,7 @@ TEST_F(BuyerCodeFetchManagerTest, async_context.result = SuccessExecutionResult(); async_context.Finish(); - return SuccessExecutionResult(); + return absl::OkStatus(); }); EXPECT_CALL(*dispatcher_, LoadSync) @@ -220,7 +228,7 @@ TEST_F(BuyerCodeFetchManagerTest, EXPECT_EQ( blob_data, GetBuyerWrappedCode(pas_data, "", AuctionType::kProtectedAppSignals, - "// No additional setup")); + kEncodedProtectedAppSignalsHandler)); return absl::OkStatus(); }); diff --git a/services/bidding_service/code_wrapper/BUILD b/services/bidding_service/code_wrapper/BUILD index 1d8718a9..120973cd 100644 --- a/services/bidding_service/code_wrapper/BUILD +++ b/services/bidding_service/code_wrapper/BUILD @@ -28,6 +28,7 @@ cc_library( "//services/bidding_service:runtime_flags", "//services/common/clients/config:config_client", "//services/common/constants:common_service_flags", + "//services/common/util:reporting_util", "//services/common/util:request_response_constants", "@com_google_absl//absl/strings", "@google_privacysandbox_servers_common//src/logger:request_context_impl", @@ -60,6 +61,7 @@ cc_test( "//services/common/constants:common_service_flags", "//services/common/test:mocks", "//services/common/util:json_util", + "//services/common/util:reporting_util", "@com_google_absl//absl/strings", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", diff --git a/services/bidding_service/code_wrapper/buyer_code_wrapper.cc b/services/bidding_service/code_wrapper/buyer_code_wrapper.cc index 02d784c6..5a32887e 100644 --- a/services/bidding_service/code_wrapper/buyer_code_wrapper.cc +++ b/services/bidding_service/code_wrapper/buyer_code_wrapper.cc @@ -21,22 +21,12 @@ #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" #include "services/bidding_service/constants.h" +#include "services/common/util/reporting_util.h" #include "src/logger/request_context_impl.h" namespace privacy_sandbox::bidding_auction_servers { namespace { -void AppendFeatureFlagValue(std::string& feature_flags, - absl::string_view feature_name, - bool is_feature_enabled) { - absl::string_view enable_feature = kFeatureDisabled; - if (is_feature_enabled) { - enable_feature = kFeatureEnabled; - } - feature_flags.append( - absl::StrCat("\"", feature_name, "\": ", enable_feature)); -} - std::string WasmBytesToJavascript(absl::string_view wasm_bytes) { std::string hex_array = ""; @@ -57,7 +47,8 @@ absl::string_view GetGenerateBidArgs(AuctionType auction_type) { case AuctionType::kProtectedAppSignals: return kProtectedAppSignalsGenerateBidsArgs; default: - PS_VLOG(1) << "Unsupported auction type: " << absl::StrCat(auction_type); + PS_LOG(ERROR) << "Unsupported auction type: " + << absl::StrCat(auction_type); return ""; } } @@ -84,14 +75,4 @@ std::string GetProtectedAppSignalsGenericBuyerWrappedCode( ad_tech_js); } -std::string GetFeatureFlagJson(bool enable_logging, - bool enable_debug_url_generation) { - std::string feature_flags = "{"; - AppendFeatureFlagValue(feature_flags, kFeatureLogging, enable_logging); - feature_flags.append(","); - AppendFeatureFlagValue(feature_flags, kFeatureDebugUrlGeneration, - enable_debug_url_generation); - feature_flags.append("}"); - return feature_flags; -} } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/bidding_service/code_wrapper/buyer_code_wrapper.h b/services/bidding_service/code_wrapper/buyer_code_wrapper.h index 070256c6..83e026ad 100644 --- a/services/bidding_service/code_wrapper/buyer_code_wrapper.h +++ b/services/bidding_service/code_wrapper/buyer_code_wrapper.h @@ -23,12 +23,6 @@ namespace privacy_sandbox::bidding_auction_servers { -constexpr char kFeatureLogging[] = "enable_logging"; -constexpr char kFeatureDebugUrlGeneration[] = "enable_debug_url_generation"; - -constexpr char kFeatureDisabled[] = "false"; -constexpr char kFeatureEnabled[] = "true"; - // Returns the complete wrapped code for Buyer. // The function adds wrappers to the Buyer provided generateBid function. // This enables: @@ -50,17 +44,12 @@ std::string GetProtectedAppSignalsGenericBuyerWrappedCode( absl::string_view ad_tech_js, absl::string_view ad_tech_wasm, absl::string_view function_name, absl::string_view args); -// Returns a JSON string for feature flags to be used by the wrapper script. -std::string GetFeatureFlagJson(bool enable_logging = true, - bool enable_debug_url_generation = false); - // Wrapper Javascript over AdTech code. // This wrapper supports the features below: //- Exporting logs to Bidding Service using console.log //- Hooks in wasm module inline constexpr absl::string_view kEntryFunction = R"JS_CODE( function generateBidEntryFunction($0, featureFlags){ - $1 var ps_logs = []; var ps_errors = []; var ps_warns = []; @@ -75,7 +64,7 @@ inline constexpr absl::string_view kEntryFunction = R"JS_CODE( ps_warns.push(JSON.stringify(args)) } } - + $1 var forDebuggingOnly_auction_loss_url = undefined; var forDebuggingOnly_auction_win_url = undefined; const forDebuggingOnly = {}; diff --git a/services/bidding_service/code_wrapper/buyer_code_wrapper_test.cc b/services/bidding_service/code_wrapper/buyer_code_wrapper_test.cc index 136283ca..f16b8a15 100644 --- a/services/bidding_service/code_wrapper/buyer_code_wrapper_test.cc +++ b/services/bidding_service/code_wrapper/buyer_code_wrapper_test.cc @@ -21,7 +21,9 @@ #include "gtest/gtest.h" #include "rapidjson/document.h" #include "services/bidding_service/code_wrapper/buyer_code_wrapper_test_constants.h" +#include "services/bidding_service/constants.h" #include "services/common/util/json_util.h" +#include "services/common/util/reporting_util.h" namespace privacy_sandbox::bidding_auction_servers { @@ -47,11 +49,11 @@ TEST(GetBuyerWrappedCode, GeneratesCompleteFinalJavascriptWithWasm) { } TEST(GetBuyerWrappedCode, GeneratesCompleteProtectedAppSignalsFinalJavaScript) { - EXPECT_EQ( - GetBuyerWrappedCode(kBuyerBaseCode_template, /*ad_tech_wasm=*/"", - AuctionType::kProtectedAppSignals, - /*auction_specific_setup=*/"// No additional setup."), - kExpectedProtectedAppSignalsGenerateBidCodeTemplate); + EXPECT_EQ(GetBuyerWrappedCode( + kBuyerBaseCode_template, /*ad_tech_wasm=*/"", + AuctionType::kProtectedAppSignals, + /*auction_specific_setup=*/kEncodedProtectedAppSignalsHandler), + kExpectedProtectedAppSignalsGenerateBidCodeTemplate); } void GenerateFeatureFlagsTestHelper(bool is_logging_enabled, diff --git a/services/bidding_service/code_wrapper/buyer_code_wrapper_test_constants.h b/services/bidding_service/code_wrapper/buyer_code_wrapper_test_constants.h index 7b1942e2..38379f52 100644 --- a/services/bidding_service/code_wrapper/buyer_code_wrapper_test_constants.h +++ b/services/bidding_service/code_wrapper/buyer_code_wrapper_test_constants.h @@ -45,7 +45,6 @@ constexpr absl::string_view kExpectedGenerateBidCode_template = R"JS_CODE( const globalWasmHelper = globalWasmHex.length ? new WebAssembly.Module(Uint8Array.from(globalWasmHex)) : null; function generateBidEntryFunction(interest_group, auction_signals, buyer_signals, trusted_bidding_signals, device_signals, featureFlags){ - device_signals.wasmHelper = globalWasmHelper; var ps_logs = []; var ps_errors = []; var ps_warns = []; @@ -60,7 +59,7 @@ constexpr absl::string_view kExpectedGenerateBidCode_template = R"JS_CODE( ps_warns.push(JSON.stringify(args)) } } - + device_signals.wasmHelper = globalWasmHelper; var forDebuggingOnly_auction_loss_url = undefined; var forDebuggingOnly_auction_win_url = undefined; const forDebuggingOnly = {}; @@ -118,8 +117,7 @@ constexpr absl::string_view const globalWasmHex = []; const globalWasmHelper = globalWasmHex.length ? new WebAssembly.Module(Uint8Array.from(globalWasmHex)) : null; - function generateBidEntryFunction(ads, sellerAuctionSignals, buyerSignals, preprocessedDataForRetrieval, protectedAppSignals, encodingVersion, featureFlags){ - // No additional setup. + function generateBidEntryFunction(ads, sellerAuctionSignals, buyerSignals, preprocessedDataForRetrieval, encodedOnDeviceSignals, encodingVersion, featureFlags){ var ps_logs = []; var ps_errors = []; var ps_warns = []; @@ -134,7 +132,15 @@ constexpr absl::string_view ps_warns.push(JSON.stringify(args)) } } - + if (encodedOnDeviceSignals) { + const convertToUint8Array = + (encodedOnDeviceSignalsIn) => + Uint8Array.from(encodedOnDeviceSignalsIn.match(/.{1,2}/g).map((byte) => + parseInt(byte, 16))); + console.log("PAS hex string: " + encodedOnDeviceSignals); + encodedOnDeviceSignals = convertToUint8Array(encodedOnDeviceSignals); + console.log("Uint8 PAS bytes: " + Array.apply([], encodedOnDeviceSignals).join(",")); + } var forDebuggingOnly_auction_loss_url = undefined; var forDebuggingOnly_auction_win_url = undefined; const forDebuggingOnly = {}; @@ -148,7 +154,7 @@ constexpr absl::string_view var generateBidResponse = {}; try { - generateBidResponse = generateBid(ads, sellerAuctionSignals, buyerSignals, preprocessedDataForRetrieval, protectedAppSignals, encodingVersion); + generateBidResponse = generateBid(ads, sellerAuctionSignals, buyerSignals, preprocessedDataForRetrieval, encodedOnDeviceSignals, encodingVersion); if( featureFlags.enable_debug_url_generation && (forDebuggingOnly_auction_loss_url || forDebuggingOnly_auction_win_url)) { diff --git a/services/bidding_service/constants.h b/services/bidding_service/constants.h index 97daa032..23dc90ec 100644 --- a/services/bidding_service/constants.h +++ b/services/bidding_service/constants.h @@ -40,7 +40,18 @@ constexpr absl::string_view kProtectedAudienceGenerateBidsArgs = "device_signals"; constexpr absl::string_view kProtectedAppSignalsGenerateBidsArgs = "ads, sellerAuctionSignals, buyerSignals, " - "preprocessedDataForRetrieval, protectedAppSignals, encodingVersion"; + "preprocessedDataForRetrieval, encodedOnDeviceSignals, encodingVersion"; + +constexpr absl::string_view kEncodedProtectedAppSignalsHandler = + R"JS_CODE(if (encodedOnDeviceSignals) { + const convertToUint8Array = + (encodedOnDeviceSignalsIn) => + Uint8Array.from(encodedOnDeviceSignalsIn.match(/.{1,2}/g).map((byte) => + parseInt(byte, 16))); + console.log("PAS hex string: " + encodedOnDeviceSignals); + encodedOnDeviceSignals = convertToUint8Array(encodedOnDeviceSignals); + console.log("Uint8 PAS bytes: " + Array.apply([], encodedOnDeviceSignals).join(",")); + })JS_CODE"; // Params related to the UDF that can be used to retrieve the protected // embeddings that can later be used to fetch the top-k ads from ads retrieval diff --git a/services/bidding_service/generate_bids_reactor.cc b/services/bidding_service/generate_bids_reactor.cc index 0dc3dde5..5444dc30 100644 --- a/services/bidding_service/generate_bids_reactor.cc +++ b/services/bidding_service/generate_bids_reactor.cc @@ -448,9 +448,10 @@ void GenerateBidsReactor::Execute() { absl::StatusOr ig_trusted_signals_map = SerializeTrustedBiddingSignalsPerIG(raw_request_, log_context_); if (!ig_trusted_signals_map.ok()) { - PS_VLOG(0, log_context_) << "Request failed while parsing bidding signals: " - << ig_trusted_signals_map.status().ToString( - absl::StatusToStringMode::kWithEverything); + PS_LOG(ERROR, log_context_) + << "Request failed while parsing bidding signals: " + << ig_trusted_signals_map.status().ToString( + absl::StatusToStringMode::kWithEverything); EncryptResponseAndFinish( server_common::FromAbslStatus(ig_trusted_signals_map.status())); return; @@ -501,7 +502,7 @@ void GenerateBidsReactor::Execute() { LogIfError(metric_context_ ->AccumulateMetric( 1, metric::kBiddingGenerateBidsFailedToDispatchCode)); - PS_VLOG(1, log_context_) + PS_LOG(ERROR, log_context_) << "Execution request failed for batch: " << raw_request_.DebugString() << status.ToString(absl::StatusToStringMode::kWithEverything); LogIfError( @@ -542,13 +543,15 @@ void GenerateBidsReactor::GenerateBidsCallback( ParseAndGetResponseJson(enable_adtech_code_logging_, result->resp, log_context_); if (!generate_bid_response.ok()) { - PS_VLOG(0, log_context_) + PS_LOG(ERROR, log_context_) << "Failed to parse response from Roma " << generate_bid_response.status().ToString( absl::StatusToStringMode::kWithEverything); } + google::protobuf::json::ParseOptions parse_options; + parse_options.ignore_unknown_fields = true; auto valid = google::protobuf::util::JsonStringToMessage( - generate_bid_response.value(), &bid); + generate_bid_response.value(), &bid, parse_options); const std::string interest_group_name = result->id; if (valid.ok()) { if (current_all_debug_urls_chars >= @@ -569,16 +572,16 @@ void GenerateBidsReactor::GenerateBidsCallback( auction_scope_ == AuctionScope::kDeviceComponentSeller && !bid.allow_component_auction()) { // TODO(b/311234165): Add metric for rejected component ads. - PS_VLOG(1, log_context_) + PS_LOG(ERROR, log_context_) << "Skipping component bid as it is not allowed for " << interest_group_name << ": " << bid.DebugString(); } else { bid.set_interest_group_name(interest_group_name); - *raw_response_.add_bids() = bid; + *raw_response_.add_bids() = std::move(bid); is_bid_zero = false; } } else { - PS_VLOG(1, log_context_) + PS_LOG(ERROR, log_context_) << "Invalid json output from code execution for interest_group " << interest_group_name << ": " << result->resp; } @@ -587,7 +590,7 @@ void GenerateBidsReactor::GenerateBidsCallback( LogIfError(metric_context_ ->AccumulateMetric( 1, metric::kBiddingGenerateBidsDispatchResponseError)); - PS_VLOG(1, log_context_) + PS_LOG(ERROR, log_context_) << "Invalid execution (possibly invalid input): " << result.status().ToString( absl::StatusToStringMode::kWithEverything); @@ -601,17 +604,18 @@ void GenerateBidsReactor::GenerateBidsCallback( LogIfError(metric_context_->LogHistogram( (static_cast(zero_bid_count)) / total_bid_count)); - PS_VLOG(1, log_context_) << "\n\nFailed of total: " << failed_requests << "/" - << output.size(); + PS_VLOG(kInfoMsg, log_context_) + << "\n\nFailed of total: " << failed_requests << "/" << output.size(); benchmarking_logger_->HandleResponseEnd(); } void GenerateBidsReactor::EncryptResponseAndFinish(grpc::Status status) { - PS_VLOG(kPlain, log_context_) << "GenerateBidsRawResponse\n" - << raw_response_.DebugString(); + PS_VLOG(kPlain, log_context_) << "GenerateBidsRawResponse:\n" + << raw_response_.ShortDebugString(); if (!EncryptResponse()) { - PS_VLOG(1, log_context_) << "Failed to encrypt the generate bids response."; + PS_LOG(ERROR, log_context_) + << "Failed to encrypt the generate bids response."; status = grpc::Status(grpc::INTERNAL, kInternalServerError); } if (status.error_code() != grpc::StatusCode::OK) { diff --git a/services/bidding_service/generate_bids_reactor_test.cc b/services/bidding_service/generate_bids_reactor_test.cc index fd70dc6c..f82aff92 100644 --- a/services/bidding_service/generate_bids_reactor_test.cc +++ b/services/bidding_service/generate_bids_reactor_test.cc @@ -83,6 +83,36 @@ std::string GetTestResponse(absl::string_view render, float bid) { render, bid); } +std::string GetTestResponseWithBuyerReportingId( + absl::string_view render, float bid, absl::string_view buyer_reporting_id) { + return absl::Substitute(R"JSON({ + "response": { + "render": "$0", + "bid": $1, + "buyerReportingId": "$2" + }, + "logs": [], + "errors": [], + "warnings":[] + })JSON", + render, bid, buyer_reporting_id); +} + +std::string GetTestResponseWithUnknownField(absl::string_view render, + float bid) { + return absl::Substitute(R"JSON({ + "response": { + "render": "$0", + "bid": $1, + "buyer_reporting_ids": "abcdef" + }, + "logs": [], + "errors": [], + "warnings":[] + })JSON", + render, bid); +} + std::string GetComponentAuctionResponse(absl::string_view render, float bid, bool allow_component_auction) { return absl::Substitute(R"JSON({ @@ -331,6 +361,65 @@ TEST_F(GenerateBidsReactorTest, GenerateBidSuccessfulWithCodeWrapper) { enable_adtech_code_logging); } +TEST_F(GenerateBidsReactorTest, BuyerReportingIdSetInResponse) { + bool enable_debug_reporting = false; + bool enable_buyer_debug_url_generation = false; + bool enable_adtech_code_logging = true; + std::string response_json = GetTestResponseWithBuyerReportingId( + kTestRenderUrl, 1, kTestBuyerReportingId); + AdWithBid bid; + bid.set_render(kTestRenderUrl); + bid.set_bid(1); + bid.set_interest_group_name("ig_name_Bar"); + bid.set_buyer_reporting_id(kTestBuyerReportingId); + GenerateBidsResponse::GenerateBidsRawResponse raw_response; + *raw_response.add_bids() = bid; + Response ads; + *ads.mutable_response_ciphertext() = raw_response.SerializeAsString(); + std::vector igs; + igs.push_back(GetIGForBiddingBar()); + + EXPECT_CALL(dispatcher_, BatchExecute) + .WillOnce([response_json](std::vector& batch, + BatchDispatchDoneCallback batch_callback) { + return FakeExecute(batch, std::move(batch_callback), response_json); + }); + RawRequest raw_request; + BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, + kTestBiddingSignals, raw_request, enable_debug_reporting); + CheckGenerateBids(raw_request, ads, enable_buyer_debug_url_generation, + enable_adtech_code_logging); +} + +TEST_F(GenerateBidsReactorTest, UnknownFieldInResponseParsedSuccessfully) { + bool enable_debug_reporting = false; + bool enable_buyer_debug_url_generation = false; + bool enable_adtech_code_logging = true; + std::string response_json = + GetTestResponseWithUnknownField(kTestRenderUrl, 1); + AdWithBid bid; + bid.set_render(kTestRenderUrl); + bid.set_bid(1); + bid.set_interest_group_name("ig_name_Bar"); + GenerateBidsResponse::GenerateBidsRawResponse raw_response; + *raw_response.add_bids() = bid; + Response ads; + *ads.mutable_response_ciphertext() = raw_response.SerializeAsString(); + std::vector igs; + igs.push_back(GetIGForBiddingBar()); + + EXPECT_CALL(dispatcher_, BatchExecute) + .WillOnce([response_json](std::vector& batch, + BatchDispatchDoneCallback batch_callback) { + return FakeExecute(batch, std::move(batch_callback), response_json); + }); + RawRequest raw_request; + BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, + kTestBiddingSignals, raw_request, enable_debug_reporting); + CheckGenerateBids(raw_request, ads, enable_buyer_debug_url_generation, + enable_adtech_code_logging); +} + TEST_F(GenerateBidsReactorTest, DoesNotGenerateBidsIfBiddingSignalsAreMalformed) { auto in_signals = R"JSON({"ig_name_Bar":1})JSON"; diff --git a/services/bidding_service/generate_bids_reactor_test_utils.cc b/services/bidding_service/generate_bids_reactor_test_utils.cc index 7fcccb02..c5950f8f 100644 --- a/services/bidding_service/generate_bids_reactor_test_utils.cc +++ b/services/bidding_service/generate_bids_reactor_test_utils.cc @@ -82,7 +82,8 @@ GenerateProtectedAppSignalsBidsRawRequest CreateRawProtectedAppSignalsRequest( const std::string& auction_signals, const std::string& buyer_signals, const ProtectedAppSignals& protected_app_signals, const std::string& seller, const std::string& publisher_name, - absl::optional contextual_pas_data) { + absl::optional contextual_pas_data, + bool enable_unlimited_egress) { GenerateProtectedAppSignalsBidsRawRequest raw_request; raw_request.set_auction_signals(auction_signals); raw_request.set_buyer_signals(buyer_signals); @@ -93,7 +94,8 @@ GenerateProtectedAppSignalsBidsRawRequest CreateRawProtectedAppSignalsRequest( *raw_request.mutable_contextual_protected_app_signals_data() = *std::move(contextual_pas_data); } - PS_VLOG(1) << "Created request:\n" << raw_request.DebugString(); + raw_request.set_enable_unlimited_egress(enable_unlimited_egress); + PS_LOG(INFO) << "Created request:\n" << raw_request.DebugString(); return raw_request; } @@ -147,24 +149,31 @@ std::string CreatePrepareDataForAdsRetrievalResponse( std::string CreateGenerateBidsUdfResponse( absl::string_view render, double bid, - absl::string_view egress_features_hex_string, - absl::string_view debug_reporting_urls) { + absl::string_view egress_payload_hex_string, + absl::string_view debug_reporting_urls, + absl::string_view temporary_egress_payload_hex_string) { std::string base64_encoded_features_bytes; - absl::Base64Escape(absl::HexStringToBytes(egress_features_hex_string), + absl::Base64Escape(absl::HexStringToBytes(egress_payload_hex_string), &base64_encoded_features_bytes); + std::string base64_encoded_temporary_features_bytes; + absl::Base64Escape( + absl::HexStringToBytes(temporary_egress_payload_hex_string), + &base64_encoded_temporary_features_bytes); return absl::Substitute(R"JSON( { "response": { "render": "$0", "bid": $1, - "egressFeatures": "$2", - "debugReportUrls": $3 + "egressPayload": "$2", + "debugReportUrls": $3, + "temporaryUnlimitedEgressPayload": "$4" }, "logs": [] } )JSON", render, bid, base64_encoded_features_bytes, - debug_reporting_urls); + debug_reporting_urls, + base64_encoded_temporary_features_bytes); } void SetupContextualProtectedAppSignalsRomaExpectations( diff --git a/services/bidding_service/generate_bids_reactor_test_utils.h b/services/bidding_service/generate_bids_reactor_test_utils.h index 98757545..a763f1ca 100644 --- a/services/bidding_service/generate_bids_reactor_test_utils.h +++ b/services/bidding_service/generate_bids_reactor_test_utils.h @@ -50,10 +50,12 @@ constexpr char kTestProtectedAppSignals[] = "test_protected_app_signals"; constexpr char kTestRetrievalData[] = "test_retrieval_data"; constexpr double kTestWinningBid = 1.25; constexpr int kTestAdRetrievalTimeoutMs = 100000; -constexpr char kTestEgressFeaturesHex[] = "bccd"; -constexpr char kTestEgressFeaturesBiggerThan3Bytes[] = "deadbeef"; -constexpr char kTestEgressFeaturesBiggerThan23bits[] = "f01020"; +constexpr char kTestEgressPayloadHex[] = "bccd"; +constexpr char kTestTemporaryEgressPayloadHex[] = "abcdefabcedfe"; +constexpr char kTestEgressPayloadBiggerThan3Bytes[] = "deadbeef"; +constexpr char kTestEgressPayloadBiggerThan23bits[] = "f01020"; constexpr char kTestAdsRetrievalAdsResponse[] = "Ads Data And Metadata"; +constexpr char kTestBuyerReportingId[] = "testBuyerReportingId"; constexpr char kTestAdsRetrievalContextualEmbeddingsResponse[] = R"JSON( { "contextualEmbeddings1": { @@ -82,7 +84,8 @@ GenerateProtectedAppSignalsBidsRawRequest CreateRawProtectedAppSignalsRequest( const ProtectedAppSignals& protected_app_signals, const std::string& seller, const std::string& publisher_name, absl::optional contextual_pas_data = - absl::nullopt); + absl::nullopt, + bool enable_unlimited_egress = false); // Creates a generate protected app signals bids request using the provided // raw request. @@ -102,8 +105,10 @@ std::string CreatePrepareDataForAdsRetrievalResponse( // Creates a mock response from `generateBid` UDF. std::string CreateGenerateBidsUdfResponse( absl::string_view render = kTestRenderUrl, double bid = kTestWinningBid, - absl::string_view egress_features_hex_string = kTestEgressFeaturesHex, - absl::string_view debug_reporting_urls = kTestDebugReportingUrls); + absl::string_view egress_payload_hex_string = kTestEgressPayloadHex, + absl::string_view debug_reporting_urls = kTestDebugReportingUrls, + absl::string_view temporary_egress_payload_hex_string = + kTestTemporaryEgressPayloadHex); // Creates a mock response from ads retrieval service. absl::StatusOr diff --git a/services/bidding_service/inference/BUILD b/services/bidding_service/inference/BUILD index e798b19d..1a6c21c5 100644 --- a/services/bidding_service/inference/BUILD +++ b/services/bidding_service/inference/BUILD @@ -26,6 +26,7 @@ cc_library( deps = [ ":inference_flags", "//services/common/blob_fetch:blob_fetcher", + "//services/common/util:request_response_constants", "@com_github_grpc_grpc//:grpc++", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/flags:flag", diff --git a/services/bidding_service/inference/inference_flags.cc b/services/bidding_service/inference/inference_flags.cc index 97102937..60a3797f 100644 --- a/services/bidding_service/inference/inference_flags.cc +++ b/services/bidding_service/inference/inference_flags.cc @@ -35,3 +35,16 @@ ABSL_FLAG(std::optional, inference_model_bucket_paths, std::nullopt, "Comma separated list of bucket paths. Used to specify a list of " "directories to fetch the blobs from."); +// The JSON string should adhere to the following format: +// { +// "num_interop_threads": , +// "num_intraop_threads": , +// "module_name": , +// ... +// } +// where each property corresponds to a field of the +// InferenceSidecarRuntimeConfig proto in +// services/inference_sidecar/common/proto/inference_sidecar.proto. +ABSL_FLAG(std::optional, inference_sidecar_runtime_config, + std::nullopt, + "JSON string configurations for the inference sidecar runtime."); diff --git a/services/bidding_service/inference/inference_flags.h b/services/bidding_service/inference/inference_flags.h index f9591482..4dac6e59 100644 --- a/services/bidding_service/inference/inference_flags.h +++ b/services/bidding_service/inference/inference_flags.h @@ -25,6 +25,7 @@ ABSL_DECLARE_FLAG(std::optional, inference_sidecar_binary_path); ABSL_DECLARE_FLAG(std::optional, inference_model_local_paths); ABSL_DECLARE_FLAG(std::optional, inference_model_bucket_name); ABSL_DECLARE_FLAG(std::optional, inference_model_bucket_paths); +ABSL_DECLARE_FLAG(std::optional, inference_sidecar_runtime_config); namespace privacy_sandbox::bidding_auction_servers { @@ -36,9 +37,12 @@ inline constexpr char INFERENCE_MODEL_BUCKET_NAME[] = "INFERENCE_MODEL_BUCKET_NAME"; inline constexpr char INFERENCE_MODEL_BUCKET_PATHS[] = "INFERENCE_MODEL_BUCKET_PATHS"; +inline constexpr char INFERENCE_SIDECAR_RUNTIME_CONFIG[] = + "INFERENCE_SIDECAR_RUNTIME_CONFIG"; inline constexpr absl::string_view kInferenceFlags[] = { INFERENCE_SIDECAR_BINARY_PATH, INFERENCE_MODEL_LOCAL_PATHS, - INFERENCE_MODEL_BUCKET_NAME, INFERENCE_MODEL_BUCKET_PATHS}; + INFERENCE_MODEL_BUCKET_NAME, INFERENCE_MODEL_BUCKET_PATHS, + INFERENCE_SIDECAR_RUNTIME_CONFIG}; } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/bidding_service/inference/inference_utils.cc b/services/bidding_service/inference/inference_utils.cc index d971a635..e7bc4ef8 100644 --- a/services/bidding_service/inference/inference_utils.cc +++ b/services/bidding_service/inference/inference_utils.cc @@ -34,6 +34,7 @@ #include "absl/synchronization/mutex.h" #include "proto/inference_sidecar.grpc.pb.h" #include "services/bidding_service/inference/inference_flags.h" +#include "services/common/util/request_response_constants.h" #include "src/logger/request_context_logger.h" #include "src/roma/interface/roma.h" #include "src/util/status_macro/status_macros.h" @@ -48,7 +49,8 @@ SandboxExecutor& Executor() { // TODO(b/317124648): Pass a SandboxExecutor object via Roma's `TMetadata`. static SandboxExecutor* executor = new SandboxExecutor( - *absl::GetFlag(FLAGS_inference_sidecar_binary_path), {""}); + *absl::GetFlag(FLAGS_inference_sidecar_binary_path), + {*absl::GetFlag(FLAGS_inference_sidecar_runtime_config)}); return *executor; } @@ -135,7 +137,7 @@ void RunInference(google::scp::roma::FunctionBindingPayload<>& wrapper) { std::unique_ptr stub = InferenceService::NewStub(InferenceChannel(executor)); - PS_VLOG(1) << "RunInference input: " << payload; + PS_VLOG(kInfoMsg) << "RunInference input: " << payload; PredictRequest predict_request; predict_request.set_input(payload); @@ -151,7 +153,7 @@ void RunInference(google::scp::roma::FunctionBindingPayload<>& wrapper) { } absl::Status status = server_common::ToAbslStatus(rpc_status); // TODO(b/321284008): Communicate inference failure with JS caller. - PS_VLOG(1) << "Response error: " << status.message(); + PS_LOG(ERROR) << "Response error: " << status.message(); } } // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/bidding_service/inference/inference_utils_test.cc b/services/bidding_service/inference/inference_utils_test.cc index 1a2ed135..1dc269bc 100644 --- a/services/bidding_service/inference/inference_utils_test.cc +++ b/services/bidding_service/inference/inference_utils_test.cc @@ -31,6 +31,8 @@ constexpr absl::string_view kInit = "non-empty"; constexpr absl::string_view kTestModelPath = "external/inference_common/testdata/models/tensorflow_1_mib_saved_model.pb"; constexpr absl::string_view kBucketName = "test_bucket"; +constexpr absl::string_view kRuntimeConfig = + R"json({"num_interop_threads": 4, "num_intraop_threads": 5, "module_name": "test"})json"; class InferenceUtilsTest : public ::testing::Test { protected: @@ -38,6 +40,7 @@ class InferenceUtilsTest : public ::testing::Test { absl::SetFlag(&FLAGS_testonly_allow_policies_for_bazel, true); absl::SetFlag(&FLAGS_inference_sidecar_binary_path, GetFilePath(kSidecarBinary)); + absl::SetFlag(&FLAGS_inference_sidecar_runtime_config, kRuntimeConfig); } private: @@ -50,6 +53,7 @@ class InferenceUtilsTest : public ::testing::Test { TEST_F(InferenceUtilsTest, ReturnValueIsSet) { SandboxExecutor& inference_executor = Executor(); CHECK_EQ(inference_executor.StartSandboxee().code(), absl::StatusCode::kOk); + ASSERT_TRUE(RegisterModelsFromLocal({std::string(kTestModelPath)}).ok()); google::scp::roma::proto::FunctionBindingIoProto input_output_proto; google::scp::roma::FunctionBindingPayload<> wrapper{input_output_proto, {}}; diff --git a/services/bidding_service/protected_app_signals_generate_bids_reactor.cc b/services/bidding_service/protected_app_signals_generate_bids_reactor.cc index 1e0e342b..9f2d6346 100644 --- a/services/bidding_service/protected_app_signals_generate_bids_reactor.cc +++ b/services/bidding_service/protected_app_signals_generate_bids_reactor.cc @@ -21,12 +21,14 @@ #include +#include "absl/flags/flag.h" #include "absl/strings/escaping.h" #include "absl/strings/numbers.h" #include "public/applications/pas/retrieval_request_builder.h" #include "public/applications/pas/retrieval_response_parser.h" #include "services/bidding_service/code_wrapper/buyer_code_wrapper.h" #include "services/bidding_service/constants.h" +#include "services/common/feature_flags.h" #include "services/common/util/json_util.h" #include "services/common/util/reporting_util.h" #include "services/common/util/request_metadata.h" @@ -60,7 +62,7 @@ inline void PopulateArgInRomaRequest( // Gets string information about a bid's well-formed-ness. inline std::string GetBidDebugInfo(const ProtectedAppSignalsAdWithBid& bid) { return absl::StrCat("Is non-zero bid: ", bid.bid() > 0.0f, - ", Num egress bytes: ", bid.egress_features().size(), + ", Num egress bytes: ", bid.egress_payload().size(), ", has debug report urls: ", bid.has_debug_report_urls()); } @@ -117,13 +119,20 @@ ProtectedAppSignalsGenerateBidsReactor::CreateAdsRetrievalRequest( ad_render_ids_list = std::vector(ad_render_ids->begin(), ad_render_ids->end()); } - return std::make_unique( - kv_server::application_pas::BuildRetrievalRequest( - prepare_data_for_ads_retrieval_response, - {{kClientIp, metadata_[kClientIpKey]}, - {kAcceptLanguage, metadata_[kAcceptLanguageKey]}, - {kUserAgent, metadata_[kUserAgentKey]}}, - raw_request_.buyer_signals(), std::move(ad_render_ids_list))); + auto ads_retrieval_request = + std::make_unique( + kv_server::application_pas::BuildRetrievalRequest( + prepare_data_for_ads_retrieval_response, + {{kClientIp, metadata_[kClientIpKey]}, + {kAcceptLanguage, metadata_[kAcceptLanguageKey]}, + {kUserAgent, metadata_[kUserAgentKey]}}, + raw_request_.buyer_signals(), std::move(ad_render_ids_list))); + // Set consented debug config. + if (raw_request_.has_consented_debug_config()) { + *ads_retrieval_request->mutable_consented_debug_config() = + raw_request_.consented_debug_config(); + } + return ads_retrieval_request; } std::unique_ptr @@ -134,6 +143,11 @@ ProtectedAppSignalsGenerateBidsReactor::CreateKVLookupRequest( auto request = std::make_unique( kv_server::application_pas::BuildLookupRequest( std::move(ad_render_ids_list))); + // Set consented debug config. + if (raw_request_.has_consented_debug_config()) { + *request->mutable_consented_debug_config() = + raw_request_.consented_debug_config(); + } PS_VLOG(8) << __func__ << " Created KV lookup request: " << request->DebugString(); return request; @@ -256,7 +270,11 @@ void ProtectedAppSignalsGenerateBidsReactor::OnFetchAdsDataDone( << "Successful non-zero protected app signals bid received"; auto* added_bid = raw_response_.add_bids(); *added_bid = bid; - added_bid->clear_egress_features(); + added_bid->clear_egress_payload(); + if (!raw_request_.enable_unlimited_egress() || + !absl::GetFlag(FLAGS_enable_temporary_unlimited_egress)) { + added_bid->clear_temporary_unlimited_egress_payload(); + } } EncryptResponseAndFinish(grpc::Status::OK); }); @@ -285,7 +303,8 @@ DispatchRequest ProtectedAppSignalsGenerateBidsReactor:: raw_request_.buyer_signals(), ArgIndex(PrepareDataForRetrievalUdfArgs::kBuyerSignals), input); PopulateArgInRomaRequest( - GetFeatureFlagJson(enable_adtech_code_logging_), + GetFeatureFlagJson(enable_adtech_code_logging_, + /*enable_debug_url_generation=*/false), ArgIndex(PrepareDataForRetrievalUdfArgs::kFeatureFlags), input); DispatchRequest request = { .id = raw_request_.log_context().generation_id(), @@ -391,8 +410,8 @@ void ProtectedAppSignalsGenerateBidsReactor::Execute() { PS_VLOG(8, log_context_) << __func__; PS_VLOG(2, log_context_) << "GenerateBidsRequest:\n" << request_->DebugString(); - PS_VLOG(1, log_context_) << "GenerateBidsRawRequest:\n" - << raw_request_.DebugString(); + PS_VLOG(kPlain, log_context_) << "GenerateBidsRawRequest:\n" + << raw_request_.ShortDebugString(); if (IsContextualRetrievalRequest()) { StartContextualAdsRetrieval(); @@ -414,7 +433,7 @@ void ProtectedAppSignalsGenerateBidsReactor::EncryptResponseAndFinish( grpc::Status status) { PS_VLOG(8, log_context_) << __func__; if (!EncryptResponse()) { - PS_VLOG(1, log_context_) + PS_LOG(ERROR, log_context_) << "Failed to encrypt the generate app signals bids response."; status = grpc::Status(grpc::INTERNAL, kInternalServerError); } diff --git a/services/bidding_service/protected_app_signals_generate_bids_reactor_test.cc b/services/bidding_service/protected_app_signals_generate_bids_reactor_test.cc index 9eb48fff..81e5dac5 100644 --- a/services/bidding_service/protected_app_signals_generate_bids_reactor_test.cc +++ b/services/bidding_service/protected_app_signals_generate_bids_reactor_test.cc @@ -20,6 +20,7 @@ #include #include +#include "absl/flags/flag.h" #include "absl/status/status.h" #include "absl/synchronization/notification.h" #include "gmock/gmock.h" @@ -33,6 +34,7 @@ #include "services/common/constants/common_service_flags.h" #include "services/common/encryption/key_fetcher_factory.h" #include "services/common/encryption/mock_crypto_client_wrapper.h" +#include "services/common/feature_flags.h" #include "services/common/metric/server_definition.h" #include "services/common/test/mocks.h" #include "services/common/test/random.h" @@ -339,7 +341,7 @@ TEST_F(GenerateBidsReactorTest, GenerateBidsInputIsCorrect) { ASSERT_EQ(num_roma_dispatches, 2); } -TEST_F(GenerateBidsReactorTest, EgressFeaturesAreNotPopulated) { +TEST_F(GenerateBidsReactorTest, egressPayloadAreNotPopulated) { int num_roma_dispatches = 0; SetupProtectedAppSignalsRomaExpectations(dispatcher_, num_roma_dispatches); @@ -373,7 +375,7 @@ TEST_F(GenerateBidsReactorTest, EgressFeaturesAreNotPopulated) { EXPECT_EQ(generated_bid.bid(), kTestWinningBid); EXPECT_EQ(generated_bid.render(), kTestRenderUrl); - ASSERT_EQ(generated_bid.egress_features().size(), 0); + ASSERT_EQ(generated_bid.egress_payload().size(), 0); } TEST_F(GenerateBidsReactorTest, ZeroBidsAreFiltered) { @@ -397,7 +399,7 @@ TEST_F(GenerateBidsReactorTest, ZeroBidsAreFiltered) { kProtectedAppSignalsGenerateBidBlobVersion, CreateGenerateBidsUdfResponse( kTestRenderUrl, /*bid=*/0.0, - /*egress_features_hex_string=*/"", + /*egress_payload_hex_string=*/"", /*debug_reporting_urls=*/"RJSON({})JSON")); } }); @@ -599,5 +601,124 @@ TEST_F(GenerateBidsReactorTest, KvInputIsCorrect) { EXPECT_EQ(generated_bid.render(), kTestRenderUrl); } +TEST_F(GenerateBidsReactorTest, TemporaryEgressVectorGetsPopulated) { + absl::SetFlag(&FLAGS_enable_temporary_unlimited_egress, true); + int num_roma_dispatches = 0; + SetupProtectedAppSignalsRomaExpectations(dispatcher_, num_roma_dispatches); + + EXPECT_CALL(ad_retrieval_client_, ExecuteInternal) + .WillOnce( + [](std::unique_ptr raw_request, + const RequestMetadata& metadata, + absl::AnyInvocable< + void(absl::StatusOr>) &&> + on_done, + absl::Duration timeout) { + auto response = CreateAdsRetrievalOrKvLookupResponse(); + EXPECT_TRUE(response.ok()) << response.status(); + std::move(on_done)( + std::make_unique(*std::move(response))); + return absl::OkStatus(); + }); + + auto raw_request = CreateRawProtectedAppSignalsRequest( + kTestAuctionSignals, kTestBuyerSignals, + CreateProtectedAppSignals(kTestAppInstallSignals, kTestEncodingVersion), + kSeller, kPublisherName, /*contextual_pas_data=*/absl::nullopt, + /*enable_unlimited_egress=*/true); + auto raw_response = RunReactorWithRequest(raw_request); + + // One dispatch to `preparedDataForAdRetrieval` and another to `generateBids` + // is expected. + ASSERT_EQ(num_roma_dispatches, 2); + + ASSERT_EQ(raw_response.bids().size(), 1); + const auto& generated_bid = raw_response.bids()[0]; + EXPECT_EQ(generated_bid.bid(), kTestWinningBid); + EXPECT_EQ(generated_bid.render(), kTestRenderUrl); + + EXPECT_GT(generated_bid.temporary_unlimited_egress_payload().size(), 0); +} + +TEST_F(GenerateBidsReactorTest, + TemporaryEgressVectorNotPopulatedWhenNotEnabled) { + absl::SetFlag(&FLAGS_enable_temporary_unlimited_egress, true); + int num_roma_dispatches = 0; + SetupProtectedAppSignalsRomaExpectations(dispatcher_, num_roma_dispatches); + + EXPECT_CALL(ad_retrieval_client_, ExecuteInternal) + .WillOnce( + [](std::unique_ptr raw_request, + const RequestMetadata& metadata, + absl::AnyInvocable< + void(absl::StatusOr>) &&> + on_done, + absl::Duration timeout) { + auto response = CreateAdsRetrievalOrKvLookupResponse(); + EXPECT_TRUE(response.ok()) << response.status(); + std::move(on_done)( + std::make_unique(*std::move(response))); + return absl::OkStatus(); + }); + + auto raw_request = CreateRawProtectedAppSignalsRequest( + kTestAuctionSignals, kTestBuyerSignals, + CreateProtectedAppSignals(kTestAppInstallSignals, kTestEncodingVersion), + kSeller, kPublisherName, /*contextual_pas_data=*/absl::nullopt, + /*enable_unlimited_egress=*/false); + auto raw_response = RunReactorWithRequest(raw_request); + + // One dispatch to `preparedDataForAdRetrieval` and another to `generateBids` + // is expected. + ASSERT_EQ(num_roma_dispatches, 2); + + ASSERT_EQ(raw_response.bids().size(), 1); + const auto& generated_bid = raw_response.bids()[0]; + EXPECT_EQ(generated_bid.bid(), kTestWinningBid); + EXPECT_EQ(generated_bid.render(), kTestRenderUrl); + + EXPECT_EQ(generated_bid.temporary_unlimited_egress_payload().size(), 0); +} + +TEST_F(GenerateBidsReactorTest, + TemporaryEgressVectorNotPopulatedWhenFeatureIsOff) { + absl::SetFlag(&FLAGS_enable_temporary_unlimited_egress, false); + int num_roma_dispatches = 0; + SetupProtectedAppSignalsRomaExpectations(dispatcher_, num_roma_dispatches); + + EXPECT_CALL(ad_retrieval_client_, ExecuteInternal) + .WillOnce( + [](std::unique_ptr raw_request, + const RequestMetadata& metadata, + absl::AnyInvocable< + void(absl::StatusOr>) &&> + on_done, + absl::Duration timeout) { + auto response = CreateAdsRetrievalOrKvLookupResponse(); + EXPECT_TRUE(response.ok()) << response.status(); + std::move(on_done)( + std::make_unique(*std::move(response))); + return absl::OkStatus(); + }); + + auto raw_request = CreateRawProtectedAppSignalsRequest( + kTestAuctionSignals, kTestBuyerSignals, + CreateProtectedAppSignals(kTestAppInstallSignals, kTestEncodingVersion), + kSeller, kPublisherName, /*contextual_pas_data=*/absl::nullopt, + /*enable_unlimited_egress=*/true); + auto raw_response = RunReactorWithRequest(raw_request); + + // One dispatch to `preparedDataForAdRetrieval` and another to `generateBids` + // is expected. + ASSERT_EQ(num_roma_dispatches, 2); + + ASSERT_EQ(raw_response.bids().size(), 1); + const auto& generated_bid = raw_response.bids()[0]; + EXPECT_EQ(generated_bid.bid(), kTestWinningBid); + EXPECT_EQ(generated_bid.render(), kTestRenderUrl); + + EXPECT_EQ(generated_bid.temporary_unlimited_egress_payload().size(), 0); +} + } // namespace } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/buyer_frontend_service/buyer_frontend_main.cc b/services/buyer_frontend_service/buyer_frontend_main.cc index 274e5892..469506ce 100644 --- a/services/buyer_frontend_service/buyer_frontend_main.cc +++ b/services/buyer_frontend_service/buyer_frontend_main.cc @@ -166,11 +166,11 @@ absl::StatusOr GetConfigClient( "support enabled"; } - PS_VLOG(1) << "Protected App Signals support enabled on the service: " - << enable_protected_app_signals; - PS_VLOG(1) << "Protected Audience support enabled on the service: " - << enable_protected_audience; - PS_VLOG(1) << "Successfully constructed the config client.\n"; + PS_LOG(INFO) << "Protected App Signals support enabled on the service: " + << enable_protected_app_signals; + PS_LOG(INFO) << "Protected Audience support enabled on the service: " + << enable_protected_audience; + PS_LOG(INFO) << "Successfully constructed the config client.\n"; return config_client; } @@ -284,7 +284,7 @@ absl::Status RunServer() { if (server == nullptr) { return absl::UnavailableError("Error starting Server."); } - PS_VLOG(1) << "Server listening on " << server_address; + PS_LOG(INFO) << "Server listening on " << server_address; // Wait for the server to shut down. Note that some other thread must be // responsible for shutting down the server for this call to ever return. diff --git a/services/buyer_frontend_service/get_bids_unary_reactor.cc b/services/buyer_frontend_service/get_bids_unary_reactor.cc index c391ec36..b6dcc064 100644 --- a/services/buyer_frontend_service/get_bids_unary_reactor.cc +++ b/services/buyer_frontend_service/get_bids_unary_reactor.cc @@ -154,10 +154,10 @@ void GetBidsUnaryReactor::OnAllBidsDone(bool any_successful_bids) { } PS_VLOG(kPlain, log_context_) << "GetBidsRawResponse:\n" - << get_bids_raw_response_->DebugString(); + << get_bids_raw_response_->ShortDebugString(); if (auto encryption_status = EncryptResponse(); !encryption_status.ok()) { - PS_VLOG(1, log_context_) << "Failed to encrypt the response"; + PS_LOG(ERROR, log_context_) << "Failed to encrypt the response"; benchmarking_logger_->End(); FinishWithStatus( grpc::Status(grpc::StatusCode::INTERNAL, encryption_status.ToString())); @@ -231,8 +231,8 @@ void GetBidsUnaryReactor::Execute() { absl::StreamFormatter())); if (!decrypt_status_.ok()) { - PS_VLOG(1, log_context_) << "Decrypting the request failed:" - << server_common::ToAbslStatus(decrypt_status_); + PS_LOG(ERROR, log_context_) << "Decrypting the request failed:" + << server_common::ToAbslStatus(decrypt_status_); FinishWithStatus(decrypt_status_); return; } @@ -303,7 +303,7 @@ void GetBidsUnaryReactor::MayGetProtectedSignalsBids() { metric::kBfeErrorCountByErrorCode>( 1, metric:: kBfeGenerateProtectedAppSignalsBidsResponseError)); - PS_VLOG(1, log_context_) + PS_LOG(ERROR, log_context_) << "Execution of GenerateProtectedAppSignalsBids request " "failed with status: " << status; @@ -387,7 +387,7 @@ void GetBidsUnaryReactor::MayGetProtectedAudienceBids() { 1, metric::kBfeBiddingSignalsResponseError)); LogInitiatedRequestErrorMetrics(metric::kKv, response.status()); // Return error to client. - PS_VLOG(1, log_context_) + PS_LOG(ERROR, log_context_) << "GetBiddingSignals request failed with status:" << response.status(); async_task_tracker_.TaskCompleted( @@ -408,7 +408,7 @@ void GetBidsUnaryReactor::PrepareAndGenerateProtectedAudienceBid( std::unique_ptr bidding_signals) { if (!bidding_signals || !bidding_signals->trusted_signals || bidding_signals->trusted_signals->empty()) { - PS_VLOG(1, log_context_) + PS_LOG(ERROR, log_context_) << "GetBiddingSignals request succeeded but was empty."; async_task_tracker_.TaskCompleted(TaskStatus::EMPTY_RESPONSE); return; @@ -447,7 +447,7 @@ void GetBidsUnaryReactor::PrepareAndGenerateProtectedAudienceBid( ->AccumulateMetric( 1, metric::kBfeGenerateBidsResponseError)); LogInitiatedRequestErrorMetrics(metric::kBs, status); - PS_VLOG(1, log_context_) + PS_LOG(ERROR, log_context_) << "Execution of GenerateBids request failed with status: " << status; async_task_tracker_.TaskCompleted( diff --git a/services/buyer_frontend_service/providers/http_bidding_signals_async_provider.cc b/services/buyer_frontend_service/providers/http_bidding_signals_async_provider.cc index 0dfdd6e3..837b41f8 100644 --- a/services/buyer_frontend_service/providers/http_bidding_signals_async_provider.cc +++ b/services/buyer_frontend_service/providers/http_bidding_signals_async_provider.cc @@ -68,7 +68,7 @@ void HttpBiddingSignalsAsyncProvider::Get( }, timeout); if (!status.ok()) { - PS_VLOG(1) << "Unable to fetch bidding signals"; + PS_LOG(ERROR) << "Unable to fetch bidding signals"; } } } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/buyer_frontend_service/util/BUILD b/services/buyer_frontend_service/util/BUILD index 32c25b81..7975535d 100644 --- a/services/buyer_frontend_service/util/BUILD +++ b/services/buyer_frontend_service/util/BUILD @@ -33,7 +33,7 @@ cc_library( ) cc_test( - name = "buyer_frontend_utils_test", + name = "proto_factory_test", size = "small", testonly = True, srcs = [ diff --git a/services/buyer_frontend_service/util/proto_factory.cc b/services/buyer_frontend_service/util/proto_factory.cc index d8385d7b..a992bd0d 100644 --- a/services/buyer_frontend_service/util/proto_factory.cc +++ b/services/buyer_frontend_service/util/proto_factory.cc @@ -187,6 +187,9 @@ CreateGenerateProtectedAppSignalsBidsRawRequest( generate_bids_raw_request->set_enable_debug_reporting( raw_request.enable_debug_reporting()); + generate_bids_raw_request->set_enable_unlimited_egress( + raw_request.enable_unlimited_egress()); + return generate_bids_raw_request; } diff --git a/services/buyer_frontend_service/util/proto_factory_test.cc b/services/buyer_frontend_service/util/proto_factory_test.cc index 82e3c506..8a0810af 100644 --- a/services/buyer_frontend_service/util/proto_factory_test.cc +++ b/services/buyer_frontend_service/util/proto_factory_test.cc @@ -460,10 +460,11 @@ TEST(CreateGenerateProtectedAppSignalsBidsRawRequestTest, kTestProtectedAppSignals); EXPECT_EQ(request->seller(), kTestSeller); EXPECT_EQ(request->publisher_name(), kTestPublisherName); - EXPECT_EQ(request->enable_debug_reporting(), true); + EXPECT_TRUE(request->enable_debug_reporting()); + EXPECT_TRUE(request->enable_unlimited_egress()); EXPECT_EQ(request->log_context().generation_id(), kTestGenerationId); EXPECT_EQ(request->log_context().adtech_debug_id(), kTestAdTechDebugId); - EXPECT_EQ(request->consented_debug_config().is_consented(), true); + EXPECT_TRUE(request->consented_debug_config().is_consented()); EXPECT_EQ(request->consented_debug_config().token(), kTestConsentedDebuggingToken); ASSERT_TRUE(request->has_contextual_protected_app_signals_data()); diff --git a/services/common/BUILD b/services/common/BUILD new file mode 100644 index 00000000..ddf1b546 --- /dev/null +++ b/services/common/BUILD @@ -0,0 +1,30 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_cc//cc:defs.bzl", "cc_library") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "feature_flags", + srcs = [ + "feature_flags.cc", + ], + hdrs = [ + "feature_flags.h", + ], + deps = [ + "@com_google_absl//absl/flags:flag", + ], +) diff --git a/services/common/blob_fetch/BUILD b/services/common/blob_fetch/BUILD index 137a3861..7aa7dade 100644 --- a/services/common/blob_fetch/BUILD +++ b/services/common/blob_fetch/BUILD @@ -30,6 +30,7 @@ cc_library( "@google_privacysandbox_servers_common//src/public/core/interface:errors", "@google_privacysandbox_servers_common//src/public/core/interface:execution_result", "@google_privacysandbox_servers_common//src/public/cpio/interface/blob_storage_client", + "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", ], ) diff --git a/services/common/blob_fetch/blob_fetcher.cc b/services/common/blob_fetch/blob_fetcher.cc index bdae3a7e..1e2abdc8 100644 --- a/services/common/blob_fetch/blob_fetcher.cc +++ b/services/common/blob_fetch/blob_fetcher.cc @@ -27,6 +27,7 @@ #include "src/public/core/interface/execution_result.h" #include "src/public/cpio/interface/blob_storage_client/blob_storage_client_interface.h" #include "src/public/cpio/proto/blob_storage_service/v1/blob_storage_service.pb.h" +#include "src/util/status_macro/status_macros.h" using ::google::cmrt::sdk::blob_storage_service::v1::GetBlobRequest; using ::google::cmrt::sdk::blob_storage_service::v1::GetBlobResponse; @@ -45,7 +46,10 @@ BlobFetcher::BlobFetcher( : bucket_name_(bucket_name), executor_(executor), blob_storage_client_(std::move(blob_storage_client)) { - InitAndRunConfigClient(); + absl::Status status = blob_storage_client_->Init(); + CHECK(status.ok()) << "Failed to init BlobStorageClient: " << status; + status = blob_storage_client_->Run(); + CHECK(status.ok()) << "Failed to run BlobStorageClient: " << status; } absl::Status BlobFetcher::FetchSync() { @@ -59,18 +63,6 @@ absl::Status BlobFetcher::FetchSync() { return status; } -void BlobFetcher::InitAndRunConfigClient() { - auto result = blob_storage_client_->Init(); - CHECK(result.Successful()) - << absl::StrFormat("Failed to init BlobStorageClient (status_code: %s)\n", - GetErrorMessage(result.status_code)); - - result = blob_storage_client_->Run(); - CHECK(result.Successful()) - << absl::StrFormat("Failed to run BlobStorageClient (status_code: %s)\n", - GetErrorMessage(result.status_code)); -} - absl::Status BlobFetcher::InternalFetch() { absl::Status status; std::vector blob_names; @@ -79,12 +71,13 @@ absl::Status BlobFetcher::InternalFetch() { auto list_blobs_request = std::make_shared(); absl::Notification notification; list_blobs_request->mutable_blob_metadata()->set_bucket_name(bucket_name_); + list_blobs_request->set_exclude_directories(true); AsyncContext list_blobs_context(list_blobs_request, [&status, &blob_names, ¬ification](auto& context) { if (!context.result.Successful()) { - PS_VLOG(0) << "Failed to list blobs: " - << GetErrorMessage(context.result.status_code); + PS_LOG(ERROR) << "Failed to list blobs: " + << GetErrorMessage(context.result.status_code); status = absl::InternalError("Failed to list blobs"); } else { PS_VLOG(10) << "BlobStorageClient ListBlobsMetadata() Response: " @@ -94,6 +87,7 @@ absl::Status BlobFetcher::InternalFetch() { context.response->blob_metadatas(i).blob_name()); } } + // The caller waits for the notification. // Please note that the callback might not be called by // ListBlobsMetadata. @@ -102,22 +96,17 @@ absl::Status BlobFetcher::InternalFetch() { notification.Notify(); }); - auto list_blobs_execution = - blob_storage_client_->ListBlobsMetadata(list_blobs_context); - if (!list_blobs_execution.Successful()) { - // If ListBlobsMetadata fails fast, we return the error early without - // waiting for `notification`. - return absl::InternalError( - absl::StrCat("BlobStorageClient -> ListBlobsMetadata() failed: ", - GetErrorMessage(list_blobs_execution.status_code))); - } + // If ListBlobsMetadata fails fast, we return the error early without + // waiting for `notification`. + PS_RETURN_IF_ERROR( + blob_storage_client_->ListBlobsMetadata(list_blobs_context)); notification.WaitForNotification(); + // Checks the error from the callback. - if (!status.ok()) { - return status; - } + PS_RETURN_IF_ERROR(status); std::vector new_file_snapshot; + // Fetches all the blobs in the bucket. // TODO(b/329674737): Fetch blobs in parallel. for (const std::string& blob_name : blob_names) { @@ -130,8 +119,8 @@ absl::Status BlobFetcher::InternalFetch() { get_blob_request, [&status, &new_file_snapshot, &per_blob_notification](auto& context) { if (!context.result.Successful()) { - PS_VLOG(0) << "Failed to fetch blobs: " - << GetErrorMessage(context.result.status_code); + PS_LOG(ERROR) << "Failed to fetch blobs: " + << GetErrorMessage(context.result.status_code); status = absl::InternalError("Failed to fetch blobs"); } else { // Should not log blob().data(), which can be very large bytes. @@ -147,19 +136,13 @@ absl::Status BlobFetcher::InternalFetch() { per_blob_notification.Notify(); }); - auto get_blob_execution = blob_storage_client_->GetBlob(get_blob_context); - if (!get_blob_execution.Successful()) { - // If GetBlob fails fast, we return the error early. We update the file - // snapshot only when all the file fetching is successfully done. - return absl::InternalError( - absl::StrCat("BlobStorageClient -> GetBlob() failed: ", - GetErrorMessage(get_blob_execution.status_code))); - } + // If GetBlob fails fast, we return the error early. We update the file + // snapshot only when all the file fetching is successfully done. + PS_RETURN_IF_ERROR(blob_storage_client_->GetBlob(get_blob_context)); per_blob_notification.WaitForNotification(); + // Checks the error from the callback. - if (!status.ok()) { - return status; - } + PS_RETURN_IF_ERROR(status); } // All the blobs are successfully fetched. diff --git a/services/common/blob_fetch/blob_fetcher.h b/services/common/blob_fetch/blob_fetcher.h index cc4eacd1..bd20decb 100644 --- a/services/common/blob_fetch/blob_fetcher.h +++ b/services/common/blob_fetch/blob_fetcher.h @@ -61,9 +61,6 @@ class BlobFetcher { // Performs bucket fetching with BlobStorageClient. // It's not thread-safe. absl::Status InternalFetch(); - // Initializes BlobStorageClient. - // Prerequisite: Caller must call google::scp::cpio::Cpio::InitCpio() ahead. - void InitAndRunConfigClient(); const std::string bucket_name_; server_common::Executor* executor_; // not owned diff --git a/services/common/blob_fetch/blob_fetcher_test.cc b/services/common/blob_fetch/blob_fetcher_test.cc index 2861901c..330ae95e 100644 --- a/services/common/blob_fetch/blob_fetcher_test.cc +++ b/services/common/blob_fetch/blob_fetcher_test.cc @@ -46,18 +46,18 @@ TEST(BlobFetcherTest, FetchBucket) { auto executor = std::make_unique(); auto blob_storage_client = std::make_unique(); - EXPECT_CALL(*blob_storage_client, Run).WillOnce([&]() { - return SuccessExecutionResult(); + EXPECT_CALL(*blob_storage_client, Run).WillOnce([]() { + return absl::OkStatus(); }); - EXPECT_CALL(*executor, Run).WillOnce([&](absl::AnyInvocable closure) { + EXPECT_CALL(*executor, Run).WillOnce([](absl::AnyInvocable closure) { closure(); }); EXPECT_CALL(*blob_storage_client, ListBlobsMetadata) .WillOnce( - [&](AsyncContext - async_context) { + [](AsyncContext + async_context) { auto async_bucket_name = async_context.request->blob_metadata().bucket_name(); EXPECT_EQ(async_bucket_name, kSampleBucketName); @@ -70,12 +70,12 @@ TEST(BlobFetcherTest, FetchBucket) { async_context.result = SuccessExecutionResult(); async_context.Finish(); - return SuccessExecutionResult(); + return absl::OkStatus(); }); EXPECT_CALL(*blob_storage_client, GetBlob) .WillOnce( - [&](AsyncContext async_context) { + [](AsyncContext async_context) { auto async_bucket_name = async_context.request->blob_metadata().bucket_name(); auto async_blob_name = @@ -88,7 +88,7 @@ TEST(BlobFetcherTest, FetchBucket) { async_context.result = SuccessExecutionResult(); async_context.Finish(); - return SuccessExecutionResult(); + return absl::OkStatus(); }); BlobFetcher bucket_fetcher(kSampleBucketName, executor.get(), @@ -100,18 +100,18 @@ TEST(BlobFetcherTest, FetchBucket_Failure) { auto executor = std::make_unique(); auto blob_storage_client = std::make_unique(); - EXPECT_CALL(*blob_storage_client, Run).WillOnce([&]() { - return SuccessExecutionResult(); + EXPECT_CALL(*blob_storage_client, Run).WillOnce([] { + return absl::OkStatus(); }); - EXPECT_CALL(*executor, Run).WillOnce([&](absl::AnyInvocable closure) { + EXPECT_CALL(*executor, Run).WillOnce([](absl::AnyInvocable closure) { closure(); }); EXPECT_CALL(*blob_storage_client, ListBlobsMetadata) .WillOnce( - [&](AsyncContext - async_context) { + [](AsyncContext + async_context) { auto async_bucket_name = async_context.request->blob_metadata().bucket_name(); EXPECT_EQ(async_bucket_name, kSampleBucketName); @@ -124,22 +124,21 @@ TEST(BlobFetcherTest, FetchBucket_Failure) { async_context.result = SuccessExecutionResult(); async_context.Finish(); - return SuccessExecutionResult(); + return absl::OkStatus(); }); EXPECT_CALL(*blob_storage_client, GetBlob) .WillOnce( - [&](AsyncContext async_context) { + [](AsyncContext async_context) { async_context.result = FailureExecutionResult(SC_UNKNOWN); async_context.Finish(); - - return FailureExecutionResult(SC_UNKNOWN); + return absl::UnknownError(""); }); BlobFetcher bucket_fetcher(kSampleBucketName, executor.get(), std::move(blob_storage_client)); auto status = bucket_fetcher.FetchSync(); - EXPECT_EQ(status.code(), absl::StatusCode::kInternal); + EXPECT_FALSE(status.ok()); } } // namespace diff --git a/services/common/clients/async_grpc/default_async_grpc_client.h b/services/common/clients/async_grpc/default_async_grpc_client.h index a7a38d10..8a476d19 100644 --- a/services/common/clients/async_grpc/default_async_grpc_client.h +++ b/services/common/clients/async_grpc/default_async_grpc_client.h @@ -81,8 +81,8 @@ class DefaultAsyncGrpcClient std::move(raw_request), *crypto_client_, *key_fetcher_manager_, cloud_platform_); if (!secret_request.ok()) { - PS_VLOG(1) << "Failed to encrypt the request: " - << secret_request.status(); + PS_LOG(ERROR) << "Failed to encrypt the request: " + << secret_request.status(); return absl::InternalError(kEncryptionFailed); } auto& [hpke_secret, request] = *secret_request; diff --git a/services/common/clients/async_grpc/default_async_grpc_client_test.cc b/services/common/clients/async_grpc/default_async_grpc_client_test.cc index 8f39133e..c5a8ad14 100644 --- a/services/common/clients/async_grpc/default_async_grpc_client_test.cc +++ b/services/common/clients/async_grpc/default_async_grpc_client_test.cc @@ -125,7 +125,7 @@ class TestDefaultAsyncGrpcClient void SendRpc(const std::string& hpke_secret, RawClientParams* params) const override { - PS_VLOG(1) << "SendRpc invoked"; + PS_LOG(INFO) << "SendRpc invoked"; EXPECT_EQ(params->RequestRef()->request_ciphertext(), req_.request_ciphertext()); absl::Duration actual_timeout = diff --git a/services/common/clients/auction_server/scoring_async_client.cc b/services/common/clients/auction_server/scoring_async_client.cc index ce5d5787..97a04de9 100644 --- a/services/common/clients/auction_server/scoring_async_client.cc +++ b/services/common/clients/auction_server/scoring_async_client.cc @@ -40,8 +40,8 @@ void ScoringAsyncGrpcClient::SendRpc( params->ContextRef(), params->RequestRef(), params->ResponseRef(), [this, params, hpke_secret](const grpc::Status& status) { if (!status.ok()) { - PS_VLOG(1) << "SendRPC completion status not ok: " - << server_common::ToAbslStatus(status); + PS_LOG(ERROR) << "SendRPC completion status not ok: " + << server_common::ToAbslStatus(status); params->OnDone(status); return; } @@ -50,7 +50,7 @@ void ScoringAsyncGrpcClient::SendRpc( auto decrypted_response = DecryptResponse(hpke_secret, params->ResponseRef()); if (!decrypted_response.ok()) { - PS_VLOG(1) << "ScoringAsyncGrpcClient Failed to decrypt response"; + PS_LOG(ERROR) << "ScoringAsyncGrpcClient Failed to decrypt response"; params->OnDone(grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, decrypted_response.status().ToString())); return; diff --git a/services/common/clients/bidding_server/bidding_async_client.cc b/services/common/clients/bidding_server/bidding_async_client.cc index e3227235..12a6771f 100644 --- a/services/common/clients/bidding_server/bidding_async_client.cc +++ b/services/common/clients/bidding_server/bidding_async_client.cc @@ -32,15 +32,15 @@ void OnRpcDone( std::function>(Response*)> decrypt_response) { if (!status.ok()) { - PS_VLOG(1) << "SendRPC completion status not ok: " - << server_common::ToAbslStatus(status); + PS_LOG(ERROR) << "SendRPC completion status not ok: " + << server_common::ToAbslStatus(status); params->OnDone(status); return; } PS_VLOG(6) << "SendRPC completion status ok"; auto decrypted_response = decrypt_response(params->ResponseRef()); if (!decrypted_response.ok()) { - PS_VLOG(1) << "BiddingAsyncGrpcClient failed to decrypt response"; + PS_LOG(ERROR) << "BiddingAsyncGrpcClient failed to decrypt response"; params->OnDone(grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, decrypted_response.status().ToString())); return; diff --git a/services/common/clients/buyer_frontend_server/buyer_frontend_async_client.cc b/services/common/clients/buyer_frontend_server/buyer_frontend_async_client.cc index ed7a3f79..68cdd84b 100644 --- a/services/common/clients/buyer_frontend_server/buyer_frontend_async_client.cc +++ b/services/common/clients/buyer_frontend_server/buyer_frontend_async_client.cc @@ -44,8 +44,8 @@ void BuyerFrontEndAsyncGrpcClient::SendRpc( params->ContextRef(), params->RequestRef(), params->ResponseRef(), [this, params, hpke_secret](const grpc::Status& status) { if (!status.ok()) { - PS_VLOG(1) << "SendRPC completion status not ok: " - << server_common::ToAbslStatus(status); + PS_LOG(ERROR) << "SendRPC completion status not ok: " + << server_common::ToAbslStatus(status); params->OnDone(status); return; } @@ -54,7 +54,7 @@ void BuyerFrontEndAsyncGrpcClient::SendRpc( auto decrypted_response = DecryptResponse(hpke_secret, params->ResponseRef()); if (!decrypted_response.ok()) { - PS_VLOG(1) + PS_LOG(ERROR) << "BuyerFrontEndAsyncGrpcClient Failed to decrypt response"; params->OnDone(grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, decrypted_response.status().ToString())); diff --git a/services/common/clients/code_dispatcher/v8_dispatcher.cc b/services/common/clients/code_dispatcher/v8_dispatcher.cc index 8c612c42..6ade2381 100644 --- a/services/common/clients/code_dispatcher/v8_dispatcher.cc +++ b/services/common/clients/code_dispatcher/v8_dispatcher.cc @@ -34,9 +34,9 @@ V8Dispatcher::V8Dispatcher(DispatchConfig&& config) : roma_service_(std::move(config)) {} V8Dispatcher::~V8Dispatcher() { - PS_VLOG(1) << "Stopping roma service..."; + PS_LOG(ERROR) << "Stopping roma service..."; absl::Status stop_status = roma_service_.Stop(); - PS_VLOG(1) << "Roma service stop status: " << stop_status; + PS_LOG(ERROR) << "Roma service stop status: " << stop_status; } absl::Status V8Dispatcher::Init() { return roma_service_.Init(); } diff --git a/services/common/clients/config/trusted_server_config_client.cc b/services/common/clients/config/trusted_server_config_client.cc index 1e9ae631..dbc4f38c 100644 --- a/services/common/clients/config/trusted_server_config_client.cc +++ b/services/common/clients/config/trusted_server_config_client.cc @@ -75,10 +75,11 @@ absl::Status TrustedServersConfigClient::Init( GetParameterRequest get_parameter_request; get_parameter_request.set_parameter_name( absl::StrCat(config_param_prefix, key)); + // Callbacks occur synchronously. // The GetParameter() call returns success given a valid request object // (e.g. a non-empty parameter name). - ExecutionResult result = config_client_->GetParameter( + absl::Status status = config_client_->GetParameter( std::move(get_parameter_request), [this, &key = key, &initial_value = value, &counter]( const ExecutionResult& result, @@ -95,11 +96,12 @@ absl::Status TrustedServersConfigClient::Init( } counter.DecrementCount(); }); + // Throw an error if key value not set before this and not found in // parameter store. - if (!result.Successful() && value == kEmptyValue) { - return HandleFailure(absl::StrFormat( - error_message, key, GetErrorMessage(result.status_code))); + if (!status.ok() && value == kEmptyValue) { + return HandleFailure( + absl::StrFormat(error_message, key, status.message())); } } @@ -131,20 +133,10 @@ int TrustedServersConfigClient::GetIntParameter( } absl::Status TrustedServersConfigClient::InitAndRunConfigClient() noexcept { - ExecutionResult result = config_client_->Init(); - if (!result.Successful()) { - return absl::UnavailableError( - absl::StrFormat("Cannot init config client (status_code: %s)\n", - GetErrorMessage(result.status_code))); - } - - result = config_client_->Run(); - if (!result.Successful()) { - return absl::UnavailableError( - absl::StrFormat("Cannot run config client (status_code: %s)\n", - GetErrorMessage(result.status_code))); - } - + PS_RETURN_IF_ERROR(config_client_->Init()).SetPrepend() + << "Cannot init config client: "; + PS_RETURN_IF_ERROR(config_client_->Run()).SetPrepend() + << "Cannot run config client"; return absl::OkStatus(); } diff --git a/services/common/clients/config/trusted_server_config_client_test.cc b/services/common/clients/config/trusted_server_config_client_test.cc index c516e07d..b7989be9 100644 --- a/services/common/clients/config/trusted_server_config_client_test.cc +++ b/services/common/clients/config/trusted_server_config_client_test.cc @@ -72,19 +72,16 @@ TEST(TrustedServerConfigClientTest, CanReadFlagsPassedThroughConstructor) { std::vector> f; TrustedServersConfigClient config_client( - kFlags, - [&f](const ParameterClientOptions& parameter_client_options) - -> std::unique_ptr { + kFlags, [&f](const ParameterClientOptions& parameter_client_options) { std::unique_ptr mock_config_client = std::make_unique(); EXPECT_CALL(*mock_config_client, Init) - .WillOnce(Return(SuccessExecutionResult())); + .WillOnce(Return(absl::OkStatus())); EXPECT_CALL(*mock_config_client, Run) - .WillOnce(Return(SuccessExecutionResult())); + .WillOnce(Return(absl::OkStatus())); EXPECT_CALL(*mock_config_client, GetParameter) .WillRepeatedly([&f](const GetParameterRequest& get_param_req, - Callback callback) - -> ExecutionResult { + Callback callback) { // async reading parameter like the real case. f.push_back(std::async(std::launch::async, [cb = std::move( callback)]() { @@ -94,9 +91,9 @@ TEST(TrustedServerConfigClientTest, CanReadFlagsPassedThroughConstructor) { google::scp::core::errors::SC_CPIO_RESOURCE_NOT_FOUND), response); })); - return SuccessExecutionResult(); + return absl::OkStatus(); }); - return std::move(mock_config_client); + return mock_config_client; }); config_client.SetFlag(FLAGS_config_param_1, "config_param_1"); config_client.SetFlag(FLAGS_config_param_2, "config_param_2"); @@ -131,21 +128,18 @@ TEST(TrustedServerConfigClientTest, FetchesConfigValueFromConfigClient) { std::vector> f; TrustedServersConfigClient config_client( - kFlags, - [&f, &expected_param_values]( - const ParameterClientOptions& parameter_client_options) - -> std::unique_ptr { + kFlags, [&f, &expected_param_values]( + const ParameterClientOptions& parameter_client_options) { std::unique_ptr mock_config_client = std::make_unique(); EXPECT_CALL(*mock_config_client, Init) - .WillOnce(Return(SuccessExecutionResult())); + .WillOnce(Return(absl::OkStatus())); EXPECT_CALL(*mock_config_client, Run) - .WillOnce(Return(SuccessExecutionResult())); + .WillOnce(Return(absl::OkStatus())); EXPECT_CALL(*mock_config_client, GetParameter) .WillRepeatedly([&f, &expected_param_values]( GetParameterRequest get_param_req, - Callback callback) - -> ExecutionResult { + Callback callback) { // async reading parameter like the real case f.push_back(std::async( std::launch::async, @@ -157,9 +151,9 @@ TEST(TrustedServerConfigClientTest, FetchesConfigValueFromConfigClient) { expected_param_values.at(req.parameter_name())); cb(SuccessExecutionResult(), response); })); - return SuccessExecutionResult(); + return absl::OkStatus(); }); - return std::move(mock_config_client); + return mock_config_client; }); ASSERT_TRUE(config_client.Init("").ok()); for (auto& each : f) { @@ -186,21 +180,18 @@ TEST(TrustedServerConfigClientTest, OverwritesConfigValueFromCloud) { }; std::vector> f; TrustedServersConfigClient config_client( - {key}, - [&f, &expected_param_values]( - const ParameterClientOptions& parameter_client_options) - -> std::unique_ptr { + {key}, [&f, &expected_param_values]( + const ParameterClientOptions& parameter_client_options) { std::unique_ptr mock_config_client = std::make_unique(); EXPECT_CALL(*mock_config_client, Init) - .WillOnce(Return(SuccessExecutionResult())); + .WillOnce(Return(absl::OkStatus())); EXPECT_CALL(*mock_config_client, Run) - .WillOnce(Return(SuccessExecutionResult())); + .WillOnce(Return(absl::OkStatus())); EXPECT_CALL(*mock_config_client, GetParameter) .WillRepeatedly([&f, &expected_param_values]( GetParameterRequest get_param_req, - Callback callback) - -> ExecutionResult { + Callback callback) { // async reading parameter like the real case f.push_back(std::async( std::launch::async, @@ -212,9 +203,9 @@ TEST(TrustedServerConfigClientTest, OverwritesConfigValueFromCloud) { expected_param_values.at(req.parameter_name())); cb(SuccessExecutionResult(), response); })); - return SuccessExecutionResult(); + return absl::OkStatus(); }); - return std::move(mock_config_client); + return mock_config_client; }); config_client.SetFlag(FLAGS_config_param_5, "config_param_5"); ASSERT_TRUE(config_client.Init("").ok()); @@ -231,16 +222,15 @@ TEST(TrustedServerConfigClientTest, ThrowsUnavailableErrorOnClientInitFail) { TrustedServersConfigClient config_client( {"config_param_1"}, - [](const ParameterClientOptions& parameter_client_options) - -> std::unique_ptr { + [](const ParameterClientOptions& parameter_client_options) { std::unique_ptr mock_config_client = std::make_unique(); EXPECT_CALL(*mock_config_client, Init()) - .WillOnce(Return(SuccessExecutionResult())); + .WillOnce(Return(absl::OkStatus())); EXPECT_CALL(*mock_config_client, Run()) - .WillOnce(Return(FailureExecutionResult(0))); + .WillOnce(Return(absl::UnknownError(""))); - return std::move(mock_config_client); + return mock_config_client; }); absl::Status init_result = config_client.Init(""); @@ -250,18 +240,16 @@ TEST(TrustedServerConfigClientTest, ThrowsUnavailableErrorOnClientInitFail) { TEST(TrustedServerConfigClientTest, PrependsFlagNamesWithTag) { TrustedServersConfigClient config_client( {"config_param_1"}, - [](const ParameterClientOptions& parameter_client_options) - -> std::unique_ptr { + [](const ParameterClientOptions& parameter_client_options) { std::unique_ptr mock_config_client = std::make_unique(); EXPECT_CALL(*mock_config_client, Init()) - .WillOnce(Return(SuccessExecutionResult())); + .WillOnce(Return(absl::OkStatus())); EXPECT_CALL(*mock_config_client, Run()) - .WillOnce(Return(SuccessExecutionResult())); + .WillOnce(Return(absl::OkStatus())); EXPECT_CALL(*mock_config_client, GetParameter) .WillOnce([](const GetParameterRequest& get_param_req, - const Callback& callback) - -> ExecutionResult { + const Callback& callback) { // Verify we query for fetched config values with the prefix. EXPECT_EQ(get_param_req.parameter_name(), "MyConfigParamPrefix-config_param_1"); @@ -269,9 +257,9 @@ TEST(TrustedServerConfigClientTest, PrependsFlagNamesWithTag) { GetParameterResponse response; response.set_parameter_value("config_value_1"); callback(SuccessExecutionResult(), response); - return SuccessExecutionResult(); + return absl::OkStatus(); }); - return std::move(mock_config_client); + return mock_config_client; }); ASSERT_TRUE(config_client.Init("MyConfigParamPrefix-").ok()); diff --git a/services/common/clients/config/trusted_server_config_client_util.cc b/services/common/clients/config/trusted_server_config_client_util.cc index 5d4159b9..b7fd9565 100644 --- a/services/common/clients/config/trusted_server_config_client_util.cc +++ b/services/common/clients/config/trusted_server_config_client_util.cc @@ -42,7 +42,6 @@ using google::cmrt::sdk::instance_service::v1::GetTagsByResourceNameResponse; using ::google::scp::core::ExecutionResult; using ::google::scp::core::errors::GetErrorMessage; using ::google::scp::cpio::InstanceClientInterface; -using ::google::scp::cpio::InstanceClientOptions; namespace { @@ -66,9 +65,10 @@ absl::StatusOr GetResourceName( std::string resource_name; absl::Notification done; - const auto& result = client->GetCurrentInstanceResourceName( + absl::Status status = client->GetCurrentInstanceResourceName( GetCurrentInstanceResourceNameRequest(), - [&](const ExecutionResult& result, + [&resource_name, &done]( + const ExecutionResult& result, const GetCurrentInstanceResourceNameResponse& response) { if (result.Successful()) { resource_name = std::string{response.instance_resource_name()}; @@ -80,9 +80,9 @@ absl::StatusOr GetResourceName( done.Notify(); }); - if (!result.Successful()) { - return HandleFailure(absl::StrFormat(kResourceNameFetchError, - GetErrorMessage(result.status_code))); + if (!status.ok()) { + return HandleFailure( + absl::StrFormat(kResourceNameFetchError, status.message())); } done.WaitForNotification(); @@ -103,8 +103,8 @@ TrustedServerConfigUtil::TrustedServerConfigUtil(bool init_config_client) } std::shared_ptr client = - google::scp::cpio::InstanceClientFactory::Create(InstanceClientOptions()); - client->Init(); + google::scp::cpio::InstanceClientFactory::Create(); + client->Init().IgnoreError(); absl::StatusOr resource_name = GetResourceName(client); CHECK_OK(resource_name) << "Could not fetch host resource name."; ComputeZone(resource_name.value()); @@ -112,10 +112,10 @@ TrustedServerConfigUtil::TrustedServerConfigUtil(bool init_config_client) GetInstanceDetailsByResourceNameRequest request; request.set_instance_resource_name(resource_name.value()); - const auto result = client->GetInstanceDetailsByResourceName( + absl::Status status = client->GetInstanceDetailsByResourceName( std::move(request), - [&](const ExecutionResult& result, - const GetInstanceDetailsByResourceNameResponse& response) { + [this, &done](const ExecutionResult& result, + const GetInstanceDetailsByResourceNameResponse& response) { if (result.Successful()) { ABSL_LOG(INFO) << response.DebugString(); instance_id_ = std::string{response.instance_details().instance_id()}; @@ -129,9 +129,9 @@ TrustedServerConfigUtil::TrustedServerConfigUtil(bool init_config_client) } done.Notify(); }); - if (!result.Successful()) { + if (!status.ok()) { ABSL_LOG(ERROR) << absl::StrFormat(kResourceTagFetchError, - GetErrorMessage(result.status_code)); + status.message()); } else { done.WaitForNotification(); } diff --git a/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client.cc b/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client.cc index 91fe49ba..587fbeec 100644 --- a/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client.cc +++ b/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client.cc @@ -111,21 +111,22 @@ BuyerKeyValueAsyncHttpClient::BuyerKeyValueAsyncHttpClient( kv_server_base_address_(kv_server_base_address) { if (pre_warm) { auto request = std::make_unique(); - auto status = Execute( - std::move(request), {}, - [](absl::StatusOr> - buyer_kv_output) mutable { - if (!buyer_kv_output.ok()) { - PS_VLOG(1) - << "BuyerKeyValueAsyncHttpClient pre-warm returned status:" - << buyer_kv_output.status().message(); - } - }, - // Longer timeout for first request - absl::Milliseconds(60000)); + auto status = + Execute( + std::move(request), {}, + [](absl::StatusOr> + buyer_kv_output) mutable { + if (!buyer_kv_output.ok()) { + PS_LOG(ERROR) + << "BuyerKeyValueAsyncHttpClient pre-warm returned status:" + << buyer_kv_output.status().message(); + } + }, + // Longer timeout for first request + absl::Milliseconds(60000)); if (!status.ok()) { - PS_VLOG(1) << "BuyerKeyValueAsyncHttpClient pre-warming failed: " - << status; + PS_LOG(ERROR) << "BuyerKeyValueAsyncHttpClient pre-warming failed: " + << status; } } } diff --git a/services/common/clients/http_kv_server/seller/seller_key_value_async_http_client.cc b/services/common/clients/http_kv_server/seller/seller_key_value_async_http_client.cc index ba3560d4..57ccfe2d 100644 --- a/services/common/clients/http_kv_server/seller/seller_key_value_async_http_client.cc +++ b/services/common/clients/http_kv_server/seller/seller_key_value_async_http_client.cc @@ -102,21 +102,22 @@ SellerKeyValueAsyncHttpClient::SellerKeyValueAsyncHttpClient( kv_server_base_address_(kv_server_base_address) { if (pre_warm) { auto request = std::make_unique(); - auto status = Execute( - std::move(request), {}, - [](absl::StatusOr> - seller_kv_output) mutable { - if (!seller_kv_output.ok()) { - PS_VLOG(1) - << "SellerKeyValueAsyncHttpClient pre-warm returned status:" - << seller_kv_output.status().message(); - } - }, - // Longer timeout for first request - absl::Milliseconds(60000)); + auto status = + Execute( + std::move(request), {}, + [](absl::StatusOr> + seller_kv_output) mutable { + if (!seller_kv_output.ok()) { + PS_LOG(ERROR) + << "SellerKeyValueAsyncHttpClient pre-warm returned status:" + << seller_kv_output.status().message(); + } + }, + // Longer timeout for first request + absl::Milliseconds(60000)); if (!status.ok()) { - PS_VLOG(1) << "SellerKeyValueAsyncHttpClient pre-warming failed:" - << status; + PS_LOG(ERROR) << "SellerKeyValueAsyncHttpClient pre-warming failed:" + << status; } } } diff --git a/services/common/clients/kv_server/kv_async_client.cc b/services/common/clients/kv_server/kv_async_client.cc index cb0e9d3d..7f1a1d28 100644 --- a/services/common/clients/kv_server/kv_async_client.cc +++ b/services/common/clients/kv_server/kv_async_client.cc @@ -53,8 +53,8 @@ void KVAsyncGrpcClient::SendRpc( auto oblivious_http_request_uptr = ObliviousHttpRequestUptr(captured_oblivious_http_context); if (!status.ok()) { - PS_VLOG(1) << "SendRPC completion status not ok: " - << server_common::ToAbslStatus(status); + PS_LOG(ERROR) << "SendRPC completion status not ok: " + << server_common::ToAbslStatus(status); params->OnDone(status); return; } @@ -64,7 +64,8 @@ void KVAsyncGrpcClient::SendRpc( FromObliviousHTTPResponse(*params->ResponseRef()->mutable_data(), *captured_oblivious_http_context); if (!plain_text_binary_http_response.ok()) { - PS_VLOG(1) << "KVAsyncGrpcClient failed to get binary HTTP response"; + PS_LOG(ERROR) + << "KVAsyncGrpcClient failed to get binary HTTP response"; params->OnDone(grpc::Status( grpc::StatusCode::INVALID_ARGUMENT, plain_text_binary_http_response.status().ToString())); @@ -72,10 +73,10 @@ void KVAsyncGrpcClient::SendRpc( } auto response = FromBinaryHTTP( - *plain_text_binary_http_response, /*from_json=*/true); + *plain_text_binary_http_response, /*from_json=*/false); PS_VLOG(7) << "Retrieved proto response: " << response->DebugString(); if (!response->has_single_partition()) { - PS_VLOG(1) + PS_LOG(ERROR) << "KVAsyncGrpcClient expected a single partition response, got: " << response->DebugString(); params->OnDone(grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, @@ -99,7 +100,7 @@ absl::Status KVAsyncGrpcClient::ExecuteInternal( absl::Duration timeout) const { PS_VLOG(6) << "Raw request:\n" << raw_request->DebugString(); PS_ASSIGN_OR_RETURN(std::string binary_http_msg, - ToBinaryHTTP(*raw_request, /*to_json=*/true)); + ToBinaryHTTP(*raw_request, /*to_json=*/false)); PS_VLOG(5) << "Fetching public Key ..."; PS_ASSIGN_OR_RETURN(auto public_key, diff --git a/services/common/code_dispatch/code_dispatch_reactor.h b/services/common/code_dispatch/code_dispatch_reactor.h index 5653c253..06a60ffe 100644 --- a/services/common/code_dispatch/code_dispatch_reactor.h +++ b/services/common/code_dispatch/code_dispatch_reactor.h @@ -55,7 +55,7 @@ class CodeDispatchReactor : public grpc::ServerUnaryReactor { if (DecryptRequest()) { PS_VLOG(3) << "Decrypted request: " << raw_request_.DebugString(); } else { - PS_VLOG(1) << "Failed to decrypt the request"; + PS_LOG(ERROR) << "Failed to decrypt the request"; } } @@ -82,12 +82,12 @@ class CodeDispatchReactor : public grpc::ServerUnaryReactor { // successful. If successful, the result is written into 'raw_request_'. bool DecryptRequest() { if (request_->key_id().empty()) { - PS_VLOG(1) << "No key ID found in the request"; + PS_LOG(ERROR) << "No key ID found in the request"; Finish( grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, kEmptyKeyIdError)); return false; } else if (request_->request_ciphertext().empty()) { - PS_VLOG(1) << "No ciphertext found in the request"; + PS_LOG(ERROR) << "No ciphertext found in the request"; Finish(grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, kEmptyCiphertextError)); return false; @@ -96,7 +96,8 @@ class CodeDispatchReactor : public grpc::ServerUnaryReactor { std::optional private_key = key_fetcher_manager_->GetPrivateKey(request_->key_id()); if (!private_key.has_value()) { - PS_VLOG(1) << "Unable to fetch private key from the key fetcher manager"; + PS_LOG(ERROR) + << "Unable to fetch private key from the key fetcher manager"; Finish( grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, kInvalidKeyIdError)); return false; @@ -106,8 +107,8 @@ class CodeDispatchReactor : public grpc::ServerUnaryReactor { decrypt_response = crypto_client_->HpkeDecrypt( *private_key, request_->request_ciphertext()); if (!decrypt_response.ok()) { - PS_VLOG(1) << "Unable to decrypt the request ciphertext: " - << decrypt_response.status(); + PS_LOG(ERROR) << "Unable to decrypt the request ciphertext: " + << decrypt_response.status(); Finish(grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, kMalformedCiphertext)); return false; @@ -124,7 +125,7 @@ class CodeDispatchReactor : public grpc::ServerUnaryReactor { absl::StatusOr aead_encrypt = crypto_client_->AeadEncrypt(payload, hpke_secret_); if (!aead_encrypt.ok()) { - PS_VLOG(1) << "AEAD encrypt failed: " << aead_encrypt.status(); + PS_LOG(ERROR) << "AEAD encrypt failed: " << aead_encrypt.status(); Finish(grpc::Status(grpc::StatusCode::INTERNAL, aead_encrypt.status().ToString())); return false; diff --git a/services/common/code_fetch/BUILD b/services/common/code_fetch/BUILD index a5bacc1a..40ea11c8 100644 --- a/services/common/code_fetch/BUILD +++ b/services/common/code_fetch/BUILD @@ -33,6 +33,7 @@ cc_library( ":code_fetcher_interface", "//services/common/clients/code_dispatcher:v8_dispatcher", "//services/common/clients/http:http_fetcher_async", + "//services/common/util:request_response_constants", "@com_google_absl//absl/functional:any_invocable", "@com_google_absl//absl/log:check", "@com_google_absl//absl/status", @@ -51,6 +52,7 @@ cc_library( deps = [ ":code_fetcher_interface", "//services/common/clients/code_dispatcher:v8_dispatcher", + "//services/common/util:request_response_constants", "@com_google_absl//absl/functional:any_invocable", "@com_google_absl//absl/log:check", "@com_google_absl//absl/status", diff --git a/services/common/code_fetch/periodic_bucket_fetcher.cc b/services/common/code_fetch/periodic_bucket_fetcher.cc index 21fe1743..4d54dea4 100644 --- a/services/common/code_fetch/periodic_bucket_fetcher.cc +++ b/services/common/code_fetch/periodic_bucket_fetcher.cc @@ -29,6 +29,7 @@ #include "absl/synchronization/blocking_counter.h" #include "absl/time/time.h" #include "services/common/code_fetch/code_fetcher_interface.h" +#include "services/common/util/request_response_constants.h" #include "src/core/interface/async_context.h" #include "src/core/interface/errors.h" #include "src/logger/request_context_logger.h" @@ -103,12 +104,12 @@ PeriodicBucketFetcher::ListBlobsSync() { notification.Notify(); }); - auto list_blobs_execution = - blob_storage_client_.ListBlobsMetadata(list_blobs_context); - if (!list_blobs_execution.Successful()) { - result = absl::InternalError( - absl::StrCat("ListBlobsMetadata attempt failed: ", - GetErrorMessage(list_blobs_execution.status_code))); + if (const absl::Status status = + blob_storage_client_.ListBlobsMetadata(list_blobs_context); + !status.ok()) { + result = absl::Status( + status.code(), + absl::StrCat("ListBlobsMetadata attempt failed: ", status.message())); PS_VLOG(5) << "List blob metadata failed."; } else { PS_VLOG(5) << "Waiting for list blob metadata done notification."; @@ -122,19 +123,20 @@ void PeriodicBucketFetcher::HandleBlobFetchResult( const AsyncContext& context) { absl::string_view version = context.request->blob_metadata().blob_name(); if (!context.result.Successful()) { - PS_VLOG(0) << "Failed to fetch blob: " << version - << GetErrorMessage(context.result.status_code); + PS_LOG(ERROR) << "Failed to fetch blob: " << version + << GetErrorMessage(context.result.status_code); return; } auto result_value = {context.response->blob().data()}; std::string wrapped_code = wrap_code_(result_value); absl::Status roma_result = dispatcher_.LoadSync(version, wrapped_code); if (!roma_result.ok()) { - PS_VLOG(0) << "Roma failed to load blob: " << roma_result; + PS_LOG(ERROR) << "Roma failed to load blob: " << roma_result; return; } - PS_VLOG(1) << "Current code loaded into Roma for version " << version << ":\n" - << wrapped_code; + PS_VLOG(kInfoMsg) << "Current code loaded into Roma for version " << version + << ":\n" + << wrapped_code; absl::MutexLock lock(&some_load_success_mu_); some_load_success_ = true; } @@ -142,9 +144,9 @@ void PeriodicBucketFetcher::HandleBlobFetchResult( void PeriodicBucketFetcher::PeriodicBucketFetchSync() { absl::StatusOr blob_list = ListBlobsSync(); if (!blob_list.ok()) { - PS_VLOG(0) << "Periodic bucket fetch failed for bucket " << bucket_name_ - << ". Will try again in " << fetch_period_ms_ - << " milliseconds."; + PS_LOG(ERROR) << "Periodic bucket fetch failed for bucket " << bucket_name_ + << ". Will try again in " << fetch_period_ms_ + << " milliseconds."; task_id_ = executor_.RunAfter(fetch_period_ms_, [this]() { PeriodicBucketFetchSync(); }); return; @@ -166,12 +168,11 @@ void PeriodicBucketFetcher::PeriodicBucketFetchSync() { blobs_remaining.DecrementCount(); }); - auto get_blob_result = blob_storage_client_.GetBlob(get_blob_context); - - if (!get_blob_result.Successful()) { + if (absl::Status status = blob_storage_client_.GetBlob(get_blob_context); + !status.ok()) { blobs_remaining.DecrementCount(); - PS_VLOG(0) << "GetBlob attempt failed for " << md.blob_name() - << GetErrorMessage(get_blob_result.status_code); + PS_LOG(ERROR) << "GetBlob attempt failed for " << md.blob_name() + << status.message(); } } diff --git a/services/common/code_fetch/periodic_bucket_fetcher_test.cc b/services/common/code_fetch/periodic_bucket_fetcher_test.cc index b8477675..f2a35e36 100644 --- a/services/common/code_fetch/periodic_bucket_fetcher_test.cc +++ b/services/common/code_fetch/periodic_bucket_fetcher_test.cc @@ -53,7 +53,6 @@ using ::google::cmrt::sdk::blob_storage_service::v1::GetBlobResponse; using ::google::cmrt::sdk::blob_storage_service::v1::ListBlobsMetadataRequest; using ::google::cmrt::sdk::blob_storage_service::v1::ListBlobsMetadataResponse; using ::google::scp::core::AsyncContext; -using ::google::scp::core::ExecutionResult; using ::google::scp::core::FailureExecutionResult; using ::google::scp::core::SuccessExecutionResult; using ::google::scp::core::errors::GetErrorMessage; @@ -89,7 +88,7 @@ TEST(PeriodicBucketFetcherTest, LoadsWrappedResultIntoV8Dispatcher) { std::move(md)); async_context.result = SuccessExecutionResult(); async_context.Finish(); - return SuccessExecutionResult(); + return absl::OkStatus(); }); EXPECT_CALL(*executor, RunAfter) @@ -115,7 +114,7 @@ TEST(PeriodicBucketFetcherTest, LoadsWrappedResultIntoV8Dispatcher) { async_context.result = SuccessExecutionResult(); async_context.Finish(); - return SuccessExecutionResult(); + return absl::OkStatus(); }); EXPECT_CALL(dispatcher, LoadSync) @@ -164,7 +163,7 @@ TEST(PeriodicBucketFetcherTest, LoadsAllBlobsInBucket) { std::move(md2)); async_context.result = SuccessExecutionResult(); async_context.Finish(); - return SuccessExecutionResult(); + return absl::OkStatus(); }); EXPECT_CALL(*blob_storage_client, GetBlob) @@ -185,7 +184,7 @@ TEST(PeriodicBucketFetcherTest, LoadsAllBlobsInBucket) { async_context.Finish(); done_get_blob.DecrementCount(); - return SuccessExecutionResult(); + return absl::OkStatus(); }); EXPECT_CALL(*executor, RunAfter) @@ -245,13 +244,12 @@ TEST(PeriodicBucketFetcherTest, ReturnsSuccessIfAtLeastOneBlobLoads) { async_context.response->mutable_blob_metadatas()->Add(std::move(md3)); async_context.result = SuccessExecutionResult(); async_context.Finish(); - return SuccessExecutionResult(); + return absl::OkStatus(); }); EXPECT_CALL(*blob_storage_client, GetBlob) .WillRepeatedly( - [](AsyncContext async_context) - -> ExecutionResult { + [](AsyncContext async_context) { async_context.response = std::make_shared(); if (async_context.request->blob_metadata().blob_name() == kSampleBlobName) { @@ -262,11 +260,11 @@ TEST(PeriodicBucketFetcherTest, ReturnsSuccessIfAtLeastOneBlobLoads) { std::string(kSampleData2)); async_context.result = SuccessExecutionResult(); } else { - return FailureExecutionResult(SC_UNKNOWN); + return absl::UnknownError(""); } async_context.Finish(); - return SuccessExecutionResult(); + return absl::OkStatus(); }); EXPECT_CALL(*executor, RunAfter) @@ -318,7 +316,7 @@ TEST(PeriodicBucketFetcherTest, FailsStartupIfNoBlobLoadedSuccessfully) { std::move(md)); async_context.result = SuccessExecutionResult(); async_context.Finish(); - return SuccessExecutionResult(); + return absl::OkStatus(); }); EXPECT_CALL(*executor, RunAfter) @@ -344,7 +342,7 @@ TEST(PeriodicBucketFetcherTest, FailsStartupIfNoBlobLoadedSuccessfully) { async_context.result = SuccessExecutionResult(); async_context.Finish(); - return SuccessExecutionResult(); + return absl::OkStatus(); }); EXPECT_CALL(dispatcher, LoadSync) diff --git a/services/common/code_fetch/periodic_code_fetcher.cc b/services/common/code_fetch/periodic_code_fetcher.cc index b7afc479..35ad77fd 100644 --- a/services/common/code_fetch/periodic_code_fetcher.cc +++ b/services/common/code_fetch/periodic_code_fetcher.cc @@ -24,6 +24,7 @@ #include "absl/log/check.h" #include "absl/status/statusor.h" #include "absl/time/time.h" +#include "services/common/util/request_response_constants.h" #include "src/logger/request_context_logger.h" namespace privacy_sandbox::bidding_auction_servers { @@ -76,13 +77,13 @@ void PeriodicCodeFetcher::PeriodicCodeFetchSync() { for (const auto& result : results) { if (!result.ok()) { - PS_VLOG(1) << "MultiCurlHttpFetcher Failure Response: " - << result.status(); + PS_LOG(ERROR) << "MultiCurlHttpFetcher Failure Response: " + << result.status(); all_status_ok = false; break; } else { - PS_VLOG(1) << "MultiCurlHttpFetcher Success Response: " - << result.status(); + PS_VLOG(kSuccess) + << "MultiCurlHttpFetcher Success Response: " << result.status(); results_value.push_back(*result); } } @@ -95,7 +96,7 @@ void PeriodicCodeFetcher::PeriodicCodeFetchSync() { std::string wrapped_code = wrap_code_(cb_results_value_); absl::Status syncResult = dispatcher_.LoadSync(version_string_, wrapped_code); - PS_VLOG(1) << "Roma Client Response: " << syncResult; + PS_VLOG(kInfoMsg) << "Roma Client Response: " << syncResult; if (syncResult.ok()) { PS_VLOG(2) << "Current code loaded into Roma:\n" << wrapped_code; absl::MutexLock lock(&some_load_success_mu_); @@ -109,10 +110,9 @@ void PeriodicCodeFetcher::PeriodicCodeFetchSync() { // Create a HTTPRequest object from the url_endpoint_ std::vector requests; for (const std::string& endpoint : url_endpoints_) { - HTTPRequest request; - PS_VLOG(1) << "Requesting UDF from: " << endpoint; - request.url = endpoint; - requests.push_back(request); + PS_VLOG(kInfoMsg) << "Requesting UDF from: " << endpoint; + requests.push_back( + {.url = endpoint, .headers = {"Cache-Control: no-cache"}}); } curl_http_fetcher_.FetchUrls(requests, time_out_ms_, diff --git a/services/common/encryption/crypto_client_factory.cc b/services/common/encryption/crypto_client_factory.cc index 14ace0c1..17221170 100644 --- a/services/common/encryption/crypto_client_factory.cc +++ b/services/common/encryption/crypto_client_factory.cc @@ -38,8 +38,8 @@ std::unique_ptr CreateCryptoClient() { std::unique_ptr cpio_crypto_client = google::scp::cpio::CryptoClientFactory::Create(options); - cpio_crypto_client->Init(); - cpio_crypto_client->Run(); + cpio_crypto_client->Init().IgnoreError(); + cpio_crypto_client->Run().IgnoreError(); return std::make_unique(std::move(cpio_crypto_client)); } diff --git a/services/common/encryption/crypto_client_wrapper.cc b/services/common/encryption/crypto_client_wrapper.cc index 8446e953..9116f872 100644 --- a/services/common/encryption/crypto_client_wrapper.cc +++ b/services/common/encryption/crypto_client_wrapper.cc @@ -52,17 +52,14 @@ using ::google::cmrt::sdk::public_key_service::v1::PublicKey; namespace { -absl::Status HandleCryptoOperationResult(const ExecutionResult& result, +absl::Status HandleCryptoOperationResult(const absl::Status& status, bool operation_successful, const std::string& operation) { - if (!result.Successful() || !operation_successful) { - // Only log the execution result error; don't log the callback failure - // twice. - if (!result.Successful()) { - const std::string error = - absl::StrFormat(kCryptoOperationFailureError, operation.data(), - GetErrorMessage(result.status_code)); - ABSL_LOG(ERROR) << error; + if (!status.ok() || !operation_successful) { + // Only log the status; don't log the callback failure twice. + if (!status.ok()) { + ABSL_LOG(ERROR) << absl::StrFormat(kCryptoOperationFailureError, + operation.data(), status.message()); } return absl::Status(absl::StatusCode::kInternal, @@ -77,11 +74,13 @@ absl::Status HandleCryptoOperationResult(const ExecutionResult& result, CryptoClientWrapper::CryptoClientWrapper( std::unique_ptr crypto_client) : crypto_client_(std::move(crypto_client)) { - crypto_client_->Init(); - crypto_client_->Run(); + crypto_client_->Init().IgnoreError(); + crypto_client_->Run().IgnoreError(); } -CryptoClientWrapper::~CryptoClientWrapper() { crypto_client_->Stop(); } +CryptoClientWrapper::~CryptoClientWrapper() { + crypto_client_->Stop().IgnoreError(); +} absl::StatusOr CryptoClientWrapper::HpkeEncrypt( const PublicKey& key, const std::string& plaintext_payload) noexcept { @@ -100,7 +99,7 @@ absl::StatusOr CryptoClientWrapper::HpkeEncrypt( HpkeEncryptResponse response; bool success = false; // HpkeEncrypt() callback is executed synchronously. - ExecutionResult execution_result = crypto_client_->HpkeEncrypt( + absl::Status status = crypto_client_->HpkeEncrypt( std::move(request), [&response, &success]( const google::scp::core::ExecutionResult& result, @@ -117,7 +116,7 @@ absl::StatusOr CryptoClientWrapper::HpkeEncrypt( }); PS_RETURN_IF_ERROR( - HandleCryptoOperationResult(execution_result, success, kHpkeEncrypt)); + HandleCryptoOperationResult(status, success, kHpkeEncrypt)); return response; } @@ -131,7 +130,7 @@ absl::StatusOr CryptoClientWrapper::AeadEncrypt( AeadEncryptResponse response; bool success = false; // AeadEncrypt() callback is executed synchronously. - ExecutionResult execution_result = crypto_client_->AeadEncrypt( + absl::Status status = crypto_client_->AeadEncrypt( std::move(request), [&response, &success](const google::scp::core::ExecutionResult& result, AeadEncryptResponse encrypt_response) { @@ -146,7 +145,7 @@ absl::StatusOr CryptoClientWrapper::AeadEncrypt( }); PS_RETURN_IF_ERROR( - HandleCryptoOperationResult(execution_result, success, kAeadEncrypt)); + HandleCryptoOperationResult(status, success, kAeadEncrypt)); return response; } @@ -184,7 +183,7 @@ absl::StatusOr CryptoClientWrapper::HpkeDecrypt( HpkeDecryptResponse response; bool success = false; // HpkeDecrypt() callback is executed synchronously. - ExecutionResult execution_result = crypto_client_->HpkeDecrypt( + absl::Status status = crypto_client_->HpkeDecrypt( std::move(request), [&response, &success]( const google::scp::core::ExecutionResult& result, @@ -201,7 +200,7 @@ absl::StatusOr CryptoClientWrapper::HpkeDecrypt( }); PS_RETURN_IF_ERROR( - HandleCryptoOperationResult(execution_result, success, kHpkeDecrypt)); + HandleCryptoOperationResult(status, success, kHpkeDecrypt)); return response; } @@ -215,7 +214,7 @@ absl::StatusOr CryptoClientWrapper::AeadDecrypt( AeadDecryptResponse response; bool success = false; // AeadDecrypt() callback is executed synchronously. - ExecutionResult execution_result = crypto_client_->AeadDecrypt( + absl::Status status = crypto_client_->AeadDecrypt( std::move(request), [&response, &success](const google::scp::core::ExecutionResult& result, AeadDecryptResponse decrypt_response) { @@ -230,7 +229,7 @@ absl::StatusOr CryptoClientWrapper::AeadDecrypt( }); PS_RETURN_IF_ERROR( - HandleCryptoOperationResult(execution_result, success, kAeadDecrypt)); + HandleCryptoOperationResult(status, success, kAeadDecrypt)); return response; } diff --git a/services/common/encryption/crypto_client_wrapper.h b/services/common/encryption/crypto_client_wrapper.h index c5e0e77e..c5c88ddd 100644 --- a/services/common/encryption/crypto_client_wrapper.h +++ b/services/common/encryption/crypto_client_wrapper.h @@ -48,29 +48,29 @@ class CryptoClientWrapper : public CryptoClientWrapperInterface { CryptoClientWrapper( std::unique_ptr crypto_client); - virtual ~CryptoClientWrapper(); + ~CryptoClientWrapper() override; // Encrypts a plaintext payload using HPKE. absl::StatusOr HpkeEncrypt(const google::cmrt::sdk::public_key_service::v1::PublicKey& key, - const std::string& plaintext_payload) noexcept; + const std::string& plaintext_payload) noexcept override; // Decrypts a ciphertext using HPKE. absl::StatusOr HpkeDecrypt(const server_common::PrivateKey& private_key, - const std::string& ciphertext) noexcept; + const std::string& ciphertext) noexcept override; // Encrypts plaintext payload using AEAD and a secret derived from the HPKE // decrypt operation. absl::StatusOr AeadEncrypt(const std::string& plaintext_payload, - const std::string& secret) noexcept; + const std::string& secret) noexcept override; // Decrypts a ciphertext using AEAD and a secret derived from the HPKE // encrypt operation. absl::StatusOr AeadDecrypt(const std::string& ciphertext, - const std::string& secret) noexcept; + const std::string& secret) noexcept override; private: std::unique_ptr crypto_client_; diff --git a/services/common/encryption/crypto_client_wrapper_test.cc b/services/common/encryption/crypto_client_wrapper_test.cc index fade6cdb..283b22a0 100644 --- a/services/common/encryption/crypto_client_wrapper_test.cc +++ b/services/common/encryption/crypto_client_wrapper_test.cc @@ -44,7 +44,6 @@ using ::google::cmrt::sdk::crypto_service::v1::HpkeEncryptResponse; using ::google::scp::cpio::Callback; using ::google::cmrt::sdk::public_key_service::v1::PublicKey; -using ::google::scp::core::ExecutionResult; using ::google::scp::core::FailureExecutionResult; using ::google::scp::core::SuccessExecutionResult; @@ -55,32 +54,32 @@ inline constexpr char kCiphertext[] = "ciphertext"; class MockCryptoClientProvider : public google::scp::cpio::CryptoClientInterface { public: - ExecutionResult init_result_mock = SuccessExecutionResult(); - ExecutionResult Init() noexcept override { return init_result_mock; } + absl::Status init_result_mock = absl::OkStatus(); + absl::Status Init() noexcept override { return init_result_mock; } - ExecutionResult run_result_mock = SuccessExecutionResult(); - ExecutionResult Run() noexcept override { return run_result_mock; } + absl::Status run_result_mock = absl::OkStatus(); + absl::Status Run() noexcept override { return run_result_mock; } - ExecutionResult stop_result_mock = SuccessExecutionResult(); - ExecutionResult Stop() noexcept override { return stop_result_mock; } + absl::Status stop_result_mock = absl::OkStatus(); + absl::Status Stop() noexcept override { return stop_result_mock; } // NOLINTNEXTLINE - MOCK_METHOD(ExecutionResult, HpkeEncrypt, + MOCK_METHOD(absl::Status, HpkeEncrypt, (HpkeEncryptRequest request, Callback callback), (override, noexcept)); // NOLINTNEXTLINE - MOCK_METHOD(ExecutionResult, HpkeDecrypt, + MOCK_METHOD(absl::Status, HpkeDecrypt, (HpkeDecryptRequest request, Callback callback), (override, noexcept)); // NOLINTNEXTLINE - MOCK_METHOD(ExecutionResult, AeadEncrypt, + MOCK_METHOD(absl::Status, AeadEncrypt, (AeadEncryptRequest request, Callback callback), (override, noexcept)); // NOLINTNEXTLINE - MOCK_METHOD(ExecutionResult, AeadDecrypt, + MOCK_METHOD(absl::Status, AeadDecrypt, (AeadDecryptRequest request, Callback callback), (override, noexcept)); @@ -111,13 +110,13 @@ TEST(CryptoClientWrapperTest, HpkeEncrypt_Success) { std::unique_ptr mock_crypto_client = std::make_unique(); EXPECT_CALL(*mock_crypto_client, HpkeEncrypt) - .WillOnce([&](const HpkeEncryptRequest& request, - const Callback& callback) - -> ExecutionResult { + .WillOnce([&expected_request, &mock_response]( + const HpkeEncryptRequest& request, + const Callback& callback) { EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals( request, expected_request)); callback(SuccessExecutionResult(), mock_response); - return SuccessExecutionResult(); + return absl::OkStatus(); }); CryptoClientWrapper crypto_client(std::move(mock_crypto_client)); @@ -132,11 +131,10 @@ TEST(CryptoClientWrapperTest, HpkeEncrypt_Failure) { std::unique_ptr mock_crypto_client = std::make_unique(); EXPECT_CALL(*mock_crypto_client, HpkeEncrypt) - .WillOnce([&](const HpkeEncryptRequest& request, - const Callback& callback) - -> ExecutionResult { + .WillOnce([](const HpkeEncryptRequest& request, + const Callback& callback) { callback(FailureExecutionResult(0), HpkeEncryptResponse()); - return SuccessExecutionResult(); + return absl::OkStatus(); }); CryptoClientWrapper crypto_client(std::move(mock_crypto_client)); @@ -185,13 +183,13 @@ TEST(CryptoClientWrapperTest, HpkeDecrypt_Success) { std::unique_ptr mock_crypto_client = std::make_unique(); EXPECT_CALL(*mock_crypto_client, HpkeDecrypt) - .WillOnce([&](const HpkeDecryptRequest& request, - const Callback& callback) - -> ExecutionResult { + .WillOnce([&expected_request, &mock_response]( + const HpkeDecryptRequest& request, + const Callback& callback) { EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals( request, expected_request)); callback(SuccessExecutionResult(), mock_response); - return SuccessExecutionResult(); + return absl::OkStatus(); }); CryptoClientWrapper crypto_client(std::move(mock_crypto_client)); @@ -218,13 +216,13 @@ TEST(CryptoClientWrapperTest, AeadEncrypt_Success) { std::unique_ptr mock_crypto_client = std::make_unique(); EXPECT_CALL(*mock_crypto_client, AeadEncrypt) - .WillOnce([&](const AeadEncryptRequest& request, - const Callback& callback) - -> ExecutionResult { + .WillOnce([&expected_request, &mock_response]( + const AeadEncryptRequest& request, + const Callback& callback) { EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals( request, expected_request)); callback(SuccessExecutionResult(), mock_response); - return SuccessExecutionResult(); + return absl::OkStatus(); }); CryptoClientWrapper crypto_client(std::move(mock_crypto_client)); @@ -252,13 +250,13 @@ TEST(CryptoClientWrapperTest, AeadDecrypt_Success) { std::unique_ptr mock_crypto_client = std::make_unique(); EXPECT_CALL(*mock_crypto_client, AeadDecrypt) - .WillOnce([&](const AeadDecryptRequest& request, - const Callback& callback) - -> ExecutionResult { + .WillOnce([&expected_request, &mock_response]( + const AeadDecryptRequest& request, + const Callback& callback) { EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals( request, expected_request)); callback(SuccessExecutionResult(), mock_response); - return SuccessExecutionResult(); + return absl::OkStatus(); }); CryptoClientWrapper crypto_client(std::move(mock_crypto_client)); diff --git a/services/common/feature_flags.cc b/services/common/feature_flags.cc new file mode 100644 index 00000000..a8ceabce --- /dev/null +++ b/services/common/feature_flags.cc @@ -0,0 +1,23 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "feature_flags.h" + +#include + +#include "absl/flags/flag.h" + +ABSL_FLAG(bool, enable_temporary_unlimited_egress, false, + "If true, temporary unlimited egress is allowed (if " + "client had also set enable_unlimited_egress flag in request)"); diff --git a/services/common/feature_flags.h b/services/common/feature_flags.h new file mode 100644 index 00000000..36a7d163 --- /dev/null +++ b/services/common/feature_flags.h @@ -0,0 +1,24 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SERVICES_COMMON_FEATURE_FLAGS_H_ +#define SERVICES_COMMON_FEATURE_FLAGS_H_ + +#include + +#include "absl/flags/declare.h" + +ABSL_DECLARE_FLAG(bool, enable_temporary_unlimited_egress); + +#endif // SERVICES_COMMON_FEATURE_FLAGS_H_ diff --git a/services/common/loggers/BUILD b/services/common/loggers/BUILD index 213ba918..e2d289e3 100644 --- a/services/common/loggers/BUILD +++ b/services/common/loggers/BUILD @@ -34,6 +34,7 @@ cc_library( ], deps = [ "//services/common/loggers:timer", + "//services/common/util:request_response_constants", "@com_google_absl//absl/strings", "@com_google_absl//absl/time", "@google_privacysandbox_servers_common//src/logger:request_context_logger", diff --git a/services/common/loggers/benchmarking_logger.cc b/services/common/loggers/benchmarking_logger.cc index c4fd45cd..1359c3a3 100644 --- a/services/common/loggers/benchmarking_logger.cc +++ b/services/common/loggers/benchmarking_logger.cc @@ -19,6 +19,7 @@ #include #include "absl/strings/str_cat.h" +#include "services/common/util/request_response_constants.h" #include "src/logger/request_context_logger.h" namespace privacy_sandbox::bidding_auction_servers { @@ -39,9 +40,9 @@ void BenchmarkingLogger::End() { BenchmarkingLogger::~BenchmarkingLogger() { for (const std::string& log_record : log_store_) { - PS_VLOG(1) << absl::StrCat("\nBenchmarking for request ID [", request_id_, - "]: ", log_record); + PS_VLOG(kInfoMsg) << absl::StrCat("\nBenchmarking for request ID [", + request_id_, "]: ", log_record); } - PS_VLOG(1) << "\n\n"; + PS_VLOG(kInfoMsg) << "\n\n"; } } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/common/metric/server_definition.h b/services/common/metric/server_definition.h index 767a1f13..dcc84de4 100644 --- a/services/common/metric/server_definition.h +++ b/services/common/metric/server_definition.h @@ -548,6 +548,7 @@ inline constexpr const server_common::metrics::DefinitionName* &kSfeWithWinnerTimeMs, &kProtectedCiphertextSize, &kAuctionConfigSize, + &kAuctionBidRejectedCount, }; template <> diff --git a/services/common/test/random.cc b/services/common/test/random.cc index cc20bd6a..fbce4c06 100644 --- a/services/common/test/random.cc +++ b/services/common/test/random.cc @@ -77,12 +77,10 @@ google::protobuf::Struct MakeAnAd(absl::string_view render_url, google::protobuf::Value ad_render_url; ad_render_url.set_string_value(render_url); ad.mutable_fields()->try_emplace("renderUrl", ad_render_url); - auto metadata_obj = MakeARandomStruct(0); google::protobuf::Value metadata_v; metadata_v.set_number_value(metadata_value); metadata_obj->mutable_fields()->try_emplace(metadata_key, metadata_v); - google::protobuf::Value metadata_obj_val; metadata_obj_val.set_allocated_struct_value(metadata_obj.release()); ad.mutable_fields()->try_emplace("metadata", metadata_obj_val); @@ -344,16 +342,30 @@ MakeARandomAdWithBidMetadata(float min_bid, float max_bid, ad_with_bid.mutable_ad()->mutable_struct_value()->MergeFrom( MakeAnAd(MakeARandomString(), MakeARandomString(), 2)); - ad_with_bid.set_interest_group_name(MakeARandomString()); - ad_with_bid.set_render(MakeARandomString()); ad_with_bid.set_bid(MakeARandomNumber(min_bid, max_bid)); - ad_with_bid.set_interest_group_owner(MakeARandomString()); + ad_with_bid.set_render( + absl::StrCat("barStandardAds.com/ads?id=", MakeARandomString())); for (int i = 0; i < num_ad_components; i++) { ad_with_bid.add_ad_components(absl::StrCat("adComponent.com/id=", i)); } - ad_with_bid.set_ad_cost(MakeARandomNumber(0.0, 2.0)); + // allow_component_auction defaults to false. + + ad_with_bid.set_interest_group_name( + absl::StrCat("interest_group_name_", MakeARandomString())); + ad_with_bid.set_interest_group_owner( + absl::StrCat("interest_group_owner_", MakeARandomString())); + + // join_count and recency left to default zeros. + ad_with_bid.set_modeling_signals(MakeARandomInt(0, 100)); + ad_with_bid.set_ad_cost(MakeARandomNumber(0.0, 2.0)); + + // No bid currency specified. + + ad_with_bid.set_interest_group_origin( + absl::StrCat("interest_group_origin_", MakeARandomString())); + return ad_with_bid; } @@ -448,6 +460,8 @@ ScoreAdsResponse::AdScore MakeARandomAdScore( ad_score.set_interest_group_name(MakeARandomString()); ad_score.set_buyer_bid(bid); ad_score.set_interest_group_owner(MakeARandomString()); + ad_score.set_interest_group_origin( + absl::StrCat("interest_group_origin_", MakeARandomString())); ad_score.set_ad_metadata(MakeARandomString()); ad_score.set_allow_component_auction(false); ad_score.set_bid(MakeARandomNumber(1, 2.5)); @@ -480,38 +494,47 @@ ScoreAdsResponse::AdScore MakeARandomAdScore( // Must manually delete/take ownership of underlying pointer // build_android_signals: If false, will build browser signals instead. std::unique_ptr MakeARandomInterestGroup( - bool build_android_signals) { + bool for_android) { auto interest_group = std::make_unique(); interest_group->set_name(absl::StrCat("ig_name_", MakeARandomString())); + interest_group->mutable_bidding_signals_keys()->Add( absl::StrCat("bidding_signal_key_", MakeARandomString())); interest_group->mutable_bidding_signals_keys()->Add( absl::StrCat("bidding_signal_key_", MakeARandomString())); - interest_group->set_allocated_user_bidding_signals( - MakeARandomStructJsonString(5).release()); + int ad_render_ids_to_generate = MakeARandomInt(1, 10); for (int i = 0; i < ad_render_ids_to_generate; i++) { *interest_group->mutable_ad_render_ids()->Add() = absl::StrCat("ad_render_id_", MakeARandomString()); } - if (build_android_signals) { + + // No component ads added. + + interest_group->set_allocated_user_bidding_signals( + MakeARandomStructJsonString(5).release()); + + if (for_android) { // Empty field right now. interest_group->mutable_android_signals(); + interest_group->set_origin( + absl::StrCat("interest_group_origin_", MakeARandomString())); } else { interest_group->mutable_browser_signals()->CopyFrom( MakeRandomBrowserSignalsForIG(interest_group->ad_render_ids())); } + return interest_group; } std::unique_ptr MakeARandomInterestGroupFromAndroid() { - return MakeARandomInterestGroup(true); + return MakeARandomInterestGroup(/*for_android=*/true); } std::unique_ptr MakeARandomInterestGroupFromBrowser() { - return MakeARandomInterestGroup(false); + return MakeARandomInterestGroup(/*for_android=*/false); } GetBidsRequest::GetBidsRawRequest MakeARandomGetBidsRawRequest() { @@ -557,7 +580,7 @@ google::protobuf::Value MakeAListValue( BuyerInput MakeARandomBuyerInput() { BuyerInput buyer_input; buyer_input.mutable_interest_groups()->AddAllocated( - MakeARandomInterestGroup(false).release()); + MakeARandomInterestGroup(/*for_android=*/false).release()); return buyer_input; } diff --git a/services/common/test/random.h b/services/common/test/random.h index 0c769238..237d2d83 100644 --- a/services/common/test/random.h +++ b/services/common/test/random.h @@ -134,7 +134,7 @@ ScoreAdsResponse::AdScore MakeARandomAdScore( // Must manually delete/take ownership of underlying pointer // build_android_signals: If false, will build browser signals instead. std::unique_ptr MakeARandomInterestGroup( - bool build_android_signals); + bool for_android); std::unique_ptr MakeARandomInterestGroupFromAndroid(); diff --git a/services/common/test/utils/test_utils.cc b/services/common/test/utils/test_utils.cc index 2e55dea8..97b2a1ae 100644 --- a/services/common/test/utils/test_utils.cc +++ b/services/common/test/utils/test_utils.cc @@ -26,6 +26,7 @@ GetBidsRequest::GetBidsRawRequest CreateGetBidsRawRequest( raw_request.set_publisher_name(kTestPublisherName); raw_request.set_seller(kTestSeller); raw_request.set_enable_debug_reporting(true); + raw_request.set_enable_unlimited_egress(true); raw_request.mutable_log_context()->set_generation_id(kTestGenerationId); raw_request.mutable_log_context()->set_adtech_debug_id(kTestAdTechDebugId); raw_request.mutable_consented_debug_config()->set_is_consented(true); @@ -82,7 +83,9 @@ ProtectedAppSignalsAdWithBid CreateProtectedAppSignalsAdWithBid() { ad_with_bid.set_modeling_signals(kTestModelingSignals2); ad_with_bid.set_ad_cost(kTestAdCost2); ad_with_bid.set_bid_currency(kTestCurrency2); - ad_with_bid.set_egress_features(kTestEgressFeature); + ad_with_bid.set_egress_payload(kTestEgressPayload); + ad_with_bid.set_temporary_unlimited_egress_payload( + kTestTemporaryEgressPayload); return ad_with_bid; } diff --git a/services/common/test/utils/test_utils.h b/services/common/test/utils/test_utils.h index 70018b21..ccdf3938 100644 --- a/services/common/test/utils/test_utils.h +++ b/services/common/test/utils/test_utils.h @@ -38,7 +38,8 @@ constexpr char kTestInterestGroupName[] = "test_ig"; constexpr int kTestBidValue1 = 10.0; constexpr int kTestAdCost1 = 2.0; constexpr int kTestModelingSignals1 = 54; -constexpr char kTestEgressFeature[] = "test_egress_features"; +constexpr char kTestEgressPayload[] = "test_egress_payload"; +constexpr char kTestTemporaryEgressPayload[] = "test_temporary_egress_payload"; constexpr char kTestRender1[] = "https://test-render.com"; constexpr char kTestMetadataKey1[] = "test_metadata_key"; constexpr char kTestAdComponent[] = "test_ad_component"; diff --git a/services/common/util/BUILD b/services/common/util/BUILD index 4ec44099..48b188a8 100644 --- a/services/common/util/BUILD +++ b/services/common/util/BUILD @@ -33,6 +33,32 @@ cc_library( ], ) +cc_library( + name = "proto_util", + hdrs = [ + "proto_util.h", + ], + visibility = ["//visibility:public"], + deps = [ + "@com_google_absl//absl/status:statusor", + "@com_google_protobuf//:protobuf", + "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", + ], +) + +cc_test( + name = "proto_util_test", + size = "small", + srcs = [ + "proto_util_test.cc", + ], + deps = [ + ":proto_util", + "@com_google_absl//absl/log:check", + "@com_google_googletest//:gtest_main", + ], +) + cc_test( name = "json_util_test", size = "small", @@ -175,6 +201,7 @@ cc_library( "//api:bidding_auction_servers_cc_proto", "//services/common/clients/http:http_fetcher_async", "//services/common/util:json_util", + "//services/common/util:request_response_constants", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/functional:any_invocable", "@com_google_absl//absl/status", diff --git a/services/common/util/error_categories.h b/services/common/util/error_categories.h index 353769d1..cf233e9b 100644 --- a/services/common/util/error_categories.h +++ b/services/common/util/error_categories.h @@ -74,6 +74,8 @@ inline constexpr char kUnsupportedAdTypeInAuctionResultError[] = "Unsupported ad type."; inline constexpr char kTopLevelWinReportingUrlsInAuctionResultError[] = "Top Level Win Reporting URLs should not be present."; +inline constexpr char kMultipleComponentAuctionResultsError[] = + "Top Level Auction contains multiple auction results for the same seller: "; // Server side errors listed here. inline constexpr char kInternalError[] = "Internal Error"; diff --git a/services/common/util/post_auction_signals.h b/services/common/util/post_auction_signals.h index 9060954b..a76151b2 100644 --- a/services/common/util/post_auction_signals.h +++ b/services/common/util/post_auction_signals.h @@ -31,9 +31,13 @@ struct PostAuctionSignals { std::string winning_ig_owner; // The winning bid from the auction. float winning_bid; + // Currency for said winning bid. + std::string winning_bid_currency; // The bid which was scored second highest in the auction. float highest_scoring_other_bid; + // Currency for said highest-scoring other bid. + std::string highest_scoring_other_bid_currency; // Owner of the interest group which made the second highest scored bid. std::string highest_scoring_other_bid_ig_owner; // Set to true if the signals has second highest scoring bid information. diff --git a/services/common/util/proto_util.h b/services/common/util/proto_util.h new file mode 100644 index 00000000..00d15bc8 --- /dev/null +++ b/services/common/util/proto_util.h @@ -0,0 +1,39 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SERVICES_COMMON_UTIL_PROTO_UTIL_H_ +#define SERVICES_COMMON_UTIL_PROTO_UTIL_H_ + +#include +#include + +#include "absl/status/statusor.h" +#include "src/util/status_macro/status_macros.h" + +namespace privacy_sandbox::bidding_auction_servers { + +// Converts the provided JSON string to a protobuf struct value. +inline absl::StatusOr JsonStringToValue( + absl::string_view json_string) { + google::protobuf::Value val; + PS_RETURN_IF_ERROR( + google::protobuf::util::JsonStringToMessage(json_string, &val)); + return val; +} + +} // namespace privacy_sandbox::bidding_auction_servers + +#endif // SERVICES_COMMON_UTIL_PROTO_UTIL_H_ diff --git a/services/common/util/proto_util_test.cc b/services/common/util/proto_util_test.cc new file mode 100644 index 00000000..5352f5a6 --- /dev/null +++ b/services/common/util/proto_util_test.cc @@ -0,0 +1,71 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "services/common/util/proto_util.h" + +#include "absl/log/check.h" +#include "gmock/gmock.h" +#include "google/protobuf/util/message_differencer.h" +#include "gtest/gtest.h" + +namespace privacy_sandbox { +namespace bidding_auction_servers { +namespace { + +using ::google::protobuf::util::MessageDifferencer; + +TEST(JsonStringToValue, ComplainsOnInvalidInput) { + auto test_value = R"JSON( + { + "k1": [ + })JSON"; + auto proto_value = JsonStringToValue(test_value); + EXPECT_FALSE(proto_value.ok()); +} + +TEST(JsonStringToValue, ConvertsWellFormedInput) { + auto test_value = R"JSON( + { + "k1": [1], + "k2": "2" + })JSON"; + auto proto_value = JsonStringToValue(test_value); + CHECK_OK(proto_value); + auto k1_it = proto_value->struct_value().fields().find("k1"); + ASSERT_NE(k1_it, proto_value->struct_value().fields().end()); + EXPECT_EQ(k1_it->second.list_value().values()[0].number_value(), 1); + + auto k2_it = proto_value->struct_value().fields().find("k2"); + ASSERT_NE(k2_it, proto_value->struct_value().fields().end()); + EXPECT_EQ(k2_it->second.string_value(), "2"); +} + +TEST(JsonStringToValue, ConvertsStringValue) { + auto test_value = R"JSON("test_string")JSON"; + auto proto_value = JsonStringToValue(test_value); + CHECK_OK(proto_value); + EXPECT_EQ(proto_value->string_value(), "test_string"); +} + +TEST(JsonStringToValue, ConvertsListValue) { + auto test_value = R"JSON([1, 2])JSON"; + auto proto_value = JsonStringToValue(test_value); + CHECK_OK(proto_value); + EXPECT_EQ(proto_value->list_value().values()[0].number_value(), 1); + EXPECT_EQ(proto_value->list_value().values()[1].number_value(), 2); +} + +} // namespace +} // namespace bidding_auction_servers +} // namespace privacy_sandbox diff --git a/services/common/util/reporting_util.cc b/services/common/util/reporting_util.cc index b9011186..817e631e 100644 --- a/services/common/util/reporting_util.cc +++ b/services/common/util/reporting_util.cc @@ -23,6 +23,7 @@ #include "absl/strings/str_cat.h" #include "absl/strings/str_replace.h" #include "services/common/util/json_util.h" +#include "services/common/util/request_response_constants.h" #include "src/util/status_macro/status_macros.h" namespace privacy_sandbox::bidding_auction_servers { @@ -34,6 +35,8 @@ inline constexpr char kWarnings[] = "warnings"; inline constexpr char kErrors[] = "errors"; inline constexpr int kNumDebugReportingReplacements = 5; inline constexpr int kNumAdditionalWinReportingReplacements = 2; +inline constexpr absl::string_view kFeatureDisabled = "false"; +inline constexpr absl::string_view kFeatureEnabled = "true"; void MayVlogAdTechCodeLogs(const rapidjson::Document& document, @@ -42,21 +45,35 @@ void MayVlogAdTechCodeLogs(const rapidjson::Document& document, auto logs_it = document.FindMember(log_type.c_str()); if (logs_it != document.MemberEnd()) { for (const auto& log : logs_it->value.GetArray()) { - PS_VLOG(1, log_context) << log_type << ": " << log.GetString(); + PS_VLOG(kInfoMsg, log_context) << log_type << ": " << log.GetString(); } } } +void AppendFeatureFlagValue(std::string& feature_flags, + absl::string_view feature_name, + bool is_feature_enabled) { + absl::string_view enable_feature = kFeatureDisabled; + if (is_feature_enabled) { + enable_feature = kFeatureEnabled; + } + feature_flags.append( + absl::StrCat("\"", feature_name, "\": ", enable_feature)); +} + } // namespace PostAuctionSignals GeneratePostAuctionSignals( - const std::optional& winning_ad_score) { + const std::optional& winning_ad_score, + absl::string_view seller_currency) { // If there is no winning ad, return with default signals values. if (!winning_ad_score.has_value()) { return {kDefaultWinningInterestGroupName, kDefaultWinningInterestGroupOwner, kDefaultWinningBid, + /*DefaultWinningBidCurrency=*/kEmptyBidCurrencyCode, /*DefaultHighestScoringOtherBid=*/0.0, + /*DefaultHighestScoringOtherBidCurrency=*/kEmptyBidCurrencyCode, kDefaultHighestScoringOtherBidInterestGroupOwner, /*DefaultHasHighestScoringOtherBid=*/false, kDefaultWinningScore, @@ -81,6 +98,13 @@ PostAuctionSignals GeneratePostAuctionSignals( } } + std::string bid_currency = (winning_ad_score->buyer_bid_currency().empty()) + ? kEmptyBidCurrencyCode + : winning_ad_score->buyer_bid_currency(); + std::string highest_scoring_other_bid_currency = + (seller_currency.empty()) ? kEmptyBidCurrencyCode + : std::string(seller_currency); + bool made_highest_scoring_other_bid = false; if (winning_ad_score->ig_owner_highest_scoring_other_bids_map().size() == 1 && winning_ad_score->ig_owner_highest_scoring_other_bids_map().contains( @@ -112,7 +136,9 @@ PostAuctionSignals GeneratePostAuctionSignals( return {winning_ad_score->interest_group_name(), winning_ad_score->interest_group_owner(), winning_bid, + bid_currency, highest_scoring_other_bid, + highest_scoring_other_bid_currency, std::move(highest_scoring_other_bid_ig_owner), has_highest_scoring_other_bid, winning_score, @@ -262,4 +288,15 @@ absl::StatusOr ParseAndGetResponseJson( return SerializeJsonDoc(document["response"]); } +std::string GetFeatureFlagJson(bool enable_logging, + bool enable_debug_url_generation) { + std::string feature_flags = "{"; + AppendFeatureFlagValue(feature_flags, kFeatureLogging, enable_logging); + feature_flags.append(","); + AppendFeatureFlagValue(feature_flags, kFeatureDebugUrlGeneration, + enable_debug_url_generation); + feature_flags.append("}"); + return feature_flags; +} + } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/common/util/reporting_util.h b/services/common/util/reporting_util.h index 443af3e8..578dcc72 100644 --- a/services/common/util/reporting_util.h +++ b/services/common/util/reporting_util.h @@ -58,6 +58,7 @@ constexpr absl::string_view kRejectionReasonBidFromScoreAdFailedCurrencyCheck = "bid-from-score-ad-failed-currency-check"; constexpr float kDefaultWinningBid = 0.0; +constexpr char kEmptyBidCurrencyCode[] = "???"; constexpr float kDefaultWinningScore = 0.0; inline constexpr char kDefaultWinningAdRenderUrl[] = ""; inline constexpr char kDefaultWinningInterestGroupName[] = ""; @@ -65,6 +66,9 @@ inline constexpr char kDefaultWinningInterestGroupOwner[] = ""; inline constexpr char kDefaultHighestScoringOtherBidInterestGroupOwner[] = ""; inline constexpr char kDefaultHighestScoringOtherBid[] = "0.00"; inline constexpr char kDefaultHasHighestScoringOtherBid[] = "false"; +inline constexpr char kFeatureLogging[] = "enable_logging"; +inline constexpr char kFeatureDebugUrlGeneration[] = + "enable_debug_url_generation"; // Captures placeholder data for debug reporting. struct DebugReportingPlaceholder { @@ -84,7 +88,8 @@ struct DebugReportingPlaceholder { // Returns post auction signals from winning ad score. // If there is no winning ad, default values are returned. PostAuctionSignals GeneratePostAuctionSignals( - const std::optional& winning_ad_score); + const std::optional& winning_ad_score, + absl::string_view seller_currency); // Returns a http request object for debug reporting after replacing placeholder // data in the url. @@ -121,5 +126,9 @@ absl::StatusOr ParseAndGetResponseJson( bool enable_ad_tech_code_logging, const std::string& response, server_common::log::ContextImpl& log_context); +// Returns a JSON string for feature flags to be used by the wrapper script. +std::string GetFeatureFlagJson(bool enable_logging, + bool enable_debug_url_generation); + } // namespace privacy_sandbox::bidding_auction_servers #endif // SERVICES_COMMON_UTIL_REPORTING_UTIL_H diff --git a/services/common/util/reporting_util_test.cc b/services/common/util/reporting_util_test.cc index 5985d93f..28e543cc 100644 --- a/services/common/util/reporting_util_test.cc +++ b/services/common/util/reporting_util_test.cc @@ -31,17 +31,21 @@ inline constexpr char kTestIgOwner[] = "test_ig_owner"; inline constexpr char kTestIgName[] = "test_ig_name"; inline constexpr char kWinningIgName[] = "winning_ig_name"; inline constexpr char kWinningIgOwner[] = "winning_ig_owner"; +inline constexpr char kSellerCurrency[] = "GBP"; +inline constexpr char kDefaultHighestScoringOtherBidCurrency[] = "???"; TEST(PostAuctionSignalsTest, HasAllSignals) { std::optional ad_score = std::make_optional(MakeARandomAdScore(2, 2, 2)); - PostAuctionSignals signals = GeneratePostAuctionSignals(ad_score); + PostAuctionSignals signals = + GeneratePostAuctionSignals(ad_score, kSellerCurrency); EXPECT_EQ(ad_score->interest_group_owner(), signals.winning_ig_owner); EXPECT_EQ(ad_score->interest_group_name(), signals.winning_ig_name); EXPECT_EQ(ad_score->buyer_bid(), signals.winning_bid); EXPECT_EQ(ad_score->desirability(), signals.winning_score); EXPECT_EQ(ad_score->render(), signals.winning_ad_render_url); EXPECT_TRUE(signals.has_highest_scoring_other_bid); + EXPECT_EQ(signals.highest_scoring_other_bid_currency, kSellerCurrency); EXPECT_NE(signals.highest_scoring_other_bid_ig_owner, ""); EXPECT_GT(signals.highest_scoring_other_bid, 0.0); EXPECT_GT(signals.rejection_reason_map.size(), 0); @@ -50,32 +54,36 @@ TEST(PostAuctionSignalsTest, HasAllSignals) { TEST(PostAuctionSignalsTest, HasWinningBidSignals) { std::optional ad_score = std::make_optional(MakeARandomAdScore(0)); - PostAuctionSignals signals = GeneratePostAuctionSignals(ad_score); + PostAuctionSignals signals = GeneratePostAuctionSignals(ad_score, ""); EXPECT_EQ(ad_score->interest_group_owner(), signals.winning_ig_owner); EXPECT_EQ(ad_score->interest_group_name(), signals.winning_ig_name); EXPECT_EQ(ad_score->buyer_bid(), signals.winning_bid); EXPECT_EQ(ad_score->desirability(), signals.winning_score); EXPECT_EQ(ad_score->render(), signals.winning_ad_render_url); + EXPECT_EQ(signals.highest_scoring_other_bid_currency, + kDefaultHighestScoringOtherBidCurrency); } TEST(PostAuctionSignalsTest, HasHighestOtherBidSignals) { std::optional ad_score = std::make_optional(MakeARandomAdScore(2)); - PostAuctionSignals signals = GeneratePostAuctionSignals(ad_score); + PostAuctionSignals signals = + GeneratePostAuctionSignals(ad_score, kSellerCurrency); EXPECT_TRUE(signals.has_highest_scoring_other_bid); EXPECT_NE(signals.highest_scoring_other_bid_ig_owner, ""); EXPECT_GT(signals.highest_scoring_other_bid, 0.0); + EXPECT_EQ(signals.highest_scoring_other_bid_currency, kSellerCurrency); } TEST(PostAuctionSignalsTest, HasRejectionReasons) { std::optional ad_score = std::make_optional(MakeARandomAdScore(0, 2, 2)); - PostAuctionSignals signals = GeneratePostAuctionSignals(ad_score); + PostAuctionSignals signals = GeneratePostAuctionSignals(ad_score, ""); EXPECT_GT(signals.rejection_reason_map.size(), 0); } TEST(PostAuctionSignalsTest, DoesNotHaveAnySignal) { - PostAuctionSignals signals = GeneratePostAuctionSignals(std::nullopt); + PostAuctionSignals signals = GeneratePostAuctionSignals(std::nullopt, ""); EXPECT_EQ(signals.winning_ig_owner, ""); EXPECT_EQ(signals.winning_ig_name, ""); EXPECT_EQ(signals.winning_bid, 0.0); @@ -83,6 +91,8 @@ TEST(PostAuctionSignalsTest, DoesNotHaveAnySignal) { EXPECT_EQ(signals.winning_ad_render_url, ""); EXPECT_FALSE(signals.has_highest_scoring_other_bid); EXPECT_EQ(signals.highest_scoring_other_bid_ig_owner, ""); + EXPECT_EQ(signals.highest_scoring_other_bid_currency, + kDefaultHighestScoringOtherBidCurrency); EXPECT_EQ(signals.highest_scoring_other_bid, 0.0); EXPECT_EQ(signals.rejection_reason_map.size(), 0); } diff --git a/services/common/util/request_response_constants.h b/services/common/util/request_response_constants.h index c5f0d9f3..14004854 100644 --- a/services/common/util/request_response_constants.h +++ b/services/common/util/request_response_constants.h @@ -34,7 +34,7 @@ inline constexpr int kNumConsentedDebugConfigKeys = 3; // Maximum number of keys that will be populated in the encoded CBOR // AuctionResult response. -inline constexpr int kNumAuctionResultKeys = 13; +inline constexpr int kNumAuctionResultKeys = 14; // Maximum number of keys that will be populated in the encoded CBOR // WinReportingUrls response. @@ -83,6 +83,7 @@ inline constexpr char kAdRenderUrl[] = "adRenderURL"; // length: 11 inline constexpr char kBidCurrency[] = "bidCurrency"; // length: 11 inline constexpr char kBiddingGroups[] = "biddingGroups"; // length: 13 inline constexpr char kTopLevelSeller[] = "topLevelSeller"; // length: 14 +inline constexpr char kBuyerReportingId[] = "buyerReportingId"; // length: 16 inline constexpr char kWinReportingUrls[] = "winReportingURLs"; // length: 16 inline constexpr char kInterestGroupName[] = "interestGroupName"; // length: 17 inline constexpr char kInterestGroupOwner[] = @@ -141,6 +142,8 @@ enum class AuctionType : int { kProtectedAudience, kProtectedAppSignals }; // log verbosity inline constexpr int kPlain = 1; +inline constexpr int kSuccess = 3; +inline constexpr int kInfoMsg = 3; inline constexpr int kEncrypted = 4; } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/inference_sidecar/README.md b/services/inference_sidecar/README.md index 78e395f4..3527c084 100644 --- a/services/inference_sidecar/README.md +++ b/services/inference_sidecar/README.md @@ -42,9 +42,12 @@ variable with `--//:inference_runtime=pytorch`. ## Start the B&A servers in GCP - Create a GCS bucket and store ML models into it. -- Set runtime flags: `INFERENCE_SIDECAR_BINARY_PATH`, `INFERENCE_MODEL_BUCKET_NAME`, and - `INFERENCE_MODEL_BUCKET_PATHS`. +- Set runtime flags: `INFERENCE_SIDECAR_BINARY_PATH`, `INFERENCE_MODEL_BUCKET_NAME`, + `INFERENCE_MODEL_BUCKET_PATHS`, and `INFERENCE_SIDECAR_RUNTIME_CONFIG`. - Set `INFERENCE_SIDECAR_BINARY_PATH` to `/server/bin/inference_sidecar`. + - `INFERENCE_SIDECAR_RUNTIME_CONFIG` has the following format: { "num_interop_threads": + , "num_intraop_threads": , "module_name": , } + Currently "module_name" can be one of "test", "tensorflow_v2_14_0", "pytorch_v2_1_1". - Refer to [README.md](https://github.com/privacysandbox/bidding-auction-servers/tree/main/production/deploy/gcp/terraform/environment/demo/README.md). @@ -53,7 +56,7 @@ variable with `--//:inference_runtime=pytorch`. The servers run locally and the ML models are directly read from the local disk. - Use the command-line flags: `--inference_sidecar_binary_path` and - `--inference_model_local_paths`. + `--inference_model_local_paths`, `--inference_sidecar_runtime_config`. - Utilize services/inference_sidecar/common/tools/debug/start_inference script. ## Trigger ML Inference @@ -133,36 +136,36 @@ See an example of a Batch Inference Request in a JSON format with 2 models: ```json { - "request" : [ - { - "model_path" : "my_bucket/models/pcvr/1/", - "tensors" : [ + "request": [ { - "tensor_name": "feature1", - "data_type": "DOUBLE", - "tensor_shape": [2, 1], - "tensor_content": ["0.454920", "-0.25752"] - } - ] - }, - { - "model_path" : "my_bucket/models/pctr/2/", - "tensors" : [ - { - "tensor_name": "feature1", - "data_type": "INT32", - "tensor_shape": [2, 1], - "tensor_content": ["5", "6"] + "model_path": "my_bucket/models/pcvr/1/", + "tensors": [ + { + "tensor_name": "feature1", + "data_type": "DOUBLE", + "tensor_shape": [2, 1], + "tensor_content": ["0.454920", "-0.25752"] + } + ] }, { - "tensor_name": "feature2", - "data_type": "FLOAT", - "tensor_shape": [2, 3], - "tensor_content": ["0.5", "0.6", "0.7", "0.8". "0.21", "-0.99"] + "model_path": "my_bucket/models/pctr/2/", + "tensors": [ + { + "tensor_name": "feature1", + "data_type": "INT32", + "tensor_shape": [2, 1], + "tensor_content": ["5", "6"] + }, + { + "tensor_name": "feature2", + "data_type": "FLOAT", + "tensor_shape": [2, 3], + "tensor_content": ["0.5", "0.6", "0.7", "0.8", "0.21", "-0.99"] + } + ] } - ] - } - ] + ] } ``` diff --git a/services/inference_sidecar/common/BUILD b/services/inference_sidecar/common/BUILD index 4464a27b..f725970e 100644 --- a/services/inference_sidecar/common/BUILD +++ b/services/inference_sidecar/common/BUILD @@ -33,6 +33,7 @@ cc_library( "//proto:inference_sidecar_cc_proto", "//sandbox:sandbox_worker", "@com_google_absl//absl/log", + "@com_google_absl//absl/strings", "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", ], ) @@ -50,6 +51,7 @@ cc_library( "@com_google_absl//absl/log", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", "@google_privacysandbox_servers_common//src/util/status_macro:status_util", ], ) @@ -94,6 +96,7 @@ cc_test( ], flaky = True, deps = [ + ":test_constants", "//proto:inference_sidecar_cc_grpc_proto", "//proto:inference_sidecar_cc_proto", "//sandbox:sandbox_executor", @@ -109,6 +112,15 @@ cc_test( ], ) +cc_library( + name = "test_constants", + testonly = True, + hdrs = ["test_constants.h"], + deps = [ + "@com_google_absl//absl/strings", + ], +) + genrule( name = "collect-logs", outs = ["collect_logs.bin"], diff --git a/services/inference_sidecar/common/WORKSPACE b/services/inference_sidecar/common/WORKSPACE index 20f3ca92..3c738610 100644 --- a/services/inference_sidecar/common/WORKSPACE +++ b/services/inference_sidecar/common/WORKSPACE @@ -39,6 +39,14 @@ load("@google_privacysandbox_servers_common//third_party:deps1.bzl", data_plane_ data_plane_shared_deps1() +load("@google_privacysandbox_servers_common//third_party:deps2.bzl", data_plane_shared_deps2 = "deps2") + +data_plane_shared_deps2() + +load("@google_privacysandbox_servers_common//third_party:deps3.bzl", data_plane_shared_deps3 = "deps3") + +data_plane_shared_deps3() + load("@com_google_googleapis//:repository_rules.bzl", "switched_rules_by_language") switched_rules_by_language( @@ -63,15 +71,3 @@ grpc_deps() load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") protobuf_deps() - -load("@google_privacysandbox_servers_common//build_defs/cc/shared:sandboxed_api.bzl", "sandboxed_api") - -### Sandbox2 depndencies -sandboxed_api() - -load("@com_google_sandboxed_api//sandboxed_api/bazel:llvm_config.bzl", "llvm_disable_optional_support_deps") -load("@com_google_sandboxed_api//sandboxed_api/bazel:sapi_deps.bzl", "sapi_deps") - -llvm_disable_optional_support_deps() - -sapi_deps() diff --git a/services/inference_sidecar/common/benchmark/BUILD b/services/inference_sidecar/common/benchmark/BUILD index bc3b5454..7e3106ac 100644 --- a/services/inference_sidecar/common/benchmark/BUILD +++ b/services/inference_sidecar/common/benchmark/BUILD @@ -14,6 +14,12 @@ load("@rules_cc//cc:defs.bzl", "cc_binary") +# We export some benchmark files. +# The benchmark can run in the workspace of each inference backend. +exports_files([ + "module_benchmark.cc", +]) + cc_binary( name = "sandbox_benchmark", srcs = [ @@ -39,6 +45,44 @@ cc_binary( ], ) +cc_binary( + name = "module_benchmark", + srcs = [ + "module_benchmark.cc", + ], + data = [ + "//:gen_test_model", + ], + deps = [ + "//modules:module_interface", + "//modules:test_module", + "//proto:inference_sidecar_cc_proto", + "//utils:file_util", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_benchmark//:benchmark", + "@com_google_benchmark//:benchmark_main", + ], +) + +cc_binary( + name = "roma_benchmark", + srcs = ["roma_benchmark.cc"], + deps = [ + "//proto:inference_sidecar_cc_proto", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/synchronization", + "@com_google_benchmark//:benchmark", + "@com_google_benchmark//:benchmark_main", + "@google_privacysandbox_servers_common//src/roma/config", + "@google_privacysandbox_servers_common//src/roma/interface", + "@google_privacysandbox_servers_common//src/roma/roma_service", + ], +) + # Builds the IPC inference sidecar for testing. cc_binary( name = "ipc_inference_sidecar_test_bin", @@ -46,6 +90,7 @@ cc_binary( deps = [ "//:ipc_sidecar", "//modules:test_module", + "//proto:inference_sidecar_cc_proto", "@com_google_absl//absl/log:check", ], ) diff --git a/services/inference_sidecar/common/benchmark/ipc_sidecar_main.cc b/services/inference_sidecar/common/benchmark/ipc_sidecar_main.cc index e2917ce9..1ae32b07 100644 --- a/services/inference_sidecar/common/benchmark/ipc_sidecar_main.cc +++ b/services/inference_sidecar/common/benchmark/ipc_sidecar_main.cc @@ -15,11 +15,14 @@ // IPC sidecar for testing. #include "absl/log/check.h" +#include "proto/inference_sidecar.pb.h" #include "ipc_sidecar.h" int main(int argc, char** argv) { - CHECK(!privacy_sandbox::bidding_auction_servers::inference::Run().ok()) + privacy_sandbox::bidding_auction_servers::inference:: + InferenceSidecarRuntimeConfig config; + CHECK(!privacy_sandbox::bidding_auction_servers::inference::Run(config).ok()) << "Unsuccessful run of the inference sidecar."; return 0; } diff --git a/services/inference_sidecar/common/benchmark/module_benchmark.cc b/services/inference_sidecar/common/benchmark/module_benchmark.cc new file mode 100644 index 00000000..35025bc3 --- /dev/null +++ b/services/inference_sidecar/common/benchmark/module_benchmark.cc @@ -0,0 +1,172 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This benchmark measures the performance of the inference module. +// +// Run the benchmark as follows: +// builders/tools/bazel-debian run //benchmark:module_benchmark -- \ +// --benchmark_counters_tabular=true --benchmark_repetitions=5 \ +// --benchmark_min_warmup_time=1 > /tmp/report.txt + +// How to read the benchmark results: +// +// Benchmark name: +// * `process_time/real_time`: Measures the wall time for latency and CPU time +// of all threads not just limited to the main thread. +// * `threads:8`: The number of threads concurrently executing the benchmark. +// +// Metrics: +// * `Time`: The total elapsed wall clock time per Iteration. +// * `CPU`: The total CPU time spent by all the threads per Iteration. +// * `Iterations`: The number of serial executions. +// * `Throughput` & `items_per_second`: The number of Iterations per second. +// * `Latency`: Average time spent per Iteration. + +#include + +#include "absl/log/check.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "benchmark/benchmark.h" +#include "modules/module_interface.h" +#include "proto/inference_sidecar.pb.h" +#include "utils/file_util.h" + +namespace privacy_sandbox::bidding_auction_servers::inference { +namespace { + +constexpr absl::string_view kTestModelPath = "test_model"; +constexpr char kJsonString[] = R"json({ + "request" : [{ + "model_path" : "test_model", + "tensors" : [ + { + "tensor_name": "serving_default_double1:0", + "data_type": "DOUBLE", + "tensor_shape": [ + 1, + 1 + ], + "tensor_content": ["3.14"] + } + ] +}] + })json"; +constexpr int kMaxThreads = 64; +constexpr int kRegisterMaxThreads = 4; + +static void ExportMetrics(benchmark::State& state) { + state.SetItemsProcessed(state.iterations()); + + state.counters["Throughput"] = + benchmark::Counter(state.iterations(), benchmark::Counter::kIsRate); + state.counters["Latency"] = + benchmark::Counter(state.iterations(), benchmark::Counter::kIsRate | + benchmark::Counter::kInvert); +} + +// Creates a new request with model files in the new model path +RegisterModelRequest CreateNewRegisterRequest( + const RegisterModelRequest& register_request, + const std::string& new_model_path) { + RegisterModelRequest new_register_request = register_request; + new_register_request.mutable_model_spec()->set_model_path(new_model_path); + + std::map new_model_files; + + absl::string_view request_model_path = + register_request.model_spec().model_path(); + for (const auto& [relative_path, file_content] : + register_request.model_files()) { + std::string new_file_path = absl::StrCat( + new_model_path, relative_path.substr(request_model_path.length())); + new_model_files[new_file_path] = file_content; + } + new_register_request.clear_model_files(); + for (const auto& [new_file_path, file_content] : new_model_files) { + (*new_register_request.mutable_model_files())[new_file_path] = file_content; + } + return new_register_request; +} + +class ModuleFixture : public benchmark::Fixture { + public: + void SetUp(::benchmark::State& state) { + if (state.thread_index() != 0) return; + + InferenceSidecarRuntimeConfig config; + module_ = ModuleInterface::Create(config); + + CHECK(PopulateRegisterModelRequest(kTestModelPath, register_model_request_) + .ok()); + absl::StatusOr response = module_->RegisterModel(register_model_request_); + CHECK(response.ok()) << response.status().message(); + } + void TearDown(::benchmark::State& state) { + if (state.thread_index() != 0) return; + module_.reset(); + } + + protected: + std::unique_ptr module_; + RegisterModelRequest register_model_request_; +}; + +BENCHMARK_DEFINE_F(ModuleFixture, BM_Predict)(benchmark::State& state) { + for (auto _ : state) { + PredictRequest predict_request; + predict_request.set_input(kJsonString); + + absl::StatusOr response = module_->Predict(predict_request); + CHECK(response.ok()) << response.status().message(); + } + + ExportMetrics(state); +} + +BENCHMARK_DEFINE_F(ModuleFixture, BM_Register)(benchmark::State& state) { + int iter = 0; + for (auto _ : state) { + state.PauseTiming(); + std::string new_model_path = + absl::StrCat(register_model_request_.model_spec().model_path(), + state.thread_index(), iter++); + RegisterModelRequest new_register_model_request = + CreateNewRegisterRequest(register_model_request_, new_model_path); + state.ResumeTiming(); + + absl::StatusOr response = + module_->RegisterModel(new_register_model_request); + CHECK(response.ok()) << response.status().message(); + } + + ExportMetrics(state); +} + +// Registers the functions to the benchmark. +BENCHMARK_REGISTER_F(ModuleFixture, BM_Register) + ->ThreadRange(1, kRegisterMaxThreads) + ->MeasureProcessCPUTime() + ->UseRealTime(); +BENCHMARK_REGISTER_F(ModuleFixture, BM_Predict) + ->ThreadRange(1, kMaxThreads) + ->MeasureProcessCPUTime() + ->UseRealTime(); + +// Runs the benchmark. +BENCHMARK_MAIN(); + +} // namespace +} // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/inference_sidecar/common/benchmark/roma_benchmark.cc b/services/inference_sidecar/common/benchmark/roma_benchmark.cc new file mode 100644 index 00000000..269345e3 --- /dev/null +++ b/services/inference_sidecar/common/benchmark/roma_benchmark.cc @@ -0,0 +1,273 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This benchmark measures the performance of the Roma used to run inference. +// +// Run the benchmark as follows: +// builders/tools/bazel-debian run //benchmark:roma_benchmark -- \ +// --benchmark_counters_tabular=true \ +// --benchmark_repetitions=5 \ +// --benchmark_min_warmup_time=1 \ +// 2>/dev/null \ +// | grep -Ev "sandbox.cc|monitor_base.cc|sandbox2.cc" \ +// > /tmp/report.txt +// +// How to read the benchmark results: +// +// Benchmark name: +// * `process_time/real_time`: Measures the wall time for latency and CPU time +// of all threads not just limited to the main thread. +// * `threads:8`: The number of threads concurrently executing the benchmark. +// +// Metrics: +// * `Time`: The total elapsed wall clock time per Iteration. +// * `CPU`: The total CPU time spent by all the threads per Iteration. +// * `Iterations`: The number of serial executions. +// * `Throughput` & `items_per_second`: The number of Iterations per second. +// * `Latency`: Average time spent per Iteration. +// +// This Roma bechmark code is re-written based on: +// https://github.com/privacysandbox/data-plane-shared-libraries/blob/main/src/roma/benchmark/ba_server_benchmark.cc + +#include +#include +#include +#include + +#include "absl/log/check.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/synchronization/notification.h" +#include "benchmark/benchmark.h" +#include "proto/inference_sidecar.pb.h" +#include "src/roma/config/config.h" +#include "src/roma/interface/roma.h" +#include "src/roma/roma_service/roma_service.h" + +namespace privacy_sandbox::bidding_auction_servers::inference { +namespace { + +using LoadRequest = ::google::scp::roma::CodeObject; +using DispatchRequest = ::google::scp::roma::InvocationSharedRequest<>; +using DispatchConfig = ::google::scp::roma::Config<>; +using LoadResponse = ::google::scp::roma::ResponseObject; +using DispatchResponse = ::google::scp::roma::ResponseObject; + +constexpr absl::string_view kVersionString = "v1"; +constexpr absl::string_view kCode = R"( + function run() { + console.log("Hello World!"); + } +)"; +// TODO(b/330364610): Use the real inference requests. +constexpr absl::string_view kInferenceCode = R"( + function run() { + console.log("Hello Inference!"); + const batchInferenceRequest = { + request: [ + { + model_path: '/model/path', + tensors: [ + { + tensor_name: 'input', + data_type: 'INT32', + tensor_shape: [1, 1], + tensor_content: ['8'], + }, + ], + }, + ], + }; + + const jsonRequest = JSON.stringify(batchInferenceRequest); + const output = runInference(jsonRequest); + return output; + } +)"; + +constexpr absl::string_view kHandlerName = "run"; +// TODO(b/330364610): Run the benchmark with variable #Roma workers. +constexpr int kNumRomaWorkers = 16; +// TODO(b/330364610): Enable multi threaded benchmark. +constexpr int kMaxThreads = 1; + +static void ExportMetrics(benchmark::State& state) { + state.SetItemsProcessed(state.iterations()); + + state.counters["Throughput"] = + benchmark::Counter(state.iterations(), benchmark::Counter::kIsRate); + state.counters["Latency"] = + benchmark::Counter(state.iterations(), benchmark::Counter::kIsRate | + benchmark::Counter::kInvert); +} + +void TestRunInference(google::scp::roma::FunctionBindingPayload<>& wrapper) { + const std::string& payload = wrapper.io_proto.input_string(); + + PredictRequest predict_request; + predict_request.set_input(payload); + + // No gRPC to the inference sidecar. + PredictResponse response; + response.set_output("0.57721"); + wrapper.io_proto.set_output_string(response.output()); +} + +class RomaFixture : public benchmark::Fixture { + public: + void SetUp(::benchmark::State& state) { + if (state.thread_index() != 0) return; + + DispatchConfig config; + config.number_of_workers = kNumRomaWorkers; + + // Registers runInference(). + auto run_inference_function_object = + std::make_unique>(); + run_inference_function_object->function_name = std::string("runInference"); + run_inference_function_object->function = TestRunInference; + config.RegisterFunctionBinding(std::move(run_inference_function_object)); + + roma_service_ = std::make_unique< + google::scp::roma::sandbox::roma_service::RomaService<>>( + std::move(config)); + CHECK(roma_service_->Init().ok()); + } + void TearDown(::benchmark::State& state) { + if (state.thread_index() != 0) return; + + CHECK(roma_service_->Stop().ok()); + roma_service_.reset(); + } + + // Loads the JS code synchronously. + // TODO(b/330364610): Re-use the B&A V8Dispatcher class. + void LoadCode(absl::string_view version, absl::string_view js_code) const { + auto request = std::make_unique(LoadRequest{ + .version_string = std::string(version), + .js = std::string(js_code), + }); + + absl::Notification load_finished; + absl::Status load_status; + + if (absl::Status try_load = roma_service_->LoadCodeObj( + std::move(request), + [&load_finished, + &load_status](absl::StatusOr res) { // NOLINT + if (!res.ok()) { + load_status.Update(res.status()); + } + load_finished.Notify(); + }); + !try_load.ok()) { + // Load callback won't be called, we can return. + return; + } + load_finished.WaitForNotification(); + CHECK(load_status.ok()); + } + + // Executes the JS code synchronously. + void BatchExecute(std::vector& batch) const { + absl::Notification notification; + auto batch_callback = + [¬ification]( + const std::vector>& result) { + for (const auto& status_or : result) { + CHECK(status_or.ok()); + } + notification.Notify(); + }; + + // This call schedules the code to be executed: + CHECK(roma_service_->BatchExecute(batch, std::move(batch_callback)).ok()); + notification.WaitForNotification(); + } + + void BatchExecute(absl::string_view handler_name, int batch_size) { + DispatchRequest request = { + .id = "id", + .version_string = std::string(kVersionString), + .handler_name = std::string{handler_name}, + }; + + std::vector batch(batch_size, request); + BatchExecute(batch); + } + + protected: + std::unique_ptr> + roma_service_; +}; + +BENCHMARK_DEFINE_F(RomaFixture, BM_LoadCode)(benchmark::State& state) { + for (auto _ : state) { + LoadCode(kVersionString, kCode); + } + ExportMetrics(state); +} + +BENCHMARK_DEFINE_F(RomaFixture, BM_LoadCode_Inference) +(benchmark::State& state) { + for (auto _ : state) { + LoadCode(kVersionString, kInferenceCode); + } + ExportMetrics(state); +} + +BENCHMARK_DEFINE_F(RomaFixture, BM_BatchExecute)(benchmark::State& state) { + LoadCode(kVersionString, kCode); + for (auto _ : state) { + BatchExecute(kHandlerName, kNumRomaWorkers); + } + ExportMetrics(state); +} + +BENCHMARK_DEFINE_F(RomaFixture, BM_BatchExecute_Inference) +(benchmark::State& state) { + LoadCode(kVersionString, kInferenceCode); + for (auto _ : state) { + BatchExecute(kHandlerName, kNumRomaWorkers); + } + ExportMetrics(state); +} + +// Registers the functions to the benchmark. +BENCHMARK_REGISTER_F(RomaFixture, BM_LoadCode) + ->ThreadRange(1, kMaxThreads) + ->MeasureProcessCPUTime() + ->UseRealTime(); + +BENCHMARK_REGISTER_F(RomaFixture, BM_LoadCode_Inference) + ->ThreadRange(1, kMaxThreads) + ->MeasureProcessCPUTime() + ->UseRealTime(); + +BENCHMARK_REGISTER_F(RomaFixture, BM_BatchExecute) + ->ThreadRange(1, kMaxThreads) + ->MeasureProcessCPUTime() + ->UseRealTime(); + +BENCHMARK_REGISTER_F(RomaFixture, BM_BatchExecute_Inference) + ->ThreadRange(1, kMaxThreads) + ->MeasureProcessCPUTime() + ->UseRealTime(); + +// Runs the benchmark. +BENCHMARK_MAIN(); + +} // namespace +} // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/inference_sidecar/common/benchmark/sandbox_benchmark.cc b/services/inference_sidecar/common/benchmark/sandbox_benchmark.cc index 85252e53..8dafb9c5 100644 --- a/services/inference_sidecar/common/benchmark/sandbox_benchmark.cc +++ b/services/inference_sidecar/common/benchmark/sandbox_benchmark.cc @@ -90,7 +90,7 @@ static void ExportMetrics(benchmark::State& state) { } static void BM_Multiworker_Predict_GRPC(benchmark::State& state) { - SandboxExecutor executor(kGrpcInferenceSidecarBinary, {""}); + SandboxExecutor executor(kGrpcInferenceSidecarBinary, {"{}"}); CHECK_EQ(executor.StartSandboxee().code(), absl::StatusCode::kOk); std::shared_ptr client_channel = @@ -133,7 +133,7 @@ static void BM_Predict_GRPC(benchmark::State& state) { static std::unique_ptr stub = nullptr; if (state.thread_index() == 0) { - const std::vector arg = {""}; + const std::vector arg = {"{}"}; executor = std::make_unique(kGrpcInferenceSidecarBinary, arg); CHECK_EQ(executor->StartSandboxee().code(), absl::StatusCode::kOk); @@ -177,7 +177,7 @@ static void BM_Predict_GRPC(benchmark::State& state) { } static void BM_Multiworker_Predict_IPC(benchmark::State& state) { - SandboxExecutor executor(kIpcInferenceSidecarBinary, {""}); + SandboxExecutor executor(kIpcInferenceSidecarBinary, {"{}"}); CHECK_EQ(executor.StartSandboxee().code(), absl::StatusCode::kOk); sandbox2::Comms comms(executor.FileDescriptor()); @@ -216,7 +216,7 @@ static void BM_Predict_IPC(benchmark::State& state) { static std::unique_ptr comms = nullptr; if (state.thread_index() == 0) { - const std::vector arg = {""}; + const std::vector arg = {"{}"}; executor = std::make_unique(kIpcInferenceSidecarBinary, arg); CHECK_EQ(executor->StartSandboxee().code(), absl::StatusCode::kOk); @@ -261,7 +261,7 @@ static void BM_Predict_IPC(benchmark::State& state) { // BM_Register_IPC is not implemented. It's too slow to run the microbenchmark // because it requires a new sandbox worker per model registration. static void BM_Register_GRPC(benchmark::State& state) { - SandboxExecutor executor(kGrpcInferenceSidecarBinary, {""}); + SandboxExecutor executor(kGrpcInferenceSidecarBinary, {"{}"}); CHECK_EQ(executor.StartSandboxee().code(), absl::StatusCode::kOk); std::shared_ptr client_channel = diff --git a/services/inference_sidecar/common/grpc_sidecar.cc b/services/inference_sidecar/common/grpc_sidecar.cc index 74d7ed7d..ccf9cd27 100644 --- a/services/inference_sidecar/common/grpc_sidecar.cc +++ b/services/inference_sidecar/common/grpc_sidecar.cc @@ -28,6 +28,7 @@ #include "absl/log/absl_log.h" #include "absl/log/check.h" #include "absl/status/status.h" +#include "absl/strings/str_cat.h" #include "absl/time/clock.h" #include "modules/module_interface.h" #include "proto/inference_sidecar.grpc.pb.h" @@ -75,12 +76,20 @@ class InferenceServiceImpl final : public InferenceService::Service { } // namespace -absl::Status Run() { +absl::Status Run(const InferenceSidecarRuntimeConfig& config) { SandboxWorker worker; grpc::ServerBuilder builder; + // Set the max receive size to unlimited; The default max size is only 4MB. + builder.SetMaxReceiveMessageSize(-1); + if (!config.module_name().empty() && + config.module_name() != ModuleInterface::GetModuleVersion()) { + return absl::FailedPreconditionError( + absl::StrCat("Expected inference module: ", config.module_name(), + ", but got : ", ModuleInterface::GetModuleVersion())); + } auto server_impl = - std::make_unique(ModuleInterface::Create()); + std::make_unique(ModuleInterface::Create(config)); builder.RegisterService(server_impl.get()); std::unique_ptr server(builder.BuildAndStart()); if (server == nullptr) { diff --git a/services/inference_sidecar/common/grpc_sidecar.h b/services/inference_sidecar/common/grpc_sidecar.h index 7d9c1960..2629d584 100644 --- a/services/inference_sidecar/common/grpc_sidecar.h +++ b/services/inference_sidecar/common/grpc_sidecar.h @@ -16,11 +16,12 @@ #define SERVICES_INFERENCE_SIDECAR_COMMON_GRPC_SIDECAR_H_ #include "absl/status/status.h" +#include "proto/inference_sidecar.pb.h" namespace privacy_sandbox::bidding_auction_servers::inference { // Runs a simple gRPC server. It is thread safe. -absl::Status Run(); +absl::Status Run(const InferenceSidecarRuntimeConfig& config); } // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/inference_sidecar/common/inference_sidecar_main.cc b/services/inference_sidecar/common/inference_sidecar_main.cc index daaa1940..b19dcf7a 100644 --- a/services/inference_sidecar/common/inference_sidecar_main.cc +++ b/services/inference_sidecar/common/inference_sidecar_main.cc @@ -14,12 +14,21 @@ // Inference sidecar binary. +#include + #include "absl/log/check.h" +#include "absl/status/status.h" +#include "proto/inference_sidecar.pb.h" #include "grpc_sidecar.h" int main(int argc, char** argv) { - CHECK(!privacy_sandbox::bidding_auction_servers::inference::Run().ok()) + privacy_sandbox::bidding_auction_servers::inference:: + InferenceSidecarRuntimeConfig config; + CHECK(google::protobuf::util::JsonStringToMessage(argv[0], &config).ok()) + << "Could not parse inference sidecar runtime config JsonString to a " + "proto message."; + CHECK(privacy_sandbox::bidding_auction_servers::inference::Run(config).ok()) << "Unsuccessful run of the inference sidecar."; return 0; } diff --git a/services/inference_sidecar/common/inference_sidecar_test.cc b/services/inference_sidecar/common/inference_sidecar_test.cc index d4003c29..3c76b543 100644 --- a/services/inference_sidecar/common/inference_sidecar_test.cc +++ b/services/inference_sidecar/common/inference_sidecar_test.cc @@ -27,6 +27,8 @@ #include "sandbox/sandbox_executor.h" #include "utils/file_util.h" +#include "test_constants.h" + namespace privacy_sandbox::bidding_auction_servers::inference { namespace { @@ -53,7 +55,8 @@ TEST(InferenceSidecarTest, RegisterModelAndRunInference_Grpc) { absl::FlagSaver flag_saver; absl::SetFlag(&FLAGS_testonly_allow_policies_for_bazel, true); - SandboxExecutor executor(kInferenceSidecarBinary, {""}); + SandboxExecutor executor(kInferenceSidecarBinary, + {std::string(kRuntimeConfig)}); ASSERT_EQ(executor.StartSandboxee().code(), absl::StatusCode::kOk); std::shared_ptr client_channel = diff --git a/services/inference_sidecar/common/ipc_sidecar.cc b/services/inference_sidecar/common/ipc_sidecar.cc index d0e54b9b..c3e59a55 100644 --- a/services/inference_sidecar/common/ipc_sidecar.cc +++ b/services/inference_sidecar/common/ipc_sidecar.cc @@ -20,6 +20,7 @@ #include "absl/log/absl_log.h" #include "absl/log/check.h" +#include "absl/strings/str_cat.h" #include "absl/time/clock.h" #include "absl/time/time.h" #include "modules/module_interface.h" @@ -38,11 +39,18 @@ constexpr int kRecvNumTrials = 10; } // namespace -absl::Status Run() { +absl::Status Run(const InferenceSidecarRuntimeConfig& config) { SandboxWorker worker; sandbox2::Comms comms(worker.FileDescriptor()); - std::unique_ptr inference_module = ModuleInterface::Create(); + if (!config.module_name().empty() && + config.module_name() != ModuleInterface::GetModuleVersion()) { + return absl::FailedPreconditionError( + absl::StrCat("Expected inference module: ", config.module_name(), + ", but got : ", ModuleInterface::GetModuleVersion())); + } + std::unique_ptr inference_module = + ModuleInterface::Create(config); // TODO(b/322109220): Handles arbitrary request proto simultaneously. // TODO(b/325123788): Remove retry logic with gRPC over IPC. diff --git a/services/inference_sidecar/common/ipc_sidecar.h b/services/inference_sidecar/common/ipc_sidecar.h index 1d81f590..f6e1e6b5 100644 --- a/services/inference_sidecar/common/ipc_sidecar.h +++ b/services/inference_sidecar/common/ipc_sidecar.h @@ -16,11 +16,12 @@ #define SERVICES_INFERENCE_SIDECAR_COMMON_IPC_SIDECAR_H_ #include "absl/status/status.h" +#include "proto/inference_sidecar.pb.h" namespace privacy_sandbox::bidding_auction_servers::inference { // Runs a simple IPC single-threaded processor. -absl::Status Run(); +absl::Status Run(const InferenceSidecarRuntimeConfig& config); } // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/inference_sidecar/common/modules/module_concurrency_test.cc b/services/inference_sidecar/common/modules/module_concurrency_test.cc index 060b4db5..475042bd 100644 --- a/services/inference_sidecar/common/modules/module_concurrency_test.cc +++ b/services/inference_sidecar/common/modules/module_concurrency_test.cc @@ -74,7 +74,8 @@ RegisterModelRequest CreateNewRegisterRequest( } TEST(ModuleConcurrencyTest, RegisterModel_Success) { - std::unique_ptr module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr module = ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_TRUE( PopulateRegisterModelRequest(kTestModelPath, register_request).ok()); @@ -99,7 +100,8 @@ TEST(ModuleConcurrencyTest, RegisterModel_Success) { } TEST(ModuleConcurrencyTest, Predict_Success) { - std::unique_ptr module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr module = ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_TRUE( PopulateRegisterModelRequest(kTestModelPath, register_request).ok()); @@ -121,7 +123,8 @@ TEST(ModuleConcurrencyTest, Predict_Success) { } TEST(ModuleConcurrencyTest, RegisterModel_Predict_Success) { - std::unique_ptr module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr module = ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_TRUE( PopulateRegisterModelRequest(kTestModelPath, register_request).ok()); diff --git a/services/inference_sidecar/common/modules/module_interface.h b/services/inference_sidecar/common/modules/module_interface.h index ab49382d..65210e05 100644 --- a/services/inference_sidecar/common/modules/module_interface.h +++ b/services/inference_sidecar/common/modules/module_interface.h @@ -29,9 +29,11 @@ class ModuleInterface { // Different implementations of ModuleInterface return specialized modules, // for example, Tensorflow module and PyTorch module. These specialized // modules can only be called with the methods defined by this interface. - static std::unique_ptr Create(); + static std::unique_ptr Create( + const InferenceSidecarRuntimeConfig& config); + // Gets inference backend module version. + static absl::string_view GetModuleVersion(); virtual ~ModuleInterface() = default; - // Executes inference on a registered ML model. virtual absl::StatusOr Predict( const PredictRequest& request) = 0; diff --git a/services/inference_sidecar/common/modules/test_module.cc b/services/inference_sidecar/common/modules/test_module.cc index 813d70e3..b4ef9ba3 100644 --- a/services/inference_sidecar/common/modules/test_module.cc +++ b/services/inference_sidecar/common/modules/test_module.cc @@ -75,8 +75,11 @@ absl::StatusOr TestModule::RegisterModel( return response; } -std::unique_ptr ModuleInterface::Create() { - return std::make_unique(); +std::unique_ptr ModuleInterface::Create( + const InferenceSidecarRuntimeConfig& config) { + return std::make_unique(config); } +absl::string_view ModuleInterface::GetModuleVersion() { return "test"; } + } // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/inference_sidecar/common/modules/test_module.h b/services/inference_sidecar/common/modules/test_module.h index 3aaaa604..5512b244 100644 --- a/services/inference_sidecar/common/modules/test_module.h +++ b/services/inference_sidecar/common/modules/test_module.h @@ -34,7 +34,8 @@ namespace privacy_sandbox::bidding_auction_servers::inference { class TestModule final : public ModuleInterface { public: // The constructor maps the embedded model file into memory. - TestModule() = default; + explicit TestModule(const InferenceSidecarRuntimeConfig& config) + : config_(config) {} ~TestModule() override; absl::StatusOr Predict( @@ -55,6 +56,7 @@ class TestModule final : public ModuleInterface { void* model_ptr_ = nullptr; // The model path. std::string model_path_ = ""; + const InferenceSidecarRuntimeConfig config_; }; } // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/inference_sidecar/common/modules/test_module_test.cc b/services/inference_sidecar/common/modules/test_module_test.cc index db789a16..5fb9b87d 100644 --- a/services/inference_sidecar/common/modules/test_module_test.cc +++ b/services/inference_sidecar/common/modules/test_module_test.cc @@ -28,21 +28,24 @@ constexpr absl::string_view kModelPath = "__main__/testdata/models/tensorflow_1_mib_saved_model.pb"; TEST(TestModule, Success_Predict) { - std::unique_ptr module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr module = ModuleInterface::Create(config); PredictRequest request; auto result = module->Predict(request); EXPECT_TRUE(result.ok()); } TEST(TestModule, Success_RegisterModel) { - std::unique_ptr module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr module = ModuleInterface::Create(config); RegisterModelRequest request; auto result = module->RegisterModel(request); EXPECT_TRUE(result.ok()); } TEST(TestModule, Success_ReadModel) { - std::unique_ptr module = std::make_unique(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr module = std::make_unique(config); module->set_model_path(kModelPath); RegisterModelRequest request; auto result = module->RegisterModel(request); diff --git a/services/inference_sidecar/common/proto/inference_payload.proto b/services/inference_sidecar/common/proto/inference_payload.proto index e1ceb74d..4e9c1c74 100644 --- a/services/inference_sidecar/common/proto/inference_payload.proto +++ b/services/inference_sidecar/common/proto/inference_payload.proto @@ -60,13 +60,15 @@ message Tensor { // should be equal to the product of tensor_shape elements, for example // a tensor of shape [1,4] will expect a flat array or 4 elements // (e.g. [1, 2, 7, 4]) and one with a shape [2,3] will expect a 6 element one. - bytes tensor_content = 4; + repeated string tensor_content = 4; } // Supported tensor data types. enum DataType { FLOAT = 0; // 32-bit floating point - INT64 = 1; // 64-bit integer (signed) - INT32 = 2; // 32-bit integer (signed) - DOUBLE = 3; // 64-bit floating point + DOUBLE = 1; // 64-bit floating point + INT8= 2; // 8-bit integer (signed) + INT16 = 3; // 16-bit integer (signed) + INT32 = 4; // 32-bit integer (signed) + INT64 = 5; // 64-bit integer (signed) } diff --git a/services/inference_sidecar/common/proto/inference_sidecar.proto b/services/inference_sidecar/common/proto/inference_sidecar.proto index 048e7daf..465e09e1 100644 --- a/services/inference_sidecar/common/proto/inference_sidecar.proto +++ b/services/inference_sidecar/common/proto/inference_sidecar.proto @@ -52,3 +52,20 @@ message RegisterModelRequest { message RegisterModelResponse { } + +message InferenceSidecarRuntimeConfig { + // The following two parameters control the threading behavior of the + // inference backend. + // For Tensorflow, they are configured at the session level. + // For PyTorch, they are configured the process level. + + // Specifies the number of threads for parallelizing individual operations. + int32 num_interop_threads = 1; + // Specifies the number of threads for parallelizing the execution within a + // single operation. + int32 num_intraop_threads = 2; + + // Specifies the inference backend module required. + // Currently supports "test", "tensorflow_v2_14_0", "pytorch_v2_1_1" + string module_name = 3; +} diff --git a/services/inference_sidecar/common/sandbox/sandbox_executor.cc b/services/inference_sidecar/common/sandbox/sandbox_executor.cc index 5d3686c3..d2d0f0fc 100644 --- a/services/inference_sidecar/common/sandbox/sandbox_executor.cc +++ b/services/inference_sidecar/common/sandbox/sandbox_executor.cc @@ -133,6 +133,11 @@ void AllowGrpc(sandbox2::PolicyBuilder& builder) { {ARG_32(2) /*sig*/, JEQ32(SIGABRT, ALLOW)}); } +void AllowAws(sandbox2::PolicyBuilder& builder) { + // TODO(b/333559278): Revisit the need for the following permissions. + builder.AllowSyscall(__NR_getsockopt); +} + void AllowPyTorch(sandbox2::PolicyBuilder& builder) { // TODO(b/323601668): Rewrite the PyTorch sidecar in order not to call // sysinfo. @@ -158,6 +163,7 @@ std::unique_ptr MakePolicy() { } AllowSamePolicyAsRoma(builder); AllowGrpc(builder); + AllowAws(builder); AllowPyTorch(builder); AllowTensorFlow(builder); builder.AllowStaticStartup().AllowLogForwarding(); diff --git a/services/inference_sidecar/common/test_constants.h b/services/inference_sidecar/common/test_constants.h new file mode 100644 index 00000000..81b0cede --- /dev/null +++ b/services/inference_sidecar/common/test_constants.h @@ -0,0 +1,29 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SERVICES_INFERENCE_SIDECAR_COMMON_TEST_CONSTANTS_H_ +#define SERVICES_INFERENCE_SIDECAR_COMMON_TEST_CONSTANTS_H_ + +#include "absl/strings/string_view.h" + +namespace privacy_sandbox::bidding_auction_servers::inference { + +constexpr absl::string_view kRuntimeConfig = + R"json({"num_interop_threads": 4, "num_intraop_threads": 5, "module_name": "test"})json"; + +} // namespace privacy_sandbox::bidding_auction_servers::inference + +#endif // SERVICES_BIDDING_SERVICE_CONSTANTS_H_ diff --git a/services/inference_sidecar/common/tools/debug/start_inference b/services/inference_sidecar/common/tools/debug/start_inference index b5026451..287e56d5 100755 --- a/services/inference_sidecar/common/tools/debug/start_inference +++ b/services/inference_sidecar/common/tools/debug/start_inference @@ -112,6 +112,11 @@ export SERVER_START_CMD=$(cat << END /server \ --inference_sidecar_binary_path="/inference_sidecar" \ --inference_model_local_paths="/test_model" \ +--inference_sidecar_runtime_config='{ + "num_interop_threads": 4, + "num_intraop_threads": 4 + "module_name": "tensorflow_v2_14_0" + }' \ --enable_bidding_service_benchmark="true" \ --init_config_client="false" --port=50057 \ --js_num_workers=4 \ diff --git a/services/inference_sidecar/common/utils/BUILD b/services/inference_sidecar/common/utils/BUILD index dfed33bb..bcd40b1e 100644 --- a/services/inference_sidecar/common/utils/BUILD +++ b/services/inference_sidecar/common/utils/BUILD @@ -26,6 +26,7 @@ cc_library( "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", + "@com_google_absl//absl/synchronization", ], ) diff --git a/services/inference_sidecar/common/utils/file_util.cc b/services/inference_sidecar/common/utils/file_util.cc index 57663e95..63dd43bf 100644 --- a/services/inference_sidecar/common/utils/file_util.cc +++ b/services/inference_sidecar/common/utils/file_util.cc @@ -22,11 +22,15 @@ #include "absl/base/log_severity.h" #include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" +#include "absl/synchronization/mutex.h" namespace privacy_sandbox::bidding_auction_servers::inference { namespace { absl::StatusOr ReadFile(absl::string_view path) { + static absl::Mutex mutex; + absl::MutexLock lock(&mutex); + std::ifstream ifs(path.data(), std::ios::in | std::ios::binary); if (!ifs.good()) { return absl::InvalidArgumentError( diff --git a/services/inference_sidecar/modules/pytorch_v2_1_1/BUILD b/services/inference_sidecar/modules/pytorch_v2_1_1/BUILD index 080fd9a6..55c60dff 100644 --- a/services/inference_sidecar/modules/pytorch_v2_1_1/BUILD +++ b/services/inference_sidecar/modules/pytorch_v2_1_1/BUILD @@ -14,6 +14,8 @@ load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") +package(default_visibility = ["//visibility:public"]) + cc_binary( name = "inference_sidecar_bin", srcs = ["@inference_common//:inference_sidecar_main.cc"], @@ -52,6 +54,7 @@ cc_test( ], flaky = True, deps = [ + ":test_constants", "@com_github_grpc_grpc//:grpc++", "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/flags:reflection", @@ -67,6 +70,15 @@ cc_test( ], ) +cc_library( + name = "test_constants", + testonly = True, + hdrs = ["test_constants.h"], + deps = [ + "@com_google_absl//absl/strings", + ], +) + cc_library( name = "pytorch", srcs = ["pytorch.cc"], diff --git a/services/inference_sidecar/modules/pytorch_v2_1_1/benchmark/BUILD b/services/inference_sidecar/modules/pytorch_v2_1_1/benchmark/BUILD new file mode 100644 index 00000000..e3a138ac --- /dev/null +++ b/services/inference_sidecar/modules/pytorch_v2_1_1/benchmark/BUILD @@ -0,0 +1,34 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_cc//cc:defs.bzl", "cc_binary") + +cc_binary( + name = "module_benchmark", + srcs = ["@inference_common//benchmark:module_benchmark.cc"], + data = [ + "//:test_model_target", + ], + deps = [ + "//:pytorch", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_benchmark//:benchmark", + "@com_google_benchmark//:benchmark_main", + "@inference_common//modules:module_interface", + "@inference_common//proto:inference_sidecar_cc_proto", + "@inference_common//utils:file_util", + ], +) diff --git a/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch.cc b/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch.cc index f05e655b..4c85870c 100644 --- a/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch.cc +++ b/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch.cc @@ -17,7 +17,7 @@ #include #include -#include +#include #include "absl/base/thread_annotations.h" #include "absl/container/flat_hash_map.h" @@ -71,8 +71,37 @@ absl::StatusOr PredictInternal(torch::jit::script::Module* model, } } +// Initializes PyTorch runtime inter-operations and intra-operations parallelism +// threading configurations. +absl::Status InitRuntimeThreadConfig( + const InferenceSidecarRuntimeConfig& config) { + try { + // `set_num_interop_threads` needs to be called before `set_num_threads`. + if (config.num_interop_threads() != 0) { + at::set_num_interop_threads(config.num_interop_threads()); + } + if (config.num_intraop_threads() != 0) { + at::set_num_threads(config.num_intraop_threads()); + } + } catch (const std::exception& e) { + return absl::InternalError(absl::StrCat( + "Error setting PyTorch threading runtime configs: ", e.what())); + } catch (...) { + return absl::InternalError( + "Unknown error during while setting PyTorch threading runtime " + "config."); + } + return absl::OkStatus(); +} + class PyTorchModule final : public ModuleInterface { public: + explicit PyTorchModule(const InferenceSidecarRuntimeConfig& config) { + absl::Status init_result = InitRuntimeThreadConfig(config); + CHECK(init_result.ok()) + << "Could not initialize runtime flags: " << init_result; + } + absl::StatusOr Predict( const PredictRequest& request) override ABSL_LOCKS_EXCLUDED(mu_); absl::StatusOr RegisterModel( @@ -168,8 +197,13 @@ absl::StatusOr PyTorchModule::RegisterModel( } // namespace -std::unique_ptr ModuleInterface::Create() { - return std::make_unique(); +std::unique_ptr ModuleInterface::Create( + const InferenceSidecarRuntimeConfig& config) { + return std::make_unique(config); +} + +absl::string_view ModuleInterface::GetModuleVersion() { + return "pytorch_v2_1_1"; } } // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch_test.cc b/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch_test.cc index c3a0800b..928d861b 100644 --- a/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch_test.cc +++ b/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch_test.cc @@ -39,9 +39,40 @@ constexpr absl::string_view kTestModelMixedInputsMixedOutputs = "mixed_inputs_mixed_outputs_model"; constexpr int kNumThreads = 100; +TEST(PyTorchModuleRuntimeConfigTest, + RuntimeConfigInitializationSucceeds_Empty) { + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); + + // PyTorch runtime threading configuration is set at the process level. + std::thread t([]() { + EXPECT_NE(at::get_num_threads(), 0); + EXPECT_NE(at::get_num_interop_threads(), 0); + }); + t.join(); +} + +TEST(PyTorchModuleRuntimeConfigTest, RuntimeConfigInitializationSucceeds) { + InferenceSidecarRuntimeConfig config; + config.set_num_intraop_threads(4); + config.set_num_interop_threads(5); + std::unique_ptr torch_module = + ModuleInterface::Create(config); + + // PyTorch runtime threading configuration is set at the process level. + std::thread t([]() { + EXPECT_EQ(at::get_num_threads(), 4); + EXPECT_EQ(at::get_num_interop_threads(), 5); + }); + t.join(); +} + TEST(PyTorchModuleRegisterModelTest, RegisterModelWithEmtpyKeyReturnsInvalidArgument) { - std::unique_ptr torch_module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_EQ(torch_module->RegisterModel(register_request).status().code(), absl::StatusCode::kInvalidArgument); @@ -49,7 +80,9 @@ TEST(PyTorchModuleRegisterModelTest, TEST(PyTorchModuleRegisterModelTest, RegisterModelWithExistingKeyReturnsAlreadyExists) { - std::unique_ptr torch_module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_TRUE( PopulateRegisterModelRequest(kSimpleModel, register_request).ok()); @@ -60,7 +93,9 @@ TEST(PyTorchModuleRegisterModelTest, TEST(PyTorchModuleRegisterModelTest, RegisterModelWithNoModelContentReturnsInvalidArgument) { - std::unique_ptr torch_module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); RegisterModelRequest register_request; register_request.mutable_model_spec()->set_model_path("e2e_model1"); EXPECT_EQ(torch_module->RegisterModel(register_request).status().code(), @@ -68,7 +103,9 @@ TEST(PyTorchModuleRegisterModelTest, } TEST(PyTorchModuleRegisterModelTest, RegisterModelOk) { - std::unique_ptr torch_module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_TRUE( PopulateRegisterModelRequest(kSimpleModel, register_request).ok()); @@ -77,7 +114,9 @@ TEST(PyTorchModuleRegisterModelTest, RegisterModelOk) { TEST(PyTorchModulePredictTest, PredictWithoutValidBatchRequestInputReturnsInvalidArgument) { - std::unique_ptr torch_module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_TRUE( PopulateRegisterModelRequest(kSimpleModel, register_request).ok()); @@ -110,7 +149,9 @@ constexpr char kInvalidTensorContentRequest[] = R"json({ TEST(PyTorchModulePredictTest, PredictWithInvalidTensorContentReturnsInvalidArgument) { - std::unique_ptr torch_module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_TRUE( PopulateRegisterModelRequest(kSimpleModel, register_request).ok()); @@ -141,7 +182,9 @@ constexpr char kSimpleRequest[] = R"json({ })json"; TEST(PyTorchModulePredictTest, PredictWithoutValidModelReturnsNotFound) { - std::unique_ptr torch_module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_EQ(torch_module->RegisterModel(register_request).status().code(), absl::StatusCode::kInvalidArgument); @@ -156,7 +199,9 @@ TEST(PyTorchModulePredictTest, PredictWithoutValidModelReturnsNotFound) { } TEST(PyTorchModulePredictTest, PredictSimpleSuccess) { - std::unique_ptr torch_module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_TRUE( PopulateRegisterModelRequest(kSimpleModel, register_request).ok()); @@ -190,7 +235,9 @@ constexpr char kNotRegisteredModelRequest[] = R"json({ })json"; TEST(PyTorchModulePredictTest, PredictReturnsNotFoundError) { - std::unique_ptr torch_module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_TRUE( PopulateRegisterModelRequest(kTestModelVariedInputs1, register_request) @@ -223,7 +270,9 @@ constexpr char kMismatchedInput[] = R"json({ })json"; TEST(PyTorchModulePredictTest, PredictReturnsInternalError) { - std::unique_ptr torch_module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_TRUE( PopulateRegisterModelRequest(kTestModelVariedInputs1, register_request) @@ -258,7 +307,9 @@ constexpr char kSimpleRequest2Elements[] = R"json({ })json"; TEST(PyTorchModulePredictTest, PredictSimpleSuccessShape1x2) { - std::unique_ptr torch_module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_TRUE( PopulateRegisterModelRequest(kSimpleModel, register_request).ok()); @@ -291,7 +342,9 @@ constexpr char kSimpleRequestBatchSize2[] = R"json({ })json"; TEST(PyTorchModulePredictTest, PredictSimpleBatchSize2Success) { - std::unique_ptr torch_module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_TRUE( PopulateRegisterModelRequest(kSimpleModel, register_request).ok()); @@ -338,7 +391,9 @@ constexpr char kSameModelSameBatchSizeMultipleRequests[] = R"json({ TEST(PyTorchModulePredictTest, PredictSameModelSameBatchSizeMultipleRequestsSuccess) { - std::unique_ptr torch_module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_TRUE( PopulateRegisterModelRequest(kSimpleModel, register_request).ok()); @@ -387,7 +442,9 @@ constexpr char kSameModelVariedBatchSizesMultipleRequests[] = R"json({ TEST(PyTorchModulePredictTest, PredictSameModelVariedBatchSizesMultipleRequestsSuccess) { - std::unique_ptr torch_module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_TRUE( PopulateRegisterModelRequest(kSimpleModel, register_request).ok()); @@ -435,7 +492,9 @@ constexpr char kRequestsWithMultipleInvalidInputs[] = R"json({ })json"; TEST(PyTorchModulePredictTest, PredictMutilpleInvalidInputsReturnsFirstError) { - std::unique_ptr torch_module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_TRUE( PopulateRegisterModelRequest(kSimpleModel, register_request).ok()); @@ -479,7 +538,9 @@ constexpr char kBothValidAndInvalidInputs[] = R"json({ })json"; TEST(PyTorchModulePredictTest, PredictBothValidAndInvalidInputsReturnsError) { - std::unique_ptr torch_module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_TRUE( PopulateRegisterModelRequest(kSimpleModel, register_request).ok()); @@ -560,7 +621,9 @@ constexpr char kVariedInputsRequestBatchSize1[] = R"json({ })json"; TEST(PyTorchModulePredictTest, PredictVariedInputsBatchSize1Success) { - std::unique_ptr torch_module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_TRUE( PopulateRegisterModelRequest(kTestModelVariedInputs1, register_request) @@ -652,7 +715,9 @@ constexpr char kVariedInputsRequestBatchSize2[] = R"json({ })json"; TEST(PyTorchModulePredictTest, PredictVariedInputsBatchSize2Success) { - std::unique_ptr torch_module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_TRUE( PopulateRegisterModelRequest(kTestModelVariedInputs1, register_request) @@ -694,7 +759,9 @@ constexpr char kMixedInputsBatchSize1[] = R"json({ })json"; TEST(PyTorchModulePredictTest, PredictMixedInputsMixedOutputsSuccess) { - std::unique_ptr torch_module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_TRUE(PopulateRegisterModelRequest(kTestModelMixedInputsMixedOutputs, register_request) @@ -860,7 +927,9 @@ constexpr char kVariedInputsMultipleModelsRequest[] = R"json({ TEST(PyTorchModulePredictTest, PredictVariedInputsMultipleModelsBatchSize1Success) { - std::unique_ptr torch_module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); RegisterModelRequest register_request1; ASSERT_TRUE( PopulateRegisterModelRequest(kTestModelVariedInputs1, register_request1) @@ -888,7 +957,9 @@ TEST(PyTorchModulePredictTest, } TEST(PyTorchModuleConcurrencyTest, RegisterSameModelWithMultipleThreads) { - std::unique_ptr torch_module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_TRUE( PopulateRegisterModelRequest(kSimpleModel, register_request).ok()); @@ -922,7 +993,9 @@ TEST(PyTorchModuleConcurrencyTest, RegisterSameModelWithMultipleThreads) { TEST(PyTorchModuleConcurrencyTest, ThreadsMakingInferenceRequestAgainstSingleModel) { - std::unique_ptr torch_module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_TRUE( PopulateRegisterModelRequest(kSimpleModel, register_request).ok()); @@ -954,7 +1027,9 @@ TEST(PyTorchModuleConcurrencyTest, TEST(PyTorchModuleConcurrencyTest, ThreadsMakingInferenceRequestAgainstMultipleModels) { - std::unique_ptr torch_module = ModuleInterface::Create(); + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); std::thread register_request_thread1([&torch_module]() { RegisterModelRequest register_request1; diff --git a/services/inference_sidecar/modules/pytorch_v2_1_1/test_constants.h b/services/inference_sidecar/modules/pytorch_v2_1_1/test_constants.h new file mode 100644 index 00000000..5a98cd58 --- /dev/null +++ b/services/inference_sidecar/modules/pytorch_v2_1_1/test_constants.h @@ -0,0 +1,29 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SERVICES_INFERENCE_SIDECAR_COMMON_TEST_CONSTANTS_H_ +#define SERVICES_INFERENCE_SIDECAR_COMMON_TEST_CONSTANTS_H_ + +#include "absl/strings/string_view.h" + +namespace privacy_sandbox::bidding_auction_servers::inference { + +constexpr absl::string_view kRuntimeConfig = + R"json({"num_interop_threads": 4, "num_intraop_threads": 5, "module_name": "pytorch_v2_1_1"})json"; + +} // namespace privacy_sandbox::bidding_auction_servers::inference + +#endif // SERVICES_BIDDING_SERVICE_CONSTANTS_H_ diff --git a/services/inference_sidecar/modules/tensorflow_v2_14_0/BUILD b/services/inference_sidecar/modules/tensorflow_v2_14_0/BUILD index fbe5d5d6..1668224b 100644 --- a/services/inference_sidecar/modules/tensorflow_v2_14_0/BUILD +++ b/services/inference_sidecar/modules/tensorflow_v2_14_0/BUILD @@ -21,6 +21,8 @@ load( ) load("@rules_pkg//pkg:zip.bzl", "pkg_zip") +package(default_visibility = ["//visibility:public"]) + cc_binary( name = "inference_sidecar_bin", srcs = ["@inference_common//:inference_sidecar_main.cc"], @@ -71,6 +73,7 @@ cc_test( ], flaky = True, deps = [ + ":test_constants", "@com_github_grpc_grpc//:grpc++", "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/flags:reflection", @@ -86,6 +89,15 @@ cc_test( ], ) +cc_library( + name = "test_constants", + testonly = True, + hdrs = ["test_constants.h"], + deps = [ + "@com_google_absl//absl/strings", + ], +) + cc_library( name = "tensorflow_parser", srcs = ["tensorflow_parser.cc"], diff --git a/services/inference_sidecar/modules/tensorflow_v2_14_0/benchmark/BUILD b/services/inference_sidecar/modules/tensorflow_v2_14_0/benchmark/BUILD new file mode 100644 index 00000000..ceb88edd --- /dev/null +++ b/services/inference_sidecar/modules/tensorflow_v2_14_0/benchmark/BUILD @@ -0,0 +1,34 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_cc//cc:defs.bzl", "cc_binary") + +cc_binary( + name = "module_benchmark", + srcs = ["@inference_common//benchmark:module_benchmark.cc"], + data = [ + "//:test_model_target", + ], + deps = [ + "//:tensorflow", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_benchmark//:benchmark", + "@com_google_benchmark//:benchmark_main", + "@inference_common//modules:module_interface", + "@inference_common//proto:inference_sidecar_cc_proto", + "@inference_common//utils:file_util", + ], +) diff --git a/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow.cc b/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow.cc index c5c4fac6..f1ef5a49 100644 --- a/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow.cc +++ b/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow.cc @@ -61,7 +61,7 @@ absl::Status SaveToRamFileSystem(const RegisterModelRequest& request) { return absl::OkStatus(); } -absl::StatusOr>> +absl::StatusOr>> PredictPerModel(const tensorflow::SavedModelBundle* model, const InferenceRequest& inference_request) { std::vector> inputs; @@ -98,12 +98,23 @@ PredictPerModel(const tensorflow::SavedModelBundle* model, return absl::InternalError(absl::StrCat( "Inference failed for model '", model_key, "': ", status.ToString())); } + std::vector zipped_vector; + if (output_names.size() != outputs.size()) { + return absl::InternalError( + "The number of output tensors doesn't match the number of output " + "tensor names"); + } + for (size_t i = 0; i < output_names.size(); ++i) { + zipped_vector.push_back(TensorWithName(output_names[i], outputs[i])); + } - return std::make_pair(std::string(model_key), outputs); + return std::make_pair(std::string(model_key), zipped_vector); } class TensorflowModule final : public ModuleInterface { public: + explicit TensorflowModule(const InferenceSidecarRuntimeConfig& config) + : runtime_config_(config) {} absl::StatusOr Predict( const PredictRequest& request) override ABSL_LOCKS_EXCLUDED(mu_); absl::StatusOr RegisterModel( @@ -117,6 +128,7 @@ class TensorflowModule final : public ModuleInterface { model_map_ ABSL_GUARDED_BY(mu_); // TODO(b/327907675) : Add a test for concurrency absl::Mutex mu_; + const InferenceSidecarRuntimeConfig runtime_config_; }; absl::StatusOr TensorflowModule::Predict( @@ -130,7 +142,7 @@ absl::StatusOr TensorflowModule::Predict( } std::vector>>>> + absl::StatusOr>>>> tasks; for (const InferenceRequest& inference_request : *parsed_requests) { @@ -144,7 +156,7 @@ absl::StatusOr TensorflowModule::Predict( it->second.get(), inference_request)); } - std::vector>> + std::vector>> batch_outputs; for (size_t task_id = 0; task_id < tasks.size(); ++task_id) { auto result_status_or = tasks[task_id].get(); @@ -172,6 +184,16 @@ absl::StatusOr TensorflowModule::Predict( absl::StatusOr TensorflowModule::RegisterModel( const RegisterModelRequest& request) { tensorflow::SessionOptions session_options; + // TODO(b/332599154): Support runtime configuration on a per-session basis. + if (runtime_config_.num_intraop_threads() != 0) { + session_options.config.set_intra_op_parallelism_threads( + runtime_config_.num_intraop_threads()); + } + if (runtime_config_.num_interop_threads() != 0) { + session_options.config.set_inter_op_parallelism_threads( + runtime_config_.num_interop_threads()); + } + const std::unordered_set tags = {"serve"}; const auto& model_path = request.model_spec().model_path(); @@ -203,8 +225,13 @@ absl::StatusOr TensorflowModule::RegisterModel( } // namespace -std::unique_ptr ModuleInterface::Create() { - return std::make_unique(); +std::unique_ptr ModuleInterface::Create( + const InferenceSidecarRuntimeConfig& config) { + return std::make_unique(config); +} + +absl::string_view ModuleInterface::GetModuleVersion() { + return "tensorflow_v2_14_0"; } } // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow_parser.cc b/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow_parser.cc index 66b1f59f..b1026c06 100644 --- a/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow_parser.cc +++ b/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow_parser.cc @@ -163,10 +163,13 @@ absl::StatusOr SerializeJsonDoc(const rapidjson::Value& document) { // Converts a single tensor to json. absl::StatusOr TensorToJsonValue( - const tensorflow::Tensor& tensor, + const std::string& tensor_name, const tensorflow::Tensor& tensor, rapidjson::MemoryPoolAllocator<>& allocator) { rapidjson::Value json_tensor(rapidjson::kObjectType); + rapidjson::Value tensor_name_value; + tensor_name_value.SetString(tensor_name.c_str(), allocator); + json_tensor.AddMember("tensor_name", tensor_name_value, allocator); tensorflow::TensorShape tensor_shape = tensor.shape(); rapidjson::Value tensor_shape_json(rapidjson::kArrayType); for (int i = 0; i < tensor_shape.dims(); ++i) { @@ -236,26 +239,23 @@ absl::StatusOr TensorToJsonValue( } absl::StatusOr ConvertTensorsToJson( - const std::vector>>& + const std::vector>>& batch_outputs) { rapidjson::Document document; document.SetObject(); rapidjson::MemoryPoolAllocator<>& allocator = document.GetAllocator(); rapidjson::Value batch(rapidjson::kArrayType); - for (const auto& pair : batch_outputs) { - const std::string model_path = pair.first; - const std::vector tensors = pair.second; - + for (const auto& [model_path, tensors] : batch_outputs) { rapidjson::Value nested_object(rapidjson::kObjectType); rapidjson::Value model_path_value; model_path_value.SetString(model_path.c_str(), allocator); nested_object.AddMember("model_path", model_path_value, allocator); rapidjson::Value tensors_value(rapidjson::kArrayType); - for (const auto& tensor : tensors) { + for (const auto& [tensor_name, tensor] : tensors) { absl::StatusOr json = - TensorToJsonValue(tensor, allocator); + TensorToJsonValue(tensor_name, tensor, allocator); if (json.ok()) { tensors_value.PushBack(json.value(), allocator); } else { diff --git a/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow_parser.h b/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow_parser.h index 7f2280ef..4975f809 100644 --- a/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow_parser.h +++ b/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow_parser.h @@ -26,6 +26,16 @@ namespace privacy_sandbox::bidding_auction_servers::inference { +struct TensorWithName { + // tensorflow::Tensor doesn't hold the name of the tensor so we need to create + // a superset struct. + std::string tensor_name; + tensorflow::Tensor tensor; + + TensorWithName(std::string tensor_name, tensorflow::Tensor tensor) + : tensor_name(tensor_name), tensor(tensor) {} +}; + // Transforms an internal ML framework agnostic dense tensor representation // into a Tensorflow tensor. absl::StatusOr ConvertFlatArrayToTensor( @@ -35,7 +45,7 @@ absl::StatusOr ConvertFlatArrayToTensor( // a JSON string. // batch_outputs contains a collection of pairs. absl::StatusOr ConvertTensorsToJson( - const std::vector>>& + const std::vector>>& batch_outputs); } // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow_parser_test.cc b/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow_parser_test.cc index e1959208..44074add 100644 --- a/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow_parser_test.cc +++ b/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow_parser_test.cc @@ -144,9 +144,9 @@ TEST(TensorflowParserTest, ConvertTensorsToJson) { for (int i = 0; i < 6; ++i) { tensor_data(i) = i * 2.0f; // Set values (i * 2) } - std::vector tensors; - tensors.push_back(tensor); - std::vector>> + std::vector tensors; + tensors.push_back(TensorWithName("output", tensor)); + std::vector>> batch_tensors; batch_tensors.emplace_back("my_bucket/models/pcvr_models/1/", tensors); @@ -154,15 +154,15 @@ TEST(TensorflowParserTest, ConvertTensorsToJson) { EXPECT_TRUE(output.ok()) << output.status(); EXPECT_EQ( output.value(), - R"({"response":[{"model_path":"my_bucket/models/pcvr_models/1/","tensors":[{"tensor_shape":[2,3],"data_type":"FLOAT","tensor_content":[0.0,2.0,4.0,6.0,8.0,10.0]}]}]})"); + R"({"response":[{"model_path":"my_bucket/models/pcvr_models/1/","tensors":[{"tensor_name":"output","tensor_shape":[2,3],"data_type":"FLOAT","tensor_content":[0.0,2.0,4.0,6.0,8.0,10.0]}]}]})"); } TEST(TensorflowParserTest, ConvertTensorsToJson_UnsupportedFloat16Type) { tensorflow::Tensor half_tensor(tensorflow::DT_BFLOAT16); - std::vector tensors; - tensors.push_back(half_tensor); - std::vector>> + std::vector tensors; + tensors.push_back(TensorWithName("output", half_tensor)); + std::vector>> batch_tensors; batch_tensors.emplace_back("my_bucket/models/pcvr_models/1/", tensors); @@ -177,9 +177,9 @@ TEST(TensorflowParserTest, ConvertTensorsToJson_int8) { auto int8_data_ptr = int8_tensor.flat().data(); std::copy(int8_data.begin(), int8_data.end(), int8_data_ptr); - std::vector tensors; - tensors.push_back(int8_tensor); - std::vector>> + std::vector tensors; + tensors.push_back(TensorWithName("output", int8_tensor)); + std::vector>> batch_tensors; batch_tensors.emplace_back("my_bucket/models/pcvr_models/1/", tensors); @@ -187,7 +187,7 @@ TEST(TensorflowParserTest, ConvertTensorsToJson_int8) { EXPECT_TRUE(output.ok()); EXPECT_EQ( output.value(), - R"({"response":[{"model_path":"my_bucket/models/pcvr_models/1/","tensors":[{"tensor_shape":[2],"data_type":"INT8","tensor_content":[10,-5]}]}]})"); + R"({"response":[{"model_path":"my_bucket/models/pcvr_models/1/","tensors":[{"tensor_name":"output","tensor_shape":[2],"data_type":"INT8","tensor_content":[10,-5]}]}]})"); } TEST(TensorflowParserTest, ConvertTensorsToJson_int16) { @@ -196,9 +196,9 @@ TEST(TensorflowParserTest, ConvertTensorsToJson_int16) { auto int16_data_ptr = int16_tensor.flat().data(); std::copy(int16_data.begin(), int16_data.end(), int16_data_ptr); - std::vector tensors; - tensors.push_back(int16_tensor); - std::vector>> + std::vector tensors; + tensors.push_back(TensorWithName("output", int16_tensor)); + std::vector>> batch_tensors; batch_tensors.emplace_back("my_bucket/models/pcvr_models/1/", tensors); @@ -206,7 +206,7 @@ TEST(TensorflowParserTest, ConvertTensorsToJson_int16) { EXPECT_TRUE(output.ok()); EXPECT_EQ( output.value(), - R"({"response":[{"model_path":"my_bucket/models/pcvr_models/1/","tensors":[{"tensor_shape":[2],"data_type":"INT16","tensor_content":[10,-5]}]}]})"); + R"({"response":[{"model_path":"my_bucket/models/pcvr_models/1/","tensors":[{"tensor_name":"output","tensor_shape":[2],"data_type":"INT16","tensor_content":[10,-5]}]}]})"); } TEST(TensorflowParserTest, TestConvertTensorsToJson_MultipleTensors) { @@ -222,11 +222,11 @@ TEST(TensorflowParserTest, TestConvertTensorsToJson_MultipleTensors) { auto tensor_data_2 = tensor_2.flat(); tensor_data_2(0) = 1000; - std::vector tensors; - tensors.push_back(tensor_1); - tensors.push_back(tensor_2); + std::vector tensors; + tensors.push_back(TensorWithName("output1", tensor_1)); + tensors.push_back(TensorWithName("output2", tensor_2)); - std::vector>> + std::vector>> batch_tensors; batch_tensors.emplace_back("my_bucket/models/pcvr_models/1/", tensors); @@ -234,7 +234,7 @@ TEST(TensorflowParserTest, TestConvertTensorsToJson_MultipleTensors) { EXPECT_TRUE(output.ok()) << output.status(); EXPECT_EQ( output.value(), - R"({"response":[{"model_path":"my_bucket/models/pcvr_models/1/","tensors":[{"tensor_shape":[1,1],"data_type":"DOUBLE","tensor_content":[3.14]},{"tensor_shape":[1,1],"data_type":"INT64","tensor_content":[1000]}]}]})"); + R"({"response":[{"model_path":"my_bucket/models/pcvr_models/1/","tensors":[{"tensor_name":"output1","tensor_shape":[1,1],"data_type":"DOUBLE","tensor_content":[3.14]},{"tensor_name":"output2","tensor_shape":[1,1],"data_type":"INT64","tensor_content":[1000]}]}]})"); } TEST(TensorflowParserTest, TestConvertTensorsToJson_BatchOdModels) { @@ -243,18 +243,18 @@ TEST(TensorflowParserTest, TestConvertTensorsToJson_BatchOdModels) { tensorflow::Tensor tensor_1(tensorflow::DT_DOUBLE, shape_1); auto tensor_data_1 = tensor_1.flat(); tensor_data_1(0) = 3.14; - std::vector tensors1; - tensors1.push_back(tensor_1); + std::vector tensors1; + tensors1.push_back(TensorWithName("output1", tensor_1)); // Int64 tensor. tensorflow::TensorShape shape_2({1, 1}); tensorflow::Tensor tensor_2(tensorflow::DT_INT64, shape_2); auto tensor_data_2 = tensor_2.flat(); tensor_data_2(0) = 1000; - std::vector tensors2; - tensors2.push_back(tensor_2); + std::vector tensors2; + tensors2.push_back(TensorWithName("output2", tensor_2)); - std::vector>> + std::vector>> batch_tensors; batch_tensors.emplace_back("my_bucket/models/pcvr_models/1/", tensors1); batch_tensors.emplace_back("my_bucket/models/pctr_models/1/", tensors2); @@ -263,7 +263,7 @@ TEST(TensorflowParserTest, TestConvertTensorsToJson_BatchOdModels) { EXPECT_TRUE(output.ok()) << output.status(); EXPECT_EQ( output.value(), - R"({"response":[{"model_path":"my_bucket/models/pcvr_models/1/","tensors":[{"tensor_shape":[1,1],"data_type":"DOUBLE","tensor_content":[3.14]}]},{"model_path":"my_bucket/models/pctr_models/1/","tensors":[{"tensor_shape":[1,1],"data_type":"INT64","tensor_content":[1000]}]}]})"); + R"({"response":[{"model_path":"my_bucket/models/pcvr_models/1/","tensors":[{"tensor_name":"output1","tensor_shape":[1,1],"data_type":"DOUBLE","tensor_content":[3.14]}]},{"model_path":"my_bucket/models/pctr_models/1/","tensors":[{"tensor_name":"output2","tensor_shape":[1,1],"data_type":"INT64","tensor_content":[1000]}]}]})"); } } // namespace diff --git a/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow_test.cc b/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow_test.cc index ead05269..a749a989 100644 --- a/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow_test.cc +++ b/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow_test.cc @@ -32,8 +32,9 @@ constexpr absl::string_view kModel2Dir = "./benchmark_models/pctr"; constexpr absl::string_view kEmbeddingModelDir = "./benchmark_models/embedding"; TEST(TensorflowModuleTest, Failure_RegisterModelWithEmptyPath) { + InferenceSidecarRuntimeConfig config; std::unique_ptr tensorflow_module = - ModuleInterface::Create(); + ModuleInterface::Create(config); RegisterModelRequest register_request; register_request.mutable_model_spec()->set_model_path(""); absl::StatusOr status_or = @@ -45,8 +46,9 @@ TEST(TensorflowModuleTest, Failure_RegisterModelWithEmptyPath) { } TEST(TensorflowModuleTest, Failure_RegisterModelAlreadyRegistered) { + InferenceSidecarRuntimeConfig config; std::unique_ptr tensorflow_module = - ModuleInterface::Create(); + ModuleInterface::Create(config); // First model registration RegisterModelRequest register_request_1; @@ -71,8 +73,9 @@ TEST(TensorflowModuleTest, Failure_RegisterModelAlreadyRegistered) { } TEST(TensorflowModuleTest, Failure_RegisterModelLoadError) { + InferenceSidecarRuntimeConfig config; std::unique_ptr tensorflow_module = - ModuleInterface::Create(); + ModuleInterface::Create(config); // Setting the model path to a non-existent directory RegisterModelRequest register_request; @@ -89,8 +92,9 @@ TEST(TensorflowModuleTest, Failure_RegisterModelLoadError) { } TEST(TensorflowModuleTest, Success_RegisterModel) { + InferenceSidecarRuntimeConfig config; std::unique_ptr tensorflow_module = - ModuleInterface::Create(); + ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_TRUE(PopulateRegisterModelRequest(kModel1Dir, register_request).ok()); absl::StatusOr status_or = @@ -99,8 +103,9 @@ TEST(TensorflowModuleTest, Success_RegisterModel) { } TEST(TensorflowModuleTest, Failure_PredictInvalidJson) { + InferenceSidecarRuntimeConfig config; std::unique_ptr tensorflow_module = - ModuleInterface::Create(); + ModuleInterface::Create(config); PredictRequest predict_request; predict_request.set_input(""); @@ -177,8 +182,9 @@ constexpr char kJsonString[] = R"json({ })json"; TEST(TensorflowModuleTest, Failure_PredictModelNotRegistered) { + InferenceSidecarRuntimeConfig config; std::unique_ptr tensorflow_module = - ModuleInterface::Create(); + ModuleInterface::Create(config); PredictRequest predict_request; predict_request.set_input(kJsonString); @@ -191,8 +197,9 @@ TEST(TensorflowModuleTest, Failure_PredictModelNotRegistered) { } TEST(TensorflowModuleTest, Success_Predict) { + InferenceSidecarRuntimeConfig config; std::unique_ptr tensorflow_module = - ModuleInterface::Create(); + ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_TRUE(PopulateRegisterModelRequest(kModel1Dir, register_request).ok()); ASSERT_TRUE(tensorflow_module->RegisterModel(register_request).ok()); @@ -205,8 +212,9 @@ TEST(TensorflowModuleTest, Success_Predict) { ASSERT_FALSE(response.output().empty()); ASSERT_EQ(response.output(), "{\"response\":[{\"model_path\":\"./benchmark_models/" - "pcvr\",\"tensors\":[{\"tensor_shape\":[1,1],\"data_type\":" - "\"FLOAT\",\"tensor_content\":[0.019116628915071489]}]}]}"); + "pcvr\",\"tensors\":[{\"tensor_name\":\"StatefulPartitionedCall:" + "0\",\"tensor_shape\":[1,1],\"data_type\":\"FLOAT\",\"tensor_" + "content\":[0.019116628915071489]}]}]}"); } constexpr char kJsonStringBatchSize2[] = R"json({ @@ -274,8 +282,9 @@ constexpr char kJsonStringBatchSize2[] = R"json({ })json"; TEST(TensorflowModuleTest, Success_PredictBatchSize2) { + InferenceSidecarRuntimeConfig config; std::unique_ptr tensorflow_module = - ModuleInterface::Create(); + ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_TRUE(PopulateRegisterModelRequest(kModel1Dir, register_request).ok()); ASSERT_TRUE(tensorflow_module->RegisterModel(register_request).ok()); @@ -286,11 +295,11 @@ TEST(TensorflowModuleTest, Success_PredictBatchSize2) { ASSERT_TRUE(predict_status.ok()); PredictResponse response = predict_status.value(); ASSERT_FALSE(response.output().empty()); - ASSERT_EQ( - response.output(), - "{\"response\":[{\"model_path\":\"./benchmark_models/" - "pcvr\",\"tensors\":[{\"tensor_shape\":[2,1],\"data_type\":\"FLOAT\"," - "\"tensor_content\":[0.019116630777716638,0.1847093403339386]}]}]}"); + ASSERT_EQ(response.output(), + "{\"response\":[{\"model_path\":\"./benchmark_models/" + "pcvr\",\"tensors\":[{\"tensor_name\":\"StatefulPartitionedCall:" + "0\",\"tensor_shape\":[2,1],\"data_type\":\"FLOAT\",\"tensor_" + "content\":[0.019116630777716638,0.1847093403339386]}]}]}"); } constexpr char kJsonStringMissingTensorName[] = R"json({ @@ -309,8 +318,9 @@ constexpr char kJsonStringMissingTensorName[] = R"json({ })json"; TEST(TensorflowModuleTest, Failure_PredictMissingTensorName) { + InferenceSidecarRuntimeConfig config; std::unique_ptr tensorflow_module = - ModuleInterface::Create(); + ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_TRUE(PopulateRegisterModelRequest(kModel1Dir, register_request).ok()); ASSERT_TRUE(tensorflow_module->RegisterModel(register_request).ok()); @@ -451,8 +461,9 @@ constexpr char kJsonStringWith2Model[] = R"json({ })json"; TEST(TensorflowModuleTest, Success_PredictWith2Models) { + InferenceSidecarRuntimeConfig config; std::unique_ptr tensorflow_module = - ModuleInterface::Create(); + ModuleInterface::Create(config); RegisterModelRequest register_request_1; ASSERT_TRUE( PopulateRegisterModelRequest(kModel1Dir, register_request_1).ok()); @@ -469,14 +480,15 @@ TEST(TensorflowModuleTest, Success_PredictWith2Models) { ASSERT_TRUE(predict_status.ok()); PredictResponse response = predict_status.value(); ASSERT_FALSE(response.output().empty()); - EXPECT_EQ( - response.output(), - "{\"response\":[{\"model_path\":\"./benchmark_models/" - "pcvr\",\"tensors\":[{\"tensor_shape\":[2,1],\"data_type\":\"FLOAT\"," - "\"tensor_content\":[0.019116630777716638,0.1847093403339386]}]},{" - "\"model_path\":\"./benchmark_models/" - "pctr\",\"tensors\":[{\"tensor_shape\":[2,1],\"data_type\":\"FLOAT\"," - "\"tensor_content\":[0.14649735391139985,0.2522672712802887]}]}]}"); + EXPECT_EQ(response.output(), + "{\"response\":[{\"model_path\":\"./benchmark_models/" + "pcvr\",\"tensors\":[{\"tensor_name\":\"StatefulPartitionedCall:" + "0\",\"tensor_shape\":[2,1],\"data_type\":\"FLOAT\",\"tensor_" + "content\":[0.019116630777716638,0.1847093403339386]}]},{\"model_" + "path\":\"./benchmark_models/" + "pctr\",\"tensors\":[{\"tensor_name\":\"StatefulPartitionedCall:" + "0\",\"tensor_shape\":[2,1],\"data_type\":\"FLOAT\",\"tensor_" + "content\":[0.14649735391139985,0.2522672712802887]}]}]}"); } constexpr char kJsonStringWith1ModelVariedSize[] = R"json({ @@ -606,8 +618,9 @@ constexpr char kJsonStringWith1ModelVariedSize[] = R"json({ TEST(TensorflowModuleTest, PredictSameModelVariedBatchSizesMultipleRequestsSuccess) { + InferenceSidecarRuntimeConfig config; std::unique_ptr tensorflow_module = - ModuleInterface::Create(); + ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_TRUE( PopulateRegisterModelRequest(std::string(kModel1Dir), register_request) @@ -622,11 +635,13 @@ TEST(TensorflowModuleTest, ASSERT_FALSE(response.output().empty()); EXPECT_EQ(response.output(), "{\"response\":[{\"model_path\":\"./benchmark_models/" - "pcvr\",\"tensors\":[{\"tensor_shape\":[2,1],\"data_type\":" - "\"FLOAT\",\"tensor_content\":[0.019116630777716638,0." - "1847093403339386]}]},{\"model_path\":\"./benchmark_models/" - "pcvr\",\"tensors\":[{\"tensor_shape\":[1,1],\"data_type\":" - "\"FLOAT\",\"tensor_content\":[0.010360434651374817]}]}]}"); + "pcvr\",\"tensors\":[{\"tensor_name\":\"StatefulPartitionedCall:" + "0\",\"tensor_shape\":[2,1],\"data_type\":\"FLOAT\",\"tensor_" + "content\":[0.019116630777716638,0.1847093403339386]}]},{\"model_" + "path\":\"./benchmark_models/" + "pcvr\",\"tensors\":[{\"tensor_name\":\"StatefulPartitionedCall:" + "0\",\"tensor_shape\":[1,1],\"data_type\":\"FLOAT\",\"tensor_" + "content\":[0.010360434651374817]}]}]}"); } constexpr char kJsonStringEmbeddingModel[] = R"json({ @@ -694,8 +709,9 @@ constexpr char kJsonStringEmbeddingModel[] = R"json({ })json"; TEST(TensorflowModuleTest, Success_PredictEmbed) { + InferenceSidecarRuntimeConfig config; std::unique_ptr tensorflow_module = - ModuleInterface::Create(); + ModuleInterface::Create(config); RegisterModelRequest register_request; ASSERT_TRUE( PopulateRegisterModelRequest(kEmbeddingModelDir, register_request).ok()); @@ -707,14 +723,18 @@ TEST(TensorflowModuleTest, Success_PredictEmbed) { ASSERT_TRUE(predict_status.ok()); PredictResponse response = predict_status.value(); ASSERT_FALSE(response.output().empty()); + // The order of output tensors in Tensorflow is not constant so we check ech + // tensor output separately. + EXPECT_TRUE(absl::StrContains( + response.output(), + "{\"tensor_name\":\"StatefulPartitionedCall:0\",\"tensor_shape\":[1,6]," + "\"data_type\":\"FLOAT\",\"tensor_content\":[0.7276111245155335,0." + "0728105902671814,0.11053494364023209,0.6876803636550903,0." + "3626940846443176,0.13941356539726258]}")); EXPECT_TRUE(absl::StrContains( response.output(), - "{\"tensor_shape\":[1,6],\"data_type\":\"FLOAT\",\"tensor_content\":[0." - "7276111245155335,0.0728105902671814,0.11053494364023209,0." - "6876803636550903,0.3626940846443176,0.13941356539726258]}")); - EXPECT_TRUE(absl::StrContains(response.output(), - "{\"tensor_shape\":[1,1],\"data_type\":" - "\"INT32\",\"tensor_content\":[0]}")); + "{\"tensor_name\":\"StatefulPartitionedCall:1\",\"tensor_shape\":[1,1]," + "\"data_type\":\"INT32\",\"tensor_content\":[0]}")); } constexpr char kJsonStringWithMultipleInvalidInputs[] = R"json({ @@ -747,8 +767,9 @@ constexpr char kJsonStringWithMultipleInvalidInputs[] = R"json({ })json"; TEST(TensorflowModuleTest, PredictMultipleInvalidInputsReturnsFirstError) { + InferenceSidecarRuntimeConfig config; std::unique_ptr tensorflow_module = - ModuleInterface::Create(); + ModuleInterface::Create(config); RegisterModelRequest register_request_1; ASSERT_TRUE( PopulateRegisterModelRequest(std::string(kModel1Dir), register_request_1) diff --git a/services/inference_sidecar/modules/tensorflow_v2_14_0/test_constants.h b/services/inference_sidecar/modules/tensorflow_v2_14_0/test_constants.h new file mode 100644 index 00000000..e50168c2 --- /dev/null +++ b/services/inference_sidecar/modules/tensorflow_v2_14_0/test_constants.h @@ -0,0 +1,29 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SERVICES_INFERENCE_SIDECAR_COMMON_TEST_CONSTANTS_H_ +#define SERVICES_INFERENCE_SIDECAR_COMMON_TEST_CONSTANTS_H_ + +#include "absl/strings/string_view.h" + +namespace privacy_sandbox::bidding_auction_servers::inference { + +constexpr absl::string_view kRuntimeConfig = + R"json({"num_interop_threads": 4, "num_intraop_threads": 5, "module_name": "tensorflow_v2_14_0"})json"; + +} // namespace privacy_sandbox::bidding_auction_servers::inference + +#endif // SERVICES_BIDDING_SERVICE_CONSTANTS_H_ diff --git a/services/seller_frontend_service/BUILD b/services/seller_frontend_service/BUILD index b922531c..809f073c 100644 --- a/services/seller_frontend_service/BUILD +++ b/services/seller_frontend_service/BUILD @@ -47,6 +47,7 @@ cc_library( "seller_frontend_service.h", ], deps = [ + ":get_component_auction_ciphertexts_reactor", ":runtime_flags", "//api:bidding_auction_servers_cc_grpc_proto", "//api:bidding_auction_servers_cc_proto", @@ -201,6 +202,7 @@ cc_test( "@com_google_absl//absl/log:scoped_mock_log", "@com_google_absl//absl/strings", "@com_google_googletest//:gtest_main", + "@google_privacysandbox_servers_common//src/encryption/key_fetcher:fake_key_fetcher_manager", "@google_privacysandbox_servers_common//src/encryption/key_fetcher/mock:mock_key_fetcher_manager", ], ) @@ -226,3 +228,38 @@ cc_test( "@google_privacysandbox_servers_common//src/encryption/key_fetcher/mock:mock_key_fetcher_manager", ], ) + +cc_library( + name = "get_component_auction_ciphertexts_reactor", + srcs = ["get_component_auction_ciphertexts_reactor.cc"], + hdrs = ["get_component_auction_ciphertexts_reactor.h"], + deps = [ + "//api:bidding_auction_servers_cc_grpc_proto", + "//services/common/clients/config:config_client", + "//services/common/util:request_response_constants", + "//services/seller_frontend_service/util:encryption_util", + "@com_github_grpc_grpc//:grpc++", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/strings", + "@google_privacysandbox_servers_common//src/encryption/key_fetcher/interface:key_fetcher_manager_interface", + "@google_privacysandbox_servers_common//src/util/status_macro:status_util", + ], +) + +cc_test( + name = "get_component_auction_ciphertexts_reactor_test", + srcs = ["get_component_auction_ciphertexts_reactor_test.cc"], + deps = [ + ":get_component_auction_ciphertexts_reactor", + "//services/common/clients/config:config_client", + "//services/common/constants:common_service_flags", + "//services/common/encryption:key_fetcher_factory", + "//services/common/test:mocks", + "//services/seller_frontend_service/util:encryption_util", + "//services/seller_frontend_service/util:select_ad_reactor_test_utils", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + "@google_privacysandbox_servers_common//src/core/test/utils", + "@google_privacysandbox_servers_common//src/encryption/key_fetcher:fake_key_fetcher_manager", + ], +) diff --git a/services/seller_frontend_service/benchmarking/select_ad_reactor_benchmarks.cc b/services/seller_frontend_service/benchmarking/select_ad_reactor_benchmarks.cc index a5a55a69..6cc9da99 100644 --- a/services/seller_frontend_service/benchmarking/select_ad_reactor_benchmarks.cc +++ b/services/seller_frontend_service/benchmarking/select_ad_reactor_benchmarks.cc @@ -50,8 +50,9 @@ void AsyncReporterStub::DoReport( const HTTPRequest& reporting_request, absl::AnyInvocable) &&> done_callback) const { - PS_VLOG(1) << "Mocking success for reporting to URL: " - << reporting_request.url << ", body: " << reporting_request.body; + PS_VLOG(kInfoMsg) << "Mocking success for reporting to URL: " + << reporting_request.url + << ", body: " << reporting_request.body; std::move(done_callback)("success"); } diff --git a/services/seller_frontend_service/get_component_auction_ciphertexts_reactor.cc b/services/seller_frontend_service/get_component_auction_ciphertexts_reactor.cc new file mode 100644 index 00000000..8b1760dc --- /dev/null +++ b/services/seller_frontend_service/get_component_auction_ciphertexts_reactor.cc @@ -0,0 +1,116 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "services/seller_frontend_service/get_component_auction_ciphertexts_reactor.h" + +#include + +#include "absl/strings/ascii.h" +#include "services/common/util/request_response_constants.h" +#include "services/seller_frontend_service/util/encryption_util.h" +#include "src/util/status_macro/status_util.h" + +namespace privacy_sandbox::bidding_auction_servers { + +void GetComponentAuctionCiphertextsReactor::Execute() { + if (seller_cloud_platforms_map_.empty()) { + FinishWithStatus(grpc::Status(grpc::UNIMPLEMENTED, kDisabledError)); + return; + } + // Validate the request + if (request_->component_sellers().empty() || + request_->protected_auction_ciphertext().empty()) { + FinishWithStatus( + grpc::Status(grpc::INVALID_ARGUMENT, kEmptyInputFieldError)); + return; + } + + // Decrypt the protected auction ciphertext + auto decrypted_data = DecryptOHTTPEncapsulatedHpkeCiphertext( + request_->protected_auction_ciphertext(), key_fetcher_manager_); + if (!decrypted_data.ok()) { + FinishWithStatus( + grpc::Status(grpc::INVALID_ARGUMENT, + absl::StrCat(kCiphertextDecryptionFailureError, + decrypted_data.status().message()))); + return; + } + + // Re-encrypt the decrypted data for each component seller and add to the + // response map + for (const auto& seller : request_->component_sellers()) { + const auto& seller_cloud_platform_itr = + seller_cloud_platforms_map_.find(absl::AsciiStrToLower(seller)); + if (seller_cloud_platform_itr == seller_cloud_platforms_map_.end()) { + // Publish metric here. + PS_LOG(WARNING, log_context_) + << "Cloud platform not configured for seller: " << seller; + continue; + } + + // Skip duplicates. + if (response_->mutable_seller_component_ciphertexts()->find(seller) != + response_->mutable_seller_component_ciphertexts()->end()) { + PS_LOG(WARNING, log_context_) + << "Duplicate component seller in input: " << seller; + continue; + } + + absl::StatusOr re_encrypted_data = + HpkeEncryptAndOHTTPEncapsulate( + (*decrypted_data)->plaintext, (*decrypted_data)->request_label, + key_fetcher_manager_, seller_cloud_platform_itr->second); + if (!re_encrypted_data.ok()) { + FinishWithStatus( + grpc::Status(server_common::FromAbslStatus(re_encrypted_data.status()) + .error_code(), + absl::StrCat(kCiphertextEncryptionError, + re_encrypted_data.status().message()))); + return; + } + (*response_->mutable_seller_component_ciphertexts())[seller] = + std::move(re_encrypted_data)->ciphertext; + } + + PS_VLOG(kSuccess, log_context_) << "Finishing RPC with success."; + FinishWithStatus(grpc::Status::OK); +} + +void GetComponentAuctionCiphertextsReactor::FinishWithStatus( + const grpc::Status& status) { + if (status.error_code() != grpc::StatusCode::OK) { + PS_LOG(ERROR, log_context_) << "RPC failed: " << status.error_message(); + } + Finish(status); +} + +void GetComponentAuctionCiphertextsReactor::OnDone() { delete this; } + +void GetComponentAuctionCiphertextsReactor::OnCancel() { + // TODO: Handle early abort and errors. +} + +GetComponentAuctionCiphertextsReactor::GetComponentAuctionCiphertextsReactor( + const GetComponentAuctionCiphertextsRequest* request, + GetComponentAuctionCiphertextsResponse* response, + server_common::KeyFetcherManagerInterface& key_fetcher_manager, + const absl::flat_hash_map& + seller_cloud_platforms_map) + : request_(request), + response_(response), + key_fetcher_manager_(key_fetcher_manager), + seller_cloud_platforms_map_(seller_cloud_platforms_map), + log_context_({}, server_common::ConsentedDebugConfiguration()) {} + +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/seller_frontend_service/get_component_auction_ciphertexts_reactor.h b/services/seller_frontend_service/get_component_auction_ciphertexts_reactor.h new file mode 100644 index 00000000..53981424 --- /dev/null +++ b/services/seller_frontend_service/get_component_auction_ciphertexts_reactor.h @@ -0,0 +1,79 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SERVICES_SELLER_FRONTEND_SERVICE_GET_COMPONENT_AUCTION_CIPHERTEXTS_REACTOR_H_ +#define SERVICES_SELLER_FRONTEND_SERVICE_GET_COMPONENT_AUCTION_CIPHERTEXTS_REACTOR_H_ + +#include + +#include + +#include "absl/container/flat_hash_map.h" +#include "api/bidding_auction_servers.grpc.pb.h" +#include "api/bidding_auction_servers.pb.h" +#include "include/grpcpp/impl/codegen/server_callback.h" +#include "src/encryption/key_fetcher/interface/key_fetcher_manager_interface.h" +#include "src/logger/request_context_impl.h" + +namespace privacy_sandbox::bidding_auction_servers { + +// Constants for service errors. +inline constexpr char kDisabledError[] = + "RPC disabled: Seller cloud platforms must " + "be configured for RPC to be enabled."; +inline constexpr char kEmptyInputFieldError[] = + "Component sellers list or ciphertext is empty"; +inline constexpr char kCiphertextDecryptionFailureError[] = + "Failed to decrypt ciphertext: "; +inline constexpr char kCiphertextEncryptionError[] = + "Failed to re-encrypt data with error: "; + +class GetComponentAuctionCiphertextsReactor : public grpc::ServerUnaryReactor { + public: + explicit GetComponentAuctionCiphertextsReactor( + const GetComponentAuctionCiphertextsRequest* request, + GetComponentAuctionCiphertextsResponse* response, + server_common::KeyFetcherManagerInterface& key_fetcher_manager, + // The keys(seller name/domain) must be lower case. + // This is currently performed while parsing the seller platform maps + // config obtained from the parameter store. + const absl::flat_hash_map& + seller_cloud_platforms_map); + void Execute(); + + private: + const GetComponentAuctionCiphertextsRequest* request_; + GetComponentAuctionCiphertextsResponse* response_; + server_common::KeyFetcherManagerInterface& key_fetcher_manager_; + const absl::flat_hash_map& + seller_cloud_platforms_map_; + server_common::log::ContextImpl log_context_; + + // Cleans up and deletes the GetComponentAuctionCiphertextsReactor. Called by + // the grpc library after the response has finished. + void OnDone() override; + + // Abandons the entire GetComponentAuctionCiphertextsReactor. Called by the + // grpc library if the client cancels the request. + void OnCancel() override; + + // Finishes the RPC call with a status. + void FinishWithStatus(const grpc::Status& status); +}; + +} // namespace privacy_sandbox::bidding_auction_servers + +#endif // SERVICES_SELLER_FRONTEND_SERVICE_GET_COMPONENT_AUCTION_CIPHERTEXTS_REACTOR_H_ diff --git a/services/seller_frontend_service/get_component_auction_ciphertexts_reactor_test.cc b/services/seller_frontend_service/get_component_auction_ciphertexts_reactor_test.cc new file mode 100644 index 00000000..6fc2f976 --- /dev/null +++ b/services/seller_frontend_service/get_component_auction_ciphertexts_reactor_test.cc @@ -0,0 +1,160 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "services/seller_frontend_service/get_component_auction_ciphertexts_reactor.h" + +#include +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "services/common/clients/config/trusted_server_config_client.h" +#include "services/common/constants/common_service_flags.h" +#include "services/common/encryption/key_fetcher_factory.h" +#include "services/common/test/mocks.h" +#include "services/seller_frontend_service/util/encryption_util.h" +#include "services/seller_frontend_service/util/select_ad_reactor_test_utils.h" +#include "src/core/test/utils/proto_test_utils.h" +#include "src/encryption/key_fetcher/fake_key_fetcher_manager.h" + +using google::scp::core::test::EqualsProto; + +namespace privacy_sandbox::bidding_auction_servers { + +static const char kSampleEncodedTest[] = "sample_protected_auction_encoding"; +static const char kSeller1[] = "seller1.example.com"; +static const char kSeller2[] = "seller2.example.com"; +static const char kUnknownSeller[] = "unknown-seller"; + +// Helper function to create a sample GetComponentAuctionCiphertextsRequest. +GetComponentAuctionCiphertextsRequest CreateSampleRequest() { + GetComponentAuctionCiphertextsRequest request; + auto [ciphertext, context] = + GetFramedInputAndOhttpContext(kSampleEncodedTest); + request.set_protected_auction_ciphertext(ciphertext); + request.add_component_sellers(kSeller1); + request.add_component_sellers(kSeller2); + return request; +} + +class GetComponentAuctionCiphertextsReactorTest : public ::testing::Test { + protected: + void SetUp() override { + // initialize + key_fetcher_manager_ = + std::make_unique(); + seller_cloud_platform_map_ = { + {kSeller1, server_common::CloudPlatform::kGcp}, + {kSeller2, server_common::CloudPlatform::kAws}}; + valid_request_ = CreateSampleRequest(); + } + + GetComponentAuctionCiphertextsResponse CreateReactorAndRun( + const GetComponentAuctionCiphertextsRequest& request) { + GetComponentAuctionCiphertextsResponse response; + auto class_under_test = GetComponentAuctionCiphertextsReactor( + &request, &response, *key_fetcher_manager_, seller_cloud_platform_map_); + class_under_test.Execute(); + return response; + } + + // Helper function to decrypt and compare a ciphertext in + // seller_component_ciphertexts map with the original encoded string. + void DecryptAndTestEncodedText( + GetComponentAuctionCiphertextsResponse& actual_response, + absl::string_view seller_name) { + ASSERT_TRUE( + actual_response.seller_component_ciphertexts().contains(seller_name)) + << "Ciphertext not found for seller " << seller_name; + auto decryption_status = DecryptOHTTPEncapsulatedHpkeCiphertext( + actual_response.seller_component_ciphertexts().at(seller_name), + *key_fetcher_manager_); + ASSERT_TRUE(decryption_status.ok()) << decryption_status.status(); + + // Unframe the framed response. + absl::StatusOr unframed_response = + server_common::DecodeRequestPayload((*decryption_status)->plaintext); + ASSERT_TRUE(unframed_response.ok()) << unframed_response.status().message(); + + EXPECT_EQ(unframed_response->compressed_data, kSampleEncodedTest); + } + + GetComponentAuctionCiphertextsRequest valid_request_; + std::unique_ptr + key_fetcher_manager_; + absl::flat_hash_map + seller_cloud_platform_map_; +}; + +TEST_F(GetComponentAuctionCiphertextsReactorTest, + EncryptsCiphertextsForAllSellers) { + auto actual_response = CreateReactorAndRun(valid_request_); + ASSERT_EQ(actual_response.seller_component_ciphertexts().size(), 2); + DecryptAndTestEncodedText(actual_response, kSeller1); + DecryptAndTestEncodedText(actual_response, kSeller2); +} + +TEST_F(GetComponentAuctionCiphertextsReactorTest, + ReturnsDifferentCiphertextForEachSeller) { + ASSERT_GT(valid_request_.protected_auction_ciphertext().size(), 0); + auto actual_response = CreateReactorAndRun(valid_request_); + + ASSERT_EQ(actual_response.seller_component_ciphertexts().size(), 2); + ASSERT_TRUE( + actual_response.seller_component_ciphertexts().contains(kSeller1)); + EXPECT_GT(actual_response.seller_component_ciphertexts().at(kSeller1).size(), + 0); + EXPECT_NE(actual_response.seller_component_ciphertexts().at(kSeller1), + valid_request_.protected_auction_ciphertext()); + + ASSERT_TRUE( + actual_response.seller_component_ciphertexts().contains(kSeller2)); + EXPECT_GT(actual_response.seller_component_ciphertexts().at(kSeller2).size(), + 0); + EXPECT_NE(actual_response.seller_component_ciphertexts().at(kSeller2), + valid_request_.protected_auction_ciphertext()); + + EXPECT_NE(actual_response.seller_component_ciphertexts().at(kSeller2), + actual_response.seller_component_ciphertexts().at(kSeller1)); + DecryptAndTestEncodedText(actual_response, kSeller1); + DecryptAndTestEncodedText(actual_response, kSeller2); +} + +TEST_F(GetComponentAuctionCiphertextsReactorTest, + ReturnsEmptyOutputForEmptySellerList) { + valid_request_.clear_component_sellers(); + + GetComponentAuctionCiphertextsResponse expected_response; + auto actual_response = CreateReactorAndRun(valid_request_); + EXPECT_THAT(actual_response, EqualsProto(expected_response)); +} + +TEST_F(GetComponentAuctionCiphertextsReactorTest, + ReturnsEmptyOutputForEmptyCiphertext) { + valid_request_.clear_protected_auction_ciphertext(); + + GetComponentAuctionCiphertextsResponse expected_response; + auto actual_response = CreateReactorAndRun(valid_request_); + EXPECT_THAT(actual_response, EqualsProto(expected_response)); +} + +TEST_F(GetComponentAuctionCiphertextsReactorTest, IgnoresUnknownSeller) { + valid_request_.add_component_sellers(kUnknownSeller); + auto actual_response = CreateReactorAndRun(valid_request_); + ASSERT_EQ(actual_response.seller_component_ciphertexts().size(), 2); + DecryptAndTestEncodedText(actual_response, kSeller1); + DecryptAndTestEncodedText(actual_response, kSeller2); +} + +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/seller_frontend_service/providers/http_scoring_signals_async_provider.cc b/services/seller_frontend_service/providers/http_scoring_signals_async_provider.cc index 5d9b10d6..83036ce6 100644 --- a/services/seller_frontend_service/providers/http_scoring_signals_async_provider.cc +++ b/services/seller_frontend_service/providers/http_scoring_signals_async_provider.cc @@ -69,7 +69,7 @@ void HttpScoringSignalsAsyncProvider::Get( }, timeout); if (!status.ok()) { - PS_VLOG(1) << "Unable to get seller KV signals: " << status; + PS_LOG(ERROR) << "Unable to get seller KV signals: " << status; } } diff --git a/services/seller_frontend_service/runtime_flags.h b/services/seller_frontend_service/runtime_flags.h index c071150b..61911629 100644 --- a/services/seller_frontend_service/runtime_flags.h +++ b/services/seller_frontend_service/runtime_flags.h @@ -45,6 +45,8 @@ inline constexpr char SFE_TLS_CERT[] = "SFE_TLS_CERT"; inline constexpr char AUCTION_EGRESS_TLS[] = "AUCTION_EGRESS_TLS"; inline constexpr char BUYER_EGRESS_TLS[] = "BUYER_EGRESS_TLS"; inline constexpr char SFE_PUBLIC_KEYS_ENDPOINTS[] = "SFE_PUBLIC_KEYS_ENDPOINTS"; +inline constexpr char SELLER_CLOUD_PLATFORMS_MAP[] = + "SELLER_CLOUD_PLATFORMS_MAP"; inline constexpr absl::string_view kFlags[] = { PORT, @@ -66,6 +68,7 @@ inline constexpr absl::string_view kFlags[] = { AUCTION_EGRESS_TLS, BUYER_EGRESS_TLS, SFE_PUBLIC_KEYS_ENDPOINTS, + SELLER_CLOUD_PLATFORMS_MAP, }; inline std::vector GetServiceFlags() { diff --git a/services/seller_frontend_service/schemas/auction_response.json b/services/seller_frontend_service/schemas/auction_response.json index 565cec0d..a3a5431b 100644 --- a/services/seller_frontend_service/schemas/auction_response.json +++ b/services/seller_frontend_service/schemas/auction_response.json @@ -77,6 +77,14 @@ "type": "string", "description": "Optional currency of the bid (three uppercase letters, ISO 4217 alpha code suggested)." }, + "buyerReportingId": { + "type": "string", + "description": "Optional BuyerReportingId of the winning Ad." + }, + "buyerAndSellerReportingId": { + "type": "string", + "description": "Optional BuyerAndSellerReportingId of the winning Ad." + }, "isChaff": { "type": "boolean", "description": "Boolean to indicate that there is no remarketing winner from the auction. AuctionResult may be ignored by the client (after decryption) if this is set to true." diff --git a/services/seller_frontend_service/select_ad_reactor.cc b/services/seller_frontend_service/select_ad_reactor.cc index 50f5a794..7d96b0dd 100644 --- a/services/seller_frontend_service/select_ad_reactor.cc +++ b/services/seller_frontend_service/select_ad_reactor.cc @@ -97,6 +97,7 @@ SelectAdReactor::SelectAdReactor( AdWithBidMetadata SelectAdReactor::BuildAdWithBidMetadata( const AdWithBid& input, absl::string_view interest_group_owner) { AdWithBidMetadata result; + // First copy all the fields that can be copied directly. if (input.has_ad()) { *result.mutable_ad() = input.ad(); } @@ -105,24 +106,34 @@ AdWithBidMetadata SelectAdReactor::BuildAdWithBidMetadata( result.set_allow_component_auction(input.allow_component_auction()); result.mutable_ad_components()->CopyFrom(input.ad_components()); result.set_interest_group_name(input.interest_group_name()); - result.set_interest_group_owner(interest_group_owner); result.set_ad_cost(input.ad_cost()); result.set_modeling_signals(input.modeling_signals()); result.set_bid_currency(input.bid_currency()); + + // Then set fields passed in externally. + result.set_interest_group_owner(interest_group_owner); + + // Finally, find the AdWithBid's IG and copy the last fields from there. const BuyerInput& buyer_input = buyer_inputs_->find(interest_group_owner)->second; for (const auto& interest_group : buyer_input.interest_groups()) { if (interest_group.name() == result.interest_group_name()) { + result.set_interest_group_origin(interest_group.origin()); if (request_->client_type() == CLIENT_TYPE_BROWSER) { result.set_join_count(interest_group.browser_signals().join_count()); - PS_VLOG(1, log_context_) << "BrowserSignal: Recency:" - << interest_group.browser_signals().recency(); + PS_VLOG(kInfoMsg, log_context_) + << "BrowserSignal: Recency:" + << interest_group.browser_signals().recency(); result.set_recency(interest_group.browser_signals().recency()); } + // May as well skip further iterations. break; } } + if (!input.buyer_reporting_id().empty()) { + result.set_buyer_reporting_id(input.buyer_reporting_id()); + } return result; } @@ -189,9 +200,11 @@ grpc::Status SelectAdReactor::DecryptRequest() { GetDecodedProtectedAudienceInput(decrypted_request_->plaintext); } std::visit( - [this](const auto& protected_auction_input) { - buyer_inputs_ = - GetDecodedBuyerinputs(protected_auction_input.buyer_input()); + [this](auto& input) { + buyer_inputs_ = GetDecodedBuyerinputs(input.buyer_input()); + for (auto& each_buyer_input : *input.mutable_buyer_input()) { + each_buyer_input.second = "encoded buyer input erased"; + } }, protected_auction_input_); return grpc::Status::OK; @@ -268,7 +281,7 @@ void SelectAdReactor::MayPopulateAdServerVisibleErrors() { void SelectAdReactor::MayLogBuyerInput() { if (!buyer_inputs_.ok()) { - PS_VLOG(1, log_context_) << "Failed to decode buyer inputs"; + PS_LOG(ERROR, log_context_) << "Failed to decode buyer inputs"; } else { PS_VLOG(6, log_context_) << "Decoded BuyerInput:\n" @@ -330,8 +343,8 @@ void SelectAdReactor::Execute() { absl::StreamFormatter())); if (!decrypt_status.ok()) { FinishWithStatus(decrypt_status); - PS_VLOG(1, log_context_) << "SelectAdRequest decryption failed:" - << server_common::ToAbslStatus(decrypt_status); + PS_LOG(ERROR, log_context_) << "SelectAdRequest decryption failed:" + << server_common::ToAbslStatus(decrypt_status); return; } std::visit( @@ -349,7 +362,7 @@ void SelectAdReactor::Execute() { LogIfError( metric_context_->AccumulateMetric( 1, metric::kSfeSelectAdRequestBadInput)); - PS_VLOG(1, log_context_) << "AdServerVisible errors found, failing now"; + PS_LOG(ERROR, log_context_) << "AdServerVisible errors found, failing now"; // Finish the GRPC request if we have received bad data from the ad tech // server. @@ -375,13 +388,13 @@ void SelectAdReactor::Execute() { MayPopulateClientVisibleErrors(); if (error_accumulator_.HasErrors()) { - PS_VLOG(1, log_context_) << "Some errors found, failing now"; + PS_LOG(ERROR, log_context_) << "Some errors found, failing now"; // Finish the GRPC request now. OnScoreAdsDone(std::make_unique()); return; } - PS_VLOG(1, log_context_) << "No client / Adtech server errors found"; + PS_VLOG(3, log_context_) << "No client / Adtech server errors found"; benchmarking_logger_->Begin(); @@ -470,6 +483,8 @@ SelectAdReactor::CreateGetBidsRequest(const std::string& buyer_ig_owner, *get_bids_request->mutable_consented_debug_config() = protected_auction_input.consented_debug_config(); } + get_bids_request->set_enable_unlimited_egress( + protected_auction_input.enable_unlimited_egress()); }, protected_auction_input_); @@ -597,7 +612,7 @@ void SelectAdReactor::OnFetchBidsDone( 1, metric::kSfeGetBidsResponseError)); LogInitiatedRequestErrorMetrics(metric::kBfe, response.status(), buyer_ig_owner); - PS_VLOG(1, log_context_) + PS_LOG(ERROR, log_context_) << "GetBidsRequest failed for buyer " << buyer_ig_owner << "\nresponse status: " << response.status(); @@ -646,7 +661,7 @@ void SelectAdReactor::OnAllBidsDone(bool any_successful_bids) { } template -void FilterBidsWithMismatchingCurrencyHelper( +void SelectAdReactor::FilterBidsWithMismatchingCurrencyHelper( google::protobuf::RepeatedPtrField* ads_with_bids, absl::string_view buyer_currency) { int i = 0; @@ -657,7 +672,11 @@ void FilterBidsWithMismatchingCurrencyHelper( buyer_currency != ad_with_bid.bid_currency()) { // Swap to last. Mark for removal and make sure we don't check it again ads_with_bids->SwapElements(i, --remove_starting_at); - // TODO: Record metric. + LogIfError( + metric_context_->AccumulateMetric( + 1, ToSellerRejectionReasonString( + SellerRejectionReason:: + BID_FROM_GENERATE_BID_FAILED_CURRENCY_CHECK))); // Leave index un-incremented so swapped element is checked. } else { i++; @@ -783,7 +802,7 @@ void SelectAdReactor::OnFetchScoringSignalsDone( metric_context_->AccumulateMetric( 1, metric::kSfeScoringSignalsResponseError)); LogInitiatedRequestErrorMetrics(metric::kKv, result.status()); - PS_VLOG(1, log_context_) + PS_LOG(ERROR, log_context_) << "Scoring signals fetch from key-value server failed: " << result.status(); @@ -1011,8 +1030,8 @@ bool SelectAdReactor::EncryptResponse(std::string plaintext_response) { ProtoCloudPlatformToScpCloudPlatform( request_->auction_config().top_level_cloud_platform())); if (!encrypted_request.ok()) { - PS_VLOG(1, log_context_) << "Error while encrypting response: " - << encrypted_request.status().message(); + PS_LOG(ERROR, log_context_) << "Error while encrypting response: " + << encrypted_request.status().message(); FinishWithStatus( grpc::Status(grpc::StatusCode::INTERNAL, kInternalServerError)); return false; @@ -1064,8 +1083,8 @@ void SelectAdReactor::PerformDebugReporting( return; } - PostAuctionSignals post_auction_signals = - GeneratePostAuctionSignals(high_score); + PostAuctionSignals post_auction_signals = GeneratePostAuctionSignals( + high_score, request_->auction_config().seller_currency()); for (const auto& [ig_owner, get_bid_response] : shared_buyer_bids_map_) { for (int i = 0; i < get_bid_response->bids_size(); i++) { const AdWithBid& ad_with_bid = get_bid_response->bids().at(i); diff --git a/services/seller_frontend_service/select_ad_reactor.h b/services/seller_frontend_service/select_ad_reactor.h index 52cd2fd9..6dd71ef0 100644 --- a/services/seller_frontend_service/select_ad_reactor.h +++ b/services/seller_frontend_service/select_ad_reactor.h @@ -124,6 +124,11 @@ class SelectAdReactor : public grpc::ServerUnaryReactor { // RETURNS: True if any bids remain to be scored and false otherwise. bool FilterBidsWithMismatchingCurrency(); + template + void FilterBidsWithMismatchingCurrencyHelper( + google::protobuf::RepeatedPtrField* ads_with_bids, + absl::string_view buyer_currency); + // Validates the mandatory fields in the request. Reports any errors to the // error accumulator. template diff --git a/services/seller_frontend_service/select_ad_reactor_app.cc b/services/seller_frontend_service/select_ad_reactor_app.cc index ad734050..525cbee3 100644 --- a/services/seller_frontend_service/select_ad_reactor_app.cc +++ b/services/seller_frontend_service/select_ad_reactor_app.cc @@ -91,7 +91,7 @@ absl::StatusOr SelectAdReactorForApp::GetNonEncryptedResponse( request_->auction_config().top_level_seller()); } PS_VLOG(kPlain, log_context_) << "AuctionResult:\n" - << auction_result.DebugString(); + << auction_result.ShortDebugString(); // Serialized the data to bytes array. std::string serialized_result = auction_result.SerializeAsString(); @@ -239,9 +239,11 @@ SelectAdReactorForApp::BuildProtectedAppSignalsAdWithBidMetadata( result.set_render(input.render()); result.set_modeling_signals(input.modeling_signals()); result.set_ad_cost(input.ad_cost()); - result.set_egress_features(input.egress_features()); result.set_owner(buyer_owner); result.set_bid_currency(input.bid_currency()); + result.set_egress_payload(input.egress_payload()); + result.set_temporary_unlimited_egress_payload( + input.temporary_unlimited_egress_payload()); return result; } diff --git a/services/seller_frontend_service/select_ad_reactor_app_test.cc b/services/seller_frontend_service/select_ad_reactor_app_test.cc index 9ae5e06f..efa310f6 100644 --- a/services/seller_frontend_service/select_ad_reactor_app_test.cc +++ b/services/seller_frontend_service/select_ad_reactor_app_test.cc @@ -66,7 +66,9 @@ inline constexpr int kTestBidValue = 10.0; inline constexpr int kTestAdCost = 2.0; inline constexpr int kTestEncodingVersion = 1; inline constexpr int kTestModelingSignals = 3; -inline constexpr char kTestEgressFeature[] = "TestEgressFeatures"; +inline constexpr char kTestEgressPayload[] = "TestegressPayload"; +inline constexpr char kTestTemporaryEgressPayload[] = + "TestTemporaryEgressPayload"; inline constexpr char kTestRender[] = "https://test-render.com"; inline constexpr char kAdRenderUrls[] = "AdRenderUrls"; inline constexpr char kTestMetadataKey[] = "TestMetadataKey"; @@ -475,7 +477,8 @@ class SelectAdReactorPASTest : public ::testing::Test { result.set_render(kTestRender); result.set_modeling_signals(kTestModelingSignals); result.set_ad_cost(kTestAdCost); - result.set_egress_features(kTestEgressFeature); + result.set_egress_payload(kTestEgressPayload); + result.set_temporary_unlimited_egress_payload(kTestTemporaryEgressPayload); return result; } @@ -715,8 +718,8 @@ TEST_F(SelectAdReactorPASTest, PASAdWithBidIsSentForScoring) { expected_bid.modeling_signals()); EXPECT_EQ(observed_bid_with_metadata.ad_cost(), expected_bid.ad_cost()); - EXPECT_EQ(observed_bid_with_metadata.egress_features(), - expected_bid.egress_features()); + EXPECT_EQ(observed_bid_with_metadata.egress_payload(), + expected_bid.egress_payload()); EXPECT_EQ(observed_bid_with_metadata.owner(), kSampleBuyer); EXPECT_TRUE(MessageDifferencer::Equals( observed_bid_with_metadata.ad(), expected_bid.ad())); diff --git a/services/seller_frontend_service/select_ad_reactor_test.cc b/services/seller_frontend_service/select_ad_reactor_test.cc index 575a07df..b6c5bb2d 100644 --- a/services/seller_frontend_service/select_ad_reactor_test.cc +++ b/services/seller_frontend_service/select_ad_reactor_test.cc @@ -58,6 +58,9 @@ inline constexpr absl::string_view kProtectedAudienceInput = using ::google::protobuf::TextFormat; using ::testing::_; +using ::testing::Eq; +using ::testing::Pointee; +using ::testing::Property; using ::testing::Return; using Request = SelectAdRequest; @@ -226,7 +229,10 @@ TYPED_TEST(SellerFrontEndServiceTest, FetchesBidsFromAllBuyers) { buyer_ig_owner, this->protected_auction_input_.buyer_input().at(buyer_ig_owner), error_accumulator); - EXPECT_FALSE(error_accumulator.HasErrors()); + EXPECT_FALSE(error_accumulator.HasErrors()) + << error_accumulator.GetAccumulatedErrorString( + ErrorVisibility::CLIENT_VISIBLE) + << "\n\n"; const BuyerInput& buyer_input = std::move(decoded_buyer_input); EXPECT_CALL(buyer_clients, Get(buyer_ig_owner)) .WillOnce([SetupMockBuyer, buyer_input](absl::string_view hostname) { @@ -589,6 +595,7 @@ TYPED_TEST(SellerFrontEndServiceTest, ScoresAdsAfterGettingSignals) { EXPECT_EQ(client_count, 2); absl::flat_hash_map bids; BuyerBidsResponseMap expected_buyer_bids; + std::string buyer_reporting_id = "testBuyerReportingId"; for (const auto& [buyer, unused] : this->protected_auction_input_.buyer_input()) { AdUrl url = buyer_to_ad_url.at(buyer); @@ -600,7 +607,7 @@ TYPED_TEST(SellerFrontEndServiceTest, ScoresAdsAfterGettingSignals) { /*bid_value = */ absl::nullopt, /*enable_event_level_debug_reporting = */ false, /*number_ad_component_render_urls = */ kDefaultNumAdComponents, - /*bid_currency = */ kEurosIsoCode); + /*bid_currency = */ kEurosIsoCode, buyer_reporting_id); bids.insert_or_assign(url, response.bids().at(0)); SetupBuyerClientMock(buyer, buyer_clients, response); expected_buyer_bids.try_emplace( @@ -620,11 +627,11 @@ TYPED_TEST(SellerFrontEndServiceTest, ScoresAdsAfterGettingSignals) { EXPECT_CALL(scoring_client, ExecuteInternal) .WillOnce([select_ad_req = this->request_, protected_auction_input = this->protected_auction_input_, - scoring_signals_value, - bids](std::unique_ptr - score_ads_raw_request, - const RequestMetadata& metadata, - ScoreAdsDoneCallback on_done, absl::Duration timeout) { + scoring_signals_value, bids, buyer_reporting_id]( + std::unique_ptr + score_ads_raw_request, + const RequestMetadata& metadata, + ScoreAdsDoneCallback on_done, absl::Duration timeout) { google::protobuf::util::MessageDifferencer diff; std::string diff_output; diff.ReportDifferencesToString(&diff_output); @@ -642,7 +649,8 @@ TYPED_TEST(SellerFrontEndServiceTest, ScoresAdsAfterGettingSignals) { score_ads_raw_request->ad_bids()) { AdWithBid actual_ad_with_bid_for_test; BuildAdWithBidFromAdWithBidMetadata(actual_ad_with_bid_metadata, - &actual_ad_with_bid_for_test); + &actual_ad_with_bid_for_test, + buyer_reporting_id); EXPECT_TRUE( diff.Compare(actual_ad_with_bid_for_test, bids.at(actual_ad_with_bid_for_test.render()))); @@ -1551,5 +1559,82 @@ TYPED_TEST(SellerFrontEndServiceTest, PassesFieldsForServerComponentAuction) { EXPECT_EQ(response.key_id(), key.key_id()); } +auto EqGetBidsRawRequestUnlimitedEgress( + const GetBidsRequest::GetBidsRawRequest& raw_request) { + return Property(&GetBidsRequest::GetBidsRawRequest::enable_unlimited_egress, + Eq(raw_request.enable_unlimited_egress())); +} + +TYPED_TEST(SellerFrontEndServiceTest, VerifyUnlimitedEgressFlagPropagates) { + MockAsyncProvider + scoring_signals_provider; + ScoringAsyncClientMock scoring_client; + BuyerFrontEndAsyncClientFactoryMock buyer_front_end_async_client_factory_mock; + BuyerBidsResponseMap expected_buyer_bids; + std::unique_ptr key_fetcher_manager = + std::make_unique(); + EXPECT_CALL(*key_fetcher_manager, GetPrivateKey) + .WillRepeatedly(Return(GetPrivateKey())); + // Reporting Client. + std::unique_ptr async_reporter = + std::make_unique( + std::make_unique()); + ClientRegistry clients{scoring_signals_provider, + scoring_client, + buyer_front_end_async_client_factory_mock, + *key_fetcher_manager, + /* crypto_client = */ nullptr, + std::move(async_reporter)}; + + // Setup expectation on buyer client from SFE. + GetBidsRequest::GetBidsRawRequest expected_get_bid_request; + expected_get_bid_request.set_enable_unlimited_egress(true); + + auto mock_get_bids = + [](std::unique_ptr get_values_request, + const RequestMetadata& metadata, GetBidDoneCallback on_done, + absl::Duration timeout) { + auto get_bids_response = + std::make_unique(); + auto* bid = get_bids_response->mutable_bids()->Add(); + bid->set_bid(10.0); + std::move(on_done)(std::move(get_bids_response)); + return absl::OkStatus(); + }; + auto setup_mock_buyer = + [&expected_get_bid_request, + &mock_get_bids](std::unique_ptr buyer) { + EXPECT_CALL(*buyer, + ExecuteInternal(Pointee(EqGetBidsRawRequestUnlimitedEgress( + expected_get_bid_request)), + _, _, _)) + .WillRepeatedly(mock_get_bids); + return buyer; + }; + auto mock_buyer_factory = [setup_mock_buyer](absl::string_view hostname) { + return setup_mock_buyer(std::make_unique()); + }; + EXPECT_CALL(buyer_front_end_async_client_factory_mock, Get) + .WillRepeatedly(mock_buyer_factory); + + // Set consented debug config that should be propagated to the downstream + // services. + auto [protected_auction_input, request, context] = + GetSampleSelectAdRequest( + ClientType::CLIENT_TYPE_ANDROID, kSellerOriginDomain, + /*is_consented_debug=*/false, + /* top_level_seller=*/"", + EncryptionCloudPlatform::ENCRYPTION_CLOUD_PLATFORM_UNSPECIFIED, + /*enable_unlimited_egress =*/true); + + request.mutable_auction_config()->set_seller_debug_id(kSampleSellerDebugId); + auto& buyer_config = (*request.mutable_auction_config() + ->mutable_per_buyer_config())[kSampleBuyer]; + buyer_config.set_buyer_debug_id(kSampleBuyerDebugId); + buyer_config.set_buyer_signals(kSampleBuyerSignals); + + RunReactorRequest(this->config_, clients, request); +} + } // namespace } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/seller_frontend_service/select_ad_reactor_web.cc b/services/seller_frontend_service/select_ad_reactor_web.cc index e6d8d57c..ca6465a5 100644 --- a/services/seller_frontend_service/select_ad_reactor_web.cc +++ b/services/seller_frontend_service/select_ad_reactor_web.cc @@ -106,7 +106,7 @@ absl::StatusOr SelectAdReactorForWeb::GetNonEncryptedResponse( encoded_data = auction_result.SerializeAsString(); PS_VLOG(kPlain, log_context_) << "AuctionResult:\n" - << auction_result.DebugString(); + << auction_result.ShortDebugString(); } else { // SINGLE_SELLER or SERVER_TOP_LEVEL Auction PS_ASSIGN_OR_RETURN( diff --git a/services/seller_frontend_service/select_auction_result_reactor.cc b/services/seller_frontend_service/select_auction_result_reactor.cc index 0f193faa..676a9f07 100644 --- a/services/seller_frontend_service/select_auction_result_reactor.cc +++ b/services/seller_frontend_service/select_auction_result_reactor.cc @@ -229,7 +229,7 @@ void SelectAuctionResultReactor::Execute() { encapsulated_req, clients_.key_fetcher_manager_); // Could not decrypt PAI. if (!decrypt_req_status.ok()) { - PS_VLOG(1, log_context_) + PS_LOG(ERROR, log_context_) << "Error decrypting the protected " << (is_protected_auction_request_ ? "auction" : "audience") << " input ciphertext" << decrypt_req_status.status(); @@ -263,7 +263,7 @@ void SelectAuctionResultReactor::Execute() { // Validate PAI. request_generation_id_ = GetGenerationId(protected_auction_input_); if (request_generation_id_.empty()) { - PS_VLOG(1, log_context_) << kMissingGenerationId; + PS_LOG(ERROR, log_context_) << kMissingGenerationId; // Client side errors. FinishWithClientVisibleErrors(kMissingGenerationId); return; @@ -280,7 +280,7 @@ void SelectAuctionResultReactor::Execute() { if (component_auction_results.empty()) { std::string error_msg = error_accumulator_.GetAccumulatedErrorString( ErrorVisibility::AD_SERVER_VISIBLE); - PS_VLOG(1, log_context_) << error_msg; + PS_LOG(ERROR, log_context_) << error_msg; FinishWithServerVisibleErrors(error_msg); return; } diff --git a/services/seller_frontend_service/select_auction_result_reactor_test.cc b/services/seller_frontend_service/select_auction_result_reactor_test.cc index 6869ea6c..8305e013 100644 --- a/services/seller_frontend_service/select_auction_result_reactor_test.cc +++ b/services/seller_frontend_service/select_auction_result_reactor_test.cc @@ -161,6 +161,30 @@ TYPED_TEST(SelectAuctionResultReactorTest, CallsScoringWithComponentAuctions) { scoring_done.WaitForNotification(); } +TYPED_TEST(SelectAuctionResultReactorTest, AbortsAuctionWithDuplicateResults) { + absl::Notification scoring_done; + this->SetupComponentAuctionResults(1); + // Add duplicate + const auto& original_car = this->request_.component_auction_results(0); + auto* duplicate_car = + this->request_.mutable_component_auction_results()->Add(); + duplicate_car->set_key_id(original_car.key_id()); + duplicate_car->set_auction_result_ciphertext( + original_car.auction_result_ciphertext()); + + EXPECT_CALL(this->scoring_client_, ExecuteInternal).Times(0); + ClientRegistry clients = { + MockAsyncProvider(), + this->scoring_client_, + BuyerFrontEndAsyncClientFactoryMock(), + this->key_fetcher_manager_, + &this->crypto_client_, + std::make_unique( + std::make_unique())}; + auto response = RunRequest(this->config_, clients, this->request_); + EXPECT_EQ(response.auction_result_ciphertext().size(), 0); +} + TYPED_TEST(SelectAuctionResultReactorTest, ReturnsErrorForNoValidComponentAuctions) { this->SetupComponentAuctionResults(0); diff --git a/services/seller_frontend_service/seller_frontend_main.cc b/services/seller_frontend_service/seller_frontend_main.cc index 1e2ad9f7..479430c5 100644 --- a/services/seller_frontend_service/seller_frontend_main.cc +++ b/services/seller_frontend_service/seller_frontend_main.cc @@ -88,6 +88,9 @@ ABSL_FLAG(std::optional, buyer_egress_tls, std::nullopt, ABSL_FLAG(std::optional, sfe_public_keys_endpoints, std::nullopt, "Endpoints serving set of public keys used for encryption across the " "supported cloud platforms"); +ABSL_FLAG(std::optional, seller_cloud_platforms_map, std::nullopt, + "Seller partner cloud platforms. Can be empty." + "Enables requests to the getComponentAuctionCiphertexts RPC."); namespace privacy_sandbox::bidding_auction_servers { @@ -158,6 +161,8 @@ absl::StatusOr GetConfigClient( config_client.SetFlag(FLAGS_enable_protected_audience, ENABLE_PROTECTED_AUDIENCE); config_client.SetFlag(FLAGS_ps_verbosity, PS_VERBOSITY); + config_client.SetFlag(FLAGS_seller_cloud_platforms_map, + SELLER_CLOUD_PLATFORMS_MAP); if (absl::GetFlag(FLAGS_init_config_client)) { PS_RETURN_IF_ERROR(config_client.Init(config_param_prefix)).LogError() @@ -175,11 +180,11 @@ absl::StatusOr GetConfigClient( ABSL_LOG(WARNING) << "Neither protected audience nor protected app signals " "is enabled"; } - PS_VLOG(1) << "Protected Audience support enabled on the service: " - << enable_protected_audience; - PS_VLOG(1) << "Protected App Signals support enabled on the service: " - << enable_protected_app_signals; - PS_VLOG(1) << "Successfully constructed the config client."; + PS_LOG(INFO) << "Protected Audience support enabled on the service: " + << enable_protected_audience; + PS_LOG(INFO) << "Protected App Signals support enabled on the service: " + << enable_protected_app_signals; + PS_LOG(INFO) << "Successfully constructed the config client."; return config_client; } @@ -252,7 +257,7 @@ absl::Status RunServer() { return absl::UnavailableError("Error starting Server."); } - PS_VLOG(1) << "Server listening on " << server_address; + PS_LOG(INFO) << "Server listening on " << server_address; // Wait for the server to shutdown. Note that some other thread must be // responsible for shutting down the server for this call to ever return. server->Wait(); diff --git a/services/seller_frontend_service/seller_frontend_service.cc b/services/seller_frontend_service/seller_frontend_service.cc index 49275c0a..a052109a 100644 --- a/services/seller_frontend_service/seller_frontend_service.cc +++ b/services/seller_frontend_service/seller_frontend_service.cc @@ -23,17 +23,20 @@ #include "services/common/clients/http_kv_server/seller/fake_seller_key_value_async_http_client.h" #include "services/common/clients/http_kv_server/seller/seller_key_value_async_http_client.h" #include "services/common/metric/server_definition.h" +#include "services/common/util/auction_scope_util.h" +#include "services/seller_frontend_service/get_component_auction_ciphertexts_reactor.h" #include "services/seller_frontend_service/select_ad_reactor.h" #include "services/seller_frontend_service/select_ad_reactor_app.h" #include "services/seller_frontend_service/select_ad_reactor_invalid_client.h" #include "services/seller_frontend_service/select_ad_reactor_web.h" +#include "services/seller_frontend_service/select_auction_result_reactor.h" #include "src/telemetry/telemetry.h" namespace privacy_sandbox::bidding_auction_servers { namespace { // Factory method to get a reactor based on the request type. -std::unique_ptr GetReactorForRequest( +std::unique_ptr GetSelectAdReactor( grpc::CallbackServerContext* context, const SelectAdRequest* request, SelectAdResponse* response, const ClientRegistry& clients, const TrustedServersConfigClient& config_client) { @@ -69,8 +72,26 @@ grpc::ServerUnaryReactor* SellerFrontEndService::SelectAd( grpc::CallbackServerContext* context, const SelectAdRequest* request, SelectAdResponse* response) { LogCommonMetric(request, response); - auto reactor = GetReactorForRequest(context, request, response, clients_, - config_client_); + if (AuctionScope auction_scope = GetAuctionScope(*request); + auction_scope == AuctionScope::AUCTION_SCOPE_SERVER_TOP_LEVEL_SELLER) { + auto reactor = std::make_unique( + context, request, response, clients_, config_client_); + reactor->Execute(); + return reactor.release(); + } + std::unique_ptr reactor = + GetSelectAdReactor(context, request, response, clients_, config_client_); + reactor->Execute(); + return reactor.release(); +} + +grpc::ServerUnaryReactor* SellerFrontEndService::GetComponentAuctionCiphertexts( + grpc::CallbackServerContext* context, + const GetComponentAuctionCiphertextsRequest* request, + GetComponentAuctionCiphertextsResponse* response) { + auto reactor = std::make_unique( + request, response, clients_.key_fetcher_manager_, + seller_cloud_platforms_map_); reactor->Execute(); return reactor.release(); } diff --git a/services/seller_frontend_service/seller_frontend_service.h b/services/seller_frontend_service/seller_frontend_service.h index 435b7a10..0abd6bab 100644 --- a/services/seller_frontend_service/seller_frontend_service.h +++ b/services/seller_frontend_service/seller_frontend_service.h @@ -39,6 +39,15 @@ namespace privacy_sandbox::bidding_auction_servers { +// Create seller cloud platform map. +static inline absl::flat_hash_map +ParseSellerCloudPlarformMap(absl::string_view seller_platforms_map_json) { + auto seller_cloud_platform_parse_status = + ParseSellerToCloudPlatformInMap(seller_platforms_map_json); + CHECK_OK(seller_cloud_platform_parse_status); + return *seller_cloud_platform_parse_status; +} + // This a utility class that acts as a wrapper for the clients that are used // by SellerFrontEndService. struct ClientRegistry { @@ -106,11 +115,20 @@ class SellerFrontEndService final : public SellerFrontEnd::CallbackService { crypto_client_.get(), std::make_unique( std::make_unique(executor_.get()))} { + if (config_client_.HasParameter(SELLER_CLOUD_PLATFORMS_MAP)) { + seller_cloud_platforms_map_ = ParseSellerCloudPlarformMap( + config_client_.GetStringParameter(SELLER_CLOUD_PLATFORMS_MAP)); + } } SellerFrontEndService(const TrustedServersConfigClient* config_client, ClientRegistry clients) - : config_client_(*config_client), clients_(std::move(clients)) {} + : config_client_(*config_client), clients_(std::move(clients)) { + if (config_client_.HasParameter(SELLER_CLOUD_PLATFORMS_MAP)) { + seller_cloud_platforms_map_ = ParseSellerCloudPlarformMap( + config_client_.GetStringParameter(SELLER_CLOUD_PLATFORMS_MAP)); + } + } std::unique_ptr> CreateKVClient(); @@ -131,6 +149,26 @@ class SellerFrontEndService final : public SellerFrontEnd::CallbackService { const bidding_auction_servers::SelectAdRequest* request, bidding_auction_servers::SelectAdResponse* response) override; + // Selects a winning ad by running an ad auction. + // + // This is an rpc endpoint which will lead to further requests (rpc and http) + // to other endpoints. + // + // context: Standard gRPC-owned testing utility parameter. + // request: Pointer to GetComponentAuctionCiphertextsRequest. + // The request contains a ciphertext encrypted using + // Hybrid Public Key Encryption (HPKE). + // response: Pointer to GetComponentAuctionCiphertextsResponse. + // The response contains the same payload encrypted to unique ciphertexts + // using HPKE. + // return: a ServerUnaryReactor that is used by the gRPC library + grpc::ServerUnaryReactor* GetComponentAuctionCiphertexts( + grpc::CallbackServerContext* context, + const bidding_auction_servers::GetComponentAuctionCiphertextsRequest* + request, + bidding_auction_servers::GetComponentAuctionCiphertextsResponse* response) + override; + private: const TrustedServersConfigClient& config_client_; std::unique_ptr @@ -142,6 +180,8 @@ class SellerFrontEndService final : public SellerFrontEnd::CallbackService { std::unique_ptr> buyer_factory_; const ClientRegistry clients_; + absl::flat_hash_map + seller_cloud_platforms_map_; }; } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/seller_frontend_service/seller_frontend_service_test.cc b/services/seller_frontend_service/seller_frontend_service_test.cc index 25198244..7430d9af 100644 --- a/services/seller_frontend_service/seller_frontend_service_test.cc +++ b/services/seller_frontend_service/seller_frontend_service_test.cc @@ -36,8 +36,10 @@ #include "services/common/test/random.h" #include "services/common/test/utils/cbor_test_utils.h" #include "services/common/test/utils/service_utils.h" +#include "services/seller_frontend_service/get_component_auction_ciphertexts_reactor.h" #include "services/seller_frontend_service/select_ad_reactor.h" #include "services/seller_frontend_service/util/select_ad_reactor_test_utils.h" +#include "src/encryption/key_fetcher/fake_key_fetcher_manager.h" #include "src/encryption/key_fetcher/mock/mock_key_fetcher_manager.h" namespace privacy_sandbox::bidding_auction_servers { @@ -68,6 +70,19 @@ using ScoreAdsDoneCallback = using EncodedBuyerInputs = ::google::protobuf::Map; using DecodedBuyerInputs = ::google::protobuf::Map; +// Maintains ownership of clients +struct SellerFrontEndClientOwner { + // Setup buyer client. + BuyerFrontEndAsyncClientFactoryMock buyer_clients; + + // Scoring signals provider mock. + MockAsyncProvider + scoring_signals_provider; + ScoringAsyncClientMock scoring_client; + + server_common::FakeKeyFetcherManager key_fetcher_manager; +}; + template class SellerFrontEndServiceTest : public ::testing::Test { protected: @@ -85,9 +100,21 @@ class SellerFrontEndServiceTest : public ::testing::Test { config_.SetFlagForTest("", CONSENTED_DEBUG_TOKEN); config_.SetFlagForTest(kFalse, ENABLE_PROTECTED_APP_SIGNALS); config_.SetFlagForTest(kTrue, ENABLE_PROTECTED_AUDIENCE); + config_.SetFlagForTest("{}", SELLER_CLOUD_PLATFORMS_MAP); + } + + ClientRegistry CreateValidClientRegistry() { + return {valid_clients_.scoring_signals_provider, + valid_clients_.scoring_client, valid_clients_.buyer_clients, + valid_clients_.key_fetcher_manager, + /* crypto_client= */ nullptr, + // Reporting Client. + std::make_unique( + std::make_unique())}; } TrustedServersConfigClient config_ = TrustedServersConfigClient({}); + SellerFrontEndClientOwner valid_clients_; }; using ProtectedAuctionInputTypes = @@ -1336,5 +1363,166 @@ TYPED_TEST(SellerFrontEndServiceTest, ReturnsErrorForAndroidComponentAuction) { kDeviceComponentAuctionWithAndroid)); } +static const char kTestComponentSeller[] = "seller1.example.com"; +static const char kSampleCloudPlatformMap[] = R"JSON({ + "seller1.example.com" :"GCP", + "seller2.example.com":"AWS" + })JSON"; + +TYPED_TEST( + SellerFrontEndServiceTest, + GetComponentAuctionCiphertexts_ReturnsErrorForEmptySellerCloudPlatforms) { + GetComponentAuctionCiphertextsRequest request; + request.set_protected_auction_ciphertext(MakeARandomString()); + request.add_component_sellers(MakeARandomString()); + GetComponentAuctionCiphertextsResponse response; + grpc::ClientContext context; + + this->config_.SetFlagForTest(kSampleSellerDomain, SELLER_ORIGIN_DOMAIN); + SellerFrontEndService seller_frontend_service( + &this->config_, this->CreateValidClientRegistry()); + auto start_sfe_result = StartLocalService(&seller_frontend_service); + auto stub = CreateServiceStub(start_sfe_result.port); + grpc::Status status = + stub->GetComponentAuctionCiphertexts(&context, request, &response); + ASSERT_EQ(status.error_code(), grpc::StatusCode::UNIMPLEMENTED); + EXPECT_TRUE(absl::StrContains(status.error_message(), kDisabledError)); +} + +TYPED_TEST( + SellerFrontEndServiceTest, + GetComponentAuctionCiphertexts_ReturnsInvalidArgumentForEmptyCiphertext) { + GetComponentAuctionCiphertextsRequest request; + request.add_component_sellers(MakeARandomString()); + GetComponentAuctionCiphertextsResponse response; + grpc::ClientContext context; + + this->config_.SetFlagForTest(kSampleSellerDomain, SELLER_ORIGIN_DOMAIN); + this->config_.SetFlagForTest(kSampleCloudPlatformMap, + SELLER_CLOUD_PLATFORMS_MAP); + SellerFrontEndService seller_frontend_service( + &this->config_, this->CreateValidClientRegistry()); + auto start_sfe_result = StartLocalService(&seller_frontend_service); + auto stub = CreateServiceStub(start_sfe_result.port); + grpc::Status status = + stub->GetComponentAuctionCiphertexts(&context, request, &response); + ASSERT_EQ(status.error_code(), grpc::StatusCode::INVALID_ARGUMENT); + EXPECT_TRUE(absl::StrContains(status.error_message(), kEmptyInputFieldError)); +} + +TYPED_TEST( + SellerFrontEndServiceTest, + GetComponentAuctionCiphertexts_ReturnsInvalidArgumentForEmptySellerList) { + GetComponentAuctionCiphertextsRequest request; + request.set_protected_auction_ciphertext(MakeARandomString()); + GetComponentAuctionCiphertextsResponse response; + grpc::ClientContext context; + + this->config_.SetFlagForTest(kSampleSellerDomain, SELLER_ORIGIN_DOMAIN); + this->config_.SetFlagForTest(kSampleCloudPlatformMap, + SELLER_CLOUD_PLATFORMS_MAP); + SellerFrontEndService seller_frontend_service( + &this->config_, this->CreateValidClientRegistry()); + auto start_sfe_result = StartLocalService(&seller_frontend_service); + auto stub = CreateServiceStub(start_sfe_result.port); + grpc::Status status = + stub->GetComponentAuctionCiphertexts(&context, request, &response); + ASSERT_EQ(status.error_code(), grpc::StatusCode::INVALID_ARGUMENT); + EXPECT_TRUE(absl::StrContains(status.error_message(), kEmptyInputFieldError)); +} + +TYPED_TEST( + SellerFrontEndServiceTest, + GetComponentAuctionCiphertexts_ReturnsInvalidArgumentForInvalidCiphertext) { + GetComponentAuctionCiphertextsRequest request; + request.set_protected_auction_ciphertext(MakeARandomString()); + request.add_component_sellers(MakeARandomString()); + GetComponentAuctionCiphertextsResponse response; + grpc::ClientContext context; + + this->config_.SetFlagForTest(kSampleSellerDomain, SELLER_ORIGIN_DOMAIN); + this->config_.SetFlagForTest(kSampleCloudPlatformMap, + SELLER_CLOUD_PLATFORMS_MAP); + SellerFrontEndService seller_frontend_service( + &this->config_, this->CreateValidClientRegistry()); + auto start_sfe_result = StartLocalService(&seller_frontend_service); + auto stub = CreateServiceStub(start_sfe_result.port); + grpc::Status status = + stub->GetComponentAuctionCiphertexts(&context, request, &response); + ASSERT_EQ(status.error_code(), grpc::StatusCode::INVALID_ARGUMENT); + EXPECT_TRUE(absl::StrContains(status.error_message(), + kCiphertextDecryptionFailureError)); +} + +TYPED_TEST(SellerFrontEndServiceTest, + GetComponentAuctionCiphertexts_ReturnsResponseForComponentSeller) { + GetComponentAuctionCiphertextsRequest request; + TypeParam protected_auction_input = + MakeARandomProtectedAuctionInput(); + auto [encrypted_protected_auction_input, encryption_context] = + GetCborEncodedEncryptedInputAndOhttpContext(protected_auction_input); + request.set_protected_auction_ciphertext(encrypted_protected_auction_input); + request.add_component_sellers(kTestComponentSeller); + request.add_component_sellers(MakeARandomString()); + + GetComponentAuctionCiphertextsResponse response; + grpc::ClientContext context; + + this->config_.SetFlagForTest(kSampleSellerDomain, SELLER_ORIGIN_DOMAIN); + this->config_.SetFlagForTest(kSampleCloudPlatformMap, + SELLER_CLOUD_PLATFORMS_MAP); + SellerFrontEndService seller_frontend_service( + &this->config_, this->CreateValidClientRegistry()); + + auto start_sfe_result = StartLocalService(&seller_frontend_service); + auto stub = CreateServiceStub(start_sfe_result.port); + grpc::Status status = + stub->GetComponentAuctionCiphertexts(&context, request, &response); + ASSERT_EQ(status.error_code(), grpc::StatusCode::OK); + ASSERT_EQ(response.seller_component_ciphertexts().size(), 1); + ASSERT_TRUE( + response.seller_component_ciphertexts().contains(kTestComponentSeller)); + EXPECT_NE(response.seller_component_ciphertexts().at(kTestComponentSeller), + encrypted_protected_auction_input); +} + +TYPED_TEST(SellerFrontEndServiceTest, + GetComponentAuctionCiphertexts_ReturnsInternalErrorForKeyNotFound) { + GetComponentAuctionCiphertextsRequest request; + TypeParam protected_auction_input = + MakeARandomProtectedAuctionInput(); + auto [encrypted_protected_auction_input, encryption_context] = + GetCborEncodedEncryptedInputAndOhttpContext(protected_auction_input); + request.set_protected_auction_ciphertext(encrypted_protected_auction_input); + request.add_component_sellers(kTestComponentSeller); + + GetComponentAuctionCiphertextsResponse response; + grpc::ClientContext context; + + this->config_.SetFlagForTest(kSampleSellerDomain, SELLER_ORIGIN_DOMAIN); + this->config_.SetFlagForTest(kSampleCloudPlatformMap, + SELLER_CLOUD_PLATFORMS_MAP); + + // Key fetcher manager does not serve public key. + server_common::MockKeyFetcherManager key_fetcher_manager; + EXPECT_CALL(key_fetcher_manager, GetPrivateKey) + .WillRepeatedly(Return(GetPrivateKey())); + SellerFrontEndService seller_frontend_service( + &this->config_, {this->valid_clients_.scoring_signals_provider, + this->valid_clients_.scoring_client, + this->valid_clients_.buyer_clients, key_fetcher_manager, + /* crypto_client= */ nullptr, + std::make_unique( + std::make_unique())}); + + auto start_sfe_result = StartLocalService(&seller_frontend_service); + auto stub = CreateServiceStub(start_sfe_result.port); + grpc::Status status = + stub->GetComponentAuctionCiphertexts(&context, request, &response); + ASSERT_EQ(status.error_code(), grpc::StatusCode::INTERNAL); + EXPECT_TRUE( + absl::StrContains(status.error_message(), kCiphertextEncryptionError)); +} + } // namespace } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/seller_frontend_service/util/config_param_parser.cc b/services/seller_frontend_service/util/config_param_parser.cc index b1cd69a6..b708f32e 100644 --- a/services/seller_frontend_service/util/config_param_parser.cc +++ b/services/seller_frontend_service/util/config_param_parser.cc @@ -16,6 +16,7 @@ #include "services/seller_frontend_service/util/config_param_parser.h" #include +#include #include #include @@ -29,6 +30,22 @@ namespace privacy_sandbox::bidding_auction_servers { +namespace { +absl::StatusOr StringToCloudPlatform( + absl::string_view cloud_platform) { + if (absl::EqualsIgnoreCase(cloud_platform, "GCP")) { + return server_common::CloudPlatform::kGcp; + } else if (absl::EqualsIgnoreCase(cloud_platform, "AWS")) { + return server_common::CloudPlatform::kAws; + } else if (absl::EqualsIgnoreCase(cloud_platform, "LOCAL")) { + return server_common::CloudPlatform::kLocal; + } else { + return absl::InvalidArgumentError( + absl::StrCat("Invalid value for cloud platform: ", cloud_platform)); + } +} +} // namespace + absl::StatusOr> ParseIgOwnerToBfeDomainMap(absl::string_view ig_owner_to_bfe_domain) { if (ig_owner_to_bfe_domain.empty()) { @@ -83,16 +100,8 @@ ParseIgOwnerToBfeDomainMap(absl::string_view ig_owner_to_bfe_domain) { PS_ASSIGN_OR_RETURN( std::string cloud_platform, GetStringMember(buyer_service_endpoint_value, "cloudPlatform")); - if (absl::EqualsIgnoreCase(cloud_platform, "GCP")) { - bfe_endpoint.cloud_platform = server_common::CloudPlatform::kGcp; - } else if (absl::EqualsIgnoreCase(cloud_platform, "AWS")) { - bfe_endpoint.cloud_platform = server_common::CloudPlatform::kAws; - } else if (absl::EqualsIgnoreCase(cloud_platform, "LOCAL")) { - bfe_endpoint.cloud_platform = server_common::CloudPlatform::kLocal; - } else { - return absl::InvalidArgumentError(absl::StrCat( - "Invalid value for BFE endpoint cloud platform: ", cloud_platform)); - } + PS_ASSIGN_OR_RETURN(bfe_endpoint.cloud_platform, + StringToCloudPlatform(cloud_platform)); auto [it_2, inserted] = ig_owner_to_bfe_endpoint_map.try_emplace(ig_owner, bfe_endpoint); @@ -124,4 +133,52 @@ std::vector FetchIgOwnerList( return ig_list; } +absl::StatusOr> +ParseSellerToCloudPlatformInMap( + absl::string_view seller_to_cloud_platform_json) { + absl::flat_hash_map + seller_to_cloud_platform; + if (seller_to_cloud_platform_json.empty()) { + return seller_to_cloud_platform; + } + rapidjson::Document seller_to_cloud_platform_json_d; + rapidjson::ParseResult parse_result = + seller_to_cloud_platform_json_d.Parse( + seller_to_cloud_platform_json.data()); + if (parse_result.IsError()) { + return absl::InvalidArgumentError( + absl::StrCat("Malformed Seller to CloudPlatform map, error: ", + rapidjson::GetParseError_En(parse_result.Code()))); + } + + for (rapidjson::Value::MemberIterator itr = + seller_to_cloud_platform_json_d.MemberBegin(); + itr != seller_to_cloud_platform_json_d.MemberEnd(); ++itr) { + if (!itr->name.IsString()) { + return absl::InvalidArgumentError( + "Expected Seller identifier string in key."); + } + + if (!itr->value.IsString()) { + return absl::InvalidArgumentError( + "Expected cloud platform string in value."); + } + + std::string seller_name = itr->name.GetString(); + if (seller_name.empty()) { + return absl::InvalidArgumentError("Encountered empty Seller Name."); + } + PS_ASSIGN_OR_RETURN(server_common::CloudPlatform platform, + StringToCloudPlatform(itr->value.GetString())); + auto [inserted_itr, inserted_bool] = + seller_to_cloud_platform.try_emplace(std::move(seller_name), platform); + if (!inserted_bool) { + return absl::InvalidArgumentError( + "Entry not inserted into Seller->Cloud platform map, check for " + "duplicate entries."); + } + } + return seller_to_cloud_platform; +} + } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/seller_frontend_service/util/config_param_parser.h b/services/seller_frontend_service/util/config_param_parser.h index 9edc1c50..fd154d55 100644 --- a/services/seller_frontend_service/util/config_param_parser.h +++ b/services/seller_frontend_service/util/config_param_parser.h @@ -35,7 +35,7 @@ struct BuyerServiceEndpoint { // BuyerFrontendAsyncClientFactory constructor expects. Used to parse the // startup config parameter in seller_frontend_main. Parsing may fail, in which // case the server should not start, hence the StatusOr. If the input to Factory -// constructor changes, so should ths function. +// constructor changes, so should this function. absl::StatusOr> ParseIgOwnerToBfeDomainMap(absl::string_view ig_owner_to_bfe_domain); @@ -45,6 +45,14 @@ std::vector FetchIgOwnerList( absl::flat_hash_map>& ig_owner_to_bfe_domain_map); +// Parses a JSON string containing a map of sellers to cloud platforms +// into an absl::flat_hash_map for getComponentAuctionCiphertexts. +// Used to parse the startup config parameter in seller_frontend_main. +// Returns parsing error if string is invalid. +absl::StatusOr> +ParseSellerToCloudPlatformInMap( + absl::string_view seller_to_cloud_platform_json); + } // namespace privacy_sandbox::bidding_auction_servers #endif // SERVICES_SELLER_FRONTEND_SERVICE_UTIL_CONFIG_PARAM_PARSER_H_ diff --git a/services/seller_frontend_service/util/config_param_parser_test.cc b/services/seller_frontend_service/util/config_param_parser_test.cc index 53d1ffd0..b05bcbec 100644 --- a/services/seller_frontend_service/util/config_param_parser_test.cc +++ b/services/seller_frontend_service/util/config_param_parser_test.cc @@ -205,5 +205,63 @@ TEST(StartupParamParserTest, InvalidCloudPlatform) { EXPECT_EQ(output.status().code(), absl::StatusCode::kInvalidArgument); } +TEST(ParseSellerToCloudPlatformInMapTest, ParsesEmptyMap) { + absl::string_view seller_platform_map = R"json()json"; + + auto output = ParseSellerToCloudPlatformInMap(seller_platform_map); + EXPECT_TRUE(output.ok()); + EXPECT_EQ(output->size(), 0); +} + +TEST(ParseSellerToCloudPlatformInMapTest, ParsesMapWithNoEntries) { + absl::string_view seller_platform_map = R"json({})json"; + + auto output = ParseSellerToCloudPlatformInMap(seller_platform_map); + EXPECT_TRUE(output.ok()); + EXPECT_EQ(output->size(), 0); +} + +TEST(ParseSellerToCloudPlatformInMapTest, DoesNotParseMalformedMap) { + absl::string_view seller_platform_map = R"json({ + "https://example-seller.com": "GCP", + })json"; + + auto output = ParseSellerToCloudPlatformInMap(seller_platform_map); + ASSERT_FALSE(output.ok()); + EXPECT_EQ(output.status().code(), absl::StatusCode::kInvalidArgument); +} + +TEST(ParseSellerToCloudPlatformInMapTest, DoesNotParseHalfCorrectMap) { + absl::string_view seller_platform_map = R"json({ + "https://example-seller.com": "GCP", + "https://bidding1.com": 3 + })json"; + + auto output = ParseSellerToCloudPlatformInMap(seller_platform_map); + ASSERT_FALSE(output.ok()); + EXPECT_EQ(output.status().code(), absl::StatusCode::kInvalidArgument); +} + +TEST(ParseSellerToCloudPlatformInMapTest, ParsesValidMap) { + absl::string_view seller_platform_map = R"json({ + "https://example-seller.com": "GCP", + "https://example-seller2.com": "AWS" + })json"; + auto output = ParseSellerToCloudPlatformInMap(seller_platform_map); + + ASSERT_TRUE(output.ok()); + auto it = output.value().find("https://example-seller.com"); + EXPECT_NE(it, output.value().end()); + if (it != output.value().end()) { + EXPECT_EQ(it->second, CloudPlatform::kGcp); + } + + auto it_2 = output.value().find("https://example-seller2.com"); + EXPECT_NE(it_2, output.value().end()); + if (it_2 != output.value().end()) { + EXPECT_EQ(it_2->second, CloudPlatform::kAws); + } +} + } // namespace } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/seller_frontend_service/util/proto_mapping_util.cc b/services/seller_frontend_service/util/proto_mapping_util.cc index a8a82b4e..b238986a 100644 --- a/services/seller_frontend_service/util/proto_mapping_util.cc +++ b/services/seller_frontend_service/util/proto_mapping_util.cc @@ -62,6 +62,8 @@ AuctionResult MapAdScoreToAuctionResult( auction_result.set_score(high_score->desirability()); auction_result.set_interest_group_name(high_score->interest_group_name()); auction_result.set_interest_group_owner(high_score->interest_group_owner()); + auction_result.set_interest_group_origin( + high_score->interest_group_origin()); auction_result.set_ad_render_url(high_score->render()); auction_result.mutable_win_reporting_urls() ->mutable_buyer_reporting_urls() @@ -132,7 +134,7 @@ absl::StatusOr PackageAuctionResultForWeb( wait_for_error_callback.WaitForNotification(); return absl::Status(serialized_data.status().code(), error_msg); } else { - PS_VLOG(1, log_context) + PS_VLOG(kPlain, log_context) << "AuctionResult:\n" << [](absl::string_view encoded_data) { auto result = CborDecodeAuctionResultToProto(encoded_data); @@ -153,7 +155,7 @@ absl::StatusOr PackageAuctionResultForApp( // Map to AuctionResult proto and serialized to bytes array. std::string serialized_result = MapAdScoreToAuctionResult(high_score, error).SerializeAsString(); - PS_VLOG(1, log_context) << "AuctionResult:\n" << serialized_result; + PS_VLOG(kPlain, log_context) << "AuctionResult:\n" << serialized_result; return PackageAuctionResultCiphertext(serialized_result, decrypted_request); } @@ -295,15 +297,15 @@ IgsWithBidsMap GetBuyerIgsWithBidsMap( IgsWithBidsMap buyer_to_ig_idx_map; for (IgsWithBidsMap& component_auction_ig_group : component_auction_bidding_groups) { - PS_VLOG(1) << "Size of input buyer bids map " - << component_auction_ig_group.size(); + PS_VLOG(kInfoMsg) << "Size of input buyer bids map " + << component_auction_ig_group.size(); for (auto& [buyer, ig_idx] : component_auction_ig_group) { // Check if the key already exists in the output map const auto& it = buyer_to_ig_idx_map.find(buyer); if (it == buyer_to_ig_idx_map.end()) { // Insert the entire entry buyer_to_ig_idx_map.insert({buyer, std::move(ig_idx)}); - PS_VLOG(1) << "Inserted for " << buyer; + PS_VLOG(kInfoMsg) << "Inserted for " << buyer; continue; } else if (auto buyer_ig_set = colliding_buyer_sets.find(buyer); buyer_ig_set == colliding_buyer_sets.end()) { @@ -311,12 +313,13 @@ IgsWithBidsMap GetBuyerIgsWithBidsMap( // Add values from previous CARs. absl::flat_hash_set ig_set(it->second.index().begin(), it->second.index().end()); - PS_VLOG(1) << "Inserted in colliding for" << buyer << ig_set.size(); + PS_VLOG(kInfoMsg) << "Inserted in colliding for" << buyer + << ig_set.size(); colliding_buyer_sets.insert({buyer, std::move(ig_set)}); } // Already in colliding buyer set. Add all values from current CAR. - PS_VLOG(1) << "Inserted in colliding for" << buyer - << ig_idx.index().size(); + PS_VLOG(kInfoMsg) << "Inserted in colliding for" << buyer + << ig_idx.index().size(); colliding_buyer_sets.at(buyer).insert(ig_idx.index().begin(), ig_idx.index().end()); } @@ -327,7 +330,8 @@ IgsWithBidsMap GetBuyerIgsWithBidsMap( buyer_to_ig_idx_map.at(buyer).mutable_index()->Assign(ig_idx_set.begin(), ig_idx_set.end()); } - PS_VLOG(1) << "Size of output buyer bids map " << buyer_to_ig_idx_map.size(); + PS_VLOG(kInfoMsg) << "Size of output buyer bids map " + << buyer_to_ig_idx_map.size(); return buyer_to_ig_idx_map; } @@ -360,6 +364,8 @@ std::vector DecryptAndValidateComponentAuctionResults( server_common::KeyFetcherManagerInterface& key_fetcher_manager, ErrorAccumulator& error_accumulator, ContextImpl& log_context) { std::vector component_auction_results; + // Keep track of encountered sellers. + absl::flat_hash_set component_sellers; component_auction_results.reserve(request->component_auction_results_size()); for (const auto& enc_auction_result : request->component_auction_results()) { auto auction_result = UnpackageServerAuctionComponentResult( @@ -379,17 +385,33 @@ std::vector DecryptAndValidateComponentAuctionResults( PS_VLOG(2, log_context) << "Successfully decrypted auction result ciphertext for: " << auction_result->auction_params().component_seller(); - PS_VLOG(1, log_context) + PS_VLOG(kInfoMsg, log_context) << "Bidding group size: " << auction_result->bidding_groups().size(); // Add errors from AuctionResult to error_accumulator in // ValidateComponentAuctionResult. if (!ValidateComponentAuctionResult(*auction_result, request_generation_id, seller_domain, error_accumulator)) { - PS_VLOG(1, log_context) + PS_LOG(ERROR, log_context) << "Auction result skipped with failed validation for: " << auction_result->auction_params().component_seller(); continue; } + auto [itr, inserted] = component_sellers.insert( + auction_result->auction_params().component_seller()); + // Duplicate seller name. + if (!inserted) { + std::string error_msg = absl::StrCat( + kMultipleComponentAuctionResultsError, + auction_result->auction_params().component_seller(), "."); + // Report error. This will be later returned to the ad server. + // Marked as CLIENT_SIDE since the error originates from the + // API client. + error_accumulator.ReportError(ErrorVisibility::AD_SERVER_VISIBLE, + std::move(error_msg), + ErrorCode::CLIENT_SIDE); + // Return empty vector to abort auction. + return std::vector(); + } PS_VLOG(2, log_context) << "Successfully validated auction result for: " << auction_result->auction_params().component_seller(); diff --git a/services/seller_frontend_service/util/proto_mapping_util_test.cc b/services/seller_frontend_service/util/proto_mapping_util_test.cc index 4b0a064c..d705189a 100644 --- a/services/seller_frontend_service/util/proto_mapping_util_test.cc +++ b/services/seller_frontend_service/util/proto_mapping_util_test.cc @@ -62,6 +62,7 @@ AuctionResult MapBasicScoreFieldsToAuctionResult( auction_result.set_score(high_score.desirability()); auction_result.set_interest_group_name(high_score.interest_group_name()); auction_result.set_interest_group_owner(high_score.interest_group_owner()); + auction_result.set_interest_group_origin(high_score.interest_group_origin()); auction_result.set_ad_render_url(high_score.render()); auction_result.mutable_win_reporting_urls() ->mutable_buyer_reporting_urls() @@ -436,8 +437,11 @@ TEST_F(CreateAuctionResultCiphertextTest, ConvertsAdScoreForAndroid) { TEST_F(CreateAuctionResultCiphertextTest, ConvertsAdScoreForWeb) { AuctionResult expected = MapBasicScoreFieldsToAuctionResult(this->valid_score_); - // Should not populate bid value for single seller auction. + // bid val is only for component and auction and will not be parsed or encoded + // for single seller auction. expected.clear_bid(); + // IG Origin is Android exclusive and will not be parsed or encoded for web. + expected.clear_interest_group_origin(); expected.mutable_bidding_groups()->insert(this->bidding_group_map_.begin(), this->bidding_group_map_.end()); auto decrypted_message = MakeDecryptedMessage(); diff --git a/services/seller_frontend_service/util/select_ad_reactor_test_utils.cc b/services/seller_frontend_service/util/select_ad_reactor_test_utils.cc index f3dee1ab..9c898036 100644 --- a/services/seller_frontend_service/util/select_ad_reactor_test_utils.cc +++ b/services/seller_frontend_service/util/select_ad_reactor_test_utils.cc @@ -92,11 +92,12 @@ GetBidsResponse::GetBidsRawResponse BuildGetBidsResponseWithSingleAd( absl::optional bid_value, const bool enable_event_level_debug_reporting, int number_ad_component_render_urls, - const absl::optional& bid_currency) { - AdWithBid bid = - BuildNewAdWithBid(ad_url, std::move(interest_group_name), bid_value, - enable_event_level_debug_reporting, - number_ad_component_render_urls, bid_currency); + const absl::optional& bid_currency, + absl::string_view buyer_reporting_id) { + AdWithBid bid = BuildNewAdWithBid( + ad_url, std::move(interest_group_name), bid_value, + enable_event_level_debug_reporting, number_ad_component_render_urls, + bid_currency, buyer_reporting_id); GetBidsResponse::GetBidsRawResponse response; response.mutable_bids()->Add(std::move(bid)); return response; @@ -146,7 +147,8 @@ void SetupBuyerClientMock( } void BuildAdWithBidFromAdWithBidMetadata(const AdWithBidMetadata& input, - AdWithBid* result) { + AdWithBid* result, + absl::string_view buyer_reporting_id) { if (input.has_ad()) { *result->mutable_ad() = input.ad(); } @@ -158,6 +160,9 @@ void BuildAdWithBidFromAdWithBidMetadata(const AdWithBidMetadata& input, result->set_interest_group_name(input.interest_group_name()); result->set_ad_cost(kAdCost); result->set_modeling_signals(kModelingSignals); + if (!buyer_reporting_id.empty()) { + result->set_buyer_reporting_id(buyer_reporting_id); + } } AdWithBid BuildNewAdWithBid( @@ -166,7 +171,8 @@ AdWithBid BuildNewAdWithBid( absl::optional bid_value, const bool enable_event_level_debug_reporting, int number_ad_component_render_urls, - const absl::optional& bid_currency) { + const absl::optional& bid_currency, + absl::string_view buyer_reporting_id) { AdWithBid bid; bid.set_render(ad_url); for (int i = 0; i < number_ad_component_render_urls; i++) { @@ -193,6 +199,9 @@ AdWithBid BuildNewAdWithBid( "https://test.com/debugLoss?render=" + ad_url); *bid.mutable_debug_report_urls() = debug_report_urls; } + if (!buyer_reporting_id.empty()) { + bid.set_buyer_reporting_id(buyer_reporting_id); + } return bid; } diff --git a/services/seller_frontend_service/util/select_ad_reactor_test_utils.h b/services/seller_frontend_service/util/select_ad_reactor_test_utils.h index b30c0d74..59212cee 100644 --- a/services/seller_frontend_service/util/select_ad_reactor_test_utils.h +++ b/services/seller_frontend_service/util/select_ad_reactor_test_utils.h @@ -97,7 +97,8 @@ GetBidsResponse::GetBidsRawResponse BuildGetBidsResponseWithSingleAd( absl::optional bid_value = absl::nullopt, const bool enable_event_level_debug_reporting = false, int number_ad_component_render_urls = kDefaultNumAdComponents, - const absl::optional& bid_currency = absl::nullopt); + const absl::optional& bid_currency = absl::nullopt, + absl::string_view buyer_reporting_id = ""); void SetupMockCrytoClient(MockCryptoClientWrapper& crypto_client); @@ -111,7 +112,7 @@ void SetupBuyerClientMock( void BuildAdWithBidFromAdWithBidMetadata( const ScoreAdsRequest::ScoreAdsRawRequest::AdWithBidMetadata& input, - AdWithBid* result); + AdWithBid* result, absl::string_view buyer_reporting_id = ""); AdWithBid BuildNewAdWithBid( const std::string& ad_url, @@ -119,7 +120,8 @@ AdWithBid BuildNewAdWithBid( absl::optional bid_value = absl::nullopt, const bool enable_event_level_debug_reporting = false, int number_ad_component_render_urls = kDefaultNumAdComponents, - const absl::optional& bid_currency = absl::nullopt); + const absl::optional& bid_currency = absl::nullopt, + absl::string_view buyer_reporting_id = ""); ProtectedAppSignalsAdWithBid BuildNewPASAdWithBid( const std::string& ad_render_url, absl::optional bid_value, @@ -216,7 +218,8 @@ EncryptedSelectAdRequestWithContext GetSampleSelectAdRequest( ClientType client_type, absl::string_view seller_origin_domain, bool is_consented_debug = false, absl::string_view top_level_seller = "", EncryptionCloudPlatform top_seller_cloud_platform = - EncryptionCloudPlatform::ENCRYPTION_CLOUD_PLATFORM_UNSPECIFIED) { + EncryptionCloudPlatform::ENCRYPTION_CLOUD_PLATFORM_UNSPECIFIED, + bool enable_unlimited_egress = false) { BuyerInput buyer_input; auto* interest_group = buyer_input.mutable_interest_groups()->Add(); interest_group->set_name(kSampleInterestGroupName); @@ -242,6 +245,7 @@ EncryptedSelectAdRequestWithContext GetSampleSelectAdRequest( SelectAdRequest request; T protected_auction_input; protected_auction_input.set_generation_id(kSampleGenerationId); + protected_auction_input.set_enable_unlimited_egress(enable_unlimited_egress); if (is_consented_debug) { auto* consented_debug_config = protected_auction_input.mutable_consented_debug_config(); diff --git a/services/seller_frontend_service/util/web_utils.cc b/services/seller_frontend_service/util/web_utils.cc index f0ebb492..9e4b2b02 100644 --- a/services/seller_frontend_service/util/web_utils.cc +++ b/services/seller_frontend_service/util/web_utils.cc @@ -319,6 +319,10 @@ absl::Status CborSerializeScoreAdResponse( error_handler, root)); PS_RETURN_IF_ERROR( CborSerializeBiddingGroups(bidding_group_map, error_handler, root)); + if (!ad_score.buyer_reporting_id().empty()) { + PS_RETURN_IF_ERROR(CborSerializeString( + kBuyerReportingId, ad_score.buyer_reporting_id(), error_handler, root)); + } PS_RETURN_IF_ERROR(CborSerializeWinReportingUrls( ad_score.win_reporting_urls(), error_handler, root)); PS_RETURN_IF_ERROR(CborSerializeString( @@ -1316,6 +1320,13 @@ absl::StatusOr CborDecodeAuctionResultToProto( } auction_result.set_bid_currency(CborDecodeString(kv.value)); } break; + case 13: { // kBuyerReportingId + if (!cbor_isa_string(kv.value)) { + return absl::InvalidArgumentError( + "Expected buyer reporting id to be a string"); + } + auction_result.set_buyer_reporting_id(CborDecodeString(kv.value)); + } break; default: // Unexpected key in the auction result CBOR return absl::Status( diff --git a/services/seller_frontend_service/util/web_utils.h b/services/seller_frontend_service/util/web_utils.h index af39afd1..b468c843 100644 --- a/services/seller_frontend_service/util/web_utils.h +++ b/services/seller_frontend_service/util/web_utils.h @@ -355,7 +355,8 @@ inline constexpr std::array kWinReportingUrls, // 9 kAdMetadata, // 10 kTopLevelSeller, // 11 - kBidCurrency // 12 + kBidCurrency, // 12 + kBuyerReportingId // 13 }; template diff --git a/services/seller_frontend_service/util/web_utils_test.cc b/services/seller_frontend_service/util/web_utils_test.cc index 6f5e55f8..a2642ab8 100644 --- a/services/seller_frontend_service/util/web_utils_test.cc +++ b/services/seller_frontend_service/util/web_utils_test.cc @@ -68,6 +68,8 @@ inline constexpr char kTestReportResultUrl[] = "http://reportResult.com"; inline constexpr char kTestReportWinUrl[] = "http://reportWin.com"; inline constexpr char kConsentedDebugToken[] = "xyz"; inline constexpr char kUsdIso[] = "USD"; +inline constexpr char kTestBuyerReportingId[] = "testBuyerReportingId"; + server_common::log::ContextImpl log_context{ {}, server_common::ConsentedDebugConfiguration()}; @@ -714,6 +716,7 @@ TEST(ChromeResponseUtils, VerifyCborEncodingWithWinReportingUrls) { ->mutable_component_seller_reporting_urls() ->mutable_interaction_reporting_urls() ->try_emplace(kTestEvent1, kTestInteractionUrl1); + winner.set_buyer_reporting_id(kTestBuyerReportingId); // Setup a bidding group map. google::protobuf::Map bidding_group_map; @@ -777,6 +780,7 @@ TEST(ChromeResponseUtils, VerifyCborEncodingWithWinReportingUrls) { .interaction_reporting_urls() .at(kTestEvent1), kTestInteractionUrl1); + EXPECT_EQ(decoded_result->buyer_reporting_id(), kTestBuyerReportingId); } TEST(ChromeResponseUtils, VerifyCborEncoding) { @@ -1313,6 +1317,63 @@ TEST(ChromeResponseUtils, VerifyMinimalComponentResponseEncodingNoBidCurrency) { "2e636f6d"); } +TEST(ChromeResponseUtils, VerifyMinimalEncodingWithBuyerReportingId) { + ScoreAdsResponse::AdScore winner; + winner.set_interest_group_owner("https://adtech.com"); + winner.set_interest_group_name("ig1"); + winner.set_desirability(156671.781); + // Should accomplish nothing as bid should not be included in the response for + // non-component auction. + winner.set_buyer_bid(0.195839122); + winner.mutable_win_reporting_urls() + ->mutable_buyer_reporting_urls() + ->set_reporting_url(kTestReportWinUrl); + winner.mutable_win_reporting_urls() + ->mutable_buyer_reporting_urls() + ->mutable_interaction_reporting_urls() + ->try_emplace(kTestEvent1, kTestInteractionUrl1); + winner.mutable_win_reporting_urls() + ->mutable_top_level_seller_reporting_urls() + ->set_reporting_url(kTestReportResultUrl); + winner.mutable_win_reporting_urls() + ->mutable_top_level_seller_reporting_urls() + ->mutable_interaction_reporting_urls() + ->try_emplace(kTestEvent1, kTestInteractionUrl1); + winner.set_buyer_reporting_id(kTestBuyerReportingId); + const std::string interest_group_owner_1 = "ig1"; + const std::string interest_group_owner_2 = "zi"; + const std::string interest_group_owner_3 = "ih1"; + AuctionResult::InterestGroupIndex indices; + indices.add_index(7); + indices.add_index(2); + google::protobuf::Map + bidding_group_map; + bidding_group_map.try_emplace(interest_group_owner_1, indices); + bidding_group_map.try_emplace(interest_group_owner_2, indices); + bidding_group_map.try_emplace(interest_group_owner_3, indices); + bidding_group_map.try_emplace("owner1", std::move(indices)); + + auto ret = Encode(std::move(winner), bidding_group_map, std::nullopt, + [](auto error) {}); + ASSERT_TRUE(ret.ok()) << ret.status(); + // Conversion can be verified at: https://cbor.me/ + EXPECT_EQ(absl::BytesToHexString(*ret), + "a96573636f7265fa4818fff26769734368616666f46a636f6d706f6e656e747" + "3806b616452656e64657255524c606d62696464696e6747726f757073a4627a6" + "98207026369673182070263696831820702666f776e657231820702706275796" + "5725265706f7274696e674964747465737442757965725265706f7274696e674" + "9647077696e5265706f7274696e6755524c73a27262757965725265706f72746" + "96e6755524c73a26c7265706f7274696e6755524c74687474703a2f2f7265706" + "f727457696e2e636f6d7818696e746572616374696f6e5265706f7274696e675" + "5524c73a165636c69636b70687474703a2f2f636c69636b2e636f6d781b746f7" + "04c6576656c53656c6c65725265706f7274696e6755524c73a26c7265706f727" + "4696e6755524c77687474703a2f2f7265706f7274526573756c742e636f6d781" + "8696e746572616374696f6e5265706f7274696e6755524c73a165636c69636b7" + "0687474703a2f2f636c69636b2e636f6d71696e74657265737447726f75704e6" + "16d656369673172696e74657265737447726f75704f776e65727268747470733" + "a2f2f6164746563682e636f6d"); +} + MATCHER_P(ProtoEq, value, "") { AuctionResult result; CHECK(google::protobuf::TextFormat::ParseFromString(value, &result)); diff --git a/third_party/bazel_clang_tidy.patch b/third_party/bazel_clang_tidy.patch deleted file mode 100644 index 5999f748..00000000 --- a/third_party/bazel_clang_tidy.patch +++ /dev/null @@ -1,18 +0,0 @@ -diff --git a/clang_tidy/run_clang_tidy.sh b/clang_tidy/run_clang_tidy.sh -index 5cddb59..c33d0cf 100755 ---- a/clang_tidy/run_clang_tidy.sh -+++ b/clang_tidy/run_clang_tidy.sh -@@ -35,6 +35,13 @@ trap 'if (($?)); then cat "$logfile" 1>&2; fi; rm "$logfile"' EXIT - set -- \ - --checks=-clang-diagnostic-builtin-macro-redefined \ - --warnings-as-errors=-clang-diagnostic-builtin-macro-redefined \ -+ -extra-arg=-Wno-unknown-warning-option \ -+ -extra-arg=-Wno-unused-result \ -+ -extra-arg=-Wno-macro-redefined \ -+ -extra-arg=-Wno-deprecated-declarations \ -+ -extra-arg=-Wno-unused-const-variable \ -+ -extra-arg=-Wno-format \ -+ -extra-arg=-Wno-unused-function \ - "$@" - - "${CLANG_TIDY_BIN}" "$@" >"$logfile" 2>&1 diff --git a/tools/debug/README.md b/tools/debug/README.md index 7ce50d8c..784bd96b 100644 --- a/tools/debug/README.md +++ b/tools/debug/README.md @@ -82,18 +82,6 @@ Docker daemon should be up and running. #### Start Buyer stack ```bash -# Find Bazel Build Path -# The path for the server binaries will be different in case of docker builds. You can find the root -# of the build directory as follows - -# Run in bidding-auction-server root to find bazel build root -ls -al -# Should print out the following as part of the output -# bazel-bin -> /bazel_root/build_ubuntu_b5f37d9/eb7c660ef3781542ec00a071f7f762a5/execroot/__main__/bazel-out/k8-opt/bin - -# Copy the directory name starting [build_ubuntu_b5f37d9/eb7c660ef3781542ec00a071f7f762a5] -# Find the root of bazel cache in your system. Should be ~/.cache/bazel/ -# Final path is a combination of the two - ~/.cache/bazel/build_ubuntu_b5f37d9/eb7c660ef3781542ec00a071f7f762a5 - # Edit the run time flags in the scripts: # Eg. change the biddingJsUrl in tools/debug/start_bidding for custom generateBid script. # Eg. change the buyer_kv_server_addr in tools/debug/start_bfe for custom KV server. @@ -101,11 +89,11 @@ ls -al # Open two new terminals at B&A project root. # Start the Bidding server in terminal 1 with bazel build folder: -./tools/debug/start_bidding ~/.cache/bazel/build_ubuntu_b5f37d9/eb7c660ef3781542ec00a071f7f762a5 +./tools/debug/start_bidding # You should see some logs in each server as it displays HTTP metrics for the first call to the generateBid JS endpoint and some errors for OTEL collectors not found. # Start the BuyerFrontEnd server in terminal 2 with bazel build folder: -./tools/debug/start_bfe ~/.cache/bazel/build_ubuntu_b5f37d9/eb7c660ef3781542ec00a071f7f762a5 +./tools/debug/start_bfe # You should see some logs in each server as it displays HTTP metrics for the first call to the KV server and some errors for OTEL collectors not found. ``` @@ -118,18 +106,6 @@ ls -al #### Start Seller stack ```bash -# Find Bazel Build Path -# The path for the server binaries will be different in case of docker builds. You can find the root -# of the build directory as follows - -# Run in bidding-auction-server root to find bazel build root -ls -al -# Should print out the following as part of the output -# bazel-bin -> /bazel_root/build_ubuntu_b5f37d9/eb7c660ef3781542ec00a071f7f762a5/execroot/__main__/bazel-out/k8-opt/bin - -# Copy the directory name starting [build_ubuntu_b5f37d9/eb7c660ef3781542ec00a071f7f762a5] -# Find the root of bazel cache in your system. Should be ~/.cache/bazel/ -# Final path is a combination of the two - ~/.cache/bazel/build_ubuntu_b5f37d9/eb7c660ef3781542ec00a071f7f762a5 - # Edit the run time flags in the scripts: # Eg. change the auctionJsUrl in tools/debug/start_auction for custom scoreAd script. # Eg. change the key_value_signals_host in tools/debug/start_sfe for custom KV server. @@ -137,11 +113,11 @@ ls -al # Open two new terminals at B&A project root. # Start the Auction server in terminal 1 with bazel build folder: -./tools/debug/start_auction ~/.cache/bazel/build_ubuntu_b5f37d9/eb7c660ef3781542ec00a071f7f762a5 +./tools/debug/start_auction # You should see some logs in each server as it displays HTTP metrics for the first call to the scoreAd JS endpoint and some errors for OTEL collectors not found. # Start the SellerFrontEnd server in terminal 2 with bazel build folder: -./tools/debug/start_sfe ~/.cache/bazel/build_ubuntu_b5f37d9/eb7c660ef3781542ec00a071f7f762a5 +./tools/debug/start_sfe # You should see some logs in each server as it displays HTTP metrics for the first call to the KV server and some errors for OTEL collectors not found. ``` @@ -180,7 +156,7 @@ there is no recommended way to get an encrypted ciphertext for testing. ### Test Seller stack -#### Plaintext request +#### Plaintext SelectAdRequest Plaintext requests need to be created manually. For the expected format for this request, please refer to the [secure_invoke] section. @@ -204,7 +180,7 @@ DOCKER_NETWORK=host ./builders/tools/bazel-debian run //tools/secure_invoke:invo -insecure=true ``` -#### Encrypted request +#### Encrypted SelectAdRequest Encrypted requests must be valid SelectAdRequests with an encrypted protectedAudienceCiphertext. The ciphertext can be obtained from the client side (Chrome/Android), and the auction config has to be @@ -230,6 +206,16 @@ populated manually. There are two ways to send encrypted requests to local serve curl --url localhost:51052/v1/selectAd -H 'X-BnA-Client-IP:' -H 'X-User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36' -H 'x-accept-language: en-US,en;q=0.9' -d "@select_ads_request.json" ``` +#### GetComponentAuctionCiphertexts + +Requests must be valid GetComponentAuctionCiphertextsRequest with an encrypted +protectedAuctionCiphertext. The encrypted ciphertext can be obtained using the [secure_invoke] tool +with -op='encrypt'. + +```bash +grpcurl --plaintext -d '{"protected_auction_ciphertext":"", "component_sellers":["component-seller1.com", "component-seller2.com", "component-seller3.com"]}' localhost:50053 privacy_sandbox.bidding_auction_servers.SellerFrontEnd/GetComponentAuctionCiphertexts +``` + Notes: - Change log level with the GLOG_v environment variable before starting the servers. To display diff --git a/tools/debug/start_auction b/tools/debug/start_auction index a807dab7..d1505b4e 100755 --- a/tools/debug/start_auction +++ b/tools/debug/start_auction @@ -32,6 +32,7 @@ export SERVER_START_CMD=$(cat << END --roma_timeout_ms="120000" \ --enable_report_win_input_noising="false" \ --seller_code_fetch_config='{ + "fetchMode": 0, "auctionJsPath": "", "auctionJsUrl": "https://pubads.g.doubleclick.net/td/sjs", "urlFetchPeriodMs": 13000000, diff --git a/tools/debug/start_sfe b/tools/debug/start_sfe index cf1c85af..9127cf71 100755 --- a/tools/debug/start_sfe +++ b/tools/debug/start_sfe @@ -27,6 +27,7 @@ export SERVER_START_CMD=$(cat << END --auction_server_host="127.0.0.1:50061" \ --key_value_signals_host="https://pubads.g.doubleclick.net/td/sts" \ --seller_origin_domain="https://securepubads.g.doubleclick.net" \ +--seller_cloud_platforms_map='{"component-seller1.com":"GCP", "component-seller2.com":"AWS"}' \ --buyer_server_hosts='{ "https://td.doubleclick.net": { "url": "127.0.0.1:50051", diff --git a/tools/secure_invoke/BUILD b/tools/secure_invoke/BUILD index a30b6a94..f0b2195f 100644 --- a/tools/secure_invoke/BUILD +++ b/tools/secure_invoke/BUILD @@ -31,9 +31,11 @@ cc_library( "//api:bidding_auction_servers_cc_grpc_proto", "//services/common/clients/seller_frontend_server:async_client", "//services/common/test/utils:ohttp_test_utils", + "//services/common/util:json_util", "//tools/secure_invoke/payload_generator:payload_packaging_lib", "//tools/secure_invoke/payload_generator:payload_packaging_utils", "@com_github_google_quiche//quiche:oblivious_http_unstable_api", + "@com_google_absl//absl/container:btree", "@google_privacysandbox_servers_common//src/encryption/key_fetcher:fake_key_fetcher_manager", ], ) diff --git a/tools/secure_invoke/flags.cc b/tools/secure_invoke/flags.cc index f31bd4fc..dac2272c 100644 --- a/tools/secure_invoke/flags.cc +++ b/tools/secure_invoke/flags.cc @@ -78,3 +78,7 @@ ABSL_FLAG(bool, enable_debug_reporting, false, ABSL_FLAG(std::string, pas_buyer_input_json, "", "PAS specific buyer input for each buyer app signal proto."); + +ABSL_FLAG(bool, enable_unlimited_egress, false, + "Set to true to indicate to server that temporary unlimited egress " + "should be enabled"); diff --git a/tools/secure_invoke/flags.h b/tools/secure_invoke/flags.h index 692a7695..361af405 100644 --- a/tools/secure_invoke/flags.h +++ b/tools/secure_invoke/flags.h @@ -37,6 +37,7 @@ ABSL_DECLARE_FLAG(std::string, public_key); ABSL_DECLARE_FLAG(std::string, private_key); ABSL_DECLARE_FLAG(std::string, key_id); ABSL_DECLARE_FLAG(bool, enable_debug_reporting); +ABSL_DECLARE_FLAG(bool, enable_unlimited_egress); ABSL_DECLARE_FLAG(std::string, pas_buyer_input_json); #endif // TOOLS_SECURE_INVOKE_FLAGS_H_ diff --git a/tools/secure_invoke/payload_generator/payload_packaging.cc b/tools/secure_invoke/payload_generator/payload_packaging.cc index 1489c833..d0cbfff6 100644 --- a/tools/secure_invoke/payload_generator/payload_packaging.cc +++ b/tools/secure_invoke/payload_generator/payload_packaging.cc @@ -84,7 +84,8 @@ SelectAdRequest::AuctionConfig GetAuctionConfig( } ProtectedAuctionInput GetProtectedAuctionInput( - rapidjson::Document* input_json, bool enable_debug_reporting = false) { + rapidjson::Document* input_json, bool enable_debug_reporting = false, + bool enable_unlimited_egress = false) { CHECK(input_json != nullptr) << "Input JSON must be non null"; rapidjson::Value& protected_auction_json = (*input_json)[kProtectedAuctionInputField]; @@ -98,6 +99,7 @@ ProtectedAuctionInput GetProtectedAuctionInput( protected_auction_json_str, &protected_auction_input, parse_options); // Enable debug reporting for all calls from this tool. protected_auction_input.set_enable_debug_reporting(enable_debug_reporting); + protected_auction_input.set_enable_unlimited_egress(enable_unlimited_egress); CHECK(protected_auction_input_parse.ok()) << protected_auction_input_parse; return protected_auction_input; } @@ -208,11 +210,12 @@ PackagePlainTextSelectAdRequest(absl::string_view input_json_str, ClientType client_type, const HpkeKeyset& keyset, bool enable_debug_reporting, - absl::string_view protected_app_signals_json) { + absl::string_view protected_app_signals_json, + bool enable_unlimited_egress) { rapidjson::Document input_json = ParseRequestInputJson(input_json_str); auto select_ad_request = std::make_unique(); - ProtectedAuctionInput protected_auction_input = - GetProtectedAuctionInput(&input_json, enable_debug_reporting); + ProtectedAuctionInput protected_auction_input = GetProtectedAuctionInput( + &input_json, enable_debug_reporting, enable_unlimited_egress); if (input_json.HasMember(kComponentAuctionsField)) { for (auto& component_auction_json : input_json[kComponentAuctionsField].GetArray()) { @@ -259,10 +262,12 @@ PackagePlainTextSelectAdRequest(absl::string_view input_json_str, std::string PackagePlainTextSelectAdRequestToJson( absl::string_view input_json_str, ClientType client_type, - const HpkeKeyset& keyset, bool enable_debug_reporting) { + const HpkeKeyset& keyset, bool enable_debug_reporting, + bool enable_unlimited_egress) { auto req = - std::move(PackagePlainTextSelectAdRequest(input_json_str, client_type, - keyset, enable_debug_reporting) + std::move(PackagePlainTextSelectAdRequest( + input_json_str, client_type, keyset, enable_debug_reporting, + /*protected_app_signals_json=*/"", enable_unlimited_egress) .first); std::string select_ad_json; auto select_ad_json_status = diff --git a/tools/secure_invoke/payload_generator/payload_packaging.h b/tools/secure_invoke/payload_generator/payload_packaging.h index bdef4ea1..b54f4a23 100644 --- a/tools/secure_invoke/payload_generator/payload_packaging.h +++ b/tools/secure_invoke/payload_generator/payload_packaging.h @@ -126,13 +126,15 @@ std::pair, PackagePlainTextSelectAdRequest( absl::string_view input_json_str, ClientType client_type, const HpkeKeyset& keyset, bool enable_debug_reporting = false, - absl::string_view protected_app_signals_json = ""); + absl::string_view protected_app_signals_json = "", + bool enable_unlimited_egress = false); // This method returns a SelectAdRequest json for testing B&A servers in // "test_mode" using the PackagePlainTextSelectAdRequest method. std::string PackagePlainTextSelectAdRequestToJson( absl::string_view input_json_str, ClientType client_type, - const HpkeKeyset& keyset, bool enable_debug_reporting = false); + const HpkeKeyset& keyset, bool enable_debug_reporting = false, + bool enable_unlimited_egress = false); } // namespace privacy_sandbox::bidding_auction_servers #endif // TOOLS_PAYLOAD_GENERATOR_PAYLOAD_PACKAGING_H_ diff --git a/tools/secure_invoke/payload_generator/payload_packaging_test.cc b/tools/secure_invoke/payload_generator/payload_packaging_test.cc index 4034232c..89d4e43f 100644 --- a/tools/secure_invoke/payload_generator/payload_packaging_test.cc +++ b/tools/secure_invoke/payload_generator/payload_packaging_test.cc @@ -515,6 +515,84 @@ TEST(PaylodPackagingTest, EXPECT_EQ(actual.component_auction_results_size(), 0); } +TEST(PaylodPackagingTest, SetsTheEnableUnlimitedEgressFlag) { + auto input = R"JSON( + { + "auction_config" : { + "auctionSignals" : "{}", + "buyerList" : [ + "1698245045005905922" + ], + "buyerTimeoutMs" : 1000, + "perBuyerConfig" : { + "1698245045005905922" : { + "buyerSignals" : "1698245045006101412" + } + }, + "seller" : "seller.com", + "sellerSignals" : "{}" + }, + "raw_protected_audience_input" : { + "raw_buyer_input" : { + "ad_tech_A.com" : { + "interestGroups" : [ + { + "adRenderIds" : [ + "ad_id1" + ], + "biddingSignalsKeys" : [ + "1" + ], + "browserSignals" : { + "bidCount" : "41", + "joinCount" : "8", + "prevWins" : "[[1,\"ad_id1\"]]", + "recency" : "1698245045" + }, + "name" : "1", + "userBiddingSignals" : "{\"1\":\"1\"}" + } + ] + } + } + } + } + )JSON"; + HpkeKeyset keyset; + auto select_ad_req = std::move(PackagePlainTextSelectAdRequest( + input, CLIENT_TYPE_ANDROID, keyset, + /*enable_debug_reporting=*/false, + /*protected_app_signals_json=*/"", + /*enable_unlimited_egress=*/true)) + .first; + + absl::StatusOr + parsed_encapsulated_request = server_common::ParseEncapsulatedRequest( + select_ad_req->protected_auction_ciphertext()); + ASSERT_TRUE(parsed_encapsulated_request.ok()) + << parsed_encapsulated_request.status(); + + // Decrypt. + server_common::PrivateKey private_key; + private_key.key_id = std::to_string(keyset.key_id); + private_key.private_key = GetHpkePrivateKey(keyset.private_key); + auto decrypted_response = server_common::DecryptEncapsulatedRequest( + private_key, *parsed_encapsulated_request); + ASSERT_TRUE(decrypted_response.ok()) << decrypted_response.status(); + + absl::StatusOr decoded_response = + server_common::DecodeRequestPayload( + decrypted_response->GetPlaintextData()); + ASSERT_TRUE(decoded_response.ok()) << decoded_response.status(); + + // Decode. + ProtectedAuctionInput protected_auction_input; + ASSERT_TRUE(protected_auction_input.ParseFromArray( + decoded_response->compressed_data.data(), + decoded_response->compressed_data.size())); + EXPECT_TRUE(protected_auction_input.enable_unlimited_egress()); +} + } // namespace } // namespace privacy_sandbox::bidding_auction_servers diff --git a/tools/secure_invoke/secure_invoke.cc b/tools/secure_invoke/secure_invoke.cc index a0680a81..37f8fd49 100644 --- a/tools/secure_invoke/secure_invoke.cc +++ b/tools/secure_invoke/secure_invoke.cc @@ -90,6 +90,7 @@ int main(int argc, char** argv) { }; bool enable_debug_reporting = absl::GetFlag(FLAGS_enable_debug_reporting); + bool enable_unlimited_egress = absl::GetFlag(FLAGS_enable_unlimited_egress); if (op == "encrypt") { if (target_service == kSfe) { json_input_str = @@ -97,22 +98,25 @@ int main(int argc, char** argv) { // LOG causes clipping of response. std::cout << privacy_sandbox::bidding_auction_servers:: PackagePlainTextSelectAdRequestToJson( - json_input_str, client_type, keyset, enable_debug_reporting); + json_input_str, client_type, keyset, enable_debug_reporting, + enable_unlimited_egress); } else { std::cout << privacy_sandbox::bidding_auction_servers:: - PackagePlainTextGetBidsRequestToJson(keyset, - enable_debug_reporting); + PackagePlainTextGetBidsRequestToJson( + keyset, enable_debug_reporting, enable_unlimited_egress); } } else if (op == "invoke") { if (target_service == kSfe) { const auto status = privacy_sandbox::bidding_auction_servers::SendRequestToSfe( - client_type, keyset, enable_debug_reporting); + client_type, keyset, enable_debug_reporting, + enable_unlimited_egress); CHECK(status.ok()) << status; } else if (target_service == kBfe) { const auto status = privacy_sandbox::bidding_auction_servers::SendRequestToBfe( - keyset, enable_debug_reporting); + keyset, enable_debug_reporting, /*stub=*/nullptr, + enable_unlimited_egress); CHECK(status.ok()) << status; } else { LOG(FATAL) << "Unsupported target service: " << target_service; diff --git a/tools/secure_invoke/secure_invoke_lib.cc b/tools/secure_invoke/secure_invoke_lib.cc index 4a9bd8af..b72de3b9 100644 --- a/tools/secure_invoke/secure_invoke_lib.cc +++ b/tools/secure_invoke/secure_invoke_lib.cc @@ -22,6 +22,7 @@ #include #include +#include "absl/container/btree_map.h" #include "absl/flags/flag.h" #include "quiche/oblivious_http/oblivious_http_client.h" #include "services/common/clients/async_grpc/grpc_client_utils.h" @@ -30,6 +31,7 @@ #include "services/common/constants/common_service_flags.h" #include "services/common/encryption/crypto_client_factory.h" #include "services/common/encryption/key_fetcher_factory.h" +#include "services/common/util/json_util.h" #include "src/encryption/key_fetcher/fake_key_fetcher_manager.h" #include "tools/secure_invoke/flags.h" #include "tools/secure_invoke/payload_generator/payload_packaging.h" @@ -41,6 +43,36 @@ namespace { constexpr char kJsonFormat[] = "JSON"; +absl::StatusOr SortAuctionResultBiddingGroups( + absl::string_view auction_result_json) { + std::string key_name = "biddingGroups"; + auto document = ParseJsonString(auction_result_json); + if (!document.ok()) { + return document.status(); + } + auto& d = *document; + absl::btree_map sorted_map; + if (d.HasMember(key_name.c_str())) { + for (auto& m : d[key_name.c_str()].GetObject()) { + sorted_map[m.name.GetString()] = m.value.GetObject(); + } + } + + d.RemoveMember(key_name.c_str()); // remove old unsorted map + + rapidjson::Value sorted_val(rapidjson::kObjectType); + + for (auto& m : sorted_map) { + rapidjson::Value key(m.first.c_str(), d.GetAllocator()); + sorted_val.AddMember(key, m.second, d.GetAllocator()); + } + + d.AddMember(rapidjson::Value().SetString(key_name.c_str(), d.GetAllocator()), + sorted_val, d.GetAllocator()); // add new sorted map + + return SerializeJsonDoc(d); +} + absl::StatusOr ParseSelectAdResponse( std::unique_ptr resp, ClientType client_type, quiche::ObliviousHttpRequest::Context& context, const HpkeKeyset& keyset) { @@ -63,7 +95,16 @@ absl::StatusOr ParseSelectAdResponse( if (!auction_result_json_status.ok()) { return auction_result_json_status; } - return auction_result_json; + // Sort bidding groups for easy comparison. + auto sorted_auction_result_json_status = + SortAuctionResultBiddingGroups(auction_result_json); + std::string debug_info_json = resp->debug_info().DebugString(); + if (!debug_info_json.empty() && sorted_auction_result_json_status.ok()) { + return absl::StrCat(R"JSON({"debug_info":")JSON", debug_info_json, + R"JSON(", "auction_result":")JSON", + *sorted_auction_result_json_status, R"JSON("})JSON"); + } + return sorted_auction_result_json_status; } } // namespace @@ -72,6 +113,7 @@ absl::Status InvokeSellerFrontEndWithRawRequest( absl::string_view raw_select_ad_request_json, const RequestOptions& request_options, ClientType client_type, const HpkeKeyset& keyset, bool enable_debug_reporting, + bool enable_unlimited_egress, absl::AnyInvocable) &&> on_done) { // Validate input if (request_options.host_addr.empty()) { @@ -95,7 +137,8 @@ absl::Status InvokeSellerFrontEndWithRawRequest( quiche::ObliviousHttpRequest::Context> request_context_pair = PackagePlainTextSelectAdRequest( raw_select_ad_request_json, client_type, keyset, - enable_debug_reporting, absl::GetFlag(FLAGS_pas_buyer_input_json)); + enable_debug_reporting, absl::GetFlag(FLAGS_pas_buyer_input_json), + enable_unlimited_egress); // Add request headers. RequestMetadata request_metadata; @@ -204,7 +247,8 @@ std::string LoadFile(absl::string_view file_path) { } absl::Status SendRequestToSfe(ClientType client_type, const HpkeKeyset& keyset, - bool enable_debug_reporting) { + bool enable_debug_reporting, + bool enable_unlimited_egress) { std::string raw_select_ad_request_json = absl::GetFlag(FLAGS_json_input_str); if (raw_select_ad_request_json.empty()) { raw_select_ad_request_json = LoadFile(absl::GetFlag(FLAGS_input_file)); @@ -219,7 +263,7 @@ absl::Status SendRequestToSfe(ClientType client_type, const HpkeKeyset& keyset, absl::Status status = privacy_sandbox::bidding_auction_servers:: InvokeSellerFrontEndWithRawRequest( raw_select_ad_request_json, options, client_type, keyset, - enable_debug_reporting, + enable_debug_reporting, enable_unlimited_egress, [¬ification](absl::StatusOr output) { if (output.ok()) { // Standard output to compare response @@ -236,12 +280,13 @@ absl::Status SendRequestToSfe(ClientType client_type, const HpkeKeyset& keyset, } GetBidsRequest::GetBidsRawRequest GetBidsRawRequestFromInput( - bool enable_debug_reporting) { + bool enable_debug_reporting, bool enable_unlimited_egress) { std::string raw_get_bids_request_str = absl::GetFlag(FLAGS_json_input_str); const bool is_json = (!raw_get_bids_request_str.empty() || absl::GetFlag(FLAGS_input_format) == kJsonFormat); GetBidsRequest::GetBidsRawRequest get_bids_raw_request; get_bids_raw_request.set_enable_debug_reporting(enable_debug_reporting); + get_bids_raw_request.set_enable_unlimited_egress(enable_unlimited_egress); if (is_json) { if (raw_get_bids_request_str.empty()) { raw_get_bids_request_str = LoadFile(absl::GetFlag(FLAGS_input_file)); @@ -263,9 +308,11 @@ GetBidsRequest::GetBidsRawRequest GetBidsRawRequestFromInput( } std::string PackagePlainTextGetBidsRequestToJson(const HpkeKeyset& keyset, - bool enable_debug_reporting) { + bool enable_debug_reporting, + bool enable_unlimited_egress) { GetBidsRequest::GetBidsRawRequest get_bids_raw_request = - GetBidsRawRequestFromInput(enable_debug_reporting); + GetBidsRawRequestFromInput(enable_debug_reporting, + enable_unlimited_egress); auto key_fetcher_manager = std::make_unique( keyset.public_key, "unused", std::to_string(keyset.key_id)); @@ -287,9 +334,11 @@ std::string PackagePlainTextGetBidsRequestToJson(const HpkeKeyset& keyset, absl::Status SendRequestToBfe( const HpkeKeyset& keyset, bool enable_debug_reporting, - std::unique_ptr stub) { + std::unique_ptr stub, + bool enable_unlimited_egress) { GetBidsRequest::GetBidsRawRequest get_bids_raw_request = - GetBidsRawRequestFromInput(enable_debug_reporting); + GetBidsRawRequestFromInput(enable_debug_reporting, + enable_unlimited_egress); privacy_sandbox::bidding_auction_servers::RequestOptions request_options; request_options.host_addr = absl::GetFlag(FLAGS_host_addr); request_options.client_ip = absl::GetFlag(FLAGS_client_ip); diff --git a/tools/secure_invoke/secure_invoke_lib.h b/tools/secure_invoke/secure_invoke_lib.h index bc7abf71..7d396cae 100644 --- a/tools/secure_invoke/secure_invoke_lib.h +++ b/tools/secure_invoke/secure_invoke_lib.h @@ -39,13 +39,15 @@ struct RequestOptions { // Sends a request to SFE. The parameters used for the request are retrieved // from absl flags that are used to run the script. absl::Status SendRequestToSfe(ClientType client_type, const HpkeKeyset& keyset, - bool enable_debug_reporting); + bool enable_debug_reporting, + bool enable_unlimited_egress); // Sends a request to BFE. The parameters used for the request are retrieved // from absl flags that are used to run the script. absl::Status SendRequestToBfe( const HpkeKeyset& keyset, bool enable_debug_reporting, - std::unique_ptr stub = nullptr); + std::unique_ptr stub = nullptr, + bool enable_unlimited_egress = false); // Gets contents of the provided file path. std::string LoadFile(absl::string_view file_path); @@ -53,7 +55,8 @@ std::string LoadFile(absl::string_view file_path); // Returns a JSON string of the OHTTP encrypted of the input GetBidsRawRequest // to the secure invoke tool. std::string PackagePlainTextGetBidsRequestToJson(const HpkeKeyset& keyset, - bool enable_debug_reporting); + bool enable_debug_reporting, + bool enable_unlimited_egress); } // namespace privacy_sandbox::bidding_auction_servers diff --git a/version.txt b/version.txt index fbcbf738..e5b82034 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -3.4.0 \ No newline at end of file +3.5.0 \ No newline at end of file