diff --git a/.bazelrc b/.bazelrc index 1f409952..105b1c82 100644 --- a/.bazelrc +++ b/.bazelrc @@ -32,13 +32,14 @@ build:clang --cxxopt=-fsized-deallocation build:clang --host_cxxopt=-fsized-deallocation build:clang --cxxopt=-fnew-alignment=8 build:clang --host_cxxopt=-fnew-alignment=8 +build:clang --per_file_copt=api/udf/.*@-fconstexpr-steps=1271242 # Flag compiler warnings as errors. build:cpp_no_warn --copt=-Werror # Ignore warnings from Roma, zlib, differential privacy repo etc. build:cpp_no_warn --per_file_copt=.*external/.*@-Wno-error # Ignore deprecated declarations warnings from Roma, ProtectedAudience, Telemetry -build:cpp_no_warn --per_file_copt=services/.*@-Wno-macro-redefined,-Wno-deprecated-declarations +build:cpp_no_warn --per_file_copt=(services|api/udf)/.*@-Wno-macro-redefined,-Wno-deprecated-declarations build:cpp_no_warn --per_file_copt=tools/secure_invoke.*@-Wno-macro-redefined,-Wno-deprecated-declarations # Telemetry has some unchecked results after registering observers. build:cpp_no_warn --per_file_copt=services/.*_main.cc@-Wno-unused-result diff --git a/CHANGELOG.md b/CHANGELOG.md index b5c40fcc..65da77d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,183 @@ All notable changes to this project will be documented in this file. See [commit-and-tag-version](https://github.com/absolute-version/commit-and-tag-version) for commit guidelines. +## 4.0.0 (2024-09-09) + + +### ⚠ BREAKING CHANGES + +* Move serialization of bidding signals per IG from Bidding to BFE + +### Features + +* [IsolateBuyerAndSellerCodeExecution] Add wrapper for buyer's reportWin() udf +* [PAS threat mitigation] Add support for egressVector and temporaryUnlimittedEgressVector by replacing the $extraArgs +* [Private Aggregation] Add all the contributions from Auction Service to AuctionResult +* [Private Aggregation] Add integration test to verify parsing of PAAPI event +* [Private Aggregation] Add PrivateAggregateReportingResponse to AuctionResult proto +* [Private Aggregation] Convert 128 bit bucket into array of 64 bit integer in wrapper +* [Private Aggregation] Convert bucket offset object to SignalBucket +* [Private Aggregation] Convert value object in contribution to SignalValue +* [PrivateAggregation] Add PrivateAggregateReportingResponse in Auction Service response +* [PrivateAggregation] Append private aggregation wrapper to existing seller's wrapper +* [reporting] Enable reportResult execution for topLevelSeller +* Add `common.privateAggregation.createContribution` function for creating a contribution object. +* Add `common.privateAggregation.ReservedEvent` object that holds reserved events string constants. +* Add a script to freeze a given TensorFlow model +* Add a script to list all TensorFlow ops +* Add a script to list all Torchscript ops +* Add AppendAdEventContributionsToPaggResponse with SignalValue only +* Add auction_service.privateAggregation.contributeToHistogram/OnEvent +* Add bucket protos (Bucket128Bit, BucketOffset, SignalBucket, PrivateAggregationBucket) and add bucket and value fields to PrivateAggregateContribution +* Add chaffing feature flags in SFE/BFE. +* Add checksum capability to model loading +* Add common utils to create test private aggregation contribution and response +* Add config flag enable_private_aggregate_reporting to bidding service's runtime_config +* Add config flag enable_private_aggregation_generation to auction service's runtime_config +* Add ConvertSignalValueToInt for calculating final value from contribution's base value, scale, and offset. +* Add distribution of IGs related metrics +* Add enable_private_aggregate_reporting for GetSellerWrappedCode and unit tests that uses GetSellerWrappedCode +* Add event code and private aggregation objects for contributeToHistogramOnEvent, also added tests for event field +* add EventMessage to log context +* Add HandlePrivateAggregateReporting with support of SignalValue only +* Add HandlePrivateAggregationContributions, iterate over AdWithBids, filter contributions, post processing, test +* Add helper functions convertEventToInt and convertBaseValueToInt and tests for them for contributeToHistogramOnEvent for bidding service +* Add Inference Metrics like count, duration, error and size to TF & Pytorch sidecars. +* Add Inference Request count, duration and failure metric partitioned by model path for Tensorflow and Pytorch sidecars. +* Add metrics for inference model fetching and registration +* add option to switch DebugInfo in secure invoke encrypt +* Add parser function for Private Aggregation's Signal Bucket Object +* Add parser function for Private Aggregation's Signal Value Object +* Add PrivateAggregateContribution proto with event field and enum EventType to the .proto file +* Add proto messages for SignalValue, PrivateAggregationValue, and BaseValue enum. +* Add request creation timestamp to ProtectedAuctionInput +* Add request/response to ExecuteInternal on async clients +* Add support for collecting metrics from the Roma callbacks. +* Add tee-container-log-redirect option in terraform +* Add ToBaseValue Helper Function to convert corresponding base value strings to BaseValue Enum +* Add ToEventTypeString for converting EventType enum into its corresponding string in JSON object returned by ROMA +* Add utility function to calculate Signal Bucket's final value post auction ([3da9ab5]( )), closes [1#L192]( ) +* Add utility function to parse and return BucketOffset from rapidjson document +* Add wrapper and test files with method headers for contributeToHistogramOnEvent for bidding service +* Added `isValidCustomEvent` validation function for common.privateAggregation +* Added isValidValue validation function for common.privateAggregation +* Added new files for JS private aggregation util for bidding service +* Bash script for ASG and Cloud Map Custom HealthChecks +* Bid Currency Support for Top-Level Seller in Server-Orchestrated Multi-Seller Auctions +* Change js helper convertEventToInt to mapEventToEnum to return string corresponding to enum instead of int +* configured public key urls are verified against an explicit allowlist +* Consented request replace enable_adtech_code_logging in Bidding Server +* contributeToHistogram with test for bidding service +* contributeToHistogramOnEvent with tests, using common private aggregation helpers +* convert all remaining docker images into OCI +* Convert Private Aggregation wrapper functions from .js to C++ string +* Create a periodic model fetcher library for inference +* Create declarative spec for generateBid +* Create inference model store for model management +* Create logging library for inference consented logs +* Debug Reporting for Bid Currency +* Declare inference JS error schema +* Declare schema for model fetching metadata file +* decrease aws/build_and_test duration by ~75% +* Enable blob fetcher to only fetch blobs with given prefixes +* Enable dynamic model loading for inference +* Enable Protected App Signals by Default +* Enable Service Mesh on AWS by Default +* export non-privacy log through otel with safe system context +* Feature Flag for TLS in Service Mesh +* Force chaffing to enabled for prod builds on BFE + update common repo dep +* Force the ML model reset with the probability of 0.1% +* Forward per request consented debugging config to roma callback +* Freeze TorchScript models before serving +* Handle chaff and new request format on BFE +* Implement aggregated Error Reporting for Tensorflow and Pytorch sidecars. +* Implement AWS Cloud Un-Map +* Implement new SFE <> BFE request format for chaffing +* Implement ParseAndProcessContribution with SignalValue only +* Implement the probabilistic model reset for PyTorch +* Implement the probabilistic model reset for TensorFlow. +* Instrument inference JS error for the PyTorch sidecar +* Integrate with Trusted KV Server in Mesh +* Load models using model metadata config for the periodic model fetcher +* Load Test Flag for AWS +* Log consented debugging information in inference sidecar +* log EventMessage in servers +* log udf log in EventMessage for non_prod debug_info +* Make chaffing flag configurable via Terraform +* move ig metric to bfe +* Move serialization of bidding signals per IG from Bidding to BFE +* Propagate inference error back to JS caller +* Remove enableAdtechCodeLogging flag value +* Remove Envoy Access Logging +* Route consented inference requests to a consented model store +* send chaff requests from SFE +* Service Mesh in AWS +* Support CPU isolation in the inference sidecar +* Unit test for loading parsing fake contributions in Roma for bidding service +* Unpad KV server responses +* update code/cloud build to use tags +* Update Demo Terraform Configs with values for integrating TEE KV Servers into Mesh +* Upgrade AWS Provider for Terraform from v3.xx to v4.xx +* Use gRPC for AWS Service Mesh Envoy HCs +* Write a fake generateBid() script which calls privateAggregation.contributeToHistogramOnEvent(event, ) +* write event message to gcs for consented request +* write event message to s3 in aws for consented request + + +### Bug Fixes + +* [IsolateBuyerAndSellerCodeExecution] Add a new class for buyer's reporting code fetch and load +* [IsolateBuyerAndSellerCodeExecution] Add a new code wrapper with only scoreAd and reportResult +* [IsolateBuyerAndSellerCodeExecution] Add config flag to enable seller and buyer code isolation +* [IsolateBuyerAndSellerCodeExecution] Modify seller_udf_manager to fetch and load buyer udfs +* [IsolateBuyerAndSellerCodeExecution] Refactor the code fetch files. +* Add BidCurrency SUT to Pre-Submit +* add check back in default grpc client +* Add createContribution inside contributeToHistogram/OnEvent and fixed typo with createContribution's value conditional statement. +* Add DebugInfo pointer for debugging log into RomaRequestContext +* Add generation_id to chaff requests +* Add handling when rejection reason is specified as base value but not available and statuscode documentation for HandlePrivateAggregationValuePostAuction +* Adds domain equality validations to buyer reporting UDF URL +* Align plaintext buyer request for basic SUT with encrypted version +* Allow up to 15 buyers when chaffing enabled +* chaffing bugs ([027d5bb]( )), closes [/github.com/abseil/abseil-cpp/blob/master/absl/container/internal/raw_hash_set.h#L1572]( ) +* Changed base value string to be converted in ToBaseValue function and test cases +* Changed raw string delimiter JSCODE to JS_CODE for consistency +* Changed the input format of HandlePrivateAggregationReporting, added support for parsing PAgg bucket, and add the logic for getting required BaseValues (winning-bid, highest-scoring-other-bid, rejection-reason). +* clean up log verbosity 3 +* Correct output_filter typo +* do not try to impersonate service accounts if TEST_MODE=true +* Don't set chaff size for non-chaff requests +* Eliminate Terraform Error Message about empty Authority Field +* Enable threat mitigation with seller and buyer code isolation +* Ensure instance id is set in logs on AWS when not using mesh. +* Execute Callback for empty HTTP request vector +* Fix bugs in reportWin URL validation in auction service +* Fix release notes by adding a dedicated tag on main to generate the changelog +* make num_chaff_requests not have a static lower bound +* Make terraform for aws delete cloud maps without error +* Populate temp rc file for inference sidecar +* Populate temp rc file for inference sidecar +* Populate temp rc file for inference sidecar +* Redirect misleading log to /dev/null +* Refactored conversion of private aggregation value post auction into returning absl::StatusOr instead of implicitly editing PrivateAggregationValue that was passed in. +* Revert wrk2 test runner to an earlier version +* scorecard.yaml version updates +* shuffle request order (real and chaff) on SFE +* truncate curl failure info for fail count metric +* undo rules_oci migration to fix hash stability issues +* Update setup_2 and demo terraform configs to valid state +* Update SUTs to include and test Experiment Group ID +* Upgrades google terraform plugin to 5.31.0 to fix crash + + +### Documentation + +* Add PAS input example to BFE +* Document how to deployment B&A without inference +* Update inference_sidecar README with model size limits +* Updated expected format for INFERENCE_MODEL_BUCKET_PATHS in README + ## 3.11.0 (2024-08-19) diff --git a/WORKSPACE b/WORKSPACE index 8b8d0154..7c1f7ef0 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,7 +1,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") ### register Python toolchain -- note this toolchain defines the path to a specific version of python -load("//builders/bazel:deps.bzl", "python_deps") +load("//builders/bazel:deps.bzl", "python_deps", "python_register_toolchains") http_archive( name = "io_bazel_rules_docker", @@ -9,7 +9,9 @@ http_archive( urls = ["https://github.com/bazelbuild/rules_docker/releases/download/v0.25.0/rules_docker-v0.25.0.tar.gz"], ) -python_deps("//builders/bazel") +python_deps() + +python_register_toolchains("//builders/bazel") # TODO: Remove bazel_clang_tidy once we sync to the common repo commit 9edb0c3 (4/3/2024) or later http_archive( @@ -23,11 +25,11 @@ http_archive( http_archive( name = "google_privacysandbox_servers_common", - # 2024-08-06 - sha256 = "776c90c2eb6961c4b834242db58ba187cf1e790c876c5e8ef4bdb912210b8266", - strip_prefix = "data-plane-shared-libraries-81262eb0b1dfad7b998eec6c6d38e902ed151482", + # 2024-08-26 + sha256 = "dcd09e9241b9e2e85dfa9bd6b12768391f20e4587cba20589c781e4be1ba64a2", + strip_prefix = "data-plane-shared-libraries-da1550404faa919ccbbdaf9e91e6225934ad1620", urls = [ - "https://github.com/privacysandbox/data-plane-shared-libraries/archive/81262eb0b1dfad7b998eec6c6d38e902ed151482.zip", + "https://github.com/privacysandbox/data-plane-shared-libraries/archive/da1550404faa919ccbbdaf9e91e6225934ad1620.zip", ], ) diff --git a/api/BUILD b/api/BUILD index c1ca3969..724aaab5 100644 --- a/api/BUILD +++ b/api/BUILD @@ -23,6 +23,7 @@ proto_library( name = "bidding_auction_servers_proto", srcs = ["bidding_auction_servers.proto"], deps = [ + "//api/udf:generate_bid_proto", "@com_google_googleapis//google/api:annotations_proto", "@com_google_protobuf//:struct_proto", "@google_privacysandbox_servers_common//src/logger:logger_proto", diff --git a/api/bidding_auction_servers.proto b/api/bidding_auction_servers.proto index 13d2d61b..e55792d8 100644 --- a/api/bidding_auction_servers.proto +++ b/api/bidding_auction_servers.proto @@ -16,6 +16,7 @@ syntax = "proto3"; package privacy_sandbox.bidding_auction_servers; +import "api/udf/generate_bid.proto"; import "google/api/annotations.proto"; import "google/protobuf/struct.proto"; import "src/logger/logger.proto"; @@ -196,6 +197,7 @@ message BuyerInput { } // Information about an Interest Group passed by the browser. +// (-- LINT.IfChange(browser_signals_bidding) --) message BrowserSignals { // Number of times the group was joined in the last 30 days. int64 join_count = 1; @@ -220,9 +222,12 @@ message BrowserSignals { // Only one of the recency or recency_ms is expected to present in the request. optional int64 recency_ms = 5; } +// (-- LINT.ThenChange(/services/bidding_service/udf/api/generate_bid.proto:browser_signals_generate_bid) --) // Information passed by Android. +// (-- LINT.IfChange(android_signals_bidding) --) message AndroidSignals {} +// (-- LINT.ThenChange(/services/bidding_service/udf/api/generate_bid.proto:android_signals_generate_bid) --) // Specifies type of the ad. It can help differentiate between ads when // B&A is supporting multiple ad targeting use cases concurrently. @@ -448,6 +453,17 @@ message AuctionResult { // Proptected App Signal: This refers to the buyer domain. string owner = 3; + // Owner of the IG to which the ad belongs to. + // Note: This is only populated for Protected Audience for + // Android apps. + string origin = 4; + + // Name of the Interest Group (Custom Audience) to which the ghost + // winner belongs to. + // Note: This is only populated for Protected Audience for + // Android apps. + string ig_name = 5; + // Private aggregation signals for the ghost winner. // Single seller auctions: This would correspond to a ghost winner // if available. @@ -461,7 +477,7 @@ message AuctionResult { int32 value = 2; } - optional GhostWinnerPrivateAggregationSignals ghost_winner_private_aggregation_signals = 4; + optional GhostWinnerPrivateAggregationSignals ghost_winner_private_aggregation_signals = 6; // In case of multiseller auction, the associated data for the ghost winner // will be returned so that the ghost winning bid can be scored @@ -496,9 +512,13 @@ message AuctionResult { // properties on the browser. string buyer_and_seller_reporting_id = 6; } - optional GhostWinnerForTopLevelAuction ghost_winner_for_top_level_auction = 5; + optional GhostWinnerForTopLevelAuction ghost_winner_for_top_level_auction = 7; } repeated KAnonGhostWinner k_anon_ghost_winners = 23; + + // This field will be populated for all seller and buyer contributions in + // single seller auctions and server orchestrated multi seller auctions. + repeated PrivateAggregateReportingResponse top_level_contributions = 24; } message GetComponentAuctionCiphertextsRequest { @@ -968,6 +988,7 @@ message PrivateAggregateReportingResponse { } // Bid for an ad candidate. +// (-- LINT.IfChange(bid_bidding) --) message AdWithBid { // Metadata of the ad, this will be passed to Seller's scoring function. // Represents an opaque object that is eventually passed to seller Adtech @@ -1016,6 +1037,7 @@ message AdWithBid { // Private aggregation object. repeated PrivateAggregateContribution private_aggregation_contributions = 12; } +// (-- LINT.ThenChange(bid_generate_bid) --) // Bidding service operated by buyer. service Bidding { @@ -1051,9 +1073,7 @@ message GenerateBidsRequest { string name = 1; // Used to fetch real time bidding signals from buyer's key/value server - // included in the request. The value of each key in this list will be - // passed from the bidding signals dictionary to the Interest Group's - // GenerateBid() function as the trustedBiddingSignals parameter. + // included in the request. repeated string trusted_bidding_signals_keys = 2; // Optional. @@ -1086,6 +1106,11 @@ message GenerateBidsRequest { // information. BrowserSignals browser_signals = 7; } + + // Real time bidding signals fetched from buyer's key/value service + // passed to the Interest Group's GenerateBid() function as the + // trustedBiddingSignals parameter. + string trusted_bidding_signals = 8; } // Interest Group is an input to bidding code. @@ -1111,7 +1136,7 @@ message GenerateBidsRequest { string buyer_signals = 3; // Real Time signals fetched from buyer's Key/Value service. - string bidding_signals = 4; + string bidding_signals = 4 [deprecated = true]; // A boolean value which indicates if event level debug reporting should be // enabled or disabled for this request. @@ -1625,8 +1650,9 @@ message ScoreAdsResponse { // Protected Audience only). string interest_group_origin = 20; - // Private Aggregate contributions to be sent to the client - PrivateAggregateReportingResponse private_aggregate_reporting_response = 21; + // This field will be populated for all seller and buyer contributions in + // single seller auctions and server orchestrated multi seller auctions. + repeated PrivateAggregateReportingResponse top_level_contributions = 21; } // The response includes the top scored ad along with other related data. @@ -1669,17 +1695,6 @@ message WinReportingUrls { ReportingUrls top_level_seller_reporting_urls = 3; } -// Urls to support debug reporting, when auction is won and auction is lost. -message DebugReportUrls { - // URL to be triggered if the interest group wins the auction. - // If undefined or malformed url it will be ignored. - string auction_debug_win_url = 1; - - // URL to be triggered if the interest grou losses the auction. - // If undefined or malformed url it will be ignored. - string auction_debug_loss_url = 2; -} - // message to store the server request/ response message EventMessage { message KvSignal { diff --git a/api/udf/BUILD b/api/udf/BUILD new file mode 100644 index 00000000..0e8b4211 --- /dev/null +++ b/api/udf/BUILD @@ -0,0 +1,53 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load( + "@google_privacysandbox_servers_common//src/roma/tools/api_plugin:roma_api.bzl", + "declare_roma_api", + "roma_byob_app_api_cc_library", +) +load("@rules_cc//cc:defs.bzl", "cc_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") + +proto_library( + name = "generate_bid_proto", + srcs = ["generate_bid.proto"], + visibility = ["//visibility:public"], + deps = [ + "@google_privacysandbox_servers_common//apis/privacysandbox/apis/roma/app_api/v1:options_proto", + ], +) + +cc_proto_library( + name = "generate_bid_cc_proto", + deps = [ + ":generate_bid_proto", + ], +) + +generate_bid_api = declare_roma_api( + cc_protos = [":generate_bid_cc_proto"], + proto_basename = "generate_bid", + protos = [":generate_bid_proto"], +) + +roma_byob_app_api_cc_library( + name = "generate_bid_roma_api", + roma_app_api = generate_bid_api, + tags = [ + "noasan", + "notsan", + ], + visibility = ["//visibility:public"], +) diff --git a/api/udf/generate_bid.proto b/api/udf/generate_bid.proto new file mode 100644 index 00000000..60d2e611 --- /dev/null +++ b/api/udf/generate_bid.proto @@ -0,0 +1,293 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package privacy_sandbox.bidding_auction_servers; + +import "apis/privacysandbox/apis/roma/app_api/v1/options.proto"; + +service GenerateProtectedAudienceBidService { + option (privacysandbox.apis.roma.app_api.v1.roma_svc_annotation) = { + name: 'ProtectedAudience GenerateBid Service', + code_id: "generateprotectedaudiencebid_service_v1", + description: + 'Service to execute generateBid() UDF in a sandbox in the' + ' Bidding service for ProtectedAudience', + cpp_namespace: 'privacy_sandbox::bidding_auction_servers::roma_service', + roma_app_name: 'GenerateProtectedAudienceBidService', + }; + + rpc GenerateProtectedAudienceBid(GenerateProtectedAudienceBidRequest) returns (GenerateProtectedAudienceBidResponse) { + option (privacysandbox.apis.roma.app_api.v1.roma_rpc_annotation) = {description: + 'RPC call to generateBid() UDF for a single Protected Audience Custom Audience (a.k.a' + ' Interest Group).' +}; + } +} + +message GenerateProtectedAudienceBidRequest { + option (privacysandbox.apis.roma.app_api.v1.roma_mesg_annotation) = {description: + 'Request to generateBid() UDF for a Protected Audience Interest Group. (similar to the' + ' parameters in the JS spec for generateBid) for a Protected Audience' + ' auction.' +}; + + ProtectedAudienceInterestGroup interest_group = 1 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'This will be prepared by the Bidding service based on the data received' + ' in the BuyerInput from the device.' +}]; + + string auction_signals = 2 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'Auction signals are sent by the seller in the Auction Config. This can' + ' be encoded any way by the seller and will be passed as-is to the' + ' generateBid() UDF.' +}]; + + string per_buyer_signals = 3 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'Per buyer signals are sent by the seller in the Auction Config. This can' + ' be encoded any way by the seller and will be passed as-is to the' + ' generateBid() UDF.' +}]; + + string trusted_bidding_signals = 4 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'This will be passed as the JSON response received from the buyer\'s' + ' key/value server.' +}]; + + oneof ProtectedAudienceDeviceSignals { + ProtectedAudienceAndroidSignals android_signals = 5 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'This will be prepared by the Bidding server based on information' + ' passed by the Android app.' +}]; + + ProtectedAudienceBrowserSignals browser_signals = 6 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'This will be prepared by the Bidding server based on information' + ' passed by the browser on desktop or Android.' +}]; + + ServerMetadata server_metadata = 7 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'This will be prepared by the Bidding server and will contain config' + ' information about the current execution environment.' +}]; + } +} + +message ProtectedAudienceInterestGroup { + option (privacysandbox.apis.roma.app_api.v1.roma_mesg_annotation) = {description: 'Custom Audience (a.k.a Interest Group) for bidding.'}; + + string name = 1 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'Unique string that identifies the Custom Audience (a.k.a Interest Group)' + ' for a buyer.' +}]; + + repeated string trusted_bidding_signals_keys = 2 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'Used to fetch real time bidding signals from the buyer\'s key/value' + ' server included in GenerateBidsRequest.' +}]; + + repeated string ad_render_ids = 3 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'Optional. Id of ad_render_url generated by the buyer and passed to the' + ' client. Then client passes this in InterestGroup if available. Note: If' + ' the buyer doesn\'t generate the ad_render_id, then their generateBid()' + ' UDF should dynamically generate the URL for the bid. The winning ad' + ' render URL returned back to the client will be validated with the' + ' Interest Group information on the client.' +}]; + + repeated string ad_component_render_ids = 4 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'Optional. Id of ad_component_render_url generated by the buyer and' + ' passed to the client.' +}]; + + string user_bidding_signals = 5 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'Optional. User bidding signal that may be ingested during bidding and/or' + ' filtering.' +}]; +} + +// (-- LINT.IfChange(android_signals_generate_bid) --) +message ProtectedAudienceAndroidSignals { + option (privacysandbox.apis.roma.app_api.v1.roma_mesg_annotation) = {description: + 'Information sent from the Android app that the generateBid() UDF may' + ' want to use or verify.' +}; + + string top_level_seller = 1 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: 'Top level seller origin/domain passed in case of component auctions.'}]; +} +// (-- LINT.ThenChange(/api/bidding_auction_servers.proto:android_signals_bidding) --) + +// (-- LINT.IfChange(browser_signals_generate_bid) --) +message ProtectedAudienceBrowserSignals { + option (privacysandbox.apis.roma.app_api.v1.roma_mesg_annotation) = {description: + 'Information sent by the browser on desktop or Android that the' + ' generateBid() UDF may want to use or verify.' +}; + + string top_window_hostname = 1 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: 'Hostname of the top window.'}]; + + string seller = 2 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: 'Seller origin/domain.'}]; + + string top_level_seller = 3 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: 'Top level seller origin/domain passed in case of component auctions.'}]; + + int64 join_count = 4 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: 'Number of times the Interest Group was joined in the last 30 days.'}]; + + int64 bid_count = 5 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'Number of times the Interest Group bid in an auction in the last 30' + ' days.' +}]; + + int64 recency = 6 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'The most recent join time for the Interest Group expressed in' + ' milliseconds before the containing auctionBlob was requested.' +}]; + + string prev_wins_ms = 7 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'Tuple of time-ad pairs for a previous win for the Interest Group that' + ' occurred in the last 30 days. The time is specified in milliseconds' + ' before the containing auctionBlob was requested.' +}]; +} +// (-- LINT.ThenChange(/api/bidding_auction_servers.proto:browser_signals_bidding) --) + +message ServerMetadata { + option (privacysandbox.apis.roma.app_api.v1.roma_mesg_annotation) = {description: + 'Config information about the current execution environment for a' + ' GenerateBidRequest.' +}; + + bool debug_reporting_enabled = 1 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'A boolean value which indicates if event level debug reporting is' + ' enabled or disabled for the request. Adtechs should only return debug' + ' URLs if this is set to true, otherwise the URLs will be ignored and' + ' creating these will be wasted compute.' +}]; + + bool logging_enabled = 2 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'A boolean value which indicates if logging is enabled or disabled for' + ' the request. If this is false, the logs returned from the RPC in the' + ' response will be ignored. Otherwise, these will be outputted to the' + ' standard logs or included in the response.' +}]; +} + +message GenerateProtectedAudienceBidResponse { + option (privacysandbox.apis.roma.app_api.v1.roma_mesg_annotation) = {description: + 'Response to GenerateProtectedAudienceBidRequest with bid(s) for ad' + ' candidate(s) corresponding to the single Custom Audience (a.k.a' + ' Interest Group) (similar to' + ' the return values from the JS spec for generateBid), for a Protected' + ' Audience auction.' +}; + + repeated ProtectedAudienceBid bids = 1 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'The generateBid() UDF can return a list of bids instead of a single bid.' + ' This is added for supporting the K-anonymity feature. The maximum' + ' number of bids allowed to be returned is specified by the seller. When' + ' K-anonymity is disabled or not implemented, only the first candidate' + ' bid will be considered.' +}]; + + LogMessages log_messages = 2 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'Adtechs can add logs to the response if logging was enabled in the' + ' request. Logs will be printed out to the console in case of non-prod' + ' builds and added to the server response in case of debug consented' + ' requests.' +}]; +} + +// (-- LINT.IfChange(bid_generate_bid) --) +message ProtectedAudienceBid { + option (privacysandbox.apis.roma.app_api.v1.roma_mesg_annotation) = {description: + 'Bid for an ad candidate corresponding to a single Custom Audience (a.k.a' + ' Interest Group).' +}; + + string ad = 1 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'Metadata of the ad. Represents an opaque object that is eventually' + ' passed to the seller\'s scoreAd() UDF. Note: API will be updated' + ' separately for Component Ads.' +}]; + + float bid = 2 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: 'Bid price corresponding to an ad.'}]; + + string render = 3 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: 'Ad render URL that identifies an ad creative.'}]; + + repeated string ad_components = 4 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'List of ad render URLs that identifies ad components. This field must' + ' not be present if no component_ad_render_id was passed in' + ' interest_group to the generateBid() UDF.' +}]; + + bool allow_component_auction = 5 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: 'Whether component auction is allowed.'}]; + + string interest_group_name = 6 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'Name of the Custom Audience (a.k.a Interest Group) this ad belongs to' + ' required by the device to validate that a winning remarketing ad' + ' actually belongs to the Custom Audience (a.k.a Interest Group) as' + ' stored on-device.' +}]; + + double ad_cost = 7 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'A numerical value used to pass reporting advertiser click or conversion' + ' cost from the generateBid() UDF to reportWin(). The precision of this' + ' number is limited to an 8-bit mantissa and 8-bit exponent, with any' + ' rounding performed stochastically.' +}]; + + int32 modeling_signals = 8 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'A 12-bit integer signal used as input to win reporting URL generation' + ' for the buyer.' +}]; + + string bid_currency = 9 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'Indicates the currency used for the bid price (expressed as ISO 4217' + ' alpha code).' +}]; + + string buyer_reporting_id = 10 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: 'Optional. buyerReportingId set in buyerReportingMetadata of reportWin().'}]; + + DebugReportUrls debug_report_urls = 11 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'Optional field for debug report URLs. This should only be populated by' + ' adtechs if debug_reporting_enabled was set to true in the request.' +}]; +} +// (-- LINT.ThenChange(/api/bidding_auction_servers.proto:bid_bidding) --) + +message DebugReportUrls { + option (privacysandbox.apis.roma.app_api.v1.roma_mesg_annotation) = {description: + 'URLs to support debug reporting, when auction is won and auction is' + ' lost.' +}; + + string auction_debug_win_url = 1 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'URL to be triggered if the Interest Group wins the auction. If undefined' + ' or malformed, it will be ignored.' +}]; + + string auction_debug_loss_url = 2 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: + 'URL to be triggered if the Interest Group loses the auction. If' + ' undefined or malformed, it will be ignored.' +}]; +} + +message LogMessages { + option (privacysandbox.apis.roma.app_api.v1.roma_mesg_annotation) = {description: 'Logs, errors, and warnings populated by the generateBid() UDF.'}; + + repeated string logs = 1 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: 'Optional list of logs.'}]; + + repeated string errors = 2 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: 'Optional list of errors.'}]; + + repeated string warnings = 3 [(privacysandbox.apis.roma.app_api.v1.roma_field_annotation) = {description: 'Optional list of warnings.'}]; +} diff --git a/builders/.pre-commit-config.yaml b/builders/.pre-commit-config.yaml index c7997b3e..3f853aab 100644 --- a/builders/.pre-commit-config.yaml +++ b/builders/.pre-commit-config.yaml @@ -21,7 +21,7 @@ exclude: (?x)^( fail_fast: true repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.6.0 hooks: - id: end-of-file-fixer - id: fix-byte-order-marker @@ -47,7 +47,7 @@ repos: - id: shellcheck - repo: https://github.com/pre-commit/mirrors-clang-format - rev: v16.0.6 + rev: v18.1.5 hooks: - id: clang-format types_or: @@ -55,7 +55,7 @@ repos: - c - repo: https://github.com/bufbuild/buf - rev: v1.23.1 + rev: v1.31.0 hooks: - id: buf-format @@ -109,7 +109,7 @@ repos: - markdown - repo: https://github.com/DavidAnson/markdownlint-cli2 - rev: v0.8.1 + rev: v0.13.0 hooks: - id: markdownlint-cli2 name: lint markdown @@ -144,7 +144,7 @@ repos: - --quiet - repo: https://github.com/psf/black - rev: 23.7.0 + rev: 24.4.2 hooks: - id: black name: black python formatter diff --git a/builders/.profiler.bazelrc b/builders/.profiler.bazelrc new file mode 100644 index 00000000..42524d63 --- /dev/null +++ b/builders/.profiler.bazelrc @@ -0,0 +1,5 @@ +build:profiler --compilation_mode=opt +build:profiler --dynamic_mode=off +build:profiler --copt=-gmlt +build:profiler --copt=-fno-omit-frame-pointer +build:profiler --strip=never diff --git a/builders/CHANGELOG.md b/builders/CHANGELOG.md index 10be7b47..3d0ac973 100644 --- a/builders/CHANGELOG.md +++ b/builders/CHANGELOG.md @@ -2,6 +2,150 @@ All notable changes to this project will be documented in this file. See [commit-and-tag-version](https://github.com/absolute-version/commit-and-tag-version) for commit guidelines. +## 0.68.1 (2024-08-21) + +### Bug Fixes + +* Fix load bazel_tools import for Python deps + +## 0.68.0 (2024-08-21) + + +### Features + +* **deps:** Split python deps and registering toolchains +* **deps:** Update rules_python to 0.35.0 + +## 0.67.0 (2024-07-31) + + +### Bug Fixes + +* Add EXTRA_CBUILD_ARGS to tools/bazel-* scripts + + +### Dependencies + +* **deps:** Update buildozer to 6.1.1 +* **deps:** Upgrade amazonlinux2023 to 5.20240722.0 + +## 0.66.1 (2024-06-24) + + +### Bug Fixes + +* Add --compilation_mode=opt to build:profiler config + +## 0.66.0 (2024-06-20) + + +### Features + +* Add cpu-profiler flags to cbuild +* Add profiler config in .profiler.bazelrc + +## 0.65.1 (2024-06-04) + + +### Bug Fixes + +* Support multiple etc files in a single image + +## 0.65.0 (2024-06-04) + + +### Features + +* Add DOCKER_NETWORK env var for test-tools + +## 0.64.1 (2024-05-29) + + +### Bug Fixes + +* Support container reuse when --cmd not specified +* Use find to identify bazel symlinks + +## 0.64.0 (2024-05-27) + + +### Features + +* Support cmd-profiler mode with/without --cmd + + +### Bug Fixes + +* cbuild should find container with exact name match +* Ensure normalize-bazel-symlinks is in the workspace dir + + +### Dependencies + +* **deps:** Upgrade clang-format pre-commit hook + +## 0.63.0 (2024-05-26) + + +### Features + +* Support cmd-profiler mode with/without --cmd + + +### Bug Fixes + +* Ensure normalize-bazel-symlinks is in the workspace dir + + +### Dependencies + +* **deps:** Upgrade clang-format pre-commit hook + +## 0.62.0 (2024-05-10) + + +### Features + +* Add --dir flag to normalize-dist + +## 0.61.1 (2024-05-10) + + +### Bug Fixes + +* Add docker flags to container name +* Set 8h ttl for long-running build container + +## 0.61.0 (2024-05-08) + + +### Features + +* Add cbuild support for container reuse + +## 0.60.0 (2024-05-07) + + +### Dependencies + +* **deps:** Upgrade coverage-tools to ubuntu 24.04 +* **deps:** Upgrade golang to 1.22.2 + +## 0.59.0 (2024-05-02) + + +### Bug Fixes + +* **deps:** Update pre-commit hooks + + +### Dependencies + +* **deps:** Upgrade alpine base image +* **deps:** Upgrade base images for Amazon Linux +* **deps:** Upgrade grpcurl to 1.9.1 +* **deps:** Upgrade presubmit to ubuntu 24.04 + ## 0.58.0 (2024-04-26) diff --git a/builders/bazel/deps.bzl b/builders/bazel/deps.bzl index 0fe6743b..9e46f9ca 100644 --- a/builders/bazel/deps.bzl +++ b/builders/bazel/deps.bzl @@ -15,21 +15,24 @@ """Load definitions for use in WORKSPACE files.""" load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") -def python_deps(bazel_package): - """Load rules_python and register container-based python toolchain +def python_deps(): + """Load rules_python. Use python_register_toolchains to also resgister container-based python toolchain.""" + maybe( + http_archive, + name = "rules_python", + sha256 = "be04b635c7be4604be1ef20542e9870af3c49778ce841ee2d92fcb42f9d9516a", + strip_prefix = "rules_python-0.35.0", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.35.0/rules_python-0.35.0.tar.gz", + ) + +def python_register_toolchains(bazel_package): + """Register container-based python toolchain. Note: the bazel_package arg will depend on the import/submodule location in your workspace Args: bazel_package: repo-relative bazel package to builders/bazel/BUILD eg. "//builders/bazel" """ - http_archive( - name = "rules_python", - sha256 = "0a8003b044294d7840ac7d9d73eef05d6ceb682d7516781a4ec62eeb34702578", - strip_prefix = "rules_python-0.24.0", - urls = [ - "https://github.com/bazelbuild/rules_python/releases/download/0.24.0/rules_python-0.24.0.tar.gz", - ], - ) native.register_toolchains("{}:py_toolchain".format(bazel_package)) diff --git a/builders/images/build-amazonlinux2/Dockerfile b/builders/images/build-amazonlinux2/Dockerfile index bb6c1edc..6ccce805 100644 --- a/builders/images/build-amazonlinux2/Dockerfile +++ b/builders/images/build-amazonlinux2/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM amazonlinux:2.0.20230822.0 +FROM amazonlinux:2.0.20240412.0 COPY /install_apps install_golang_apps install_go.sh generate_system_bazelrc .bazelversion /scripts/ COPY get_workspace_mount /usr/local/bin diff --git a/builders/images/build-amazonlinux2/install_apps b/builders/images/build-amazonlinux2/install_apps index a5e6ff35..43fa2166 100755 --- a/builders/images/build-amazonlinux2/install_apps +++ b/builders/images/build-amazonlinux2/install_apps @@ -22,14 +22,8 @@ while [[ $# -gt 0 ]]; do VERBOSE=1 shift ;; - -h | --help) - usage 0 - break - ;; - *) - usage - break - ;; + -h | --help) usage 0 ;; + *) usage ;; esac done diff --git a/builders/images/build-amazonlinux2023/Dockerfile b/builders/images/build-amazonlinux2023/Dockerfile index d24ba9af..c7d3509b 100644 --- a/builders/images/build-amazonlinux2023/Dockerfile +++ b/builders/images/build-amazonlinux2023/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM amazonlinux:2023.1.20230825.0 +FROM amazonlinux:2023.5.20240722.0 COPY /install_apps install_golang_apps install_go.sh generate_system_bazelrc .bazelversion /scripts/ COPY get_workspace_mount /usr/local/bin diff --git a/builders/images/build-amazonlinux2023/install_apps b/builders/images/build-amazonlinux2023/install_apps index 51c64377..16378fa5 100755 --- a/builders/images/build-amazonlinux2023/install_apps +++ b/builders/images/build-amazonlinux2023/install_apps @@ -22,14 +22,8 @@ while [[ $# -gt 0 ]]; do VERBOSE=1 shift ;; - -h | --help) - usage 0 - break - ;; - *) - usage - break - ;; + -h | --help) usage 0 ;; + *) usage ;; esac done @@ -53,8 +47,8 @@ function install_python() { function install_nitro() { dnf install -y \ - "aws-nitro-enclaves-cli-1.2.*" \ - "aws-nitro-enclaves-cli-devel-1.2.*" + "aws-nitro-enclaves-cli-1.3.*" \ + "aws-nitro-enclaves-cli-devel-1.3.*" } function install_gcc() { diff --git a/builders/images/coverage-tools/Dockerfile b/builders/images/coverage-tools/Dockerfile index c2f25551..6915b8d2 100644 --- a/builders/images/coverage-tools/Dockerfile +++ b/builders/images/coverage-tools/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ubuntu:20.04 +FROM ubuntu:24.04 COPY install_apps /scripts/ diff --git a/builders/images/coverage-tools/install_apps b/builders/images/coverage-tools/install_apps index 72fd822a..5f8713b8 100755 --- a/builders/images/coverage-tools/install_apps +++ b/builders/images/coverage-tools/install_apps @@ -33,8 +33,8 @@ function apt_update() { function install_misc() { DEBIAN_FRONTEND=noninteractive apt-get --quiet install -y --no-install-recommends \ - lcov="1.*" \ - google-perftools="2.*" + google-perftools="2.*" \ + lcov="2.*" } function clean_debian() { diff --git a/builders/images/generate_system_bazelrc b/builders/images/generate_system_bazelrc index 9c8e419f..b68d9405 100755 --- a/builders/images/generate_system_bazelrc +++ b/builders/images/generate_system_bazelrc @@ -22,14 +22,12 @@ while [[ $# -gt 0 ]]; do case "$1" in --user-root-name) USER_ROOT_NAME="$2" - shift - shift + shift 2 || usage ;; -h | --help) usage 0 ;; *) printf "unrecognized arg: %s\n" "$1" usage - break ;; esac done diff --git a/builders/images/install_go.sh b/builders/images/install_go.sh index a436d20a..dc30cd73 100644 --- a/builders/images/install_go.sh +++ b/builders/images/install_go.sh @@ -22,12 +22,12 @@ function _golang_install_dir() { function install_golang() { declare -r _ARCH="$1" declare -r FNAME=gobin.tar.gz - declare -r VERSION=1.20.4 + declare -r VERSION=1.22.2 # shellcheck disable=SC2155 declare -r GO_INSTALL_DIR="$(_golang_install_dir)" declare -r -A GO_HASHES=( - [amd64]="698ef3243972a51ddb4028e4a1ac63dc6d60821bf18e59a807e051fee0a385bd" - [arm64]="105889992ee4b1d40c7c108555222ca70ae43fccb42e20fbf1eebb822f5e72c6" + [amd64]="5901c52b7a78002aeff14a21f93e0f064f74ce1360fce51c6ee68cd471216a17" + [arm64]="36e720b2d564980c162a48c7e97da2e407dfcc4239e1e58d98082dfa2486a0c1" ) declare -r GO_HASH=${GO_HASHES[${_ARCH}]} if [[ -z ${GO_HASH} ]]; then diff --git a/builders/images/install_golang_apps b/builders/images/install_golang_apps index 9d75eb94..a6bb3cfd 100755 --- a/builders/images/install_golang_apps +++ b/builders/images/install_golang_apps @@ -23,10 +23,7 @@ while [[ $# -gt 0 ]]; do shift ;; -h | --help) usage 0 ;; - *) - usage - break - ;; + *) usage ;; esac done diff --git a/builders/images/presubmit/.bazelversion b/builders/images/presubmit/.bazelversion new file mode 120000 index 00000000..8f79a5af --- /dev/null +++ b/builders/images/presubmit/.bazelversion @@ -0,0 +1 @@ +../../etc/.bazelversion \ No newline at end of file diff --git a/builders/images/presubmit/Dockerfile b/builders/images/presubmit/Dockerfile index 024c05fe..827e2584 100644 --- a/builders/images/presubmit/Dockerfile +++ b/builders/images/presubmit/Dockerfile @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ubuntu:20.04 +FROM ubuntu:24.04 -COPY install_apps install_go.sh .pre-commit-config.yaml /scripts/ +COPY install_apps install_go.sh install_golang_apps .bazelversion .pre-commit-config.yaml /scripts/ COPY gitconfig /etc ARG PRE_COMMIT_VENV_DIR=/usr/pre-commit-venv @@ -28,6 +28,7 @@ ENV BUILD_ARCH="${TARGETARCH}" \ RUN \ chmod 644 /etc/gitconfig && \ /usr/bin/env -v PRE_COMMIT_VENV_DIR=${PRE_COMMIT_VENV_DIR} /scripts/install_apps && \ + /scripts/install_golang_apps && \ rm -rf /scripts ENV PATH="${PATH}:/usr/local/go/bin" diff --git a/builders/images/presubmit/install_apps b/builders/images/presubmit/install_apps index c0d8d0d8..3986ec65 100755 --- a/builders/images/presubmit/install_apps +++ b/builders/images/presubmit/install_apps @@ -51,18 +51,17 @@ function apt_update() { function install_packages() { DEBIAN_FRONTEND=noninteractive apt-get --quiet install -y --no-install-recommends \ - apt-transport-https="2.0.*" \ + apt-transport-https="2.7.*" \ ca-certificates \ - libcurl4="7.68.*" \ - curl="7.68.*" \ - gnupg="2.2.*" \ - lsb-release="11.1.*" \ + libcurl4t64="8.5.*" \ + curl="8.5.*" \ + lsb-release="12.0*" \ openjdk-11-jre="11.0.*" \ - python3.9-venv="3.9.*" \ - shellcheck="0.7.*" \ + python3.12-venv="3.12.*" \ + shellcheck="0.9.*" \ software-properties-common="0.99.*" \ - wget="1.20.*" - update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 100 + wget="1.21.*" + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 100 } # Install Docker (https://docs.docker.com/engine/install/debian/) @@ -81,9 +80,9 @@ function install_docker() { } function install_precommit() { - /usr/bin/python3.9 -m venv "${PRE_COMMIT_VENV_DIR}" + /usr/bin/python3.12 -m venv "${PRE_COMMIT_VENV_DIR}" "${PRE_COMMIT_VENV_DIR}"/bin/pip install \ - pre-commit~=3.1 \ + pre-commit~=3.7 \ pylint~=3.1.0 "${PRE_COMMIT_TOOL}" --version diff --git a/builders/images/presubmit/install_golang_apps b/builders/images/presubmit/install_golang_apps new file mode 120000 index 00000000..acc9d5a3 --- /dev/null +++ b/builders/images/presubmit/install_golang_apps @@ -0,0 +1 @@ +../install_golang_apps \ No newline at end of file diff --git a/builders/images/release/install_release_apps b/builders/images/release/install_release_apps index e6c12253..dc99134f 100755 --- a/builders/images/release/install_release_apps +++ b/builders/images/release/install_release_apps @@ -5,4 +5,4 @@ npm install --global commit-and-tag-version@10.1.0 # Install the GitHub CLI tool (https://cli.github.com/) apk add github-cli -GOBIN=/usr/local/go/bin go install github.com/bazelbuild/buildtools/buildozer@6.0.1 +GOBIN=/usr/local/go/bin go install github.com/bazelbuild/buildtools/buildozer@6.1.1 diff --git a/builders/images/test-tools/Dockerfile b/builders/images/test-tools/Dockerfile index 45fab3cf..69399eb4 100644 --- a/builders/images/test-tools/Dockerfile +++ b/builders/images/test-tools/Dockerfile @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM alpine:3.18 as slowhttptest_builder +FROM alpine:3.19 as slowhttptest_builder # hadolint ignore=DL3018 RUN apk add --no-cache autoconf automake build-base git openssl-dev WORKDIR /build ADD https://github.com/shekyan/slowhttptest/archive/refs/tags/v1.9.0.tar.gz /build/src.tar.gz RUN tar xz --strip-components 1 -f src.tar.gz && ./configure && make -FROM alpine:3.18 as wrk_builder +FROM alpine:3.19 as wrk_builder ARG TARGETARCH ENV BUILD_ARCH="${TARGETARCH}" COPY build_wrk /build/ @@ -27,14 +27,14 @@ WORKDIR /build ADD https://github.com/giltene/wrk2/archive/44a94c17d8e6a0bac8559b53da76848e430cb7a7.tar.gz /build/src.tar.gz RUN /build/build_wrk -FROM golang:1.21-alpine3.18 AS golang +FROM golang:1.22-alpine3.19 AS golang ENV GOBIN=/usr/local/go/bin COPY build_golang_apps /scripts/ RUN /scripts/build_golang_apps -FROM fullstorydev/grpcurl:v1.8.9-alpine AS grpcurl +FROM fullstorydev/grpcurl:v1.9.1-alpine AS grpcurl -FROM alpine:3.18 +FROM alpine:3.19 COPY --from=golang /usr/local/go/bin/* /usr/local/bin/ COPY --from=grpcurl /bin/grpcurl /usr/local/bin/ ARG TARGETARCH diff --git a/builders/images/utils/Dockerfile b/builders/images/utils/Dockerfile index 0d1defd6..ea4bee81 100644 --- a/builders/images/utils/Dockerfile +++ b/builders/images/utils/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM alpine:3.16 +FROM alpine:3.19 RUN apk --no-cache add \ unzip~=6.0 \ diff --git a/builders/tests/data/hashes/build-amazonlinux2 b/builders/tests/data/hashes/build-amazonlinux2 index 7aed7b0f..71f56dfb 100644 --- a/builders/tests/data/hashes/build-amazonlinux2 +++ b/builders/tests/data/hashes/build-amazonlinux2 @@ -1 +1 @@ -3efa00f3a5dbe0a4708be523aa32aca91dcd56d403d3ff32e0202756b8321b3b +57ca4f2381a0fc193b0476663171c4d339b6ef66c0d1f1c24bb3f48d368b38ab diff --git a/builders/tests/data/hashes/build-amazonlinux2023 b/builders/tests/data/hashes/build-amazonlinux2023 index 1bcc412e..7d1e615d 100644 --- a/builders/tests/data/hashes/build-amazonlinux2023 +++ b/builders/tests/data/hashes/build-amazonlinux2023 @@ -1 +1 @@ -57396ff1c765f7b63905963cfe4498912f7f75b5cb9f7bc36bd6879af69872e7 +59a82d2db8173784b0b49959c9f82ead6c2e6da78a6be21cdc78520aa43741e3 diff --git a/builders/tests/data/hashes/build-debian b/builders/tests/data/hashes/build-debian index 8836d286..57095aed 100644 --- a/builders/tests/data/hashes/build-debian +++ b/builders/tests/data/hashes/build-debian @@ -1 +1 @@ -35e001149b0e33cba53e9a393c157ef920b3ab2adabcebd07eee0e3d4d9fccf3 +c194dafd287978093f8fe6e16e981fb22028e37345e20a4d7ca84caa43f0d4c0 diff --git a/builders/tests/data/hashes/coverage-tools b/builders/tests/data/hashes/coverage-tools index f0336331..e0127b80 100644 --- a/builders/tests/data/hashes/coverage-tools +++ b/builders/tests/data/hashes/coverage-tools @@ -1 +1 @@ -cd3fb189dd23793af3bdfa02d6774ccb35bddbec7059761e25c4f7be4c1e8ca1 +b768060d602e2ed1b60573edfa6afad5379e96a9d6153cd721b2a0665075fe98 diff --git a/builders/tests/data/hashes/presubmit b/builders/tests/data/hashes/presubmit index a35c6c86..a3e4b6ff 100644 --- a/builders/tests/data/hashes/presubmit +++ b/builders/tests/data/hashes/presubmit @@ -1 +1 @@ -d9dab1c798d51f79e68fd8eb3bb83312086808d789bbc09d0f2dbf708ef5f114 +560a5a1726e7b6fd2a507f72f2563eb3938d153a7d4b4aada575a7fe772873b0 diff --git a/builders/tests/data/hashes/release b/builders/tests/data/hashes/release index e5218ba2..affaf218 100644 --- a/builders/tests/data/hashes/release +++ b/builders/tests/data/hashes/release @@ -1 +1 @@ -d60fb40a53b1704f7ac353d0d036f49eac10bfd08ccb19f9b436acf8bdf2cb79 +398787e442bb10bcf7383bc3beec85ab27fb0145f80a23d8ee0eeb4992e5cb81 diff --git a/builders/tests/data/hashes/test-tools b/builders/tests/data/hashes/test-tools index fc5e0b5c..63f4e4bd 100644 --- a/builders/tests/data/hashes/test-tools +++ b/builders/tests/data/hashes/test-tools @@ -1 +1 @@ -dd1ec6137d4dd22fec555044cd85f484adfa6c7b686880ea5449cff936bad34e +c1111c91dcb1e9f4df65f9fd5eab60b2545b0e716cfaf59fb88c1006a6496a5e diff --git a/builders/tests/data/hashes/utils b/builders/tests/data/hashes/utils index da29b2fa..188febac 100644 --- a/builders/tests/data/hashes/utils +++ b/builders/tests/data/hashes/utils @@ -1 +1 @@ -9fca27d931acc2bc96fa0560466cc0914a0d1cc73fb8749af057caacf2911f85 +f4b8d15b26c7bef3bc94038be9b71aaf8ba8ba8d33663b7d6fb55ebdff9a902e diff --git a/builders/tests/run-tests b/builders/tests/run-tests index 037067d2..b7c1a7f9 100755 --- a/builders/tests/run-tests +++ b/builders/tests/run-tests @@ -19,18 +19,20 @@ set -o pipefail set -o errexit -trap _cleanup EXIT +# shellcheck disable=SC2317 function _cleanup() { - local -r -i STATUS=$? - if [[ -d ${TMP_HASHES_DIR1} ]]; then + local -r -i _status=$? + if [[ -d "${TMP_HASHES_DIR1}" ]]; then rm -rf "${TMP_HASHES_DIR1}" "${TMP_HASHES_DIR2}" fi - if [[ ${STATUS} -ne 0 ]]; then - printf "Error: run-tests status code: %d\n" "${STATUS}" + if [[ ${_status} -ne 0 ]]; then + printf "Error: run-tests status code: %d\n" "${_status}" sleep 5s fi - exit ${STATUS} + # shellcheck disable=SC2086 + exit ${_status} } +trap _cleanup EXIT function get_image_list() { local -r _images_dir="$1" diff --git a/builders/tools/bazel-debian b/builders/tools/bazel-debian index 843b3b6e..c32cf7b9 100755 --- a/builders/tools/bazel-debian +++ b/builders/tools/bazel-debian @@ -19,6 +19,7 @@ # BAZEL_STARTUP_ARGS Additional startup arguments to pass to bazel invocations # BAZEL_EXTRA_ARGS Additional command arguments to pass to bazel invocations # EXTRA_DOCKER_RUN_ARGS Additional arguments to pass to docker run invocations +# EXTRA_CBUILD_ARGS Additional arguments to pass to tools/cbuild set -o pipefail set -o errexit @@ -63,7 +64,8 @@ declare -a APP_ARGS declare -r -a ARGLIST=("$@") partition_array ARGLIST BAZEL_ARGS APP_ARGS -"${CBUILD}" --seccomp-unconfined --image "${IMAGE}" --cmd " +# shellcheck disable=SC2086 +"${CBUILD}" ${EXTRA_CBUILD_ARGS} --seccomp-unconfined --image "${IMAGE}" --cmd " printf 'bazel output_base: [%s]\n' \"\$(bazel info output_base 2>/dev/null)\" bazel ${BAZEL_STARTUP_ARGS} ${BAZEL_ARGS[*]@Q} ${BAZEL_EXTRA_ARGS} ${APP_ARGS[*]@Q} " diff --git a/builders/tools/cbuild b/builders/tools/cbuild index c40a3aed..31ee0fcb 100755 --- a/builders/tools/cbuild +++ b/builders/tools/cbuild @@ -36,7 +36,6 @@ function usage() { usage: $0 --cmd bash command string to execute within the docker container - --cmd-profiler enable profiler for the command --image Image name for the build runtime. Valid names: USAGE @@ -57,6 +56,11 @@ USAGE --seccomp-unconfined Run docker container without a seccomp profile --verbose Enable verbose output + Profiler flags: + --cmd-profiler enable profiler for the command + --cpu-profiler-signal unix signal to use to trigger profiler output. Default: ${CPU_PROFILER_SIGNAL} + --cpu-profiler-filename path for the cpu profiler output. Default: ${CPU_PROFILER_FILENAME} + Environment variables (all optional): WORKSPACE Full path to the workspace (repo root) WORKSPACE_MOUNT Full path to the workspace on the host filesystem @@ -77,6 +81,10 @@ declare -i WITH_DOCKER_SOCK=1 declare -i WITH_CMD_PROFILER=0 DOCKER_NETWORK="${DOCKER_NETWORK:-bridge}" declare -i DOCKER_SECCOMP_UNCONFINED=0 +declare -i KEEP_CONTAINER_RUNNING=0 +declare LONG_RUNNING_CONTAINER_TIMEOUT=8h +declare CPU_PROFILER_FILENAME=benchmark.prof +declare -i CPU_PROFILER_SIGNAL=12 while [[ $# -gt 0 ]]; do case "$1" in @@ -88,12 +96,23 @@ while [[ $# -gt 0 ]]; do WITH_CMD_PROFILER=1 shift ;; + --cpu-profiler-filename) + CPU_PROFILER_FILENAME="$2" + shift 2 || usage + ;; + --cpu-profiler-signal) + CPU_PROFILER_SIGNAL=$2 + shift 2 || usage + ;; --env) ENV_VARS+=("$2") shift 2 || usage ;; --image) IMAGE="$2" + if [[ ${IMAGE} =~ ^build-* ]]; then + KEEP_CONTAINER_RUNNING=1 + fi shift 2 || usage ;; --without-shared-cache) @@ -171,12 +190,14 @@ if [[ ${PWD_WORKSPACE_REL_PATH:0:1} != / ]]; then fi readonly WORKDIR -declare -a DOCKER_RUN_ARGS -DOCKER_RUN_ARGS+=( +# DOCKER_EXEC_RUN_ARGS applies to both `docker run` and `docker exec` +declare -a DOCKER_EXEC_RUN_ARGS=( + "--workdir=${WORKDIR}" +) +declare -a DOCKER_RUN_ARGS=( "--rm" "--entrypoint=/bin/bash" "--volume=${WORKSPACE_MOUNT}:/src/workspace" - "--workdir=${WORKDIR}" "--network=${DOCKER_NETWORK}" "$(echo "${EXTRA_DOCKER_RUN_ARGS}" | envsubst)" ) @@ -192,7 +213,8 @@ if [[ ${WITH_CMD_PROFILER} -eq 1 ]]; then fi DOCKER_RUN_ARGS+=( "--env=CMD_PROFILER=LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libprofiler.so" - "--env=CPUPROFILE=benchmark.prof" + "--env=CPUPROFILE=${CPU_PROFILER_FILENAME}" + "--env=CPUPROFILESIGNAL=${CPU_PROFILER_SIGNAL}" ) fi @@ -200,53 +222,127 @@ fi readonly BAZEL_ROOT=/bazel_root if [[ ${WITH_SHARED_CACHE} -eq 0 ]]; then # use tmpfs for as temporary, container-bound bazel cache - DOCKER_RUN_ARGS+=( - "--tmpfs ${BAZEL_ROOT}:exec" - ) + DOCKER_RUN_ARGS+=("--tmpfs=${BAZEL_ROOT}:exec") else # mount host filesystem for "shared" use by multiple docker container invocations - DOCKER_RUN_ARGS+=( - "--volume ${HOME}/.cache/bazel:${BAZEL_ROOT}" - ) + DOCKER_RUN_ARGS+=("--volume=${HOME}/.cache/bazel:${BAZEL_ROOT}") fi if [[ ${WITH_DOCKER_SOCK} -eq 1 ]]; then - DOCKER_RUN_ARGS+=( - "--volume /var/run/docker.sock:/var/run/docker.sock" - ) + DOCKER_RUN_ARGS+=("--volume=/var/run/docker.sock:/var/run/docker.sock") fi for evar in "${ENV_VARS[@]}" do - DOCKER_RUN_ARGS+=( - "--env=${evar}" - ) + DOCKER_EXEC_RUN_ARGS+=("--env=${evar}") done if [[ -t 0 ]] && [[ -t 1 ]]; then # stdin and stdout are open, assume it's an interactive tty session - DOCKER_RUN_ARGS+=( - --interactive - --tty + DOCKER_EXEC_RUN_ARGS+=( + "--interactive" + "--tty" ) fi +function get_container_name() { + local -r mount="$(echo "${WORKSPACE_MOUNT}" | sha256sum)" + local -r image_sha="${IMAGE_TAGGED##*-}" + local -r docker_args_sha="$({ +cat </dev/stderr + docker container rm --force "${name}" >/dev/null + printf "finished removing docker container: %s\n" "${name}" &>/dev/stderr + fi + docker "${docker_args[@]}" --filter "status=running" +} + +function long_running_container() { + local -r container_name="$1" + local -r docker_running_container="$(running_container_for "${container_name}")" + if [[ -z ${docker_running_container} ]]; then + printf "starting a new container [%s]\n" "${container_name}" &>/dev/stderr + if [[ -n ${CMD} ]]; then + # shellcheck disable=SC2068 + docker run \ + ${DOCKER_RUN_ARGS[@]} \ + "${DOCKER_EXEC_RUN_ARGS[@]}" \ + --detach \ + "${IMAGE_TAGGED}" \ + --login -c " +declare -i -r pid=\$(bazel info server_pid 2>/dev/null) +# wait for pid, even if it's not a child process of this shell +timeout ${LONG_RUNNING_CONTAINER_TIMEOUT} tail --pid=\${pid} -f /dev/null +" &>/dev/null + fi + fi + running_container_for "${DOCKER_CONTAINER_NAME}" +} + +if [[ ${KEEP_CONTAINER_RUNNING} -eq 1 ]]; then + if [[ -z ${CMD} ]]; then + # shellcheck disable=SC2068 + docker run \ + ${DOCKER_RUN_ARGS[@]} \ + "${DOCKER_EXEC_RUN_ARGS[@]}" \ + "${IMAGE_TAGGED}" + else + DOCKER_RUNNING_CONTAINER="$(long_running_container "${DOCKER_CONTAINER_NAME}")" + if [[ ${WITH_CMD_PROFILER} -eq 1 ]]; then + docker exec \ + "${DOCKER_EXEC_RUN_ARGS[@]}" \ + "${DOCKER_RUNNING_CONTAINER}" \ + /bin/bash -c "'${TOOLS_RELDIR}'/normalize-bazel-symlinks; env \${CMD_PROFILER} ${CMD:-/bin/sh}" + else + docker exec \ + "${DOCKER_EXEC_RUN_ARGS[@]}" \ + "${DOCKER_RUNNING_CONTAINER}" \ + /bin/bash -c "${CMD:-/bin/sh}" + fi + fi else - # shellcheck disable=SC2068 - docker run \ - ${DOCKER_RUN_ARGS[@]} \ - "${IMAGE_TAGGED}" \ - --login -c "${CMD}" + if [[ -z ${CMD} ]]; then + # shellcheck disable=SC2068 + docker run \ + ${DOCKER_RUN_ARGS[@]} \ + "${DOCKER_EXEC_RUN_ARGS[@]}" \ + "${IMAGE_TAGGED}" \ + --login + elif [[ ${WITH_CMD_PROFILER} -eq 1 ]]; then + # shellcheck disable=SC2068 + docker run \ + ${DOCKER_RUN_ARGS[@]} \ + "${DOCKER_EXEC_RUN_ARGS[@]}" \ + "${IMAGE_TAGGED}" \ + --login -c "'${TOOLS_RELDIR}'/normalize-bazel-symlinks; env \${CMD_PROFILER} ${CMD}" + else + # shellcheck disable=SC2068 + docker run \ + ${DOCKER_RUN_ARGS[@]} \ + "${DOCKER_EXEC_RUN_ARGS[@]}" \ + "${IMAGE_TAGGED}" \ + --login -c "${CMD}" + fi fi diff --git a/builders/tools/get-builder-image-tagged b/builders/tools/get-builder-image-tagged index 35371aea..bc14958a 100755 --- a/builders/tools/get-builder-image-tagged +++ b/builders/tools/get-builder-image-tagged @@ -199,7 +199,7 @@ function _tar_for_dir() { # shellcheck disable=SC2012 ls -A -1 "${FILEPATH}" "${ETC_DIR}" | sort | uniq -d ls -A -1 "${WORKSPACE}" - } | sort | uniq -d)" + } | sort | uniq -d | tr '\n' ' ')" # create a deterministic tarball of the collected files docker run \ --rm \ diff --git a/builders/tools/normalize-bazel-symlinks b/builders/tools/normalize-bazel-symlinks index 8506ac96..05e3e7ef 100755 --- a/builders/tools/normalize-bazel-symlinks +++ b/builders/tools/normalize-bazel-symlinks @@ -43,14 +43,11 @@ if [[ -f /.dockerenv ]]; then _normalize_fn=normalize_symlink_docker fi -declare -a -r LINK_DIRS=( - bazel-bin - bazel-out - bazel-testlogs - bazel-workspace -) -for link in "${LINK_DIRS[@]}"; do - if [[ -L ${link} ]]; then - ${_normalize_fn} "${link}" - fi +source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"/builder.sh +cd "${WORKSPACE}" || true + +declare -a links +mapfile -t links < <(find . -maxdepth 1 -type l -name "bazel-*" -exec basename {} \;) +for link in "${links[@]}"; do + ${_normalize_fn} "${link}" done diff --git a/builders/tools/normalize-dist b/builders/tools/normalize-dist index 93627d25..cfa86e11 100755 --- a/builders/tools/normalize-dist +++ b/builders/tools/normalize-dist @@ -19,13 +19,40 @@ set -o pipefail set -o errexit +declare TOP_LEVEL_DIR=dist + +function usage() { + local exitval=${1-1} + cat &>/dev/stderr < + --dir directory to normalize recursively. Default: ${TOP_LEVEL_DIR} +USAGE + # shellcheck disable=SC2086 + exit ${exitval} +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --dir) + TOP_LEVEL_DIR="$2" + shift 2 || usage + ;; + -h | --help) usage 0 ;; + *) + printf "unrecognized arg: %s\n" "$1" + usage + ;; + esac +done + trap _cleanup EXIT function _cleanup() { local -r -i STATUS=$? if [[ ${STATUS} -eq 0 ]]; then - printf "normalize-dist completed successfully\n" &>/dev/stderr + printf "normalize-dist [%s] completed successfully\n" "${TOP_LEVEL_DIR}" &>/dev/stderr else - printf "Error: normalize-dist completed with status code: %s\n" "${STATUS}" &>/dev/stderr + printf "Error: normalize-dist [%s] completed with status code: %s\n" "${TOP_LEVEL_DIR}" "${STATUS}" &>/dev/stderr fi exit 0 } @@ -40,9 +67,7 @@ readonly GROUP USER="$(builder::id u)" readonly USER -readonly TOP_LEVEL_DIRS="dist" - -printf "Setting file ownership [%s], group [%s] in dirs [%s]\n" "${USER}" "${GROUP}" "${TOP_LEVEL_DIRS}" +printf "Setting file ownership [%s], group [%s] in dirs [%s]\n" "${USER}" "${GROUP}" "${TOP_LEVEL_DIR}" declare -a runner=() if [[ -f /.dockerenv ]]; then runner+=(bash -c) @@ -51,11 +76,9 @@ else fi "${runner[@]}" " -for TOP_LEVEL_DIR in ${TOP_LEVEL_DIRS}; do - find \${TOP_LEVEL_DIR} -type f ! -executable -exec chmod 644 {} \; - find \${TOP_LEVEL_DIR} -type f -executable -exec chmod 755 {} \; - find \${TOP_LEVEL_DIR} -type d -exec chmod 755 {} \; - chgrp --recursive ${GROUP} \${TOP_LEVEL_DIR} - chown --recursive ${USER} \${TOP_LEVEL_DIR} -done +find ${TOP_LEVEL_DIR} -type f ! -executable -exec chmod 644 {} \; +find ${TOP_LEVEL_DIR} -type f -executable -exec chmod 755 {} \; +find ${TOP_LEVEL_DIR} -type d -exec chmod 755 {} \; +chgrp --recursive ${GROUP} ${TOP_LEVEL_DIR} +chown --recursive ${USER} ${TOP_LEVEL_DIR} " diff --git a/builders/tools/terraform b/builders/tools/terraform index 71a3e708..6cfc214c 100755 --- a/builders/tools/terraform +++ b/builders/tools/terraform @@ -82,4 +82,5 @@ DOCKER_RUN_ARGS+=( # shellcheck disable=SC2068 docker run \ "${DOCKER_RUN_ARGS[@]}" \ - ${IMAGE_TAGGED} "$@" + "${IMAGE_TAGGED}" \ + "$@" diff --git a/builders/tools/test-tool b/builders/tools/test-tool index 6a275370..908249db 100755 --- a/builders/tools/test-tool +++ b/builders/tools/test-tool @@ -13,7 +13,9 @@ # limitations under the License. # environment variables (all optional): -# WORKSPACE repo root directory, must be an absolute path +# WORKSPACE repo root directory, must be an absolute path +# DOCKER_NETWORK docker run --network arg, defaults to "host", set to +# blank to avoid setting --network set -o errexit @@ -57,6 +59,9 @@ readonly REL_PWD WORKSPACE_MOUNT="$(builder::get_docker_workspace_mount)" readonly WORKSPACE_MOUNT +# respect an empty DOCKER_NETWORK value +DOCKER_NETWORK=${DOCKER_NETWORK-host} + declare -a DOCKER_RUN_ARGS=( "--rm" "--interactive" @@ -64,6 +69,11 @@ declare -a DOCKER_RUN_ARGS=( "--volume=${WORKSPACE_MOUNT}:/src/workspace" "--workdir=/src/workspace/${REL_PWD}" ) +if [[ -n ${DOCKER_NETWORK} ]]; then + DOCKER_RUN_ARGS+=( + "--network=${DOCKER_NETWORK}" + ) +fi if [[ -n ${EXTRA_DOCKER_RUN_ARGS} ]]; then # shellcheck disable=SC2207 DOCKER_RUN_ARGS+=( diff --git a/builders/version.txt b/builders/version.txt index e8f304f8..65529192 100644 --- a/builders/version.txt +++ b/builders/version.txt @@ -1 +1 @@ -0.58.0 \ No newline at end of file +0.68.1 \ No newline at end of file diff --git a/production/deploy/aws/terraform/environment/demo/buyer/buyer.tf b/production/deploy/aws/terraform/environment/demo/buyer/buyer.tf index ddc5e0b0..551d421b 100644 --- a/production/deploy/aws/terraform/environment/demo/buyer/buyer.tf +++ b/production/deploy/aws/terraform/environment/demo/buyer/buyer.tf @@ -168,4 +168,5 @@ module "buyer" { BFE_TCMALLOC_BACKGROUND_RELEASE_RATE_BYTES_PER_SECOND = "4096" BFE_TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES = "10737418240" } + consented_request_s3_bucket = "" # Example: ${name of a s3 bucket} } diff --git a/production/deploy/aws/terraform/environment/demo/seller/seller.tf b/production/deploy/aws/terraform/environment/demo/seller/seller.tf index 7a260f05..141ec0d8 100644 --- a/production/deploy/aws/terraform/environment/demo/seller/seller.tf +++ b/production/deploy/aws/terraform/environment/demo/seller/seller.tf @@ -148,4 +148,5 @@ module "seller" { SFE_TCMALLOC_BACKGROUND_RELEASE_RATE_BYTES_PER_SECOND = "4096" SFE_TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES = "10737418240" } + consented_request_s3_bucket = "" # Example: ${name of a s3 bucket} } diff --git a/production/deploy/aws/terraform/modules/buyer/service.tf b/production/deploy/aws/terraform/modules/buyer/service.tf index 4de61235..90058db8 100644 --- a/production/deploy/aws/terraform/modules/buyer/service.tf +++ b/production/deploy/aws/terraform/modules/buyer/service.tf @@ -197,6 +197,7 @@ module "autoscaling_bidding" { healthcheck_healthy_threshold = var.healthcheck_healthy_threshold healthcheck_unhealthy_threshold = var.healthcheck_unhealthy_threshold healthcheck_grace_period_sec = var.healthcheck_grace_period_sec + consented_request_s3_bucket = var.consented_request_s3_bucket } ################ Buyer FrontEnd operator Setup ################ @@ -271,6 +272,7 @@ module "autoscaling_bfe" { healthcheck_healthy_threshold = var.healthcheck_healthy_threshold healthcheck_unhealthy_threshold = var.healthcheck_unhealthy_threshold healthcheck_grace_period_sec = var.healthcheck_grace_period_sec + consented_request_s3_bucket = var.consented_request_s3_bucket } ################ Parameter Setup ################ diff --git a/production/deploy/aws/terraform/modules/buyer/service_vars.tf b/production/deploy/aws/terraform/modules/buyer/service_vars.tf index 627328ac..7cdcb73e 100644 --- a/production/deploy/aws/terraform/modules/buyer/service_vars.tf +++ b/production/deploy/aws/terraform/modules/buyer/service_vars.tf @@ -237,3 +237,8 @@ variable "ad_retrieval_kv_server_virtual_service_name" { description = "Full name of the virtual service for the Ad Retrieval KV server." type = string } + +variable "consented_request_s3_bucket" { + description = "s3 bucket to export event message for consented request" + type = string +} diff --git a/production/deploy/aws/terraform/modules/seller/service.tf b/production/deploy/aws/terraform/modules/seller/service.tf index 88b86873..144dfe18 100644 --- a/production/deploy/aws/terraform/modules/seller/service.tf +++ b/production/deploy/aws/terraform/modules/seller/service.tf @@ -271,6 +271,8 @@ module "autoscaling_auction" { healthcheck_healthy_threshold = var.healthcheck_healthy_threshold healthcheck_unhealthy_threshold = var.healthcheck_unhealthy_threshold healthcheck_grace_period_sec = var.healthcheck_grace_period_sec + consented_request_s3_bucket = var.consented_request_s3_bucket + } @@ -347,6 +349,7 @@ module "autoscaling_sfe" { healthcheck_healthy_threshold = var.healthcheck_healthy_threshold healthcheck_unhealthy_threshold = var.healthcheck_unhealthy_threshold healthcheck_grace_period_sec = var.healthcheck_grace_period_sec + consented_request_s3_bucket = var.consented_request_s3_bucket } ################ Parameter Setup ################ diff --git a/production/deploy/aws/terraform/modules/seller/service_vars.tf b/production/deploy/aws/terraform/modules/seller/service_vars.tf index ec75cc6a..68371d33 100644 --- a/production/deploy/aws/terraform/modules/seller/service_vars.tf +++ b/production/deploy/aws/terraform/modules/seller/service_vars.tf @@ -249,3 +249,8 @@ variable "kv_server_virtual_service_name" { type = string default = "unused" } + +variable "consented_request_s3_bucket" { + description = "s3 bucket to export event message for consented request" + type = string +} diff --git a/production/deploy/aws/terraform/services/autoscaling/instance_init_script.tftpl b/production/deploy/aws/terraform/services/autoscaling/instance_init_script.tftpl index 70c148f5..13ccd445 100644 --- a/production/deploy/aws/terraform/services/autoscaling/instance_init_script.tftpl +++ b/production/deploy/aws/terraform/services/autoscaling/instance_init_script.tftpl @@ -96,10 +96,13 @@ systemctl enable vsockproxy.service # Update otel collector config sed -i -e 's/$SERVICE/'${service}'/g' /opt/privacysandbox/otel_collector_config.yaml +sed -i -e 's/$S3_REGION/'${region}'/g' /opt/privacysandbox/otel_collector_config.yaml +sed -i -e 's/$S3_BUCKET/'${consented_request_s3_bucket}'/g' /opt/privacysandbox/otel_collector_config.yaml +sed -i -e 's/$S3_PREFIX/'${s3_prefix}'/g' /opt/privacysandbox/otel_collector_config.yaml +sed -i -e 's/$FILE_PREFIX/'${file_prefix}'/g' /opt/privacysandbox/otel_collector_config.yaml + # Start the otel collector -sudo /opt/aws/aws-otel-collector/bin/aws-otel-collector-ctl \ - -c /opt/privacysandbox/otel_collector_config.yaml \ - -a start +sudo systemctl restart otelcol-contrib if [[ "${enclave_debug_mode}" == "true" ]]; then # Create cloudwatch config diff --git a/production/deploy/aws/terraform/services/autoscaling/main.tf b/production/deploy/aws/terraform/services/autoscaling/main.tf index 69d518d4..854bde71 100644 --- a/production/deploy/aws/terraform/services/autoscaling/main.tf +++ b/production/deploy/aws/terraform/services/autoscaling/main.tf @@ -48,6 +48,9 @@ resource "aws_launch_template" "instance_launch_template" { healthcheck_healthy_threshold = var.healthcheck_healthy_threshold healthcheck_unhealthy_threshold = var.healthcheck_unhealthy_threshold healthcheck_grace_period_sec = var.healthcheck_grace_period_sec + consented_request_s3_bucket = var.consented_request_s3_bucket + s3_prefix = "${var.environment}" + file_prefix = "${var.service}-${var.operator}" })) metadata_options { diff --git a/production/deploy/aws/terraform/services/autoscaling/variables.tf b/production/deploy/aws/terraform/services/autoscaling/variables.tf index def9d3d1..2d3bcdf2 100644 --- a/production/deploy/aws/terraform/services/autoscaling/variables.tf +++ b/production/deploy/aws/terraform/services/autoscaling/variables.tf @@ -143,3 +143,9 @@ variable "healthcheck_grace_period_sec" { description = "Amount of time to wait for service inside enclave to start up before starting health checks, in seconds." type = number } + +variable "consented_request_s3_bucket" { + description = "s3 bucket to export event message for consented request" + type = string + # no default value, to enforce configuration at buyer and seller module +} diff --git a/production/deploy/aws/terraform/services/iam_role_policies/main.tf b/production/deploy/aws/terraform/services/iam_role_policies/main.tf index e19ee74c..36286202 100644 --- a/production/deploy/aws/terraform/services/iam_role_policies/main.tf +++ b/production/deploy/aws/terraform/services/iam_role_policies/main.tf @@ -138,6 +138,12 @@ data "aws_iam_policy_document" "instance_policy_doc" { actions = ["route53:ChangeResourceRecordSets"] resources = ["*"] } + statement { + sid = "AllowWriteS3" + effect = "Allow" + actions = ["s3:PutObject"] + resources = ["*"] + } } resource "aws_iam_policy" "instance_policy" { diff --git a/production/deploy/gcp/terraform/environment/demo/buyer/buyer.tf b/production/deploy/gcp/terraform/environment/demo/buyer/buyer.tf index df7619ac..4c0a320a 100644 --- a/production/deploy/gcp/terraform/environment/demo/buyer/buyer.tf +++ b/production/deploy/gcp/terraform/environment/demo/buyer/buyer.tf @@ -157,7 +157,12 @@ module "buyer" { collector_service_port = 4317 collector_startup_script = templatefile("../../../services/autoscaling/collector_startup.tftpl", { collector_port = 4317 - otel_collector_image_uri = "otel/opentelemetry-collector-contrib:0.81.0" + otel_collector_image_uri = "otel/opentelemetry-collector-contrib:0.105.0" + gcs_hmac_key = module.secrets.gcs_hmac_key + gcs_hmac_secret = module.secrets.gcs_hmac_secret + gcs_bucket = "" # Example: ${name of a gcs bucket} + gcs_bucket_prefix = "" # Example: "consented-eventmessage-${local.environment}" + file_prefix = "" # Example: "operator-name" }) region_config = { # Example config provided for us-central1 and you may add your own regions. diff --git a/production/deploy/gcp/terraform/environment/demo/seller/seller.tf b/production/deploy/gcp/terraform/environment/demo/seller/seller.tf index f28bf00d..c5d4d713 100644 --- a/production/deploy/gcp/terraform/environment/demo/seller/seller.tf +++ b/production/deploy/gcp/terraform/environment/demo/seller/seller.tf @@ -147,7 +147,12 @@ module "seller" { collector_service_port = 4317 collector_startup_script = templatefile("../../../services/autoscaling/collector_startup.tftpl", { collector_port = 4317 - otel_collector_image_uri = "otel/opentelemetry-collector-contrib:0.81.0" + otel_collector_image_uri = "otel/opentelemetry-collector-contrib:0.105.0" + gcs_hmac_key = module.secrets.gcs_hmac_key + gcs_hmac_secret = module.secrets.gcs_hmac_secret + gcs_bucket = "" # Example: ${name of a gcs bucket} + gcs_bucket_prefix = "" # Example: "consented-eventmessage-${local.environment}" + file_prefix = "" # Example: "operator-name" }) region_config = { # Example config provided for us-central1 and you may add your own regions. diff --git a/production/deploy/gcp/terraform/modules/secrets/secrets.tf b/production/deploy/gcp/terraform/modules/secrets/secrets.tf index 586583e6..654745bd 100644 --- a/production/deploy/gcp/terraform/modules/secrets/secrets.tf +++ b/production/deploy/gcp/terraform/modules/secrets/secrets.tf @@ -22,6 +22,16 @@ data "google_secret_manager_secret_version" "tls_cert" { secret = "envoy-tls-termination-cert" } +data "google_secret_manager_secret_version" "gcs_hmac_key" { + provider = google-beta + secret = "gcs-hmac-key" +} + +data "google_secret_manager_secret_version" "gcs_hmac_secret" { + provider = google-beta + secret = "gcs-hmac-secret" +} + output "tls_key" { value = data.google_secret_manager_secret_version.tls_key.secret_data description = "The TLS Private Key used by BFE/SFE to terminate connections from the load balancer." @@ -33,3 +43,15 @@ output "tls_cert" { description = "The TLS Private Certificate used by BFE/SFE. May be self-signed, the parent chain is not validated." sensitive = true } + +output "gcs_hmac_key" { + value = data.google_secret_manager_secret_version.gcs_hmac_key.secret_data + description = "hmac_key used by otel collector to write to GCS. see cloud.google.com/storage/docs/authentication/managing-hmackeys#create" + sensitive = true +} + +output "gcs_hmac_secret" { + value = data.google_secret_manager_secret_version.gcs_hmac_secret.secret_data + description = "the secret of hmac_key used by otel collector to write to GCS. see cloud.google.com/storage/docs/authentication/managing-hmackeys#create" + sensitive = true +} diff --git a/production/deploy/gcp/terraform/services/autoscaling/collector_startup.tftpl b/production/deploy/gcp/terraform/services/autoscaling/collector_startup.tftpl index bdbf2c66..0ac6ad10 100644 --- a/production/deploy/gcp/terraform/services/autoscaling/collector_startup.tftpl +++ b/production/deploy/gcp/terraform/services/autoscaling/collector_startup.tftpl @@ -15,6 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Even though this setup is for a GCP collector, `awss3` is used to write to GCS via interoperability (https://cloud.google.com/storage/docs/interoperability). Therefore, the $AWS_ACCESS_KEY_ID and $AWS_SECRET_ACCESS_KEY are configured using the HMAC key created in GCP (https://cloud.google.com/storage/docs/authentication/managing-hmackeys#create) + write_files: - path: /etc/systemd/system/otelcol-contrib.service permissions: "0644" @@ -24,7 +26,7 @@ write_files: Description=Start a simple otel collector [Service] - ExecStart=/usr/bin/docker run --name otel -p ${collector_port}:${collector_port} -v /tmp/config.yaml:/etc/otelcol-contrib/config.yaml ${otel_collector_image_uri} + ExecStart=/usr/bin/docker run --name otel -p ${collector_port}:${collector_port} -v /tmp/config.yaml:/etc/otelcol-contrib/config.yaml --env AWS_ACCESS_KEY_ID=${gcs_hmac_key} --env AWS_SECRET_ACCESS_KEY=${gcs_hmac_secret} ${otel_collector_image_uri} ExecStop=/usr/bin/docker stop otel ExecStopPost=/usr/bin/docker rm otel @@ -40,8 +42,27 @@ write_files: processors: batch: + filter/drop_event: + error_mode: ignore + logs: + log_record: + - 'attributes["ps_tee_log_type"] == "event_message"' + filter/drop_non_event: + error_mode: ignore + logs: + log_record: + - 'attributes["ps_tee_log_type"] != "event_message"' exporters: + awss3: + s3uploader: + region: us-east-1 + s3_bucket: ${gcs_bucket} + s3_prefix: ${gcs_bucket_prefix} + s3_partition: minute + file_prefix: ${file_prefix} + endpoint: https://storage.googleapis.com/ + marshaler: body googlecloud: metric: resource_filters: @@ -61,10 +82,14 @@ write_files: receivers: [otlp] processors: [batch] exporters: [googlecloud] - logs: + logs/1: receivers: [otlp] - processors: [batch] + processors: [batch, filter/drop_event] exporters: [googlecloud] + logs/2: + receivers: [otlp] + processors: [filter/drop_non_event] + exporters: [awss3] runcmd: - systemctl daemon-reload diff --git a/production/deploy/gcp/terraform/services/dashboards/buyer_dashboard/main.tf b/production/deploy/gcp/terraform/services/dashboards/buyer_dashboard/main.tf index d6f04c55..5b7bfac1 100644 --- a/production/deploy/gcp/terraform/services/dashboards/buyer_dashboard/main.tf +++ b/production/deploy/gcp/terraform/services/dashboards/buyer_dashboard/main.tf @@ -1459,6 +1459,180 @@ resource "google_monitoring_dashboard" "environment_dashboard" { }, "width": 24, "yPos": 323 + }, + { + "height": 19, + "widget": { + "title": "inference.errors_count [MEAN]", + "xyChart": { + "chartOptions": {}, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "300s", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"workload.googleapis.com/inference.errors_count\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "secondaryAggregation": { + "alignmentPeriod": "300s", + "crossSeriesReducer": "REDUCE_MEAN", + "groupByFields": [ + "metric.label.\"error_code\"", + "metric.label.\"service_name\"", + "metric.label.\"deployment_environment\"", + "metric.label.\"operator\"", + "metric.label.\"Noise\"", + "resource.label.\"task_id\"", + "metric.label.\"service_version\"" + ], + "perSeriesAligner": "ALIGN_MEAN" + } + } + } + } + ], + "yAxis": { + "scale": "LINEAR" + } + } + }, + "width": 24, + "xPos": 24, + "yPos": 323 + }, + { + "height": 19, + "widget": { + "title": "inference.request.count_by_model [MEAN]", + "xyChart": { + "chartOptions": {}, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "300s", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"workload.googleapis.com/inference.request.count_by_model\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "secondaryAggregation": { + "alignmentPeriod": "300s", + "crossSeriesReducer": "REDUCE_MEAN", + "groupByFields": [ + "metric.label.\"model\"", + "metric.label.\"service_name\"", + "metric.label.\"deployment_environment\"", + "metric.label.\"operator\"", + "metric.label.\"Noise\"", + "resource.label.\"task_id\"", + "metric.label.\"service_version\"" + ], + "perSeriesAligner": "ALIGN_MEAN" + } + } + } + } + ], + "yAxis": { + "scale": "LINEAR" + } + } + }, + "width": 24, + "yPos": 342 + }, + { + "height": 19, + "widget": { + "title": "inference.request.duration_ms_by_model [95TH PERCENTILE]", + "xyChart": { + "chartOptions": {}, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "300s", + "crossSeriesReducer": "REDUCE_PERCENTILE_95", + "groupByFields": [ + "metric.label.\"model\"", + "metric.label.\"service_name\"", + "metric.label.\"deployment_environment\"", + "metric.label.\"operator\"", + "metric.label.\"Noise\"", + "resource.label.\"task_id\"", + "metric.label.\"service_version\"" + ], + "perSeriesAligner": "ALIGN_DELTA" + }, + "filter": "metric.type=\"workload.googleapis.com/inference.request.duration_ms_by_model\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"" + } + } + } + ], + "yAxis": { + "scale": "LINEAR" + } + } + }, + "width": 24, + "xPos": 24, + "yPos": 342 + }, + { + "height": 19, + "widget": { + "title": "inference.request.failed_count_by_model [MEAN]", + "xyChart": { + "chartOptions": {}, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "300s", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"workload.googleapis.com/inference.request.failed_count_by_model\" resource.type=\"generic_task\" metric.label.\"deployment_environment\"=\"${var.environment}\"", + "secondaryAggregation": { + "alignmentPeriod": "300s", + "crossSeriesReducer": "REDUCE_MEAN", + "groupByFields": [ + "metric.label.\"model\"", + "metric.label.\"service_name\"", + "metric.label.\"deployment_environment\"", + "metric.label.\"operator\"", + "metric.label.\"Noise\"", + "resource.label.\"task_id\"", + "metric.label.\"service_version\"" + ], + "perSeriesAligner": "ALIGN_MEAN" + } + } + } + } + ], + "yAxis": { + "scale": "LINEAR" + } + } + }, + "width": 24, + "yPos": 361 } ] } diff --git a/production/packaging/aws/auction_service/BUILD b/production/packaging/aws/auction_service/BUILD index 3f7410ef..f8fe71ec 100644 --- a/production/packaging/aws/auction_service/BUILD +++ b/production/packaging/aws/auction_service/BUILD @@ -12,8 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("@container_structure_test//:defs.bzl", "container_structure_test") -load("@rules_oci//oci:defs.bzl", "oci_image", "oci_tarball") +load( + "@io_bazel_rules_docker//container:container.bzl", + "container_image", + "container_layer", +) +load("@io_bazel_rules_docker//contrib:test.bzl", "container_test") load( "@rules_pkg//pkg:mappings.bzl", "pkg_attributes", @@ -55,13 +59,21 @@ pkg_tar( srcs = server_binaries, ) +container_layer( + name = "server_binary_layer", + directory = "/", + tars = [ + ":server_binaries_tar", + ], +) + # This image target is meant for testing running the server in an enclave using # aws proxy to abstract vsock communication. -oci_image( +container_image( name = "server_docker_image", base = select({ - "@platforms//cpu:arm64": "@runtime-debian-debug-root-arm64//:runtime-debian-debug-root-arm64", - "@platforms//cpu:x86_64": "@runtime-debian-debug-root-amd64//:runtime-debian-debug-root-amd64", + "@platforms//cpu:arm64": "@runtime-cc-debian-arm64//image", + "@platforms//cpu:x86_64": "@runtime-cc-debian-amd64//image", }), cmd = [ "/proxify", @@ -72,32 +84,28 @@ oci_image( entrypoint = [ "/busybox/sh", ], - tars = [ - "@google_privacysandbox_servers_common//src/aws/proxy:libnsm_and_proxify_tar", - "@google_privacysandbox_servers_common//src/cpio/client_providers/kms_client_provider/aws:kms_binaries", - ":server_binaries_tar", + layers = [ + ":server_binary_layer", ] + select({ "//:e2e_build": [ ], "//conditions:default": [], }), + tars = [ + "@google_privacysandbox_servers_common//src/aws/proxy:libnsm_and_proxify_tar", + "@google_privacysandbox_servers_common//src/cpio/client_providers/kms_client_provider/aws:kms_binaries", + ], ) -oci_tarball( - name = "server_docker_tarball", - image = ":server_docker_image", - repo_tags = ["bazel/production/packaging/aws/auction_service:server_docker_image"], -) - -container_structure_test( +container_test( name = "structure_test", size = "large", configs = ["test/structure.yaml"], driver = "tar", - image = ":server_docker_tarball", + image = ":server_docker_image", ) -container_structure_test( +container_test( name = "commands_test", size = "large", configs = ["test/commands.yaml"], @@ -115,14 +123,14 @@ genrule( name = "copy_to_dist", srcs = [ ":server_artifacts", - ":server_docker_tarball", + ":server_docker_image.tar", "//api:bidding_auction_servers_descriptor_set", ], outs = ["copy_to_dist.bin"], cmd_bash = """cat << EOF > '$@' mkdir -p dist/debian cp $(execpath :server_artifacts) dist/debian/$$(basename $(RULEDIR))_artifacts.zip -cp $(execpath :server_docker_tarball) dist/debian/$$(basename $(RULEDIR))_image.tar +cp $(execpath :server_docker_image.tar) dist/debian/$$(basename $(RULEDIR))_image.tar cp $(execpath //api:bidding_auction_servers_descriptor_set) dist builders/tools/normalize-dist EOF""", diff --git a/production/packaging/aws/bidding_service/BUILD b/production/packaging/aws/bidding_service/BUILD index 8208340c..838583d9 100644 --- a/production/packaging/aws/bidding_service/BUILD +++ b/production/packaging/aws/bidding_service/BUILD @@ -13,8 +13,12 @@ # limitations under the License. load("@bazel_skylib//rules:copy_file.bzl", "copy_file") -load("@container_structure_test//:defs.bzl", "container_structure_test") -load("@rules_oci//oci:defs.bzl", "oci_image", "oci_tarball") +load( + "@io_bazel_rules_docker//container:container.bzl", + "container_image", + "container_layer", +) +load("@io_bazel_rules_docker//contrib:test.bzl", "container_test") load( "@rules_pkg//pkg:mappings.bzl", "pkg_attributes", @@ -71,6 +75,19 @@ pkg_tar( srcs = server_binaries, ) +container_layer( + name = "server_binary_layer", + directory = "/", + env = { + "GLOG_logtostderr": "1", + "GLOG_stderrthreshold": "0", + "GRPC_DNS_RESOLVER": "native", + }, + tars = [ + ":server_binaries_tar", + ], +) + cddl_specs = [ "//services/bidding_service:packaged_cddl_specs", ] @@ -85,6 +102,14 @@ pkg_tar( srcs = cddl_specs, ) +container_layer( + name = "cddl_specs_layer", + directory = "/", + tars = [ + ":cddl_specs_tar", + ], +) + copy_file( name = "libcddl_so_lib", src = "@cddl_lib//:cddl", @@ -101,17 +126,23 @@ pkg_zip( pkg_tar( name = "cddl_lib_artifacts_tar", srcs = [":libcddl_so_lib"], - package_dir = "/usr/lib/x86_64-linux-gnu/", +) + +container_layer( + name = "cddl_lib_layer", + directory = "/usr/lib/x86_64-linux-gnu/", + tars = [ + ":cddl_lib_artifacts_tar", + ], ) # This image target is meant for testing running the server in an enclave using # aws proxy to abstract vsock communication. -oci_image( - # Name must be server_docker_image to be compatible with build scripts. +container_image( name = "server_docker_image", base = select({ - "@platforms//cpu:arm64": "@runtime-debian-debug-root-arm64//:runtime-debian-debug-root-arm64", - "@platforms//cpu:x86_64": "@runtime-debian-debug-root-amd64//:runtime-debian-debug-root-amd64", + "@platforms//cpu:arm64": "@runtime-cc-debian-arm64//image", + "@platforms//cpu:x86_64": "@runtime-cc-debian-amd64//image", }), cmd = [ "/proxify", @@ -122,33 +153,30 @@ oci_image( entrypoint = [ "/busybox/sh", ], - env = { - "GLOG_logtostderr": "1", - "GLOG_stderrthreshold": "0", - "GRPC_DNS_RESOLVER": "native", - }, - tars = [ - "@google_privacysandbox_servers_common//src/aws/proxy:libnsm_and_proxify_tar", - "@google_privacysandbox_servers_common//src/cpio/client_providers/kms_client_provider/aws:kms_binaries", - ":cddl_lib_artifacts_tar", - ":cddl_specs_tar", - ":server_binaries_tar", + layers = [ + ":cddl_lib_layer", + ":cddl_specs_layer", + ":server_binary_layer", ] + select({ "//:e2e_build": [ ], "//conditions:default": [], }), + tars = [ + "@google_privacysandbox_servers_common//src/aws/proxy:libnsm_and_proxify_tar", + "@google_privacysandbox_servers_common//src/cpio/client_providers/kms_client_provider/aws:kms_binaries", + ], ) -container_structure_test( +container_test( name = "structure_test", size = "large", configs = ["test/structure.yaml"], driver = "tar", - image = ":server_docker_tarball", + image = ":server_docker_image", ) -container_structure_test( +container_test( name = "commands_test", size = "large", configs = ["test/commands.yaml"], @@ -162,24 +190,18 @@ pkg_zip( srcs = server_binaries, ) -oci_tarball( - name = "server_docker_tarball", - image = ":server_docker_image", - repo_tags = ["bazel/production/packaging/aws/bidding_service:server_docker_image"], -) - genrule( name = "copy_to_dist", srcs = [ ":server_artifacts", - ":server_docker_tarball", + ":server_docker_image.tar", "//api:bidding_auction_servers_descriptor_set", ], outs = ["copy_to_dist.bin"], cmd_bash = """cat << EOF > '$@' mkdir -p dist/debian cp $(execpath :server_artifacts) dist/debian/$$(basename $(RULEDIR))_artifacts.zip -cp $(execpath :server_docker_tarball) dist/debian/$$(basename $(RULEDIR))_image.tar +cp $(execpath :server_docker_image.tar) dist/debian/$$(basename $(RULEDIR))_image.tar cp $(execpath //api:bidding_auction_servers_descriptor_set) dist builders/tools/normalize-dist EOF""", diff --git a/production/packaging/aws/buyer_frontend_service/BUILD b/production/packaging/aws/buyer_frontend_service/BUILD index c0a21132..4e90b59b 100644 --- a/production/packaging/aws/buyer_frontend_service/BUILD +++ b/production/packaging/aws/buyer_frontend_service/BUILD @@ -12,8 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("@container_structure_test//:defs.bzl", "container_structure_test") -load("@rules_oci//oci:defs.bzl", "oci_image", "oci_tarball") +load( + "@io_bazel_rules_docker//container:container.bzl", + "container_image", + "container_layer", +) +load("@io_bazel_rules_docker//contrib:test.bzl", "container_test") load( "@rules_pkg//pkg:mappings.bzl", "pkg_attributes", @@ -55,13 +59,26 @@ pkg_tar( srcs = server_binaries, ) +container_layer( + name = "server_binary_layer", + directory = "/", + env = { + "GLOG_logtostderr": "1", + "GLOG_stderrthreshold": "0", + "GRPC_DNS_RESOLVER": "native", + }, + tars = [ + ":server_binaries_tar", + ], +) + # This image target is meant for testing running the server in an enclave using # aws proxy to abstract vsock communication. -oci_image( +container_image( name = "server_docker_image", base = select({ - "@platforms//cpu:arm64": "@runtime-debian-debug-root-arm64//:runtime-debian-debug-root-arm64", - "@platforms//cpu:x86_64": "@runtime-debian-debug-root-amd64//:runtime-debian-debug-root-amd64", + "@platforms//cpu:arm64": "@runtime-cc-debian-arm64//image", + "@platforms//cpu:x86_64": "@runtime-cc-debian-amd64//image", }), cmd = [ "/proxify", @@ -72,37 +89,28 @@ oci_image( entrypoint = [ "/busybox/sh", ], - env = { - "GLOG_logtostderr": "1", - "GLOG_stderrthreshold": "0", - "GRPC_DNS_RESOLVER": "native", - }, - tars = [ - "@google_privacysandbox_servers_common//src/aws/proxy:libnsm_and_proxify_tar", - "@google_privacysandbox_servers_common//src/cpio/client_providers/kms_client_provider/aws:kms_binaries", - ":server_binaries_tar", + layers = [ + ":server_binary_layer", ] + select({ "//:e2e_build": [ ], "//conditions:default": [], }), + tars = [ + "@google_privacysandbox_servers_common//src/aws/proxy:libnsm_and_proxify_tar", + "@google_privacysandbox_servers_common//src/cpio/client_providers/kms_client_provider/aws:kms_binaries", + ], ) -oci_tarball( - name = "server_docker_tarball", - image = ":server_docker_image", - repo_tags = ["bazel/production/packaging/aws/buyer_frontend_service:server_docker_image"], -) - -container_structure_test( +container_test( name = "structure_test", size = "large", configs = ["test/structure.yaml"], driver = "tar", - image = ":server_docker_tarball", + image = ":server_docker_image", ) -container_structure_test( +container_test( name = "commands_test", size = "large", configs = ["test/commands.yaml"], @@ -120,14 +128,14 @@ genrule( name = "copy_to_dist", srcs = [ ":server_artifacts", - ":server_docker_tarball", + ":server_docker_image.tar", "//api:bidding_auction_servers_descriptor_set", ], outs = ["copy_to_dist.bin"], cmd_bash = """cat << EOF > '$@' mkdir -p dist/debian cp $(execpath :server_artifacts) dist/debian/$$(basename $(RULEDIR))_artifacts.zip -cp $(execpath :server_docker_tarball) dist/debian/$$(basename $(RULEDIR))_image.tar +cp $(execpath :server_docker_image.tar) dist/debian/$$(basename $(RULEDIR))_image.tar cp $(execpath //api:bidding_auction_servers_descriptor_set) dist builders/tools/normalize-dist EOF""", diff --git a/production/packaging/aws/common/ami/otel_collector_config.yaml b/production/packaging/aws/common/ami/otel_collector_config.yaml index 1476dc7d..3502b799 100644 --- a/production/packaging/aws/common/ami/otel_collector_config.yaml +++ b/production/packaging/aws/common/ami/otel_collector_config.yaml @@ -33,6 +33,16 @@ processors: timeout: 60s batch/logs: timeout: 60s + filter/drop_event: + error_mode: ignore + logs: + log_record: + - 'attributes["ps_tee_log_type"] == "event_message"' + filter/drop_non_event: + error_mode: ignore + logs: + log_record: + - 'attributes["ps_tee_log_type"] != "event_message"' exporters: awsxray: @@ -44,6 +54,14 @@ exporters: awscloudwatchlogs: log_group_name: "bidding-auction" log_stream_name: "consented-log-stream" + awss3: + s3uploader: + region: $S3_REGION + s3_bucket: $S3_BUCKET + s3_prefix: $S3_PREFIX + s3_partition: minute + file_prefix: $FILE_PREFIX + marshaler: body service: pipelines: @@ -55,9 +73,13 @@ service: receivers: [otlp] processors: [batch/metrics] exporters: [awsemf] - logs: + logs/1: receivers: [otlp] - processors: [batch/logs] + processors: [batch/logs, filter/drop_event] exporters: [awscloudwatchlogs] + logs/2: + receivers: [otlp] + processors: [filter/drop_non_event] + exporters: [awss3] extensions: [health_check] diff --git a/production/packaging/aws/common/ami/setup.sh b/production/packaging/aws/common/ami/setup.sh index fa309a14..5f23a2bc 100644 --- a/production/packaging/aws/common/ami/setup.sh +++ b/production/packaging/aws/common/ami/setup.sh @@ -21,8 +21,13 @@ sudo yum update -y sudo yum install -y \ amazon-cloudwatch-agent \ docker -wget -O /tmp/aws-otel-collector.rpm https://aws-otel-collector.s3.amazonaws.com/amazon_linux/amd64/latest/aws-otel-collector.rpm -sudo yum localinstall -y /tmp/aws-otel-collector.rpm + +wget -O /tmp/otelcol-contrib_0.105.0_linux_amd64.rpm https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.105.0/otelcol-contrib_0.105.0_linux_amd64.rpm +sudo yum localinstall -y /tmp/otelcol-contrib_0.105.0_linux_amd64.rpm +ENV_FILE="/etc/otelcol-contrib/otelcol-contrib.conf" +NEW_OTELCOL_OPTIONS="OTELCOL_OPTIONS=\"--config=/opt/privacysandbox/otel_collector_config.yaml\"" +sudo bash -c "sudo sed -i 's|^OTELCOL_OPTIONS=.*|${NEW_OTELCOL_OPTIONS}|' $ENV_FILE" + sudo amazon-linux-extras install -y aws-nitro-enclaves-cli sudo usermod -a -G docker ec2-user diff --git a/production/packaging/aws/seller_frontend_service/BUILD b/production/packaging/aws/seller_frontend_service/BUILD index 23c9aa51..9df37cf0 100644 --- a/production/packaging/aws/seller_frontend_service/BUILD +++ b/production/packaging/aws/seller_frontend_service/BUILD @@ -12,8 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("@container_structure_test//:defs.bzl", "container_structure_test") -load("@rules_oci//oci:defs.bzl", "oci_image", "oci_tarball") +load( + "@io_bazel_rules_docker//container:container.bzl", + "container_image", + "container_layer", +) +load("@io_bazel_rules_docker//contrib:test.bzl", "container_test") load( "@rules_pkg//pkg:mappings.bzl", "pkg_attributes", @@ -55,13 +59,26 @@ pkg_tar( srcs = server_binaries, ) +container_layer( + name = "server_binary_layer", + directory = "/", + env = { + "GLOG_logtostderr": "1", + "GLOG_stderrthreshold": "0", + "GRPC_DNS_RESOLVER": "native", + }, + tars = [ + ":server_binaries_tar", + ], +) + # This image target is meant for testing running the server in an enclave using # aws proxy to abstract vsock communication. -oci_image( +container_image( name = "server_docker_image", base = select({ - "@platforms//cpu:arm64": "@runtime-debian-debug-root-arm64//:runtime-debian-debug-root-arm64", - "@platforms//cpu:x86_64": "@runtime-debian-debug-root-amd64//:runtime-debian-debug-root-amd64", + "@platforms//cpu:arm64": "@runtime-cc-debian-arm64//image", + "@platforms//cpu:x86_64": "@runtime-cc-debian-amd64//image", }), cmd = [ "/proxify", @@ -72,37 +89,28 @@ oci_image( entrypoint = [ "/busybox/sh", ], - env = { - "GLOG_logtostderr": "1", - "GLOG_stderrthreshold": "0", - "GRPC_DNS_RESOLVER": "native", - }, - tars = [ - "@google_privacysandbox_servers_common//src/aws/proxy:libnsm_and_proxify_tar", - "@google_privacysandbox_servers_common//src/cpio/client_providers/kms_client_provider/aws:kms_binaries", - ":server_binaries_tar", + layers = [ + ":server_binary_layer", ] + select({ "//:e2e_build": [ ], "//conditions:default": [], }), + tars = [ + "@google_privacysandbox_servers_common//src/aws/proxy:libnsm_and_proxify_tar", + "@google_privacysandbox_servers_common//src/cpio/client_providers/kms_client_provider/aws:kms_binaries", + ], ) -oci_tarball( - name = "server_docker_tarball", - image = ":server_docker_image", - repo_tags = ["bazel/production/packaging/aws/seller_frontend_service:server_docker_image"], -) - -container_structure_test( +container_test( name = "structure_test", size = "large", configs = ["test/structure.yaml"], driver = "tar", - image = ":server_docker_tarball", + image = ":server_docker_image", ) -container_structure_test( +container_test( name = "commands_test", size = "large", configs = ["test/commands.yaml"], @@ -120,14 +128,14 @@ genrule( name = "copy_to_dist", srcs = [ ":server_artifacts", - ":server_docker_tarball", + ":server_docker_image.tar", "//api:bidding_auction_servers_descriptor_set", ], outs = ["copy_to_dist.bin"], cmd_bash = """cat << EOF > '$@' mkdir -p dist/debian cp $(execpath :server_artifacts) dist/debian/$$(basename $(RULEDIR))_artifacts.zip -cp $(execpath :server_docker_tarball) dist/debian/$$(basename $(RULEDIR))_image.tar +cp $(execpath :server_docker_image.tar) dist/debian/$$(basename $(RULEDIR))_image.tar cp $(execpath //api:bidding_auction_servers_descriptor_set) dist builders/tools/normalize-dist EOF""", diff --git a/production/packaging/gcp/auction_service/BUILD b/production/packaging/gcp/auction_service/BUILD index 5a0974c8..21c77c41 100644 --- a/production/packaging/gcp/auction_service/BUILD +++ b/production/packaging/gcp/auction_service/BUILD @@ -12,8 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("@container_structure_test//:defs.bzl", "container_structure_test") -load("@rules_oci//oci:defs.bzl", "oci_image", "oci_tarball") +load( + "@io_bazel_rules_docker//container:container.bzl", + "container_image", + "container_layer", +) +load("@io_bazel_rules_docker//contrib:test.bzl", "container_test") load( "@rules_pkg//pkg:mappings.bzl", "pkg_attributes", @@ -45,11 +49,24 @@ pkg_tar( srcs = server_binaries, ) -oci_image( +container_layer( + name = "server_binary_layer", + directory = "/", + env = { + "GLOG_logtostderr": "1", + "GLOG_stderrthreshold": "0", + "GRPC_DNS_RESOLVER": "native", + }, + tars = [ + ":server_binaries_tar", + ], +) + +container_image( name = "server_docker_image", base = select({ - "@platforms//cpu:arm64": "@runtime-debian-debug-root-arm64//:runtime-debian-debug-root-arm64", - "@platforms//cpu:x86_64": "@runtime-debian-debug-root-amd64//:runtime-debian-debug-root-amd64", + "@platforms//cpu:arm64": "@runtime-cc-debian-arm64//image", + "@platforms//cpu:x86_64": "@runtime-cc-debian-amd64//image", }), cmd = [ "--init_config_client=true", @@ -57,37 +74,26 @@ oci_image( entrypoint = [ "/server/bin/server", ], - env = { - "GLOG_logtostderr": "1", - "GLOG_stderrthreshold": "0", - "GRPC_DNS_RESOLVER": "native", - }, - exposed_ports = ["50051/tcp"], labels = {"tee.launch_policy.log_redirect": "always"}, - tars = [ - ":server_binaries_tar", + layers = [ + ":server_binary_layer", ] + select({ "//:e2e_build": [ ], "//conditions:default": [], }), + ports = ["50051"], ) -oci_tarball( - name = "server_docker_tarball", - image = ":server_docker_image", - repo_tags = ["bazel/production/packaging/gcp/auction_service:server_docker_image"], -) - -container_structure_test( +container_test( name = "structure_test", size = "large", configs = ["test/structure.yaml"], driver = "tar", - image = ":server_docker_tarball", + image = ":server_docker_image", ) -container_structure_test( +container_test( name = "commands_test", size = "large", configs = ["test/commands.yaml"], @@ -105,14 +111,14 @@ genrule( name = "copy_to_dist", srcs = [ ":server_artifacts", - ":server_docker_tarball", + ":server_docker_image.tar", "//api:bidding_auction_servers_descriptor_set", ], outs = ["copy_to_dist.bin"], cmd_bash = """cat << EOF > '$@' mkdir -p dist/debian cp $(execpath :server_artifacts) dist/debian/$$(basename $(RULEDIR))_artifacts.zip -cp $(execpath :server_docker_tarball) dist/debian/$$(basename $(RULEDIR))_image.tar +cp $(execpath :server_docker_image.tar) dist/debian/$$(basename $(RULEDIR))_image.tar cp $(execpath //api:bidding_auction_servers_descriptor_set) dist builders/tools/normalize-dist EOF""", diff --git a/production/packaging/gcp/bidding_service/BUILD b/production/packaging/gcp/bidding_service/BUILD index 642eb863..a3349130 100644 --- a/production/packaging/gcp/bidding_service/BUILD +++ b/production/packaging/gcp/bidding_service/BUILD @@ -13,8 +13,12 @@ # limitations under the License. load("@bazel_skylib//rules:copy_file.bzl", "copy_file") -load("@container_structure_test//:defs.bzl", "container_structure_test") -load("@rules_oci//oci:defs.bzl", "oci_image", "oci_tarball") +load( + "@io_bazel_rules_docker//container:container.bzl", + "container_image", + "container_layer", +) +load("@io_bazel_rules_docker//contrib:test.bzl", "container_test") load( "@rules_pkg//pkg:mappings.bzl", "pkg_attributes", @@ -63,6 +67,19 @@ pkg_tar( srcs = server_binaries, ) +container_layer( + name = "server_binary_layer", + directory = "/", + env = { + "GLOG_logtostderr": "1", + "GLOG_stderrthreshold": "0", + "GRPC_DNS_RESOLVER": "native", + }, + tars = [ + ":server_binaries_tar", + ], +) + cddl_specs = [ "//services/bidding_service:packaged_cddl_specs", ] @@ -77,6 +94,14 @@ pkg_tar( srcs = cddl_specs, ) +container_layer( + name = "cddl_specs_layer", + directory = "/", + tars = [ + ":cddl_specs_tar", + ], +) + copy_file( name = "libcddl_so_lib", src = "@cddl_lib//:cddl", @@ -93,14 +118,21 @@ pkg_zip( pkg_tar( name = "cddl_lib_artifacts_tar", srcs = [":libcddl_so_lib"], - package_dir = "/lib", ) -oci_image( +container_layer( + name = "cddl_lib_layer", + directory = "/lib", + tars = [ + ":cddl_lib_artifacts_tar", + ], +) + +container_image( name = "server_docker_image", base = select({ - "@platforms//cpu:arm64": "@runtime-debian-debug-root-arm64//:runtime-debian-debug-root-arm64", - "@platforms//cpu:x86_64": "@runtime-debian-debug-root-amd64//:runtime-debian-debug-root-amd64", + "@platforms//cpu:arm64": "@runtime-cc-debian-arm64//image", + "@platforms//cpu:x86_64": "@runtime-cc-debian-amd64//image", }), cmd = [ "/server/bin/init_server_basic", @@ -108,39 +140,28 @@ oci_image( entrypoint = [ "sh", ], - env = { - "GLOG_logtostderr": "1", - "GLOG_stderrthreshold": "0", - "GRPC_DNS_RESOLVER": "native", - }, - exposed_ports = ["50051/tcp"], labels = {"tee.launch_policy.log_redirect": "always"}, - tars = [ - ":cddl_lib_artifacts_tar", - ":cddl_specs_tar", - ":server_binaries_tar", + layers = [ + ":cddl_lib_layer", + ":cddl_specs_layer", + ":server_binary_layer", ] + select({ "//:e2e_build": [ ], "//conditions:default": [], }), + ports = ["50051"], ) -oci_tarball( - name = "server_docker_tarball", - image = ":server_docker_image", - repo_tags = ["bazel/production/packaging/gcp/bidding_service:server_docker_image"], -) - -container_structure_test( +container_test( name = "structure_test", size = "large", configs = ["test/structure.yaml"], driver = "tar", - image = ":server_docker_tarball", + image = ":server_docker_image", ) -container_structure_test( +container_test( name = "commands_test", size = "large", configs = ["test/commands.yaml"], @@ -158,14 +179,14 @@ genrule( name = "copy_to_dist", srcs = [ ":server_artifacts", - ":server_docker_tarball", + ":server_docker_image.tar", "//api:bidding_auction_servers_descriptor_set", ], outs = ["copy_to_dist.bin"], cmd_bash = """cat << EOF > '$@' mkdir -p dist/debian cp $(execpath :server_artifacts) dist/debian/$$(basename $(RULEDIR))_artifacts.zip -cp $(execpath :server_docker_tarball) dist/debian/$$(basename $(RULEDIR))_image.tar +cp $(execpath :server_docker_image.tar) dist/debian/$$(basename $(RULEDIR))_image.tar cp $(execpath //api:bidding_auction_servers_descriptor_set) dist builders/tools/normalize-dist EOF""", diff --git a/production/packaging/gcp/buyer_frontend_service/BUILD b/production/packaging/gcp/buyer_frontend_service/BUILD index 69334f5f..4a39763c 100644 --- a/production/packaging/gcp/buyer_frontend_service/BUILD +++ b/production/packaging/gcp/buyer_frontend_service/BUILD @@ -12,8 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("@container_structure_test//:defs.bzl", "container_structure_test") -load("@rules_oci//oci:defs.bzl", "oci_image", "oci_tarball") +load( + "@io_bazel_rules_docker//container:container.bzl", + "container_image", + "container_layer", +) +load("@io_bazel_rules_docker//contrib:test.bzl", "container_test") load( "@rules_pkg//pkg:mappings.bzl", "pkg_attributes", @@ -47,46 +51,48 @@ pkg_tar( srcs = server_binaries, ) -oci_image( +container_layer( + name = "server_binary_layer", + directory = "/", + tars = [ + ":server_binaries_tar", + ], +) + +container_image( name = "server_docker_image", base = select({ - "@platforms//cpu:arm64": "@runtime-debian-debug-root-arm64//:runtime-debian-debug-root-arm64", - "@platforms//cpu:x86_64": "@runtime-debian-debug-root-amd64//:runtime-debian-debug-root-amd64", + "@platforms//cpu:arm64": "@runtime-cc-debian-arm64//image", + "@platforms//cpu:x86_64": "@runtime-cc-debian-amd64//image", }), cmd = [ "/server/bin/init_server_basic", ], entrypoint = ["sh"], - # BFE and BFE Non-TLS Healthcheck ports, respectively: - exposed_ports = [ - "50051/tcp", - "50050/tcp", - ], labels = {"tee.launch_policy.log_redirect": "always"}, - tars = [ - ":server_binaries_tar", + layers = [ + ":server_binary_layer", ] + select({ "//:e2e_build": [ ], "//conditions:default": [], }), + # BFE and BFE Non-TLS Healthcheck ports, respectively: + ports = [ + "50051", + "50050", + ], ) -oci_tarball( - name = "server_docker_tarball", - image = ":server_docker_image", - repo_tags = ["bazel/production/packaging/gcp/buyer_frontend_service:server_docker_image"], -) - -container_structure_test( +container_test( name = "structure_test", size = "large", configs = ["test/structure.yaml"], driver = "tar", - image = ":server_docker_tarball", + image = ":server_docker_image", ) -container_structure_test( +container_test( name = "commands_test", size = "large", configs = ["test/commands.yaml"], @@ -104,14 +110,14 @@ genrule( name = "copy_to_dist", srcs = [ ":server_artifacts", - ":server_docker_tarball", + ":server_docker_image.tar", "//api:bidding_auction_servers_descriptor_set", ], outs = ["copy_to_dist.bin"], cmd_bash = """cat << EOF > '$@' mkdir -p dist/debian cp $(execpath :server_artifacts) dist/debian/$$(basename $(RULEDIR))_artifacts.zip -cp $(execpath :server_docker_tarball) dist/debian/$$(basename $(RULEDIR))_image.tar +cp $(execpath :server_docker_image.tar) dist/debian/$$(basename $(RULEDIR))_image.tar cp $(execpath //api:bidding_auction_servers_descriptor_set) dist builders/tools/normalize-dist EOF""", diff --git a/production/packaging/gcp/seller_frontend_service/BUILD b/production/packaging/gcp/seller_frontend_service/BUILD index 91f51315..0a862608 100644 --- a/production/packaging/gcp/seller_frontend_service/BUILD +++ b/production/packaging/gcp/seller_frontend_service/BUILD @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + load( "@io_bazel_rules_docker//container:container.bzl", "container_flatten", diff --git a/production/packaging/gcp/seller_frontend_service/test/structure.yaml b/production/packaging/gcp/seller_frontend_service/test/structure.yaml index 494b5aa2..2e70adbb 100644 --- a/production/packaging/gcp/seller_frontend_service/test/structure.yaml +++ b/production/packaging/gcp/seller_frontend_service/test/structure.yaml @@ -37,15 +37,10 @@ fileExistenceTests: shouldExist: true isExecutableBy: any - - name: envoy-config + - name: envoy path: /etc/envoy/envoy.yaml shouldExist: true - - name: envoy-binary - path: /usr/local/bin/envoy - shouldExist: true - isExecutableBy: any - - name: ca-certs path: /etc/ssl/certs/ca-certificates.crt shouldExist: true diff --git a/services/auction_service/BUILD b/services/auction_service/BUILD index f6d6b882..9810fa32 100644 --- a/services/auction_service/BUILD +++ b/services/auction_service/BUILD @@ -79,6 +79,7 @@ cc_library( "//services/auction_service/reporting:reporting_response", "//services/auction_service/reporting/buyer:buyer_reporting_helper", "//services/auction_service/reporting/buyer:pa_buyer_reporting_manager", + "//services/auction_service/reporting/buyer:pas_buyer_reporting_manager", "//services/auction_service/reporting/seller:component_seller_reporting_manager", "//services/auction_service/reporting/seller:seller_reporting_manager", "//services/auction_service/reporting/seller:top_level_seller_reporting_manager", diff --git a/services/auction_service/auction_main.cc b/services/auction_service/auction_main.cc index a71ec37a..63a5af4f 100644 --- a/services/auction_service/auction_main.cc +++ b/services/auction_service/auction_main.cc @@ -1,3 +1,4 @@ + // Copyright 2022 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -95,7 +96,14 @@ using ::google::scp::cpio::LogOption; using ::grpc::Server; using ::grpc::ServerBuilder; -bool kEnableSellerAndBuyerUdfIsolation = false; +namespace { +bool ShouldEnableSellerAndBuyerUdfIsolation(bool test_mode) { + if (test_mode) { + return false; + } + return true; +} +} // namespace absl::StatusOr GetConfigClient( absl::string_view config_param_prefix) { @@ -227,8 +235,12 @@ absl::Status RunServer() { code_fetch_proto.enable_private_aggregate_reporting(); const bool enable_protected_app_signals = config_client.GetBooleanParameter(ENABLE_PROTECTED_APP_SIGNALS); + bool test_mode = config_client.GetBooleanParameter(TEST_MODE); + bool enable_seller_and_buyer_udf_isolation = + ShouldEnableSellerAndBuyerUdfIsolation(test_mode); + code_fetch_proto.set_enable_seller_and_buyer_udf_isolation( - kEnableSellerAndBuyerUdfIsolation); + enable_seller_and_buyer_udf_isolation); MultiCurlHttpFetcherAsync http_fetcher = MultiCurlHttpFetcherAsync(executor.get()); HttpFetcherAsync* seller_udf_fetcher = &http_fetcher; @@ -296,7 +308,7 @@ absl::Status RunServer() { config_client.GetIntParameter(MAX_ALLOWED_SIZE_ALL_DEBUG_URLS_KB), .default_code_version = default_code_version, .enable_seller_and_buyer_udf_isolation = - kEnableSellerAndBuyerUdfIsolation, + enable_seller_and_buyer_udf_isolation, .enable_private_aggregate_reporting = enable_private_aggregate_reporting}; AuctionService auction_service( std::move(score_ads_reactor_factory), diff --git a/services/auction_service/auction_service_integration_test_util.cc b/services/auction_service/auction_service_integration_test_util.cc index 598ec93c..6f495abe 100644 --- a/services/auction_service/auction_service_integration_test_util.cc +++ b/services/auction_service/auction_service_integration_test_util.cc @@ -48,6 +48,8 @@ namespace { using ::google::protobuf::TextFormat; using AdWithBidMetadata = ScoreAdsRequest::ScoreAdsRawRequest::AdWithBidMetadata; +using ProtectedAppSignalsAdWithBidMetadata = + ScoreAdsRequest::ScoreAdsRawRequest::ProtectedAppSignalsAdWithBidMetadata; using ::testing::AnyNumber; constexpr absl::string_view kKeyId = "key_id"; @@ -55,6 +57,12 @@ constexpr absl::string_view kSecret = "secret"; constexpr absl::string_view kTestConsentToken = "testConsentToken"; constexpr absl::string_view kEurosIsoCode = "EUR"; constexpr absl::string_view kPublisherHostname = "fenceStreetJournal.com"; +constexpr absl::string_view kEgressPayload = + "{\"features\": [{\"type\": \"boolean-feature\", \"value\": true}, " + "{\"type\": \"unsigned-integer-feature\", \"value\": 127}]}"; +constexpr absl::string_view kTemporaryUnlimitedEgressPayload = + "{\"features\": [{\"type\": \"boolean-feature\", \"value\": " + "true},{\"type\": \"unsigned-integer-feature\", \"value\": 2}]}"; AdWithBidMetadata GetTestAdWithBidMetadata( const TestScoreAdsRequestConfig& test_score_ads_request_config) { @@ -139,6 +147,70 @@ ScoreAdsRequest BuildScoreAdsRequest( return request; } +ProtectedAppSignalsAdWithBidMetadata GetTestAdWithBidMetadataForPAS( + const TestScoreAdsRequestConfig& test_score_ads_request_config) { + ScoreAdsRequest::ScoreAdsRawRequest::ProtectedAppSignalsAdWithBidMetadata ad; + ad.mutable_ad()->mutable_struct_value()->MergeFrom( + MakeAnAd(MakeARandomString(), MakeARandomString(), 2)); + ad.set_bid(1.0); + ad.set_bid_currency(kEurosIsoCode); + ad.set_ad_cost( + test_score_ads_request_config.test_buyer_reporting_signals.ad_cost); + ad.set_modeling_signals(test_score_ads_request_config + .test_buyer_reporting_signals.modeling_signals); + ad.set_owner(test_score_ads_request_config.test_buyer_reporting_signals + .interest_group_name); + ad.set_owner(test_score_ads_request_config.interest_group_owner.c_str()); + ad.set_render(absl::StrFormat( + "%s/ads", test_score_ads_request_config.interest_group_owner)); + ad.set_egress_payload(kEgressPayload); + ad.set_temporary_unlimited_egress_payload(kTemporaryUnlimitedEgressPayload); + return ad; +} + +ScoreAdsRequest BuildScoreAdsRequestForPAS( + const TestScoreAdsRequestConfig& test_score_ads_request_config, + const std::vector& ads) { + ScoreAdsRequest::ScoreAdsRawRequest raw_request; + std::string trusted_scoring_signals = + R"json({"renderUrls":{"placeholder_url":[123])json"; + for (const auto& ad : ads) { + raw_request.mutable_per_buyer_signals()->try_emplace( + ad.owner(), test_score_ads_request_config.test_buyer_reporting_signals + .buyer_signals); + std::string ad_signal = absl::StrFormat( + "\"%s\":%s", ad.render(), R"JSON(["short", "test", "signal"])JSON"); + absl::StrAppend(&trusted_scoring_signals, + absl::StrFormat(", %s", ad_signal)); + *raw_request.mutable_protected_app_signals_ad_bids()->Add() = ad; + } + absl::StrAppend(&trusted_scoring_signals, + R"json(},"adComponentRenderUrls":{}})json"); + raw_request.set_scoring_signals(trusted_scoring_signals); + if (test_score_ads_request_config.enable_debug_reporting) { + raw_request.set_enable_debug_reporting( + test_score_ads_request_config.enable_debug_reporting); + } + raw_request.set_seller("http://seller.com"); + raw_request.set_publisher_hostname(kPublisherHostname); + raw_request.set_auction_signals( + test_score_ads_request_config.test_buyer_reporting_signals + .auction_signals); + raw_request.mutable_consented_debug_config()->set_is_consented( + test_score_ads_request_config.is_consented); + raw_request.mutable_consented_debug_config()->set_token(kTestConsentToken); + // top_level_seller is expected to be set only for component auctions + if (!test_score_ads_request_config.top_level_seller.empty()) { + raw_request.set_top_level_seller( + test_score_ads_request_config.top_level_seller); + } + ScoreAdsRequest request; + *request.mutable_request_ciphertext() = raw_request.SerializeAsString(); + request.set_key_id(kKeyId); + return request; +} + void SetupMockCryptoClientWrapper(MockCryptoClientWrapper& crypto_client) { // Mock the HpkeDecrypt() call on the crypto_client. This is used by the // service to decrypt the incoming request. @@ -173,7 +245,7 @@ void InitV8Dispatcher(V8Dispatcher& dispatcher) { ASSERT_TRUE(dispatcher.Init().ok()); } -void LoadTestSellerUdfWrapper( +void LoadWrapperWithMockSellerUdf( absl::string_view adtech_code_blob, const AuctionServiceRuntimeConfig& auction_service_runtime_config, V8Dispatcher& dispatcher) { @@ -184,7 +256,7 @@ void LoadTestSellerUdfWrapper( ASSERT_TRUE(dispatcher.LoadSync(kScoreAdBlobVersion, wrapper_js_blob).ok()); } -void LoadTestBuyerUdfWrapper( +void LoadWrapperWithMockReportWinUdf( V8Dispatcher& dispatcher, const ScoreAdsRequest& request, absl::string_view buyer_udf, const AuctionServiceRuntimeConfig& auction_service_runtime_config, @@ -193,13 +265,13 @@ void LoadTestBuyerUdfWrapper( ASSERT_TRUE(raw_request.ParseFromString(request.request_ciphertext())); if (!auction_service_runtime_config.enable_report_win_url_generation) { return; + } else if (buyer_udf.empty()) { + PS_VLOG(kNoisyInfo) << "No buyer code loaded"; + return; } for (const auto& ad_bid : raw_request.ad_bids()) { - if (buyer_udf.empty()) { - PS_VLOG(kNoisyInfo) << "No buyer code loaded"; - break; - } - std::string wrapper_js_blob = GetBuyerWrappedCode(buyer_udf); + std::string wrapper_js_blob = GetBuyerWrappedCode( + buyer_udf, auction_service_runtime_config.enable_protected_app_signals); absl::StatusOr version = GetBuyerReportWinVersion(ad_bid.interest_group_owner(), auction_type); ASSERT_TRUE(version.ok()); @@ -207,6 +279,28 @@ void LoadTestBuyerUdfWrapper( } } +void LoadWrapperWithMockReportWinUdfForPAS( + V8Dispatcher& dispatcher, const ScoreAdsRequest& request, + absl::string_view buyer_udf, + const AuctionServiceRuntimeConfig& auction_service_runtime_config, + AuctionType auction_type) { + ScoreAdsRequest::ScoreAdsRawRequest raw_request; + ASSERT_TRUE(raw_request.ParseFromString(request.request_ciphertext())); + if (!auction_service_runtime_config.enable_report_win_url_generation) { + return; + } else if (buyer_udf.empty()) { + PS_VLOG(kNoisyInfo) << "No buyer code loaded"; + return; + } + for (const auto& ad_bid : raw_request.protected_app_signals_ad_bids()) { + std::string wrapper_js_blob = GetBuyerWrappedCode(buyer_udf); + absl::StatusOr version = + GetBuyerReportWinVersion(ad_bid.owner(), auction_type); + ASSERT_TRUE(version.ok()); + ASSERT_TRUE(dispatcher.LoadSync(*version, wrapper_js_blob).ok()); + } +} + void RunTestScoreAds( CodeDispatchClient& client, ScoreAdsRequest& request, const AuctionServiceRuntimeConfig& auction_service_runtime_config, @@ -252,9 +346,21 @@ void LoadPABuyerAndSellerCode(const ScoreAdsRequest& request, absl::string_view buyer_udf, absl::string_view seller_udf) { InitV8Dispatcher(dispatcher); - LoadTestSellerUdfWrapper(seller_udf, runtime_config, dispatcher); - LoadTestBuyerUdfWrapper(dispatcher, request, buyer_udf, runtime_config, - AuctionType::kProtectedAudience); + LoadWrapperWithMockSellerUdf(seller_udf, runtime_config, dispatcher); + LoadWrapperWithMockReportWinUdf(dispatcher, request, buyer_udf, + runtime_config, + AuctionType::kProtectedAudience); +} + +void LoadPASBuyerAndSellerCode( + const ScoreAdsRequest& request, V8Dispatcher& dispatcher, + const AuctionServiceRuntimeConfig& runtime_config, + absl::string_view buyer_udf, absl::string_view seller_udf) { + InitV8Dispatcher(dispatcher); + LoadWrapperWithMockSellerUdf(seller_udf, runtime_config, dispatcher); + LoadWrapperWithMockReportWinUdfForPAS(dispatcher, request, buyer_udf, + runtime_config, + AuctionType::kProtectedAppSignals); } } // namespace @@ -285,4 +391,20 @@ void LoadAndRunScoreAdsForPA( seller_udf); RunTestScoreAds(dispatch_client, request, runtime_config, response); } + +void LoadAndRunScoreAdsForPAS( + const AuctionServiceRuntimeConfig& runtime_config, + const TestScoreAdsRequestConfig& test_score_ads_request_config, + absl::string_view buyer_udf, absl::string_view seller_udf, + ScoreAdsResponse& response) { + V8Dispatcher dispatcher; + CodeDispatchClient dispatch_client(dispatcher); + ProtectedAppSignalsAdWithBidMetadata test_ad = + GetTestAdWithBidMetadataForPAS(test_score_ads_request_config); + ScoreAdsRequest request = + BuildScoreAdsRequestForPAS(test_score_ads_request_config, {test_ad}); + LoadPASBuyerAndSellerCode(request, dispatcher, runtime_config, buyer_udf, + seller_udf); + RunTestScoreAds(dispatch_client, request, runtime_config, response); +} } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/auction_service_integration_test_util.h b/services/auction_service/auction_service_integration_test_util.h index 8bd0a5af..21ff0e37 100644 --- a/services/auction_service/auction_service_integration_test_util.h +++ b/services/auction_service/auction_service_integration_test_util.h @@ -119,6 +119,12 @@ void LoadAndRunScoreAdsForPA( const TestScoreAdsRequestConfig& test_score_ads_request_config, absl::string_view buyer_udf, absl::string_view seller_udf, ScoreAdsResponse& response); + +void LoadAndRunScoreAdsForPAS( + const AuctionServiceRuntimeConfig& runtime_config, + const TestScoreAdsRequestConfig& test_score_ads_request_config, + absl::string_view buyer_udf, absl::string_view seller_udf, + ScoreAdsResponse& response); } // namespace privacy_sandbox::bidding_auction_servers #endif // SERVICES_AUCTION_SERVICE_INTEGRATION_TEST_UTIL_H_ diff --git a/services/auction_service/auction_service_reporting_integration_test.cc b/services/auction_service/auction_service_reporting_integration_test.cc index 1502209c..36a986f9 100644 --- a/services/auction_service/auction_service_reporting_integration_test.cc +++ b/services/auction_service/auction_service_reporting_integration_test.cc @@ -222,6 +222,37 @@ TEST_F(AuctionServiceReportingIntegrationTest, EXPECT_GT(score_ad.desirability(), 0); } +TEST_F(AuctionServiceReportingIntegrationTest, + ScoresAdsSuccessWithProtectedAppSignalsEnabled) { + ScoreAdsResponse response; + AuctionServiceRuntimeConfig runtime_config = { + .enable_report_result_url_generation = true, + .enable_report_win_url_generation = true, + .enable_protected_app_signals = true, + .enable_seller_and_buyer_udf_isolation = true}; + TestBuyerReportingSignals test_buyer_reporting_signals; + TestScoreAdsRequestConfig test_score_ads_request_config = { + .test_buyer_reporting_signals = test_buyer_reporting_signals, + .buyer_reporting_id = kBuyerReportingId, + .interest_group_owner = kTestIgOwner}; + LoadAndRunScoreAdsForPAS(runtime_config, test_score_ads_request_config, + kProtectedAppSignalsBuyerBaseCode, kSellerBaseCode, + response); + ScoreAdsResponse::ScoreAdsRawResponse raw_response; + ASSERT_TRUE(raw_response.ParseFromString(response.response_ciphertext())); + const auto& score_ad = raw_response.ad_score(); + EXPECT_GT(score_ad.desirability(), 0); + const auto& top_level_seller_reporting_urls = + score_ad.win_reporting_urls().top_level_seller_reporting_urls(); + EXPECT_EQ(top_level_seller_reporting_urls.reporting_url(), + kExpectedReportResultUrl); + EXPECT_EQ(top_level_seller_reporting_urls.interaction_reporting_urls().size(), + 1); + EXPECT_EQ(top_level_seller_reporting_urls.interaction_reporting_urls().at( + kTestInteractionEvent), + kTestInteractionReportingUrl); +} + TEST_F( AuctionServiceReportingIntegrationTest, ReportingSuccessWithSellerAndBuyerCodeIsolationAndEmptySignalsForWinner) { diff --git a/services/auction_service/benchmarking/BUILD b/services/auction_service/benchmarking/BUILD index c576e7ff..e854efaa 100644 --- a/services/auction_service/benchmarking/BUILD +++ b/services/auction_service/benchmarking/BUILD @@ -53,6 +53,7 @@ cc_binary( "//services/common/encryption:mock_crypto_client_wrapper", "//services/common/test:mocks", "//services/common/test:random", + "//services/common/test/utils:test_init", "@google_benchmark//:benchmark", "@google_benchmark//:benchmark_main", "@google_privacysandbox_servers_common//src/encryption/key_fetcher/mock:mock_key_fetcher_manager", diff --git a/services/auction_service/benchmarking/score_ads_reactor_benchmarks.cc b/services/auction_service/benchmarking/score_ads_reactor_benchmarks.cc index cdd3b427..a475b613 100644 --- a/services/auction_service/benchmarking/score_ads_reactor_benchmarks.cc +++ b/services/auction_service/benchmarking/score_ads_reactor_benchmarks.cc @@ -31,6 +31,7 @@ #include "services/common/reporters/async_reporter.h" #include "services/common/test/mocks.h" #include "services/common/test/random.h" +#include "services/common/test/utils/test_init.h" namespace privacy_sandbox::bidding_auction_servers { namespace { @@ -319,6 +320,7 @@ RawRequest BuildRawRequest( static void BM_ScoreAdsProtectedAudience(benchmark::State& state) { // Setup. + CommonTestInit(); ScoreAdsRequest score_ads_request; server_common::telemetry::TelemetryConfig config_proto; config_proto.set_mode(server_common::telemetry::TelemetryConfig::PROD); @@ -364,6 +366,7 @@ static void BM_ScoreAdsProtectedAudience(benchmark::State& state) { static void BM_ScoreAdsProtectedAudienceAndAppSignals(benchmark::State& state) { // Setup. + CommonTestInit(); ScoreAdsRequest score_ads_request; server_common::telemetry::TelemetryConfig config_proto; config_proto.set_mode(server_common::telemetry::TelemetryConfig::PROD); diff --git a/services/auction_service/code_wrapper/buyer_reporting_test_constants.h b/services/auction_service/code_wrapper/buyer_reporting_test_constants.h index e1f93d21..90d39575 100644 --- a/services/auction_service/code_wrapper/buyer_reporting_test_constants.h +++ b/services/auction_service/code_wrapper/buyer_reporting_test_constants.h @@ -63,7 +63,7 @@ constexpr absl::string_view kTestReportWinUdfWithValidation = } )JS_CODE"; -constexpr absl::string_view kExpectedBuyerCodeWithReportWin = R"JS_CODE( +constexpr absl::string_view kExpectedBuyerCodeWithReportWinForPA = R"JS_CODE( function reportWinEntryFunction( auctionConfig, perBuyerSignals, signalsForWinner, buyerReportingSignals, @@ -158,5 +158,131 @@ reportWin = function(auctionSignals, perBuyerSignals, signalsForWinner, buyerRep } )JS_CODE"; +constexpr absl::string_view kExpectedBuyerCodeWithReportWinForPAS = R"JS_CODE( + +function reportWinEntryFunction( + auctionConfig, perBuyerSignals, signalsForWinner, buyerReportingSignals, + directFromSellerSignals, enable_logging, egressVector, temporaryUnlimitedEgressVector) { + ps_sendReportToInvoked = false + ps_registerAdBeaconInvoked = false + const ps_report_win_response = { + reportWinUrl: '', + interactionReportingUrls: {}, + }; + const ps_logs = []; + const ps_errors = []; + const ps_warns = []; + if (enable_logging) { + console.log = (...args) => ps_logs.push(JSON.stringify(args)); + console.warn = (...args) => ps_warns.push(JSON.stringify(args)); + console.error = (...args) => ps_errors.push(JSON.stringify(args)); + } else { + console.log = console.warn = console.error = function() {}; + } + globalThis.sendReportTo = function sendReportTo(url) { + if (ps_sendReportToInvoked) { + throw new Error('sendReportTo function invoked more than once'); + } + ps_report_win_response.reportWinUrl = url; + ps_sendReportToInvoked = true; + }; + globalThis.registerAdBeacon = function registerAdBeacon(eventUrlMap) { + if (ps_registerAdBeaconInvoked) { + throw new Error( + 'registerAdBeaconInvoked function invoked more than once'); + } + ps_report_win_response.interactionReportingUrls = eventUrlMap; + ps_registerAdBeaconInvoked = true; + }; + try { + reportWin( + auctionConfig.auctionSignals, perBuyerSignals, signalsForWinner, + buyerReportingSignals, directFromSellerSignals, egressVector, temporaryUnlimitedEgressVector); + } catch (ex) { + console.error(ex.message); + } + return { + response: ps_report_win_response, + logs: ps_logs, + errors: ps_errors, + warnings: ps_warns + }; +} +reportWin = function(auctionSignals, perBuyerSignals, signalsForWinner, buyerReportingSignals, + directFromSellerSignals){ + if(!buyerReportingSignals.seller){ + console.error("Missing seller in input to reportWin") + return + } + if(!buyerReportingSignals.interestGroupName && !buyerReportingSignals.buyerReportingId){ + console.error("Missing both interestGroupName and buyerReportingId in input to reportWin") + return + } + if(!buyerReportingSignals.adCost || buyerReportingSignals.adCost < 1){ + console.error("Missing adCost in input to reportWin") + return + } + var reportWinUrl = "Invalid buyerReportingSignals" + if(validateInputs(buyerReportingSignals)){ + reportWinUrl = "http://test.com?seller="+buyerReportingSignals.seller+ + "&interestGroupName="+buyerReportingSignals.interestGroupName+ + "&buyerReportingId="+buyerReportingSignals.buyerReportingId+ + "&adCost="+buyerReportingSignals.adCost+ + "&highestScoringOtherBid="+buyerReportingSignals.highestScoringOtherBid+ + "&madeHighestScoringOtherBid="+buyerReportingSignals.madeHighestScoringOtherBid+ + "&signalsForWinner="+JSON.stringify(signalsForWinner)+ + "&perBuyerSignals="+perBuyerSignals+"&auctionSignals="+auctionSignals+"&desirability="+buyerReportingSignals.desirability+"&topLevelSeller="+buyerReportingSignals.topLevelSeller+"&modifiedBid="+buyerReportingSignals.modifiedBid; + } + console.log("Logging from ReportWin"); + console.error("Logging error from ReportWin") + console.warn("Logging warning from ReportWin") + sendReportTo(reportWinUrl) + registerAdBeacon({"clickEvent":"http://click.com"}) + } + function validateInputs(buyerReportingSignals){ + if(buyerReportingSignals.modelingSignals === undefined || buyerReportingSignals.modelingSignals<0 || buyerReportingSignals.modelingSignals>65535){ + return false + } + if(buyerReportingSignals.recency===undefined || buyerReportingSignals.recency<0){ + return false + } + if(buyerReportingSignals.joinCount===undefined || buyerReportingSignals.joinCount<0){ + return false + } + return true + } +)JS_CODE"; + +constexpr absl::string_view kProtectedAppSignalsBuyerBaseCode = R"JS_CODE( +reportWin = function( + auctionSignals, perBuyerSignals, signalsForWinner, buyerReportingSignals, + directFromSellerSignals, egressPayload, temporaryUnlimitedEgressPayload) { + console.log('Testing Protected App Signals'); + var reportWinUrl = 'http://test.com?seller=' + buyerReportingSignals.seller + + '&interestGroupName=' + buyerReportingSignals.interestGroupName + + '&buyerReportingId=' + buyerReportingSignals.buyerReportingId + + '&adCost=' + buyerReportingSignals.adCost + '&highestScoringOtherBid=' + + buyerReportingSignals.highestScoringOtherBid + + '&madeHighestScoringOtherBid=' + + buyerReportingSignals.madeHighestScoringOtherBid + + '&signalsForWinner=' + JSON.stringify(signalsForWinner) + + '&perBuyerSignals=' + perBuyerSignals + + '&auctionSignals=' + auctionSignals + + '&desirability=' + buyerReportingSignals.desirability + + '&topLevelSeller=' + buyerReportingSignals.topLevelSeller + + '&modifiedBid=' + buyerReportingSignals.modifiedBid + + '&egressPayload=' + egressPayload + + '&temporaryUnlimitedEgressPayload=' + temporaryUnlimitedEgressPayload; + console.log('Logging from ReportWin'); + console.error('Logging error from ReportWin'); + console.warn('Logging warning from ReportWin'); + sendReportTo(reportWinUrl); + registerAdBeacon({'clickEvent': 'http://click.com'}); + return { + 'testSignal': 'testValue' + }; +} +)JS_CODE"; + } // namespace privacy_sandbox::bidding_auction_servers #endif // SERVICES_BUYER_REPORTING_TEST_CONSTANTS_H_ diff --git a/services/auction_service/code_wrapper/buyer_reporting_udf_wrapper.cc b/services/auction_service/code_wrapper/buyer_reporting_udf_wrapper.cc index 39816983..118266a6 100644 --- a/services/auction_service/code_wrapper/buyer_reporting_udf_wrapper.cc +++ b/services/auction_service/code_wrapper/buyer_reporting_udf_wrapper.cc @@ -15,13 +15,21 @@ #include "services/auction_service/code_wrapper/buyer_reporting_udf_wrapper.h" #include +#include +#include "absl/strings/str_replace.h" #include "absl/strings/string_view.h" namespace privacy_sandbox::bidding_auction_servers { -std::string GetBuyerWrappedCode(absl::string_view buyer_js_code) { - return absl::StrCat(kReportWinWrapperFunction, buyer_js_code); +std::string GetBuyerWrappedCode(absl::string_view buyer_js_code, + bool enable_protected_app_signals) { + std::string wrap_code{kReportWinWrapperFunction}; + if (enable_protected_app_signals) { + wrap_code = absl::StrReplaceAll(wrap_code, {{"$extraArgs", kEgressArgs}}); + } + wrap_code.append(buyer_js_code); + return wrap_code; } } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/code_wrapper/buyer_reporting_udf_wrapper.h b/services/auction_service/code_wrapper/buyer_reporting_udf_wrapper.h index 242dd375..cb241b94 100644 --- a/services/auction_service/code_wrapper/buyer_reporting_udf_wrapper.h +++ b/services/auction_service/code_wrapper/buyer_reporting_udf_wrapper.h @@ -23,6 +23,8 @@ namespace privacy_sandbox::bidding_auction_servers { constexpr char kReportWinEntryFunction[] = "reportWinEntryFunction"; +inline constexpr char kEgressArgs[] = + "egressVector, temporaryUnlimitedEgressVector"; // The wrapper function which enables the below features for reportWin: // - Exporting console.log, console.error and console.warn logs from UDF @@ -86,7 +88,8 @@ function reportWinEntryFunction( // The wrapper supports: // - Generation of event level reporting urls for buyer // - Exporting console.logs from the AdTech execution. -std::string GetBuyerWrappedCode(absl::string_view buyer_js_code); +std::string GetBuyerWrappedCode(absl::string_view buyer_js_code, + bool enable_protected_app_signals = false); } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/code_wrapper/buyer_reporting_udf_wrapper_test.cc b/services/auction_service/code_wrapper/buyer_reporting_udf_wrapper_test.cc index 8dc9642f..dcbcde52 100644 --- a/services/auction_service/code_wrapper/buyer_reporting_udf_wrapper_test.cc +++ b/services/auction_service/code_wrapper/buyer_reporting_udf_wrapper_test.cc @@ -13,17 +13,26 @@ // limitations under the License. #include "services/auction_service/code_wrapper/buyer_reporting_udf_wrapper.h" +#include + #include "gtest/gtest.h" #include "services/auction_service/code_wrapper/buyer_reporting_test_constants.h" namespace privacy_sandbox::bidding_auction_servers { namespace { -TEST(GetBuyerWrappedCode, GeneratesCompleteFinalCode) { + +using ::testing::StrEq; + +TEST(GetBuyerWrappedCode, GeneratesCompleteFinalCodeForPA) { std::string observed = GetBuyerWrappedCode(kTestReportWinUdfWithValidation); - EXPECT_EQ(observed, kExpectedBuyerCodeWithReportWin) - << "Observed:\n" - << observed << "\n\n" - << "Expected: " << kExpectedBuyerCodeWithReportWin; + EXPECT_THAT(observed, StrEq(kExpectedBuyerCodeWithReportWinForPA)); +} + +TEST(GetBuyerWrappedCode, GeneratesCompleteFinalCodeForPAS) { + bool enable_protected_app_signals = true; + std::string observed = GetBuyerWrappedCode(kTestReportWinUdfWithValidation, + enable_protected_app_signals); + EXPECT_THAT(observed, StrEq(kExpectedBuyerCodeWithReportWinForPAS)); } } // namespace } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/code_wrapper/js/private_aggregation_wrapper_test.js b/services/auction_service/code_wrapper/js/private_aggregation_wrapper_test.js index e62c465e..c36fbb43 100644 --- a/services/auction_service/code_wrapper/js/private_aggregation_wrapper_test.js +++ b/services/auction_service/code_wrapper/js/private_aggregation_wrapper_test.js @@ -26,23 +26,40 @@ testSuite({ value: 30, }; - const customEventName = 'my_custom_event'; - // Test with ReservedEventConstants const reservedWin = 'reserved.win'; assertEquals(ReservedEventConstants.WIN, reservedWin); privateAggregation.contributeToHistogramOnEvent(ReservedEventConstants.WIN, contribution); assertEquals(privateAggregationContributions.win.length, 1); // Assert the expected fields and values directly - assertEquals(privateAggregationContributions.win[0].bucket_128_bit, 10); - assertEquals(privateAggregationContributions.win[0].int_value, 30); + assertEquals(privateAggregationContributions.win[0].bucket.bucket_128_bit.bucket_128_bits[0], 10); + assertEquals(privateAggregationContributions.win[0].bucket.bucket_128_bit.bucket_128_bits[1], 0); + assertEquals(privateAggregationContributions.win[0].value.int_value, 30); + }, + + /** @return {void} */ + /** @return {void} */ + testContributeToHistogramOnEventWithCustomEvent_ValidNumericBucketAndValue() { + const contribution = { + bucket: 10, + value: 30, + }; + + const customEventName = 'my_custom_event'; // Test with custom event privateAggregation.contributeToHistogramOnEvent(customEventName, contribution); assertEquals(privateAggregationContributions.custom_events.get(customEventName).length, 1); // Assert the expected fields and values directly - assertEquals(privateAggregationContributions.custom_events.get(customEventName)[0].bucket_128_bit, 10); - assertEquals(privateAggregationContributions.custom_events.get(customEventName)[0].int_value, 30); + assertEquals( + privateAggregationContributions.custom_events.get(customEventName)[0].bucket.bucket_128_bit.bucket_128_bits[0], + 10 + ); + assertEquals( + privateAggregationContributions.custom_events.get(customEventName)[0].bucket.bucket_128_bit.bucket_128_bits[1], + 0 + ); + assertEquals(privateAggregationContributions.custom_events.get(customEventName)[0].value.int_value, 30); }, /** @return {void} */ @@ -75,78 +92,77 @@ testSuite({ /** @return {void} */ testContributeToHistogramOnEvent_ValidObjectBucketAndValue() { const bucket_object = { - base_value: 'winning-bid', + baseValue: 'winning-bid', + scale: 1.0, + offset: 10, + }; + const signal_bucket = { + base_value: 'BASE_VALUE_WINNING_BID', scale: 1.0, offset: { - value: [10, 20], + value: [10, 0], is_negative: false, }, }; - const value_object = { - base_value: 'winning-bid', + baseValue: 'winning-bid', + scale: 1.0, offset: 30, + }; + const signal_value = { + base_value: 'BASE_VALUE_WINNING_BID', scale: 1.0, + offset: 30, }; const contribution = { bucket: bucket_object, value: value_object, }; - const customEventName = 'my_custom_event'; - // Test with ReservedEventConstants privateAggregation.contributeToHistogramOnEvent(ReservedEventConstants.WIN, contribution); assertEquals(privateAggregationContributions.win.length, 1); // Assert the expected fields and values directly - assertObjectEquals(privateAggregationContributions.win[0].signal_bucket, bucket_object); - assertObjectEquals(privateAggregationContributions.win[0].extended_value, value_object); - - // Test with custom event - privateAggregation.contributeToHistogramOnEvent(customEventName, contribution); - assertEquals(privateAggregationContributions.custom_events.get(customEventName).length, 1); - // Assert the expected fields and values directly - assertObjectEquals( - privateAggregationContributions.custom_events.get(customEventName)[0].signal_bucket, - bucket_object - ); - assertObjectEquals( - privateAggregationContributions.custom_events.get(customEventName)[0].extended_value, - value_object - ); + assertObjectEquals(privateAggregationContributions.win[0].bucket.signal_bucket, signal_bucket); + assertObjectEquals(privateAggregationContributions.win[0].value.extended_value, signal_value); }, /** @return {void} */ testContributeToHistogramOnEvent_ValidObjectBucketAndNumericValue() { const contribution = { bucket: { - base_value: 'winning-bid', + baseValue: 'winning-bid', scale: 1.0, - offset: { - value: [10, 20], - is_negative: false, - }, + offset: 10, }, value: 30, }; + const signal_bucket = { + base_value: 'BASE_VALUE_WINNING_BID', + scale: 1.0, + offset: { + value: [10, 0], + is_negative: false, + }, + }; const customEventName = 'my_custom_event'; // Test with ReservedEventConstants privateAggregation.contributeToHistogramOnEvent(ReservedEventConstants.WIN, contribution); + // Test with custom event + privateAggregation.contributeToHistogramOnEvent(customEventName, contribution); assertEquals(privateAggregationContributions.win.length, 1); // Assert the expected fields and values directly - assertObjectEquals(privateAggregationContributions.win[0].signal_bucket, contribution.bucket); - assertEquals(privateAggregationContributions.win[0].int_value, 30); + assertObjectEquals(privateAggregationContributions.win[0].bucket.signal_bucket, signal_bucket); + assertEquals(privateAggregationContributions.win[0].value.int_value, 30); - // Test with custom event - privateAggregation.contributeToHistogramOnEvent(customEventName, contribution); assertEquals(privateAggregationContributions.custom_events.get(customEventName).length, 1); // Assert the expected fields and values directly assertObjectEquals( - privateAggregationContributions.custom_events.get(customEventName)[0].signal_bucket, - contribution.bucket + privateAggregationContributions.custom_events.get(customEventName)[0].bucket.signal_bucket, + signal_bucket ); - assertEquals(privateAggregationContributions.custom_events.get(customEventName)[0].int_value, 30); + assertEquals(privateAggregationContributions.custom_events.get(customEventName)[0].value.int_value, 30); }, /** @return {void} */ @@ -154,28 +170,40 @@ testSuite({ const contribution = { bucket: 5, value: { - base_value: 'winning-bid', - value: 30, - is_negative: false, + baseValue: 'winning-bid', + scale: 10, + offset: 20, }, }; - + const signalValue = { + base_value: 'BASE_VALUE_WINNING_BID', + scale: 10, + offset: 20, + }; const customEventName = 'my_custom_event'; // Test with ReservedEventConstants privateAggregation.contributeToHistogramOnEvent(ReservedEventConstants.WIN, contribution); assertEquals(privateAggregationContributions.win.length, 1); - assertEquals(privateAggregationContributions.win[0].bucket_128_bit, 5); - assertObjectEquals(privateAggregationContributions.win[0].extended_value, contribution.value); + assertEquals(privateAggregationContributions.win[0].bucket.bucket_128_bit.bucket_128_bits[0], 5); + assertEquals(privateAggregationContributions.win[0].bucket.bucket_128_bit.bucket_128_bits[1], 0); + assertObjectEquals(privateAggregationContributions.win[0].value.extended_value, signalValue); // Test with custom event privateAggregation.contributeToHistogramOnEvent(customEventName, contribution); assertEquals(privateAggregationContributions.custom_events.get(customEventName).length, 1); // Assert the expected fields and values directly - assertEquals(privateAggregationContributions.custom_events.get(customEventName)[0].bucket_128_bit, 5); + assertEquals( + privateAggregationContributions.custom_events.get(customEventName)[0].bucket.bucket_128_bit.bucket_128_bits[0], + 5 + ); + assertEquals( + privateAggregationContributions.custom_events.get(customEventName)[0].bucket.bucket_128_bit.bucket_128_bits[1], + 0 + ); assertObjectEquals( - privateAggregationContributions.custom_events.get(customEventName)[0].extended_value, - contribution.value + privateAggregationContributions.custom_events.get(customEventName)[0].value.extended_value, + signalValue ); }, @@ -201,8 +229,9 @@ testSuite({ privateAggregation.contributeToHistogram(contribution); assertEquals(privateAggregationContributions.always.length, 1); // Assert the expected fields and values directly - assertEquals(privateAggregationContributions.always[0].bucket_128_bit, 10); - assertEquals(privateAggregationContributions.always[0].int_value, 30); + assertEquals(privateAggregationContributions.always[0].bucket.bucket_128_bit.bucket_128_bits[0], 10); + assertEquals(privateAggregationContributions.always[0].bucket.bucket_128_bit.bucket_128_bits[1], 0); + assertEquals(privateAggregationContributions.always[0].value.int_value, 30); const expectedErrorType = 'TypeError'; const expectedErrorMessage = 'Attempting to define property on object that is not extensible.'; diff --git a/services/auction_service/private_aggregation/BUILD b/services/auction_service/private_aggregation/BUILD new file mode 100644 index 00000000..2ec99a29 --- /dev/null +++ b/services/auction_service/private_aggregation/BUILD @@ -0,0 +1,64 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") + +package(default_visibility = ["//services/auction_service:__pkg__"]) + +cc_library( + name = "private_aggregation_manager", + srcs = [ + "private_aggregation_manager.cc", + ], + hdrs = [ + "private_aggregation_manager.h", + ], + deps = [ + "//api:bidding_auction_servers_cc_proto", + "//services/common/code_dispatch:code_dispatch_reactor", + "//services/common/loggers:request_log_context", + "//services/common/private_aggregation:private_aggregation_post_auction_util", + "//services/common/util:json_util", + "@com_google_absl//absl/strings", + "@google_privacysandbox_servers_common//src/logger:request_context_impl", + "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", + ], +) + +cc_test( + name = "private_aggregation_manager_test", + timeout = "short", + srcs = [ + "private_aggregation_manager_test.cc", + ], + deps = [ + ":private_aggregation_manager_test_util", + ], +) + +cc_library( + name = "private_aggregation_manager_test_util", + testonly = True, + srcs = ["private_aggregation_manager_test_util.cc"], + hdrs = ["private_aggregation_manager_test_util.h"], + visibility = [ + "//services/auction_service/private_aggregation:__pkg__", + ], + deps = [ + ":private_aggregation_manager", + "@com_google_googletest//:gtest_main", + "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", + "@rapidjson", + ], +) diff --git a/services/auction_service/private_aggregation/private_aggregation_manager.cc b/services/auction_service/private_aggregation/private_aggregation_manager.cc new file mode 100644 index 00000000..3021f6c9 --- /dev/null +++ b/services/auction_service/private_aggregation/private_aggregation_manager.cc @@ -0,0 +1,345 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "services/auction_service/private_aggregation/private_aggregation_manager.h" + +#include +#include + +#include "api/bidding_auction_servers.pb.h" +#include "include/rapidjson/document.h" +#include "services/common/loggers/request_log_context.h" +#include "services/common/private_aggregation/private_aggregation_post_auction_util.h" +#include "services/common/util/json_util.h" +#include "src/logger/request_context_logger.h" +#include "src/util/status_macro/status_macros.h" + +namespace privacy_sandbox::bidding_auction_servers { + +namespace { + +// Processes a list of Private Aggregation API contributions from an +// array of contributions and append them to `pagg_response`. +// +// contributions: A rapidjson ConstArray object representing an array of +// PrivateAggregation contributions. +// base_values: The base values to use for processing extended values. +// pagg_response: The PrivateAggregateReportingResponse object to append the +// processed contributions to. +void ProcessAndAppendContributionsToPaggResponse( + const rapidjson::GenericValue>::ConstArray& contributions, + const BaseValues& base_values, + PrivateAggregateReportingResponse& pagg_response) { + for (const auto& event_obj : contributions) { + absl::StatusOr new_contribution = + ParseAndProcessContribution(base_values, event_obj); + if (!new_contribution.ok()) { + // TODO(b/362803378): Add metrics for PAgg contributions. + PS_VLOG(kNoisyWarn) << "Invalid contribution: " + << new_contribution.status(); + continue; + } + *pagg_response.add_contributions() = std::move(new_contribution.value()); + } +} + +// Parses and processes contributions for a specific ad event type and appends +// them to the `private_aggregation_reporting_response`. +// +// event_type: The type of ad event (e.g., win, loss, always). +// id: The ID of the ad. +// contribution_obj_doc: The JSON document containing the contributions for +// the specified event type. +// base_values: The BaseValues struct to use for processing extended values. +// private_aggregation_reporting_response: The PrivateAggregateReportingResponse +// object to append the processed contributions to. +void HandleContribution( + EventType event_type, const std::string& id, + const rapidjson::Document& contribution_obj_doc, + const BaseValues& base_values, + PrivateAggregateReportingResponse& private_aggregation_reporting_response) { + absl::Status result = AppendAdEventContributionsToPaggResponse( + event_type, contribution_obj_doc, base_values, + private_aggregation_reporting_response); + if (!result.ok()) { + PS_VLOG(kNoisyWarn) << "Error from Parsing Contributions from ID " << id + << ": " << result; + } +} + +} // namespace + +BaseValue ToBaseValue(absl::string_view base_value_str) { + if (base_value_str.empty()) { + return BaseValue::BASE_VALUE_UNSPECIFIED; + } + + if (base_value_str == kBaseValueWinningBid) { + return BaseValue::BASE_VALUE_WINNING_BID; + } else if (base_value_str == kBaseValueHighestScoringOtherBid) { + return BaseValue::BASE_VALUE_HIGHEST_SCORING_OTHER_BID; + } else if (base_value_str == kBaseValueScriptRunTime) { + return BaseValue::BASE_VALUE_SCRIPT_RUN_TIME; + } else if (base_value_str == kBaseValueSignalsFetchTime) { + return BaseValue::BASE_VALUE_SIGNALS_FETCH_TIME; + } else if (base_value_str == kBaseValueBidRejectionReason) { + return BaseValue::BASE_VALUE_BID_REJECTION_REASON; + } else { + return BaseValue::BASE_VALUE_UNSPECIFIED; + } +} + +// The return absl::string_view must be a global inline variable +// See https://abseil.io/tips/168 and https://abseil.io/tips/140 +absl::string_view EventTypeToString(EventType event_type) { + switch (event_type) { + case EventType::EVENT_TYPE_WIN: + return kEventTypeWin; + case EventType::EVENT_TYPE_LOSS: + return kEventTypeLoss; + case EventType::EVENT_TYPE_ALWAYS: + return kEventTypeAlways; + case EventType::EVENT_TYPE_CUSTOM: + return kEventTypeCustom; + default: + return kEventTypeUnspecified; + } +} + +void HandlePrivateAggregationReporting( + const PrivateAggregationHandlerMetadata& metadata, + const rapidjson::Document& contribution_obj_doc, + ScoreAdsResponse::AdScore& score_ads_response) { + PrivateAggregateReportingResponse private_aggregation_reporting_response; + // Parses contribution_obj_doc based on whether the ad won or lost + // Use the static function instead of the lambda + if (metadata.ad_id != metadata.most_desirable_ad_score_id) { + HandleContribution(EventType::EVENT_TYPE_LOSS, metadata.ad_id, + contribution_obj_doc, metadata.base_values, + private_aggregation_reporting_response); + HandleContribution(EventType::EVENT_TYPE_ALWAYS, metadata.ad_id, + contribution_obj_doc, metadata.base_values, + private_aggregation_reporting_response); + } else { + HandleContribution(EventType::EVENT_TYPE_WIN, metadata.ad_id, + contribution_obj_doc, metadata.base_values, + private_aggregation_reporting_response); + HandleContribution(EventType::EVENT_TYPE_ALWAYS, metadata.ad_id, + contribution_obj_doc, metadata.base_values, + private_aggregation_reporting_response); + HandleContribution(EventType::EVENT_TYPE_CUSTOM, metadata.ad_id, + contribution_obj_doc, metadata.base_values, + private_aggregation_reporting_response); + } + score_ads_response.add_top_level_contributions()->Swap( + &private_aggregation_reporting_response); +} + +absl::Status AppendAdEventContributionsToPaggResponse( + EventType event_type, const rapidjson::Value& contributions, + const BaseValues& base_values, + PrivateAggregateReportingResponse& pagg_response) { + if (event_type == EventType::EVENT_TYPE_UNSPECIFIED) { + return absl::InvalidArgumentError("Event type unspecified."); + } + + absl::string_view event_type_str = EventTypeToString(event_type); + + // data() and length() are used because event_type_str is a global inline + // constant. + auto contributions_it = contributions.FindMember(event_type_str.data()); + + if (contributions_it == contributions.MemberEnd()) { + return absl::InternalError(absl::StrCat( + "Unexpected response from Private Aggregation API execution. " + "\"event_type\" field not found for event type: ", + event_type_str)); + } + const auto& contributions_obj = contributions_it->value; + + if (event_type == EventType::EVENT_TYPE_CUSTOM) { + for (const auto& member : contributions_obj.GetObject()) { + const rapidjson::Value& event_array = member.value; + if (!event_array.IsArray()) { + absl::StatusOr result = SerializeJsonDoc(event_array); + if (result.ok()) { + PS_VLOG(kNoisyWarn) << "Custom Event " << member.name.GetString() + << " does not contain an array. Value of the " + "custom event field: \n" + << result.value(); + } else { + PS_VLOG(kNoisyWarn) + << "Custom Event " << member.name.GetString() + << " does not contain an array. And there's an error " + "logging value of the field: " + << result.status(); + } + + continue; + } + ProcessAndAppendContributionsToPaggResponse(event_array.GetArray(), + base_values, pagg_response); + } + } else { + // Handle win, loss, always events + if (!contributions_obj.IsArray()) { + absl::StatusOr result = SerializeJsonDoc(contributions_obj); + if (result.ok()) { + PS_VLOG(kNoisyWarn) << "Reserved Event " << event_type_str + << " does not contain an array. Value of the " + "reserved event field: \n" + << result.value(); + } else { + PS_VLOG(kNoisyWarn) + << "Reserved Event " << event_type_str + << " does not contain an array. And there's an error " + "logging value of the field: " + << result.status(); + } + } + ProcessAndAppendContributionsToPaggResponse(contributions_obj.GetArray(), + base_values, pagg_response); + } + + return absl::OkStatus(); +} + +absl::StatusOr ParseAndProcessContribution( + const BaseValues& base_values, const rapidjson::Value& pagg_response) { + PrivateAggregateContribution contribution; + std::optional int_value; + SignalValue signal_value; + + auto value_it = pagg_response.FindMember(kPrivateAggregationValue.data()); + if (value_it == pagg_response.MemberEnd()) { + return absl::InvalidArgumentError(kInvalidPrivateAggregationValueType); + } + + PS_ASSIGN_IF_PRESENT(int_value, value_it->value, kSignalValueIntValue, Int64); + if (int_value.has_value()) { + contribution.mutable_value()->set_int_value(*int_value); + } else if (!value_it->value.HasMember(kSignalValueExtendedValue)) { + return absl::InvalidArgumentError(kInvalidPrivateAggregationValueType); + } else { + PS_ASSIGN_OR_RETURN( + signal_value, + ParseSignalValue(value_it->value[kSignalValueExtendedValue]), + absl::InvalidArgumentError(kInvalidPrivateAggregationValueType)); + contribution.mutable_value()->mutable_extended_value()->Swap(&signal_value); + PS_ASSIGN_OR_RETURN(int final_int_value, + GetPrivateAggregationValuePostAuction( + base_values, contribution.value())); + contribution.mutable_value()->set_int_value(final_int_value); + } + + // Parse SignalBucket if present. + auto bucket_it = pagg_response.FindMember(kPrivateAggregationBucket.data()); + if (bucket_it != pagg_response.MemberEnd()) { + auto signal_bucket_it = + bucket_it->value.FindMember(kSignalBucketSignalBucket); + auto bucket_128_bit_it = + bucket_it->value.FindMember(kSignalBucketBucket128Bit); + if (signal_bucket_it != bucket_it->value.MemberEnd()) { + PS_ASSIGN_OR_RETURN( + auto signal_bucket, + ParseSignalBucket(bucket_it->value[kSignalBucketSignalBucket]), + absl::InvalidArgumentError(kInvalidPrivateAggregationBucketType)); + contribution.mutable_bucket()->mutable_signal_bucket()->Swap( + &signal_bucket); + } else if (bucket_128_bit_it != bucket_it->value.MemberEnd()) { + PS_ASSIGN_OR_RETURN( + auto bucket_128_bit, + GetArrayMember(bucket_it->value, kSignalBucketBucket128Bit), + absl::InvalidArgumentError(kInvalidPrivateAggregationBucketType)); + for (auto& bit : bucket_128_bit) { + contribution.mutable_bucket() + ->mutable_bucket_128_bit() + ->add_bucket_128_bits(bit.GetUint64()); + } + } else { + return absl::InvalidArgumentError(kInvalidPrivateAggregationBucketType); + } + PS_ASSIGN_OR_RETURN(Bucket128Bit final_bucket_128_bits, + GetPrivateAggregationBucketPostAuction( + base_values, contribution.bucket())); + contribution.mutable_bucket()->mutable_bucket_128_bit()->Swap( + &final_bucket_128_bits); + } + + return contribution; +} + +absl::StatusOr ParseSignalValue( + const rapidjson::Value& signal_value_json) { + SignalValue signal_value; + std::optional base_value_str; + + PS_ASSIGN_OR_RETURN(base_value_str, GetStringMember(signal_value_json, + kSignalValueBaseValue)); + PS_ASSIGN_OR_RETURN(int offset, + GetIntMember(signal_value_json, kSignalValueOffset)); + PS_ASSIGN_OR_RETURN(double scale, + GetDoubleMember(signal_value_json, kSignalValueScale)); + + signal_value.set_base_value(ToBaseValue(*base_value_str)); + signal_value.set_offset(offset); + signal_value.set_scale(scale); + + return signal_value; +} + +absl::StatusOr ParseSignalBucket( + const rapidjson::Value& signal_bucket_json) { + SignalBucket signal_bucket; + std::optional base_value_str; + + auto bucket_offset_it = signal_bucket_json.FindMember(rapidjson::StringRef( + kSignalBucketOffset.data(), kSignalBucketOffset.length())); + + if (bucket_offset_it == signal_bucket_json.MemberEnd()) { + return absl::InvalidArgumentError("Missing Signal Bucket's offset"); + } + + PS_ASSIGN_OR_RETURN(BucketOffset offset, + ParseBucketOffset(bucket_offset_it->value)); + + PS_ASSIGN_OR_RETURN(base_value_str, GetStringMember(signal_bucket_json, + kSignalBucketBaseValue)); + PS_ASSIGN_OR_RETURN(double scale, + GetDoubleMember(signal_bucket_json, kSignalBucketScale)); + + signal_bucket.set_base_value(ToBaseValue(*base_value_str)); + signal_bucket.mutable_offset()->Swap(&offset); + signal_bucket.set_scale(scale); + + return signal_bucket; +} + +absl::StatusOr ParseBucketOffset( + const rapidjson::Value& signal_bucket_json) { + BucketOffset bucket_offset; + + PS_ASSIGN_OR_RETURN(auto value_array, + GetArrayMember(signal_bucket_json, + std::string(kBucketOffsetValue).c_str())); + PS_ASSIGN_OR_RETURN(bool is_negative, GetBoolMember(signal_bucket_json, + kBucketOffsetIsNegative)); + + bucket_offset.add_value(value_array[0].GetUint64()); + bucket_offset.add_value(value_array[1].GetUint64()); + bucket_offset.set_is_negative(is_negative); + return bucket_offset; +} + +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/private_aggregation/private_aggregation_manager.h b/services/auction_service/private_aggregation/private_aggregation_manager.h new file mode 100644 index 00000000..e18b3792 --- /dev/null +++ b/services/auction_service/private_aggregation/private_aggregation_manager.h @@ -0,0 +1,143 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SERVICES_AUCTION_SERVICE_PRIVATE_AGGREGATION_PRIVATE_AGGREGATION_MANAGER_H_ +#define SERVICES_AUCTION_SERVICE_PRIVATE_AGGREGATION_PRIVATE_AGGREGATION_MANAGER_H_ + +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "api/bidding_auction_servers.pb.h" +#include "rapidjson/document.h" +#include "services/common/code_dispatch/code_dispatch_reactor.h" +#include "services/common/private_aggregation/private_aggregation_post_auction_util.h" + +namespace privacy_sandbox::bidding_auction_servers { + +inline constexpr absl::string_view kBaseValueWinningBid = + "BASE_VALUE_WINNING_BID"; +inline constexpr absl::string_view kBaseValueHighestScoringOtherBid = + "BASE_VALUE_HIGHEST_SCORING_OTHER_BID"; +inline constexpr absl::string_view kBaseValueScriptRunTime = + "BASE_VALUE_SCRIPT_RUN_TIME"; +inline constexpr absl::string_view kBaseValueSignalsFetchTime = + "BASE_VALUE_SIGNALS_FETCH_TIME"; +inline constexpr absl::string_view kBaseValueBidRejectionReason = + "BASE_VALUE_BID_REJECTION_REASON"; +inline constexpr absl::string_view kPrivateAggregateContributionMissingValue = + "Missing value in contribution."; + +inline constexpr char kSignalValueIntValue[] = "int_value"; +inline constexpr char kSignalValueExtendedValue[] = "extended_value"; +inline constexpr char kSignalBucketBucket128Bit[] = "bucket_128_bit"; +inline constexpr char kSignalBucketSignalBucket[] = "signal_bucket"; + +inline constexpr absl::string_view kPrivateAggregationBucket = "bucket"; +inline constexpr absl::string_view kPrivateAggregationValue = "value"; + +inline constexpr absl::string_view kSignalValueBaseValue = "base_value"; +inline constexpr absl::string_view kSignalValueScale = "scale"; +inline constexpr absl::string_view kSignalValueOffset = "offset"; + +inline constexpr absl::string_view kSignalBucketBaseValue = "base_value"; +inline constexpr absl::string_view kSignalBucketScale = "scale"; +inline constexpr absl::string_view kSignalBucketOffset = "offset"; + +inline constexpr absl::string_view kBucketOffsetValue = "value"; +inline constexpr absl::string_view kBucketOffsetIsNegative = "is_negative"; + +inline constexpr absl::string_view kEventTypeWin = "win"; +inline constexpr absl::string_view kEventTypeLoss = "lose"; +inline constexpr absl::string_view kEventTypeAlways = "always"; +inline constexpr absl::string_view kEventTypeCustom = "custom_events"; +inline constexpr absl::string_view kEventTypeUnspecified = "not-specified"; + +using RawRequest = ScoreAdsRequest::ScoreAdsRawRequest; + +struct PrivateAggregationHandlerMetadata { + std::string most_desirable_ad_score_id; + int private_aggregate_report_limit; + BaseValues base_values; + std::string ad_id; +}; + +// Convert baseValue string into enum. +// For more information, see +// https://github.com/WICG/turtledove/blob/main/FLEDGE_extended_PA_reporting.md#reporting-api-informal-specification. +BaseValue ToBaseValue(absl::string_view base_value_str); + +// Convert EventType enum into string. +// Default is "not-specified". +absl::string_view EventTypeToString(EventType event_type); + +// This function handles parsing,filtering and processing of paapicontributions +// exported from Roma: +// - Parses paapi_contributions_doc based on whether the ad won or lost. +// - Processes the contributions to convert SignalBucket/SignalValue into +// numeric type. +// - Populates the contributions in the winning_ad_score +void HandlePrivateAggregationReporting( + const PrivateAggregationHandlerMetadata& metadata, + const rapidjson::Document& contribution_obj_doc, + ScoreAdsResponse::AdScore& score_ads_response); + +// Parses, Processes, and Appends the selected contributions (win, loss, +// always, or custom_events) to input pagg_response. +absl::Status AppendAdEventContributionsToPaggResponse( + EventType event_type, const rapidjson::Value& contributions, + const BaseValues& base_values, + PrivateAggregateReportingResponse& pagg_response); + +// Parses and processes a JSON containing Private Aggregate Contribution Object +// and returns a PrivateAggregateContribution object. +// +// SignalBucket and SignalValue will be converted to numerical value as +// base_value * scale + offset. +absl::StatusOr ParseAndProcessContribution( + const BaseValues& base_values, const rapidjson::Value& pagg_response); + +// Parses a SignalValue object from a rapidjson::Document. +// +// signal_value_json: The rapidjson::Value node from Private Aggregation +// Response that contains SignalValue data. +// +// Returns the parsed and processed SignalValue. +// This function assumes input JSON string contains all required fields for +// SignalValue, and all values are valid. +absl::StatusOr ParseSignalValue( + const rapidjson::Value& signal_value_json); + +// Parses a SignalBucket object from a rapidjson::Value node. +// +// signal_bucket_json: The rapidjson::Value node from Private Aggregation +// Response that contains SignalBucket data. +// - Example: `{"offset": {"value": [1, 2],"is_negative": true}, "base_value": +// "BASE_VALUE_WINNING_BID","scale": 2.5}` +// +// Returns the parsed and processed SignalBucket. +absl::StatusOr ParseSignalBucket( + const rapidjson::Value& signal_bucket_json); + +// Parses a BucketOffset object from a rapidjson::Value node that contains +// Bucket Offset's fields. +absl::StatusOr ParseBucketOffset( + const rapidjson::Value& signal_bucket_json); + +} // namespace privacy_sandbox::bidding_auction_servers + +#endif // SERVICES_AUCTION_SERVICE_PRIVATE_AGGREGATION_PRIVATE_AGGREGATION_MANAGER_H_ diff --git a/services/auction_service/private_aggregation/private_aggregation_manager_test.cc b/services/auction_service/private_aggregation/private_aggregation_manager_test.cc new file mode 100644 index 00000000..c3aa85fe --- /dev/null +++ b/services/auction_service/private_aggregation/private_aggregation_manager_test.cc @@ -0,0 +1,485 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "services/auction_service/private_aggregation/private_aggregation_manager.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "services/auction_service/private_aggregation/private_aggregation_manager_test_util.h" +#include "services/common/util/json_util.h" +#include "src/util/status_macro/status_macros.h" + +namespace privacy_sandbox::bidding_auction_servers { +namespace { + +TEST(PrivateAggregationManagerTest, ToBaseValueUnspecified) { + EXPECT_EQ(ToBaseValue(""), BaseValue::BASE_VALUE_UNSPECIFIED); + EXPECT_EQ(ToBaseValue("unknown-value"), BaseValue::BASE_VALUE_UNSPECIFIED); +} + +TEST(PrivateAggregationManagerTest, ToBaseValueWinningBid) { + EXPECT_EQ(ToBaseValue("BASE_VALUE_WINNING_BID"), + BaseValue::BASE_VALUE_WINNING_BID); +} + +TEST(PrivateAggregationManagerTest, ToBaseValueHighestScoringOtherBid) { + EXPECT_EQ(ToBaseValue("BASE_VALUE_HIGHEST_SCORING_OTHER_BID"), + BaseValue::BASE_VALUE_HIGHEST_SCORING_OTHER_BID); +} + +TEST(PrivateAggregationManagerTest, ToBaseValueScriptRunTime) { + EXPECT_EQ(ToBaseValue("BASE_VALUE_SCRIPT_RUN_TIME"), + BaseValue::BASE_VALUE_SCRIPT_RUN_TIME); +} + +TEST(PrivateAggregationManagerTest, ToBaseValueSignalsFetchTime) { + EXPECT_EQ(ToBaseValue("BASE_VALUE_SIGNALS_FETCH_TIME"), + BaseValue::BASE_VALUE_SIGNALS_FETCH_TIME); +} + +TEST(PrivateAggregationManagerTest, ToBaseValueBidRejectionReason) { + EXPECT_EQ(ToBaseValue("BASE_VALUE_BID_REJECTION_REASON"), + BaseValue::BASE_VALUE_BID_REJECTION_REASON); + EXPECT_EQ(ToBaseValue("unknown-value"), BaseValue::BASE_VALUE_UNSPECIFIED); + EXPECT_EQ(BaseValue::BASE_VALUE_UNSPECIFIED, 0); +} + +TEST(PrivateAggregationManagerTest, EventTypeWinToString) { + EXPECT_EQ(EventTypeToString(EventType::EVENT_TYPE_WIN), kEventTypeWin); +} + +TEST(PrivateAggregationManagerTest, EventTypeLossToString) { + EXPECT_EQ(EventTypeToString(EventType::EVENT_TYPE_LOSS), kEventTypeLoss); +} + +TEST(PrivateAggregationManagerTest, EventTypeAlwaysToString) { + EXPECT_EQ(EventTypeToString(EventType::EVENT_TYPE_ALWAYS), kEventTypeAlways); +} + +TEST(PrivateAggregationManagerTest, EventTypeCustomToString) { + EXPECT_EQ(EventTypeToString(EventType::EVENT_TYPE_CUSTOM), kEventTypeCustom); +} + +TEST(PrivateAggregationManagerTest, EventTypeUnspecifiedToString) { + EXPECT_EQ(EventTypeToString(EventType::EVENT_TYPE_UNSPECIFIED), + kEventTypeUnspecified); +} + +TEST(HandlePrivateAggregationReportingTest, + EmptyContributionResponseJSONSuccess) { + BaseValues base_values = MakeDefaultBaseValues(); + PrivateAggregationHandlerMetadata metadata = { + .most_desirable_ad_score_id = kTestWinningId.data(), + .base_values = base_values, + .ad_id = kTestWinningId.data(), + }; + + ScoreAdsResponse::AdScore score_ads_response; + rapidjson::Document doc; + doc.SetObject(); + HandlePrivateAggregationReporting(metadata, doc, score_ads_response); + EXPECT_EQ(score_ads_response.top_level_contributions(0).contributions_size(), + 0); +} + +TEST(HandlePrivateAggregationReportingTest, ValidWinningAdSuccess) { + BaseValues base_values = MakeDefaultBaseValues(); + PrivateAggregationHandlerMetadata metadata = { + .most_desirable_ad_score_id = kTestWinningId.data(), + .base_values = base_values, + .ad_id = kTestWinningId.data(), + }; + + PrivateAggregateContribution contribution1; + contribution1.mutable_value()->set_int_value(23); + contribution1.mutable_bucket()->mutable_bucket_128_bit()->add_bucket_128_bits( + 1); + contribution1.mutable_bucket()->mutable_bucket_128_bit()->add_bucket_128_bits( + 1); + + PrivateAggregateContribution contribution2; + contribution2.mutable_value()->mutable_extended_value()->set_base_value( + BaseValue::BASE_VALUE_WINNING_BID); + contribution2.mutable_value()->mutable_extended_value()->set_scale(2.0); + contribution2.mutable_value()->mutable_extended_value()->set_offset(10); + contribution2.mutable_bucket()->mutable_bucket_128_bit()->add_bucket_128_bits( + 1); + contribution2.mutable_bucket()->mutable_bucket_128_bit()->add_bucket_128_bits( + 1); + + std::vector win_contributions = {contribution1, + contribution2}; + std::vector always_contributions = { + contribution1, contribution2}; + absl::flat_hash_map> + custom_event_contributions; + custom_event_contributions.try_emplace("custom_event_1", win_contributions); + + rapidjson::Document doc = CreateContributionResponseDocument( + win_contributions, {}, always_contributions, custom_event_contributions); + + ScoreAdsResponse::AdScore score_ads_response; + HandlePrivateAggregationReporting(metadata, doc, score_ads_response); + EXPECT_EQ(score_ads_response.top_level_contributions(0).contributions_size(), + 6); + EXPECT_EQ(score_ads_response.top_level_contributions(0) + .contributions(0) + .value() + .int_value(), + contribution1.value().int_value()); + EXPECT_EQ( + score_ads_response.top_level_contributions(0) + .contributions(1) + .value() + .int_value(), + static_cast(kTestBidInSellerCurrency * + contribution2.value().extended_value().scale() + + contribution2.value().extended_value().offset())); +} + +TEST(HandlePrivateAggregationReportingTest, ValidLosingAdSuccess) { + BaseValues base_values = MakeDefaultBaseValues(); + PrivateAggregationHandlerMetadata metadata = { + .most_desirable_ad_score_id = kTestWinningId.data(), + .base_values = base_values, + .ad_id = kTestLossId.data(), + }; + + PrivateAggregateContribution contribution1; + contribution1.mutable_value()->set_int_value(23); + contribution1.mutable_bucket()->mutable_bucket_128_bit()->add_bucket_128_bits( + 1); + contribution1.mutable_bucket()->mutable_bucket_128_bit()->add_bucket_128_bits( + 1); + + std::vector loss_contributions = { + contribution1}; + + PrivateAggregateContribution contribution2; + contribution2.mutable_value()->mutable_extended_value()->set_base_value( + BaseValue::BASE_VALUE_WINNING_BID); + contribution2.mutable_value()->mutable_extended_value()->set_scale(2.0); + contribution2.mutable_value()->mutable_extended_value()->set_offset(10); + contribution2.mutable_bucket()->mutable_bucket_128_bit()->add_bucket_128_bits( + 1); + contribution2.mutable_bucket()->mutable_bucket_128_bit()->add_bucket_128_bits( + 1); + + std::vector always_contributions = { + contribution2}; + + ScoreAdsResponse::AdScore score_ads_response; + HandlePrivateAggregationReporting( + metadata, + CreateContributionResponseDocument({}, loss_contributions, + always_contributions, {}), + score_ads_response); + EXPECT_EQ(score_ads_response.top_level_contributions(0).contributions_size(), + 2); + EXPECT_EQ(score_ads_response.top_level_contributions(0) + .contributions(0) + .value() + .int_value(), + 23); + EXPECT_EQ( + score_ads_response.top_level_contributions(0) + .contributions(1) + .value() + .int_value(), + static_cast(kTestBidInSellerCurrency * + contribution2.value().extended_value().scale() + + contribution2.value().extended_value().offset())); +} + +TEST(AppendAdEventContributionsToPaggResponseTest, IntValueWithWinEvent) { + BaseValues base_values = MakeDefaultBaseValues(); + + ValidatePAGGContributions( + EventType::EVENT_TYPE_WIN, + R"json({"win": [{"value": {"int_value": 10}}, {"value": {"int_value": 20}}]})json", + base_values, 2, absl::StatusCode::kOk); +} + +TEST(AppendAdEventContributionsToPaggResponseTest, ExtendedValueWithWinEvent) { + BaseValues base_values = MakeDefaultBaseValues(); + + ValidatePAGGContributions( + EventType::EVENT_TYPE_WIN, + R"json({"win": [{"value": {"extended_value": {"base_value": "BASE_VALUE_WINNING_BID", "offset": 1, "scale": 2.0}}}, {"value": {"extended_value": {"base_value": "BASE_VALUE_HIGHEST_SCORING_OTHER_BID", "offset": 1, "scale": 2.0}}}]})json", + base_values, 2, absl::StatusCode::kOk); +} + +TEST(AppendAdEventContributionsToPaggResponseTest, UnsupportedBaseValue) { + BaseValues base_values = MakeDefaultBaseValues(); + + // Expected contribution count is 0 + // because BASE_VALUE_SCRIPT_RUN_TIME is not yet supported. + ValidatePAGGContributions( + EventType::EVENT_TYPE_WIN, + R"json({"win": [{"value": {"extended_value": {"base_value": "BASE_VALUE_SCRIPT_RUN_TIME", "offset": 1, "scale": 2.0}}}]})json", + base_values, 0, absl::StatusCode::kOk); +} + +TEST(AppendAdEventContributionsToPaggResponseTest, InvalidEventType) { + BaseValues base_values = MakeDefaultBaseValues(); + + ValidatePAGGContributions( + EventType::EVENT_TYPE_UNSPECIFIED, + R"json({"win": [{"value": {"int_value": 10}}]})json", base_values, 0, + absl::StatusCode::kInvalidArgument); +} + +TEST(AppendAdEventContributionsToPaggResponseTest, CustomEventType) { + BaseValues base_values = MakeDefaultBaseValues(); + + ValidatePAGGContributions( + EventType::EVENT_TYPE_CUSTOM, + R"json({"custom_events": {"event1": [{"value": {"int_value": 10}}], "event2": [{"value": {"int_value": 20}}] }})json", + base_values, 2, absl::StatusCode::kOk); +} + +TEST(AppendAdEventContributionsToPaggResponseTest, + CustomEventTypeWithInvalidEvent) { + BaseValues base_values = MakeDefaultBaseValues(); + + ValidatePAGGContributions( + EventType::EVENT_TYPE_CUSTOM, + R"json({"custom_events": {"event1": [{"value": {"int_value": 10}}], "event2": 20}})json", + base_values, 1, absl::StatusCode::kOk); +} + +TEST(AppendAdEventContributionsToPaggResponseTest, MissingSpecifiedEventType) { + BaseValues base_values = MakeDefaultBaseValues(); + + ValidatePAGGContributions(EventType::EVENT_TYPE_WIN, R"json({})json", + base_values, 0, absl::StatusCode::kInternal); +} + +TEST(AppendAdEventContributionsToPaggResponseTest, + RejectionReasonAsBaseValueButNoRejectionReason) { + BaseValues base_values = MakeDefaultBaseValuesWithNoRejectionReason(); + + ValidatePAGGContributions( + EventType::EVENT_TYPE_WIN, + R"json({"win": [{"value": {"extended_value": {"base_value": "BASE_VALUE_BID_REJECTION_REASON", "offset": 1, "scale": 2.0}}}]})json", + base_values, 0, absl::StatusCode::kOk); +} + +TEST(ParseAndProcessContributionTest, + PAggValueAsIntValueAndBucketAsSignalBucketSuccess) { + BaseValues base_values = MakeDefaultBaseValues(); + + TestParseAndProcessContribution( + /* base_values */ base_values, + /* json_string */ + R"json({"value": {"int_value": 10}, "bucket": {"bucket_128_bit": [1, 2]}})json", + /* expected_int_value */ 10, /* expected lower bits*/ 1, + /* expected upper bits */ 2, absl::StatusCode::kOk); +} + +TEST(ParseAndProcessContributionTest, PAggValueAsExtendedValueSuccess) { + BaseValues base_values = MakeDefaultBaseValues(); + + TestParseAndProcessContribution( + /* base_values */ base_values, + /* json_string */ + R"json({"value": {"extended_value": {"base_value": "BASE_VALUE_WINNING_BID", "offset": 1, "scale": 2.0}}, "bucket": {"bucket_128_bit": [1, 2]}})json", + /* expected_int_value */ 21, + /* expected lower bits*/ 1, /* expected upper bits */ 2, + absl::StatusCode::kOk); +} + +TEST(ParseAndProcessContributionTest, + PAggValueWithInvalidBaseValueReturnInvalidArgumentError) { + BaseValues base_values = MakeDefaultBaseValues(); + + TestParseAndProcessContribution( + /* base_values */ base_values, + /* json_string */ + R"json({"value": {"extended_value": {"base_value": "BASE_VALUE_SCRIPT_RUN_TIME", "offset": 1, "scale": 2.0}}, "bucket": {"bucket_128_bit": [1, 2]}})json", + /* expected_int_value */ 21, + /* expected lower bits*/ 1, /* expected upper bits */ 2, + absl::StatusCode::kInvalidArgument); +} + +TEST(ParseAndProcessContributionTest, + ContributionWithInvalidValueFieldNameReturnInvalidArgumentError) { + BaseValues base_values = MakeDefaultBaseValues(); + + TestParseAndProcessContribution( + /* base_values */ base_values, + /* json_string */ + R"json({"value": {"invalid_field_name": {"base_value": "BASE_VALUE_WINNING_BID", "offset": 1, "scale": 2.0}}, "bucket": {"bucket_128_bit": [1, 2]}})json", + /* expected_int_value */ 21, + /* expected lower bits*/ 1, /* expected upper bits */ 2, + absl::StatusCode::kInvalidArgument); +} + +TEST(ParseAndProcessContributionTest, + PAggValueExtendedValueMissingBaseValueReturnInvalidArgumentError) { + BaseValues base_values = MakeDefaultBaseValues(); + + TestParseAndProcessContribution( + /* base_values */ base_values, + /* json_string */ + R"json({"value": {"extended_value": {"offset": 1, "scale": 2.0}}, "bucket": {"bucket_128_bit": [1, 2]}})json", + /* expected_int_value */ 21, + /* expected lower bits*/ 1, /* expected upper bits */ 2, + absl::StatusCode::kInvalidArgument); +} + +TEST(ParseAndProcessContributionTest, + PAggValueExtendedValueMissingOffsetReturnInvalidArgumentError) { + BaseValues base_values = MakeDefaultBaseValues(); + + TestParseAndProcessContribution( + /* base_values */ base_values, + /* json_string */ + R"json({"value": {"extended_value": {"base_value": "BASE_VALUE_WINNING_BID", "offset": "invalid_offset", "scale": 2.0}}, "bucket": {"bucket_128_bit": [1, 2]}})json", + /* expected_int_value */ 21, + /* expected lower bits*/ 1, /* expected upper bits */ 2, + absl::StatusCode::kInvalidArgument); +} + +TEST(ParseAndProcessContributionTest, + PAggValueExtendedValueMissingScaleReturnInvalidArgumentError) { + BaseValues base_values = MakeDefaultBaseValues(); + + TestParseAndProcessContribution( + /* base_values */ base_values, + /* json_string */ + R"json({"value": {"extended_value": {"base_value": "BASE_VALUE_WINNING_BID", "offset": 1}}, "bucket": {"bucket_128_bit": [1, 2]}})json", + /* expected_int_value */ 21, + /* expected lower bits*/ 1, /* expected upper bits */ 2, + absl::StatusCode::kInvalidArgument); +} + +TEST(ParseAndProcessContributionTest, + PAggValueExtendedValueInvalidOffsetFieldReturnInvalidArgumentError) { + BaseValues base_values = MakeDefaultBaseValues(); + + TestParseAndProcessContribution( + /* base_values */ base_values, + /* json_string */ + R"json({"value": {"extended_value": {"base_value": "BASE_VALUE_WINNING_BID", "offset": "invalid_offset_field", "scale": 2.0}}, "bucket": {"bucket_128_bit": [1, 2]}})json", + /* expected_int_value */ 21, + /* expected lower bits*/ 1, /* expected upper bits */ 2, + absl::StatusCode::kInvalidArgument); +} + +TEST(ParseAndProcessContributionTest, + PAggValueExtendedValueInvalidScaleFieldReturnInvalidArgumentError) { + BaseValues base_values = MakeDefaultBaseValues(); + + TestParseAndProcessContribution( + /* base_values */ base_values, + /* json_string */ + R"json({"value": {"extended_value": {"base_value": "BASE_VALUE_WINNING_BID", "offset": 1, "scale": "invalid_scale_field"}}, "bucket": {"bucket_128_bit": [1, 2]}})json", + /* expected_int_value */ 21, + /* expected lower bits*/ 1, /* expected upper bits */ 2, + absl::StatusCode::kInvalidArgument); +} + +TEST(ParseAndProcessContributionTest, + PAggValueRejectionReasonAsBaseValueWithoutOneReturnInvalidArgumentError) { + BaseValues base_values = MakeDefaultBaseValuesWithNoRejectionReason(); + + TestParseAndProcessContribution( + /* base_values */ base_values, + /* json_string */ + R"json({"value": {"extended_value": {"base_value": "BASE_VALUE_BID_REJECTION_REASON", "offset": 1, "scale": 2.0}}, "bucket": {"bucket_128_bit": [1, 2]}})json", + /* expected_int_value */ 21, + /* expected lower bits*/ 1, /* expected upper bits */ 2, + absl::StatusCode::kInvalidArgument); +} + +TEST(PrivateAggregationManagerTest, ParsePrivateAggregationSignalValueSuccess) { + TestParseSignalValue( + R"JSON({"base_value": "BASE_VALUE_WINNING_BID","offset": 10,"scale": 2.5})JSON", + BaseValue::BASE_VALUE_WINNING_BID, 10, 2.5, absl::StatusCode::kOk); +} + +TEST(PrivateAggregationManagerTest, + ParsePrivateAggregationSignalValueMissingBaseValue) { + TestParseSignalValue(R"JSON({"offset": 10,"scale": 2.5})JSON", + BaseValue::BASE_VALUE_UNSPECIFIED, 0, 0, + absl::StatusCode::kInvalidArgument); +} + +TEST(PrivateAggregationManagerTest, + ParsePrivateAggregationSignalValueMissingOffset) { + TestParseSignalValue( + R"JSON({"base_value": "BASE_VALUE_WINNING_BID","scale": 2.5})JSON", + BaseValue::BASE_VALUE_UNSPECIFIED, 0, 0, + absl::StatusCode::kInvalidArgument); +} + +TEST(PrivateAggregationManagerTest, + ParsePrivateAggregationSignalValueMissingScale) { + TestParseSignalValue( + R"JSON({"base_value": "BASE_VALUE_WINNING_BID","offset": 10})JSON", + BaseValue::BASE_VALUE_UNSPECIFIED, 0, 0, + absl::StatusCode::kInvalidArgument); +} + +TEST(PrivateAggregationManagerTest, ParseSignalBucketSuccess) { + TestParseSignalBucket( + R"json({"offset": {"value": [1, 2],"is_negative": true}, "base_value": "BASE_VALUE_WINNING_BID","scale": 2.5})json", + BaseValue::BASE_VALUE_WINNING_BID, 1, 2, true, 2.5, + absl::StatusCode::kOk); +} + +TEST(PrivateAggregationManagerTest, ParseSignalBucketMissingOffset) { + TestParseSignalBucket( + R"json({"base_value": "BASE_VALUE_WINNING_BID","scale": 2.5})json", + BaseValue::BASE_VALUE_UNSPECIFIED, 0, 0, false, 0, + absl::StatusCode::kInvalidArgument); +} + +TEST(PrivateAggregationManagerTest, ParseSignalBucketMissingBaseValue) { + TestParseSignalBucket( + R"json({"offset": {"value": [1, 2],"is_negative": true}, "scale": 2.5})json", + BaseValue::BASE_VALUE_UNSPECIFIED, 0, 0, false, 0, + absl::StatusCode::kInvalidArgument); +} + +TEST(PrivateAggregationManagerTest, ParseSignalBucketMissingScale) { + TestParseSignalBucket( + R"json({"offset": {"value": [1, 2],"is_negative": true}, "base_value": "BASE_VALUE_WINNING_BID"})json", + BaseValue::BASE_VALUE_UNSPECIFIED, 0, 0, false, 0, + absl::StatusCode::kInvalidArgument); +} + +TEST(PrivateAggregationManagerTest, ParseEmptySignalBucket) { + TestParseSignalBucket(R"json({})json", BaseValue::BASE_VALUE_UNSPECIFIED, 0, + 0, false, 0, absl::StatusCode::kInvalidArgument); +} + +TEST(PrivateAggregationManagerTest, ParseBucketOffsetSuccess) { + TestParseBucketOffset(R"json({"value": [1, 2],"is_negative": true})json", 1, + 2, true, absl::StatusCode::kOk); +} + +TEST(PrivateAggregationManagerTest, ParseBucketOffsetMissingIsNegative) { + TestParseBucketOffset(R"json({"value": [1, 2]})json", 1, 2, false, + absl::StatusCode::kInvalidArgument); +} + +TEST(PrivateAggregationManagerTest, ParseEmptyBucketOffset) { + TestParseBucketOffset(R"json({})json", 1, 2, false, + absl::StatusCode::kInvalidArgument); +} + +} // namespace +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/private_aggregation/private_aggregation_manager_test_util.cc b/services/auction_service/private_aggregation/private_aggregation_manager_test_util.cc new file mode 100644 index 00000000..db36b196 --- /dev/null +++ b/services/auction_service/private_aggregation/private_aggregation_manager_test_util.cc @@ -0,0 +1,263 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "services/auction_service/private_aggregation/private_aggregation_manager_test_util.h" + +#include "gtest/gtest.h" +#include "include/rapidjson/document.h" +#include "services/common/util/json_util.h" + +namespace privacy_sandbox::bidding_auction_servers { + +namespace { + +// Converts a PrivateAggregateContribution object to its JSON representation. +rapidjson::Value ContributionToJson( + const PrivateAggregateContribution& contribution, + rapidjson::Document::AllocatorType& allocator) { + rapidjson::Value contribution_json(rapidjson::kObjectType); + rapidjson::Value value_json(rapidjson::kObjectType); + + if (contribution.has_value()) { + if (contribution.value().has_int_value()) { + value_json.AddMember(rapidjson::StringRef(kSignalValueIntValue, + strlen(kSignalValueIntValue)), + rapidjson::Value(contribution.value().int_value()), + allocator); + } else if (contribution.value().has_extended_value()) { + rapidjson::Value signal_value = CreateSignalValue( + BaseValue_Name(contribution.value().extended_value().base_value()), + contribution.value().extended_value().offset(), + contribution.value().extended_value().scale(), allocator); + value_json.AddMember( + rapidjson::StringRef(kSignalValueExtendedValue, + strlen(kSignalValueExtendedValue)), + signal_value, allocator); + } + } + contribution_json.AddMember( + rapidjson::StringRef(kPrivateAggregationValue.data(), + kPrivateAggregationValue.length()), + value_json, allocator); + + // Add bucket information if available. + if (contribution.has_bucket()) { + rapidjson::Value bucket_json(rapidjson::kObjectType); + if (contribution.bucket().has_bucket_128_bit()) { + rapidjson::Value bucket_128_bit_json(rapidjson::kArrayType); + for (const auto& bucket_value : + contribution.bucket().bucket_128_bit().bucket_128_bits()) { + bucket_128_bit_json.PushBack(bucket_value, allocator); + } + bucket_json.AddMember( + rapidjson::StringRef(kSignalBucketBucket128Bit, + strlen(kSignalBucketBucket128Bit)), + bucket_128_bit_json, allocator); + } + contribution_json.AddMember( + rapidjson::StringRef(kPrivateAggregationBucket.data(), + kPrivateAggregationBucket.length()), + bucket_json, allocator); + } + return contribution_json; +} + +} // namespace + +void ValidatePAGGContributions(EventType event_type, + const std::string& pagg_response_json_string, + BaseValues& base_values, + int expected_contribution_count, + absl::StatusCode expected_status_code) { + absl::StatusOr document = + ParseJsonString(pagg_response_json_string.c_str()); + ASSERT_TRUE(document.ok()); + PrivateAggregateReportingResponse pagg_response; + absl::Status status = AppendAdEventContributionsToPaggResponse( + event_type, document.value(), base_values, pagg_response); + ASSERT_EQ(status.code(), expected_status_code); + // Exit early if the status code is not kOk. + // Only check value if status is ok. + if (status.ok()) { + EXPECT_EQ(pagg_response.contributions_size(), expected_contribution_count); + } +} + +void TestParseAndProcessContribution(const BaseValues& base_values, + const std::string& json_string, + int expected_int_value, + uint64_t expected_lower_bits, + uint64_t expected_upper_bits, + absl::StatusCode expected_status_code) { + absl::StatusOr document = + ParseJsonString(json_string.c_str()); + ASSERT_TRUE(document.ok()); + absl::StatusOr contribution = + ParseAndProcessContribution(base_values, document.value()); + ASSERT_EQ(contribution.status().code(), expected_status_code); + // Exit early if the status code is not Ok. + if (expected_status_code != absl::StatusCode::kOk) { + return; + } + EXPECT_EQ(contribution.value().value().int_value(), expected_int_value); + EXPECT_FALSE(contribution.value().value().has_extended_value()); + + EXPECT_EQ(contribution.value().bucket().bucket_128_bit().bucket_128_bits(0), + expected_lower_bits); + EXPECT_EQ(contribution.value().bucket().bucket_128_bit().bucket_128_bits(1), + expected_upper_bits); +} + +void TestParseSignalValue(const std::string& json_string, + BaseValue expected_base_value, int expected_offset, + double expected_scale, + absl::StatusCode expected_status_code) { + absl::StatusOr document = + ParseJsonString(json_string.c_str()); + ASSERT_TRUE(document.ok()); + absl::StatusOr signal_value = ParseSignalValue(document.value()); + ASSERT_EQ(signal_value.status().code(), expected_status_code); + // Exit early if the status code is not Ok. + if (expected_status_code != absl::StatusCode::kOk) { + return; + } + EXPECT_EQ(signal_value.value().base_value(), expected_base_value); + EXPECT_EQ(signal_value.value().offset(), expected_offset); + EXPECT_EQ(signal_value.value().scale(), expected_scale); +} + +void TestParseSignalBucket(const std::string& json_string, + BaseValue expected_base_value, + int64_t expected_offset_0, int64_t expected_offset_1, + bool expected_is_negative, double expected_scale, + absl::StatusCode expected_status_code) { + absl::StatusOr document = + ParseJsonString(json_string.c_str()); + ASSERT_TRUE(document.ok()); + absl::StatusOr signal_bucket = + ParseSignalBucket(document.value()); + ASSERT_EQ(signal_bucket.status().code(), expected_status_code); + + // Exit early if the status code is not Ok. + if (expected_status_code != absl::StatusCode::kOk) { + return; + } + + EXPECT_EQ(signal_bucket.value().base_value(), expected_base_value); + EXPECT_EQ(signal_bucket.value().offset().value()[0], expected_offset_0); + EXPECT_EQ(signal_bucket.value().offset().value()[1], expected_offset_1); + EXPECT_EQ(signal_bucket.value().offset().is_negative(), expected_is_negative); + EXPECT_EQ(signal_bucket.value().scale(), expected_scale); +} + +void TestParseBucketOffset(absl::string_view json_string, + int64_t expected_value_0, int64_t expected_value_1, + bool expected_is_negative, + absl::StatusCode expected_status_code) { + absl::StatusOr document = ParseJsonString(json_string); + absl::StatusOr bucket_offset = + ParseBucketOffset(document.value()); + EXPECT_EQ(bucket_offset.status().code(), expected_status_code); + + // Exit early if the status code is not Ok. + if (expected_status_code != absl::StatusCode::kOk) { + return; + } + + EXPECT_EQ(bucket_offset.value().value().size(), 2); + EXPECT_EQ(bucket_offset.value().value()[0], expected_value_0); + EXPECT_EQ(bucket_offset.value().value()[1], expected_value_1); + EXPECT_EQ(bucket_offset.value().is_negative(), expected_is_negative); +} + +rapidjson::Value CreateSignalValue( + absl::string_view base_value, int offset, double scale, + rapidjson::Document::AllocatorType& allocator) { + rapidjson::Value signal_value(rapidjson::kObjectType); + + signal_value.AddMember( + rapidjson::StringRef(kSignalValueBaseValue.data(), + kSignalValueBaseValue.length()), + rapidjson::Value(kBaseValueWinningBid.data(), allocator), allocator); + signal_value.AddMember(rapidjson::StringRef(kSignalValueOffset.data(), + kSignalValueOffset.length()), + rapidjson::Value(offset), allocator); + signal_value.AddMember(rapidjson::StringRef(kSignalValueScale.data(), + kSignalValueScale.length()), + rapidjson::Value(scale), allocator); + return signal_value; +} + +rapidjson::Document CreateContributionResponseDocument( + const std::vector& win_contributions, + const std::vector& loss_contributions, + const std::vector& always_contributions, + const absl::flat_hash_map>& + custom_event_contributions) { + rapidjson::Document doc(rapidjson::kObjectType); + auto& allocator = doc.GetAllocator(); + auto addContributionsToJson = + [&doc, &allocator]( + const absl::string_view event_type, + const std::vector& contributions) { + rapidjson::Value contribution_array(rapidjson::kArrayType); + for (const auto& contribution : contributions) { + contribution_array.PushBack( + ContributionToJson(contribution, allocator), allocator); + } + doc.AddMember( + rapidjson::StringRef(event_type.data(), event_type.length()), + contribution_array, allocator); + }; + + addContributionsToJson(kEventTypeWin, win_contributions); + addContributionsToJson(kEventTypeLoss, loss_contributions); + addContributionsToJson(kEventTypeAlways, always_contributions); + + if (!custom_event_contributions.empty()) { + rapidjson::Value custom_events(rapidjson::kObjectType); + for (const auto& [event_name, contributions] : custom_event_contributions) { + rapidjson::Value contribution_array(rapidjson::kArrayType); + for (const auto& contribution : contributions) { + contribution_array.PushBack(ContributionToJson(contribution, allocator), + allocator); + } + custom_events.AddMember( + rapidjson::StringRef(event_name.c_str(), event_name.length()), + contribution_array, allocator); + } + doc.AddMember(rapidjson::StringRef(kEventTypeCustom.data(), + kEventTypeCustom.length()), + custom_events, allocator); + } + return doc; +} + +BaseValues MakeDefaultBaseValues() { + BaseValues base_values; + base_values.winning_bid = kTestBidInSellerCurrency; + base_values.highest_scoring_other_bid = kTestHighestScoringOtherBid; + base_values.reject_reason = SellerRejectionReason::INVALID_BID; + return base_values; +} + +BaseValues MakeDefaultBaseValuesWithNoRejectionReason() { + BaseValues base_values; + base_values.winning_bid = kTestBidInSellerCurrency; + base_values.highest_scoring_other_bid = kTestHighestScoringOtherBid; + return base_values; +} + +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/private_aggregation/private_aggregation_manager_test_util.h b/services/auction_service/private_aggregation/private_aggregation_manager_test_util.h new file mode 100644 index 00000000..19cb4115 --- /dev/null +++ b/services/auction_service/private_aggregation/private_aggregation_manager_test_util.h @@ -0,0 +1,136 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SERVICES_AUCTION_SERVICE_PRIVATE_AGGREGATION_PRIVATE_AGGREGATION_MANAGER_TEST_UTIL_H_ +#define SERVICES_AUCTION_SERVICE_PRIVATE_AGGREGATION_PRIVATE_AGGREGATION_MANAGER_TEST_UTIL_H_ + +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "api/bidding_auction_servers.pb.h" +#include "services/auction_service/private_aggregation/private_aggregation_manager.h" + +namespace privacy_sandbox::bidding_auction_servers { + +using RejectionReasonMap = absl::flat_hash_map< + std::string, absl::flat_hash_map>; + +using AdWithBidMetadataMap = absl::flat_hash_map< + std::string, + std::unique_ptr>; + +inline constexpr absl::string_view kTestIgOwner1 = "ig_owner_1"; +inline constexpr absl::string_view kTestIgName1 = "ig_name_1"; + +// Winning and Losing Dispatch Request ID for testing. +inline constexpr absl::string_view kTestWinningId = "1234"; +inline constexpr absl::string_view kTestLossId = "5678"; + +inline constexpr float kTestBidInSellerCurrency = 10.0f; +inline constexpr float kTestBidInBuyerCurrency = 7.5f; +inline constexpr float kTestHighestScoringOtherBid = 5.0f; + +// Helper function to test AppendAdEventContributionsToPaggResponse with +// different input values. +void ValidatePAGGContributions(EventType event_type, + const std::string& json_string, + BaseValues& base_values, + int expected_contribution_count, + absl::StatusCode expected_status_code); + +// Helper function to parse JSON and test ParseAndProcessContribution +void TestParseAndProcessContribution(const BaseValues& base_values, + const std::string& json_string, + int expected_int_value, + uint64_t expected_lower_bits, + uint64_t expected_upper_bits, + absl::StatusCode expected_status_code); + +// Helper function to parse JSON and test ParseSignalValue +void TestParseSignalValue(const std::string& json_string, + BaseValue expected_base_value, int expected_offset, + double expected_scale, + absl::StatusCode expected_status_code); + +// Helper function to parse JSON and test ParseSignalBucket +void TestParseSignalBucket(const std::string& json_string, + BaseValue expected_base_value, + int64_t expected_offset_0, int64_t expected_offset_1, + bool expected_is_negative, double expected_scale, + absl::StatusCode expected_status_code); + +// Helper function to parse JSON and test ParseBucketOffset +void TestParseBucketOffset(absl::string_view json_string, + int64_t expected_value_0, int64_t expected_value_1, + bool expected_is_negative, + absl::StatusCode expected_status_code); + +// Helper function to create rapidjson Value for SignalValue +rapidjson::Value CreateSignalValue( + absl::string_view base_value, int offset, double scale, + rapidjson::Document::AllocatorType& allocator); + +// Creates a rapidjson::Document representing a Private Aggregation API +// contribution response. +// +// `win_contributions`: A vector of PrivateAggregateContribution objects for the +// "win" event type. +// `loss_contributions`: A vector of PrivateAggregateContribution objects for +// the "loss" event type. `always_contributions`: A vector of +// PrivateAggregateContribution objects for the "always" event type. +// `custom_event_contributions`: A map of custom event names to their +// corresponding vectors of PrivateAggregateContribution objects. +// +// Returns a rapidjson::Document representing the contribution response. +// The document will have the following structure: +// ```javascript +// { +// "win": [ /* ... array of win contributions */ ], +// "loss": [ /* ... array of loss contributions */ ], +// "always": [ /* ... array of always contributions */ ], +// "custom_events": { +// "custom_event_1": [ /* ... array of contributions */ ], +// "custom_event_2": [ /* ... array of contributions */ ] +// // ... more custom events +// } +// } +// ``` +rapidjson::Document CreateContributionResponseDocument( + const std::vector& win_contributions, + const std::vector& loss_contributions, + const std::vector& always_contributions, + const absl::flat_hash_map>& + custom_event_contributions); + +// Creates a BaseValues struct with default values for testing. +// The default values are: +// winning_bid: 10.0f +// highest_scoring_other_bid: 5.0f +// reject_reason: SellerRejectionReason::INVALID_BID +BaseValues MakeDefaultBaseValues(); + +// Creates a BaseValues struct with default values for testing, but without +// setting the `reject_reason` field. +// The default values are: +// winning_bid: 10.0f +// highest_scoring_other_bid: 5.0f +BaseValues MakeDefaultBaseValuesWithNoRejectionReason(); + +} // namespace privacy_sandbox::bidding_auction_servers + +#endif // SERVICES_AUCTION_SERVICE_PRIVATE_AGGREGATION_PRIVATE_AGGREGATION_MANAGER_TEST_UTIL_H_ diff --git a/services/auction_service/reporting/buyer/BUILD b/services/auction_service/reporting/buyer/BUILD index 037bf017..1d80e176 100644 --- a/services/auction_service/reporting/buyer/BUILD +++ b/services/auction_service/reporting/buyer/BUILD @@ -16,13 +16,19 @@ load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") cc_library( name = "pas_buyer_reporting_manager", + srcs = [ + "pas_buyer_reporting_manager.cc", + ], hdrs = [ "pas_buyer_reporting_manager.h", ], visibility = ["//visibility:public"], deps = [ + ":buyer_reporting_helper", + "//services/auction_service/code_wrapper:buyer_reporting_udf_wrapper", "//services/auction_service/reporting:reporting_helper", "//services/auction_service/reporting:reporting_response", + "//services/auction_service/udf_fetcher:adtech_code_version_util", "//services/common/clients/code_dispatcher:code_dispatch_client", "@com_google_absl//absl/status:statusor", "@rapidjson", @@ -68,6 +74,32 @@ cc_library( ], ) +cc_test( + name = "pas_buyer_reporting_manager_test", + size = "small", + srcs = [ + "pas_buyer_reporting_manager_test.cc", + ], + visibility = ["//visibility:public"], + deps = [ + ":pas_buyer_reporting_manager", + "//services/auction_service/reporting:reporting_helper", + "//services/auction_service/reporting:reporting_helper_test", + "//services/auction_service/reporting:reporting_response", + "//services/auction_service/reporting:reporting_test_util", + "//services/auction_service/udf_fetcher:adtech_code_version_util", + "//services/common/clients/code_dispatcher:code_dispatch_client", + "//services/common/code_dispatch:code_dispatch_reactor", + "//services/common/test:mocks", + "//services/common/test/utils:test_init", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + "@rapidjson", + ], +) + cc_test( name = "pa_buyer_reporting_manager_test", size = "small", diff --git a/services/auction_service/reporting/buyer/pas_buyer_reporting_manager.cc b/services/auction_service/reporting/buyer/pas_buyer_reporting_manager.cc new file mode 100644 index 00000000..a963899d --- /dev/null +++ b/services/auction_service/reporting/buyer/pas_buyer_reporting_manager.cc @@ -0,0 +1,131 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License + +#include "services/auction_service/reporting/buyer/pas_buyer_reporting_manager.h" + +#include + +#include "absl/status/statusor.h" +#include "rapidjson/document.h" +#include "services/auction_service/code_wrapper/buyer_reporting_udf_wrapper.h" +#include "services/auction_service/reporting/buyer/buyer_reporting_helper.h" +#include "services/auction_service/reporting/reporting_helper.h" +#include "services/auction_service/reporting/reporting_response.h" +#include "services/auction_service/udf_fetcher/adtech_code_version_util.h" +#include "services/common/util/json_util.h" +#include "services/common/util/request_response_constants.h" + +namespace privacy_sandbox::bidding_auction_servers { +namespace { + +inline std::vector> GetPASReportWinInput( + std::shared_ptr buyer_device_signals, + const ReportingDispatchRequestConfig& dispatch_request_config, + const BuyerReportingDispatchRequestData& dispatch_request_data) { + std::vector> input(kPASReportWinArgSize); + input[PASReportWinArgIndex(PASReportWinArgs::kAuctionConfig)] = + dispatch_request_data.auction_config; + input[PASReportWinArgIndex(PASReportWinArgs::kPerBuyerSignals)] = + std::make_shared(dispatch_request_data.buyer_signals); + input[PASReportWinArgIndex(PASReportWinArgs::kSignalsForWinner)] = + std::make_shared(dispatch_request_data.signals_for_winner); + input[PASReportWinArgIndex(PASReportWinArgs::kBuyerReportingSignals)] = + std::move(buyer_device_signals); + // This is only added to prevent errors in the ReportWin ad script, and + // will always be an empty object. + input[PASReportWinArgIndex(PASReportWinArgs::kDirectFromSellerSignals)] = + std::make_shared("{}"); + input[PASReportWinArgIndex(PASReportWinArgs::kEnableLogging)] = + std::make_shared( + dispatch_request_config.enable_adtech_code_logging ? "true" + : "false"); + std::string egress_payload(*dispatch_request_data.egress_payload); + input[PASReportWinArgIndex(PASReportWinArgs::kEgressPayload)] = + std::make_shared(std::move(egress_payload)); + std::string temporary_unlimited_egress_payload( + *dispatch_request_data.temporary_unlimited_egress_payload); + input[PASReportWinArgIndex( + PASReportWinArgs::kTemporaryUnlimitedEgressPayload)] = + std::make_shared( + std::move(temporary_unlimited_egress_payload)); + PS_VLOG(kDispatch, dispatch_request_data.log_context) + << "\n\nReportWin Input Args:" << "\nAuction Config:\n" + << *(input[PASReportWinArgIndex(PASReportWinArgs::kAuctionConfig)]) + << "\nBuyer Reporting Signals:\n" + << *(input[PASReportWinArgIndex( + PASReportWinArgs::kBuyerReportingSignals)]) + << "\nSignals for winner:\n" + << *(input[PASReportWinArgIndex(PASReportWinArgs::kSignalsForWinner)]) + << "\nBuyer reporting signals:\n" + << *(input[PASReportWinArgIndex( + PASReportWinArgs::kBuyerReportingSignals)]) + << "\nEnable logging:\n" + << *(input[PASReportWinArgIndex(PASReportWinArgs::kEnableLogging)]) + << "\nEgress payload:\n" + << *(input[PASReportWinArgIndex(PASReportWinArgs::kEgressPayload)]) + << "\nTemporary unlimited egress payload:\n" + << *(input[PASReportWinArgIndex( + PASReportWinArgs::kTemporaryUnlimitedEgressPayload)]); + + return input; +} + +inline absl::StatusOr GetPASReportWinDispatchRequest( + const ReportingDispatchRequestConfig& dispatch_request_config, + const BuyerReportingDispatchRequestData& request_data, + std::shared_ptr buyer_device_signals) { + absl::StatusOr version = GetBuyerReportWinVersion( + request_data.buyer_origin, AuctionType::kProtectedAppSignals); + if (!version.ok()) { + // Todo(b/352227374) Add a metric when udf version generation fails. + return absl::Status(absl::StatusCode::kInternal, + absl::StrCat("No udf version found for winning buyer: ", + request_data.buyer_origin)); + } + // Construct the wrapper struct for our V8 Dispatch Request. + DispatchRequest dispatch_request = { + .id = request_data.winning_ad_render_url, + .version_string = *version, + .handler_name = kReportWinEntryFunction, + .input = GetPASReportWinInput(std::move(buyer_device_signals), + dispatch_request_config, request_data)}; + return dispatch_request; +} +} // namespace + +absl::Status PerformPASReportWin( + const ReportingDispatchRequestConfig& dispatch_request_config, + const BuyerReportingDispatchRequestData& request_data, + rapidjson::Document& seller_device_signals, + absl::AnyInvocable< + void(const std::vector>&)> + report_win_callback, + CodeDispatchClient& dispatcher) { + std::shared_ptr buyer_device_signals; + PS_ASSIGN_OR_RETURN( + buyer_device_signals, + GenerateBuyerDeviceSignals(request_data, seller_device_signals)); + DispatchRequest dispatch_request; + PS_ASSIGN_OR_RETURN( + dispatch_request, + GetPASReportWinDispatchRequest(dispatch_request_config, request_data, + buyer_device_signals)); + dispatch_request.tags[kRomaTimeoutMs] = + dispatch_request_config.roma_timeout_ms; + std::vector dispatch_requests = { + std::move(dispatch_request)}; + return dispatcher.BatchExecute(dispatch_requests, + std::move(report_win_callback)); +} +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/reporting/buyer/pas_buyer_reporting_manager.h b/services/auction_service/reporting/buyer/pas_buyer_reporting_manager.h index ccdcbbc8..ebcd4009 100644 --- a/services/auction_service/reporting/buyer/pas_buyer_reporting_manager.h +++ b/services/auction_service/reporting/buyer/pas_buyer_reporting_manager.h @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef SERVICES_AUCTION_SERVICE_REPORTING_BUYER_REPORTING_MANAGER_H_ -#define SERVICES_AUCTION_SERVICE_REPORTING_BUYER_REPORTING_MANAGER_H_ +#ifndef SERVICES_AUCTION_SERVICE_REPORTING_PAS_BUYER_REPORTING_MANAGER_H_ +#define SERVICES_AUCTION_SERVICE_REPORTING_PAS_BUYER_REPORTING_MANAGER_H_ #include #include @@ -26,14 +26,28 @@ #include "services/common/clients/code_dispatcher/code_dispatch_client.h" namespace privacy_sandbox::bidding_auction_servers { +inline constexpr int kPASReportWinArgSize = 8; +enum class PASReportWinArgs : int { + kAuctionConfig, + kPerBuyerSignals, + kSignalsForWinner, + kBuyerReportingSignals, + kDirectFromSellerSignals, + kEnableLogging, + kEgressPayload, + kTemporaryUnlimitedEgressPayload +}; + +inline constexpr int PASReportWinArgIndex(PASReportWinArgs arg) { + return static_cast>(arg); +} // Generates the DispatchRequest and invokes reportWin() udf with // report_win_callback for Protected App Signal auctions absl::Status PerformPASReportWin( const ReportingDispatchRequestConfig& dispatch_request_config, - const BuyerReportingMetadata& buyer_reporting_metadata, - const std::string& seller_signals, - const rapidjson::Document& seller_device_signals, + const BuyerReportingDispatchRequestData& request_data, + rapidjson::Document& seller_device_signals, absl::AnyInvocable< void(const std::vector>&)> report_win_callback, @@ -41,4 +55,4 @@ absl::Status PerformPASReportWin( } // namespace privacy_sandbox::bidding_auction_servers -#endif // SERVICES_AUCTION_SERVICE_REPORTING_BUYER_REPORTING_MANAGER_H_ +#endif // SERVICES_AUCTION_SERVICE_REPORTING_PAS_BUYER_REPORTING_MANAGER_H_ diff --git a/services/auction_service/reporting/buyer/pas_buyer_reporting_manager_test.cc b/services/auction_service/reporting/buyer/pas_buyer_reporting_manager_test.cc new file mode 100644 index 00000000..9d0e478f --- /dev/null +++ b/services/auction_service/reporting/buyer/pas_buyer_reporting_manager_test.cc @@ -0,0 +1,240 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache-form License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "services/auction_service/reporting/buyer/pas_buyer_reporting_manager.h" + +#include +#include +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "rapidjson/document.h" +#include "rapidjson/stringbuffer.h" +#include "services/auction_service/auction_constants.h" +#include "services/auction_service/code_wrapper/buyer_reporting_udf_wrapper.h" +#include "services/auction_service/reporting/reporting_helper.h" +#include "services/auction_service/reporting/reporting_helper_test_constants.h" +#include "services/auction_service/reporting/reporting_response.h" +#include "services/auction_service/reporting/reporting_test_util.h" +#include "services/auction_service/udf_fetcher/adtech_code_version_util.h" +#include "services/common/clients/code_dispatcher/v8_dispatcher.h" +#include "services/common/test/mocks.h" +#include "services/common/test/utils/test_init.h" +#include "services/common/util/json_util.h" + +namespace privacy_sandbox::bidding_auction_servers { + +namespace { + +constexpr absl::string_view kInvalidUrl = + "example.com:80:https://path/to/2?query=1&2#fragment"; + +struct ReportWinArgIndices { + int kAuctionConfigArgIdx = + PASReportWinArgIndex(PASReportWinArgs::kAuctionConfig); + int kPerBuyerSignalsArgIdx = + PASReportWinArgIndex(PASReportWinArgs::kPerBuyerSignals); + int kSignalsForWinnerArgIdx = + PASReportWinArgIndex(PASReportWinArgs::kSignalsForWinner); + int kBuyerReportingSignalsArgIdx = + PASReportWinArgIndex(PASReportWinArgs::kBuyerReportingSignals); + int kDirectFromSellerSignalsArgIdx = + PASReportWinArgIndex(PASReportWinArgs::kDirectFromSellerSignals); + int kEnableLoggingArgIdx = + PASReportWinArgIndex(PASReportWinArgs::kEnableLogging); + int kEgressPayloadArgIdx = + PASReportWinArgIndex(PASReportWinArgs::kEgressPayload); + int kTemporaryUnlimitedEgressPayloadArgIdx = + PASReportWinArgIndex(PASReportWinArgs::kTemporaryUnlimitedEgressPayload); +}; + +struct TestData { + std::string expected_auction_config = kTestAuctionConfig; + std::string expected_seller_device_signals; + std::string expected_direct_from_seller_signals = "{}"; + bool enable_adtech_code_logging = false; + absl::string_view id = kTestRender; + std::string expected_handler_name = kReportWinEntryFunction; + ReportWinArgIndices indexes; + std::string expected_buyer_signals = kTestBuyerSignals; + std::string expected_version; + std::string expected_egress_payload = kTestEgressPayload; + std::string expected_temporary_unlimited_egress_payload = + kTestTemporaryUnlimitedEgressPayload; +}; + +void VerifyDispatchRequest( + const TestData& test_data, + const BuyerReportingDispatchRequestData& dispatch_request_data, + const SellerReportingDispatchRequestData& seller_dispatch_request_data, + std::vector& batch) { + EXPECT_EQ(batch.size(), 1); + std::string expected_auction_config = kTestAuctionConfig; + EXPECT_EQ(batch[0].id, test_data.id); + EXPECT_EQ(batch[0].version_string, test_data.expected_version); + EXPECT_EQ(batch[0].handler_name, test_data.expected_handler_name); + std::vector> response_vector = batch[0].input; + EXPECT_EQ(*(response_vector[PASReportWinArgIndex( + PASReportWinArgs::kAuctionConfig)]), + test_data.expected_auction_config); + EXPECT_EQ(*(response_vector[PASReportWinArgIndex( + PASReportWinArgs::kPerBuyerSignals)]), + test_data.expected_buyer_signals); + EXPECT_EQ(*(response_vector[PASReportWinArgIndex( + PASReportWinArgs::kDirectFromSellerSignals)]), + test_data.expected_direct_from_seller_signals); + EXPECT_EQ(*(response_vector[PASReportWinArgIndex( + PASReportWinArgs::kEnableLogging)]), + test_data.enable_adtech_code_logging ? "true" : "false"); + VerifyPASBuyerReportingSignalsJson( + *(response_vector[PASReportWinArgIndex( + PASReportWinArgs::kBuyerReportingSignals)]), + dispatch_request_data, seller_dispatch_request_data); + EXPECT_EQ(*(response_vector[PASReportWinArgIndex( + PASReportWinArgs::kPerBuyerSignals)]), + test_data.expected_buyer_signals); + EXPECT_EQ(*(response_vector[PASReportWinArgIndex( + PASReportWinArgs::kEgressPayload)]), + test_data.expected_egress_payload); + EXPECT_EQ(*(response_vector[PASReportWinArgIndex( + PASReportWinArgs::kTemporaryUnlimitedEgressPayload)]), + test_data.expected_temporary_unlimited_egress_payload); +} + +class PerformReportWin : public ::testing::Test { + protected: + void SetUp() override { CommonTestInit(); } +}; + +TEST_F(PerformReportWin, DispatchesRequestToReportWin) { + auto report_win_callback = + [](const std::vector>& result) {}; + RequestLogContext log_context({}, + server_common::ConsentedDebugConfiguration()); + ScoreAdsResponse::AdScore winning_ad_score = GetTestWinningScoreAdsResponse(); + PostAuctionSignals post_auction_signals = + GeneratePostAuctionSignals(winning_ad_score, kUsdIsoCode); + SellerReportingDispatchRequestData seller_dispatch_request_data = + GetTestSellerDispatchRequestData(post_auction_signals, log_context); + + rapidjson::Document document = + GenerateTestSellerDeviceSignals(seller_dispatch_request_data); + + BuyerReportingDispatchRequestData dispatch_request_data = + GetTestBuyerDispatchRequestData(log_context); + + ReportingDispatchRequestConfig config; + + absl::StatusOr expected_version = GetBuyerReportWinVersion( + dispatch_request_data.buyer_origin, AuctionType::kProtectedAppSignals); + ASSERT_TRUE(expected_version.ok()); + TestData test_data = { + .expected_version = *expected_version, + }; + + absl::Notification notification; + MockCodeDispatchClient mock_dispatch_client; + EXPECT_CALL(mock_dispatch_client, BatchExecute) + .WillOnce([¬ification, &test_data, &dispatch_request_data, + &seller_dispatch_request_data]( + std::vector& batch, + BatchDispatchDoneCallback done_callback) { + VerifyDispatchRequest(test_data, dispatch_request_data, + seller_dispatch_request_data, batch); + notification.Notify(); + return absl::OkStatus(); + }); + absl::Status status = + PerformPASReportWin(config, dispatch_request_data, document, + std::move(report_win_callback), mock_dispatch_client); + + notification.WaitForNotification(); + ASSERT_TRUE(status.ok()); +} + +TEST_F(PerformReportWin, DispatchRequestFailsAndStatusNotOkReturned) { + auto report_win_callback = + [](const std::vector>& result) {}; + RequestLogContext log_context({}, + server_common::ConsentedDebugConfiguration()); + ScoreAdsResponse::AdScore winning_ad_score = GetTestWinningScoreAdsResponse(); + PostAuctionSignals post_auction_signals = + GeneratePostAuctionSignals(winning_ad_score, kUsdIsoCode); + SellerReportingDispatchRequestData seller_dispatch_request_data = + GetTestSellerDispatchRequestData(post_auction_signals, log_context); + + rapidjson::Document document = + GenerateTestSellerDeviceSignals(seller_dispatch_request_data); + + BuyerReportingDispatchRequestData dispatch_request_data = + GetTestBuyerDispatchRequestData(log_context); + + ReportingDispatchRequestConfig config; + + absl::StatusOr expected_version = GetBuyerReportWinVersion( + dispatch_request_data.buyer_origin, AuctionType::kProtectedAppSignals); + ASSERT_TRUE(expected_version.ok()); + TestData test_data = { + .expected_version = *expected_version, + }; + + absl::Notification notification; + MockCodeDispatchClient mock_dispatch_client; + EXPECT_CALL(mock_dispatch_client, BatchExecute) + .WillOnce([¬ification, &test_data, &dispatch_request_data, + &seller_dispatch_request_data]( + std::vector& batch, + BatchDispatchDoneCallback done_callback) { + VerifyDispatchRequest(test_data, dispatch_request_data, + seller_dispatch_request_data, batch); + notification.Notify(); + return absl::InternalError("Something went wrong"); + }); + absl::Status status = + PerformPASReportWin(config, dispatch_request_data, document, + std::move(report_win_callback), mock_dispatch_client); + + notification.WaitForNotification(); + ASSERT_FALSE(status.ok()); +} + +TEST_F(PerformReportWin, ReportWinNotExecutedWhenVersionLookupFails) { + auto report_win_callback = + [](const std::vector>& result) {}; + RequestLogContext log_context({}, + server_common::ConsentedDebugConfiguration()); + ScoreAdsResponse::AdScore winning_ad_score = GetTestWinningScoreAdsResponse(); + PostAuctionSignals post_auction_signals = + GeneratePostAuctionSignals(winning_ad_score, kUsdIsoCode); + SellerReportingDispatchRequestData seller_dispatch_request_data = + GetTestSellerDispatchRequestData(post_auction_signals, log_context); + + rapidjson::Document document = + GenerateTestSellerDeviceSignals(seller_dispatch_request_data); + + BuyerReportingDispatchRequestData dispatch_request_data = + GetTestBuyerDispatchRequestData(log_context); + + ReportingDispatchRequestConfig config; + dispatch_request_data.buyer_origin = kInvalidUrl; + MockCodeDispatchClient mock_dispatch_client; + EXPECT_CALL(mock_dispatch_client, BatchExecute).Times(0); + absl::Status status = + PerformPASReportWin(config, dispatch_request_data, document, + std::move(report_win_callback), mock_dispatch_client); + ASSERT_FALSE(status.ok()); +} + +} // namespace +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/reporting/reporting_helper.cc b/services/auction_service/reporting/reporting_helper.cc index baebad6f..7bc65248 100644 --- a/services/auction_service/reporting/reporting_helper.cc +++ b/services/auction_service/reporting/reporting_helper.cc @@ -112,13 +112,12 @@ absl::StatusOr ParseAndGetReportingResponse( void HandleUdfLogs(const rapidjson::Document& document, const std::string& log_type, absl::string_view udf_name, RequestLogContext& log_context) { + if (!AllowAnyUdfLogging(log_context)) { + return; + } auto LogUdf = [&log_context, prefix(absl::StrCat(log_type, " from udf:", udf_name))]( absl::string_view adtech_log) { - if (!server_common::log::PS_VLOG_IS_ON(kUdfLog) && - !log_context.is_debug_response()) { - return; - } PS_VLOG(kUdfLog, log_context) << prefix << adtech_log; log_context.SetEventMessageField(absl::StrCat(prefix, adtech_log)); }; diff --git a/services/auction_service/reporting/reporting_helper.h b/services/auction_service/reporting/reporting_helper.h index 1d5a53b4..368cf4f5 100644 --- a/services/auction_service/reporting/reporting_helper.h +++ b/services/auction_service/reporting/reporting_helper.h @@ -186,6 +186,8 @@ struct BuyerReportingDispatchRequestData { std::string buyer_origin; std::string signals_for_winner = "null"; std::string winning_ad_render_url; + std::optional egress_payload; + std::optional temporary_unlimited_egress_payload; }; inline const std::string kDefaultBuyerReportingMetadata = absl::StrFormat( diff --git a/services/auction_service/reporting/reporting_helper_test_constants.h b/services/auction_service/reporting/reporting_helper_test_constants.h index a47a3781..f44b1772 100644 --- a/services/auction_service/reporting/reporting_helper_test_constants.h +++ b/services/auction_service/reporting/reporting_helper_test_constants.h @@ -57,6 +57,9 @@ constexpr float kTestModifiedBid = 1.0; constexpr char kUsdIsoCode[] = "USD"; constexpr char kEurosIsoCode[] = "EUR"; constexpr bool kTestEnableReportWinInputNoisingTrue = true; +constexpr char kTestEgressPayload[] = "testEgressPayload"; +constexpr char kTestTemporaryUnlimitedEgressPayload[] = + "testTemporaryUnlimitedEgressPayload"; } // namespace privacy_sandbox::bidding_auction_servers #endif // SERVICES_AUCTION_SERVICE_REPORTING_REPORTING_HELPER_TEST_CONSTANTS_H_ diff --git a/services/auction_service/reporting/reporting_test_util.cc b/services/auction_service/reporting/reporting_test_util.cc index 41ac942f..39b75b58 100644 --- a/services/auction_service/reporting/reporting_test_util.cc +++ b/services/auction_service/reporting/reporting_test_util.cc @@ -199,6 +199,16 @@ void VerifyBuyerReportingSignals( } EXPECT_EQ(observed_buyer_device_signals.made_highest_scoring_other_bid, expected_buyer_device_signals.made_highest_scoring_other_bid); + if (observed_buyer_device_signals.egress_payload.has_value()) { + EXPECT_EQ(*observed_buyer_device_signals.egress_payload, + *expected_buyer_device_signals.egress_payload); + } + if (observed_buyer_device_signals.temporary_unlimited_egress_payload + .has_value()) { + EXPECT_EQ( + *observed_buyer_device_signals.temporary_unlimited_egress_payload, + *expected_buyer_device_signals.temporary_unlimited_egress_payload); + } } void ParseBuyerReportingSignals( @@ -292,6 +302,27 @@ void VerifyPABuyerReportingSignalsJson( expected_buyer_dispatch_request_data); } +void VerifyPASBuyerReportingSignalsJson( + const std::string& buyer_reporting_signals_json, + const BuyerReportingDispatchRequestData& + expected_buyer_dispatch_request_data, + const SellerReportingDispatchRequestData& seller_dispatch_request_data) { + RequestLogContext log_context(/*context_map=*/{}, + server_common::ConsentedDebugConfiguration()); + BuyerReportingDispatchRequestData observed_buyer_device_signals{ + .log_context = log_context}; + absl::StatusOr document = + ParseJsonString(buyer_reporting_signals_json); + ASSERT_TRUE(document.ok()); + TestSellerDeviceSignals observed_seller_device_signals = + ParseSellerDeviceSignals(*document); + VerifySellerDeviceSignals(observed_seller_device_signals, + seller_dispatch_request_data); + ParseBuyerReportingSignals(observed_buyer_device_signals, *document); + VerifyBuyerReportingSignals(observed_buyer_device_signals, + expected_buyer_dispatch_request_data); +} + BuyerReportingDispatchRequestData GetTestBuyerDispatchRequestData( RequestLogContext& log_context) { std::shared_ptr auction_config = @@ -309,7 +340,10 @@ BuyerReportingDispatchRequestData GetTestBuyerDispatchRequestData( .log_context = log_context, .buyer_origin = kTestInterestGroupOwner, .signals_for_winner = kTestSignalsForWinner, - .winning_ad_render_url = kTestRender}; + .winning_ad_render_url = kTestRender, + .egress_payload = kTestEgressPayload, + .temporary_unlimited_egress_payload = + kTestTemporaryUnlimitedEgressPayload}; } } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/auction_service/reporting/reporting_test_util.h b/services/auction_service/reporting/reporting_test_util.h index 5f3cf3c7..425b072a 100644 --- a/services/auction_service/reporting/reporting_test_util.h +++ b/services/auction_service/reporting/reporting_test_util.h @@ -83,6 +83,11 @@ void VerifyPABuyerReportingSignalsJson( const BuyerReportingDispatchRequestData& expected_buyer_dispatch_request_data, const SellerReportingDispatchRequestData& seller_dispatch_request_data); +void VerifyPASBuyerReportingSignalsJson( + const std::string& buyer_reporting_signals_json, + const BuyerReportingDispatchRequestData& + expected_buyer_dispatch_request_data, + const SellerReportingDispatchRequestData& seller_dispatch_request_data); } // namespace privacy_sandbox::bidding_auction_servers #endif // SERVICES_AUCTION_SERVICE_REPORTING_TEST_UTIL_H_ diff --git a/services/auction_service/reporting/seller/seller_reporting_manager.cc b/services/auction_service/reporting/seller/seller_reporting_manager.cc index 32b960f1..4e753d09 100644 --- a/services/auction_service/reporting/seller/seller_reporting_manager.cc +++ b/services/auction_service/reporting/seller/seller_reporting_manager.cc @@ -103,8 +103,13 @@ rapidjson::Document GenerateSellerDeviceSignals( document.GetAllocator()); document.AddMember(kRenderUrlTag, render_url_value.Move(), document.GetAllocator()); - double bid = GetEightBitRoundedValue( - dispatch_request_data.post_auction_signals.winning_bid); + double bid = + dispatch_request_data.post_auction_signals + .winning_bid_in_seller_currency > 0 + ? GetEightBitRoundedValue(dispatch_request_data.post_auction_signals + .winning_bid_in_seller_currency) + : GetEightBitRoundedValue( + dispatch_request_data.post_auction_signals.winning_bid); if (bid > -1) { document.AddMember(kBidTag, bid, document.GetAllocator()); } diff --git a/services/auction_service/score_ads_reactor.cc b/services/auction_service/score_ads_reactor.cc index 4ac268ba..137a7d59 100644 --- a/services/auction_service/score_ads_reactor.cc +++ b/services/auction_service/score_ads_reactor.cc @@ -31,6 +31,7 @@ #include "services/auction_service/code_wrapper/seller_code_wrapper.h" #include "services/auction_service/reporting/buyer/buyer_reporting_helper.h" #include "services/auction_service/reporting/buyer/pa_buyer_reporting_manager.h" +#include "services/auction_service/reporting/buyer/pas_buyer_reporting_manager.h" #include "services/auction_service/reporting/reporting_response.h" #include "services/auction_service/reporting/seller/component_seller_reporting_manager.h" #include "services/auction_service/reporting/seller/seller_reporting_manager.h" @@ -494,11 +495,11 @@ void ScoreAdsReactor::Execute() { return; } - PS_VLOG(kEncrypted, log_context_) << "Encrypted ScoreAdsRequest:\n" - << request_->ShortDebugString(); + PS_VLOG(kEncrypted, log_context_) + << "Encrypted ScoreAdsRequest exported in EventMessage"; log_context_.SetEventMessageField(*request_); - PS_VLOG(kPlain, log_context_) << "ScoreAdsRawRequest:\n" - << raw_request_.ShortDebugString(); + PS_VLOG(kPlain, log_context_) + << "ScoreAdsRawRequest exported in EventMessage"; log_context_.SetEventMessageField(raw_request_); if (!raw_request_.protected_app_signals_ad_bids().empty() && !enable_protected_app_signals_) { @@ -602,14 +603,31 @@ void ScoreAdsReactor::Execute() { } } +void ScoreAdsReactor::InitializeBuyerReportingDispatchRequestData( + const ScoreAdsResponse::AdScore& winning_ad_score) { + buyer_reporting_dispatch_request_data_.buyer_signals = + raw_request_.per_buyer_signals().at( + winning_ad_score.interest_group_owner()); + buyer_reporting_dispatch_request_data_.seller = raw_request_.seller(); + buyer_reporting_dispatch_request_data_.interest_group_name = + winning_ad_score.interest_group_name(); + buyer_reporting_dispatch_request_data_.buyer_origin = + winning_ad_score.interest_group_owner(); + buyer_reporting_dispatch_request_data_.made_highest_scoring_other_bid = + post_auction_signals_.made_highest_scoring_other_bid; +} + void ScoreAdsReactor::PerformReportingWithSellerAndBuyerCodeIsolation( const ScoreAdsResponse::AdScore& winning_ad_score, absl::string_view id) { + reporting_dispatch_request_config_ = { + .enable_report_win_url_generation = enable_report_win_url_generation_, + .enable_protected_app_signals = enable_protected_app_signals_, + .enable_report_win_input_noising = enable_report_win_input_noising_, + .enable_adtech_code_logging = enable_adtech_code_logging_}; + buyer_reporting_dispatch_request_data_.auction_config = + BuildAuctionConfig(raw_request_); + if (auto ad_it = ad_data_.find(id); ad_it != ad_data_.end()) { - reporting_dispatch_request_config_ = { - .enable_report_win_url_generation = enable_report_win_url_generation_, - .enable_protected_app_signals = enable_protected_app_signals_, - .enable_report_win_input_noising = enable_report_win_input_noising_, - .enable_adtech_code_logging = enable_adtech_code_logging_}; if (auction_scope_ == AuctionScope::AUCTION_SCOPE_SERVER_TOP_LEVEL_SELLER) { post_auction_signals_ = GeneratePostAuctionSignalsForTopLevelSeller(winning_ad_score); @@ -618,26 +636,15 @@ void ScoreAdsReactor::PerformReportingWithSellerAndBuyerCodeIsolation( } post_auction_signals_ = GeneratePostAuctionSignals( winning_ad_score, raw_request_.seller_currency()); - auto auction_config = BuildAuctionConfig(raw_request_); + InitializeBuyerReportingDispatchRequestData(winning_ad_score); + const auto& ad = ad_it->second; - buyer_reporting_dispatch_request_data_.auction_config = - std::move(auction_config); - buyer_reporting_dispatch_request_data_.buyer_signals = - raw_request_.per_buyer_signals().at( - winning_ad_score.interest_group_owner()); buyer_reporting_dispatch_request_data_.join_count = ad->join_count(); buyer_reporting_dispatch_request_data_.recency = ad->recency(); buyer_reporting_dispatch_request_data_.modeling_signals = ad->modeling_signals(); - buyer_reporting_dispatch_request_data_.seller = raw_request_.seller(); - buyer_reporting_dispatch_request_data_.interest_group_name = - winning_ad_score.interest_group_name(); buyer_reporting_dispatch_request_data_.ad_cost = ad->ad_cost(); - buyer_reporting_dispatch_request_data_.buyer_origin = - winning_ad_score.interest_group_owner(); buyer_reporting_dispatch_request_data_.winning_ad_render_url = ad->render(); - buyer_reporting_dispatch_request_data_.made_highest_scoring_other_bid = - post_auction_signals_.made_highest_scoring_other_bid; if (!ad->buyer_reporting_id().empty()) { // Set buyer_reporting_id in the response. @@ -660,8 +667,35 @@ void ScoreAdsReactor::PerformReportingWithSellerAndBuyerCodeIsolation( protected_app_signals_ad_data_.find(id); protected_app_signals_ad_it != protected_app_signals_ad_data_.end()) { - PS_LOG(ERROR, log_context_) << "Reporting with seller and buyer udf " - "isolation is unavailable for PAS"; + post_auction_signals_ = GeneratePostAuctionSignals( + winning_ad_score, raw_request_.seller_currency()); + InitializeBuyerReportingDispatchRequestData(winning_ad_score); + + const auto& protected_app_signals_ad = protected_app_signals_ad_it->second; + buyer_reporting_dispatch_request_data_.modeling_signals = + protected_app_signals_ad->modeling_signals(); + buyer_reporting_dispatch_request_data_.ad_cost = + protected_app_signals_ad->ad_cost(); + buyer_reporting_dispatch_request_data_.winning_ad_render_url = + protected_app_signals_ad->render(); + buyer_reporting_dispatch_request_data_.egress_payload = + protected_app_signals_ad->egress_payload(); + buyer_reporting_dispatch_request_data_.temporary_unlimited_egress_payload = + protected_app_signals_ad->temporary_unlimited_egress_payload(); + if (auction_scope_ == + AuctionScope::AUCTION_SCOPE_DEVICE_COMPONENT_MULTI_SELLER || + auction_scope_ == + AuctionScope::AUCTION_SCOPE_SERVER_COMPONENT_MULTI_SELLER) { + PS_LOG(ERROR, log_context_) + << "Multi seller auction is unavailable for PAS"; + } else { + DispatchReportResultRequest(winning_ad_score); + } + } else { + PS_LOG(ERROR, log_context_) + << "Following id didn't map to any ProtectedAudience or " + "ProtectedAppSignals Ad: " + << id; FinishWithStatus( grpc::Status(grpc::StatusCode::INTERNAL, kInternalServerError)); return; @@ -929,7 +963,8 @@ ScoringData ScoreAdsReactor::FindWinningAd( scoring_data, response->id); } else { HandleScoredAd(index, protected_app_signals_ad_with_bid->bid(), - /*ad_with_bid_currency=*/"", /*interest_group_name=*/"", + protected_app_signals_ad_with_bid->bid_currency(), + /*interest_group_name=*/"", protected_app_signals_ad_with_bid->owner(), /*interest_group_origin=*/"", *response_json, AdType::AD_TYPE_PROTECTED_APP_SIGNALS_AD, *ad_score, @@ -1118,7 +1153,7 @@ void ScoreAdsReactor::ReportingCallback( if (!enable_adtech_code_logging_) { return; } - if (!PS_VLOG_IS_ON(kUdfLog) && !log_context_.is_debug_response()) { + if (!AllowAnyUdfLogging(log_context_)) { return; } for (const std::string& log : adtech_logs) { @@ -1305,13 +1340,24 @@ void ScoreAdsReactor::CancellableReportResultCallback( EncryptAndFinishOK(); return; } - absl::Status report_win_status = PerformPAReportWin( - reporting_dispatch_request_config_, - buyer_reporting_dispatch_request_data_, seller_device_signals_, - [this](const std::vector>& result) { - ReportWinCallback(result); - }, - dispatcher_); + absl::Status report_win_status; + if (!buyer_reporting_dispatch_request_data_.egress_payload.has_value()) { + report_win_status = PerformPAReportWin( + reporting_dispatch_request_config_, + buyer_reporting_dispatch_request_data_, seller_device_signals_, + [this](const std::vector>& result) { + ReportWinCallback(result); + }, + dispatcher_); + } else { + report_win_status = PerformPASReportWin( + reporting_dispatch_request_config_, + buyer_reporting_dispatch_request_data_, seller_device_signals_, + [this](const std::vector>& result) { + ReportWinCallback(result); + }, + dispatcher_); + } if (!report_win_status.ok()) { PS_VLOG(kDispatch, log_context_) << "Execution failed for reportWin: " << report_win_status.message(); @@ -1376,10 +1422,11 @@ void ScoreAdsReactor::PerformDebugReporting( } void ScoreAdsReactor::EncryptAndFinishOK() { - PS_VLOG(kPlain, log_context_) << "ScoreAdsRawResponse:\n" - << raw_response_.ShortDebugString(); + PS_VLOG(kPlain, log_context_) + << "ScoreAdsRawResponse exported in EventMessage"; log_context_.SetEventMessageField(raw_response_); - log_context_.ExportEventMessage(); + // ExportEventMessage before encrypt response + log_context_.ExportEventMessage(/*if_export_consented=*/true); EncryptResponse(); PS_VLOG(kEncrypted, log_context_) << "Encrypted ScoreAdsResponse\n" << response_->ShortDebugString(); diff --git a/services/auction_service/score_ads_reactor.h b/services/auction_service/score_ads_reactor.h index 4820c201..9ef25318 100644 --- a/services/auction_service/score_ads_reactor.h +++ b/services/auction_service/score_ads_reactor.h @@ -171,6 +171,12 @@ class ScoreAdsReactor // PerformReportingWithSellerAndBuyerCodeIsolation instead")]] void PerformReporting(const ScoreAdsResponse::AdScore& winning_ad_score, absl::string_view id); + + // Initializes buyer_reporting_dispatch_request_data_ with the + // fields included in winning_ad_score. + void InitializeBuyerReportingDispatchRequestData( + const ScoreAdsResponse::AdScore& winning_ad_score); + // Performs reportResult and reportWin udf execution with seller and buyer // code isolation void PerformReportingWithSellerAndBuyerCodeIsolation( diff --git a/services/auction_service/score_ads_reactor_test.cc b/services/auction_service/score_ads_reactor_test.cc index a3778196..15afd7ad 100644 --- a/services/auction_service/score_ads_reactor_test.cc +++ b/services/auction_service/score_ads_reactor_test.cc @@ -34,6 +34,7 @@ #include "services/auction_service/code_wrapper/buyer_reporting_udf_wrapper.h" #include "services/auction_service/code_wrapper/seller_udf_wrapper.h" #include "services/auction_service/reporting/buyer/pa_buyer_reporting_manager.h" +#include "services/auction_service/reporting/buyer/pas_buyer_reporting_manager.h" #include "services/auction_service/reporting/reporting_helper.h" #include "services/auction_service/reporting/reporting_helper_test_constants.h" #include "services/auction_service/reporting/reporting_test_util.h" @@ -3469,9 +3470,9 @@ TEST_F(ScoreAdsReactorTest, .resp = absl::Substitute(kComponentWithCurrencyAdScoreTemplate, kTestDesirability, // NOLINTNEXTLINE - /*bid=*/1.0, kEuroIsoCode, + /*bid=*/2.0, kEuroIsoCode, // NOLINTNEXTLINE - /*incomingBidInSellerCurrency=*/3, + /*incomingBidInSellerCurrency=*/1, // NOLINTNEXTLINE /*allowComponentAuction=*/"true")}; done_callback({response}); diff --git a/services/auction_service/udf_fetcher/seller_udf_fetch_manager.cc b/services/auction_service/udf_fetcher/seller_udf_fetch_manager.cc index d34adbc6..f1c21060 100644 --- a/services/auction_service/udf_fetcher/seller_udf_fetch_manager.cc +++ b/services/auction_service/udf_fetcher/seller_udf_fetch_manager.cc @@ -91,8 +91,9 @@ absl::Status SellerUdfFetchManager::End() { } WrapSingleCodeBlobForDispatch SellerUdfFetchManager::GetUdfWrapperForBuyer() { - return [](const std::string& ad_tech_code_blob) { - return GetBuyerWrappedCode(ad_tech_code_blob); + return [enable_protected_app_signals = enable_protected_app_signals_]( + const std::string& ad_tech_code_blob) { + return GetBuyerWrappedCode(ad_tech_code_blob, enable_protected_app_signals); }; } diff --git a/services/auction_service/udf_fetcher/seller_udf_fetch_manager_test.cc b/services/auction_service/udf_fetcher/seller_udf_fetch_manager_test.cc index 1d3bec67..cc5505d7 100644 --- a/services/auction_service/udf_fetcher/seller_udf_fetch_manager_test.cc +++ b/services/auction_service/udf_fetcher/seller_udf_fetch_manager_test.cc @@ -14,6 +14,8 @@ #include "services/auction_service/udf_fetcher/seller_udf_fetch_manager.h" +#include + #include "absl/strings/str_cat.h" #include "absl/synchronization/blocking_counter.h" #include "absl/synchronization/notification.h" @@ -49,6 +51,7 @@ using ::google::scp::core::FailureExecutionResult; using ::google::scp::core::SuccessExecutionResult; using ::google::scp::cpio::MockBlobStorageClient; using ::testing::Return; +using ::testing::StrEq; struct TestSellerUdfConfig { std::string pa_buyer_origin = "http://PABuyerOrigin.com"; @@ -315,18 +318,24 @@ reportWin = function(auctionSignals, perBuyerSignals, signalsForWinner, buyerRep EXPECT_CALL(*dispatcher_, LoadSync) .Times(3) - .WillOnce([&expected_buyer_udf, &expected_pa_version]( - std::string_view version, absl::string_view blob_data) { + .WillOnce([&expected_buyer_udf, &expected_pa_version, + &enable_protected_app_signals](std::string_view version, + absl::string_view blob_data) { EXPECT_EQ(version, expected_pa_version); - EXPECT_EQ(blob_data, GetBuyerWrappedCode(expected_buyer_udf)); + EXPECT_THAT(blob_data, + StrEq(GetBuyerWrappedCode(expected_buyer_udf, + enable_protected_app_signals))); return absl::OkStatus(); }) - .WillOnce([&expected_buyer_udf, &expected_pas_version]( - std::string_view version, absl::string_view blob_data) { + .WillOnce([&expected_buyer_udf, &expected_pas_version, + &enable_protected_app_signals](std::string_view version, + absl::string_view blob_data) { EXPECT_EQ(version, expected_pas_version); - EXPECT_EQ(blob_data, GetBuyerWrappedCode(expected_buyer_udf)); + EXPECT_THAT(blob_data, + StrEq(GetBuyerWrappedCode(expected_buyer_udf, + enable_protected_app_signals))); return absl::OkStatus(); }) .WillOnce([&udf_config, &expected_seller_udf]( @@ -402,18 +411,24 @@ reportWin = function(auctionSignals, perBuyerSignals, signalsForWinner, buyerRep EXPECT_CALL(*dispatcher_, LoadSync) .Times(3) - .WillOnce([&expected_buyer_udf, &expected_pa_version]( - std::string_view version, absl::string_view blob_data) { + .WillOnce([&expected_buyer_udf, &expected_pa_version, + &enable_protected_app_signals](std::string_view version, + absl::string_view blob_data) { EXPECT_EQ(version, expected_pa_version); - EXPECT_EQ(blob_data, GetBuyerWrappedCode(expected_buyer_udf)); + EXPECT_THAT(blob_data, + StrEq(GetBuyerWrappedCode(expected_buyer_udf, + enable_protected_app_signals))); return absl::OkStatus(); }) - .WillOnce([&expected_buyer_udf, &expected_pas_version]( - std::string_view version, absl::string_view blob_data) { + .WillOnce([&expected_buyer_udf, &expected_pas_version, + &enable_protected_app_signals](std::string_view version, + absl::string_view blob_data) { EXPECT_EQ(version, expected_pas_version); - EXPECT_EQ(blob_data, GetBuyerWrappedCode(expected_buyer_udf)); + EXPECT_THAT(blob_data, + StrEq(GetBuyerWrappedCode(expected_buyer_udf, + enable_protected_app_signals))); return absl::OkStatus(); }) .WillOnce([&udf_config, &expected_seller_udf]( diff --git a/services/bidding_service/BUILD b/services/bidding_service/BUILD index f4426b71..90cc4fd2 100644 --- a/services/bidding_service/BUILD +++ b/services/bidding_service/BUILD @@ -292,6 +292,7 @@ cc_binary( "//services/bidding_service/data:runtime_config", "//services/bidding_service/egress_features:adtech_schema_fetcher", "//services/bidding_service/inference:inference_utils", + "//services/bidding_service/inference:model_fetcher_metric", "//services/bidding_service/inference:periodic_model_fetcher", "//services/common:feature_flags", "//services/common/blob_fetch:blob_fetcher", diff --git a/services/bidding_service/benchmarking/BUILD b/services/bidding_service/benchmarking/BUILD index 2c6e80a2..0b673e2d 100644 --- a/services/bidding_service/benchmarking/BUILD +++ b/services/bidding_service/benchmarking/BUILD @@ -56,6 +56,7 @@ cc_binary( "//services/common/encryption:mock_crypto_client_wrapper", "//services/common/test:mocks", "//services/common/test:random", + "//services/common/test/utils:test_init", "@com_google_benchmark//:benchmark", "@com_google_benchmark//:benchmark_main", "@google_privacysandbox_servers_common//src/concurrent:executor", diff --git a/services/bidding_service/benchmarking/generate_bids_reactor_benchmarks.cc b/services/bidding_service/benchmarking/generate_bids_reactor_benchmarks.cc index 7a336ca1..35edbeff 100644 --- a/services/bidding_service/benchmarking/generate_bids_reactor_benchmarks.cc +++ b/services/bidding_service/benchmarking/generate_bids_reactor_benchmarks.cc @@ -30,6 +30,7 @@ #include "services/common/reporters/async_reporter.h" #include "services/common/test/mocks.h" #include "services/common/test/random.h" +#include "services/common/test/utils/test_init.h" #include "src/concurrent/event_engine_executor.h" namespace privacy_sandbox::bidding_auction_servers { @@ -175,6 +176,8 @@ InterestGroupForBidding GetInterestGroupForBidding(const std::string& name) { ig_for_bidding.set_name(name); ig_for_bidding.mutable_trusted_bidding_signals_keys()->Add( kTrustedBiddingSignalsKey); + ig_for_bidding.set_trusted_bidding_signals( + MakeTrustedBiddingSignalsForIG(ig_for_bidding)); ig_for_bidding.set_user_bidding_signals( R"JSON({"years": [1776, 1868], "name": "winston", "someId": 1789})JSON"); for (int i = 0; i < kNumAdRenderIds; ++i) { @@ -215,15 +218,12 @@ GetGenerateBidsRawRequestForAndroid() { std::string signals = MakeAStructJsonString(); raw_request.set_auction_signals(signals); raw_request.set_buyer_signals(std::move(signals)); - auto bidding_signals = MakeRandomTrustedBiddingSignals(raw_request); - CHECK_OK(bidding_signals); - raw_request.set_bidding_signals(*std::move(bidding_signals)); - return raw_request; } static void BM_ProtectedAudience(benchmark::State& state) { // Setup. + CommonTestInit(); server_common::telemetry::TelemetryConfig config_proto; config_proto.set_mode(server_common::telemetry::TelemetryConfig::PROD); diff --git a/services/bidding_service/bidding_main.cc b/services/bidding_service/bidding_main.cc index 4c2c7356..ea8ebfce 100644 --- a/services/bidding_service/bidding_main.cc +++ b/services/bidding_service/bidding_main.cc @@ -46,6 +46,7 @@ #include "services/bidding_service/egress_schema_cache.h" #include "services/bidding_service/egress_schema_fetch_config.pb.h" #include "services/bidding_service/inference/inference_utils.h" +#include "services/bidding_service/inference/model_fetcher_metric.h" #include "services/bidding_service/inference/periodic_model_fetcher.h" #include "services/bidding_service/protected_app_signals_generate_bids_reactor.h" #include "services/bidding_service/runtime_flags.h" @@ -274,13 +275,31 @@ absl::string_view GetStringParameterSafe( return ""; } +// TODO(b/356153749): Deprecate once we support dynamic partitioning metrics. +std::vector GetModels( + std::string_view inference_model_bucket_paths) { + if (!inference_model_bucket_paths.empty()) { + std::vector models; + for (absl::string_view path : + absl::StrSplit(inference_model_bucket_paths, ',')) { + models.emplace_back(path); + } + return models; + } + return {}; +} + // Brings up the gRPC BiddingService on FLAGS_port. absl::Status RunServer() { TrustedServerConfigUtil config_util(absl::GetFlag(FLAGS_init_config_client)); PS_ASSIGN_OR_RETURN(TrustedServersConfigClient config_client, GetConfigClient(config_util.GetConfigParameterPrefix())); + const std::string_view inference_model_bucket_paths = + GetStringParameterSafe(config_client, INFERENCE_MODEL_BUCKET_PATHS); + std::vector models = GetModels(inference_model_bucket_paths); // InitTelemetry right after config_client being initialized - InitTelemetry(config_util, config_client, metric::kBs); + InitTelemetry(config_util, config_client, metric::kBs, + /* buyer_list */ {}, models); PS_LOG(INFO, SystemLogContext()) << "server parameters:\n" << config_client.DebugString(); @@ -386,14 +405,11 @@ absl::Status RunServer() { if (init_config_client) { std::string_view bucket_name = GetStringParameterSafe(config_client, INFERENCE_MODEL_BUCKET_NAME); - std::string_view bucket_paths = - GetStringParameterSafe(config_client, INFERENCE_MODEL_BUCKET_PATHS); std::string_view model_config_path = GetStringParameterSafe(config_client, INFERENCE_MODEL_CONFIG_PATH); // TODO(b/356153749): Deprecate static model fetcher. - if (!bucket_name.empty() && !bucket_paths.empty()) { - std::vector models = absl::StrSplit(bucket_paths, ','); + if (!bucket_name.empty() && !inference_model_bucket_paths.empty()) { std::unique_ptr blob_storage_client = BlobStorageClientFactory::Create(); auto blob_fetcher = std::make_unique( @@ -408,6 +424,8 @@ absl::Status RunServer() { << status.message(); } } else if (!bucket_name.empty() && !model_config_path.empty()) { + PS_RETURN_IF_ERROR(inference::AddModelFetcherMetricToBidding()) + << "Failed to initialize model fetching metrics"; model_fetcher = std::make_unique( model_config_path, std::make_unique(bucket_name, executor.get(), diff --git a/services/bidding_service/bidding_service_integration_test.cc b/services/bidding_service/bidding_service_integration_test.cc index 5d9f5f7c..a090f11c 100644 --- a/services/bidding_service/bidding_service_integration_test.cc +++ b/services/bidding_service/bidding_service_integration_test.cc @@ -400,6 +400,28 @@ constexpr absl::string_view kJsCodeWithComponentBidNotAllowed = R"JS_CODE( } )JS_CODE"; +SignalBucket GetExpectedSignalBucket() { + BucketOffset bucket_offset; + bucket_offset.add_value(1); // Add first 64-bit value + bucket_offset.add_value(0); // Add second 64-bit value + bucket_offset.set_is_negative(false); + + SignalBucket signal_bucket; + signal_bucket.set_base_value(BASE_VALUE_WINNING_BID); + signal_bucket.set_scale(2.0); + *signal_bucket.mutable_offset() = bucket_offset; + + return signal_bucket; +} + +SignalValue GetExpectedSignalValue() { + SignalValue signal_value; + signal_value.set_base_value(BASE_VALUE_HIGHEST_SCORING_OTHER_BID); + signal_value.set_scale(3.0); + signal_value.set_offset(2); + return signal_value; +} + void SetupMockCryptoClientWrapper(MockCryptoClientWrapper& crypto_client) { EXPECT_CALL(crypto_client, HpkeEncrypt) .Times(testing::AnyNumber()) @@ -469,10 +491,12 @@ function generateBid( interest_group, )JS_CODE"; void SetupV8Dispatcher(V8Dispatcher* dispatcher, absl::string_view adtech_js, - std::string adtech_wasm = "") { + std::string adtech_wasm = "", + bool enable_private_aggregate_reporting = false) { ASSERT_TRUE(dispatcher->Init().ok()); - BuyerCodeWrapperConfig wrapper_config = {.ad_tech_wasm = - std::move(adtech_wasm)}; + BuyerCodeWrapperConfig wrapper_config = { + .ad_tech_wasm = std::move(adtech_wasm), + .enable_private_aggregate_reporting = enable_private_aggregate_reporting}; std::string wrapper_blob = GetBuyerWrappedCode(adtech_js, wrapper_config); ASSERT_TRUE( dispatcher @@ -497,9 +521,6 @@ BuildGenerateBidsRequestFromBrowser( *raw_request.mutable_interest_group_for_bidding()->Add() = std::move(interest_group); } - PS_ASSIGN_OR_RETURN(std::string bidding_signals, - MakeRandomTrustedBiddingSignals(raw_request)); - raw_request.set_bidding_signals(std::move(bidding_signals)); if (enable_adtech_code_logging) { raw_request.mutable_consented_debug_config()->set_token(kTestConsentToken); raw_request.mutable_consented_debug_config()->set_is_consented(true); @@ -807,7 +828,11 @@ TEST_F(GenerateBidsReactorIntegrationTest, ReceivesTrustedBiddingSignals) { ASSERT_TRUE(req.ok()) << req.status(); auto raw_request = *std::move(req); request.set_request_ciphertext(raw_request.SerializeAsString()); - ASSERT_GT(raw_request.bidding_signals().length(), 0); + ASSERT_EQ(raw_request.interest_group_for_bidding_size(), 1); + ASSERT_GT(raw_request.interest_group_for_bidding(0) + .trusted_bidding_signals() + .length(), + 0); auto generate_bids_reactor_factory = [&client](grpc::CallbackServerContext* context, @@ -1060,9 +1085,6 @@ TEST_F(GenerateBidsReactorIntegrationTest, GeneratesBidsFromDevice) { // This fails in production, the user Bidding Signals are not being set. // use logging to figure out why. } - auto sigs = MakeRandomTrustedBiddingSignals(raw_request); - ASSERT_TRUE(sigs.ok()) << sigs.status(); - raw_request.set_bidding_signals(*std::move(sigs)); GenerateBidsResponse response; *request.mutable_request_ciphertext() = raw_request.SerializeAsString(); auto generate_bids_reactor_factory = @@ -1135,24 +1157,34 @@ TEST_F(GenerateBidsReactorIntegrationTest, GeneratesBidsFromDevice) { } } +struct GenerateBidHelperConfig { + bool enable_debug_reporting = false; + bool enable_buyer_debug_url_generation = false; + bool enable_adtech_code_logging = false; + std::string wasm_blob = ""; + int desired_bid_count = 5; + bool component_auction = false; + bool enable_private_aggregate_reporting = false; +}; + void GenerateBidCodeWrapperTestHelper( GenerateBidsResponse* response, absl::string_view js_blob, - bool enable_debug_reporting, bool enable_buyer_debug_url_generation, - bool enable_adtech_code_logging = false, std::string wasm_blob = "", - int desired_bid_count = 5, bool component_auction = false) { + const GenerateBidHelperConfig& test_config) { grpc::CallbackServerContext context; V8Dispatcher dispatcher; CodeDispatchClient client(dispatcher); - SetupV8Dispatcher(&dispatcher, js_blob, std::move(wasm_blob)); + SetupV8Dispatcher(&dispatcher, js_blob, test_config.wasm_blob, + test_config.enable_private_aggregate_reporting); GenerateBidsRequest request; request.set_key_id(kKeyId); absl::flat_hash_map> interest_group_to_ad; auto req = BuildGenerateBidsRequestFromBrowser( - &interest_group_to_ad, desired_bid_count, enable_debug_reporting, - enable_adtech_code_logging); + &interest_group_to_ad, test_config.desired_bid_count, + test_config.enable_debug_reporting, + test_config.enable_adtech_code_logging); ASSERT_TRUE(req.ok()) << req.status(); - if (component_auction) { + if (test_config.component_auction) { req.value().set_top_level_seller(kTopLevelSeller); } *request.mutable_request_ciphertext() = req->SerializeAsString(); @@ -1198,7 +1230,10 @@ void GenerateBidCodeWrapperTestHelper( config_client, /* public_key_fetcher= */ nullptr); BiddingServiceRuntimeConfig runtime_config{ - .enable_buyer_debug_url_generation = enable_buyer_debug_url_generation}; + .enable_buyer_debug_url_generation = + test_config.enable_buyer_debug_url_generation, + .enable_private_aggregate_reporting = + test_config.enable_private_aggregate_reporting}; BiddingService service( std::move(generate_bids_reactor_factory), std::move(key_fetcher_manager), std::move(crypto_client), std::move(runtime_config), @@ -1212,11 +1247,14 @@ TEST_F(GenerateBidsReactorIntegrationTest, BuyerDebugUrlGenerationDisabled) { GenerateBidsResponse response; bool enable_debug_reporting = true; bool enable_buyer_debug_url_generation = false; + GenerateBidHelperConfig test_config = { + .enable_debug_reporting = enable_debug_reporting, + .enable_buyer_debug_url_generation = enable_buyer_debug_url_generation}; GenerateBidCodeWrapperTestHelper( &response, absl::StrFormat(js_code_with_debug_urls_template, kAdRenderUrlPrefixForTest), - enable_debug_reporting, enable_buyer_debug_url_generation); + test_config); GenerateBidsResponse::GenerateBidsRawResponse raw_response; raw_response.ParseFromString(response.response_ciphertext()); @@ -1231,11 +1269,15 @@ TEST_F(GenerateBidsReactorIntegrationTest, EventLevelDebugReportingDisabled) { GenerateBidsResponse response; bool enable_debug_reporting = false; bool enable_buyer_debug_url_generation = true; + GenerateBidHelperConfig test_config = { + .enable_debug_reporting = enable_debug_reporting, + .enable_buyer_debug_url_generation = enable_buyer_debug_url_generation}; + GenerateBidCodeWrapperTestHelper( &response, absl::StrFormat(js_code_with_debug_urls_template, kAdRenderUrlPrefixForTest), - enable_debug_reporting, enable_buyer_debug_url_generation); + test_config); GenerateBidsResponse::GenerateBidsRawResponse raw_response; raw_response.ParseFromString(response.response_ciphertext()); EXPECT_GT(raw_response.bids_size(), 0); @@ -1250,11 +1292,14 @@ TEST_F(GenerateBidsReactorIntegrationTest, GenerateBidsResponse response; bool enable_debug_reporting = true; bool enable_buyer_debug_url_generation = true; + GenerateBidHelperConfig test_config = { + .enable_debug_reporting = enable_debug_reporting, + .enable_buyer_debug_url_generation = enable_buyer_debug_url_generation}; GenerateBidCodeWrapperTestHelper( &response, absl::StrFormat(js_code_with_debug_urls_template, kAdRenderUrlPrefixForTest), - enable_debug_reporting, enable_buyer_debug_url_generation); + test_config); GenerateBidsResponse::GenerateBidsRawResponse raw_response; raw_response.ParseFromString(response.response_ciphertext()); EXPECT_GT(raw_response.bids_size(), 0); @@ -1267,16 +1312,93 @@ TEST_F(GenerateBidsReactorIntegrationTest, } } +TEST_F(GenerateBidsReactorIntegrationTest, + ReturnsNumericPAAPIBucketAndValueInTheResponse) { + GenerateBidsResponse response; + bool enable_debug_reporting = true; + bool enable_buyer_debug_url_generation = true; + GenerateBidHelperConfig test_config = { + .enable_debug_reporting = enable_debug_reporting, + .enable_buyer_debug_url_generation = enable_buyer_debug_url_generation, + .enable_private_aggregate_reporting = true}; + GenerateBidCodeWrapperTestHelper( + &response, + absl::StrFormat(kBuyerBaseCodeForPrivateAggregation, + kAdRenderUrlPrefixForTest), + test_config); + GenerateBidsResponse::GenerateBidsRawResponse raw_response; + raw_response.ParseFromString(response.response_ciphertext()); + EXPECT_GT(raw_response.bids_size(), 0); + for (const auto& adWithBid : raw_response.bids()) { + EXPECT_GT(adWithBid.bid(), 0); + auto& contributions = adWithBid.private_aggregation_contributions(); + EXPECT_EQ(contributions.size(), 1); + EXPECT_EQ(contributions.at(0).event().event_type(), + EventType::EVENT_TYPE_WIN); + ASSERT_EQ( + contributions.at(0).bucket().bucket_128_bit().bucket_128_bits_size(), 2) + << "bucket_128_bit has not been set correctly in the contribution"; + EXPECT_EQ( + contributions.at(0).bucket().bucket_128_bit().bucket_128_bits().at(0), + 100) + << "bucket_128_bit has not been set in the contribution"; + EXPECT_EQ(contributions.at(0).value().int_value(), 200) + << "int_value has not been set correctly in the contribution"; + } +} + +TEST_F(GenerateBidsReactorIntegrationTest, + ReturnsPAAPISignalBucketAndSignalValueInTheResponse) { + GenerateBidsResponse response; + bool enable_debug_reporting = true; + bool enable_buyer_debug_url_generation = true; + GenerateBidHelperConfig test_config = { + .enable_debug_reporting = enable_debug_reporting, + .enable_buyer_debug_url_generation = enable_buyer_debug_url_generation, + .enable_private_aggregate_reporting = true}; + GenerateBidCodeWrapperTestHelper( + &response, + absl::StrFormat(kBuyerBaseCodeWithSignalValueAndSignalBucket, + kAdRenderUrlPrefixForTest), + test_config); + GenerateBidsResponse::GenerateBidsRawResponse raw_response; + raw_response.ParseFromString(response.response_ciphertext()); + EXPECT_GT(raw_response.bids_size(), 0); + for (const auto& adWithBid : raw_response.bids()) { + EXPECT_GT(adWithBid.bid(), 0); + auto& contributions = adWithBid.private_aggregation_contributions(); + EXPECT_EQ(contributions.size(), 1); + EXPECT_EQ(contributions.at(0).event().event_type(), + EventType::EVENT_TYPE_WIN); + ASSERT_TRUE(contributions.at(0).bucket().has_signal_bucket()) + << "SignalBucket has not been set in the contribution"; + ASSERT_TRUE(contributions.at(0).value().has_extended_value()) + << "SignalValue has not been set in the contribution"; + google::protobuf::util::MessageDifferencer diff; + std::string diff_output; + diff.ReportDifferencesToString(&diff_output); + EXPECT_TRUE(diff.Compare(contributions.at(0).bucket().signal_bucket(), + GetExpectedSignalBucket())) + << diff_output; + EXPECT_TRUE(diff.Compare(contributions.at(0).value().extended_value(), + GetExpectedSignalValue())) + << diff_output; + } +} + TEST_F(GenerateBidsReactorIntegrationTest, GeneratesBidsDoesNotReturnLargeDebugReportingUrls) { GenerateBidsResponse response; bool enable_debug_reporting = true; bool enable_buyer_debug_url_generation = true; + GenerateBidHelperConfig test_config = { + .enable_debug_reporting = enable_debug_reporting, + .enable_buyer_debug_url_generation = enable_buyer_debug_url_generation}; GenerateBidCodeWrapperTestHelper( &response, absl::StrFormat(js_code_with_very_large_debug_urls, kAdRenderUrlPrefixForTest), - enable_debug_reporting, enable_buyer_debug_url_generation); + test_config); GenerateBidsResponse::GenerateBidsRawResponse raw_response; raw_response.ParseFromString(response.response_ciphertext()); EXPECT_GT(raw_response.bids_size(), 0); @@ -1296,12 +1418,16 @@ TEST_F(GenerateBidsReactorIntegrationTest, bool enable_debug_reporting = true; bool enable_buyer_debug_url_generation = true; int desired_bid_count = 100; + GenerateBidHelperConfig test_config = { + .enable_debug_reporting = enable_debug_reporting, + .enable_buyer_debug_url_generation = enable_buyer_debug_url_generation, + .desired_bid_count = desired_bid_count}; + GenerateBidCodeWrapperTestHelper( &response, absl::StrFormat(js_code_with_very_large_debug_urls, kAdRenderUrlPrefixForTest), - enable_debug_reporting, enable_buyer_debug_url_generation, false, "", - desired_bid_count); + test_config); GenerateBidsResponse::GenerateBidsRawResponse raw_response; raw_response.ParseFromString(response.response_ciphertext()); @@ -1322,11 +1448,15 @@ TEST_F(GenerateBidsReactorIntegrationTest, GenerateBidsResponse response; bool enable_debug_reporting = true; bool enable_buyer_debug_url_generation = true; + GenerateBidHelperConfig test_config = { + .enable_debug_reporting = enable_debug_reporting, + .enable_buyer_debug_url_generation = enable_buyer_debug_url_generation, + }; GenerateBidCodeWrapperTestHelper( &response, absl::StrFormat(js_code_with_global_this_debug_urls_template, kAdRenderUrlPrefixForTest), - enable_debug_reporting, enable_buyer_debug_url_generation); + test_config); GenerateBidsResponse::GenerateBidsRawResponse raw_response; raw_response.ParseFromString(response.response_ciphertext()); EXPECT_GT(raw_response.bids_size(), 0); @@ -1344,9 +1474,12 @@ TEST_F(GenerateBidsReactorIntegrationTest, GenerateBidsResponse response; bool enable_debug_reporting = true; bool enable_buyer_debug_url_generation = true; + GenerateBidHelperConfig test_config = { + .enable_debug_reporting = enable_debug_reporting, + .enable_buyer_debug_url_generation = enable_buyer_debug_url_generation, + }; GenerateBidCodeWrapperTestHelper( - &response, js_code_throws_exception_with_debug_urls, - enable_debug_reporting, enable_buyer_debug_url_generation); + &response, js_code_throws_exception_with_debug_urls, test_config); GenerateBidsResponse::GenerateBidsRawResponse raw_response; raw_response.ParseFromString(response.response_ciphertext()); EXPECT_EQ(raw_response.bids_size(), 0); @@ -1357,9 +1490,12 @@ TEST_F(GenerateBidsReactorIntegrationTest, GenerateBidsResponse response; bool enable_debug_reporting = true; bool enable_buyer_debug_url_generation = true; + GenerateBidHelperConfig test_config = { + .enable_debug_reporting = enable_debug_reporting, + .enable_buyer_debug_url_generation = enable_buyer_debug_url_generation, + }; GenerateBidCodeWrapperTestHelper(&response, js_code_throws_exception, - enable_debug_reporting, - enable_buyer_debug_url_generation); + test_config); GenerateBidsResponse::GenerateBidsRawResponse raw_response; raw_response.ParseFromString(response.response_ciphertext()); EXPECT_EQ(raw_response.bids_size(), 0); @@ -1370,11 +1506,14 @@ TEST_F(GenerateBidsReactorIntegrationTest, GenerateBidsResponse response; bool enable_debug_reporting = false; bool enable_buyer_debug_url_generation = false; + GenerateBidHelperConfig test_config = { + .enable_debug_reporting = enable_debug_reporting, + .enable_buyer_debug_url_generation = enable_buyer_debug_url_generation, + .enable_adtech_code_logging = true}; GenerateBidCodeWrapperTestHelper( &response, absl::StrFormat(js_code_with_logs_template, kAdRenderUrlPrefixForTest), - enable_debug_reporting, enable_buyer_debug_url_generation, - /*enable_adtech_code_logging=*/true); + test_config); GenerateBidsResponse::GenerateBidsRawResponse raw_response; raw_response.ParseFromString(response.response_ciphertext()); EXPECT_GT(raw_response.bids_size(), 0); @@ -1388,10 +1527,13 @@ TEST_F(GenerateBidsReactorIntegrationTest, std::string raw_wasm_bytes; ASSERT_TRUE(absl::Base64Unescape(base64_wasm_plus_one, &raw_wasm_bytes)); - GenerateBidCodeWrapperTestHelper( - &response, js_code_runs_wasm_helper, enable_debug_reporting, - enable_buyer_debug_url_generation, - /*enable_adtech_code_logging=*/true, raw_wasm_bytes); + GenerateBidHelperConfig test_config = { + .enable_debug_reporting = enable_debug_reporting, + .enable_buyer_debug_url_generation = enable_buyer_debug_url_generation, + .enable_adtech_code_logging = true, + .wasm_blob = raw_wasm_bytes}; + GenerateBidCodeWrapperTestHelper(&response, js_code_runs_wasm_helper, + test_config); GenerateBidsResponse::GenerateBidsRawResponse raw_response; raw_response.ParseFromString(response.response_ciphertext()); EXPECT_GT(raw_response.bids_size(), 0); @@ -1402,9 +1544,9 @@ TEST_F(GenerateBidsReactorIntegrationTest, TEST_F(GenerateBidsReactorIntegrationTest, ParsesFieldsForComponentAuction) { GenerateBidsResponse response; - GenerateBidCodeWrapperTestHelper(&response, kJsCodeWithTopLevelSeller, false, - false, false, "", 5, - /* component_auction = */ true); + GenerateBidHelperConfig test_config = {.component_auction = true}; + GenerateBidCodeWrapperTestHelper(&response, kJsCodeWithTopLevelSeller, + test_config); GenerateBidsResponse::GenerateBidsRawResponse raw_response; raw_response.ParseFromString(response.response_ciphertext()); @@ -1422,9 +1564,9 @@ TEST_F(GenerateBidsReactorIntegrationTest, ParsesFieldsForComponentAuction) { TEST_F(GenerateBidsReactorIntegrationTest, FiltersUnallowedAdsForComponentAuction) { GenerateBidsResponse response; + GenerateBidHelperConfig test_config = {.component_auction = true}; GenerateBidCodeWrapperTestHelper(&response, kJsCodeWithComponentBidNotAllowed, - false, false, false, "", 5, - /* component_auction = */ true); + test_config); GenerateBidsResponse::GenerateBidsRawResponse raw_response; raw_response.ParseFromString(response.response_ciphertext()); diff --git a/services/bidding_service/cddl_spec_cache_test.cc b/services/bidding_service/cddl_spec_cache_test.cc index 4876be88..1ed1d7ae 100644 --- a/services/bidding_service/cddl_spec_cache_test.cc +++ b/services/bidding_service/cddl_spec_cache_test.cc @@ -68,7 +68,7 @@ boolean-feature-type = { } boolean-feature = { - type: boolean-feature-type, + name: text .regexp "^boolean-feature$", value: bool, } @@ -78,7 +78,7 @@ unsigned-integer-feature-type = { } unsigned-integer-feature = { - type: unsigned-integer-feature-type, + name: text .regexp "^unsigned-integer-feature$", value: uint, } @@ -88,17 +88,17 @@ signed-integer-feature-type = { } signed-integer-feature = { - type: signed-integer-feature-type, + name: text .regexp "^signed-integer-feature$", value: int, } bucket-feature-type = { - name: text .regexp "^bucket-feature$" + name: text .regexp "^bucket-feature$", size: uint, ; number of buckets. } bucket-feature = { - type: bucket-feature-type, + name: text .regexp "^bucket-feature$", value: [* bool], } @@ -111,7 +111,7 @@ histogram-feature-type = { histogram-feature-subtype /= unsigned-integer-feature-type / signed-integer-feature-type histogram-feature = { - type: histogram-feature-type, + name: text .regexp "^histogram-feature$", value: [1* unsigned-integer-feature / signed-integer-feature ], } diff --git a/services/bidding_service/code_wrapper/buyer_code_wrapper.h b/services/bidding_service/code_wrapper/buyer_code_wrapper.h index 60da5f54..54151f3d 100644 --- a/services/bidding_service/code_wrapper/buyer_code_wrapper.h +++ b/services/bidding_service/code_wrapper/buyer_code_wrapper.h @@ -90,7 +90,7 @@ inline constexpr absl::string_view kEntryFunction = R"JS_CODE( //Add private aggregate contributions to the response. //If the private aggregation is enabled, the contributions are expected to be set in ps_response.private_aggregation_contributions. if(ps_response.paapicontributions){ - generate_bid_response.private_aggregation_contributions = ps_response.private_aggregation_contributions; + generate_bid_response.private_aggregation_contributions = ps_response.paapicontributions; } ps_response.response = generate_bid_response if( featureFlags.enable_debug_url_generation && diff --git a/services/bidding_service/code_wrapper/buyer_code_wrapper_test_constants.h b/services/bidding_service/code_wrapper/buyer_code_wrapper_test_constants.h index ca4b2a83..94137cb8 100644 --- a/services/bidding_service/code_wrapper/buyer_code_wrapper_test_constants.h +++ b/services/bidding_service/code_wrapper/buyer_code_wrapper_test_constants.h @@ -40,6 +40,7 @@ constexpr absl::string_view kBuyerBaseCode_template = R"JS_CODE( }; } )JS_CODE"; + constexpr absl::string_view kBuyerBaseCodeForPrivateAggregation = R"JS_CODE( function fibonacci(num) { if (num <= 1) return 1; @@ -63,6 +64,32 @@ constexpr absl::string_view kBuyerBaseCodeForPrivateAggregation = R"JS_CODE( }; } )JS_CODE"; + +constexpr absl::string_view kBuyerBaseCodeWithSignalValueAndSignalBucket = + R"JS_CODE( + function fibonacci(num) { + if (num <= 1) return 1; + return fibonacci(num - 1) + fibonacci(num - 2); + } + function generateBid(interest_group, auction_signals,buyer_signals, + trusted_bidding_signals, + device_signals){ + const bid = fibonacci(Math.floor(Math.random() * 30 + 1)); + console.log("Logging from generateBid"); + const contribution = { + bucket: {baseValue:"winning-bid",scale:2.0, offset:1}, + value: {baseValue:"highest-scoring-other-bid",scale:3.0, offset:2}, + }; + privateAggregation.contributeToHistogramOnEvent('reserved.win', contribution); + return { + render: "%s" + interest_group.adRenderIds[0], + ad: {"arbitraryMetadataField": 1}, + bid: bid, + allowComponentAuction: false + }; + } +)JS_CODE"; + constexpr absl::string_view kExpectedGenerateBidCode_template = R"JS_CODE( const globalWasmHex = []; const globalWasmHelper = globalWasmHex.length ? new WebAssembly.Module(Uint8Array.from(globalWasmHex)) : null; @@ -102,7 +129,7 @@ constexpr absl::string_view kExpectedGenerateBidCode_template = R"JS_CODE( //Add private aggregate contributions to the response. //If the private aggregation is enabled, the contributions are expected to be set in ps_response.private_aggregation_contributions. if(ps_response.paapicontributions){ - generate_bid_response.private_aggregation_contributions = ps_response.private_aggregation_contributions; + generate_bid_response.private_aggregation_contributions = ps_response.paapicontributions; } ps_response.response = generate_bid_response if( featureFlags.enable_debug_url_generation && @@ -189,7 +216,7 @@ constexpr absl::string_view //Add private aggregate contributions to the response. //If the private aggregation is enabled, the contributions are expected to be set in ps_response.private_aggregation_contributions. if(ps_response.paapicontributions){ - generate_bid_response.private_aggregation_contributions = ps_response.private_aggregation_contributions; + generate_bid_response.private_aggregation_contributions = ps_response.paapicontributions; } ps_response.response = generate_bid_response if( featureFlags.enable_debug_url_generation && diff --git a/services/bidding_service/code_wrapper/js/bidding_private_aggregation_util.js b/services/bidding_service/code_wrapper/js/bidding_private_aggregation_util.js index 92d885fe..0859b8d1 100644 --- a/services/bidding_service/code_wrapper/js/bidding_private_aggregation_util.js +++ b/services/bidding_service/code_wrapper/js/bidding_private_aggregation_util.js @@ -46,31 +46,6 @@ class BiddingPrivateAggregationUtilImpl { return 'EVENT_TYPE_CUSTOM'; // Custom event } } - - /** Converts string baseValue type to an int using the values from enum BaseValue. - * @param {string} value - * @return {number} - * @throws {TypeError} - * @override - */ - convertBaseValueToInt(value) { - switch (value) { - case 'winning-bid': - return 0; - case 'highest-scoring-other-bid': - return 1; - case 'bid-rejection-reason': - return 2; - //TBD - case 'script-run-time': - return 3; - //TBD - case 'signals-fetch-time': - return 4; - default: - throw new TypeError('Base value type not supported.'); - } - } } /** diff --git a/services/bidding_service/code_wrapper/js/bidding_private_aggregation_util_externs.js b/services/bidding_service/code_wrapper/js/bidding_private_aggregation_util_externs.js index 46851843..f71113da 100644 --- a/services/bidding_service/code_wrapper/js/bidding_private_aggregation_util_externs.js +++ b/services/bidding_service/code_wrapper/js/bidding_private_aggregation_util_externs.js @@ -23,11 +23,4 @@ class BiddingPrivateAggregationUtil { * @throws {TypeError} */ mapEventToEnum(event) {} - - /** - * @param {string} value - * @return {number} - * @throws {TypeError} - */ - convertBaseValueToInt(value) {} } diff --git a/services/bidding_service/code_wrapper/js/bidding_private_aggregation_util_test.js b/services/bidding_service/code_wrapper/js/bidding_private_aggregation_util_test.js index d87679a3..5d234737 100644 --- a/services/bidding_service/code_wrapper/js/bidding_private_aggregation_util_test.js +++ b/services/bidding_service/code_wrapper/js/bidding_private_aggregation_util_test.js @@ -55,43 +55,4 @@ testSuite({ assertTrue(error instanceof TypeError); } }, - /** @return {void} */ - /** @suppress {reportUnknownTypes, checkTypes} */ - testConvertBaseValueToInt() { - // Test defined base values - var result = new BiddingPrivateAggregationUtilImpl().convertBaseValueToInt('winning-bid'); - assertEquals(result, 0); - result = new BiddingPrivateAggregationUtilImpl().convertBaseValueToInt('highest-scoring-other-bid'); - assertEquals(result, 1); - - // Test TBD cases (assuming these return values are defined later) - result = new BiddingPrivateAggregationUtilImpl().convertBaseValueToInt('script-run-time'); - assertEquals(result, 3); - result = new BiddingPrivateAggregationUtilImpl().convertBaseValueToInt('signals-fetch-time'); - assertEquals(result, 4); - - // Test unsupported base value type - try { - new BiddingPrivateAggregationUtilImpl().convertBaseValueToInt('something-else'); - fail('Expected TypeError for unsupported base value'); - } catch (error) { - assertTrue(error instanceof TypeError); - } - - // Test int (invalid) base value - try { - new BiddingPrivateAggregationUtilImpl().convertBaseValueToInt(1); - fail('Expected TypeError for null base value'); - } catch (error) { - assertTrue(error instanceof TypeError); - } - - // Test null base value - try { - new BiddingPrivateAggregationUtilImpl().convertBaseValueToInt(null); - fail('Expected TypeError for null base value'); - } catch (error) { - assertTrue(error instanceof TypeError); - } - }, }); diff --git a/services/bidding_service/code_wrapper/js/private_aggregation_wrapper_test.js b/services/bidding_service/code_wrapper/js/private_aggregation_wrapper_test.js index 02650c88..4b83efb6 100644 --- a/services/bidding_service/code_wrapper/js/private_aggregation_wrapper_test.js +++ b/services/bidding_service/code_wrapper/js/private_aggregation_wrapper_test.js @@ -26,19 +26,15 @@ testSuite({ }; const expectedContribution1 = { - bucket_128_bit: numberBucket, - signal_bucket: null, - int_value: numberValue, - extended_value: null, + bucket: { bucket_128_bit: { bucket_128_bits: [numberBucket, 0] } }, + value: { int_value: numberValue }, event: { event_type: 'EVENT_TYPE_WIN', }, }; const expectedContribution2 = { - bucket_128_bit: numberBucket, - signal_bucket: null, - int_value: numberValue, - extended_value: null, + bucket: { bucket_128_bit: { bucket_128_bits: [numberBucket, 0] } }, + value: { int_value: numberValue }, event: { event_type: 'EVENT_TYPE_CUSTOM', event_name: 'a-custom-event-test', @@ -53,40 +49,43 @@ testSuite({ }, testContributeToHistogramOnEvent_ValidObjectBucketAndValue() { const objectBucket = { - base_value: 'winning-bid', + baseValue: 'winning-bid', + scale: 1.0, + offset: 10, + }; + const signalBucket = { + base_value: 'BASE_VALUE_WINNING_BID', scale: 1.0, offset: { - value: [10, 20], + value: [10, 0], is_negative: false, }, }; const objectValue = { - extended_value: { - base_value: 'winning-bid', - int_value: 30, - is_negative: false, - }, + baseValue: 'winning-bid', + scale: 30, + offset: 5, + }; + const signalValue = { + base_value: 'BASE_VALUE_WINNING_BID', + scale: 30, + offset: 5, }; - const contribution = { bucket: objectBucket, value: objectValue, }; const expectedContribution1 = { - bucket_128_bit: null, - signal_bucket: objectBucket, - int_value: null, - extended_value: objectValue, + bucket: { signal_bucket: signalBucket }, + value: { extended_value: signalValue }, event: { event_type: 'EVENT_TYPE_WIN', }, }; const expectedContribution2 = { - bucket_128_bit: null, - signal_bucket: objectBucket, - int_value: null, - extended_value: objectValue, + bucket: { signal_bucket: signalBucket }, + value: { extended_value: signalValue }, event: { event_type: 'EVENT_TYPE_CUSTOM', event_name: 'a-custom-event-test', @@ -94,11 +93,12 @@ testSuite({ }; // Test with win. privateAggregation.contributeToHistogramOnEvent('reserved.win', contribution); + privateAggregation.contributeToHistogramOnEvent('a-custom-event-test', contribution); assertObjectEquals(expectedContribution1, private_aggregation_contributions[0]); // Test with custom. - privateAggregation.contributeToHistogramOnEvent('a-custom-event-test', contribution); assertObjectEquals(expectedContribution2, private_aggregation_contributions[1]); }, + testContributeToHistogram() { const numberBucket = 5; const numberValue = 10; @@ -109,10 +109,8 @@ testSuite({ }; const expectedContribution1 = { - bucket_128_bit: numberBucket, - signal_bucket: null, - int_value: numberValue, - extended_value: null, + bucket: { bucket_128_bit: { bucket_128_bits: [numberBucket, 0] } }, + value: { int_value: numberValue }, event: { event_type: 'EVENT_TYPE_ALWAYS', // reserved.always }, diff --git a/services/bidding_service/egress_cddl_spec/1.0.0 b/services/bidding_service/egress_cddl_spec/1.0.0 index c00ffde0..fc3421fb 100644 --- a/services/bidding_service/egress_cddl_spec/1.0.0 +++ b/services/bidding_service/egress_cddl_spec/1.0.0 @@ -36,7 +36,7 @@ boolean-feature-type = { } boolean-feature = { - type: boolean-feature-type, + name: text .regexp "^boolean-feature$", value: bool, } @@ -46,7 +46,7 @@ unsigned-integer-feature-type = { } unsigned-integer-feature = { - type: unsigned-integer-feature-type, + name: text .regexp "^unsigned-integer-feature$", value: uint, } @@ -56,17 +56,17 @@ signed-integer-feature-type = { } signed-integer-feature = { - type: signed-integer-feature-type, + name: text .regexp "^signed-integer-feature$", value: int, } bucket-feature-type = { - name: text .regexp "^bucket-feature$" + name: text .regexp "^bucket-feature$", size: uint, ; number of buckets. } bucket-feature = { - type: bucket-feature-type, + name: text .regexp "^bucket-feature$", value: [* bool], } @@ -79,7 +79,7 @@ histogram-feature-type = { histogram-feature-subtype /= unsigned-integer-feature-type / signed-integer-feature-type histogram-feature = { - type: histogram-feature-type, + name: text .regexp "^histogram-feature$", value: [1* unsigned-integer-feature / signed-integer-feature ], } diff --git a/services/bidding_service/egress_features/boolean_feature_test.cc b/services/bidding_service/egress_features/boolean_feature_test.cc index e3fd2822..83446d8e 100644 --- a/services/bidding_service/egress_features/boolean_feature_test.cc +++ b/services/bidding_service/egress_features/boolean_feature_test.cc @@ -34,7 +34,7 @@ class BooleanFeatureTest : public ::testing::Test { std::string test_schema = absl::Substitute(R"JSON( { "test_wrapper": { - "type": "boolean-feature", + "name": "boolean-feature", "value": $0 } } diff --git a/services/bidding_service/egress_features/bucket_feature_test.cc b/services/bidding_service/egress_features/bucket_feature_test.cc index 11f9cbfc..cb1560eb 100644 --- a/services/bidding_service/egress_features/bucket_feature_test.cc +++ b/services/bidding_service/egress_features/bucket_feature_test.cc @@ -43,7 +43,7 @@ class BucketFeatureTest : public ::testing::Test { }); std::string test_schema = absl::Substitute(R"JSON( { - "test_wrapper": {"type": "bucket-feature", "value": [ $0 ]} + "test_wrapper": {"name": "bucket-feature", "value": [ $0 ]} } )JSON", bool_feat_json); diff --git a/services/bidding_service/egress_features/egress_feature.cc b/services/bidding_service/egress_features/egress_feature.cc index 22ee9825..f208ff9e 100644 --- a/services/bidding_service/egress_features/egress_feature.cc +++ b/services/bidding_service/egress_features/egress_feature.cc @@ -39,7 +39,7 @@ uint32_t EgressFeature::Size() const { return size_; } absl::Status EgressFeature::SetValue(rapidjson::Value value, bool verify_type) { DCHECK(!is_value_set_); std::string observed_feat_type; - PS_ASSIGN_IF_PRESENT(observed_feat_type, value, "type", String); + PS_ASSIGN_IF_PRESENT(observed_feat_type, value, "name", String); if (verify_type) { absl::string_view expected_feat_type = Type(); if (observed_feat_type != expected_feat_type) { diff --git a/services/bidding_service/egress_features/histogram_feature_test.cc b/services/bidding_service/egress_features/histogram_feature_test.cc index c55ab98e..5074210c 100644 --- a/services/bidding_service/egress_features/histogram_feature_test.cc +++ b/services/bidding_service/egress_features/histogram_feature_test.cc @@ -37,7 +37,7 @@ class HistogramFeatureTest : public ::testing::Test { std::shared_ptr TestSchema() { std::string test_schema = R"JSON( { - "type": "histogram-feature", + "name": "histogram-feature", "test_wrapper": { "value": [ { @@ -66,7 +66,7 @@ class HistogramFeatureTest : public ::testing::Test { absl::StrAppend(out, absl::Substitute(R"JSON( { - "type": "$0", + "name": "$0", "value": $1 } )JSON", @@ -74,7 +74,7 @@ class HistogramFeatureTest : public ::testing::Test { }); std::string test_schema = absl::Substitute(R"JSON( { - "test_wrapper": {"type": "histogram-feature", "value": [ $0 ]} + "test_wrapper": {"name": "histogram-feature", "value": [ $0 ]} } )JSON", histogram_feat_json); diff --git a/services/bidding_service/egress_features/signed_int_feature_test.cc b/services/bidding_service/egress_features/signed_int_feature_test.cc index a5bfa2d0..1ab433d1 100644 --- a/services/bidding_service/egress_features/signed_int_feature_test.cc +++ b/services/bidding_service/egress_features/signed_int_feature_test.cc @@ -34,7 +34,7 @@ class SignedIntFeatureTest : public ::testing::Test { std::string test_schema = absl::Substitute(R"JSON( { "test_wrapper": { - "type": "signed-integer-feature", + "name": "signed-integer-feature", "value": $0 } } diff --git a/services/bidding_service/egress_features/unsigned_int_feature_test.cc b/services/bidding_service/egress_features/unsigned_int_feature_test.cc index 2545f7a6..3059635d 100644 --- a/services/bidding_service/egress_features/unsigned_int_feature_test.cc +++ b/services/bidding_service/egress_features/unsigned_int_feature_test.cc @@ -34,7 +34,7 @@ class UnsignedIntFeatureTest : public ::testing::Test { std::string test_schema = absl::Substitute(R"JSON( { "test_wrapper": { - "type": "unsigned-integer-feature", + "name": "unsigned-integer-feature", "value": $0 } } diff --git a/services/bidding_service/generate_bids_reactor.cc b/services/bidding_service/generate_bids_reactor.cc index 5b3d2a1f..5fd78f89 100644 --- a/services/bidding_service/generate_bids_reactor.cc +++ b/services/bidding_service/generate_bids_reactor.cc @@ -41,13 +41,6 @@ using ::google::protobuf::TextFormat; using RawRequest = GenerateBidsRequest::GenerateBidsRawRequest; using IGForBidding = GenerateBidsRequest::GenerateBidsRawRequest::InterestGroupForBidding; -struct ParsedTrustedBiddingSignals { - std::shared_ptr json = std::make_shared(); - absl::flat_hash_set keys; -}; -using TrustedBiddingSignalsByIg = - absl::flat_hash_map>; constexpr int kArgsSizeWithWrapper = 6; absl::StatusOr ProtoToJson( @@ -59,52 +52,6 @@ absl::StatusOr ProtoToJson( return json; } -// Creates a json of trusted bidding signals for a single IG. Queries the -// bidding signals for - -// 1. IG Name and moves the bidding signal values to the new json document. -// 2. Bidding signal keys in the IG and copies them to the new json document. -absl::StatusOr GetSignalsForIG( - const GenerateBidsRequest::GenerateBidsRawRequest::InterestGroupForBidding& - ig, - rapidjson::Value* bidding_signals_obj, long avg_signal_str_size) { - // Insert bidding signal values for this Interest Group. - ParsedTrustedBiddingSignals parsed_trusted_bidding_signals; - rapidjson::Document ig_signals; - ig_signals.SetObject(); - // If no bidding signals passed, return empty document. - if (bidding_signals_obj == nullptr) { - return parsed_trusted_bidding_signals; - } - - // Copy bidding signals with key name in bidding signal keys. - for (const auto& key : ig.trusted_bidding_signals_keys()) { - if (parsed_trusted_bidding_signals.keys.contains(key)) { - // Do not process duplicate keys. - continue; - } - rapidjson::Value::ConstMemberIterator trusted_bidding_signals_key_itr = - bidding_signals_obj->FindMember(key.c_str()); - if (trusted_bidding_signals_key_itr != bidding_signals_obj->MemberEnd()) { - rapidjson::Value json_key; - // Keep string reference. Assumes safe lifecycle. - json_key.SetString(rapidjson::StringRef(key.c_str())); - rapidjson::Value json_value; - // Copy instead of move, could be referenced by multiple IGs. - json_value.CopyFrom(trusted_bidding_signals_key_itr->value, - ig_signals.GetAllocator()); - // AddMember moves Values, do not reference them anymore. - ig_signals.AddMember(json_key, json_value, ig_signals.GetAllocator()); - parsed_trusted_bidding_signals.keys.emplace(key); - } - } - if (ig_signals.MemberCount() > 0) { - absl::StatusOr> ig_signals_str = - SerializeJsonDoc(ig_signals, avg_signal_str_size); - PS_ASSIGN_OR_RETURN(parsed_trusted_bidding_signals.json, ig_signals_str); - } - return parsed_trusted_bidding_signals; -} - constexpr char kTopWindowHostname[] = "topWindowHostname"; constexpr char kSeller[] = "seller"; constexpr char kTopLevelSeller[] = "topLevelSeller"; @@ -217,46 +164,6 @@ absl::StatusOr SerializeIG(const IGForBidding& ig) { return serialized_ig; } -// Creates a map of Interest Group names -> trusted bidding signals json -// strings. Parses the trusted bidding signals string and calls GetSignalsForIG -// in a loop for all Interest Groups. -absl::StatusOr -DeserializeAndGetTrustedBiddingSignalsPerIG( - const GenerateBidsRequest::GenerateBidsRawRequest& raw_request, - RequestLogContext& log_context) { - // Parse into JSON. - auto start_parse_time = absl::Now(); - PS_ASSIGN_OR_RETURN((rapidjson::Document parsed_signals), - ParseJsonString(raw_request.bidding_signals())); - PS_VLOG(kStats, log_context) - << "\nTrusted Bidding Signals Deserialize Time: " - << ToInt64Microseconds((absl::Now() - start_parse_time)) - << " microseconds for " << raw_request.bidding_signals().size() - << " bytes."; - - // Select root key. - if (!parsed_signals.HasMember("keys")) { - PS_VLOG(kNoisyWarn, log_context) - << "Trusted bidding signals JSON validate error (Missing property " - "\"keys\")"; - - return absl::InvalidArgumentError( - "Malformed trusted bidding signals (Missing property \"keys\")"); - } - rapidjson::Value& bidding_signals_obj = parsed_signals["keys"]; - - // Create IG -> TrustedBiddingSignals Map. - TrustedBiddingSignalsByIg per_ig_signals_map; - long avg_signal_size_per_ig = raw_request.bidding_signals().size() / - raw_request.interest_group_for_bidding_size(); - for (const auto& ig : raw_request.interest_group_for_bidding()) { - per_ig_signals_map.try_emplace( - ig.name(), - GetSignalsForIG(ig, &bidding_signals_obj, avg_signal_size_per_ig)); - } - return per_ig_signals_map; -} - // Builds a vector containing commonly shared inputs, following the description // here: // https://github.com/privacysandbox/fledge-docs/blob/main/bidding_auction_services_api.md#generatebids @@ -286,7 +193,6 @@ std::vector> BuildBaseInput( absl::StatusOr BuildGenerateBidRequest( IGForBidding& interest_group, const RawRequest& raw_request, const std::vector>& base_input, - const TrustedBiddingSignalsByIg& ig_trusted_signals_map, const bool enable_buyer_debug_url_generation, RequestLogContext& log_context, const bool enable_adtech_code_logging, const std::string& version) { @@ -298,28 +204,11 @@ absl::StatusOr BuildGenerateBidRequest( // Copy base input and amend with custom interest_group generate_bid_request.input = base_input; - // IG must have trusted bidding signals to participate in Bidding. - const auto& trusted_bidding_signals_itr = - ig_trusted_signals_map.find(interest_group.name()); - if (trusted_bidding_signals_itr == ig_trusted_signals_map.end()) { - return absl::InvalidArgumentError( - "Interest Group must contain non-empty trusted bidding " - "signals to generate bids."); - } - - if (!trusted_bidding_signals_itr->second.ok()) { - return trusted_bidding_signals_itr->second.status(); - } - - if (trusted_bidding_signals_itr->second.value().json->empty()) { - return absl::InvalidArgumentError( - "Interest Group must contain non-empty trusted bidding " - "signals to generate bids."); - } - generate_bid_request .input[ArgIndex(GenerateBidArgs::kTrustedBiddingSignals)] = - trusted_bidding_signals_itr->second.value().json; + std::make_shared( + std::move(*interest_group.mutable_trusted_bidding_signals())); + interest_group.clear_trusted_bidding_signals(); // IG must have device signals to participate in Bidding. google::protobuf::util::MessageDifferencer differencer; @@ -334,7 +223,7 @@ absl::StatusOr BuildGenerateBidRequest( } else if (interest_group.has_android_signals() && interest_group.android_signals().IsInitialized() && !differencer.Equals(AndroidSignals::default_instance(), - interest_group.browser_signals())) { + interest_group.android_signals())) { PS_ASSIGN_OR_RETURN(std::string serialized_android_signals, ProtoToJson(interest_group.android_signals())); generate_bid_request.input[ArgIndex(GenerateBidArgs::kDeviceSignals)] = @@ -353,11 +242,6 @@ absl::StatusOr BuildGenerateBidRequest( generate_bid_request.handler_name = kDispatchHandlerFunctionNameWithCodeWrapper; - // Only add parsed keys. - interest_group.clear_trusted_bidding_signals_keys(); - interest_group.mutable_trusted_bidding_signals_keys()->Add( - trusted_bidding_signals_itr->second.value().keys.begin(), - trusted_bidding_signals_itr->second.value().keys.end()); auto start_parse_time = absl::Now(); auto serialized_ig = SerializeIG(interest_group); if (!serialized_ig.ok()) { @@ -372,6 +256,7 @@ absl::StatusOr BuildGenerateBidRequest( << generate_bid_request.input[ArgIndex(GenerateBidArgs::kInterestGroup)] ->size() << " bytes."; + if (server_common::log::PS_VLOG_IS_ON(10)) { PS_VLOG(10, log_context) << "\n\nGenerateBid Input Args:"; for (const auto& it : generate_bid_request.input) { @@ -453,25 +338,21 @@ void GenerateBidsReactor::Execute() { } benchmarking_logger_->BuildInputBegin(); - PS_VLOG(kEncrypted, log_context_) << "Encrypted GenerateBidsRequest:\n" - << request_->ShortDebugString(); + PS_VLOG(kEncrypted, log_context_) + << "Encrypted GenerateBidsRequest exported in EventMessage"; log_context_.SetEventMessageField(*request_); - PS_VLOG(kPlain, log_context_) << "GenerateBidsRawRequest:\n" - << raw_request_.ShortDebugString(); + PS_VLOG(kPlain, log_context_) + << "GenerateBidsRawRequest exported in EventMessage"; log_context_.SetEventMessageField(raw_request_); auto interest_groups = raw_request_.interest_group_for_bidding(); - - // Parse trusted bidding signals - absl::StatusOr ig_trusted_signals_map = - DeserializeAndGetTrustedBiddingSignalsPerIG(raw_request_, log_context_); - if (!ig_trusted_signals_map.ok()) { - PS_LOG(ERROR, log_context_) - << "Request failed while parsing bidding signals: " - << ig_trusted_signals_map.status().ToString( - absl::StatusToStringMode::kWithEverything); + if (interest_groups.empty()) { + // This is unlikely to happen since we already have this check in place + // in BFE. + PS_LOG(ERROR, log_context_) << "Request has no interest groups."; EncryptResponseAndFinish( - server_common::FromAbslStatus(ig_trusted_signals_map.status())); + server_common::FromAbslStatus(absl::InvalidArgumentError( + "Request must have at least one interest group."))); return; } @@ -481,7 +362,6 @@ void GenerateBidsReactor::Execute() { for (int i = 0; i < interest_groups.size(); i++) { absl::StatusOr generate_bid_request = BuildGenerateBidRequest(interest_groups.at(i), raw_request_, base_input, - ig_trusted_signals_map.value(), enable_buyer_debug_url_generation_, log_context_, enable_adtech_code_logging_, protected_auction_generate_bid_version_); @@ -659,10 +539,11 @@ void GenerateBidsReactor::GenerateBidsCallback( } void GenerateBidsReactor::EncryptResponseAndFinish(grpc::Status status) { - PS_VLOG(kPlain, log_context_) << "GenerateBidsRawResponse:\n" - << raw_response_.ShortDebugString(); + PS_VLOG(kPlain, log_context_) + << "GenerateBidsRawResponse exported in EventMessage"; log_context_.SetEventMessageField(raw_response_); - log_context_.ExportEventMessage(); + // ExportEventMessage before encrypt response + log_context_.ExportEventMessage(/*if_export_consented=*/true); if (!EncryptResponse()) { PS_LOG(ERROR, log_context_) diff --git a/services/bidding_service/generate_bids_reactor_test.cc b/services/bidding_service/generate_bids_reactor_test.cc index 6a09e850..1b444b78 100644 --- a/services/bidding_service/generate_bids_reactor_test.cc +++ b/services/bidding_service/generate_bids_reactor_test.cc @@ -44,8 +44,8 @@ namespace privacy_sandbox::bidding_auction_servers { namespace { // Bidding signals must be contained in "keys" in root object. -constexpr char kTestBiddingSignals[] = - R"json({"keys":{"trusted_bidding_signal_key": "some_trusted_bidding_signal_value"}})json"; +constexpr char kTestTrustedBiddingSignals[] = + R"json({"trusted_bidding_signal_key": "some_trusted_bidding_signal_value"})json"; constexpr char kTopLevelSeller[] = "https://www.example-top-ssp.com"; constexpr char kTestConsentToken[] = "testConsentToken"; @@ -271,7 +271,7 @@ IGForBidding GetIGForBiddingFoo() { interest_group.set_user_bidding_signals(kUserBiddingSignals); interest_group.mutable_trusted_bidding_signals_keys()->Add( "trusted_bidding_signal_key"); - + interest_group.set_trusted_bidding_signals(kTestTrustedBiddingSignals); interest_group.mutable_ad_render_ids()->Add("1689"); interest_group.mutable_ad_render_ids()->Add("1776"); @@ -295,6 +295,7 @@ IGForBidding GetIGForBiddingBar(bool make_browser_signals = true) { interest_group.set_user_bidding_signals(kUserBiddingSignals); interest_group.mutable_trusted_bidding_signals_keys()->Add( "trusted_bidding_signal_key"); + interest_group.set_trusted_bidding_signals(kTestTrustedBiddingSignals); interest_group.mutable_ad_render_ids()->Add("1868"); interest_group.mutable_ad_render_ids()->Add("1954"); @@ -321,10 +322,8 @@ void CheckForAndReplaceUBSWithEmptyString( EXPECT_NE(index_of_ubs, std::string::npos); // UBS will not deserialize into a string (hence the custom serialization // logic, so we excise it from the string before going back to a message. - ABSL_LOG(INFO) << "\nDebugging test: Before:\n" << serialized_ig; serialized_ig.replace(index_of_ubs, user_bidding_signals.length(), R"JSON("")JSON"); - ABSL_LOG(INFO) << "\nDebugging test: After:\n" << serialized_ig; } void CheckCorrectnessOfIg(std::string& serialized_actual, @@ -336,9 +335,11 @@ void CheckCorrectnessOfIg(std::string& serialized_actual, CHECK_OK(google::protobuf::util::JsonStringToMessage( serialized_actual, &reconstituted_actual_ig)) << "Could not reconstitute IG: " << serialized_actual; - // Expected IG needs device signals cleared since they will not be in - // the actual bar. + // Expected IG needs trusted bidding signals and device signals cleared since + // they will not be in the actual bar. These are not passed as part of the + // serialized IG, but as separate parameters to GenerateBid. expected.clear_DeviceSignals(); + expected.clear_trusted_bidding_signals(); // Since UBS will not be equal after re-serialization, clear those as well in // both. reconstituted_actual_ig.clear_user_bidding_signals(); @@ -359,8 +360,7 @@ void CheckCorrectnessOfIg(std::string& serialized_actual, void BuildRawRequest(const std::vector& interest_groups_to_add, absl::string_view auction_signals, - absl::string_view buyer_signals, - absl::string_view bidding_signals, RawRequest& raw_request, + absl::string_view buyer_signals, RawRequest& raw_request, bool enable_debug_reporting = false, bool enable_adtech_code_logging = false) { for (int i = 0; i < interest_groups_to_add.size(); i++) { @@ -369,7 +369,6 @@ void BuildRawRequest(const std::vector& interest_groups_to_add, } raw_request.set_auction_signals(auction_signals); raw_request.set_buyer_signals(buyer_signals); - raw_request.set_bidding_signals(bidding_signals); raw_request.set_enable_debug_reporting(enable_debug_reporting); raw_request.set_seller(kSeller); raw_request.set_publisher_name(kPublisherName); @@ -382,41 +381,12 @@ void BuildRawRequest(const std::vector& interest_groups_to_add, void BuildRawRequestForComponentAuction( const std::vector& interest_groups_to_add, absl::string_view auction_signals, absl::string_view buyer_signals, - absl::string_view bidding_signals, RawRequest& raw_request, - bool enable_debug_reporting = false) { + RawRequest& raw_request, bool enable_debug_reporting = false) { BuildRawRequest(interest_groups_to_add, auction_signals, buyer_signals, - bidding_signals, raw_request, enable_debug_reporting); + raw_request, enable_debug_reporting); raw_request.set_top_level_seller(kTopLevelSeller); } -TEST_F(GenerateBidsReactorTest, DoesNotGenerateBidsForIGWithNoAds) { - Response ads; - RawRequest raw_request; - IGForBidding baz; - baz.set_name("baz"); - std::vector igs; - igs.push_back(std::move(baz)); - BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, - kTestBiddingSignals, raw_request); - - EXPECT_CALL(dispatcher_, BatchExecute).Times(0); - CheckGenerateBids(raw_request, ads); -} - -TEST_F(GenerateBidsReactorTest, DoesNotGenerateBidsForIGWithNoBiddingSignals) { - Response ads; - RawRequest raw_request; - auto ig_for_bidding_foo = GetIGForBiddingFoo(); - ig_for_bidding_foo.mutable_trusted_bidding_signals_keys()->Clear(); - std::vector igs; - igs.push_back(ig_for_bidding_foo); - BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, - kTestBiddingSignals, raw_request); - - EXPECT_CALL(dispatcher_, BatchExecute).Times(0); - CheckGenerateBids(raw_request, ads); -} - TEST_F(GenerateBidsReactorTest, GenerateBidSuccessfulWithCodeWrapper) { bool enable_debug_reporting = false; bool enable_buyer_debug_url_generation = false; @@ -440,9 +410,8 @@ TEST_F(GenerateBidsReactorTest, GenerateBidSuccessfulWithCodeWrapper) { return FakeExecute(batch, std::move(batch_callback), response_json); }); RawRequest raw_request; - BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, - kTestBiddingSignals, raw_request, enable_debug_reporting, - enable_adtech_code_logging); + BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, raw_request, + enable_debug_reporting, enable_adtech_code_logging); CheckGenerateBids(raw_request, ads, enable_buyer_debug_url_generation); } @@ -474,9 +443,8 @@ TEST_F(GenerateBidsReactorTest, PrivateAggregationObjectSetInResponse) { return FakeExecute(batch, std::move(batch_callback), response_json); }); RawRequest raw_request; - BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, - kTestBiddingSignals, raw_request, enable_debug_reporting, - enable_adtech_code_logging); + BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, raw_request, + enable_debug_reporting, enable_adtech_code_logging); CheckGenerateBids(raw_request, ads, enable_buyer_debug_url_generation); } @@ -504,9 +472,8 @@ TEST_F(GenerateBidsReactorTest, BuyerReportingIdSetInResponse) { return FakeExecute(batch, std::move(batch_callback), response_json); }); RawRequest raw_request; - BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, - kTestBiddingSignals, raw_request, enable_debug_reporting, - enable_adtech_code_logging); + BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, raw_request, + enable_debug_reporting, enable_adtech_code_logging); CheckGenerateBids(raw_request, ads, enable_buyer_debug_url_generation); } @@ -533,25 +500,21 @@ TEST_F(GenerateBidsReactorTest, UnknownFieldInResponseParsedSuccessfully) { return FakeExecute(batch, std::move(batch_callback), response_json); }); RawRequest raw_request; - BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, - kTestBiddingSignals, raw_request, enable_debug_reporting, - enable_adtech_code_logging); + BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, raw_request, + enable_debug_reporting, enable_adtech_code_logging); CheckGenerateBids(raw_request, ads, enable_buyer_debug_url_generation); } -TEST_F(GenerateBidsReactorTest, - DoesNotGenerateBidsIfBiddingSignalsAreMalformed) { - auto in_signals = R"JSON({"ig_name_Bar":1})JSON"; +TEST_F(GenerateBidsReactorTest, DoesNotValidateBiddingSignalsStructure) { Response ads; RawRequest raw_request; - IGForBidding foo, bar; + IGForBidding foo = GetIGForBiddingFoo(); + foo.set_trusted_bidding_signals("Invalid JSON"); std::vector igs; - igs.push_back(GetIGForBiddingFoo()); - igs.push_back(GetIGForBiddingBar()); - BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, in_signals, - raw_request); + igs.push_back(foo); + BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, raw_request); - EXPECT_CALL(dispatcher_, BatchExecute).Times(0); + EXPECT_CALL(dispatcher_, BatchExecute).Times(igs.size()); CheckGenerateBids(raw_request, ads); } @@ -574,8 +537,7 @@ TEST_F(GenerateBidsReactorTest, GeneratesBidForSingleIGForBidding) { RawRequest raw_request; std::vector igs; igs.push_back(GetIGForBiddingFoo()); - BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, - kTestBiddingSignals, raw_request); + BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, raw_request); CheckGenerateBids(raw_request, ads); } @@ -613,8 +575,7 @@ TEST_F(GenerateBidsReactorTest, IGSerializationLatencyBenchmark) { }); RawRequest raw_request; - BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, - kTestBiddingSignals, raw_request); + BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, raw_request); CheckGenerateBids(raw_request, ads); } @@ -648,8 +609,7 @@ TEST_F(GenerateBidsReactorTest, GeneratesBidsForMultipleIGForBiddings) { std::vector igs; igs.push_back(GetIGForBiddingBar()); igs.push_back(GetIGForBiddingFoo()); - BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, - kTestBiddingSignals, raw_request); + BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, raw_request); CheckGenerateBids(raw_request, ads); } @@ -689,8 +649,7 @@ TEST_F(GenerateBidsReactorTest, FiltersBidsWithZeroBidPrice) { std::vector igs; igs.push_back(GetIGForBiddingBar()); igs.push_back(GetIGForBiddingFoo()); - BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, - kTestBiddingSignals, raw_request); + BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, raw_request); CheckGenerateBids(raw_request, ads); } @@ -717,16 +676,13 @@ TEST_F(GenerateBidsReactorTest, CreatesGenerateBidInputsInCorrectOrder) { CheckCorrectnessOfIg(*input[0], GetIGForBiddingBar()); EXPECT_EQ(*input[1], R"JSON({"auction_signal": "test 1"})JSON"); EXPECT_EQ(*input[2], R"JSON({"buyer_signal": "test 2"})JSON"); - EXPECT_EQ( - *input[3], - R"JSON({"trusted_bidding_signal_key":"some_trusted_bidding_signal_value"})JSON"); + EXPECT_EQ(*input[3], kTestTrustedBiddingSignals); EXPECT_EQ(*input[4], bar_browser_signals); } return FakeExecute(batch, std::move(batch_callback), response_json); }); RawRequest raw_request; - BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, - kTestBiddingSignals, raw_request); + BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, raw_request); CheckGenerateBids(raw_request, ads); } @@ -756,16 +712,13 @@ TEST_F(GenerateBidsReactorTest, CheckCorrectnessOfIg(*input[0], GetIGForBiddingBar()); EXPECT_EQ(*input[1], R"JSON({"auction_signal": "test 1"})JSON"); EXPECT_EQ(*input[2], R"JSON({"buyer_signal": "test 2"})JSON"); - EXPECT_EQ( - *input[3], - R"JSON({"trusted_bidding_signal_key":"some_trusted_bidding_signal_value"})JSON"); + EXPECT_EQ(*input[3], kTestTrustedBiddingSignals); EXPECT_EQ(*input[4], kExpectedBrowserSignalsWithRecencyMs); } return FakeExecute(batch, std::move(batch_callback), response_json); }); RawRequest raw_request; - BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, - kTestBiddingSignals, raw_request); + BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, raw_request); CheckGenerateBids(raw_request, ads); } @@ -800,17 +753,15 @@ TEST_F(GenerateBidsReactorTest, R"JSON({"auction_signal": "test 1"})JSON"); EXPECT_EQ(*input[ArgIndex(GenerateBidArgs::kBuyerSignals)], R"JSON({"buyer_signal": "test 2"})JSON"); - EXPECT_EQ( - *input[ArgIndex(GenerateBidArgs::kTrustedBiddingSignals)], - R"JSON({"trusted_bidding_signal_key":"some_trusted_bidding_signal_value"})JSON"); + EXPECT_EQ(*input[ArgIndex(GenerateBidArgs::kTrustedBiddingSignals)], + kTestTrustedBiddingSignals); EXPECT_EQ(*input[ArgIndex(GenerateBidArgs::kDeviceSignals)], kComponentBrowserSignals); return FakeExecute(batch, std::move(batch_callback), json); }); RawRequest raw_request; BuildRawRequestForComponentAuction(igs, kTestAuctionSignals, - kTestBuyerSignals, kTestBiddingSignals, - raw_request); + kTestBuyerSignals, raw_request); CheckGenerateBids(raw_request, ads); } @@ -846,17 +797,15 @@ TEST_F(GenerateBidsReactorTest, R"JSON({"auction_signal": "test 1"})JSON"); EXPECT_EQ(*input[ArgIndex(GenerateBidArgs::kBuyerSignals)], R"JSON({"buyer_signal": "test 2"})JSON"); - EXPECT_EQ( - *input[ArgIndex(GenerateBidArgs::kTrustedBiddingSignals)], - R"JSON({"trusted_bidding_signal_key":"some_trusted_bidding_signal_value"})JSON"); + EXPECT_EQ(*input[ArgIndex(GenerateBidArgs::kTrustedBiddingSignals)], + kTestTrustedBiddingSignals); EXPECT_EQ(*input[ArgIndex(GenerateBidArgs::kDeviceSignals)], kComponentBrowserSignalsWithRecencyMs); return FakeExecute(batch, std::move(batch_callback), json); }); RawRequest raw_request; BuildRawRequestForComponentAuction(igs, kTestAuctionSignals, - kTestBuyerSignals, kTestBiddingSignals, - raw_request); + kTestBuyerSignals, raw_request); CheckGenerateBids(raw_request, ads); } @@ -883,8 +832,7 @@ TEST_F(GenerateBidsReactorTest, }); RawRequest raw_request; BuildRawRequestForComponentAuction(igs, kTestAuctionSignals, - kTestBuyerSignals, kTestBiddingSignals, - raw_request); + kTestBuyerSignals, raw_request); CheckGenerateBids(raw_request, ads); } @@ -904,53 +852,13 @@ TEST_F(GenerateBidsReactorTest, SkipsUnallowedAdForComponentAuction) { }); RawRequest raw_request; BuildRawRequestForComponentAuction(igs, kTestAuctionSignals, - kTestBuyerSignals, kTestBiddingSignals, - raw_request); + kTestBuyerSignals, raw_request); CheckGenerateBids(raw_request, ads); } -TEST_F(GenerateBidsReactorTest, - GeneratesBidAndBuildsSignalsForIGNameBiddingSignals) { - auto in_signals = - R"JSON({"keys": {"trusted_bidding_signal_key":{"someKey":"someValue"}}, "perInterestGroupData":{"ig_name_Bar":{"priorityVector":{"NotYet":"Supported"}}}})JSON"; - auto expected_signals = - R"JSON({"trusted_bidding_signal_key":{"someKey":"someValue"}})JSON"; - - const float bid_val = 1.234; - AdWithBid bid; - bid.set_render("https://adTech.com/ad?id=123"); - bid.set_bid(bid_val); - bid.set_interest_group_name("ig_name_Bar"); - Response ads; - GenerateBidsResponse::GenerateBidsRawResponse raw_response; - *raw_response.add_bids() = bid; - *ads.mutable_response_ciphertext() = raw_response.SerializeAsString(); - std::vector igs{GetIGForBiddingBar()}; - - std::string response_json = GetTestResponse(kTestRenderUrl, bid_val); - EXPECT_CALL(dispatcher_, BatchExecute) - .WillOnce([response_json, expected_signals]( - std::vector& batch, - BatchDispatchDoneCallback batch_callback) { - auto input = batch.at(0).input; - IGForBidding received; - EXPECT_EQ(*input[3], expected_signals); - return FakeExecute(batch, std::move(batch_callback), response_json); - }); - RawRequest raw_request; - BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, in_signals, - raw_request); - CheckGenerateBids(raw_request, ads, false); -} - // TODO (b/288954720): Once android signals message is defined and signals are -// required, change this test to expect to fail. +// required, change this test to expect to fail. TEST_F(GenerateBidsReactorTest, GeneratesBidDespiteNoBrowserSignals) { - auto in_signals = - R"JSON({"keys": {"trusted_bidding_signal_key":{"someKey":"someValue"}}, "perInterestGroupData":{"ig_name_Bar":{"priorityVector":{"NotYet":"Supported"}}}})JSON"; - auto expected_signals = - R"JSON({"trusted_bidding_signal_key":{"someKey":"someValue"}})JSON"; - AdWithBid bid; bid.set_render("https://adTech.com/ad?id=123"); bid.set_bid(1); @@ -964,56 +872,16 @@ TEST_F(GenerateBidsReactorTest, GeneratesBidDespiteNoBrowserSignals) { std::string response_json = GetTestResponse(kTestRenderUrl, 1); EXPECT_CALL(dispatcher_, BatchExecute) - .WillOnce([response_json, expected_signals]( - std::vector& batch, - BatchDispatchDoneCallback batch_callback) { + .WillOnce([response_json](std::vector& batch, + BatchDispatchDoneCallback batch_callback) { auto input = batch.at(0).input; IGForBidding received; - EXPECT_EQ(*input[3], expected_signals); // Check that device signals are an empty JSON object. EXPECT_EQ(*input[4], R"JSON({})JSON"); return FakeExecute(batch, std::move(batch_callback), response_json); }); RawRequest raw_request; - BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, in_signals, - raw_request); - CheckGenerateBids(raw_request, ads, false); -} - -TEST_F(GenerateBidsReactorTest, HandlesDuplicateKeysInBiddingSignalsKeys) { - auto in_signals = - R"JSON({"keys": {"trusted_bidding_signal_key":{"someKey":"someValue"}}, "perInterestGroupData":{"ig_name_Bar":{"priorityVector":{"NotYet":"Supported"}}}})JSON"; - auto expected_signals = - R"JSON({"trusted_bidding_signal_key":{"someKey":"someValue"}})JSON"; - - AdWithBid bid; - bid.set_render(kTestRenderUrl); - bid.set_bid(1); - bid.set_interest_group_name("ig_name_Bar"); - Response ads; - GenerateBidsResponse::GenerateBidsRawResponse raw_response; - *raw_response.add_bids() = bid; - *ads.mutable_response_ciphertext() = raw_response.SerializeAsString(); - std::vector igs; - auto bar = GetIGForBiddingBar(); - // Add key to bidding signals keys two more times (duplicate lookup). - bar.add_trusted_bidding_signals_keys("trusted_bidding_signal_key"); - bar.add_trusted_bidding_signals_keys("trusted_bidding_signal_key"); - igs.push_back(std::move(bar)); - - std::string response_json = GetTestResponse(kTestRenderUrl, 1); - EXPECT_CALL(dispatcher_, BatchExecute) - .WillOnce([response_json, expected_signals]( - std::vector& batch, - BatchDispatchDoneCallback batch_callback) { - auto input = batch.at(0).input; - IGForBidding received; - EXPECT_EQ(*input[3], expected_signals); - return FakeExecute(batch, std::move(batch_callback), response_json); - }); - RawRequest raw_request; - BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, in_signals, - raw_request); + BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, raw_request); CheckGenerateBids(raw_request, ads, false); } @@ -1053,8 +921,8 @@ TEST_F(GenerateBidsReactorTest, GenerateBidResponseWithDebugUrls) { return FakeExecute(batch, std::move(batch_callback), response_json); }); RawRequest raw_request; - BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, - kTestBiddingSignals, raw_request, enable_debug_reporting); + BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, raw_request, + enable_debug_reporting); CheckGenerateBids(raw_request, ads, enable_buyer_debug_url_generation); } @@ -1080,8 +948,8 @@ TEST_F(GenerateBidsReactorTest, GenerateBidResponseWithoutDebugUrls) { return FakeExecute(batch, std::move(batch_callback), response_json); }); RawRequest raw_request; - BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, - kTestBiddingSignals, raw_request, enable_debug_reporting); + BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, raw_request, + enable_debug_reporting); CheckGenerateBids(raw_request, ads, enable_buyer_debug_url_generation); } @@ -1091,8 +959,7 @@ TEST_F(GenerateBidsReactorTest, AddsTrustedBiddingSignalsKeysToScriptInput) { std::vector igs; igs.push_back(GetIGForBiddingFoo()); std::string response_json = GetTestResponse(kTestRenderUrl, 1); - BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, - kTestBiddingSignals, raw_request); + BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, raw_request); *request_.mutable_request_ciphertext() = raw_request.SerializeAsString(); absl::Notification notification; // Verify that serialized IG contains trustedBiddingSignalKeys. @@ -1138,8 +1005,7 @@ TEST_F(GenerateBidsReactorTest, RawRequest raw_request; std::vector igs; igs.push_back(GetIGForBiddingFoo()); - BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, - kTestBiddingSignals, raw_request); + BuildRawRequest(igs, kTestAuctionSignals, kTestBuyerSignals, raw_request); request_.set_request_ciphertext(raw_request.SerializeAsString()); absl::Notification notification; diff --git a/services/bidding_service/inference/BUILD b/services/bidding_service/inference/BUILD index 39a63812..31003ab6 100644 --- a/services/bidding_service/inference/BUILD +++ b/services/bidding_service/inference/BUILD @@ -45,6 +45,7 @@ cc_library( "@inference_common//sandbox:sandbox_executor", "@inference_common//utils:error", "@inference_common//utils:file_util", + "@inference_common//utils:inference_error_code", "@rapidjson", ], ) @@ -93,6 +94,7 @@ cc_library( visibility = ["//services/bidding_service:__subpackages__"], deps = [ ":inference_utils", + ":model_fetcher_metric", "//services/common/blob_fetch:blob_fetcher_base", "//services/common/data_fetch:fetcher_interface", "@com_github_grpc_grpc//:grpc++", @@ -104,6 +106,7 @@ cc_library( "@google_privacysandbox_servers_common//src/concurrent:executor", "@google_privacysandbox_servers_common//src/logger:request_context_logger", "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", + "@google_privacysandbox_servers_common//src/util/status_macro:status_util", "@inference_common//proto:inference_sidecar_cc_grpc_proto", "@inference_common//proto:inference_sidecar_cc_proto", "@inference_common//proto:model_metadata_cc_proto", @@ -125,3 +128,28 @@ cc_test( "@inference_common//proto:inference_sidecar_cc_grpc_proto", ], ) + +cc_library( + name = "model_fetcher_metric", + hdrs = ["model_fetcher_metric.h"], + visibility = ["//services/bidding_service:__subpackages__"], + deps = [ + "//services/common/metric:server_definition", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/synchronization", + ], +) + +cc_test( + name = "model_fetcher_metric_test", + size = "small", + srcs = ["model_fetcher_metric_test.cc"], + deps = [ + ":model_fetcher_metric", + "@com_google_absl//absl/status", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/services/bidding_service/inference/inference_utils.cc b/services/bidding_service/inference/inference_utils.cc index 635b6d1a..779b0e7d 100644 --- a/services/bidding_service/inference/inference_utils.cc +++ b/services/bidding_service/inference/inference_utils.cc @@ -48,6 +48,7 @@ #include "src/util/status_macro/status_util.h" #include "utils/error.h" #include "utils/file_util.h" +#include "utils/inference_error_code.h" namespace privacy_sandbox::bidding_auction_servers::inference { @@ -72,6 +73,26 @@ void LogMetrics( log_status = metric_context->AccumulateMetric( metric_value.value()); + } else if (key == "kInferenceErrorCountByErrorCode") { + log_status = + metric_context + ->AccumulateMetric( + metric_value.value(), metric_value.partition()); + } else if (key == "kInferenceRequestCountByModel") { + log_status = + metric_context + ->AccumulateMetric( + metric_value.value(), metric_value.partition()); + } else if (key == "kInferenceRequestDurationByModel") { + log_status = + metric_context + ->AccumulateMetric( + metric_value.value(), metric_value.partition()); + } else if (key == "kInferenceRequestFailedCountByModel") { + log_status = + metric_context + ->AccumulateMetric( + metric_value.value(), metric_value.partition()); } else { log_status = absl::NotFoundError("Unrecognized metric key: " + key); } diff --git a/services/bidding_service/inference/model_fetcher_metric.h b/services/bidding_service/inference/model_fetcher_metric.h new file mode 100644 index 00000000..cb1bd6a6 --- /dev/null +++ b/services/bidding_service/inference/model_fetcher_metric.h @@ -0,0 +1,143 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SERVICES_BIDDING_SERVICE_INFERENCE_MODEL_FETCHER_METRIC_H_ +#define SERVICES_BIDDING_SERVICE_INFERENCE_MODEL_FETCHER_METRIC_H_ + +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/status/status.h" +#include "absl/synchronization/mutex.h" +#include "services/common/metric/server_definition.h" + +namespace privacy_sandbox::bidding_auction_servers::inference { + +// Class that records metrics associated with the inference model fetcher. +// It keeps track of metrics such as cloud fetch success counts, cloud fetch +// failure counts, model registration success counts, and model registration +// failure counts. +// These metrics are not in the critical request path. +class ModelFetcherMetric { + public: + static absl::flat_hash_map GetCloudFetchSuccessCount() + ABSL_LOCKS_EXCLUDED(mu_) { + absl::MutexLock lock(&mu_); + return absl::flat_hash_map{ + {"cloud fetch", cloud_fetch_success_count_}}; + } + + static absl::flat_hash_map + GetCloudFetchFailedCountByStatus() ABSL_LOCKS_EXCLUDED(mu_) { + absl::MutexLock lock(&mu_); + return cloud_fetch_failure_count_by_error_code_; + } + + static absl::flat_hash_map + GetRecentModelRegistrationSuccess() ABSL_LOCKS_EXCLUDED(mu_) { + absl::MutexLock lock(&mu_); + return recent_model_registration_success_; + } + + static absl::flat_hash_map + GetRecentModelRegistrationFailure() ABSL_LOCKS_EXCLUDED(mu_) { + absl::MutexLock lock(&mu_); + return recent_model_registration_failure_; + } + + static absl::flat_hash_map + GetModelRegistrationFailedCountByStatus() ABSL_LOCKS_EXCLUDED(mu_) { + absl::MutexLock lock(&mu_); + return model_registration_failure_count_by_error_code_; + } + + static void IncrementCloudFetchFailedCountByStatus( + absl::StatusCode error_code) ABSL_LOCKS_EXCLUDED(mu_) { + absl::MutexLock lock(&mu_); + ++cloud_fetch_failure_count_by_error_code_[absl::StatusCodeToString( + error_code)]; + } + + static void IncrementCloudFetchSuccessCount() ABSL_LOCKS_EXCLUDED(mu_) { + absl::MutexLock lock(&mu_); + ++cloud_fetch_success_count_; + } + + static void UpdateRecentModelRegistrationSuccess( + const std::vector& models) ABSL_LOCKS_EXCLUDED(mu_) { + absl::MutexLock lock(&mu_); + recent_model_registration_success_.clear(); + for (const auto& model : models) { + ++recent_model_registration_success_[model]; + } + } + + static void UpdateRecentModelRegistrationFailure( + const std::vector& models) ABSL_LOCKS_EXCLUDED(mu_) { + absl::MutexLock lock(&mu_); + recent_model_registration_failure_.clear(); + for (const auto& model : models) { + ++recent_model_registration_failure_[model]; + } + } + + static void IncrementModelRegistrationFailedCountByStatus( + absl::StatusCode error_code) ABSL_LOCKS_EXCLUDED(mu_) { + absl::MutexLock lock(&mu_); + ++model_registration_failure_count_by_error_code_[absl::StatusCodeToString( + error_code)]; + } + + private: + ABSL_CONST_INIT static inline absl::Mutex mu_{absl::kConstInit}; + + static inline int cloud_fetch_success_count_ ABSL_GUARDED_BY(mu_){0}; + + static inline absl::flat_hash_map + cloud_fetch_failure_count_by_error_code_ ABSL_GUARDED_BY(mu_){}; + + static inline absl::flat_hash_map + recent_model_registration_success_ ABSL_GUARDED_BY(mu_){}; + + static inline absl::flat_hash_map + recent_model_registration_failure_ ABSL_GUARDED_BY(mu_){}; + + static inline absl::flat_hash_map + model_registration_failure_count_by_error_code_ ABSL_GUARDED_BY(mu_){}; +}; + +// Adds model fetcher metrics to a metric context map to bidding server. +inline absl::Status AddModelFetcherMetricToBidding() { + auto* context_map = metric::BiddingContextMap(); + PS_RETURN_IF_ERROR(context_map->AddObserverable( + metric::kInferenceCloudFetchSuccessCount, + inference::ModelFetcherMetric::GetCloudFetchSuccessCount)); + PS_RETURN_IF_ERROR(context_map->AddObserverable( + metric::kInferenceCloudFetchFailedCountByStatus, + inference::ModelFetcherMetric::GetCloudFetchFailedCountByStatus)); + PS_RETURN_IF_ERROR(context_map->AddObserverable( + metric::kInferenceRecentModelRegistrationSuccess, + inference::ModelFetcherMetric::GetRecentModelRegistrationSuccess)); + PS_RETURN_IF_ERROR(context_map->AddObserverable( + metric::kInferenceRecentModelRegistrationFailure, + inference::ModelFetcherMetric::GetRecentModelRegistrationFailure)); + return context_map->AddObserverable( + metric::kInferenceModelRegistrationFailedCountByStatus, + inference::ModelFetcherMetric::GetModelRegistrationFailedCountByStatus); +} + +} // namespace privacy_sandbox::bidding_auction_servers::inference + +#endif // SERVICES_BIDDING_SERVICE_INFERENCE_MODEL_FETCHER_METRIC_H_ diff --git a/services/bidding_service/inference/model_fetcher_metric_test.cc b/services/bidding_service/inference/model_fetcher_metric_test.cc new file mode 100644 index 00000000..7ed8cc7b --- /dev/null +++ b/services/bidding_service/inference/model_fetcher_metric_test.cc @@ -0,0 +1,112 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "services/bidding_service/inference/model_fetcher_metric.h" + +#include +#include + +#include "absl/status/status.h" + +namespace privacy_sandbox::bidding_auction_servers::inference { +namespace { + +using ::testing::Pair; +using ::testing::UnorderedElementsAre; + +TEST(ModelFetcherMetricTest, GetCloudFetchSuccessCount) { + EXPECT_THAT(ModelFetcherMetric::GetCloudFetchSuccessCount(), + UnorderedElementsAre(Pair("cloud fetch", 0))); + + ModelFetcherMetric::IncrementCloudFetchSuccessCount(); + EXPECT_THAT(ModelFetcherMetric::GetCloudFetchSuccessCount(), + UnorderedElementsAre(Pair("cloud fetch", 1))); +} + +TEST(ModelFetcherMetricTest, GetCloudFetchFailedCountByStatus) { + EXPECT_THAT(ModelFetcherMetric::GetCloudFetchFailedCountByStatus(), + UnorderedElementsAre()); + + ModelFetcherMetric::IncrementCloudFetchFailedCountByStatus( + absl::StatusCode::kUnavailable); + ModelFetcherMetric::IncrementCloudFetchFailedCountByStatus( + absl::StatusCode::kInternal); + + EXPECT_THAT( + ModelFetcherMetric::GetCloudFetchFailedCountByStatus(), + UnorderedElementsAre(Pair("UNAVAILABLE", 1), Pair("INTERNAL", 1))); + + ModelFetcherMetric::IncrementCloudFetchFailedCountByStatus( + absl::StatusCode::kUnavailable); + + EXPECT_THAT( + ModelFetcherMetric::GetCloudFetchFailedCountByStatus(), + UnorderedElementsAre(Pair("UNAVAILABLE", 2), Pair("INTERNAL", 1))); +} + +TEST(ModelFetcherMetricTest, GetRecentModelRegistrationSuccess) { + EXPECT_THAT(ModelFetcherMetric::GetRecentModelRegistrationSuccess(), + UnorderedElementsAre()); + + ModelFetcherMetric::UpdateRecentModelRegistrationSuccess( + {"model1", "model2"}); + + EXPECT_THAT(ModelFetcherMetric::GetRecentModelRegistrationSuccess(), + UnorderedElementsAre(Pair("model1", 1), Pair("model2", 1))); + + ModelFetcherMetric::UpdateRecentModelRegistrationSuccess({"model1"}); + + EXPECT_THAT(ModelFetcherMetric::GetRecentModelRegistrationSuccess(), + UnorderedElementsAre(Pair("model1", 1))); +} + +TEST(ModelFetcherMetricTest, GetRecentModelRegistrationFailure) { + EXPECT_THAT(ModelFetcherMetric::GetRecentModelRegistrationFailure(), + UnorderedElementsAre()); + + ModelFetcherMetric::UpdateRecentModelRegistrationFailure( + {"model1", "model2"}); + + EXPECT_THAT(ModelFetcherMetric::GetRecentModelRegistrationFailure(), + UnorderedElementsAre(Pair("model1", 1), Pair("model2", 1))); + + ModelFetcherMetric::UpdateRecentModelRegistrationFailure({"model1"}); + + EXPECT_THAT(ModelFetcherMetric::GetRecentModelRegistrationFailure(), + UnorderedElementsAre(Pair("model1", 1))); +} + +TEST(ModelFetcherMetricTest, GetModelRegistrationFailedCountByErroCode) { + EXPECT_THAT(ModelFetcherMetric::GetModelRegistrationFailedCountByStatus(), + UnorderedElementsAre()); + + ModelFetcherMetric::IncrementModelRegistrationFailedCountByStatus( + absl::StatusCode::kUnavailable); + ModelFetcherMetric::IncrementModelRegistrationFailedCountByStatus( + absl::StatusCode::kInternal); + + EXPECT_THAT( + ModelFetcherMetric::GetModelRegistrationFailedCountByStatus(), + UnorderedElementsAre(Pair("UNAVAILABLE", 1), Pair("INTERNAL", 1))); + + ModelFetcherMetric::IncrementModelRegistrationFailedCountByStatus( + absl::StatusCode::kUnavailable); + + EXPECT_THAT( + ModelFetcherMetric::GetModelRegistrationFailedCountByStatus(), + UnorderedElementsAre(Pair("UNAVAILABLE", 2), Pair("INTERNAL", 1))); +} + +} // namespace +} // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/bidding_service/inference/periodic_model_fetcher.cc b/services/bidding_service/inference/periodic_model_fetcher.cc index 7aa2c8ec..277ffab9 100644 --- a/services/bidding_service/inference/periodic_model_fetcher.cc +++ b/services/bidding_service/inference/periodic_model_fetcher.cc @@ -35,9 +35,11 @@ #include "proto/inference_sidecar.grpc.pb.h" #include "proto/inference_sidecar.pb.h" #include "proto/model_metadata.pb.h" +#include "services/bidding_service/inference/model_fetcher_metric.h" #include "services/common/blob_fetch/blob_fetcher_base.h" #include "src/logger/request_context_impl.h" #include "src/util/status_macro/status_macros.h" +#include "src/util/status_macro/status_util.h" namespace privacy_sandbox::bidding_auction_servers::inference { @@ -83,15 +85,19 @@ void PeriodicModelFetcher::InternalModelFetchAndRegistration() { absl::StatusOr config = FetchModelConfig(); if (!config.ok()) { PS_LOG(ERROR) << "Failed to fetch model config: " << config.status(); + ModelFetcherMetric::IncrementCloudFetchFailedCountByStatus( + config.status().code()); return; } BlobFetcherBase::FilterOptions filter_options; + std::vector pending_model_metadata; for (const ModelMetadata& metadata : config->model_metadata()) { const std::string& model_path = metadata.model_path(); if (!model_path.empty() && current_models_.find(model_path) == current_models_.end()) { filter_options.included_prefixes.push_back(model_path); + pending_model_metadata.push_back(metadata); } } @@ -103,16 +109,28 @@ void PeriodicModelFetcher::InternalModelFetchAndRegistration() { absl::Status cloud_fetch_status = blob_fetcher_->FetchSync(filter_options); if (!cloud_fetch_status.ok()) { PS_LOG(ERROR) << "Cloud model fetching fails: " << cloud_fetch_status; + ModelFetcherMetric::IncrementCloudFetchFailedCountByStatus( + cloud_fetch_status.code()); return; } + ModelFetcherMetric::IncrementCloudFetchSuccessCount(); + + // Keeps track of models that are registered successfully and those that are + // not for metric purposes. + std::vector success_models; + std::vector failure_models; const std::vector& bucket_snapshot = blob_fetcher_->snapshot(); // A single model can consist of multiple model files and hence data blobs. - for (const ModelMetadata& metadata : config->model_metadata()) { + for (const ModelMetadata& metadata : pending_model_metadata) { const std::string& model_path = metadata.model_path(); RegisterModelRequest request; request.mutable_model_spec()->set_model_path(model_path); + if (!metadata.warm_up_batch_request_json().empty()) { + request.set_warm_up_batch_request_json( + metadata.warm_up_batch_request_json()); + } PS_VLOG(10) << "Start registering model for: " << model_path; std::vector blob_views; @@ -128,8 +146,11 @@ void PeriodicModelFetcher::InternalModelFetchAndRegistration() { absl::StatusOr model_checksum = ComputeChecksumForBlobs(blob_views); if (!model_checksum.ok() || *model_checksum != metadata.checksum()) { - PS_LOG(INFO) << "Model rejected due to incorrect checksum."; - return; + PS_LOG(ERROR) << "Model rejected due to incorrect checksum."; + failure_models.push_back(model_path); + ModelFetcherMetric::IncrementModelRegistrationFailedCountByStatus( + absl::StatusCode::kFailedPrecondition); + continue; } } @@ -140,11 +161,18 @@ void PeriodicModelFetcher::InternalModelFetchAndRegistration() { if (!status.ok()) { PS_LOG(ERROR) << "Registering model failure for: " << model_path; + failure_models.push_back(model_path); + ModelFetcherMetric::IncrementModelRegistrationFailedCountByStatus( + server_common::ToAbslStatus(status).code()); } else { PS_VLOG(10) << "Registering model success for: " << model_path; current_models_.insert(model_path); + success_models.push_back(model_path); } } + + ModelFetcherMetric::UpdateRecentModelRegistrationSuccess(success_models); + ModelFetcherMetric::UpdateRecentModelRegistrationFailure(failure_models); } void PeriodicModelFetcher::InternalPeriodicModelFetchAndRegistration() { diff --git a/services/bidding_service/inference/periodic_model_fetcher_test.cc b/services/bidding_service/inference/periodic_model_fetcher_test.cc index bd2622de..bf8acd7e 100644 --- a/services/bidding_service/inference/periodic_model_fetcher_test.cc +++ b/services/bidding_service/inference/periodic_model_fetcher_test.cc @@ -41,6 +41,7 @@ constexpr char kTestModelContent1[] = "bytes1"; constexpr char kTestModelName2[] = "model2"; constexpr char kTestModelContent2[] = "bytes2"; constexpr char kModelConfigPath[] = "model_metadata_config.json"; +constexpr char kModelWarmUpRequestJson[] = "model_warm_up_request_json"; using ::google::scp::core::test::EqualsProto; using ::testing::_; @@ -415,5 +416,85 @@ TEST(PeriodicModelFetcherTest, CorrectChecksumShouldRegisterModel) { model_fetcher.End(); } +TEST(PeriodicModelFetcherTest, RegisterModelWithWarmUpRequestIfProvided) { + const std::string model_metadata_config = R"({ + "model_metadata": [ + {"model_path": "model1"}, + {"model_path": "model2", "warm_up_batch_request_json": "model_warm_up_request_json"} + ] + })"; + + const std::vector config_fetch_result = { + BlobFetcherBase::Blob(kModelConfigPath, model_metadata_config)}; + + const std::vector mock_snapshot = { + BlobFetcherBase::Blob(kTestModelName1, kTestModelContent1), + BlobFetcherBase::Blob(kTestModelName2, kTestModelContent2)}; + + auto blob_fetcher = std::make_unique(); + auto mock_inference_stub = std::make_unique(); + auto executor = std::make_unique(); + + absl::BlockingCounter done(1); + { + InSequence s; + + EXPECT_CALL( + *blob_fetcher, + FetchSync(Field(&BlobFetcherBase::FilterOptions::included_prefixes, + ElementsAre(kModelConfigPath)))) + .WillOnce(Return(absl::OkStatus())); + + EXPECT_CALL(*blob_fetcher, snapshot()) + .WillOnce(ReturnRef(config_fetch_result)); + + EXPECT_CALL( + *blob_fetcher, + FetchSync(Field(&BlobFetcherBase::FilterOptions::included_prefixes, + ElementsAre(kTestModelName1, kTestModelName2)))) + .WillOnce(Return(absl::OkStatus())); + + EXPECT_CALL(*blob_fetcher, snapshot()).WillOnce(ReturnRef(mock_snapshot)); + } + + RegisterModelRequest register_model_request_1; + register_model_request_1.mutable_model_spec()->set_model_path( + kTestModelName1); + (*register_model_request_1.mutable_model_files())[kTestModelName1] = + kTestModelContent1; + + RegisterModelRequest register_model_request_2; + register_model_request_2.mutable_model_spec()->set_model_path( + kTestModelName2); + (*register_model_request_2.mutable_model_files())[kTestModelName2] = + kTestModelContent2; + register_model_request_2.set_warm_up_batch_request_json( + kModelWarmUpRequestJson); + + EXPECT_CALL(*mock_inference_stub, + RegisterModel(_, EqualsProto(register_model_request_1), _)) + .WillOnce(Return(grpc::Status::OK)); + + EXPECT_CALL(*mock_inference_stub, + RegisterModel(_, EqualsProto(register_model_request_2), _)) + .WillOnce(Return(grpc::Status::OK)); + + EXPECT_CALL(*executor, RunAfter) + .WillOnce( + [&done](absl::Duration duration, absl::AnyInvocable closure) { + EXPECT_EQ(duration, kFetchPeriod); + done.DecrementCount(); + return server_common::TaskId(); + }); + + PeriodicModelFetcher model_fetcher(kModelConfigPath, std::move(blob_fetcher), + std::move(mock_inference_stub), + executor.get(), kFetchPeriod); + auto status = model_fetcher.Start(); + ASSERT_TRUE(status.ok()) << status; + done.Wait(); + model_fetcher.End(); +} + } // namespace } // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/bidding_service/protected_app_signals_generate_bids_reactor.cc b/services/bidding_service/protected_app_signals_generate_bids_reactor.cc index 34dfff89..0b468629 100644 --- a/services/bidding_service/protected_app_signals_generate_bids_reactor.cc +++ b/services/bidding_service/protected_app_signals_generate_bids_reactor.cc @@ -369,10 +369,10 @@ ProtectedAppSignalsGenerateBidsReactor:: absl::StatusOr ProtectedAppSignalsGenerateBidsReactor::GetSerializedEgressPayload( uint32_t schema_version, absl::string_view egress_payload, - const EgressSchemaCache& egress_schema_cache, int egress_bit_limit) { + EgressSchemaCache& egress_schema_cache, int egress_bit_limit) { PS_VLOG(5) << "Fetching egress schema from cache"; PS_ASSIGN_OR_RETURN(auto egress_features, - egress_schema_cache_->Get(schema_version)); + egress_schema_cache.Get(schema_version)); PS_VLOG(5) << "Fetched egress schema successfully from cache, retreiving " "features in egress payload: " << egress_payload; @@ -389,7 +389,7 @@ ProtectedAppSignalsGenerateBidsReactor::GetSerializedEgressPayload( void ProtectedAppSignalsGenerateBidsReactor::PopulateSerializedEgressPayload( uint32_t schema_version, std::string& egress_payload_in_proto, - const EgressSchemaCache& egress_schema_cache, int egress_bit_limit) { + EgressSchemaCache& egress_schema_cache, int egress_bit_limit) { PS_VLOG(5) << "Egress payload from generateBid: " << egress_payload_in_proto; if (egress_payload_in_proto.empty()) { PS_VLOG(5) << "Egress payload received from generateBid is empty, " @@ -616,11 +616,11 @@ void ProtectedAppSignalsGenerateBidsReactor::Execute() { } PS_VLOG(8, log_context_) << __func__; - PS_VLOG(kEncrypted, log_context_) << "GenerateBidsRequest:\n" - << request_->ShortDebugString(); + PS_VLOG(kEncrypted, log_context_) + << "GenerateBidsRequest exported in EventMessage"; log_context_.SetEventMessageField(*request_); - PS_VLOG(kPlain, log_context_) << "GenerateBidsRawRequest:\n" - << raw_request_.ShortDebugString(); + PS_VLOG(kPlain, log_context_) + << "GenerateBidsRawRequest exported in EventMessage"; log_context_.SetEventMessageField(raw_request_); if (IsContextualRetrievalRequest()) { @@ -641,10 +641,10 @@ void ProtectedAppSignalsGenerateBidsReactor::EncryptResponseAndFinish( grpc::Status status) { PS_VLOG(8, log_context_) << __func__; PS_VLOG(kPlain, log_context_) - << "GenerateProtectedAppSignalsBidsRawResponse:\n" - << raw_response_.ShortDebugString(); + << "GenerateProtectedAppSignalsBidsRawResponse exported in EventMessage"; log_context_.SetEventMessageField(raw_response_); - log_context_.ExportEventMessage(); + // ExportEventMessage before encrypt response + log_context_.ExportEventMessage(/*if_export_consented=*/true); if (!EncryptResponse()) { PS_LOG(ERROR, log_context_) << "Failed to encrypt the generate app signals bids response."; diff --git a/services/bidding_service/protected_app_signals_generate_bids_reactor.h b/services/bidding_service/protected_app_signals_generate_bids_reactor.h index 934933a7..d7861a80 100644 --- a/services/bidding_service/protected_app_signals_generate_bids_reactor.h +++ b/services/bidding_service/protected_app_signals_generate_bids_reactor.h @@ -110,14 +110,14 @@ class ProtectedAppSignalsGenerateBidsReactor // set to an empty string. void PopulateSerializedEgressPayload( uint32_t schema_version, std::string& egress_payload_in_proto, - const EgressSchemaCache& egress_schema_cache, + EgressSchemaCache& egress_schema_cache, int egress_bit_limit = std::numeric_limits::max()); // Converts the JSON string egress payload received from the response of // generateBid to wire format. absl::StatusOr GetSerializedEgressPayload( uint32_t schema_version, absl::string_view egress_payload, - const EgressSchemaCache& egress_schema_cache, + EgressSchemaCache& egress_schema_cache, int egress_bit_limit = std::numeric_limits::max()); CLASS_CANCELLATION_WRAPPER(FetchAds, enable_cancellation_, context_, diff --git a/services/bidding_service/protected_app_signals_generate_bids_reactor_test.cc b/services/bidding_service/protected_app_signals_generate_bids_reactor_test.cc index 740ac208..407a879e 100644 --- a/services/bidding_service/protected_app_signals_generate_bids_reactor_test.cc +++ b/services/bidding_service/protected_app_signals_generate_bids_reactor_test.cc @@ -65,30 +65,7 @@ using kv_server::v2::GetValuesRequest; using kv_server::v2::GetValuesResponse; using kv_server::v2::ObliviousGetValuesRequest; -class GenerateBidsReactorTest : public ::testing::Test { - protected: - void SetUp() override { - CommonTestInit(); - TrustedServersConfigClient config_client({}); - config_client.SetOverride(kTrue, TEST_MODE); - config_client.SetOverride(kTrue, ENABLE_PROTECTED_APP_SIGNALS); - server_common::log::SetGlobalPSVLogLevel(20); - - key_fetcher_manager_ = - CreateKeyFetcherManager(config_client, /*public_key_fetcher=*/nullptr); - SetupMockCryptoClientWrapper(crypto_client_); - egress_schema_cache_ = PopulateEgressSchemaCache(); - limited_egress_schema_cache_ = PopulateEgressSchemaCache(); - } - - std::unique_ptr PopulateEgressSchemaCache() { - std::unique_ptr cddl_spec_cache = - std::make_unique( - "services/bidding_service/egress_cddl_spec/"); - CHECK_OK(cddl_spec_cache->Init()); - auto egress_schema_cache = - std::make_unique(std::move(cddl_spec_cache)); - CHECK_OK(egress_schema_cache->Update(R"JSON( +constexpr absl::string_view kLimitedEgressSchema = R"JSON( { "cddl_version": "1.0.0", "version": 2, @@ -112,7 +89,60 @@ class GenerateBidsReactorTest : public ::testing::Test { } ] } - )JSON")); + )JSON"; + +constexpr absl::string_view kUnlimitedEgressSchema = R"JSON( + { + "cddl_version": "1.0.0", + "version": 2, + "features": [ + { + "name": "unsigned-integer-feature", + "size": 7 + }, + { + "name": "boolean-feature" + }, + { + "name": "histogram-feature", + "size": 1, + "value": [ + { + "name": "signed-integer-feature", + "size": 1 + } + ] + } + ] + } + )JSON"; + +class GenerateBidsReactorTest : public ::testing::Test { + protected: + void SetUp() override { + CommonTestInit(); + TrustedServersConfigClient config_client({}); + config_client.SetOverride(kTrue, TEST_MODE); + config_client.SetOverride(kTrue, ENABLE_PROTECTED_APP_SIGNALS); + server_common::log::SetGlobalPSVLogLevel(20); + + key_fetcher_manager_ = + CreateKeyFetcherManager(config_client, /*public_key_fetcher=*/nullptr); + SetupMockCryptoClientWrapper(crypto_client_); + egress_schema_cache_ = PopulateEgressSchemaCache(kUnlimitedEgressSchema); + limited_egress_schema_cache_ = + PopulateEgressSchemaCache(kLimitedEgressSchema); + } + + std::unique_ptr PopulateEgressSchemaCache( + absl::string_view egress_schema) { + std::unique_ptr cddl_spec_cache = + std::make_unique( + "services/bidding_service/egress_cddl_spec/"); + CHECK_OK(cddl_spec_cache->Init()); + auto egress_schema_cache = + std::make_unique(std::move(cddl_spec_cache)); + CHECK_OK(egress_schema_cache->Update(egress_schema)); return egress_schema_cache; } @@ -665,12 +695,12 @@ TEST_F(GenerateBidsReactorTest, TemporaryEgressVectorGetsPopulated) { "temporaryUnlimitedEgressPayload" : "{\"features\": [ - {\"type\": \"boolean-feature\", \"value\": true}, - {\"type\": \"unsigned-integer-feature\", \"value\": 2}, - {\"type\": \"histogram-feature\", + {\"name\": \"unsigned-integer-feature\", \"value\": 2}, + {\"name\": \"boolean-feature\", \"value\": true}, + {\"name\": \"histogram-feature\", \"value\": [ { - \"type\": \"signed-integer-feature\", + \"name\": \"signed-integer-feature\", \"value\": -1 } ] @@ -811,12 +841,12 @@ TEST_F(GenerateBidsReactorTest, SerializesEgressVector) { "temporaryUnlimitedEgressPayload" : "{\"features\": [ - {\"type\": \"boolean-feature\", \"value\": true}, - {\"type\": \"unsigned-integer-feature\", \"value\": 2}, - {\"type\": \"histogram-feature\", + {\"name\": \"unsigned-integer-feature\", \"value\": 2}, + {\"name\": \"boolean-feature\", \"value\": true}, + {\"name\": \"histogram-feature\", \"value\": [ { - \"type\": \"signed-integer-feature\", + \"name\": \"signed-integer-feature\", \"value\": -1 } ] @@ -859,13 +889,13 @@ TEST_F(GenerateBidsReactorTest, SerializesEgressVector) { // bits for each feature are populated (if not explicitly set, then a // default of 0 is used). // - // Base64 encoded payload: AQVA - // Wire representation: 00000001 00000101 01000000 + // Base64 encoded payload: AYJA + // Wire representation: 00000001 10000010 01000000 // In the wire representation, we expect the schema version (2 is used for // this test by default) in 3 MSB of header and in the payload, there is a - // single boolean present, unsigned integer of value 2 and width 7 and a + // unsigned integer of value 2 with width 7, a single boolean present and a // histogram feature with a signed integer of value 1. - EXPECT_EQ(generated_bid.temporary_unlimited_egress_payload(), "AQVA"); + EXPECT_EQ(generated_bid.temporary_unlimited_egress_payload(), "AYJA"); } TEST_F(GenerateBidsReactorTest, SerializesMultipleFeatures) { @@ -881,12 +911,12 @@ TEST_F(GenerateBidsReactorTest, SerializesMultipleFeatures) { "render": "$1", "egressPayload" : "{\"features\": [ - {\"type\": \"boolean-feature\", \"value\": true}, - {\"type\": \"unsigned-integer-feature\", \"value\": 127}, - {\"type\": \"histogram-feature\", + {\"name\": \"boolean-feature\", \"value\": true}, + {\"name\": \"unsigned-integer-feature\", \"value\": 127}, + {\"name\": \"histogram-feature\", \"value\": [ { - \"type\": \"signed-integer-feature\", + \"name\": \"signed-integer-feature\", \"value\": -1 } ] @@ -950,7 +980,7 @@ TEST_F(GenerateBidsReactorTest, EgressClearedIfOverLimit) { { "bid" : $0, "render": "$1", - "egressPayload" : "{\"features\": [{\"type\": \"boolean-feature\", \"value\": true}, {\"type\": \"unsigned-integer-feature\", \"value\": 127}]}" + "egressPayload" : "{\"features\": [{\"name\": \"boolean-feature\", \"value\": true}, {\"name\": \"unsigned-integer-feature\", \"value\": 127}]}" } )JSON", kTestWinningBid, kTestRenderUrl)); diff --git a/services/bidding_service/utils/egress_test.cc b/services/bidding_service/utils/egress_test.cc index e4647719..0376ed31 100644 --- a/services/bidding_service/utils/egress_test.cc +++ b/services/bidding_service/utils/egress_test.cc @@ -52,19 +52,19 @@ boolean-feature-type = { ; ; 1 bit of information. boolean-feature = { - type: boolean-feature-type, + name: text .regexp "^boolean-feature$" value: bool, } ; Type for `bucket-feature`. bucket-feature-type = { - name: text .regexp "^bucket-feature$" + name: text .regexp "^bucket-feature$", size: uint, ; number of buckets. } ; A `feature` representing a bucketized value. bucket-feature = { - type: bucket-feature-type, + name: text .regexp "^bucket-feature$", value: [* bool], } diff --git a/services/buyer_frontend_service/buyer_frontend_main.cc b/services/buyer_frontend_service/buyer_frontend_main.cc index 42719936..472179aa 100644 --- a/services/buyer_frontend_service/buyer_frontend_main.cc +++ b/services/buyer_frontend_service/buyer_frontend_main.cc @@ -73,8 +73,8 @@ ABSL_FLAG(std::optional, enable_buyer_frontend_benchmarking, std::nullopt, ABSL_FLAG( std::optional, create_new_event_engine, std::nullopt, "Share the event engine with gprc when false , otherwise create new one"); -ABSL_FLAG(std::optional, enable_bidding_compression, true, - "Flag to enable bidding client compression. True by default."); +ABSL_FLAG(std::optional, enable_bidding_compression, std::nullopt, + "Flag to enable bidding client compression. Turned off by default."); ABSL_FLAG(std::optional, bfe_ingress_tls, std::nullopt, "If true, frontend gRPC service terminates TLS"); ABSL_FLAG(std::optional, bfe_tls_key, std::nullopt, diff --git a/services/buyer_frontend_service/buyer_frontend_service_test.cc b/services/buyer_frontend_service/buyer_frontend_service_test.cc index 6ea25fa5..e7c2e9d9 100644 --- a/services/buyer_frontend_service/buyer_frontend_service_test.cc +++ b/services/buyer_frontend_service/buyer_frontend_service_test.cc @@ -64,7 +64,7 @@ using GenerateProtectedAppSignalsBidsRawResponse = using GetBidsRawResponse = GetBidsResponse::GetBidsRawResponse; constexpr char valid_bidding_signals[] = - R"JSON({"keys":{"ig_name":[123,456]}})JSON"; + R"JSON({"keys":{"key":[123,456]}})JSON"; TrustedServersConfigClient CreateTrustedServerConfigClient() { TrustedServersConfigClient config_client({}); diff --git a/services/buyer_frontend_service/get_bids_unary_reactor.cc b/services/buyer_frontend_service/get_bids_unary_reactor.cc index b72a189e..3c0fbc7a 100644 --- a/services/buyer_frontend_service/get_bids_unary_reactor.cc +++ b/services/buyer_frontend_service/get_bids_unary_reactor.cc @@ -215,11 +215,11 @@ void GetBidsUnaryReactor::OnAllBidsDone(bool any_successful_bids) { return; } - PS_VLOG(kPlain, log_context_) << "GetBidsRawResponse:\n" - << get_bids_raw_response_->ShortDebugString(); + PS_VLOG(kPlain, log_context_) + << "GetBidsRawResponse exported in EventMessage"; log_context_.SetEventMessageField(*get_bids_raw_response_); - log_context_.ExportEventMessage(); - + // ExportEventMessage before encrypt response + log_context_.ExportEventMessage(/*if_export_consented=*/true); if (auto encryption_status = EncryptResponse(); !encryption_status.ok()) { PS_LOG(ERROR, log_context_) << "Failed to encrypt the response"; benchmarking_logger_->End(); @@ -285,7 +285,7 @@ grpc::Status GetBidsUnaryReactor::DecryptRequest() { return grpc::Status::OK; } -int GetBidsUnaryReactor::GetNumberOfBiddingCalls() { +int GetBidsUnaryReactor::GetNumberOfMaximumBiddingCalls() { int num_expected_calls = 0; if (config_.is_protected_audience_enabled && raw_request_.buyer_input().interest_groups_size() > 0) { @@ -302,8 +302,8 @@ int GetBidsUnaryReactor::GetNumberOfBiddingCalls() { void GetBidsUnaryReactor::CancellableExecute() { benchmarking_logger_->Begin(); - PS_VLOG(kEncrypted, log_context_) << "Encrypted GetBidsRequest:\n" - << request_->ShortDebugString(); + PS_VLOG(kEncrypted, log_context_) + << "Encrypted GetBidsRequest exported in EventMessage"; log_context_.SetEventMessageField(*request_); PS_VLOG(kPlain, log_context_) << "Headers:\n" @@ -318,8 +318,7 @@ void GetBidsUnaryReactor::CancellableExecute() { return; } PS_VLOG(5, log_context_) << "Successfully decrypted the request"; - PS_VLOG(kPlain, log_context_) << "GetBidsRawRequest:\n" - << raw_request_.ShortDebugString(); + PS_VLOG(kPlain, log_context_) << "GetBidsRawRequest exported in EventMessage"; log_context_.SetEventMessageField(raw_request_); LogIgMetric(raw_request_, *metric_context_); @@ -329,7 +328,7 @@ void GetBidsUnaryReactor::CancellableExecute() { return; } - const int num_bidding_calls = GetNumberOfBiddingCalls(); + const int num_bidding_calls = GetNumberOfMaximumBiddingCalls(); if (num_bidding_calls == 0) { // This is unlikely to happen since we already have this check in place // in SFE. @@ -555,21 +554,24 @@ void GetBidsUnaryReactor::PrepareAndGenerateProtectedAudienceBid( async_task_tracker_.TaskCompleted(TaskStatus::EMPTY_RESPONSE); return; } - const auto& log_context = raw_request_.log_context(); std::unique_ptr - raw_bidding_input = - CreateGenerateBidsRawRequest(raw_request_, raw_request_.buyer_input(), - std::move(bidding_signals), log_context); - + raw_bidding_input = CreateGenerateBidsRawRequest( + raw_request_, raw_request_.buyer_input(), std::move(bidding_signals), + log_context_); PS_VLOG(kOriginated, log_context_) << "GenerateBidsRequest:\n" << raw_bidding_input->ShortDebugString(); + if (raw_bidding_input->interest_group_for_bidding_size() == 0) { + PS_LOG(INFO, log_context_) + << "No interest groups for bidding found in the request."; + async_task_tracker_.TaskCompleted(TaskStatus::EMPTY_RESPONSE); + return; + } + auto bidding_request = metric::MakeInitiatedRequest(metric::kBs, metric_context_.get()) .release(); bidding_request->SetRequestSize((int)raw_bidding_input->ByteSizeLong()); - grpc::ClientContext* client_context = client_contexts_.Add(); - absl::Status execute_result = bidding_async_client_->ExecuteInternal( std::move(raw_bidding_input), client_context, [this, bidding_request]( diff --git a/services/buyer_frontend_service/get_bids_unary_reactor.h b/services/buyer_frontend_service/get_bids_unary_reactor.h index 12f941cd..5c19d194 100644 --- a/services/buyer_frontend_service/get_bids_unary_reactor.h +++ b/services/buyer_frontend_service/get_bids_unary_reactor.h @@ -133,8 +133,9 @@ class GetBidsUnaryReactor : public grpc::ServerUnaryReactor { // Finishes the RPC call with a status. void FinishWithStatus(const grpc::Status& status); - // Gets the number of outbound bidding requests to the bidding service. - int GetNumberOfBiddingCalls(); + // Gets the maximum number of outbound bidding requests to the bidding + // service. + int GetNumberOfMaximumBiddingCalls(); // References for state, request, response and context from gRPC. // Should be released by gRPC @@ -192,9 +193,6 @@ class GetBidsUnaryReactor : public grpc::ServerUnaryReactor { const bool enable_cancellation_; - // Logs GetBidsRawRequest if the consented debugging is enabled. - void MayLogRawRequest(); - // Gets Protected Audience Bids. void MayGetProtectedAudienceBids(); diff --git a/services/buyer_frontend_service/get_bids_unary_reactor_test.cc b/services/buyer_frontend_service/get_bids_unary_reactor_test.cc index db315554..52178655 100644 --- a/services/buyer_frontend_service/get_bids_unary_reactor_test.cc +++ b/services/buyer_frontend_service/get_bids_unary_reactor_test.cc @@ -74,7 +74,7 @@ constexpr int kTestModelingSignals2 = 3; constexpr char kTestRender2[] = "https://test-render-2.com"; constexpr char kTestCurrency2[] = "RS"; constexpr char bidding_signals_to_be_returned[] = - R"JSON({"keys":{"ig_name":[123,456]}})JSON"; + R"JSON({"keys":{"key":[123,456]}})JSON"; void SetupMockCryptoClientWrapper(MockCryptoClientWrapper& crypto_client) { EXPECT_CALL(crypto_client, HpkeEncrypt) @@ -140,17 +140,18 @@ class GetBidUnaryReactorTest : public ::testing::Test { config_client, /* public_key_fetcher= */ nullptr); SetupMockCryptoClientWrapper(*crypto_client_); - GetBidsRequest::GetBidsRawRequest raw_request = - MakeARandomGetBidsRawRequest(); - raw_request.mutable_buyer_input()->mutable_interest_groups()->Add(); - request_.set_request_ciphertext(raw_request.SerializeAsString()); + raw_request_ = MakeARandomGetBidsRawRequest(); + auto interest_group = + raw_request_.mutable_buyer_input()->mutable_interest_groups()->Add(); + interest_group->set_name("ig_name"); + interest_group->add_bidding_signals_keys("key"); + request_.set_request_ciphertext(raw_request_.SerializeAsString()); request_.set_key_id(MakeARandomString()); } grpc::CallbackServerContext context_; GetBidsRequest request_ = MakeARandomGetBidsRequest(); - GetBidsRequest::GetBidsRawRequest raw_request_ = - MakeARandomGetBidsRawRequest(); + GetBidsRequest::GetBidsRawRequest raw_request_; GetBidsResponse response_; BiddingAsyncClientMock bidding_client_mock_; MockAsyncProvider @@ -263,8 +264,7 @@ TEST_F(GetBidUnaryReactorTest, VerifyLogContextPropagates) { auto* log_context = raw_request_.mutable_log_context(); log_context->set_adtech_debug_id(kSampleBuyerDebugId); log_context->set_generation_id(kSampleGenerationId); - raw_request_.mutable_buyer_input()->mutable_interest_groups()->Add(); - *request_.mutable_request_ciphertext() = raw_request_.SerializeAsString(); + request_.set_request_ciphertext(raw_request_.SerializeAsString()); SetupBiddingProviderMock( /*provider=*/bidding_signals_provider_, @@ -276,8 +276,8 @@ TEST_F(GetBidUnaryReactorTest, VerifyLogContextPropagates) { expected_generate_bids_raw_request; auto* expected_log_context = expected_generate_bids_raw_request.mutable_log_context(); - expected_log_context->set_generation_id(kSampleGenerationId); expected_log_context->set_adtech_debug_id(kSampleBuyerDebugId); + expected_log_context->set_generation_id(kSampleGenerationId); EXPECT_CALL(bidding_client_mock_, ExecuteInternal(Pointee(EqGenerateBidsRawRequestWithLogContext( expected_generate_bids_raw_request)), diff --git a/services/buyer_frontend_service/util/BUILD b/services/buyer_frontend_service/util/BUILD index 7975535d..b6ab81af 100644 --- a/services/buyer_frontend_service/util/BUILD +++ b/services/buyer_frontend_service/util/BUILD @@ -27,8 +27,12 @@ cc_library( deps = [ "//api:bidding_auction_servers_cc_grpc_proto", "//services/buyer_frontend_service/data:buyer_frontend_data", + "//services/common/loggers:request_log_context", + "//services/common/util:json_util", + "//services/common/util:request_response_constants", "@com_github_grpc_grpc//:grpc++", "@com_google_absl//absl/strings", + "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", ], ) diff --git a/services/buyer_frontend_service/util/proto_factory.cc b/services/buyer_frontend_service/util/proto_factory.cc index 08933f7a..c88fd5b6 100644 --- a/services/buyer_frontend_service/util/proto_factory.cc +++ b/services/buyer_frontend_service/util/proto_factory.cc @@ -14,6 +14,12 @@ #include "services/buyer_frontend_service/util/proto_factory.h" +#include "absl/container/flat_hash_set.h" +#include "absl/strings/string_view.h" +#include "services/common/util/json_util.h" +#include "services/common/util/request_response_constants.h" +#include "src/util/status_macro/status_macros.h" + namespace privacy_sandbox::bidding_auction_servers { using GetBidsRawRequest = GetBidsRequest::GetBidsRawRequest; using GetBidsRawResponse = GetBidsResponse::GetBidsRawResponse; @@ -21,6 +27,10 @@ using GenerateBidsRawRequest = GenerateBidsRequest::GenerateBidsRawRequest; using GenerateProtectedAppSignalsBidsRawRequest = GenerateProtectedAppSignalsBidsRequest:: GenerateProtectedAppSignalsBidsRawRequest; +struct ParsedTrustedBiddingSignals { + std::string json; + absl::flat_hash_set keys; +}; std::unique_ptr CreateGetBidsRawResponse( std::unique_ptr @@ -37,7 +47,55 @@ std::unique_ptr CreateGetBidsRawResponse( return get_bids_raw_response; } +// Parses trusted bidding signals for a single Interest Group (IG). +// Queries the bidding signals for trusted bidding signal keys in the IG. +// If found, +// 1. Copies key to parsed trusted keys hash set +// 2. Key-value pair to parsed trusted JSON string +absl::StatusOr GetSignalsForIG( + const ::google::protobuf::RepeatedPtrField& + bidding_signals_keys, + rapidjson::Value* bidding_signals_obj, long avg_signal_str_size) { + // Insert bidding signal values for this IG. + ParsedTrustedBiddingSignals parsed_signals; + rapidjson::Document ig_signals; + ig_signals.SetObject(); + // If no bidding signals passed, return empty document. + if (bidding_signals_obj == nullptr) { + return parsed_signals; + } + + // Copy bidding signals with key name in bidding signal keys. + for (const auto& key : bidding_signals_keys) { + if (parsed_signals.keys.contains(key)) { + // Do not process duplicate keys. + continue; + } + rapidjson::Value::ConstMemberIterator bidding_signals_key_itr = + bidding_signals_obj->FindMember(key.c_str()); + if (bidding_signals_key_itr != bidding_signals_obj->MemberEnd()) { + rapidjson::Value json_key; + // Keep string reference. Assumes safe lifecycle. + json_key.SetString(rapidjson::StringRef(key.c_str())); + rapidjson::Value json_value; + // Copy instead of move, could be referenced by multiple IGs. + json_value.CopyFrom(bidding_signals_key_itr->value, + ig_signals.GetAllocator()); + // AddMember moves Values, do not reference them anymore. + ig_signals.AddMember(json_key, json_value, ig_signals.GetAllocator()); + parsed_signals.keys.emplace(key); + } + } + if (ig_signals.MemberCount() > 0) { + absl::StatusOr ig_signals_str = + SerializeJsonDocToReservedString(ig_signals, avg_signal_str_size); + PS_ASSIGN_OR_RETURN(parsed_signals.json, ig_signals_str); + } + return parsed_signals; +} + // Copy properties from IG from device to IG for Bidding. +// Note: trusted bidding signals and keys are not copied. void CopyIGFromDeviceToIGForBidding( const BuyerInput::InterestGroup& ig_from_device, GenerateBidsRequest::GenerateBidsRawRequest::InterestGroupForBidding* @@ -53,17 +111,13 @@ void CopyIGFromDeviceToIGForBidding( mutable_ig_for_bidding->mutable_ad_render_ids()->CopyFrom( ig_from_device.ad_render_ids()); } + if (!ig_from_device.component_ads().empty()) { mutable_ig_for_bidding->mutable_ad_component_render_ids()->CopyFrom( ig_from_device.component_ads()); } - if (!ig_from_device.bidding_signals_keys().empty()) { - mutable_ig_for_bidding->mutable_trusted_bidding_signals_keys()->MergeFrom( - ig_from_device.bidding_signals_keys()); - } - - // Set Device Signals. + // Set device signals. if (ig_from_device.has_browser_signals() && ig_from_device.browser_signals().IsInitialized()) { mutable_ig_for_bidding->mutable_browser_signals()->CopyFrom( @@ -78,30 +132,83 @@ std::unique_ptr CreateGenerateBidsRawRequest( const GetBidsRawRequest& get_bids_raw_request, const BuyerInput& buyer_input, std::unique_ptr bidding_signals, - const server_common::LogContext& log_context) { + RequestLogContext& log_context) { auto generate_bids_raw_request = std::make_unique(); - // 1. Set Interest Group for bidding - for (int i = 0; i < buyer_input.interest_groups_size(); i++) { - const auto& interest_group_from_device = buyer_input.interest_groups(i); - // IG must have a name. - if (interest_group_from_device.name().empty()) { - continue; - } - // Add InterestGroupForBidding. - auto mutable_interest_group_for_bidding = - generate_bids_raw_request->mutable_interest_group_for_bidding()->Add(); + // Deserialize trusted bidding signals. + // Note: Null checks for bidding_signals and bidding_signals->trusted signals + // expected to be already performed elsewhere. + auto start_deserialize_time = absl::Now(); + absl::StatusOr trusted_signals_status = + ParseJsonString(*bidding_signals->trusted_signals); + rapidjson::Document trusted_signals; + if (trusted_signals_status.ok()) { + trusted_signals = *std::move(trusted_signals_status); + PS_VLOG(kStats, log_context) + << "\nTrusted Bidding Signals Deserialize Time: " + << ToInt64Microseconds((absl::Now() - start_deserialize_time)) + << " microseconds for " << trusted_signals.Size() << " bytes."; + } else { + // TODO(b/308793587): Publish a metric for this error. + PS_VLOG(kNoisyWarn, log_context) + << "Trusted bidding signals JSON deserialize error: " + << trusted_signals_status.status(); + return generate_bids_raw_request; + } + rapidjson::Value bidding_signals_obj; + if (trusted_signals.HasMember("keys")) { + bidding_signals_obj = std::move(trusted_signals["keys"]); + } else { + // TODO(b/308793587): Publish a metric for this error. + PS_VLOG(kNoisyWarn, log_context) + << "Trusted bidding signals JSON validate error (Missing property " + "\"keys\")"; + return generate_bids_raw_request; + } - // Copy from IG from device. - CopyIGFromDeviceToIGForBidding(interest_group_from_device, - mutable_interest_group_for_bidding); + // 1. Set interest groups (IGs) for bidding. + if (buyer_input.interest_groups_size() > 0) { + long avg_signal_size_per_ig = bidding_signals->trusted_signals->size() / + buyer_input.interest_groups_size(); + // Iterate through each IG from device. + for (int i = 0; i < buyer_input.interest_groups_size(); i++) { + // Skip IG if it has no name or bidding signals keys. + const auto& ig_from_device = buyer_input.interest_groups(i); + if (ig_from_device.name().empty() || + ig_from_device.bidding_signals_keys().empty()) { + continue; + } + // Get parsed trusted bidding signals for this IG. + absl::StatusOr parsed_signals = + GetSignalsForIG(ig_from_device.bidding_signals_keys(), + &bidding_signals_obj, avg_signal_size_per_ig); + // Skip IG if it has no parsed trusted bidding signals. + if (!parsed_signals.ok() || parsed_signals->json.empty()) { + continue; + } + // Initialize IG for bidding. + auto mutable_ig_for_bidding = + generate_bids_raw_request->mutable_interest_group_for_bidding() + ->Add(); + // Only add trusted bidding signals keys that are parsed. + // TODO(b/308793587): Optimize. + for (const std::string& key : parsed_signals->keys) { + mutable_ig_for_bidding->add_trusted_bidding_signals_keys(key); + } + // Set trusted bidding signals to include only those signals that are + // parsed. + mutable_ig_for_bidding->set_trusted_bidding_signals( + std::move(parsed_signals->json)); + // Copy other properties from IG from device to IG for bidding. + CopyIGFromDeviceToIGForBidding(ig_from_device, mutable_ig_for_bidding); + } } - // 2. Set Auction Signals. + // 2. Set auction signals. generate_bids_raw_request->set_auction_signals( get_bids_raw_request.auction_signals()); - // 3. Set Buyer Signals. + // 3. Set buyer signals. if (!get_bids_raw_request.buyer_signals().empty()) { generate_bids_raw_request->set_buyer_signals( get_bids_raw_request.buyer_signals()); @@ -109,36 +216,37 @@ std::unique_ptr CreateGenerateBidsRawRequest( generate_bids_raw_request->set_buyer_signals(""); } - // 4. Set Bidding Signals - generate_bids_raw_request->set_allocated_bidding_signals( - bidding_signals->trusted_signals.release()); - - // 5. Set Debug Reporting Flag + // 5. Set debug reporting flag. generate_bids_raw_request->set_enable_debug_reporting( get_bids_raw_request.enable_debug_reporting()); + // 6. Set seller domain. + generate_bids_raw_request->set_seller(get_bids_raw_request.seller()); + + // 7. Set publisher name. generate_bids_raw_request->set_publisher_name( get_bids_raw_request.publisher_name()); - generate_bids_raw_request->set_seller(get_bids_raw_request.seller()); - generate_bids_raw_request->set_top_level_seller( - get_bids_raw_request.top_level_seller()); - // 6. Set logging context. - if (!log_context.adtech_debug_id().empty()) { + // 8. Set logging context. + if (!get_bids_raw_request.log_context().adtech_debug_id().empty()) { generate_bids_raw_request->mutable_log_context()->set_adtech_debug_id( - log_context.adtech_debug_id()); + get_bids_raw_request.log_context().adtech_debug_id()); } - if (!log_context.generation_id().empty()) { + if (!get_bids_raw_request.log_context().generation_id().empty()) { generate_bids_raw_request->mutable_log_context()->set_generation_id( - log_context.generation_id()); + get_bids_raw_request.log_context().generation_id()); } - // 7. Set consented debug config. + // 9. Set consented debug config. if (get_bids_raw_request.has_consented_debug_config()) { *generate_bids_raw_request->mutable_consented_debug_config() = get_bids_raw_request.consented_debug_config(); } + // 10. Set top level seller. + generate_bids_raw_request->set_top_level_seller( + get_bids_raw_request.top_level_seller()); + return generate_bids_raw_request; } @@ -169,14 +277,13 @@ CreateGenerateProtectedAppSignalsBidsRawRequest( generate_bids_raw_request->set_publisher_name(raw_request.publisher_name()); - const auto& log_context = raw_request.log_context(); - if (!log_context.adtech_debug_id().empty()) { + if (!raw_request.log_context().adtech_debug_id().empty()) { generate_bids_raw_request->mutable_log_context()->set_adtech_debug_id( - log_context.adtech_debug_id()); + raw_request.log_context().adtech_debug_id()); } - if (!log_context.generation_id().empty()) { + if (!raw_request.log_context().generation_id().empty()) { generate_bids_raw_request->mutable_log_context()->set_generation_id( - log_context.generation_id()); + raw_request.log_context().generation_id()); } if (raw_request.has_consented_debug_config()) { diff --git a/services/buyer_frontend_service/util/proto_factory.h b/services/buyer_frontend_service/util/proto_factory.h index bcb963f7..ada0d840 100644 --- a/services/buyer_frontend_service/util/proto_factory.h +++ b/services/buyer_frontend_service/util/proto_factory.h @@ -20,9 +20,9 @@ #include #include -#include "absl/strings/string_view.h" #include "api/bidding_auction_servers.pb.h" #include "services/buyer_frontend_service/data/bidding_signals.h" +#include "services/common/loggers/request_log_context.h" namespace privacy_sandbox::bidding_auction_servers { @@ -38,7 +38,7 @@ CreateGenerateBidsRawRequest( const GetBidsRequest::GetBidsRawRequest& get_bid_raw_request, const BuyerInput& buyer_input, std::unique_ptr bidding_signals, - const server_common::LogContext& log_context); + RequestLogContext& log_context); // Creates a request to generate bid for protected app signals. std::unique_ptr(); + bidding_signals->trusted_signals = std::make_unique( + GetBiddingSignalsFromGenerateBidsRequest(expected_raw_output)); GetBidsRequest::GetBidsRawRequest input; // 1. Set Interest Group For Bidding @@ -127,18 +129,14 @@ TEST(CreateGenerateBidsRequestTest, SetsAllFieldsFromInputParamsForAndroid) { input.mutable_buyer_input()->mutable_interest_groups()->AddAllocated( input_ig.release()); } - // 2. Set Auction Signals. input.set_auction_signals(expected_raw_output.auction_signals()); // 3. Set Buyer Signals. input.set_buyer_signals(expected_raw_output.buyer_signals()); - // 4. Set Bidding Signals - bidding_signals->trusted_signals = - std::make_unique(expected_raw_output.bidding_signals()); auto raw_output = CreateGenerateBidsRawRequest(input, input.buyer_input(), std::move(bidding_signals), - server_common::LogContext{}); + NoOpContext().log); std::string difference; MessageDifferencer differencer; @@ -147,156 +145,12 @@ TEST(CreateGenerateBidsRequestTest, SetsAllFieldsFromInputParamsForAndroid) { << difference; } -TEST(CreateGenerateBidsRequestTest, SetsAllFieldsFromInputParamsForTestIG) { - GenBidsRawReq expected_raw_output = privacy_sandbox::bidding_auction_servers:: - MakeARandomGenerateBidsRequestForBrowser(); - auto bidding_signals = std::make_unique(); - - GetBidsRequest::GetBidsRawRequest input; - - // Create a test IG with ads. - auto ig_with_two_ads = MakeAnInterestGroupSentFromDevice(); - - // Check that IG parsed correctly. - ASSERT_FALSE(ig_with_two_ads->name().empty()); - ASSERT_TRUE(ig_with_two_ads->has_browser_signals()); - ASSERT_TRUE(ig_with_two_ads->browser_signals().IsInitialized()); - ASSERT_EQ(ig_with_two_ads->ad_render_ids_size(), 2); - ASSERT_GT(ig_with_two_ads->bidding_signals_keys_size(), 0); - - // Now transform the IG into the expected output IGForBidding and add it to - // expected output object. - expected_raw_output.mutable_interest_group_for_bidding()->Clear(); - - GenBidsRawReq::InterestGroupForBidding* ig_for_bidding = - expected_raw_output.mutable_interest_group_for_bidding()->Add(); - - ig_for_bidding->set_name(ig_with_two_ads->name()); - ig_for_bidding->mutable_browser_signals()->CopyFrom( - ig_with_two_ads->browser_signals()); - ig_for_bidding->mutable_ad_render_ids()->MergeFrom( - ig_with_two_ads->ad_render_ids()); - ig_for_bidding->mutable_trusted_bidding_signals_keys()->MergeFrom( - ig_with_two_ads->bidding_signals_keys()); - - // Move Input Interest Group to Buyer Input. - input.mutable_buyer_input()->mutable_interest_groups()->AddAllocated( - ig_with_two_ads.release()); - // Check that exactly 1 IG is in the input. - ASSERT_EQ(input.buyer_input().interest_groups().size(), 1); - - // 2. Set Auction Signals. - input.set_auction_signals(expected_raw_output.auction_signals()); - // 3. Set Buyer Signals. - input.set_buyer_signals(expected_raw_output.buyer_signals()); - // 4. Set Bidding Signals - bidding_signals->trusted_signals = - std::make_unique(expected_raw_output.bidding_signals()); - - input.set_seller(expected_raw_output.seller()); - input.set_publisher_name(expected_raw_output.publisher_name()); - - auto raw_output = CreateGenerateBidsRawRequest(input, input.buyer_input(), - std::move(bidding_signals), - server_common::LogContext{}); - - ASSERT_GT(expected_raw_output.interest_group_for_bidding().size(), 0); - ASSERT_GT(raw_output->interest_group_for_bidding().size(), 0); - - EXPECT_TRUE(MessageDifferencer::Equals(expected_raw_output, *raw_output)); - - if (!(MessageDifferencer::Equals(expected_raw_output, *raw_output))) { - std::string expected_output_str, output_str; - - CHECK_OK(MessageToJsonString( - expected_raw_output.interest_group_for_bidding().at(0), - &expected_output_str)); - CHECK_OK(MessageToJsonString(raw_output->interest_group_for_bidding().at(0), - &output_str)); - - ABSL_LOG(INFO) << "\nExpected First IG:\n" << expected_output_str; - ABSL_LOG(INFO) << "\nActual First IG:\n" << output_str; - - ABSL_LOG(INFO) << "\nExpected seller:\n" << expected_raw_output.seller(); - ABSL_LOG(INFO) << "\nActual seller:\n" << raw_output->seller(); - } -} - -TEST(CreateGenerateBidsRequestTest, SetsTopLevelSellerForComponentAuction) { - GenBidsRawReq expected_raw_output = privacy_sandbox::bidding_auction_servers:: - MakeARandomGenerateBidsRequestForBrowser(); - GetBidsRequest::GetBidsRawRequest input; - input.set_top_level_seller(MakeARandomString()); - - auto raw_output = CreateGenerateBidsRawRequest( - input, input.buyer_input(), std::make_unique(), - server_common::LogContext{}); - - EXPECT_EQ(input.top_level_seller(), raw_output->top_level_seller()); -} - -TEST(CreateGenerateBidsRequestTest, SetsEmptyBiddingSignalKeysForBrowserIG) { - GetBidsRequest::GetBidsRawRequest input; - - // Create a test IG. - auto input_ig = MakeARandomInterestGroupFromBrowser(); - input_ig->mutable_bidding_signals_keys()->Clear(); - - // Check that IG created correctly. - ASSERT_EQ(input_ig->bidding_signals_keys_size(), 0); - - // Move Input Interest Group to Buyer Input. - input.mutable_buyer_input()->mutable_interest_groups()->AddAllocated( - input_ig.release()); - // Check that exactly 1 IG is in the input. - ASSERT_EQ(input.buyer_input().interest_groups().size(), 1); - - auto raw_output = CreateGenerateBidsRawRequest( - input, input.buyer_input(), std::make_unique(), - server_common::LogContext{}); - - ASSERT_EQ(raw_output->interest_group_for_bidding().size(), 1); - // Expect no bidding signal keys in output. - EXPECT_EQ(raw_output->interest_group_for_bidding(0) - .trusted_bidding_signals_keys() - .size(), - 0); -} - -TEST(CreateGenerateBidsRequestTest, SetsEmptyBiddingSignalKeysForAndroidIG) { - GetBidsRequest input; - - // Create a test IG. - auto input_ig = MakeARandomInterestGroupFromAndroid(); - input_ig->mutable_bidding_signals_keys()->Clear(); - - // Check that IG created correctly. - ASSERT_EQ(input_ig->bidding_signals_keys_size(), 0); - - // Move Input Interest Group to Buyer Input. - GetBidsRequest::GetBidsRawRequest get_bids_raw_request; - get_bids_raw_request.mutable_buyer_input() - ->mutable_interest_groups() - ->AddAllocated(input_ig.release()); - // Check that exactly 1 IG is in the input. - ASSERT_EQ(get_bids_raw_request.buyer_input().interest_groups().size(), 1); - - auto raw_output = CreateGenerateBidsRawRequest( - get_bids_raw_request, get_bids_raw_request.buyer_input(), - std::make_unique(), server_common::LogContext{}); - - ASSERT_EQ(raw_output->interest_group_for_bidding().size(), 1); - // Expect no bidding signal keys in output. - EXPECT_EQ(raw_output->interest_group_for_bidding(0) - .trusted_bidding_signals_keys() - .size(), - 0); -} - TEST(CreateGenerateBidsRequestTest, SetsAllFieldsFromInputParamsForBrowser) { GenBidsRawReq expected_raw_output = privacy_sandbox::bidding_auction_servers:: MakeARandomGenerateBidsRequestForBrowser(); auto bidding_signals = std::make_unique(); + bidding_signals->trusted_signals = std::make_unique( + GetBiddingSignalsFromGenerateBidsRequest(expected_raw_output)); GetBidsRequest::GetBidsRawRequest input; // 1. Set Interest Group For Bidding @@ -356,16 +210,12 @@ TEST(CreateGenerateBidsRequestTest, SetsAllFieldsFromInputParamsForBrowser) { input.set_auction_signals(expected_raw_output.auction_signals()); // 3. Set Buyer Signals. input.set_buyer_signals(expected_raw_output.buyer_signals()); - // 4. Set Bidding Signals - bidding_signals->trusted_signals = - std::make_unique(expected_raw_output.bidding_signals()); - input.set_seller(expected_raw_output.seller()); input.set_publisher_name(expected_raw_output.publisher_name()); auto raw_output = CreateGenerateBidsRawRequest(input, input.buyer_input(), std::move(bidding_signals), - server_common::LogContext{}); + NoOpContext().log); EXPECT_TRUE(MessageDifferencer::Equals(expected_raw_output, *raw_output)); @@ -394,49 +244,221 @@ TEST(CreateGenerateBidsRequestTest, SetsAllFieldsFromInputParamsForBrowser) { } } -TEST(CreateGenerateBidsRequestTest, - SetsEmptyBiddingSignalsForNullTrustedSignals) { +TEST(CreateGenerateBidsRequestTest, SetsAllFieldsFromInputParamsForTestIG) { + GenBidsRawReq expected_raw_output = privacy_sandbox::bidding_auction_servers:: + MakeARandomGenerateBidsRequestForBrowser(); + GetBidsRequest::GetBidsRawRequest input; + + // Create a test IG with ads. + auto ig_with_two_ads = MakeAnInterestGroupSentFromDevice(); + // Check that IG parsed correctly. + ASSERT_FALSE(ig_with_two_ads->name().empty()); + ASSERT_TRUE(ig_with_two_ads->has_browser_signals()); + ASSERT_TRUE(ig_with_two_ads->browser_signals().IsInitialized()); + ASSERT_EQ(ig_with_two_ads->ad_render_ids_size(), 2); + ASSERT_GT(ig_with_two_ads->bidding_signals_keys_size(), 0); + + // Get bidding signals for test IG auto bidding_signals = std::make_unique(); + bidding_signals->trusted_signals = std::make_unique( + MakeBiddingSignalsForIGFromDevice(*ig_with_two_ads.get())); + + // Now transform the IG into the expected output IGForBidding and add it to + // expected output object. + expected_raw_output.mutable_interest_group_for_bidding()->Clear(); + GenBidsRawReq::InterestGroupForBidding* ig_for_bidding = + expected_raw_output.mutable_interest_group_for_bidding()->Add(); + ig_for_bidding->set_name(ig_with_two_ads->name()); + ig_for_bidding->mutable_browser_signals()->CopyFrom( + ig_with_two_ads->browser_signals()); + ig_for_bidding->mutable_ad_render_ids()->MergeFrom( + ig_with_two_ads->ad_render_ids()); + ig_for_bidding->mutable_trusted_bidding_signals_keys()->MergeFrom( + ig_with_two_ads->bidding_signals_keys()); + ig_for_bidding->set_trusted_bidding_signals( + MakeTrustedBiddingSignalsForIG(*ig_for_bidding)); + + // Move Input Interest Group to Buyer Input. + input.mutable_buyer_input()->mutable_interest_groups()->AddAllocated( + ig_with_two_ads.release()); + // Check that exactly 1 IG is in the input. + ASSERT_EQ(input.buyer_input().interest_groups().size(), 1); + + // 2. Set Auction Signals. + input.set_auction_signals(expected_raw_output.auction_signals()); + // 3. Set Buyer Signals. + input.set_buyer_signals(expected_raw_output.buyer_signals()); + input.set_seller(expected_raw_output.seller()); + input.set_publisher_name(expected_raw_output.publisher_name()); + + auto raw_output = CreateGenerateBidsRawRequest(input, input.buyer_input(), + std::move(bidding_signals), + NoOpContext().log); + + ASSERT_GT(expected_raw_output.interest_group_for_bidding().size(), 0); + ASSERT_GT(raw_output->interest_group_for_bidding().size(), 0); + + EXPECT_TRUE(MessageDifferencer::Equals(expected_raw_output, *raw_output)); + + if (!(MessageDifferencer::Equals(expected_raw_output, *raw_output))) { + std::string expected_output_str, output_str; + + CHECK_OK(MessageToJsonString( + expected_raw_output.interest_group_for_bidding().at(0), + &expected_output_str)); + CHECK_OK(MessageToJsonString(raw_output->interest_group_for_bidding().at(0), + &output_str)); + + ABSL_LOG(INFO) << "\nExpected First IG:\n" << expected_output_str; + ABSL_LOG(INFO) << "\nActual First IG:\n" << output_str; + + ABSL_LOG(INFO) << "\nExpected seller:\n" << expected_raw_output.seller(); + ABSL_LOG(INFO) << "\nActual seller:\n" << raw_output->seller(); + } +} + +TEST(CreateGenerateBidsRequestTest, IsEmptyForMalformedBiddingSignals) { + auto bidding_signals = std::make_unique(); + bidding_signals->trusted_signals = + std::make_unique("Malformed JSON"); GetBidsRequest::GetBidsRawRequest input; - server_common::LogContext log_context; + auto input_ig = MakeARandomInterestGroupFromBrowser(); + input.mutable_buyer_input()->mutable_interest_groups()->AddAllocated( + input_ig.release()); + ASSERT_EQ(input.buyer_input().interest_groups().size(), 1); + auto raw_output = CreateGenerateBidsRawRequest(input, input.buyer_input(), std::move(bidding_signals), - server_common::LogContext{}); + NoOpContext().log); - EXPECT_TRUE(raw_output->bidding_signals().empty()); + EXPECT_EQ(raw_output->ByteSizeLong(), 0); +} + +TEST(CreateGenerateBidsRequestTest, SkipsIGWithEmptyBiddingSignalsKeys) { + GetBidsRequest::GetBidsRawRequest input; + auto input_ig = MakeARandomInterestGroupFromBrowser(); + auto bidding_signals = std::make_unique(); + bidding_signals->trusted_signals = std::make_unique( + MakeBiddingSignalsForIGFromDevice(*input_ig.get())); + + input_ig->mutable_bidding_signals_keys()->Clear(); + ASSERT_EQ(input_ig->bidding_signals_keys_size(), 0); + input.mutable_buyer_input()->mutable_interest_groups()->AddAllocated( + input_ig.release()); + ASSERT_EQ(input.buyer_input().interest_groups().size(), 1); + + auto raw_output = CreateGenerateBidsRawRequest(input, input.buyer_input(), + std::move(bidding_signals), + NoOpContext().log); + + EXPECT_EQ(raw_output->interest_group_for_bidding().size(), 0); +} + +constexpr char kTestBiddingSignals[] = + R"JSON({"keys":{"key1":["val1"]}, "perInterestGroupData":{"ig":["val1"]}})JSON"; +constexpr char kTestTrustedBiddingSignals[] = R"JSON({"key1":["val1"]})JSON"; + +TEST(CreateGenerateBidsRequestTest, HandlesIGWithDuplicateBiddingSignalsKeys) { + GetBidsRequest::GetBidsRawRequest input; + auto input_ig = MakeARandomInterestGroupFromBrowser(); + input_ig->set_name("ig"); + input_ig->mutable_bidding_signals_keys()->Clear(); + input_ig->add_bidding_signals_keys("key1"); + ASSERT_EQ(input_ig->bidding_signals_keys_size(), 1); + input_ig->add_bidding_signals_keys("key1"); + ASSERT_EQ(input_ig->bidding_signals_keys_size(), 2); + input.mutable_buyer_input()->mutable_interest_groups()->AddAllocated( + input_ig.release()); + ASSERT_EQ(input.buyer_input().interest_groups().size(), 1); + + auto bidding_signals = std::make_unique(); + bidding_signals->trusted_signals = + std::make_unique(kTestBiddingSignals); + + auto raw_output = CreateGenerateBidsRawRequest(input, input.buyer_input(), + std::move(bidding_signals), + NoOpContext().log); + + EXPECT_EQ(raw_output->interest_group_for_bidding(0) + .trusted_bidding_signals_keys_size(), + 1); + EXPECT_EQ(raw_output->interest_group_for_bidding(0).trusted_bidding_signals(), + kTestTrustedBiddingSignals); +} + +TEST(CreateGenerateBidsRequestTest, + HandlesIGWithBiddingSignalsKeysWithoutValue) { + GetBidsRequest::GetBidsRawRequest input; + auto input_ig = MakeARandomInterestGroupFromBrowser(); + input_ig->set_name("ig"); + input_ig->mutable_bidding_signals_keys()->Clear(); + input_ig->add_bidding_signals_keys("key1"); + input_ig->add_bidding_signals_keys("key2"); + ASSERT_EQ(input_ig->bidding_signals_keys_size(), 2); + input.mutable_buyer_input()->mutable_interest_groups()->AddAllocated( + input_ig.release()); + ASSERT_EQ(input.buyer_input().interest_groups().size(), 1); + + auto bidding_signals = std::make_unique(); + bidding_signals->trusted_signals = + std::make_unique(kTestBiddingSignals); + + auto raw_output = CreateGenerateBidsRawRequest(input, input.buyer_input(), + std::move(bidding_signals), + NoOpContext().log); + + EXPECT_EQ(raw_output->interest_group_for_bidding(0) + .trusted_bidding_signals_keys_size(), + 1); + EXPECT_EQ(raw_output->interest_group_for_bidding(0).trusted_bidding_signals(), + kTestTrustedBiddingSignals); } TEST(CreateGenerateBidsRequestTest, SetsEnableEventLevelDebugReporting) { + GenBidsRawReq expected_raw_output = privacy_sandbox::bidding_auction_servers:: + MakeARandomGenerateBidsRequestForBrowser(); auto bidding_signals = std::make_unique(); + bidding_signals->trusted_signals = std::make_unique( + GetBiddingSignalsFromGenerateBidsRequest(expected_raw_output)); GetBidsRequest::GetBidsRawRequest input; input.set_enable_debug_reporting(true); + auto raw_output = CreateGenerateBidsRawRequest(input, input.buyer_input(), std::move(bidding_signals), - server_common::LogContext{}); + NoOpContext().log); EXPECT_TRUE(raw_output->enable_debug_reporting()); } TEST(CreateGenerateBidsRequestTest, SetsLogContext) { + GenBidsRawReq expected_raw_output = privacy_sandbox::bidding_auction_servers:: + MakeARandomGenerateBidsRequestForBrowser(); auto bidding_signals = std::make_unique(); + bidding_signals->trusted_signals = std::make_unique( + GetBiddingSignalsFromGenerateBidsRequest(expected_raw_output)); GetBidsRequest::GetBidsRawRequest input; server_common::LogContext log_context; log_context.set_generation_id(kSampleGenerationId); log_context.set_adtech_debug_id(kSampleAdtechDebugId); - auto raw_output = CreateGenerateBidsRawRequest( - input, input.buyer_input(), std::move(bidding_signals), log_context); + *input.mutable_log_context() = std::move(log_context); + + auto raw_output = CreateGenerateBidsRawRequest(input, input.buyer_input(), + std::move(bidding_signals), + NoOpContext().log); - EXPECT_EQ(raw_output->log_context().generation_id(), - log_context.generation_id()); - EXPECT_EQ(raw_output->log_context().adtech_debug_id(), - log_context.adtech_debug_id()); + EXPECT_EQ(raw_output->log_context().generation_id(), kSampleGenerationId); + EXPECT_EQ(raw_output->log_context().adtech_debug_id(), kSampleAdtechDebugId); } TEST(CreateGenerateBidsRequestTest, SetsConsentedDebugConfig) { + GenBidsRawReq expected_raw_output = privacy_sandbox::bidding_auction_servers:: + MakeARandomGenerateBidsRequestForBrowser(); auto bidding_signals = std::make_unique(); + bidding_signals->trusted_signals = std::make_unique( + GetBiddingSignalsFromGenerateBidsRequest(expected_raw_output)); GetBidsRequest::GetBidsRawRequest input; auto* consented_debug_config = input.mutable_consented_debug_config(); @@ -445,13 +467,30 @@ TEST(CreateGenerateBidsRequestTest, SetsConsentedDebugConfig) { auto raw_output = CreateGenerateBidsRawRequest(input, input.buyer_input(), std::move(bidding_signals), - server_common::LogContext{}); + NoOpContext().log); EXPECT_EQ(raw_output->consented_debug_config().is_consented(), kIsConsentedDebug); EXPECT_EQ(raw_output->consented_debug_config().token(), kConsentedDebugToken); } +TEST(CreateGenerateBidsRequestTest, SetsTopLevelSellerForComponentAuction) { + GenBidsRawReq expected_raw_output = privacy_sandbox::bidding_auction_servers:: + MakeARandomGenerateBidsRequestForBrowser(); + auto bidding_signals = std::make_unique(); + bidding_signals->trusted_signals = std::make_unique( + GetBiddingSignalsFromGenerateBidsRequest(expected_raw_output)); + + GetBidsRequest::GetBidsRawRequest input; + input.set_top_level_seller(MakeARandomString()); + + auto raw_output = CreateGenerateBidsRawRequest(input, input.buyer_input(), + std::move(bidding_signals), + NoOpContext().log); + + EXPECT_EQ(input.top_level_seller(), raw_output->top_level_seller()); +} + TEST(CreateGenerateProtectedAppSignalsBidsRawRequestTest, SetsAppropriateFields) { auto get_bids_request = CreateGetBidsRawRequest(); diff --git a/services/common/clients/code_dispatcher/v8_dispatcher.cc b/services/common/clients/code_dispatcher/v8_dispatcher.cc index 803dd492..a02a3221 100644 --- a/services/common/clients/code_dispatcher/v8_dispatcher.cc +++ b/services/common/clients/code_dispatcher/v8_dispatcher.cc @@ -69,7 +69,8 @@ absl::Status V8Dispatcher::LoadSync(absl::string_view version, absl::Status V8Dispatcher::Execute(std::unique_ptr request, DispatchDoneCallback done_callback) { - return roma_service_.Execute(std::move(request), std::move(done_callback)); + return roma_service_.Execute(std::move(request), std::move(done_callback)) + .status(); } absl::Status V8Dispatcher::BatchExecute( diff --git a/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client.cc b/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client.cc index 405040d2..2d9dbe0b 100644 --- a/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client.cc +++ b/services/common/clients/http_kv_server/buyer/buyer_key_value_async_http_client.cc @@ -82,7 +82,7 @@ absl::Status BuyerKeyValueAsyncHttpClient::Execute( request_size += request.url.size(); EventMessage::KvSignal bid_signal = [&]() { EventMessage::KvSignal signal; - if (!context.log.is_debug_response()) { + if (!AllowAnyEventLogging(context.log)) { return signal; } signal.set_request(request.url); @@ -96,9 +96,8 @@ absl::Status BuyerKeyValueAsyncHttpClient::Execute( absl::StatusOr resultStr) mutable { if (resultStr.ok()) { PS_VLOG(kKVLog, context.log) - << "BuyerKeyValueAsyncHttpClient Success Response:\n" - << resultStr.value(); - if (context.log.is_debug_response()) { + << "BuyerKeyValueAsyncHttpClient response exported in EventMessage"; + if (AllowAnyEventLogging(context.log)) { bid_signal.set_response(resultStr.value()); context.log.SetEventMessageField(std::move(bid_signal)); } @@ -111,6 +110,7 @@ absl::Status BuyerKeyValueAsyncHttpClient::Execute( PS_VLOG(kNoisyWarn, context.log) << "BuyerKeyValueAsyncHttpClient Failure Response: " << resultStr.status(); + context.log.SetEventMessageField(std::move(bid_signal)); std::move(on_done)(resultStr.status()); } }; diff --git a/services/common/clients/http_kv_server/seller/seller_key_value_async_http_client.cc b/services/common/clients/http_kv_server/seller/seller_key_value_async_http_client.cc index cc9f750c..345531b1 100644 --- a/services/common/clients/http_kv_server/seller/seller_key_value_async_http_client.cc +++ b/services/common/clients/http_kv_server/seller/seller_key_value_async_http_client.cc @@ -78,7 +78,7 @@ absl::Status SellerKeyValueAsyncHttpClient::Execute( request_size += request.url.size(); EventMessage::KvSignal score_signal = [&]() { EventMessage::KvSignal signal; - if (!context.log.is_debug_response()) { + if (!AllowAnyEventLogging(context.log)) { return signal; } signal.set_request(request.url); @@ -92,8 +92,8 @@ absl::Status SellerKeyValueAsyncHttpClient::Execute( absl::StatusOr resultStr) mutable { if (resultStr.ok()) { PS_VLOG(kKVLog, context.log) - << "SellerKeyValueAsyncHttpClient Response: " << resultStr.value(); - if (context.log.is_debug_response()) { + << "SellerKeyValueAsyncHttpClient response exported in EventMessage"; + if (AllowAnyEventLogging(context.log)) { score_signal.set_response(resultStr.value()); context.log.SetEventMessageField(std::move(score_signal)); } @@ -106,6 +106,7 @@ absl::Status SellerKeyValueAsyncHttpClient::Execute( PS_VLOG(kNoisyWarn, context.log) << "SellerKeyValueAsyncHttpClients Response fail: " << resultStr.status(); + context.log.SetEventMessageField(std::move(score_signal)); std::move(on_done)(resultStr.status()); } }; diff --git a/services/common/compression/compression_utils.cc b/services/common/compression/compression_utils.cc new file mode 100644 index 00000000..9c071043 --- /dev/null +++ b/services/common/compression/compression_utils.cc @@ -0,0 +1,73 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "services/common/compression/compression_utils.h" + +#include + +#include "absl/strings/str_cat.h" +#include "services/common/compression/gzip.h" + +namespace privacy_sandbox::bidding_auction_servers { + +absl::StatusOr ToCompressionType(int num) { + switch (num) { + case 0: + return CompressionType::kUncompressed; + case 1: + return CompressionType::kGzip; + default: + return absl::InvalidArgumentError( + absl::StrCat("Cannot convert value to CompressionType enum: ", num)); + } +} + +absl::StatusOr Compress(std::string uncompressed, + CompressionType compression_type) { + switch (compression_type) { + case kUncompressed: + return std::move(uncompressed); + case kGzip: + return GzipCompress(uncompressed); + default: + return absl::InvalidArgumentError( + "Invalid compression type supplied during compression"); + } +} + +absl::StatusOr Decompress(std::string compressed, + CompressionType compression_type) { + switch (compression_type) { + case kUncompressed: + return std::move(compressed); + case kGzip: + return GzipDecompress(compressed); + default: + return absl::InvalidArgumentError( + "Invalid compression type supplied during decompression"); + } +} + +absl::StatusOr Decompress(CompressionType compression_type, + absl::string_view compressed) { + switch (compression_type) { + case kGzip: + return GzipDecompress(compressed); + default: + return absl::InvalidArgumentError( + "Invalid compression type supplied during decompression"); + } +} + +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/common/compression/compression_utils.h b/services/common/compression/compression_utils.h new file mode 100644 index 00000000..c32c1e32 --- /dev/null +++ b/services/common/compression/compression_utils.h @@ -0,0 +1,57 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SERVICES_COMMON_CLIENTS_COMPRESSION_UTILS_H_ +#define SERVICES_COMMON_CLIENTS_COMPRESSION_UTILS_H_ + +#include + +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" + +namespace privacy_sandbox::bidding_auction_servers { + +enum CompressionType { kUncompressed = 0, kGzip = 1 }; + +absl::StatusOr ToCompressionType(int num); + +// Compresses std::string using the specified compression algorithm. +absl::StatusOr Compress(std::string uncompressed, + CompressionType compression_type); + +// Two versions of Decompress() are provided below. +// Some callers have std::strings, some have absl::string_view. +// absl::string_view signatures are provided so callers with string_views don't +// have to create a copy of their string to compress/decompress. The reversed +// ordering of the parameters is to clearly differentiate which signature a +// client is calling. + +// The signatures accepting absl::string_views cannot be called with +// CompressionType::kUncompressed because the methods cannot create an +// std::string from an absl::string_view *without creating* a copy of the input +// string. + +// Decompresses std::string using the specified compression algorithm. +absl::StatusOr Decompress(std::string compressed, + CompressionType compression_type); +// Decompresses absl::string_view. This method *cannot* be called with +// compression_type = kUncompressed. +absl::StatusOr Decompress(CompressionType compression_type, + absl::string_view compressed); + +} // namespace privacy_sandbox::bidding_auction_servers + +#endif // SERVICES_COMMON_CLIENTS_COMPRESSION_UTILS_H_ diff --git a/services/common/loggers/BUILD b/services/common/loggers/BUILD index 03402db1..38d05be0 100644 --- a/services/common/loggers/BUILD +++ b/services/common/loggers/BUILD @@ -53,11 +53,11 @@ cc_library( "no_ops_logger.h", ], deps = [ + "//services/common/loggers:request_log_context", "//services/common/loggers:timer", "//services/common/util:request_response_constants", "@com_google_absl//absl/strings", "@com_google_absl//absl/time", - "@google_privacysandbox_servers_common//src/logger:request_context_logger", ], ) diff --git a/services/common/loggers/benchmarking_logger.cc b/services/common/loggers/benchmarking_logger.cc index 67534aa6..a9da9e34 100644 --- a/services/common/loggers/benchmarking_logger.cc +++ b/services/common/loggers/benchmarking_logger.cc @@ -19,8 +19,8 @@ #include #include "absl/strings/str_cat.h" +#include "services/common/loggers/request_log_context.h" #include "services/common/util/request_response_constants.h" -#include "src/logger/request_context_logger.h" namespace privacy_sandbox::bidding_auction_servers { diff --git a/services/common/loggers/request_log_context.h b/services/common/loggers/request_log_context.h index 16ae2a0f..e7e1562f 100644 --- a/services/common/loggers/request_log_context.h +++ b/services/common/loggers/request_log_context.h @@ -43,7 +43,7 @@ inline server_common::log::SystemLogContext& SystemLogContext() { class EventMessageProvider { public: - EventMessage Get() { return event_message_; } + const EventMessage& Get() { return event_message_; } EVENT_MESSAGE_PROVIDER_SET(SelectAdRequest, select_ad_request); EVENT_MESSAGE_PROVIDER_SET(ProtectedAuctionInput, protected_auction); @@ -101,6 +101,33 @@ inline RequestContext NoOpContext() { // use single ']' as separator constexpr absl::string_view kFailCurl = "Failed to curl]"; +// log verbosity + +inline constexpr int kPlain = 1; // plaintext B&A request and response served +inline constexpr int kNoisyWarn = + 2; // non-critical error, use PS_LOG(ERROR, *) for critical error +inline constexpr int kUdfLog = 3; +inline constexpr int kSuccess = 3; +inline constexpr int kNoisyInfo = 4; +inline constexpr int kDispatch = 4; // UDF dispatch request and response +inline constexpr int kOriginated = + 5; // plaintext B&A request and response originated from server +inline constexpr int kKVLog = 5; // KV request response +inline constexpr int kStats = 5; // Stats log like time , byte size, etc. +inline constexpr int kEncrypted = 6; + +inline bool AllowAnyEventLogging(RequestLogContext& log_context) { + // if is_debug_response in non prod, it logs to debug kNoisyInfo + // if is_consented, it logs to event message + return (log_context.is_debug_response() && !server_common::log::IsProd()) || + log_context.is_consented(); +} + +inline bool AllowAnyUdfLogging(RequestLogContext& log_context) { + return server_common::log::PS_VLOG_IS_ON(kUdfLog) || + AllowAnyEventLogging(log_context); +} + } // namespace privacy_sandbox::bidding_auction_servers #endif // SERVICES_COMMON_LOGGERS_REQUEST_LOG_CONTEXT_H_ diff --git a/services/common/metric/BUILD b/services/common/metric/BUILD index be56f50b..e8e97cb9 100644 --- a/services/common/metric/BUILD +++ b/services/common/metric/BUILD @@ -37,6 +37,7 @@ cc_library( "//services/common/util:reporting_util", "@google_privacysandbox_servers_common//src/metric:context_map", "@google_privacysandbox_servers_common//src/metric:key_fetch", + "@inference_common//utils:inference_error_code", ], ) diff --git a/services/common/metric/server_definition.h b/services/common/metric/server_definition.h index 8b934f08..cde6da31 100644 --- a/services/common/metric/server_definition.h +++ b/services/common/metric/server_definition.h @@ -27,6 +27,7 @@ #include "src/metric/context_map.h" #include "src/metric/definition.h" #include "src/metric/key_fetch.h" +#include "utils/inference_error_code.h" // Defines API used by Bidding auction servers, and B&A specific metrics. namespace privacy_sandbox::bidding_auction_servers { @@ -193,6 +194,44 @@ inline constexpr server_common::metrics::Definition< "Time taken by Roma callback to execute inference", server_common::metrics::kTimeHistogram, 300, 0); +inline constexpr server_common::metrics::Definition< + double, server_common::metrics::Privacy::kNonImpacting, + server_common::metrics::Instrument::kGauge> + kInferenceCloudFetchSuccessCount( + "inference.cloud_fetch.success_count", + "Success count for fetching model blobs from the cloud bucket"); + +inline constexpr server_common::metrics::Definition< + double, server_common::metrics::Privacy::kNonImpacting, + server_common::metrics::Instrument::kGauge> + kInferenceCloudFetchFailedCountByStatus( + "inference.cloud_fetch.failed_count_by_status", + "Failed count for fetching model blobs from the cloud bucket by " + "status"); + +inline constexpr server_common::metrics::Definition< + double, server_common::metrics::Privacy::kNonImpacting, + server_common::metrics::Instrument::kGauge> + kInferenceRecentModelRegistrationSuccess( + "inference.model_registration.recent_success_by_model", + "Success count for registering models with inference sidecar by model " + "in the most recent fetch"); + +inline constexpr server_common::metrics::Definition< + double, server_common::metrics::Privacy::kNonImpacting, + server_common::metrics::Instrument::kGauge> + kInferenceModelRegistrationFailedCountByStatus( + "inference.model_registration.failed_count_by_status", + "Failed count for registering models with inference sidecar by status"); + +inline constexpr server_common::metrics::Definition< + double, server_common::metrics::Privacy::kNonImpacting, + server_common::metrics::Instrument::kGauge> + kInferenceRecentModelRegistrationFailure( + "inference.model_registration.recent_failure_by_model", + "Failed count for registering models with inference sidecar by model " + "in the most recent fetch"); + inline constexpr absl::string_view kSellerRejectReasons[] = { kRejectionReasonBidBelowAuctionFloor, kRejectionReasonBidFromGenBidFailedCurrencyCheck, @@ -329,6 +368,20 @@ inline constexpr server_common::metrics::Definition< /*lower_bound*/ 0, /*min_noise_to_output*/ 0.99); +inline constexpr server_common::metrics::Definition< + int, server_common::metrics::Privacy::kImpacting, + server_common::metrics::Instrument::kPartitionedCounter> + kInferenceErrorCountByErrorCode( + /*name*/ "inference.errors_count", + /*description*/ + "Number of errors in the inference sidecar by error code", + /*partition_type*/ "error_code", + /*max_partitions_contributed*/ 1, + /*public_partitions*/ inference::kInferenceErrorCode, + /*upper_bound*/ 1, + /*lower_bound*/ 0, + /*min_noise_to_output*/ 0.99); + inline constexpr server_common::metrics::Definition< int, server_common::metrics::Privacy::kImpacting, server_common::metrics::Instrument::kPartitionedCounter> @@ -541,6 +594,46 @@ inline constexpr server_common::metrics::Definition< "Size of the inference response going out of the sidecar", server_common::metrics::kSizeHistogram, 100'000, 100); +inline constexpr server_common::metrics::Definition< + int, server_common::metrics::Privacy::kImpacting, + server_common::metrics::Instrument::kPartitionedCounter> + kInferenceRequestCountByModel( + /*name*/ "inference.request.count_by_model", + /*description*/ + "Total number of inference request per model", + /*partition_type*/ "model", + /*max_partitions_contributed*/ 1, + /*public_partitions*/ server_common::metrics::kEmptyPublicPartition, + /*upper_bound*/ 1, + /*lower_bound*/ 0); + +inline constexpr server_common::metrics::Definition< + int, server_common::metrics::Privacy::kImpacting, + server_common::metrics::Instrument::kPartitionedCounter> + kInferenceRequestDurationByModel( + /*name*/ "inference.request.duration_ms_by_model", + /*description*/ + "Time taken by inference sidecar to execute inference by model", + /*partition_type*/ "model", + /*max_partitions_contributed*/ 1, + /*public_partitions*/ server_common::metrics::kEmptyPublicPartition, + /*upper_bound*/ 300, + /*lower_bound*/ 0); + +inline constexpr server_common::metrics::Definition< + int, server_common::metrics::Privacy::kImpacting, + server_common::metrics::Instrument::kPartitionedCounter> + kInferenceRequestFailedCountByModel( + /*name*/ "inference.request.failed_count_by_model", + /*description*/ + "otal number of inference requests resulted in failure partitioned by " + "model", + /*partition_type*/ "model", + /*max_partitions_contributed*/ 1, + /*public_partitions*/ server_common::metrics::kEmptyPublicPartition, + /*upper_bound*/ 1, + /*lower_bound*/ 0); + template struct RequestMetric; @@ -571,11 +664,20 @@ inline constexpr const server_common::metrics::DefinitionName* &kJSExecutionErrorCount, &kBiddingErrorCountByErrorCode, &kBiddingInferenceRequestDuration, + &kInferenceCloudFetchSuccessCount, + &kInferenceCloudFetchFailedCountByStatus, + &kInferenceRecentModelRegistrationSuccess, + &kInferenceModelRegistrationFailedCountByStatus, + &kInferenceRecentModelRegistrationFailure, &kInferenceRequestCount, &kInferenceRequestDuration, &kInferenceRequestSize, &kInferenceResponseSize, &kInferenceRequestFailedCountByStatus, + &kInferenceErrorCountByErrorCode, + &kInferenceRequestCountByModel, + &kInferenceRequestDurationByModel, + &kInferenceRequestFailedCountByModel, }; template <> @@ -847,6 +949,19 @@ inline void AddBuyerPartition( buyer_list_view); } +inline void AddModelPartition( + server_common::telemetry::BuildDependentConfig& telemetry_config, + const std::vector& model_list) { + std::vector model_list_view = {model_list.begin(), + model_list.end()}; + telemetry_config.SetPartition(metric::kInferenceRequestCountByModel.name_, + model_list_view); + telemetry_config.SetPartition(metric::kInferenceRequestDurationByModel.name_, + model_list_view); + telemetry_config.SetPartition( + metric::kInferenceRequestFailedCountByModel.name_, model_list_view); +} + inline std::vector GetErrorList() { std::vector error_list; diff --git a/services/common/private_aggregation/BUILD b/services/common/private_aggregation/BUILD index baaa8cd0..2cecf195 100644 --- a/services/common/private_aggregation/BUILD +++ b/services/common/private_aggregation/BUILD @@ -25,6 +25,7 @@ cc_library( deps = [ "//api:bidding_auction_servers_cc_grpc_proto", "//services/common/util:reporting_util", + "@com_google_absl//absl/numeric:int128", "@com_google_absl//absl/strings", "@com_google_absl//absl/types:optional", "@rapidjson", @@ -44,3 +45,17 @@ cc_test( "@com_google_googletest//:gtest_main", ], ) + +cc_library( + name = "private_aggregation_test_util", + srcs = ["private_aggregation_test_util.cc"], + hdrs = [ + "private_aggregation_test_util.h", + ], + deps = [ + "//api:bidding_auction_servers_cc_grpc_proto", + "//services/common/util:reporting_util", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:optional", + ], +) diff --git a/services/common/private_aggregation/js/private_aggregation_helper_functions.js b/services/common/private_aggregation/js/private_aggregation_helper_functions.js index 89010f3c..03cea059 100644 --- a/services/common/private_aggregation/js/private_aggregation_helper_functions.js +++ b/services/common/private_aggregation/js/private_aggregation_helper_functions.js @@ -43,11 +43,13 @@ const ReservedEventConstants = Object.freeze({ Object.freeze(BaseValue); Object.freeze(ReservedEventConstants); -/** @implements {PrivateAggregationUtil} */ +/** @implements {PrivateAggregationUtil} + * @suppress {checkTypes} + * */ class PrivateAggregationUtilImpl { /** * Checks if the number is an 128-bit unsigned integer. - * @param {number|bigint} number The number to check. + * @param {bigint} number The number to check. * @return {boolean} True if the number is an unsigned 128-bit integer, false otherwise. * * @override @@ -58,24 +60,105 @@ class PrivateAggregationUtilImpl { /** * Converts a 128-bit unsigned integer to an array of two 64-bit integers. * - * @param {number|bigint} number The 128-bit unsigned integer to convert. + * @param {bigint} bucket The 128-bit unsigned integer to convert. * @return {Array} An array of two 64-bit integers representing the 128-bit integer. + * The first element of the array is the high 64 bits of the 128-bit integer, and the second + * element is the low 64 bits of the 128-bit integer. * @override */ - convertTo128BitArray(number) { - return null; + convertTo128BitArray(bucket) { + //Assume that the type of the bucket is BigInt. The type cannot be checked since + // closure_js_test does not support any features >ES6 including the BigInt type. + const highBits = bucket >>> 64; // Right shift 64 bits (zero-fill) + const lowBits = bucket & 0xffffffffffffffff; // Mask to get the lower 64 bits + return [highBits, lowBits]; } /** * Converts numerical offset to BucketOffset. * - * @param {number|bigint} offset The numerical offset to convert. + * @param {bigint} offset The numerical offset to convert. * @return {{value: Array, is_negative: boolean}} The BucketOffset Object. * @override */ convertToBucketOffset(offset) { return { - value: null, - is_negative: false, + value: this.convertTo128BitArray(Math.abs(offset)), + is_negative: offset < 0, + }; + } + + /** Converts string baseValue type to equivalent ENUM BaseValue string. + * @param {string} value + * @return {string} + * @throws {TypeError} + * @override + */ + convertBaseValueToEnumString(value) { + switch (value) { + case 'winning-bid': + return 'BASE_VALUE_WINNING_BID'; + case 'highest-scoring-other-bid': + return 'BASE_VALUE_HIGHEST_SCORING_OTHER_BID'; + case 'bid-rejection-reason': + return 'BASE_VALUE_BID_REJECTION_REASON'; + //TBD + case 'script-run-time': + return 'BASE_VALUE_SCRIPT_RUN_TIME'; + //TBD + case 'signals-fetch-time': + return 'BASE_VALUE_SIGNALS_FETCH_TIME'; + default: + throw new TypeError('Base value type not supported.'); + } + } + + /** + * Converts bucket object in contribution to SignalBucket. + * + * @param {Object} bucket_obj The bucket object. + * @return {{base_value: string, scale: number, offset: {value: Array, is_negative: boolean}}} The SignalBucket Object. + * @override + */ + convertToSignalBucket(bucket_obj) { + var scale = 1.0; + var offset = this.convertToBucketOffset(0); + // Optional scale factor for the bucket. Default value will be 1.0 + if (bucket_obj.scale) { + scale = bucket_obj.scale; + } + // Optional offset for the bucket. Default value will be 0 + if (bucket_obj.offset) { + offset = this.convertToBucketOffset(bucket_obj.offset); + } + return { + base_value: this.convertBaseValueToEnumString(bucket_obj.baseValue), + scale: scale, + offset: offset, + }; + } + + /** + * Converts value object in contribution to SignalValue. + * + * @param {Object} value_obj The value object. + * @return {{base_value: string, scale: number, offset: number}} The SignalValue Object. + * @override + */ + convertToSignalValue(value_obj) { + var scale = 1.0; + var offset = 0; + // Optional scale factor for the value. Default value will be 1.0 + if (value_obj.scale) { + scale = value_obj.scale; + } + // Optional offset for the value. Default value will be 0 + if (value_obj.offset) { + offset = value_obj.offset; + } + return { + base_value: this.convertBaseValueToEnumString(value_obj.baseValue), + scale: scale, + offset: offset, }; } @@ -113,7 +196,7 @@ class PrivateAggregationUtilImpl { return false; } // Check if it contains the required base_value field. - if (value.base_value != null && Object.values(BaseValue).includes(value.base_value)) { + if (value.baseValue != null && Object.values(BaseValue).includes(value.baseValue)) { return true; } } @@ -142,9 +225,6 @@ class PrivateAggregationUtilImpl { return false; } - /** - * TODO(b/355034881): All usage of "number" type for bucket should be replaced as "bigint" once bigint support is enabled. - */ /** * Create a Private Aggregation contribution object with the input Private Aggregation bucket and value. * @@ -173,24 +253,21 @@ class PrivateAggregationUtilImpl { * @property {?Object} extended_value */ const contribution = { - bucket_128_bit: null, - signal_bucket: null, - int_value: null, - extended_value: null, + bucket: {}, + value: {}, }; - - if (typeof bucket === 'number') { - contribution.bucket_128_bit = bucket; // TODO(b/355034881): Once bigint is supported, should be Array returned from PrivateAggregationUtilImpl.prototype.convertTo128BitArray(bucket); - } else if (typeof bucket === 'object') { - contribution.signal_bucket = bucket; - } else { + if (typeof bucket === 'object') { + contribution.bucket.signal_bucket = this.convertToSignalBucket(bucket); + } else if (typeof bucket === 'string') { throw new TypeError('Invalid type for Private Aggregation bucket.'); + } else { + contribution.bucket.bucket_128_bit = {}; + contribution.bucket.bucket_128_bit.bucket_128_bits = privateAggregationUtil.convertTo128BitArray(bucket); } - if (typeof value === 'number') { - contribution.int_value = value; + contribution.value.int_value = value; } else if (typeof value === 'object') { - contribution.extended_value = value; + contribution.value.extended_value = privateAggregationUtil.convertToSignalValue(value); } else { throw new TypeError('Invalid type for Private Aggregation value.'); } diff --git a/services/common/private_aggregation/js/private_aggregation_helper_functions_externs.js b/services/common/private_aggregation/js/private_aggregation_helper_functions_externs.js index dde894ce..d4e15bbc 100644 --- a/services/common/private_aggregation/js/private_aggregation_helper_functions_externs.js +++ b/services/common/private_aggregation/js/private_aggregation_helper_functions_externs.js @@ -27,11 +27,34 @@ class PrivateAggregationUtil { isUnsigned128BitInteger(value) {} /** - * @param {number|bigint} value + * @param {bigint} bucket * @return {?Array} */ - convertTo128BitArray(value) {} + convertTo128BitArray(bucket) {} + /** Converts string baseValue type to equivalent ENUM BaseValue string. + * @param {string} value + * @return {string} + * @throws {TypeError} + * @override + */ + convertBaseValueToEnumString(value) {} + /** + * Converts value object in contribution to SignalValue. + * + * @param {Object} value_obj The value object. + * @return {{base_value: string, scale: number, offset: {value: Array, is_negative: boolean}}} The SignalValue Object. + * @override + */ + convertToSignalValue(value_obj) {} + /** + * Converts bucket object in contribution to SignalBucket. + * + * @param {Object} bucket_obj The bucket object. + * @return {{base_value: string, scale: number, offset: {value: Array, is_negative: boolean}}} The SignalBucket Object. + * @override + */ + convertToSignalBucket(bucket_obj) {} /** * @param {number|bigint} offset * @return {{is_negative: boolean, value: ?Array}} diff --git a/services/common/private_aggregation/js/private_aggregation_helper_functions_test.js b/services/common/private_aggregation/js/private_aggregation_helper_functions_test.js index 4291b8a4..d33be787 100644 --- a/services/common/private_aggregation/js/private_aggregation_helper_functions_test.js +++ b/services/common/private_aggregation/js/private_aggregation_helper_functions_test.js @@ -60,7 +60,7 @@ testSuite({ /** @return {void} */ testIsValidValue_ValidSignalValue() { const validSignalValue = { - base_value: BaseValue.WINNING_BID, + baseValue: BaseValue.WINNING_BID, scale: 10, offset: 5, }; @@ -79,7 +79,7 @@ testSuite({ /** @return {void} */ testIsValidValue_invalidSignalValue_InvalidBaseValue() { const invalidSignalValue = { - base_value: 'invalid', + baseValue: 'invalid', scale: 10, offset: 5, }; @@ -89,7 +89,7 @@ testSuite({ /** @return {void} */ testIsValidValue_InvalidSignalValue_InvalidScale() { const invalidSignalValue = { - base_value: BaseValue.WINNING_BID, + baseValue: BaseValue.WINNING_BID, scale: 'invalid', offset: 5, }; @@ -99,7 +99,7 @@ testSuite({ /** @return {void} */ testIsValidValue_InvalidSignalValue_InvalidOffset() { const invalidSignalValue = { - base_value: BaseValue.WINNING_BID, + baseValue: BaseValue.WINNING_BID, scale: 10, offset: 'invalid', }; @@ -132,51 +132,173 @@ testSuite({ assertFalse(privateAggregationUtil.isValidCustomEvent(undefined)); }, - /** @return {void} */ - testIsValidContribution() {}, - - /** - * TODO(b/355034881): All usage and test cases of "number" type for bucket should be replaced as "bigint" once bigint support is enabled. - * Validation of SignalBucket and SignalValue's fields should also be added. - */ /** * @return {void} * @suppress {reportUnknownTypes} */ testCreateContribution() { - const numberBucket = 5; + const numberBucket = 12345; const numberValue = 5; - const expectedContribution = { - bucket_128_bit: numberBucket, - signal_bucket: null, - int_value: numberValue, - extended_value: null, + bucket: { bucket_128_bit: { bucket_128_bits: [12345, 0] } }, + value: { int_value: numberValue }, }; - const contribution = privateAggregationUtil.createContribution(numberBucket, numberValue); assertObjectEquals(expectedContribution, contribution); + }, + /** @return {void} */ + /** @suppress {reportUnknownTypes, checkTypes} */ + testConvertBaseValueForWinningBid() { + var result = privateAggregationUtil.convertBaseValueToEnumString('winning-bid'); + assertEquals(result, 'BASE_VALUE_WINNING_BID'); + }, + /** @return {void} */ + /** @suppress {reportUnknownTypes, checkTypes} */ + testConvertBaseValueForHighestScoringOtherBid() { + var result = privateAggregationUtil.convertBaseValueToEnumString('highest-scoring-other-bid'); + assertEquals(result, 'BASE_VALUE_HIGHEST_SCORING_OTHER_BID'); + }, + /** @return {void} */ + /** @suppress {reportUnknownTypes, checkTypes} */ + testConvertBaseValueForScriptRunTime() { + var result = privateAggregationUtil.convertBaseValueToEnumString('script-run-time'); + assertEquals(result, 'BASE_VALUE_SCRIPT_RUN_TIME'); + }, + /** @return {void} */ + /** @suppress {reportUnknownTypes, checkTypes} */ + testConvertBaseValueForInvalidString() { + try { + privateAggregationUtil.convertBaseValueToEnumString('something-else'); + fail('Expected TypeError for unsupported base value'); + } catch (error) { + assertTrue(error instanceof TypeError); + } + }, + /** @return {void} */ + /** @suppress {reportUnknownTypes, checkTypes} */ + testConvertBaseValueWithNullInput() { + try { + privateAggregationUtil.convertBaseValueToEnumString(null); + fail('Expected TypeError for null base value'); + } catch (error) { + assertTrue(error instanceof TypeError); + } + }, + /** + * @return {void} + * @suppress {reportUnknownTypes} + */ + testCreateContributionWhenBucketAndValueAreObjects() { const objectBucket = { - value: [5, 0], + baseValue: 'winning-bid', + scale: 2.0, + offset: 1, + }; + const bucketOffset = { + value: [1, 0], is_negative: false, }; + const signalBucket = { + base_value: 'BASE_VALUE_WINNING_BID', + scale: 2.0, + offset: bucketOffset, + }; const objectValue = { - base_value: 'winning-bid', - int_value: 10, + baseValue: 'winning-bid', + scale: 10, + offset: 5, + }; + const signalValue = { + base_value: 'BASE_VALUE_WINNING_BID', + scale: 10, + offset: 5, + }; + const expectedContribution2 = { + bucket: { signal_bucket: signalBucket }, + value: { extended_value: signalValue }, + }; + + const contribution2 = privateAggregationUtil.createContribution(objectBucket, objectValue); + assertObjectEquals(expectedContribution2, contribution2); + }, + + /** + * @return {void} + * @suppress {reportUnknownTypes} + */ + testCreateContributionWithDefaultSignalBucketScaleAndOffset() { + const objectBucket = { + baseValue: 'winning-bid', + }; + const objectValue = { + baseValue: 'winning-bid', + }; + const signalValue = { + base_value: 'BASE_VALUE_WINNING_BID', + scale: 1.0, + offset: 0, + }; + const bucketOffset = { + value: [0, 0], is_negative: false, }; + const signalBucket = { + base_value: 'BASE_VALUE_WINNING_BID', + scale: 1.0, + offset: bucketOffset, + }; + const expectedContribution2 = { + bucket: { signal_bucket: signalBucket }, + value: { extended_value: signalValue }, + }; + const contribution2 = privateAggregationUtil.createContribution(objectBucket, objectValue); + assertObjectEquals(expectedContribution2, contribution2); + }, + /** + * @return {void} + * @suppress {reportUnknownTypes} + */ + testCreateContributionWithNegativeSignalBucketOffset() { + const objectBucket = { + baseValue: 'winning-bid', + scale: 2.0, + offset: -1, + }; + const objectValue = { + baseValue: 'winning-bid', + scale: 10, + offset: -5, + }; + const signalValue = { + base_value: 'BASE_VALUE_WINNING_BID', + scale: 10, + offset: -5, + }; + const bucketOffset = { + value: [1, 0], + is_negative: true, + }; + const signalBucket = { + base_value: 'BASE_VALUE_WINNING_BID', + scale: 2.0, + offset: bucketOffset, + }; const expectedContribution2 = { - bucket_128_bit: null, - signal_bucket: objectBucket, - int_value: null, - extended_value: objectValue, + bucket: { signal_bucket: signalBucket }, + value: { extended_value: signalValue }, }; const contribution2 = privateAggregationUtil.createContribution(objectBucket, objectValue); assertObjectEquals(expectedContribution2, contribution2); + }, + /** + * @return {void} + * @suppress {reportUnknownTypes} + */ + testCreateContributionWhenBucketAndValueAreInvalid() { const invalidBucket = 'invalid'; const invalidValue = 'invalid'; @@ -186,6 +308,14 @@ testSuite({ assertEquals(e.message, 'Invalid type for Private Aggregation bucket.'); assertEquals(e.name, 'TypeError'); } + }, + /** + * @return {void} + * @suppress {reportUnknownTypes} + */ + testCreateContributionWithInvalidValue() { + const numberBucket = 12345; + const invalidValue = 'invalid'; try { privateAggregationUtil.createContribution(numberBucket, invalidValue); diff --git a/services/common/private_aggregation/private_aggregation_post_auction_util.cc b/services/common/private_aggregation/private_aggregation_post_auction_util.cc index 0335c8e9..585d0425 100644 --- a/services/common/private_aggregation/private_aggregation_post_auction_util.cc +++ b/services/common/private_aggregation/private_aggregation_post_auction_util.cc @@ -14,16 +14,81 @@ #include "private_aggregation_post_auction_util.h" +#include "absl/numeric/int128.h" #include "rapidjson/document.h" #include "services/common/util/reporting_util.h" namespace privacy_sandbox::bidding_auction_servers { -absl::Status HandlePrivateAggregationValuePostAuction( +absl::StatusOr GetPrivateAggregationBucketPostAuction( + const BaseValues& base_values, const PrivateAggregationBucket& bucket) { + Bucket128Bit result; + if (bucket.has_bucket_128_bit()) { + result.CopyFrom(bucket.bucket_128_bit()); + return result; + } + + if (!bucket.has_signal_bucket()) { + return absl::InvalidArgumentError(kInvalidPrivateAggregationBucketType); + } + + BaseValue base_value = bucket.signal_bucket().base_value(); + + // base_value indicates which value the browser + // should use to calculate the resulting bucket or value. + // For example, if base_value is BASE_VALUE_WINNING_BID, the extended value is + // the winning bid * scale + offset. + // For more information, see + // https://github.com/WICG/turtledove/blob/44a00e4e8f02ec03c05075fd994a8abad7b2a6cd/FLEDGE_extended_PA_reporting.md?plain=1#L192. + float base_value_numerical_value; + + switch (base_value) { + case BaseValue::BASE_VALUE_WINNING_BID: + base_value_numerical_value = base_values.winning_bid; + break; + case BaseValue::BASE_VALUE_HIGHEST_SCORING_OTHER_BID: + base_value_numerical_value = base_values.highest_scoring_other_bid; + break; + case BaseValue::BASE_VALUE_BID_REJECTION_REASON: + if (!base_values.reject_reason.has_value()) { + return absl::InvalidArgumentError(kBaseValuesRejectReasonNotAvailable); + } + base_value_numerical_value = base_values.reject_reason.value(); + break; + default: + return absl::InvalidArgumentError(kBaseValueNotSupported); + } + + // Calculate the offset value as a 128-bit integer. + absl::uint128 offset_value = + absl::MakeUint128(bucket.signal_bucket().offset().value(1), + bucket.signal_bucket().offset().value(0)); + + absl::uint128 final_value = absl::MakeUint128( + 0, static_cast(base_value_numerical_value * + bucket.signal_bucket().scale())); + if (bucket.signal_bucket().offset().is_negative()) { + if (final_value < offset_value) { + return absl::InvalidArgumentError( + kPrivateAggregationBucketNegativeFinalValue); + } + final_value -= offset_value; + } else { + final_value += offset_value; + } + + // Convert the final value to an array of 2 uint64 integers. + result.add_bucket_128_bits(absl::Uint128Low64(final_value)); + result.add_bucket_128_bits(absl::Uint128High64(final_value)); + + return result; +} + +absl::StatusOr GetPrivateAggregationValuePostAuction( const BaseValues& base_values, - PrivateAggregationValue& private_aggregation_value) { + const PrivateAggregationValue& private_aggregation_value) { if (private_aggregation_value.has_int_value()) { - return absl::OkStatus(); + return private_aggregation_value.int_value(); } if (!private_aggregation_value.has_extended_value()) { @@ -34,15 +99,25 @@ absl::Status HandlePrivateAggregationValuePostAuction( private_aggregation_value.extended_value().base_value(); int final_value = 0; + // base_value indicates which value the browser + // should use to calculate the resulting bucket or value. + // For example, if base_value is BASE_VALUE_WINNING_BID, the extended value is + // the winning bid + // * scale + offset. + // For more information, see + // https://github.com/WICG/turtledove/blob/44a00e4e8f02ec03c05075fd994a8abad7b2a6cd/FLEDGE_extended_PA_reporting.md?plain=1#L192. switch (base_value) { - case 1: + case BaseValue::BASE_VALUE_WINNING_BID: final_value = base_values.winning_bid; break; - case 2: + case BaseValue::BASE_VALUE_HIGHEST_SCORING_OTHER_BID: final_value = base_values.highest_scoring_other_bid; break; - case 5: - final_value = base_values.reject_reason; + case BaseValue::BASE_VALUE_BID_REJECTION_REASON: + if (!base_values.reject_reason.has_value()) { + return absl::InvalidArgumentError(kBaseValuesRejectReasonNotAvailable); + } + final_value = base_values.reject_reason.value(); break; default: return absl::InvalidArgumentError(kBaseValueNotSupported); @@ -52,8 +127,7 @@ absl::Status HandlePrivateAggregationValuePostAuction( static_cast(final_value * private_aggregation_value.extended_value().scale()) + private_aggregation_value.extended_value().offset(); - private_aggregation_value.set_int_value(final_value); - return absl::OkStatus(); + return final_value; } } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/common/private_aggregation/private_aggregation_post_auction_util.h b/services/common/private_aggregation/private_aggregation_post_auction_util.h index fb7cda48..759e3faf 100644 --- a/services/common/private_aggregation/private_aggregation_post_auction_util.h +++ b/services/common/private_aggregation/private_aggregation_post_auction_util.h @@ -15,6 +15,8 @@ #ifndef SERVICES_COMMON_PRIVATE_AGGREGATION_PRIVATE_AGGREGATION_POST_AUCTION_UTIL_H_ #define SERVICES_COMMON_PRIVATE_AGGREGATION_PRIVATE_AGGREGATION_POST_AUCTION_UTIL_H_ +#include + #include "absl/strings/string_view.h" #include "api/bidding_auction_servers.pb.h" #include "src/util/status_macro/status_macros.h" @@ -22,25 +24,75 @@ namespace privacy_sandbox::bidding_auction_servers { -inline constexpr std::string_view kInvalidPrivateAggregationValueType = +inline constexpr absl::string_view kInvalidPrivateAggregationBucketType = + "Invalid Private Aggregation Bucket with neither bucket_128_bit or " + "signal_bucket."; +inline constexpr absl::string_view kPrivateAggregationBucketNegativeFinalValue = + "Invalid Private Aggregation Signal Bucket BaseValue * scale - offset < 0."; +inline constexpr absl::string_view kInvalidPrivateAggregationValueType = "Invalid Private Aggregation Value with neither Int or Extended Value."; -inline constexpr std::string_view kBaseValueNotSupported = +inline constexpr absl::string_view kBaseValueNotSupported = "Input base value is not supported."; + +inline constexpr absl::string_view kBaseValuesRejectReasonNotAvailable = + "Reject reason is not available."; struct BaseValues { float winning_bid; float highest_scoring_other_bid; - SellerRejectionReason reject_reason; + std::optional reject_reason; }; -// Handle Private Aggregation Value post auction to convert all signalValue into -// numerical values. -// - If Private Aggregation Value is already numerical, no operation is done. +// Calculates and returns the final 128-bit bucket value for private aggregation +// based on the provided BaseValues and PrivateAggregationBucket. +// +// If the bucket 128 bit field is already set, the function simply returns its +// value. +// +// If the bucket contains signal bucket, the function calculates the final +// bucket value using the following equation: +// final_value = base_value_numerical_value * scale + offset +// +// where: +// * base_value_numerical_value is determined by the base_value field of the +// signal_bucket and the corresponding value in the BaseValues object. +// * scale is the scale field of the signal_bucket. +// * offset is the offset field of the signal_bucket. +// +// The final_value is then converted to a Bucket128Bit object and returned. +// +// Args: +// base_values: The BaseValues object containing the winning bid, highest +// scoring other bid, and other relevant values. +// bucket: The PrivateAggregationBucket object containing the bucket +// configuration. +// +// Returns: +// An absl::StatusOr object containing the final 128-bit +// bucket value, or an error status if the calculation fails. +absl::StatusOr GetPrivateAggregationBucketPostAuction( + const BaseValues& base_values, + const PrivateAggregationBucket& private_aggregation_bucket); + +// Convert Private Aggregation Value to integer value after post auction signals +// are available. +// - If Private Aggregation Value is already an integer value, no operation is +// done. // - If Private Aggregation Value is signalValue dependent on post auction -// signals, final numerical value is calculated with (baseValue * scale) + +// signals, final integer value is calculated with (baseValue * scale) + // offset -absl::Status HandlePrivateAggregationValuePostAuction( +// Returns: +// - the integer value if the conversion is successful. +// - absl::InvalidArgumentError(kInvalidPrivateAggregationValueType) if the +// Private Aggregation Value is invalid. +// - absl::InvalidArgumentError(kBaseValueNotSupported) if the base value is +// not supported. +// - absl::InvalidArgumentError(kBaseValuesRejectReasonNotAvailable) if the +// base value is BaseValue::BASE_VALUE_BID_REJECTION_REASON and the +// reject_reason is not available in base_values. This happens when the ad +// is not rejected but reject reason is specified as placeholder base value. +absl::StatusOr GetPrivateAggregationValuePostAuction( const BaseValues& base_values, - PrivateAggregationValue& private_aggregation_value); + const PrivateAggregationValue& private_aggregation_value); } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/common/private_aggregation/private_aggregation_post_auction_util_test.cc b/services/common/private_aggregation/private_aggregation_post_auction_util_test.cc index c8df1c30..d1316b5c 100644 --- a/services/common/private_aggregation/private_aggregation_post_auction_util_test.cc +++ b/services/common/private_aggregation/private_aggregation_post_auction_util_test.cc @@ -19,6 +19,7 @@ #include #include #include +#include #include "api/bidding_auction_servers.pb.h" #include "include/gtest/gtest.h" @@ -27,8 +28,136 @@ namespace privacy_sandbox::bidding_auction_servers { namespace { -TEST(PrivateAggregationPostAuctionUtilTest, - HandlePrivateAggregationValuePostAuctionSuccess) { +void TestGetPrivateAggregationBucketPostAuction( + const BaseValues& base_values, BaseValue base_value, float scale, + const std::vector& offset, bool is_negative_offset, + const std::vector& expected_bucket) { + PrivateAggregationBucket bucket; + bucket.mutable_signal_bucket()->set_base_value(base_value); + bucket.mutable_signal_bucket()->set_scale(scale); + for (uint64_t value : offset) { + bucket.mutable_signal_bucket()->mutable_offset()->add_value(value); + } + bucket.mutable_signal_bucket()->mutable_offset()->set_is_negative( + is_negative_offset); + + absl::StatusOr result = + GetPrivateAggregationBucketPostAuction(base_values, bucket); + EXPECT_TRUE(result.ok()); + EXPECT_EQ(result->bucket_128_bits_size(), expected_bucket.size()); + for (size_t i = 0; i < expected_bucket.size(); ++i) { + EXPECT_EQ(result->bucket_128_bits(i), expected_bucket[i]); + } +} + +void TestGetPrivateAggregationValuePostAuction(BaseValues base_values, + BaseValue base_value, + double scale, int offset, + int expected_value) { + PrivateAggregationValue private_aggregation_value; + + private_aggregation_value.mutable_extended_value()->set_base_value( + base_value); + private_aggregation_value.mutable_extended_value()->set_scale(scale); + private_aggregation_value.mutable_extended_value()->set_offset(offset); + + EXPECT_EQ(GetPrivateAggregationValuePostAuction(base_values, + private_aggregation_value) + .value(), + expected_value); +} + +TEST(GetPrivateAggregationBucketPostAuctionTest, SupportsPositiveOffset) { + BaseValues base_values; + base_values.winning_bid = 10.0f; + base_values.highest_scoring_other_bid = 5.0f; + base_values.reject_reason = SellerRejectionReason::INVALID_BID; + + TestGetPrivateAggregationBucketPostAuction(base_values, + BaseValue::BASE_VALUE_WINNING_BID, + 2.0f, {1, 0}, false, {21, 0}); +} + +TEST(GetPrivateAggregationBucketPostAuctionTest, SupportsNegativeOffset) { + BaseValues base_values; + base_values.winning_bid = 10.0f; + base_values.highest_scoring_other_bid = 5.0f; + base_values.reject_reason = SellerRejectionReason::INVALID_BID; + + TestGetPrivateAggregationBucketPostAuction(base_values, + BaseValue::BASE_VALUE_WINNING_BID, + 2.0f, {1, 0}, true, {19, 0}); +} + +TEST(GetPrivateAggregationBucketPostAuctionTest, Supports128BitAlreadySet) { + Bucket128Bit bucket_128_bit; + bucket_128_bit.add_bucket_128_bits(1); + bucket_128_bit.add_bucket_128_bits(2); + BaseValues base_values; + base_values.winning_bid = 10.0f; + base_values.highest_scoring_other_bid = 5.0f; + base_values.reject_reason = SellerRejectionReason::INVALID_BID; + + PrivateAggregationBucket bucket; + *bucket.mutable_bucket_128_bit() = bucket_128_bit; + + absl::StatusOr result = + GetPrivateAggregationBucketPostAuction(base_values, bucket); + ASSERT_TRUE(result.ok()); + EXPECT_EQ(result->bucket_128_bits(0), 1); + EXPECT_EQ(result->bucket_128_bits(1), 2); +} + +TEST(GetPrivateAggregationBucketPostAuctionTest, + UnsupportedBaseValueReturnFails) { + PrivateAggregationBucket bucket; + bucket.mutable_signal_bucket()->set_base_value( + BaseValue::BASE_VALUE_SCRIPT_RUN_TIME); + bucket.mutable_signal_bucket()->set_scale(2.0f); + bucket.mutable_signal_bucket()->mutable_offset()->add_value(1); + bucket.mutable_signal_bucket()->mutable_offset()->add_value(0); + BaseValues base_values; + base_values.winning_bid = 10.0f; + base_values.highest_scoring_other_bid = 5.0f; + base_values.reject_reason = SellerRejectionReason::INVALID_BID; + EXPECT_EQ( + GetPrivateAggregationBucketPostAuction(base_values, bucket).status(), + absl::Status(absl::StatusCode::kInvalidArgument, kBaseValueNotSupported)); +} + +TEST(GetPrivateAggregationBucketPostAuctionTest, + InvalidPrivateAggregationBucketFails) { + PrivateAggregationBucket bucket; + BaseValues base_values; + base_values.winning_bid = 10.0f; + base_values.highest_scoring_other_bid = 5.0f; + base_values.reject_reason = SellerRejectionReason::INVALID_BID; + EXPECT_EQ( + GetPrivateAggregationBucketPostAuction(base_values, bucket).status(), + absl::Status(absl::StatusCode::kInvalidArgument, + kInvalidPrivateAggregationBucketType)); +} + +TEST(GetPrivateAggregationBucketPostAuctionTest, + NeedRejectionReasonForSignalBucketTypeRejectionReason) { + BaseValues base_values; + base_values.winning_bid = 10.0f; + base_values.highest_scoring_other_bid = 5.0f; + + PrivateAggregationBucket bucket; + bucket.mutable_signal_bucket()->set_base_value( + BaseValue::BASE_VALUE_BID_REJECTION_REASON); + bucket.mutable_signal_bucket()->set_scale(2.0f); + bucket.mutable_signal_bucket()->mutable_offset()->add_value(1); + bucket.mutable_signal_bucket()->mutable_offset()->add_value(0); + + EXPECT_EQ( + GetPrivateAggregationBucketPostAuction(base_values, bucket).status(), + absl::Status(absl::StatusCode::kInvalidArgument, + kBaseValuesRejectReasonNotAvailable)); +} + +TEST(GetPrivateAggregationValuePostAuctionTest, IntValueAlreadySet) { PrivateAggregationValue private_aggregation_value; BaseValues base_values; base_values.winning_bid = 10.0f; @@ -37,54 +166,61 @@ TEST(PrivateAggregationPostAuctionUtilTest, // int_value is already set private_aggregation_value.set_int_value(10); - ASSERT_TRUE(private_aggregation_value.has_int_value()); - ASSERT_EQ(private_aggregation_value.int_value(), 10); - ASSERT_THAT(HandlePrivateAggregationValuePostAuction( - base_values, private_aggregation_value), - absl::OkStatus()); - ASSERT_EQ(private_aggregation_value.int_value(), 10); - - // extended_value is set with base_value = BaseValue::BASE_VALUE_WINNING_BID - private_aggregation_value.mutable_extended_value()->set_base_value( - BaseValue::BASE_VALUE_WINNING_BID); - private_aggregation_value.mutable_extended_value()->set_scale(2.0f); - private_aggregation_value.mutable_extended_value()->set_offset(1.0f); - ASSERT_TRUE(private_aggregation_value.has_extended_value()); - ASSERT_THAT(HandlePrivateAggregationValuePostAuction( - base_values, private_aggregation_value), - absl::OkStatus()); - ASSERT_EQ(private_aggregation_value.int_value(), 21); - ASSERT_FALSE(private_aggregation_value.has_extended_value()); - - // extended_value is set with base_value = - // BaseValue::BASE_VALUE_HIGHEST_SCORING_OTHER_BID - private_aggregation_value.mutable_extended_value()->set_base_value( - BaseValue::BASE_VALUE_HIGHEST_SCORING_OTHER_BID); - private_aggregation_value.mutable_extended_value()->set_scale(2.0f); - private_aggregation_value.mutable_extended_value()->set_offset(1.0f); - ASSERT_TRUE(private_aggregation_value.has_extended_value()); - ASSERT_THAT(HandlePrivateAggregationValuePostAuction( - base_values, private_aggregation_value), - absl::OkStatus()); - ASSERT_EQ(private_aggregation_value.int_value(), 11); - ASSERT_FALSE(private_aggregation_value.has_extended_value()); - - // extended_value is set with base_value = - // BaseValue::BASE_VALUE_BID_REJECTION_REASON - private_aggregation_value.mutable_extended_value()->set_base_value( - BaseValue::BASE_VALUE_BID_REJECTION_REASON); - private_aggregation_value.mutable_extended_value()->set_scale(2.0f); - private_aggregation_value.mutable_extended_value()->set_offset(1.0f); - ASSERT_TRUE(private_aggregation_value.has_extended_value()); - ASSERT_THAT(HandlePrivateAggregationValuePostAuction( - base_values, private_aggregation_value), - absl::OkStatus()); - ASSERT_EQ(private_aggregation_value.int_value(), 3); - ASSERT_FALSE(private_aggregation_value.has_extended_value()); + ASSERT_THAT(GetPrivateAggregationValuePostAuction(base_values, + private_aggregation_value), + private_aggregation_value.int_value()); +} + +TEST(GetPrivateAggregationValuePostAuctionTest, + GetFinalValueBaseValueWinningBid) { + BaseValues base_values; + base_values.winning_bid = 10.0f; + base_values.highest_scoring_other_bid = 5.0f; + base_values.reject_reason = SellerRejectionReason::INVALID_BID; + + double scale = 2.0; + int offset = 1; + + TestGetPrivateAggregationValuePostAuction( + base_values, BaseValue::BASE_VALUE_WINNING_BID, scale, offset, + static_cast(base_values.winning_bid * scale + offset)); } -TEST(PrivateAggregationPostAuctionUtilTest, - HandlePrivateAggregationValuePostAuctionUnsupportedBaseValue) { +TEST(GetPrivateAggregationValuePostAuctionTest, + GetFinalValueBaseValueHighestScoringOtherBid) { + BaseValues base_values; + base_values.winning_bid = 10.0f; + base_values.highest_scoring_other_bid = 5.0f; + base_values.reject_reason = SellerRejectionReason::INVALID_BID; + + double scale = 2.0; + int offset = 1; + + TestGetPrivateAggregationValuePostAuction( + base_values, BaseValue::BASE_VALUE_HIGHEST_SCORING_OTHER_BID, scale, + offset, + static_cast(base_values.highest_scoring_other_bid * scale + offset)); + TestGetPrivateAggregationValuePostAuction( + base_values, BaseValue::BASE_VALUE_BID_REJECTION_REASON, scale, offset, + static_cast(base_values.reject_reason.value() * scale + offset)); +} + +TEST(GetPrivateAggregationValuePostAuctionTest, + GetFinalValueBaseValueBidRejectionReason) { + BaseValues base_values; + base_values.winning_bid = 10.0f; + base_values.highest_scoring_other_bid = 5.0f; + base_values.reject_reason = SellerRejectionReason::INVALID_BID; + + double scale = 2.0; + int offset = 1; + + TestGetPrivateAggregationValuePostAuction( + base_values, BaseValue::BASE_VALUE_BID_REJECTION_REASON, scale, offset, + static_cast(base_values.reject_reason.value() * scale + offset)); +} + +TEST(GetPrivateAggregationValuePostAuctionTest, UnsupportedBaseValue) { PrivateAggregationValue private_aggregation_value; BaseValues base_values; base_values.winning_bid = 10.0f; @@ -97,25 +233,46 @@ TEST(PrivateAggregationPostAuctionUtilTest, BaseValue::BASE_VALUE_SCRIPT_RUN_TIME); private_aggregation_value.mutable_extended_value()->set_scale(2.0f); private_aggregation_value.mutable_extended_value()->set_offset(1.0f); - ASSERT_TRUE(private_aggregation_value.has_extended_value()); ASSERT_THAT( - HandlePrivateAggregationValuePostAuction(base_values, - private_aggregation_value), + GetPrivateAggregationValuePostAuction(base_values, + private_aggregation_value), absl::Status(absl::StatusCode::kInvalidArgument, kBaseValueNotSupported)); } -TEST(PrivateAggregationPostAuctionUtilTest, - HandlePrivateAggregationValuePostAuctionInvalidValue) { +TEST(GetPrivateAggregationValuePostAuctionTest, + GetPrivateAggregationValuePostAuctionInvalidValue) { PrivateAggregationValue private_aggregation_value; BaseValues base_values; base_values.winning_bid = 10.0f; base_values.highest_scoring_other_bid = 5.0f; base_values.reject_reason = SellerRejectionReason::INVALID_BID; - ASSERT_THAT(HandlePrivateAggregationValuePostAuction( - base_values, private_aggregation_value), + ASSERT_THAT(GetPrivateAggregationValuePostAuction(base_values, + private_aggregation_value), absl::Status(absl::StatusCode::kInvalidArgument, kInvalidPrivateAggregationValueType)); } +TEST(GetPrivateAggregationValuePostAuctionTest, + ReturnsInValidArgumentIfRejectReasonNotAvailable) { + PrivateAggregationValue private_aggregation_value; + BaseValues base_values; + base_values.winning_bid = 10.0f; + base_values.highest_scoring_other_bid = 5.0f; + + BaseValue base_value = BaseValue::BASE_VALUE_BID_REJECTION_REASON; + double scale = 2.0; + int offset = 1; + + private_aggregation_value.mutable_extended_value()->set_base_value( + base_value); + private_aggregation_value.mutable_extended_value()->set_scale(scale); + private_aggregation_value.mutable_extended_value()->set_offset(offset); + + EXPECT_THAT(GetPrivateAggregationValuePostAuction(base_values, + private_aggregation_value), + absl::Status(absl::StatusCode::kInvalidArgument, + kBaseValuesRejectReasonNotAvailable)); +} + } // namespace } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/common/private_aggregation/private_aggregation_test_util.cc b/services/common/private_aggregation/private_aggregation_test_util.cc new file mode 100644 index 00000000..157250ea --- /dev/null +++ b/services/common/private_aggregation/private_aggregation_test_util.cc @@ -0,0 +1,165 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "services/common/private_aggregation/private_aggregation_test_util.h" + +#include +#include +#include + +#include "api/bidding_auction_servers.pb.h" + +namespace privacy_sandbox::bidding_auction_servers { + +Bucket128Bit GetTestBucket128Bit() { + Bucket128Bit bucket; + // Set default values for the first 64 bits + bucket.add_bucket_128_bits(0x123456789ABCDEF0); + // Set default values for the second 64 bits + bucket.add_bucket_128_bits(0x0123456789ABCDEF); + return bucket; +} + +PrivateAggregateContribution GetTestContributionWithIntegers( + EventType event_type, absl::string_view event_name = "") { + // Create the example Bucket128Bit + Bucket128Bit bucket_128_bit = GetTestBucket128Bit(); + + // Create the PrivateAggregationValue with int32 value + PrivateAggregationValue private_aggregation_value; + private_aggregation_value.set_int_value(10); + + // Create the PrivateAggregationBucket with Bucket128Bit + PrivateAggregationBucket private_aggregation_bucket; + *private_aggregation_bucket.mutable_bucket_128_bit() = bucket_128_bit; + + // Create a PrivateAggregateContribution and set its fields + PrivateAggregateContribution contribution; + *contribution.mutable_bucket() = private_aggregation_bucket; + *contribution.mutable_value() = private_aggregation_value; + contribution.mutable_event()->set_event_type(event_type); + contribution.mutable_event()->set_event_name(event_name); + return contribution; +} + +SignalValue GetTestSignalValue() { + SignalValue signal_value; + signal_value.set_base_value(BASE_VALUE_WINNING_BID); // Set the base_value + signal_value.set_offset(10); // Set some offset value + signal_value.set_scale(1.5); // Set scale factor + return signal_value; +} + +SignalBucket GetTestSignalBucket() { + SignalBucket signal_bucket; + signal_bucket.set_base_value( + BASE_VALUE_HIGHEST_SCORING_OTHER_BID); // Set the base_value + signal_bucket.set_scale(2.0); // Set scale factor + return signal_bucket; +} + +PrivateAggregateContribution GetTestContributionWithSignalObjects( + EventType event_type, absl::string_view event_name = "") { + // Create the example SignalValue + SignalValue signal_value = GetTestSignalValue(); + + // Create the example SignalBucket + SignalBucket signal_bucket = GetTestSignalBucket(); + + // Create the BucketOffset + BucketOffset bucket_offset; + bucket_offset.add_value( + static_cast(12345678901234567890ULL)); // Add 64-bit values + bucket_offset.add_value(static_cast( + 112233445566778899ULL)); // Add another 64-bit value + bucket_offset.set_is_negative(true); // Set the is_negative flag + + // Set the BucketOffset in SignalBucket + *signal_bucket.mutable_offset() = bucket_offset; + + // Create the PrivateAggregationValue with SignalValue + PrivateAggregationValue private_aggregation_value; + *private_aggregation_value.mutable_extended_value() = signal_value; + + // Create the PrivateAggregationBucket with SignalBucket + PrivateAggregationBucket private_aggregation_bucket; + *private_aggregation_bucket.mutable_signal_bucket() = signal_bucket; + + // Create a PrivateAggregateContribution and set its fields + PrivateAggregateContribution contribution; + *contribution.mutable_bucket() = private_aggregation_bucket; + *contribution.mutable_value() = private_aggregation_value; + contribution.mutable_event()->set_event_type(event_type); + contribution.mutable_event()->set_event_name(event_name); + return contribution; +} + +PrivateAggregateReportingResponse GetTestPrivateAggregateResponse( + std::vector& contributions, + absl::string_view ad_tech_origin) { + PrivateAggregateReportingResponse response; + for (const auto& contribution : contributions) { + *response.add_contributions() = contribution; + } + response.set_adtech_origin(ad_tech_origin); + return response; +} + +AdWithBid GetAdWithBidWithPrivateAggregationContributions( + absl::string_view ig_name) { + PrivateAggregateContribution win_object_contribution = + GetTestContributionWithSignalObjects(EVENT_TYPE_WIN, ""); + PrivateAggregateContribution loss_object_contribution = + GetTestContributionWithSignalObjects(EVENT_TYPE_LOSS, ""); + PrivateAggregateContribution win_int_contribution = + GetTestContributionWithIntegers(EVENT_TYPE_WIN, ""); + PrivateAggregateContribution loss_int_contribution = + GetTestContributionWithIntegers(EVENT_TYPE_LOSS, ""); + AdWithBid ad_with_bid; + *ad_with_bid.add_private_aggregation_contributions() = + win_object_contribution; + *ad_with_bid.add_private_aggregation_contributions() = + loss_object_contribution; + *ad_with_bid.add_private_aggregation_contributions() = win_int_contribution; + *ad_with_bid.add_private_aggregation_contributions() = loss_int_contribution; + ad_with_bid.set_interest_group_name(ig_name); + return ad_with_bid; +} + +GetBidsResponse::GetBidsRawResponse +GetBidsRawResponseWithPrivateAggregationContributions( + absl::string_view ig_name_win, absl::string_view ig_name_loss) { + GetBidsResponse::GetBidsRawResponse get_bids_raw_response; + AdWithBid ad_with_bid_win = + GetAdWithBidWithPrivateAggregationContributions(ig_name_win); + AdWithBid ad_with_bid_loss = + GetAdWithBidWithPrivateAggregationContributions(ig_name_loss); + + *get_bids_raw_response.add_bids() = ad_with_bid_win; + *get_bids_raw_response.add_bids() = ad_with_bid_loss; + + return get_bids_raw_response; +} + +HighestScoringOtherBidsMap GetHighestScoringOtherBidsMap( + absl::string_view ig_owner) { + HighestScoringOtherBidsMap ig_owner_highest_scoring_other_bids_map; + ig_owner_highest_scoring_other_bids_map.try_emplace( + ig_owner, google::protobuf::ListValue()); + ig_owner_highest_scoring_other_bids_map.at(ig_owner) + .add_values() + ->set_number_value(1.0); + return ig_owner_highest_scoring_other_bids_map; +} + +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/common/private_aggregation/private_aggregation_test_util.h b/services/common/private_aggregation/private_aggregation_test_util.h new file mode 100644 index 00000000..07ac91e3 --- /dev/null +++ b/services/common/private_aggregation/private_aggregation_test_util.h @@ -0,0 +1,67 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#ifndef SERVICES_COMMON_PRIVATE_AGGREGATION_PRIVATE_AGGREGATION_TEST_UTIL_H_ +#define SERVICES_COMMON_PRIVATE_AGGREGATION_PRIVATE_AGGREGATION_TEST_UTIL_H_ + +#include +#include +#include + +#include "api/bidding_auction_servers.pb.h" + +namespace privacy_sandbox::bidding_auction_servers { + +using HighestScoringOtherBidsMap = + ::google::protobuf::Map; + +// Creates a test Bucket128Bit object. +Bucket128Bit GetTestBucket128Bit(); + +// Creates a test PrivateAggregateContribution object with int32 value and +// Bucket128Bit. +PrivateAggregateContribution GetTestContributionWithIntegers( + EventType event_type, absl::string_view event_name); + +// Creates a test SignalValue object. +SignalValue GetTestSignalValue(); + +// Creates a test SignalBucket object. +SignalBucket GetTestSignalBucket(); + +// Creates a test PrivateAggregateContribution object with SignalValue and +// SignalBucket. +PrivateAggregateContribution GetTestContributionWithSignalObjects( + EventType event_type, absl::string_view event_name); + +// Creates a test PrivateAggregateReportingResponse object. +PrivateAggregateReportingResponse GetTestPrivateAggregateResponse( + std::vector& contributions, + absl::string_view ad_tech_origin); + +// Creates a test AdWithBid object with input interest group name. +AdWithBid GetAdWithBidWithPrivateAggregationContributions( + absl::string_view ig_name); + +// Creates test pointer to GetBidsRawResponse object. +GetBidsResponse::GetBidsRawResponse +GetBidsRawResponseWithPrivateAggregationContributions( + absl::string_view ig_name_win, absl::string_view ig_name_loss); + +// Creates test highest scoring other bids map with one value. +HighestScoringOtherBidsMap GetHighestScoringOtherBidsMap( + absl::string_view ig_owner); + +} // namespace privacy_sandbox::bidding_auction_servers + +#endif // SERVICES_COMMON_PRIVATE_AGGREGATION_PRIVATE_AGGREGATION_TEST_UTIL_H_ diff --git a/services/common/telemetry/configure_telemetry.h b/services/common/telemetry/configure_telemetry.h index 05fb2971..a2fa0c6e 100644 --- a/services/common/telemetry/configure_telemetry.h +++ b/services/common/telemetry/configure_telemetry.h @@ -44,7 +44,8 @@ template void InitTelemetry(const TrustedServerConfigUtil& config_util, const TrustedServersConfigClient& config_client, absl::string_view server, - const std::vector& buyer_list = {}) { + const std::vector& buyer_list = {}, + const std::vector& model_list = {}) { using ::opentelemetry::logs::LoggerProvider; using ::opentelemetry::sdk::metrics::PeriodicExportingMetricReaderOptions; using ::opentelemetry::sdk::resource::Resource; @@ -110,6 +111,9 @@ void InitTelemetry(const TrustedServerConfigUtil& config_util, if constexpr (std::is_same_v) { AddBuyerPartition(context_map->metric_config(), buyer_list); } + if constexpr (std::is_same_v) { + AddModelPartition(context_map->metric_config(), model_list); + } AddErrorTypePartition(context_map->metric_config(), server); } diff --git a/services/common/test/random.cc b/services/common/test/random.cc index 390fdf9a..3e277e14 100644 --- a/services/common/test/random.cc +++ b/services/common/test/random.cc @@ -22,6 +22,7 @@ namespace privacy_sandbox::bidding_auction_servers { constexpr char kTestIgWithTwoAds[] = R"json({"ad_render_ids":["adg_id=134256827445&cr_id=594352291621&cv_id=4", "adg_id=134256827445&cr_id=605048329089&cv_id=2"],"browser_signals":{"joinCount":1,"bidCount":100,"prevWins":"[[1472425.0,{\"metadata\":[134256827485.0,594352291615.0,null,16996677067.0]}],[1475389.0,{\"metadata\":[134256827445.0,594352291621.0,null,16996677067.0]}],[1487572.0,{\"metadata\":[134256827445.0,605490292974.0,null,16996677067.0]}],[1451707.0,{\"metadata\":[134256827445.0,605048329092.0,null,16996677067.0]}],[1485996.0,{\"metadata\":[134256827445.0,605048329089.0,null,16996677067.0]}],[1450931.0,{\"metadata\":[134256827485.0,605490292980.0,null,16996677067.0]}],[1473069.0,{\"metadata\":[134256827485.0,605048328957.0,null,16996677067.0]}],[1461197.0,{\"metadata\":[134256827485.0,605048329080.0,null,16996677067.0]}]]"},"name":"1j1043317685"})json"; + std::string MakeARandomString() { return std::to_string(ToUnixNanos(absl::Now())); } @@ -163,46 +164,120 @@ std::unique_ptr MakeAnInterestGroupSentFromDevice() { return u_ptr_to_ig; } -InterestGroupForBidding MakeAnInterestGroupForBiddingSentFromDevice() { - std::unique_ptr ig_with_two_ads = - MakeAnInterestGroupSentFromDevice(); - InterestGroupForBidding ig_for_bidding_from_device; +std::string MakeBiddingSignalsValuesForKey(const std::string& key) { + std::string values = "["; + int length = key.back() - '0'; + if (length < 1) { + length = 1; + } + if (length > 9) { + length = 9; + } + for (int j = 0; j < length; j++) { + absl::StrAppend(&values, absl::StrFormat(R"JSON("%s_val%d",)JSON", key, j)); + } + absl::StrAppend(&values, + absl::StrFormat(R"JSON("%s_val%d"])JSON", key, length)); + return values; +} - ig_for_bidding_from_device.set_name(ig_with_two_ads->name()); - ig_for_bidding_from_device.mutable_browser_signals()->CopyFrom( - ig_with_two_ads->browser_signals()); - ig_for_bidding_from_device.mutable_ad_render_ids()->MergeFrom( - ig_with_two_ads->ad_render_ids()); - ig_for_bidding_from_device.mutable_trusted_bidding_signals_keys()->Add( - MakeARandomString()); - return ig_for_bidding_from_device; +std::string MakeBiddingSignalsForIGFromDevice( + const BuyerInput::InterestGroup& interest_group) { + std::string signals_dest; + + // First add the values for the bidding signals keys. + absl::StrAppend(&signals_dest, absl::StrFormat(R"JSON({"keys":{)JSON")); + for (int i = 0; i < interest_group.bidding_signals_keys_size(); i++) { + const auto& bidding_signal_key = interest_group.bidding_signals_keys(i); + + // Append key-value pair. + absl::StrAppend( + &signals_dest, + absl::StrFormat(R"JSON("%s":%s)JSON", bidding_signal_key, + MakeBiddingSignalsValuesForKey(bidding_signal_key))); + + // Add a comma for all but the last key-value pair in this IG. + if (i < interest_group.bidding_signals_keys_size() - 1) { + absl::StrAppend(&signals_dest, ","); + } + } + + // Then add the list of values for the IG. + absl::StrAppend(&signals_dest, + absl::StrFormat(R"JSON(},"perInterestGroupData":{)JSON")); + + // Iterate through all of the trusted bidding signals keys and flatten their + // values into raw_values. + std::string raw_values = "["; + for (int i = 0; i < interest_group.bidding_signals_keys_size(); i++) { + const auto& bidding_signal_key = interest_group.bidding_signals_keys(i); + + // Read in values for this key and append them to raw_values without the + // list brackets on either side. + std::string raw_values_for_key = + MakeBiddingSignalsValuesForKey(bidding_signal_key); + absl::StrAppend(&raw_values, + absl::StrFormat(R"JSON(%s)JSON", + raw_values_for_key.substr( + 1, raw_values_for_key.size() - 2))); + + // Unless we are at the final key, add the last comma for this key's list + // of values. + if (i < interest_group.bidding_signals_keys_size() - 1) { + absl::StrAppend(&raw_values, ","); + } + } + absl::StrAppend(&raw_values, "]"); + + // Add the name-values pair for the IG. + absl::StrAppend(&signals_dest, + absl::StrFormat(R"JSON("%s":%s}})JSON", interest_group.name(), + raw_values)); + + return signals_dest; } -absl::StatusOr MakeRandomTrustedBiddingSignals( +std::string MakeTrustedBiddingSignalsForIG( + const InterestGroupForBidding& interest_group) { + std::string signals_dest = "{"; + // Iterate over each bidding signal key. + for (int i = 0; i < interest_group.trusted_bidding_signals_keys_size(); i++) { + const auto& trusted_bidding_signal_key = + interest_group.trusted_bidding_signals_keys(i); + + // Append key-value pair to trusted bidding signals JSON string. + absl::StrAppend( + &signals_dest, + absl::StrFormat( + R"JSON("%s":%s)JSON", trusted_bidding_signal_key, + MakeBiddingSignalsValuesForKey(trusted_bidding_signal_key))); + + // Add a comma for all but the last key-value pair. + if (i < interest_group.trusted_bidding_signals_keys_size() - 1) { + absl::StrAppend(&signals_dest, ","); + } + } + absl::StrAppend(&signals_dest, "}"); + return signals_dest; +} + +std::string GetBiddingSignalsFromGenerateBidsRequest( const GenerateBidsRequest::GenerateBidsRawRequest& raw_request) { std::string signals_dest; // First add the values for the trusted bidding signals keys for each IG. - absl::StrAppend(&signals_dest, "{\"keys\":{"); - // Iterate through all of the IGs. + absl::StrAppend(&signals_dest, absl::StrFormat(R"JSON({"keys":{)JSON")); for (int i = 0; i < raw_request.interest_group_for_bidding_size(); i++) { const auto& interest_group = raw_request.interest_group_for_bidding(i); - // Now iterate through all of their trusted bidding signals keys. - for (int j = 0; j < interest_group.trusted_bidding_signals_keys_size(); - j++) { - const auto& trusted_bidding_signal_key = - interest_group.trusted_bidding_signals_keys(j); - std::string raw_random_signals; - PS_RETURN_IF_ERROR(google::protobuf::util::MessageToJsonString( - MakeARandomListOfStrings(), &raw_random_signals)); - absl::StrAppend(&signals_dest, absl::StrFormat(R"JSON("%s":%s)JSON", - trusted_bidding_signal_key, - raw_random_signals)); - // Add a comma for all but the last signal-value pair in this IG. - if (j < interest_group.trusted_bidding_signals_keys_size() - 1) { - absl::StrAppend(&signals_dest, ","); - } - } + + // Read in key-value pairs for this IG and append them to signals_dest + // without the JSON brackets on either side. + absl::string_view raw_signals = interest_group.trusted_bidding_signals(); + absl::StrAppend( + &signals_dest, + absl::StrFormat(R"JSON(%s)JSON", + raw_signals.substr(1, raw_signals.size() - 2))); + // Unless we are at the final IG, add the last comma for this IG's key-value // pairs. if (i < raw_request.interest_group_for_bidding_size() - 1) { @@ -210,16 +285,44 @@ absl::StatusOr MakeRandomTrustedBiddingSignals( } } - // Then add the values for the IG names. - absl::StrAppend(&signals_dest, "},\"perInterestGroupData\":{"); + // Then add the list of values for each IG. + absl::StrAppend(&signals_dest, + absl::StrFormat(R"JSON(},"perInterestGroupData":{)JSON")); for (int i = 0; i < raw_request.interest_group_for_bidding_size(); i++) { const auto& interest_group = raw_request.interest_group_for_bidding(i); - std::string raw_random_signals; - PS_RETURN_IF_ERROR(google::protobuf::util::MessageToJsonString( - MakeARandomListOfStrings(), &raw_random_signals)); + + // Iterate through all of the trusted bidding signals keys of this IG and + // flatten their values into raw_values. + std::string raw_values = "["; + for (int j = 0; j < interest_group.trusted_bidding_signals_keys_size(); + j++) { + const auto& trusted_bidding_signal_key = + interest_group.trusted_bidding_signals_keys(j); + + // Read in values for this key and append them to raw_values without the + // list brackets on either side. + std::string raw_values_for_key = + MakeBiddingSignalsValuesForKey(trusted_bidding_signal_key); + absl::StrAppend(&raw_values, + absl::StrFormat(R"JSON(%s)JSON", + raw_values_for_key.substr( + 1, raw_values_for_key.size() - 2))); + + // Unless we are at the final key, add the last comma for this key's list + // of values. + if (j < interest_group.trusted_bidding_signals_keys_size() - 1) { + absl::StrAppend(&raw_values, ","); + } + } + absl::StrAppend(&raw_values, "]"); + + // Add the name-values pair for this IG. absl::StrAppend(&signals_dest, absl::StrFormat(R"JSON("%s":%s)JSON", interest_group.name(), - raw_random_signals)); + raw_values)); + + // Unless we are at the final IG, add the last comma for this IG's + // name-values pair. if (i < raw_request.interest_group_for_bidding_size() - 1) { absl::StrAppend(&signals_dest, ","); } @@ -229,12 +332,31 @@ absl::StatusOr MakeRandomTrustedBiddingSignals( return signals_dest; } +InterestGroupForBidding MakeAnInterestGroupForBiddingSentFromDevice() { + std::unique_ptr ig_with_two_ads = + MakeAnInterestGroupSentFromDevice(); + InterestGroupForBidding ig_for_bidding_from_device; + + ig_for_bidding_from_device.set_name(ig_with_two_ads->name()); + ig_for_bidding_from_device.mutable_browser_signals()->CopyFrom( + ig_with_two_ads->browser_signals()); + ig_for_bidding_from_device.mutable_ad_render_ids()->MergeFrom( + ig_with_two_ads->ad_render_ids()); + ig_for_bidding_from_device.mutable_trusted_bidding_signals_keys()->Add( + MakeARandomString()); + ig_for_bidding_from_device.set_trusted_bidding_signals( + MakeTrustedBiddingSignalsForIG(ig_for_bidding_from_device)); + return ig_for_bidding_from_device; +} + InterestGroupForBidding MakeARandomInterestGroupForBidding( bool build_android_signals, bool set_user_bidding_signals_to_empty_struct) { InterestGroupForBidding ig_for_bidding; ig_for_bidding.set_name(absl::StrCat("ig_name_random_", MakeARandomString())); ig_for_bidding.mutable_trusted_bidding_signals_keys()->Add( absl::StrCat("trusted_bidding_signals_key_random_", MakeARandomString())); + ig_for_bidding.set_trusted_bidding_signals( + MakeTrustedBiddingSignalsForIG(ig_for_bidding)); if (!set_user_bidding_signals_to_empty_struct) { // Takes ownership of pointer. ig_for_bidding.set_user_bidding_signals( @@ -278,6 +400,9 @@ InterestGroupForBidding MakeALargeInterestGroupForBiddingForLatencyTesting() { ig_for_bidding.mutable_trusted_bidding_signals_keys()->Add( absl::StrCat("biddingSignalsKey", i + 1)); } + // Bidding signals. + ig_for_bidding.set_trusted_bidding_signals( + MakeTrustedBiddingSignalsForIG(ig_for_bidding)); // User bidding signals. ig_for_bidding.set_user_bidding_signals( MakeAFixedSetOfUserBiddingSignals(num_user_bidding_signals)); @@ -308,7 +433,6 @@ MakeARandomGenerateBidsRawRequestForAndroid() { std::move(MakeARandomStructJsonString(MakeARandomInt(0, 100))).release()); raw_request.set_allocated_buyer_signals( std::move(MakeARandomStructJsonString(MakeARandomInt(0, 100))).release()); - raw_request.set_bidding_signals(MakeARandomString()); return raw_request; } @@ -326,7 +450,6 @@ MakeARandomGenerateBidsRequestForBrowser() { std::move(MakeARandomStructJsonString(MakeARandomInt(0, 100))).release()); raw_request.set_allocated_buyer_signals( std::move(MakeARandomStructJsonString(MakeARandomInt(0, 10))).release()); - raw_request.set_bidding_signals(MakeARandomString()); raw_request.set_seller(MakeARandomString()); raw_request.set_publisher_name(MakeARandomString()); diff --git a/services/common/test/random.h b/services/common/test/random.h index 5ffb52d2..ebe1b4e2 100644 --- a/services/common/test/random.h +++ b/services/common/test/random.h @@ -24,7 +24,10 @@ #include #include "absl/container/flat_hash_map.h" +#include "absl/log/check.h" +#include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" #include "absl/time/time.h" #include "api/bidding_auction_servers.pb.h" #include "services/common/test/utils/cbor_test_utils.h" @@ -81,12 +84,35 @@ BrowserSignals MakeRandomBrowserSignalsForIG( // Must manually delete/take ownership of underlying pointer std::unique_ptr MakeAnInterestGroupSentFromDevice(); -InterestGroupForBidding MakeAnInterestGroupForBiddingSentFromDevice(); - -// Build random trusted bidding signals with interest group names. -absl::StatusOr MakeRandomTrustedBiddingSignals( +// Build bidding signals values for given key. +std::string MakeBiddingSignalsValuesForKey(const std::string& key); + +// Build bidding signals for an interest group sent from device. Returns a JSON +// string. Example: +// {"keys": {"key1": ["val1", "val2"], +// "key2": ["val3"]}, +// "perInterestGroupData": {"ig1": ["val1", "val2", "val3"]}} +std::string MakeBiddingSignalsForIGFromDevice( + const BuyerInput::InterestGroup& interest_group); + +// Build trusted bidding signals for an interest group for bidding. Returns a +// JSON string. Example: +// {"key1": ["val1", "val2"], +// "key2": ["val3"]} +std::string MakeTrustedBiddingSignalsForIG( + const InterestGroupForBidding& interest_group); + +// Get bidding signals for all the IGs in given GenerateBidsRawRequest. Returns +// a JSON string. Example: +// {"keys": {"key1": ["val1", "val2"], +// "key2": ["val3"]}, +// "perInterestGroupData": {"ig1": ["val1", "val2", "val3"]} +// "ig2": ["val3"]}} +std::string GetBiddingSignalsFromGenerateBidsRequest( const GenerateBidsRequest::GenerateBidsRawRequest& raw_request); +InterestGroupForBidding MakeAnInterestGroupForBiddingSentFromDevice(); + // build_android_signals: If false, will insert random values into // browser signals, otherwise will insert random values into android signals. InterestGroupForBidding MakeARandomInterestGroupForBidding( diff --git a/services/common/test/utils/test_utils.cc b/services/common/test/utils/test_utils.cc index 97b2a1ae..b4a96214 100644 --- a/services/common/test/utils/test_utils.cc +++ b/services/common/test/utils/test_utils.cc @@ -46,6 +46,10 @@ GetBidsRequest::GetBidsRawRequest CreateGetBidsRawRequest( } if (add_protected_audience_input) { *raw_request.mutable_buyer_input() = MakeARandomBuyerInput(); + auto interest_group = + raw_request.mutable_buyer_input()->mutable_interest_groups()->Add(); + interest_group->set_name("ig_name"); + interest_group->add_bidding_signals_keys("key"); } return raw_request; } diff --git a/services/common/util/binary_http_utils.h b/services/common/util/binary_http_utils.h index 9e00e15d..091cfac4 100644 --- a/services/common/util/binary_http_utils.h +++ b/services/common/util/binary_http_utils.h @@ -23,8 +23,8 @@ #include "absl/status/statusor.h" #include "google/protobuf/util/json_util.h" #include "quiche/binary_http/binary_http_message.h" +#include "services/common/loggers/request_log_context.h" #include "services/common/util/request_response_constants.h" -#include "src/logger/request_context_logger.h" #include "src/util/status_macro/status_macros.h" namespace privacy_sandbox::bidding_auction_servers { diff --git a/services/common/util/json_util.h b/services/common/util/json_util.h index f76e33aa..9134b9c1 100644 --- a/services/common/util/json_util.h +++ b/services/common/util/json_util.h @@ -102,7 +102,21 @@ inline absl::StatusOr> SerializeJsonDoc( return absl::InternalError("Unknown JSON to string serialization error"); } -// Converts rapidjson::Value& to a string +// Converts rapidjson::Document to a string. The reserve_string_len argument +// helps reserve a large string size up front to prevent reallocation and +// copying. +inline absl::StatusOr SerializeJsonDocToReservedString( + const rapidjson::Document& document, int reserve_string_len) { + rapidjson::StringBuffer string_buffer; + string_buffer.Reserve(reserve_string_len); + rapidjson::Writer writer(string_buffer); + if (document.Accept(writer)) { + return std::string(string_buffer.GetString()); + } + return absl::InternalError("Unknown JSON to string serialization error"); +} + +// Converts rapidjson::Value& to a string. inline absl::StatusOr SerializeJsonDoc( const rapidjson::Value& document) { rapidjson::StringBuffer string_buffer; @@ -110,7 +124,7 @@ inline absl::StatusOr SerializeJsonDoc( if (document.Accept(writer)) { return std::string(string_buffer.GetString()); } - return absl::InternalError("Error converting inner Json to String."); + return absl::InternalError("Error converting inner JSON to String."); } // Converts rapidjson::Document to a string. @@ -208,6 +222,44 @@ inline absl::StatusOr GetIntMember(const T& document, return it->value.GetInt(); } +// Retrieves the double value of the specified member in the document. +template +inline absl::StatusOr GetDoubleMember(const T& document, + absl::string_view member_name) { + auto it = document.FindMember(member_name.data()); + if (it == document.MemberEnd()) { + return absl::InvalidArgumentError( + absl::StrFormat(kMissingMember, member_name)); + } + + if (!it->value.IsDouble()) { + return absl::InvalidArgumentError( + absl::StrFormat(kUnexpectedMemberType, member_name, + rapidjson::kNumberType, it->value.GetType())); + } + + return it->value.GetDouble(); +} + +// Retrieves the boolean value of the specified member in the document. +template +inline absl::StatusOr GetBoolMember(const T& document, + absl::string_view member_name) { + auto it = document.FindMember(member_name.data()); + if (it == document.MemberEnd()) { + return absl::InvalidArgumentError( + absl::StrFormat(kMissingMember, member_name)); + } + + if (!it->value.IsBool()) { + return absl::InvalidArgumentError( + absl::StrFormat(kUnexpectedMemberType, member_name, + rapidjson::kNumberType, it->value.GetType())); + } + + return it->value.GetBool(); +} + } // namespace privacy_sandbox::bidding_auction_servers #endif // SERVICES_COMMON_UTIL_JSON_UTIL_H_ diff --git a/services/common/util/json_util_test.cc b/services/common/util/json_util_test.cc index 87fd97ae..97ff3e45 100644 --- a/services/common/util/json_util_test.cc +++ b/services/common/util/json_util_test.cc @@ -57,6 +57,25 @@ TEST(SerializeJsonDoc, WorksForValidDocWithSize) { EXPECT_STREQ(output.value()->c_str(), expected_output.c_str()); } +TEST(SerializeJsonDoc, WorksForValidDocWithSizeToReserveString) { + std::string key = MakeARandomString(); + std::string value = MakeARandomString(); + std::string expected_output = "{\"" + key + "\":\"" + value + "\"}"; + + rapidjson::Document document; + document.SetObject(); + rapidjson::Value key_v; + key_v.SetString(key.c_str(), document.GetAllocator()); + rapidjson::Value val_v; + val_v.SetString(value.c_str(), document.GetAllocator()); + document.AddMember(key_v, val_v.Move(), document.GetAllocator()); + + absl::StatusOr output = + SerializeJsonDocToReservedString(document, 20); + ASSERT_TRUE(output.ok()) << output.status(); + EXPECT_STREQ(output.value().c_str(), expected_output.c_str()); +} + TEST(SerializeJsonDoc, WorksForValidDoc) { std::string key = MakeARandomString(); std::string value = MakeARandomString(); @@ -196,5 +215,61 @@ TEST(SerializeJsonDoc, GetNumberMember_ComplainsOnNonIntType) { EXPECT_FALSE(actual_value.ok()) << actual_value.status(); } +TEST(SerializeJsonDoc, GetDoubleMember_ParsesTheNumberSuccessfully) { + std::string json_str = R"json({"key": 123.67})json"; + auto document = ParseJsonString(json_str); + ASSERT_TRUE(document.ok()) << document.status(); + + auto actual_value = GetDoubleMember(*document, "key"); + ASSERT_TRUE(actual_value.ok()); + EXPECT_EQ(*actual_value, 123.67); +} + +TEST(SerializeJsonDoc, GetDoubleMember_ComplainsOnMissingKey) { + std::string json_str = R"json({"key": 123.67})json"; + auto document = ParseJsonString(json_str); + ASSERT_TRUE(document.ok()) << document.status(); + + auto actual_value = GetDoubleMember(*document, "MissingKey"); + EXPECT_FALSE(actual_value.ok()) << actual_value.status(); +} + +TEST(SerializeJsonDoc, GetNumberMember_ComplainsOnNonDoubleType) { + std::string json_str = R"json({"key": 123})json"; + auto document = ParseJsonString(json_str); + ASSERT_TRUE(document.ok()) << document.status(); + + auto actual_value = GetDoubleMember(*document, "Key"); + EXPECT_FALSE(actual_value.ok()) << actual_value.status(); +} + +TEST(SerializeJsonDoc, GetBoolMember_ParsesTheNumberSuccessfully) { + std::string json_str = R"json({"key": true})json"; + auto document = ParseJsonString(json_str); + ASSERT_TRUE(document.ok()) << document.status(); + + auto actual_value = GetBoolMember(*document, "key"); + ASSERT_TRUE(actual_value.ok()); + EXPECT_EQ(*actual_value, true); +} + +TEST(SerializeJsonDoc, GetBoolMember_ComplainsOnMissingKey) { + std::string json_str = R"json({"key": 123.67})json"; + auto document = ParseJsonString(json_str); + ASSERT_TRUE(document.ok()) << document.status(); + + auto actual_value = GetBoolMember(*document, "MissingKey"); + EXPECT_FALSE(actual_value.ok()) << actual_value.status(); +} + +TEST(SerializeJsonDoc, GetBoolMember_ComplainsOnNonBoolType) { + std::string json_str = R"json({"key": 123})json"; + auto document = ParseJsonString(json_str); + ASSERT_TRUE(document.ok()) << document.status(); + + auto actual_value = GetBoolMember(*document, "key"); + EXPECT_FALSE(actual_value.ok()) << actual_value.status(); +} + } // namespace } // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/common/util/request_response_constants.h b/services/common/util/request_response_constants.h index f468bf5a..de5677c9 100644 --- a/services/common/util/request_response_constants.h +++ b/services/common/util/request_response_constants.h @@ -138,20 +138,6 @@ inline constexpr std::array enum class AuctionType : int { kProtectedAudience, kProtectedAppSignals }; -// log verbosity - -inline constexpr int kPlain = 1; // plaintext B&A request and response served -inline constexpr int kNoisyWarn = - 2; // non-critical error, use PS_LOG(ERROR, *) for critical error -inline constexpr int kUdfLog = 3; -inline constexpr int kSuccess = 3; -inline constexpr int kNoisyInfo = 4; -inline constexpr int kDispatch = 4; // UDF dispatch request and response -inline constexpr int kOriginated = - 5; // plaintext B&A request and response originated from server -inline constexpr int kKVLog = 5; // KV request response -inline constexpr int kStats = 5; // Stats log like time , byte size, etc. -inline constexpr int kEncrypted = 6; } // namespace privacy_sandbox::bidding_auction_servers #endif // SERVICES_COMMON_UTIL_REQUEST_RESPONSE_CONSTANTS_H_ diff --git a/services/inference_sidecar/README.md b/services/inference_sidecar/README.md index cd465f57..0624142a 100644 --- a/services/inference_sidecar/README.md +++ b/services/inference_sidecar/README.md @@ -204,7 +204,7 @@ one with a shape `[2,3]` will expect a 6 element one. Currently we support 6 tensor types (both in Tensorflow and PyTorch): double, float, int8, int16, int32, int64. -All numbers in `tensor_content` MUST be surrounded by quotes. +All numbers in `tensor_content` MUST be in string format. See an example of a Batch Inference Request in a JSON format with 2 models: @@ -243,6 +243,37 @@ See an example of a Batch Inference Request in a JSON format with 2 models: } ``` +The response for the above request would look like this : + +```json +{ + "response": [ + { + "model_path": "my_bucket/models/pcvr/1/", + "tensors": [ + { + "tensor_name": "StatefulPartitionedCall:0", + "data_type": "FLOAT", + "tensor_shape": [2, 1], + "tensor_content": [0.11673324, 0.178222424] + } + ] + }, + { + "model_path": "my_bucket/models/pctr/2/", + "tensors": [ + { + "tensor_name": "StatefulPartitionedCall:0", + "data_type": "FLOAT", + "tensor_shape": [2, 1], + "tensor_content": [0.342842, 0.23468234] + } + ] + } + ] +} +``` + - Please refer to [request_parser.h](https://github.com/privacysandbox/bidding-auction-servers/tree/main/services/inference_sidecar/common/utils/request_parser.h) and diff --git a/services/inference_sidecar/common/model/BUILD b/services/inference_sidecar/common/model/BUILD index 51a757d4..f65f17b0 100644 --- a/services/inference_sidecar/common/model/BUILD +++ b/services/inference_sidecar/common/model/BUILD @@ -23,6 +23,8 @@ cc_library( "//proto:inference_sidecar_cc_proto", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/functional:any_invocable", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/random", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", @@ -38,6 +40,7 @@ cc_test( "//proto:inference_sidecar_cc_proto", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", + "@com_google_absl//absl/time", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", ], diff --git a/services/inference_sidecar/common/model/model_store.h b/services/inference_sidecar/common/model/model_store.h index 603c63ac..0165254d 100644 --- a/services/inference_sidecar/common/model/model_store.h +++ b/services/inference_sidecar/common/model/model_store.h @@ -15,17 +15,22 @@ #ifndef SERVICES_INFERENCE_SIDECAR_COMMON_MODEL_MODEL_STORE_H_ #define SERVICES_INFERENCE_SIDECAR_COMMON_MODEL_MODEL_STORE_H_ +#include #include #include +#include #include #include #include "absl/container/flat_hash_map.h" #include "absl/functional/any_invocable.h" +#include "absl/log/check.h" +#include "absl/random/random.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/synchronization/mutex.h" +#include "absl/synchronization/notification.h" #include "proto/inference_sidecar.pb.h" #include "src/util/status_macro/status_macros.h" @@ -41,13 +46,40 @@ template class ModelStore { public: using ModelConstructor = - absl::AnyInvocable>( + absl::AnyInvocable>( const InferenceSidecarRuntimeConfig&, const RegisterModelRequest&) const>; explicit ModelStore(const InferenceSidecarRuntimeConfig& config, ModelConstructor model_constructor) - : config_(config), model_constructor_(std::move(model_constructor)) {} + : config_(config), + model_constructor_(std::move(model_constructor)), + model_reset_background_thread_running_(true) { + model_reset_background_thread_ = std::thread([this]() { + while (model_reset_background_thread_running_) { + // ResetModels() is continually called in the background thread. + ResetModels(); + // Waits for the completed inference execution. + // + // Moves on after 1 second and check if the background thread should + // terminate. The destructor terminates this thread by setting + // |model_reset_background_thread_running_|. + inference_notification_.WaitForNotificationWithTimeout( + absl::Seconds(1)); + } + }); + } + + virtual ~ModelStore() { + model_reset_background_thread_running_ = false; + model_reset_background_thread_.join(); + } + + ModelStore(const ModelStore&) = delete; + ModelStore& operator=(const ModelStore&) = delete; + + ModelStore(ModelStore&&) = delete; + ModelStore& operator=(ModelStore&&) = delete; // Puts model into the model store. Creates both a copy for consented traffic // and a copy for prod traffic. Future reset calls use the saved RegisterModel @@ -55,9 +87,9 @@ class ModelStore { // This method is thread-safe. absl::Status PutModel(absl::string_view key, const RegisterModelRequest& request) { - PS_ASSIGN_OR_RETURN(std::unique_ptr prod_model, + PS_ASSIGN_OR_RETURN(std::shared_ptr prod_model, model_constructor_(config_, request)); - PS_ASSIGN_OR_RETURN(std::unique_ptr consented_model, + PS_ASSIGN_OR_RETURN(std::shared_ptr consented_model, model_constructor_(config_, request)); absl::MutexLock model_data_lock(&model_data_mutex_); @@ -101,7 +133,7 @@ class ModelStore { "' fails because it has not been registered")); } const RegisterModelRequest& request = it->second; - PS_ASSIGN_OR_RETURN(std::unique_ptr model, + PS_ASSIGN_OR_RETURN(std::shared_ptr model, model_constructor_(config_, request)); absl::flat_hash_map>& model_map = @@ -121,11 +153,14 @@ class ModelStore { return model_keys; } - ModelStore(const ModelStore&) = delete; - ModelStore& operator=(const ModelStore&) = delete; - - ModelStore(ModelStore&&) = delete; - ModelStore& operator=(ModelStore&&) = delete; + // Increments the inference count associated with a model key for model + // resetting purposes. Note that model reset currently only happens for prod + // models so no consented flag is required for this method. + void IncrementModelInferenceCount(absl::string_view key) { + absl::MutexLock lock(&per_model_inference_count_mu_); + ++per_model_inference_count_[std::string(key)]; + inference_notification_.Notify(); + } protected: // Used for test only, the caller needs to ensure thread-safety. @@ -134,6 +169,38 @@ class ModelStore { } private: + // Called periodically by `model_reset_background_thread_` to reset models in + // a probabilistic fashion. + void ResetModels() { + const double reset_probability = config_.model_reset_probability(); + if (reset_probability == 0.0) { + // Model reset is disabled. + return; + } + + std::vector models = ListModels(); + for (const auto& model_key : models) { + int count = 0; + { + absl::MutexLock lock(&per_model_inference_count_mu_); + count = per_model_inference_count_[model_key]; + // Sets the per-model counter to 0. + per_model_inference_count_[model_key] = 0; + } + if (count <= 0) continue; + double random = absl::Uniform(bitgen_, 0.0, 1.0); + // Boosts the chance of reset multiplied by the number of inferences as + // approximation. + if (reset_probability != 1.0 && random >= reset_probability * count) { + continue; + } + + // We should make sure the model reset is successfully done. + // Otherwise, we terminate the program to preserve user privacy. + CHECK_OK(ResetModel(model_key)) << "Failed to reset model: " << model_key; + } + } + // Always lock on `model_data_mutex_` before `prod_model_mutex_` and // then `consented_model_mutex_` to avoid deadlock. mutable absl::Mutex model_data_mutex_ ABSL_ACQUIRED_BEFORE(prod_model_mutex_); @@ -149,6 +216,19 @@ class ModelStore { const InferenceSidecarRuntimeConfig config_; ModelConstructor model_constructor_; + + // Background thread for model reset. + std::thread model_reset_background_thread_; + // The background thread is shutdown if set to false. + std::atomic model_reset_background_thread_running_; + // Exclusively used in a single backgrond thread. No need of a mutex. + absl::BitGen bitgen_; + // Notification to trigger model reset. + absl::Notification inference_notification_; + // Counts the number of inferences per model. Used for model reset. + absl::flat_hash_map per_model_inference_count_ + ABSL_GUARDED_BY(per_model_inference_count_mu_); + absl::Mutex per_model_inference_count_mu_; }; } // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/inference_sidecar/common/model/model_store_test.cc b/services/inference_sidecar/common/model/model_store_test.cc index 371267c0..b638757f 100644 --- a/services/inference_sidecar/common/model/model_store_test.cc +++ b/services/inference_sidecar/common/model/model_store_test.cc @@ -27,6 +27,8 @@ #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" +#include "absl/time/clock.h" +#include "absl/time/time.h" #include "proto/inference_sidecar.pb.h" namespace privacy_sandbox::bidding_auction_servers::inference { @@ -403,5 +405,56 @@ TEST_F(ModelStoreConcurrencyTest, } } +class ModelStoreBackgroundModelResetTest : public ::testing::Test { + protected: + ModelStoreBackgroundModelResetTest() + : store_(BuildAlwaysResetInferenceSidecarRuntimeConfig(), + MockModelConstructorWithInitCounter) {} + + MockModelStore store_; + + private: + InferenceSidecarRuntimeConfig + BuildAlwaysResetInferenceSidecarRuntimeConfig() { + InferenceSidecarRuntimeConfig config; + config.set_model_reset_probability(1.0); + return config; + } +}; + +TEST_F(ModelStoreBackgroundModelResetTest, ResetSuccessWithStatefulModels) { + for (int i = 0; i < kNumThreads; ++i) { + EXPECT_TRUE(store_ + .PutModel(absl::StrCat(kTestModelName, i), + BuildMockModelRegisterModelRequest(1)) + .ok()); + } + + std::vector threads; + threads.reserve(kNumThreads); + for (int i = 0; i < kNumThreads; ++i) { + threads.push_back(std::thread([this, i]() { + absl::StatusOr> prod_model = store_.GetModel( + absl::StrCat(kTestModelName, i), /*is_consented=*/false); + ASSERT_TRUE(prod_model.ok()); + EXPECT_EQ((*prod_model)->GetCounter(), 1); + (*prod_model)->Increment(); + store_.IncrementModelInferenceCount(absl::StrCat(kTestModelName, i)); + })); + } + + for (auto& thread : threads) { + thread.join(); + } + + absl::SleepFor(absl::Seconds(1)); + for (int i = 0; i < kNumThreads; ++i) { + absl::StatusOr> prod_model = store_.GetModel( + absl::StrCat(kTestModelName, i), /*is_consented=*/false); + ASSERT_TRUE(prod_model.ok()); + EXPECT_EQ((*prod_model)->GetCounter(), 1); + } +} + } // namespace } // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/inference_sidecar/common/modules/module_interface.h b/services/inference_sidecar/common/modules/module_interface.h index d54d015c..47da604c 100644 --- a/services/inference_sidecar/common/modules/module_interface.h +++ b/services/inference_sidecar/common/modules/module_interface.h @@ -42,8 +42,6 @@ class ModuleInterface { // Registers a new model. virtual absl::StatusOr RegisterModel( const RegisterModelRequest& request) = 0; - // Resets models. - virtual void ResetModels() = 0; }; } // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/inference_sidecar/common/modules/test_module.cc b/services/inference_sidecar/common/modules/test_module.cc index 73f30783..cf10615d 100644 --- a/services/inference_sidecar/common/modules/test_module.cc +++ b/services/inference_sidecar/common/modules/test_module.cc @@ -79,12 +79,6 @@ absl::StatusOr TestModule::RegisterModel( return response; } -void TestModule::ResetModels() { - if (config_.model_reset_probability() == 0.0) { - ABSL_LOG(INFO) << "Model reset is disabled."; - } -} - std::unique_ptr ModuleInterface::Create( const InferenceSidecarRuntimeConfig& config) { return std::make_unique(config); diff --git a/services/inference_sidecar/common/modules/test_module.h b/services/inference_sidecar/common/modules/test_module.h index c0aedc9f..adfad629 100644 --- a/services/inference_sidecar/common/modules/test_module.h +++ b/services/inference_sidecar/common/modules/test_module.h @@ -43,7 +43,6 @@ class TestModule final : public ModuleInterface { const RequestContext& request_context) override; absl::StatusOr RegisterModel( const RegisterModelRequest& request) override; - void ResetModels() override; void set_model_path(absl::string_view path) { model_path_ = path; } diff --git a/services/inference_sidecar/common/modules/test_module_test.cc b/services/inference_sidecar/common/modules/test_module_test.cc index 2c1f360c..5fb9b87d 100644 --- a/services/inference_sidecar/common/modules/test_module_test.cc +++ b/services/inference_sidecar/common/modules/test_module_test.cc @@ -53,11 +53,5 @@ TEST(TestModule, Success_ReadModel) { EXPECT_GT(module->model_size(), 0); } -TEST(TestModule, Success_ResetModels) { - InferenceSidecarRuntimeConfig config; - std::unique_ptr module = std::make_unique(config); - module->ResetModels(); -} - } // namespace } // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/inference_sidecar/common/proto/inference_sidecar.proto b/services/inference_sidecar/common/proto/inference_sidecar.proto index dc889d04..1dd3e06d 100644 --- a/services/inference_sidecar/common/proto/inference_sidecar.proto +++ b/services/inference_sidecar/common/proto/inference_sidecar.proto @@ -69,7 +69,7 @@ message RegisterModelRequest { // Batch inference requests used to perform warm up for target model during model // registration, schema should follow BatchInferenceRequest in inference_payload.proto // request text should be in json format. - bytes warm_up_batch_request_json = 3; + string warm_up_batch_request_json = 3; } message RegisterModelResponse { diff --git a/services/inference_sidecar/common/proto/model_metadata.proto b/services/inference_sidecar/common/proto/model_metadata.proto index c2627bfb..2d2c5342 100644 --- a/services/inference_sidecar/common/proto/model_metadata.proto +++ b/services/inference_sidecar/common/proto/model_metadata.proto @@ -42,6 +42,5 @@ message ModelMetadata { // Batch inference requests used to perform warm up for target model during model // registration, schema should follow BatchInferenceRequest in inference_payload.proto // request text should be in json format. - bytes warm_up_batch_request_json = 3; - // TODO(b/346816785): Add metadata for model prewarming. + string warm_up_batch_request_json = 3; } diff --git a/services/inference_sidecar/common/testdata/BUILD b/services/inference_sidecar/common/testdata/BUILD index 53435e74..fe42dcb3 100644 --- a/services/inference_sidecar/common/testdata/BUILD +++ b/services/inference_sidecar/common/testdata/BUILD @@ -27,4 +27,8 @@ exports_files([ "models/pytorch_mixed_inputs_mixed_outputs_model.pt", # This PyTorch model returns the incrementing counter. "models/pytorch_stateful_model.pt", + # This PyTorch model returns itself and is not freezable. + "models/pytorch_unfreezable_model.pt", + # This PyTorch model returns is already frozen. + "models/pytorch_frozen_model.pt", ]) diff --git a/services/inference_sidecar/common/testdata/models/pytorch_frozen_model.pt b/services/inference_sidecar/common/testdata/models/pytorch_frozen_model.pt new file mode 100644 index 00000000..48d8c339 Binary files /dev/null and b/services/inference_sidecar/common/testdata/models/pytorch_frozen_model.pt differ diff --git a/services/inference_sidecar/common/testdata/models/pytorch_unfreezable_model.pt b/services/inference_sidecar/common/testdata/models/pytorch_unfreezable_model.pt new file mode 100644 index 00000000..4ae5b18c Binary files /dev/null and b/services/inference_sidecar/common/testdata/models/pytorch_unfreezable_model.pt differ diff --git a/services/inference_sidecar/common/utils/BUILD b/services/inference_sidecar/common/utils/BUILD index 2fc96f85..e366b0eb 100644 --- a/services/inference_sidecar/common/utils/BUILD +++ b/services/inference_sidecar/common/utils/BUILD @@ -114,12 +114,24 @@ cc_test( ], ) +cc_library( + name = "inference_error_code", + hdrs = ["inference_error_code.h"], + visibility = ["//visibility:public"], + deps = [ + "@com_google_absl//absl/strings", + ], +) + cc_library( name = "inference_metric_util", srcs = ["inference_metric_util.cc"], hdrs = ["inference_metric_util.h"], + visibility = ["//visibility:public"], deps = [ + ":inference_error_code", "//proto:inference_sidecar_cc_proto", + "@com_google_absl//absl/strings", "@com_google_googletest//:gtest", ], ) @@ -129,6 +141,7 @@ cc_test( size = "small", srcs = ["inference_metric_util_test.cc"], deps = [ + ":inference_error_code", ":inference_metric_util", "//proto:inference_sidecar_cc_proto", "@com_google_googletest//:gtest", diff --git a/services/inference_sidecar/common/utils/inference_error_code.h b/services/inference_sidecar/common/utils/inference_error_code.h new file mode 100644 index 00000000..e6b0da4a --- /dev/null +++ b/services/inference_sidecar/common/utils/inference_error_code.h @@ -0,0 +1,60 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SERVICES_INFERENCE_SIDECAR_COMMON_UTILS_INFERENCE_ERROR_CODE_H_ +#define SERVICES_INFERENCE_SIDECAR_COMMON_UTILS_INFERENCE_ERROR_CODE_H_ + +#include + +#include "absl/strings/string_view.h" + +namespace privacy_sandbox::bidding_auction_servers::inference { + +inline constexpr absl::string_view kInferenceUnableToParseRequest = + "Unable to Parse Request"; // Input request format errors +inline constexpr absl::string_view kInferenceOutputParsingError = + "Output Parsing Error"; // Failures in parsing or formatting the output + +inline constexpr absl::string_view kInferenceTensorInputNameError = + "Tensor Input Name Error"; // Missing tensor name in inputs +inline constexpr absl::string_view kInferenceModelNotFoundError = + "Model Not Found Error"; // Model could not be loaded or found +inline constexpr absl::string_view kInferenceModelExecutionError = + "Model Execution Error"; // Errors during model execution +inline constexpr absl::string_view kInferenceOutputTensorMismatchError = + "Output Tensor Mismatch Error"; // Output tensor count mismatches +inline constexpr absl::string_view kInferenceSignatureNotFoundError = + "Signature Not Found Error"; // Model signature missing +inline constexpr absl::string_view kInferenceInputTensorConversionError = + "Input Tensor Conversion Error"; // Issues converting data to tensors +inline constexpr absl::string_view kInferenceUnknownError = + "Unknown Error"; // Default error + +inline constexpr absl::string_view kInferenceErrorCode[]{ + kInferenceInputTensorConversionError, + kInferenceModelExecutionError, + kInferenceModelNotFoundError, + kInferenceOutputParsingError, + kInferenceOutputTensorMismatchError, + kInferenceSignatureNotFoundError, + kInferenceTensorInputNameError, + kInferenceUnableToParseRequest, + kInferenceUnknownError, +}; + +} // namespace privacy_sandbox::bidding_auction_servers::inference + +#endif // SERVICES_INFERENCE_SIDECAR_COMMON_UTILS_INFERENCE_ERROR_CODE_H_ diff --git a/services/inference_sidecar/common/utils/inference_metric_util.cc b/services/inference_sidecar/common/utils/inference_metric_util.cc index b7073959..636d6f2f 100644 --- a/services/inference_sidecar/common/utils/inference_metric_util.cc +++ b/services/inference_sidecar/common/utils/inference_metric_util.cc @@ -14,8 +14,11 @@ #include "utils/inference_metric_util.h" +#include "absl/strings/match.h" +#include "absl/strings/string_view.h" #include "gtest/gtest.h" #include "proto/inference_sidecar.pb.h" +#include "utils/inference_error_code.h" namespace privacy_sandbox::bidding_auction_servers::inference { @@ -29,4 +32,24 @@ void AddMetric(PredictResponse& response, const std::string& key, int32_t value, response.mutable_metrics()->insert({key, metric}); } +std::string ExtractErrorCodeFromMessage(absl::string_view errorMessage) { + if (absl::StartsWith(errorMessage, kInferenceTensorInputNameError)) { + return std::string(kInferenceTensorInputNameError); + } + if (absl::StartsWith(errorMessage, kInferenceInputTensorConversionError)) { + return std::string(kInferenceInputTensorConversionError); + } + if (absl::StartsWith(errorMessage, kInferenceSignatureNotFoundError)) { + return std::string(kInferenceSignatureNotFoundError); + } + if (absl::StartsWith(errorMessage, kInferenceModelExecutionError)) { + return std::string(kInferenceModelExecutionError); + } + if (absl::StartsWith(errorMessage, kInferenceOutputTensorMismatchError)) { + return std::string(kInferenceOutputTensorMismatchError); + } + return std::string( + kInferenceUnknownError); // Return an Unknown error if no match is found +} + } // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/inference_sidecar/common/utils/inference_metric_util.h b/services/inference_sidecar/common/utils/inference_metric_util.h index 81c38fc6..9d7ed30d 100644 --- a/services/inference_sidecar/common/utils/inference_metric_util.h +++ b/services/inference_sidecar/common/utils/inference_metric_util.h @@ -20,6 +20,7 @@ #include #include +#include "absl/strings/string_view.h" #include "proto/inference_sidecar.pb.h" namespace privacy_sandbox::bidding_auction_servers::inference { @@ -28,6 +29,9 @@ namespace privacy_sandbox::bidding_auction_servers::inference { void AddMetric(PredictResponse& response, const std::string& key, int32_t value, std::optional partition = std::nullopt); +// Extracts the error code from the error message for Error Reporting. +std::string ExtractErrorCodeFromMessage(absl::string_view errorMessage); + } // namespace privacy_sandbox::bidding_auction_servers::inference #endif // SERVICES_INFERENCE_SIDECAR_COMMON_UTILS_INFERENCE_METRIC_UTIL_H_ diff --git a/services/inference_sidecar/modules/pytorch_v2_1_1/BUILD b/services/inference_sidecar/modules/pytorch_v2_1_1/BUILD index 4d0593b6..68ecb3f0 100644 --- a/services/inference_sidecar/modules/pytorch_v2_1_1/BUILD +++ b/services/inference_sidecar/modules/pytorch_v2_1_1/BUILD @@ -82,18 +82,17 @@ cc_library( cc_library( name = "pytorch", srcs = ["pytorch.cc"], + hdrs = ["pytorch.h"], deps = [ ":pytorch_parser", "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/random", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_absl//absl/synchronization", "@inference_common//model:model_store", "@inference_common//modules:module_interface", "@inference_common//proto:inference_sidecar_cc_proto", + "@inference_common//utils:inference_error_code", "@inference_common//utils:inference_metric_util", "@inference_common//utils:log", "@inference_common//utils:request_parser", @@ -129,6 +128,20 @@ genrule( cmd = "cp $< $@", ) +genrule( + name = "test_unfreezable_model_target", + srcs = ["@inference_common//testdata:models/pytorch_unfreezable_model.pt"], + outs = ["unfreezable_model"], + cmd = "cp $< $@", +) + +genrule( + name = "test_frozen_model_target", + srcs = ["@inference_common//testdata:models/pytorch_frozen_model.pt"], + outs = ["frozen_model"], + cmd = "cp $< $@", +) + genrule( name = "test_mixed_inputs_mixed_outputs_target", srcs = ["@inference_common//testdata:models/pytorch_mixed_inputs_mixed_outputs_model.pt"], @@ -143,9 +156,11 @@ cc_test( data = [ ":test_e2e_model1_target", ":test_e2e_model2_target", + ":test_frozen_model_target", ":test_mixed_inputs_mixed_outputs_target", ":test_simple_model_target", ":test_stateful_model_target", + ":test_unfreezable_model_target", ], deps = [ ":pytorch", @@ -160,6 +175,7 @@ cc_test( "@inference_common//utils:file_util", "@inference_common//utils:inference_metric_util", "@inference_common//utils:test_util", + "@pytorch_v2_1_1//:torch", ], ) diff --git a/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch.cc b/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch.cc index 8ff5b1c8..7367459c 100644 --- a/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch.cc +++ b/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch.cc @@ -14,30 +14,27 @@ * limitations under the License. */ -#include +#include "pytorch.h" + #include #include +#include #include #include -#include +#include #include #include -#include "absl/base/thread_annotations.h" -#include "absl/container/flat_hash_map.h" -#include "absl/random/random.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/numbers.h" #include "absl/strings/str_cat.h" -#include "absl/synchronization/mutex.h" -#include "absl/synchronization/notification.h" -#include "model/model_store.h" #include "modules/module_interface.h" #include "proto/inference_sidecar.pb.h" #include "src/util/status_macro/status_macros.h" #include "utils/error.h" +#include "utils/inference_error_code.h" #include "utils/inference_metric_util.h" #include "utils/log.h" #include "utils/request_parser.h" @@ -61,7 +58,8 @@ absl::StatusOr PredictInternal( ConvertFlatArrayToTensor(tensor); if (!torch_tensor.ok()) { return absl::InvalidArgumentError(absl::StrCat( - "Model ", model_key, " encounters tensor parsing error: ", + kInferenceInputTensorConversionError, ". Message: ", "Model ", + model_key, " encounters tensor parsing error: ", torch_tensor.status().message())); } inputs.push_back(*std::move(torch_tensor)); @@ -74,12 +72,12 @@ absl::StatusOr PredictInternal( return model->forward(inputs); } catch (const std::exception& e) { return absl::InternalError(absl::StrCat( - "Model ", model_key, + kInferenceModelExecutionError, ". Message: ", "Model ", model_key, " encounters an exception during evaluation: ", std::string(e.what()))); } catch (...) { - return absl::InternalError( - absl::StrCat("Model ", model_key, - " encounters an unknown exception during evaluation")); + return absl::InternalError(absl::StrCat( + kInferenceModelExecutionError, ". Message: ", "Model ", model_key, + " encounters an unknown exception during evaluation")); } } @@ -106,82 +104,71 @@ absl::Status InitRuntimeThreadConfig( return absl::OkStatus(); } -absl::StatusOr> +absl::StatusOr> PyTorchModelConstructor(const InferenceSidecarRuntimeConfig& config, const RegisterModelRequest& request) { + torch::jit::script::Module model; // Converts PyTorch exception to absl status. try { const std::string& model_payload = request.model_files().begin()->second; std::istringstream is(model_payload); - auto model = - std::make_unique(torch::jit::load(is)); + model = torch::jit::load(is); // Turn on eval model for layers that behave differently during train and // eval times, for example, dropout and batch norm layers. - model->eval(); - return model; + model.eval(); } catch (...) { return absl::InternalError("Error loading model"); } -} -class PyTorchModule final : public ModuleInterface { - public: - explicit PyTorchModule(const InferenceSidecarRuntimeConfig& config) - : runtime_config_(config), - store_(config, PyTorchModelConstructor), - background_thread_running_(true) { - absl::Status init_result = InitRuntimeThreadConfig(config); - CHECK(init_result.ok()) - << "Could not initialize runtime flags: " << init_result; - - // TODO(b/332328206): Move reset model logic into the ModelStore. - background_thread_ = std::thread([this]() { - while (background_thread_running_) { - // ResetModels() is continually called in the background thread. - ResetModels(); - // Waits for the completed inference execution. - // - // Moves on after 1 second and check if the background thread should - // terminate. The destructor terminates this thread by setting - // |background_thread_running_|. - inference_notification_.WaitForNotificationWithTimeout( - absl::Seconds(1)); - } - }); + std::shared_ptr frozen_model; + try { + frozen_model = + std::make_shared(torch::jit::freeze(model)); + } catch (...) { + return absl::InternalError("Error encountered during model freeze"); } - ~PyTorchModule() override { - background_thread_running_ = false; - background_thread_.join(); + if (frozen_model->attributes().size() > 0) { + return absl::FailedPreconditionError("Failed to inline model graph."); } - absl::StatusOr Predict( - const PredictRequest& request, - const RequestContext& request_context) override; - absl::StatusOr RegisterModel( - const RegisterModelRequest& request) override; - void ResetModels() override; - - private: - const InferenceSidecarRuntimeConfig runtime_config_; - - // Stores a set of models. It's thread safe. - ModelStore store_; + // Model warm-up. + absl::string_view model_key = request.model_spec().model_path(); + if (!request.warm_up_batch_request_json().empty()) { + absl::StatusOr> parsed_requests = + ParseJsonInferenceRequest(request.warm_up_batch_request_json()); + if (!parsed_requests.ok()) { + return absl::InvalidArgumentError(absl::StrCat( + "Encounters warm up batch inference request parsing error: ", + parsed_requests.status().message())); + } + // Process warm up for each inference request. + // TODO(b/362338463): Add optional execute mode choice. + for (const InferenceRequest& inference_request : (*parsed_requests)) { + if (inference_request.model_path != model_key) { + return absl::InvalidArgumentError( + "Warm up request using different model path."); + } + auto inference_response = + PredictInternal(frozen_model, inference_request); + if (!inference_response.ok()) { + return inference_response.status(); + } + } + } + return frozen_model; +} - // Counts the number of inferences per model. Used for model reset. - absl::flat_hash_map per_model_inference_count_ - ABSL_GUARDED_BY(per_model_inference_count_mu_); - absl::Mutex per_model_inference_count_mu_; - // Notification to trigger model reset. - absl::Notification inference_notification_; +} // namespace - // The background thread continuously running ResetModels(). - std::thread background_thread_; - // The background thread is shutdown if set to false. - std::atomic background_thread_running_; - // Exclusively used in a single backgrond thread. No need of a mutex. - absl::BitGen bitgen_; -}; +PyTorchModule::PyTorchModule(const InferenceSidecarRuntimeConfig& config) + : runtime_config_(config), + store_(std::make_unique>( + config, PyTorchModelConstructor)) { + absl::Status init_result = InitRuntimeThreadConfig(config); + CHECK(init_result.ok()) << "Could not initialize runtime flags: " + << init_result; +} absl::StatusOr PyTorchModule::Predict( const PredictRequest& request, const RequestContext& request_context) { @@ -191,6 +178,8 @@ absl::StatusOr PyTorchModule::Predict( absl::StatusOr> parsed_requests = ParseJsonInferenceRequest(request.input()); if (!parsed_requests.ok()) { + AddMetric(predict_response, "kInferenceErrorCountByErrorCode", 1, + std::string(kInferenceUnableToParseRequest)); INFERENCE_LOG(ERROR, request_context) << parsed_requests.status(); predict_response.set_output(CreateBatchErrorString( Error{.error_type = Error::INPUT_PARSING, @@ -207,13 +196,19 @@ absl::StatusOr PyTorchModule::Predict( for (size_t task_id = 0; task_id < parsed_requests->size(); ++task_id) { const InferenceRequest& inference_request = (*parsed_requests)[task_id]; const std::string& model_key = inference_request.model_path; + AddMetric(predict_response, "kInferenceRequestCountByModel", 1, model_key); INFERENCE_LOG(INFO, request_context) << "Received inference request to model: " << model_key; absl::StatusOr> model = - store_.GetModel(model_key, request.is_consented()); + store_->GetModel(model_key, request.is_consented()); if (!model.ok()) { + AddMetric(predict_response, "kInferenceRequestFailedCountByModel", 1, + model_key); + AddMetric(predict_response, "kInferenceErrorCountByErrorCode", 1, + std::string(kInferenceModelNotFoundError)); INFERENCE_LOG(ERROR, request_context) - << "Fails to get model : " << model_key << model.status(); + << "Fails to get model: " << model_key + << " Reason: " << model.status(); batch_outputs[task_id] = PerModelOutput{ .model_path = model_key, .error = Error{.error_type = Error::MODEL_NOT_FOUND, @@ -230,9 +225,14 @@ absl::StatusOr PyTorchModule::Predict( absl::StatusOr task_result = tasks[task_id].get(); const std::string& model_path = (*parsed_requests)[task_id].model_path; if (!task_result.ok()) { + AddMetric(predict_response, "kInferenceRequestFailedCountByModel", 1, + model_path); + AddMetric(predict_response, "kInferenceErrorCountByErrorCode", 1, + std::optional(ExtractErrorCodeFromMessage( + task_result.status().message()))); INFERENCE_LOG(ERROR, request_context) << "Inference fails for model: " << model_path - << task_result.status(); + << " Reason: " << task_result.status(); Error::ErrorType error_type = task_result.status().code() == absl::StatusCode::kInvalidArgument ? Error::INPUT_PARSING @@ -243,6 +243,11 @@ absl::StatusOr PyTorchModule::Predict( .error_type = error_type, .description = std::string(task_result.status().message())}}; } else { + int model_execution_time_ms = + (absl::Now() - start_inference_execution_time) / + absl::Milliseconds(1); + AddMetric(predict_response, "kInferenceRequestDurationByModel", + model_execution_time_ms, model_path); batch_outputs[task_id] = PerModelOutput{ .model_path = model_path, .inference_output = *task_result}; } @@ -252,6 +257,8 @@ absl::StatusOr PyTorchModule::Predict( absl::StatusOr output_json = ConvertBatchOutputsToJson(batch_outputs); if (!output_json.ok()) { + AddMetric(predict_response, "kInferenceErrorCountByErrorCode", 1, + std::string(kInferenceOutputParsingError)); INFERENCE_LOG(ERROR, request_context) << output_json.status(); predict_response.set_output(CreateBatchErrorString( Error{.error_type = Error::OUTPUT_PARSING, @@ -259,14 +266,9 @@ absl::StatusOr PyTorchModule::Predict( return predict_response; } - { - absl::MutexLock lock(&per_model_inference_count_mu_); - for (const InferenceRequest& inference_request : *parsed_requests) { - const std::string& model_key = inference_request.model_path; - ++per_model_inference_count_[model_key]; - } + for (const InferenceRequest& inference_request : *parsed_requests) { + store_->IncrementModelInferenceCount(inference_request.model_path); } - inference_notification_.Notify(); predict_response.set_output(*output_json); int inference_execution_time_ms = @@ -290,48 +292,14 @@ absl::StatusOr PyTorchModule::RegisterModel( request.model_files().size())); } - if (store_.GetModel(model_key).ok()) { + if (store_->GetModel(model_key).ok()) { return absl::AlreadyExistsError( absl::StrCat("Model ", model_key, " has already been registered")); } - PS_RETURN_IF_ERROR(store_.PutModel(model_key, request)); + PS_RETURN_IF_ERROR(store_->PutModel(model_key, request)); return RegisterModelResponse(); } -// TODO(b/346813356): Refactor duplicate logic into a shared library/class. -void PyTorchModule::ResetModels() { - const double reset_probability = runtime_config_.model_reset_probability(); - if (reset_probability == 0.0) { - // Model reset is disabled. - return; - } - - std::vector models = store_.ListModels(); - for (const auto& model_key : models) { - int count = 0; - { - absl::MutexLock lock(&per_model_inference_count_mu_); - count = per_model_inference_count_[model_key]; - // Sets the per-model counter to 0. - per_model_inference_count_[model_key] = 0; - } - if (count <= 0) continue; - double random = absl::Uniform(bitgen_, 0.0, 1.0); - // Boosts the chance of reset multiplied by the number of inferences as - // approximation. - if (reset_probability != 1.0 && random >= reset_probability * count) { - continue; - } - - // We should make sure the model reset is successfully done. - // Otherwise, we terminate the program to preserve user privacy. - CHECK(store_.ResetModel(model_key).ok()) - << "Failed to reset model: " << model_key; - } -} - -} // namespace - std::unique_ptr ModuleInterface::Create( const InferenceSidecarRuntimeConfig& config) { return std::make_unique(config); diff --git a/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch.h b/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch.h new file mode 100644 index 00000000..37e7d2f0 --- /dev/null +++ b/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch.h @@ -0,0 +1,56 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SERVICES_INFERENCE_SIDECAR_MODULES_PYTORCH_V2_1_1_PYTORCH_H_ +#define SERVICES_INFERENCE_SIDECAR_MODULES_PYTORCH_V2_1_1_PYTORCH_H_ + +#include +#include + +#include + +#include "absl/status/statusor.h" +#include "model/model_store.h" +#include "modules/module_interface.h" +#include "proto/inference_sidecar.pb.h" + +namespace privacy_sandbox::bidding_auction_servers::inference { + +class PyTorchModule final : public ModuleInterface { + public: + explicit PyTorchModule(const InferenceSidecarRuntimeConfig& config); + + absl::StatusOr Predict( + const PredictRequest& request, + const RequestContext& request_context) override; + absl::StatusOr RegisterModel( + const RegisterModelRequest& request) override; + + private: + const InferenceSidecarRuntimeConfig runtime_config_; + + // Stores a set of models. It's thread safe. + std::unique_ptr> store_; + friend class PyTorchModuleResetModelTest; + void SetModelStoreForTestOnly( + std::unique_ptr> store) { + store_ = std::move(store); + } +}; + +} // namespace privacy_sandbox::bidding_auction_servers::inference + +#endif // SERVICES_INFERENCE_SIDECAR_MODULES_PYTORCH_V2_1_1_PYTORCH_H_ diff --git a/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch_test.cc b/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch_test.cc index 4652ad21..d643134f 100644 --- a/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch_test.cc +++ b/services/inference_sidecar/modules/pytorch_v2_1_1/pytorch_test.cc @@ -12,11 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "pytorch.h" + #include +#include #include +#include +#include -#include +#include #include "absl/status/status.h" #include "absl/status/statusor.h" @@ -26,10 +31,12 @@ #include "absl/time/clock.h" #include "absl/time/time.h" #include "gtest/gtest.h" +#include "model/model_store.h" #include "modules/module_interface.h" #include "proto/inference_sidecar.pb.h" #include "utils/file_util.h" #include "utils/inference_metric_util.h" +#include "utils/log.h" #include "utils/test_util.h" namespace privacy_sandbox::bidding_auction_servers::inference { @@ -44,7 +51,9 @@ constexpr absl::string_view kTestModelVariedInputs1 = "e2e_model1"; constexpr absl::string_view kTestModelVariedInputs2 = "e2e_model2"; constexpr absl::string_view kTestModelMixedInputsMixedOutputs = "mixed_inputs_mixed_outputs_model"; -constexpr absl::string_view kStatefulModelDir = "stateful_model"; +constexpr absl::string_view kStatefulModel = "stateful_model"; +constexpr absl::string_view kUnfreezableModel = "unfreezable_model"; +constexpr absl::string_view kFrozenModel = "frozen_model"; constexpr int kNumThreads = 100; TEST(PyTorchModuleRuntimeConfigTest, @@ -110,6 +119,43 @@ TEST(PyTorchModuleRegisterModelTest, absl::StatusCode::kInvalidArgument); } +TEST(PyTorchModuleRegisterModelTest, + RegisterModelWithAttributesReturnsFailedPrecondition) { + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); + RegisterModelRequest register_request; + ASSERT_TRUE( + PopulateRegisterModelRequest(kStatefulModel, register_request).ok()); + EXPECT_EQ(torch_module->RegisterModel(register_request).status().code(), + absl::StatusCode::kFailedPrecondition); +} + +TEST(PyTorchModuleRegisterModelTest, + RegisterModelWithUnfreezableModelReturnsInternalError) { + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); + RegisterModelRequest register_request; + // Model that returns itself is not freezable. + ASSERT_TRUE( + PopulateRegisterModelRequest(kUnfreezableModel, register_request).ok()); + EXPECT_EQ(torch_module->RegisterModel(register_request).status().code(), + absl::StatusCode::kInternal); +} + +TEST(PyTorchModuleRegisterModelTest, + RegisterModelWithFrozenModelReturnsOkStatus) { + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); + RegisterModelRequest register_request; + // Model that is already frozen can be loaded. + ASSERT_TRUE( + PopulateRegisterModelRequest(kFrozenModel, register_request).ok()); + EXPECT_TRUE(torch_module->RegisterModel(register_request).ok()); +} + TEST(PyTorchModuleRegisterModelTest, RegisterModelOk) { InferenceSidecarRuntimeConfig config; std::unique_ptr torch_module = @@ -232,7 +278,7 @@ TEST(PyTorchModulePredictTest, PredictSimpleSuccess) { EXPECT_TRUE(result.ok()); EXPECT_EQ(result->output(), kSimpleRequestResponse); ASSERT_FALSE(result->metrics().empty()); - EXPECT_EQ(result->metrics().size(), 4); + EXPECT_EQ(result->metrics().size(), 6); } TEST(PyTorchModulePredictTest, PredictSimpleSuccess_ValidateMetrics) { @@ -252,10 +298,10 @@ TEST(PyTorchModulePredictTest, PredictSimpleSuccess_ValidateMetrics) { EXPECT_TRUE(result.ok()); EXPECT_EQ(result->output(), kSimpleRequestResponse); ASSERT_FALSE(result->metrics().empty()); - EXPECT_EQ(result->metrics().size(), 4); + EXPECT_EQ(result->metrics().size(), 6); CheckMetric(result->metrics(), "kInferenceRequestCount", 1); CheckMetric(result->metrics(), "kInferenceRequestSize", 204); - CheckMetric(result->metrics(), "kInferenceResponseSize", 215); + CheckMetric(result->metrics(), "kInferenceResponseSize", 318); auto it = result->metrics().find("kInferenceRequestDuration"); ASSERT_NE(it, result->metrics().end()) << "kInferenceRequestDuration metric is missing."; @@ -279,38 +325,7 @@ TEST(PyTorchModulePredictTest, PredictConsentedRequestSuccess) { EXPECT_TRUE(result.ok()); EXPECT_EQ(result->output(), kSimpleRequestResponse); ASSERT_FALSE(result->metrics().empty()); - EXPECT_EQ(result->metrics().size(), 4); -} - -TEST(PyTorchModuleResetModelTest, NoModel) { - InferenceSidecarRuntimeConfig config; - config.set_model_reset_probability(1.0); - std::unique_ptr torch_module = - ModuleInterface::Create(config); - torch_module->ResetModels(); -} - -TEST(PyTorchModuleResetModelTest, ResetModelOk) { - InferenceSidecarRuntimeConfig config; - config.set_model_reset_probability(1.0); - std::unique_ptr torch_module = - ModuleInterface::Create(config); - RegisterModelRequest register_request; - ASSERT_TRUE( - PopulateRegisterModelRequest(kSimpleModel, register_request).ok()); - ASSERT_TRUE(torch_module->RegisterModel(register_request).ok()); - - PredictRequest predict_request; - predict_request.set_input(kSimpleRequest); - - const absl::StatusOr result = - torch_module->Predict(predict_request); - EXPECT_TRUE(result.ok()); - EXPECT_EQ(result->output(), kSimpleRequestResponse); - ASSERT_FALSE(result->metrics().empty()); - EXPECT_EQ(result->metrics().size(), 4); - - torch_module->ResetModels(); + EXPECT_EQ(result->metrics().size(), 6); } constexpr char kNotRegisteredModelRequest[] = R"json({ @@ -400,7 +415,7 @@ TEST(PyTorchModulePredictTest, PredictSimpleSuccessShape1x2) { "\"tensor_shape\":[1,2],\"data_type\":\"DOUBLE\",\"tensor_" "content\":[3.14,2.718]}]}]}"); ASSERT_FALSE(result->metrics().empty()); - EXPECT_EQ(result->metrics().size(), 4); + EXPECT_EQ(result->metrics().size(), 6); } constexpr char kSimpleRequestBatchSize2[] = R"json({ @@ -438,7 +453,7 @@ TEST(PyTorchModulePredictTest, PredictSimpleBatchSize2Success) { "\"tensor_shape\":[2,1],\"data_type\":\"DOUBLE\",\"tensor_" "content\":[3.14,2.718]}]}]}"); ASSERT_FALSE(result->metrics().empty()); - EXPECT_EQ(result->metrics().size(), 4); + EXPECT_EQ(result->metrics().size(), 6); } constexpr char kSameModelSameBatchSizeMultipleRequests[] = R"json({ @@ -491,7 +506,7 @@ TEST(PyTorchModulePredictTest, "\"model_path\":\"simple_model\",\"tensors\":[{\"tensor_shape\":[1]," "\"data_type\":\"DOUBLE\",\"tensor_content\":[2.718]}]}]}"); ASSERT_FALSE(result->metrics().empty()); - EXPECT_EQ(result->metrics().size(), 4); + EXPECT_EQ(result->metrics().size(), 6); } constexpr char kSameModelVariedBatchSizesMultipleRequests[] = R"json({ @@ -544,7 +559,7 @@ TEST(PyTorchModulePredictTest, "{\"model_path\":\"simple_model\",\"tensors\":[{\"tensor_shape\":[3,1]," "\"data_type\":\"DOUBLE\",\"tensor_content\":[2.718,1.0,1.0]}]}]}"); ASSERT_FALSE(result->metrics().empty()); - EXPECT_EQ(result->metrics().size(), 4); + EXPECT_EQ(result->metrics().size(), 6); } constexpr char kRequestsWithMultipleInvalidInputs[] = R"json({ @@ -726,7 +741,7 @@ TEST(PyTorchModulePredictTest, PredictVariedInputsBatchSize1Success) { "\"tensor_shape\":[1,1],\"data_type\":\"FLOAT\",\"tensor_content\":" "[0.4846605658531189]}]}]}"); ASSERT_FALSE(result->metrics().empty()); - EXPECT_EQ(result->metrics().size(), 4); + EXPECT_EQ(result->metrics().size(), 6); } constexpr char kVariedInputsRequestBatchSize2[] = R"json({ @@ -822,7 +837,173 @@ TEST(PyTorchModulePredictTest, PredictVariedInputsBatchSize2Success) { "\"tensor_shape\":[2,1],\"data_type\":\"FLOAT\",\"tensor_content\":" "[0.5412338972091675,0.4757022559642792]}]}]}"); ASSERT_FALSE(result->metrics().empty()); - EXPECT_EQ(result->metrics().size(), 4); + EXPECT_EQ(result->metrics().size(), 6); +} + +TEST(PyTorchModulePredictTest, RegisterModelWithWarmupDataSuccess) { + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); + RegisterModelRequest register_request; + ASSERT_TRUE( + PopulateRegisterModelRequest(kTestModelVariedInputs1, register_request) + .ok()); + register_request.set_warm_up_batch_request_json( + kVariedInputsRequestBatchSize2); + ASSERT_TRUE(torch_module->RegisterModel(register_request).ok()); +} + +constexpr char kVariedInputsRequestBatchSize2WithWrongPath[] = R"json({ + "request" : [{ + "model_path" : "e2e_model1_non_exist_path", + "tensors" : [ + { + "tensor_name": "serving_default_int_input1:0", + "data_type": "INT64", + "tensor_shape": [ + 2 + ], + "tensor_content": ["8", "6"] + }, + { + "tensor_name": "serving_default_int_input2:0", + "data_type": "INT64", + "tensor_shape": [ + 2 + ], + "tensor_content": ["18", "11"] + }, + { + "tensor_name": "serving_default_int_input3:0", + "data_type": "INT64", + "tensor_shape": [ + 2 + ], + "tensor_content": ["9", "14"] + }, + { + "tensor_name": "serving_default_int_input4:0", + "data_type": "INT64", + "tensor_shape": [ + 2 + ], + "tensor_content": ["2", "1"] + }, + { + "tensor_name": "serving_default_int_input5:0", + "data_type": "INT64", + "tensor_shape": [ + 2 + ], + "tensor_content": ["10", "4"] + }, + { + "tensor_name": "serving_default_double1:0", + "data_type": "FLOAT", + "tensor_shape": [ + 2, 10 + ], + "tensor_content": [ + "0.7862", "0.6386", "0.9695", "0.0469", "0.5807", "0.8201", "0.8321", + "0.1021", "0.6779", "0.2152", "0.4805", "0.3957", "0.0825", "0.0230", + "0.1711", "0.7269", "0.7287", "0.0651", "0.3122", "0.5082" + ] + } + ] +}] + })json"; + +constexpr char kVariedInputsRequestBatchSize2WithWrongTensor[] = R"json({ + "request" : [{ + "model_path" : "e2e_model1", + "tensors" : [ + { + "tensor_name": "serving_default_int_input1:0", + "data_type": "INT64", + "tensor_shape": [ + 2 + ], + "tensor_content": ["8", "6"] + }, + { + "tensor_name": "serving_default_int_input2:0", + "data_type": "INT64", + "tensor_shape": [ + 2 + ], + "tensor_content": ["18", "11"] + }, + { + "tensor_name": "serving_default_int_input3:0", + "data_type": "INT64", + "tensor_shape": [ + 2 + ], + "tensor_content": ["9", "14"] + }, + { + "tensor_name": "serving_default_int_input4:0", + "data_type": "INT64", + "tensor_shape": [ + 2 + ], + "tensor_content": ["2", "1"] + }, + { + "tensor_name": "serving_default_int_input5:0", + "data_type": "INT64", + "tensor_shape": [ + 2 + ], + "tensor_content": ["10", "4"] + }, + { + "tensor_name": "serving_default_double1:0", + "data_type": "FLOAT", + "tensor_shape": [ + 2, 10 + ], + "tensor_content": [ + "0.7862", "0.6386", "0.9695", "0.0469", "0.5807", "0.8201", "0.8321", + "0.1021", "0.6779", "0.2152", "0.4805", "0.3957", "0.0825", "0.0230", + "0.1711", "0.7269", "0.7287", "0.0651", "0.3122", "0.5082" + ] + }, + { + "tensor_name": "serving_default_double2:0", + "data_type": "FLOAT", + "tensor_shape": [ + 2, 10 + ], + "tensor_content": [ + "0.0677", "0.7817", "0.9529", "0.5884", "0.1285", "0.6166", "0.6815", + "0.8959", "0.2340", "0.0520", "0.7197", "0.5311", "0.3371", "0.2905", + "0.2422", "0.9047", "0.4137", "0.8606", "0.9463", "0.5633" + ] + } + ] +}] + })json"; + +TEST(PyTorchModulePredictTest, RegisterModelWithInavlidWarmupDataFail) { + InferenceSidecarRuntimeConfig config; + std::unique_ptr torch_module = + ModuleInterface::Create(config); + RegisterModelRequest register_request_wrong_path; + ASSERT_TRUE(PopulateRegisterModelRequest(kTestModelVariedInputs1, + register_request_wrong_path) + .ok()); + register_request_wrong_path.set_warm_up_batch_request_json( + kVariedInputsRequestBatchSize2WithWrongPath); + ASSERT_FALSE(torch_module->RegisterModel(register_request_wrong_path).ok()); + + RegisterModelRequest register_request_wrong_tensor; + ASSERT_TRUE(PopulateRegisterModelRequest(kTestModelVariedInputs1, + register_request_wrong_tensor) + .ok()); + register_request_wrong_tensor.set_warm_up_batch_request_json( + kVariedInputsRequestBatchSize2WithWrongPath); + ASSERT_FALSE(torch_module->RegisterModel(register_request_wrong_tensor).ok()); } constexpr char kMixedInputsBatchSize1[] = R"json({ @@ -872,7 +1053,7 @@ TEST(PyTorchModulePredictTest, PredictMixedInputsMixedOutputsSuccess) { "\"data_" "type\":\"INT64\",\"tensor_content\":[0]}]}]}"); ASSERT_FALSE(result->metrics().empty()); - EXPECT_EQ(result->metrics().size(), 4); + EXPECT_EQ(result->metrics().size(), 6); } constexpr char kVariedInputsMultipleModelsRequest[] = R"json({ @@ -1046,7 +1227,7 @@ TEST(PyTorchModulePredictTest, "\"tensors\":[{\"tensor_shape\":[2,1],\"data_type\":\"FLOAT\",\"tensor_" "content\":[0.3857767879962921,0.458008348941803]}]}]}"); ASSERT_FALSE(result->metrics().empty()); - EXPECT_EQ(result->metrics().size(), 4); + EXPECT_EQ(result->metrics().size(), 6); } constexpr char kStatefulModelRequest[] = R"json({ @@ -1064,65 +1245,6 @@ constexpr char kStatefulModelRequest[] = R"json({ }] })json"; -TEST(PyTorchModuleResetModelTest, NoResetWithStatefulModel) { - const int kIterations = 100; - InferenceSidecarRuntimeConfig config; - config.set_model_reset_probability(0.0); - std::unique_ptr torch_module = - ModuleInterface::Create(config); - RegisterModelRequest register_request; - ASSERT_TRUE( - PopulateRegisterModelRequest(kStatefulModelDir, register_request).ok()); - ASSERT_TRUE(torch_module->RegisterModel(register_request).ok()); - - for (int count = 1; count < kIterations; count++) { - PredictRequest predict_request; - predict_request.set_input(kStatefulModelRequest); - absl::StatusOr predict_status = torch_module->Predict(predict_request); - ASSERT_TRUE(predict_status.ok()); - PredictResponse response = predict_status.value(); - ASSERT_FALSE(response.output().empty()); - - EXPECT_TRUE(absl::StrContains( - response.output(), - absl::StrCat( - "{\"tensor_shape\":[],\"data_type\":\"INT32\",\"tensor_content\":[", - count, "]}"))) - << response.output(); - ASSERT_FALSE(response.metrics().empty()); - EXPECT_EQ(response.metrics().size(), 4); - } -} - -TEST(PyTorchModuleResetModelTest, ResetSuccessWithStatefulModel) { - const int kIterations = 10; - InferenceSidecarRuntimeConfig config; - config.set_model_reset_probability(1.0); - std::unique_ptr torch_module = - ModuleInterface::Create(config); - RegisterModelRequest register_request; - ASSERT_TRUE( - PopulateRegisterModelRequest(kStatefulModelDir, register_request).ok()); - ASSERT_TRUE(torch_module->RegisterModel(register_request).ok()); - - for (int i = 0; i < kIterations; i++) { - PredictRequest predict_request; - predict_request.set_input(kStatefulModelRequest); - absl::StatusOr predict_status = torch_module->Predict(predict_request); - ASSERT_TRUE(predict_status.ok()); - PredictResponse response = predict_status.value(); - ASSERT_FALSE(response.output().empty()); - - EXPECT_TRUE(absl::StrContains( - response.output(), - "{\"tensor_shape\":[],\"data_type\":\"INT32\",\"tensor_content\":[1]}")) - << response.output(); - ASSERT_FALSE(response.metrics().empty()); - EXPECT_EQ(response.metrics().size(), 4); - absl::SleepFor(absl::Seconds(1)); - } -} - TEST(PyTorchModuleConcurrencyTest, RegisterSameModelWithMultipleThreads) { InferenceSidecarRuntimeConfig config; std::unique_ptr torch_module = @@ -1186,7 +1308,7 @@ TEST(PyTorchModuleConcurrencyTest, "[{\"tensor_shape\":[1],\"data_type\":\"DOUBLE\",\"tensor_" "content\":[3.14]}]}]}"); ASSERT_FALSE(result->metrics().empty()); - EXPECT_EQ(result->metrics().size(), 4); + EXPECT_EQ(result->metrics().size(), 6); } })); } @@ -1238,7 +1360,7 @@ TEST(PyTorchModuleConcurrencyTest, "shape\":[2,1],\"data_type\":\"FLOAT\",\"tensor_content\":[0." "3857767879962921,0.458008348941803]}]}]}"); ASSERT_FALSE(result->metrics().empty()); - EXPECT_EQ(result->metrics().size(), 4); + EXPECT_EQ(result->metrics().size(), 6); } })); } @@ -1249,4 +1371,98 @@ TEST(PyTorchModuleConcurrencyTest, } } // namespace + +class PyTorchModuleResetModelTest : public ::testing::Test { + protected: + void SetUp() override { + torch_module_ = + std::make_unique(InferenceSidecarRuntimeConfig()); + } + + void SetResetProbability(float prob) { + InferenceSidecarRuntimeConfig config; + config.set_model_reset_probability(prob); + torch_module_->SetModelStoreForTestOnly( + std::make_unique>( + config, MockModelConstructor)); + } + + // No model freezing to allow stateful models to be loaded. + static absl::StatusOr> + MockModelConstructor(const InferenceSidecarRuntimeConfig& config, + const RegisterModelRequest& request) { + try { + const std::string& model_payload = request.model_files().begin()->second; + std::istringstream is(model_payload); + auto model = + std::make_shared(torch::jit::load(is)); + model->eval(); + return model; + } catch (...) { + return absl::InternalError("Error loading model"); + } + } + + std::unique_ptr torch_module_; +}; + +TEST_F(PyTorchModuleResetModelTest, NoResetWithStatefulModel) { + const int kIterations = 100; + + SetResetProbability(0.0); + + RegisterModelRequest register_request; + ASSERT_TRUE( + PopulateRegisterModelRequest(kStatefulModel, register_request).ok()); + ASSERT_TRUE(torch_module_->RegisterModel(register_request).ok()); + + for (int count = 1; count < kIterations; count++) { + PredictRequest predict_request; + predict_request.set_input(kStatefulModelRequest); + absl::StatusOr predict_status = + torch_module_->Predict(predict_request, RequestContext()); + ASSERT_TRUE(predict_status.ok()); + PredictResponse response = predict_status.value(); + ASSERT_FALSE(response.output().empty()); + + EXPECT_TRUE(absl::StrContains( + response.output(), + absl::StrCat( + "{\"tensor_shape\":[],\"data_type\":\"INT32\",\"tensor_content\":[", + count, "]}"))) + << response.output(); + ASSERT_FALSE(response.metrics().empty()); + EXPECT_EQ(response.metrics().size(), 6); + } +} + +TEST_F(PyTorchModuleResetModelTest, ResetSuccessWithStatefulModel) { + const int kIterations = 10; + + SetResetProbability(1.0); + + RegisterModelRequest register_request; + ASSERT_TRUE( + PopulateRegisterModelRequest(kStatefulModel, register_request).ok()); + ASSERT_TRUE(torch_module_->RegisterModel(register_request).ok()); + + for (int i = 0; i < kIterations; i++) { + PredictRequest predict_request; + predict_request.set_input(kStatefulModelRequest); + absl::StatusOr predict_status = + torch_module_->Predict(predict_request, RequestContext()); + ASSERT_TRUE(predict_status.ok()); + PredictResponse response = predict_status.value(); + ASSERT_FALSE(response.output().empty()); + + EXPECT_TRUE(absl::StrContains( + response.output(), + "{\"tensor_shape\":[],\"data_type\":\"INT32\",\"tensor_content\":[1]}")) + << response.output(); + ASSERT_FALSE(response.metrics().empty()); + EXPECT_EQ(response.metrics().size(), 6); + absl::SleepFor(absl::Seconds(1)); + } +} + } // namespace privacy_sandbox::bidding_auction_servers::inference diff --git a/services/inference_sidecar/modules/pytorch_v2_1_1/tools/BUILD b/services/inference_sidecar/modules/pytorch_v2_1_1/tools/BUILD new file mode 100644 index 00000000..1a42a4df --- /dev/null +++ b/services/inference_sidecar/modules/pytorch_v2_1_1/tools/BUILD @@ -0,0 +1,25 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_cc//cc:defs.bzl", "cc_binary") + +package(default_visibility = ["//visibility:public"]) + +cc_binary( + name = "list_ops", + srcs = ["list_ops.cc"], + deps = [ + "@pytorch_v2_1_1//:torch", + ], +) diff --git a/services/inference_sidecar/modules/pytorch_v2_1_1/tools/list_ops.cc b/services/inference_sidecar/modules/pytorch_v2_1_1/tools/list_ops.cc new file mode 100644 index 00000000..e0a8c0e1 --- /dev/null +++ b/services/inference_sidecar/modules/pytorch_v2_1_1/tools/list_ops.cc @@ -0,0 +1,42 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Lists all registered TensorFlow operations. +// Usage: ./builders/tools/bazel-debian run //tools:list_ops + +#include +#include + +#include + +int main() { + std::cout << "PyTorch Version: " << TORCH_VERSION_MAJOR << "." + << TORCH_VERSION_MINOR << std::endl; + + const std::vector> op_list = + torch::jit::getAllOperators(); + + // An op with the same name can be overloaded with multiple signatures. + std::set op_names; + std::cout << "Op Name" << std::endl; + for (const auto& op : op_list) { + op_names.insert(op->schema().name()); + } + + for (const auto& op_name : op_names) { + std::cout << op_name << std::endl; + } + + return 0; +} diff --git a/services/inference_sidecar/modules/tensorflow_v2_14_0/BUILD b/services/inference_sidecar/modules/tensorflow_v2_14_0/BUILD index 849e8add..83285e8f 100644 --- a/services/inference_sidecar/modules/tensorflow_v2_14_0/BUILD +++ b/services/inference_sidecar/modules/tensorflow_v2_14_0/BUILD @@ -136,18 +136,16 @@ cc_library( deps = [ ":tensorflow_parser", "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/log:absl_log", - "@com_google_absl//absl/random", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_absl//absl/synchronization", "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", "@inference_common//model:model_store", "@inference_common//modules:module_interface", "@inference_common//proto:inference_sidecar_cc_proto", "@inference_common//utils:error", + "@inference_common//utils:inference_error_code", "@inference_common//utils:inference_metric_util", "@inference_common//utils:request_parser", "@org_tensorflow//tensorflow/cc:cc_ops", diff --git a/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow.cc b/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow.cc index c577fa80..7f0561de 100644 --- a/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow.cc +++ b/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow.cc @@ -14,22 +14,17 @@ * limitations under the License. */ -#include #include #include #include #include -#include "absl/base/thread_annotations.h" -#include "absl/container/flat_hash_map.h" #include "absl/log/absl_log.h" -#include "absl/random/random.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/numbers.h" #include "absl/strings/str_cat.h" -#include "absl/synchronization/mutex.h" -#include "absl/synchronization/notification.h" +#include "absl/strings/string_view.h" #include "model/model_store.h" #include "modules/module_interface.h" #include "proto/inference_sidecar.pb.h" @@ -43,6 +38,7 @@ #include "tensorflow/tsl/platform/env.h" #include "tensorflow/tsl/platform/file_system.h" #include "utils/error.h" +#include "utils/inference_error_code.h" #include "utils/inference_metric_util.h" #include "utils/log.h" #include "utils/request_parser.h" @@ -72,12 +68,15 @@ absl::StatusOr> PredictPerModel( std::vector> inputs; for (const auto& tensor : inference_request.inputs) { if (tensor.tensor_name.empty()) { - return absl::InvalidArgumentError( - "Name is required for each TensorFlow tensor input"); + return absl::InvalidArgumentError(absl::StrCat( + kInferenceTensorInputNameError, + ". Message: ", "Name is required for each TensorFlow tensor input.")); } auto tf_tensor = ConvertFlatArrayToTensor(tensor); if (!tf_tensor.ok()) { - return absl::InvalidArgumentError(tf_tensor.status().message()); + return absl::InvalidArgumentError( + absl::StrCat(kInferenceInputTensorConversionError, + ". Message: ", tf_tensor.status().message())); } inputs.emplace_back(tensor.tensor_name, *tf_tensor); } @@ -86,6 +85,7 @@ absl::StatusOr> PredictPerModel( const auto& signature_map = model->meta_graph_def.signature_def(); if (signature_map.find("serving_default") == signature_map.end()) { return absl::InternalError(absl::StrCat( + kInferenceSignatureNotFoundError, ". Message: ", "The 'serving_default' signature was not found for model '", model_key)); } @@ -101,13 +101,15 @@ absl::StatusOr> PredictPerModel( model->session->Run(inputs, output_names, {}, &outputs); if (!status.ok()) { return absl::InternalError(absl::StrCat( + kInferenceModelExecutionError, ". Message: ", "Inference failed for model '", model_key, "': ", status.ToString())); } std::vector zipped_vector; if (output_names.size() != outputs.size()) { return absl::InternalError( - "The number of output tensors doesn't match the number of output " - "tensor names"); + absl::StrCat(kInferenceOutputTensorMismatchError, ". Message: ", + "The number of output tensors doesn't match the number of " + "output tensor names. ")); } for (size_t i = 0; i < output_names.size(); ++i) { zipped_vector.push_back(TensorWithName(output_names[i], outputs[i])); @@ -116,7 +118,7 @@ absl::StatusOr> PredictPerModel( return zipped_vector; } -absl::StatusOr> +absl::StatusOr> TensorFlowModelConstructor(const InferenceSidecarRuntimeConfig& config, const RegisterModelRequest& request) { tensorflow::SessionOptions session_options; @@ -131,7 +133,7 @@ TensorFlowModelConstructor(const InferenceSidecarRuntimeConfig& config, } const std::unordered_set tags = {"serve"}; const auto& model_path = request.model_spec().model_path(); - auto model_bundle = std::make_unique(); + auto model_bundle = std::make_shared(); if (auto status = tensorflow::LoadSavedModel( session_options, {}, absl::StrCat(kRamFileSystemScheme, model_path), tags, model_bundle.get()); @@ -139,6 +141,29 @@ TensorFlowModelConstructor(const InferenceSidecarRuntimeConfig& config, return absl::InternalError( absl::StrCat("Error loading model: ", model_path)); } + // perform warm up if metadata been provided. + // TODO(b/362338463): Add optional execute mode choice. + if (!request.warm_up_batch_request_json().empty()) { + absl::StatusOr> parsed_requests = + ParseJsonInferenceRequest(request.warm_up_batch_request_json()); + if (!parsed_requests.ok()) { + return absl::InvalidArgumentError(absl::StrCat( + "Encounters warm up batch inference request parsing error: ", + parsed_requests.status().message())); + } + // Process warm up for each inference request. + for (const InferenceRequest& inference_request : (*parsed_requests)) { + if (inference_request.model_path != model_path) { + return absl::InvalidArgumentError( + "Warm up request using different model path."); + } + auto inference_response = + PredictPerModel(model_bundle, inference_request); + if (!inference_response.ok()) { + return inference_response.status(); + } + } + } return model_bundle; } @@ -146,57 +171,21 @@ class TensorflowModule final : public ModuleInterface { public: explicit TensorflowModule(const InferenceSidecarRuntimeConfig& config) : runtime_config_(config), - store_(config, TensorFlowModelConstructor), - background_thread_running_(true) { - // TODO(b/332328206): Move reset model logic into the ModelStore. - background_thread_ = std::thread([this]() { - while (background_thread_running_) { - // ResetModels() is continually called in the background thread. - ResetModels(); - // Waits for the completed inference execution. - // - // Moves on after 1 second and check if the background thread should - // terminate. The destructor terminates this thread by setting - // |background_thread_running_|. - inference_notification_.WaitForNotificationWithTimeout( - absl::Seconds(1)); - } - }); - } - - ~TensorflowModule() override { - background_thread_running_ = false; - background_thread_.join(); - } + store_(std::make_unique>( + config, TensorFlowModelConstructor)) {} absl::StatusOr Predict( const PredictRequest& request, const RequestContext& request_context) override; absl::StatusOr RegisterModel( const RegisterModelRequest& request) override; - void ResetModels() override; private: const InferenceSidecarRuntimeConfig runtime_config_; // Stores a set of models. It's thread safe. // TODO(b/327907675) : Add a test for concurrency - ModelStore store_; - - // Counts the number of inferences per model. Used for model reset. - absl::flat_hash_map per_model_inference_count_ - ABSL_GUARDED_BY(per_model_inference_count_mu_); - absl::Mutex per_model_inference_count_mu_; - - // Notification to trigger model reset. - absl::Notification inference_notification_; - - // The background thread continuously running ResetModels(). - std::thread background_thread_; - // The background thread is shutdown if set to false. - std::atomic background_thread_running_; - // Exclusively used in a single backgrond thread. No need of a mutex. - absl::BitGen bitgen_; + std::unique_ptr> store_; }; absl::StatusOr TensorflowModule::Predict( @@ -207,6 +196,8 @@ absl::StatusOr TensorflowModule::Predict( absl::StatusOr> parsed_requests = ParseJsonInferenceRequest(request.input()); if (!parsed_requests.ok()) { + AddMetric(predict_response, "kInferenceErrorCountByErrorCode", 1, + std::string(kInferenceUnableToParseRequest)); INFERENCE_LOG(ERROR, request_context) << parsed_requests.status(); predict_response.set_output(CreateBatchErrorString( Error{.error_type = Error::INPUT_PARSING, @@ -223,13 +214,19 @@ absl::StatusOr TensorflowModule::Predict( for (size_t task_id = 0; task_id < parsed_requests->size(); ++task_id) { const InferenceRequest& inference_request = (*parsed_requests)[task_id]; const std::string& model_path = inference_request.model_path; + AddMetric(predict_response, "kInferenceRequestCountByModel", 1, model_path); INFERENCE_LOG(INFO, request_context) << "Received inference request to model: " << model_path; absl::StatusOr> model = - store_.GetModel(model_path, request.is_consented()); + store_->GetModel(model_path, request.is_consented()); if (!model.ok()) { + AddMetric(predict_response, "kInferenceErrorCountByErrorCode", 1, + std::string(kInferenceModelNotFoundError)); + AddMetric(predict_response, "kInferenceRequestFailedCountByModel", 1, + model_path); INFERENCE_LOG(ERROR, request_context) - << "Fails to get model : " << model_path << model.status(); + << "Fails to get model: " << model_path + << " Reason: " << model.status(); batch_outputs[task_id] = TensorsOrError{ .model_path = model_path, .error = Error{.error_type = Error::MODEL_NOT_FOUND, @@ -245,19 +242,32 @@ absl::StatusOr TensorflowModule::Predict( absl::StatusOr> tensors = tasks[task_id].get(); const std::string& model_path = (*parsed_requests)[task_id].model_path; + if (!tensors.ok()) { + AddMetric(predict_response, "kInferenceErrorCountByErrorCode", 1, + std::optional( + ExtractErrorCodeFromMessage(tensors.status().message()))); + AddMetric(predict_response, "kInferenceRequestFailedCountByModel", 1, + model_path); INFERENCE_LOG(ERROR, request_context) - << "Inference fails for model: " << model_path << tensors.status(); + << "Inference fails for model: " << model_path + << " Reason: " << tensors.status(); Error::ErrorType error_type = tensors.status().code() == absl::StatusCode::kInvalidArgument ? Error::INPUT_PARSING : Error::MODEL_EXECUTION; + batch_outputs[task_id] = TensorsOrError{ .model_path = model_path, .error = Error{.error_type = error_type, .description = std::string(tensors.status().message())}}; } else { + int model_execution_time_ms = + (absl::Now() - start_inference_execution_time) / + absl::Milliseconds(1); + AddMetric(predict_response, "kInferenceRequestDurationByModel", + model_execution_time_ms, model_path); batch_outputs[task_id] = TensorsOrError{.model_path = model_path, .tensors = *tensors}; } @@ -266,21 +276,18 @@ absl::StatusOr TensorflowModule::Predict( auto output_json = ConvertTensorsOrErrorToJson(batch_outputs); if (!output_json.ok()) { + AddMetric(predict_response, "kInferenceErrorCountByErrorCode", 1, + std::string(kInferenceOutputParsingError)); + INFERENCE_LOG(ERROR, request_context) << output_json.status(); predict_response.set_output(CreateBatchErrorString( Error{.error_type = Error::OUTPUT_PARSING, .description = "Error during output parsing to json."})); return predict_response; } - - { - absl::MutexLock lock(&per_model_inference_count_mu_); - for (const InferenceRequest& inference_request : *parsed_requests) { - const std::string& model_key = inference_request.model_path; - ++per_model_inference_count_[model_key]; - } + for (const InferenceRequest& inference_request : *parsed_requests) { + store_->IncrementModelInferenceCount(inference_request.model_path); } - inference_notification_.Notify(); predict_response.set_output(output_json.value()); int inference_execution_time_ms = @@ -299,7 +306,7 @@ absl::StatusOr TensorflowModule::RegisterModel( return absl::InvalidArgumentError("Model path is empty"); } - if (store_.GetModel(model_path).ok()) { + if (store_->GetModel(model_path).ok()) { return absl::AlreadyExistsError( absl::StrCat("Model ", model_path, " has already been registered")); } @@ -307,40 +314,12 @@ absl::StatusOr TensorflowModule::RegisterModel( RegisterModelRequest model_request; *model_request.mutable_model_spec() = request.model_spec(); - PS_RETURN_IF_ERROR(store_.PutModel(model_path, model_request)); - return RegisterModelResponse(); -} - -// TODO(b/346813356): Refactor duplicate logic into a shared library/class. -void TensorflowModule::ResetModels() { - const double reset_probability = runtime_config_.model_reset_probability(); - if (reset_probability == 0.0) { - // Model reset is disabled. - return; - } - - std::vector models = store_.ListModels(); - for (const auto& model_key : models) { - int count = 0; - { - absl::MutexLock lock(&per_model_inference_count_mu_); - count = per_model_inference_count_[model_key]; - // Sets the per-model counter to 0. - per_model_inference_count_[model_key] = 0; - } - if (count <= 0) continue; - double random = absl::Uniform(bitgen_, 0.0, 1.0); - // Boosts the chance of reset multiplied by the number of inferences as - // approximation. - if (reset_probability != 1.0 && random >= reset_probability * count) { - continue; - } - - // We should make sure the model reset is successfully done. - // Otherwise, we terminate the program to preserve user privacy. - CHECK(store_.ResetModel(model_key).ok()) - << "Failed to reset model: " << model_key; + if (!request.warm_up_batch_request_json().empty()) { + model_request.set_warm_up_batch_request_json( + request.warm_up_batch_request_json()); } + PS_RETURN_IF_ERROR(store_->PutModel(model_path, model_request)); + return RegisterModelResponse(); } } // namespace diff --git a/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow_test.cc b/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow_test.cc index d2792efb..3d88cd2a 100644 --- a/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow_test.cc +++ b/services/inference_sidecar/modules/tensorflow_v2_14_0/tensorflow_test.cc @@ -230,7 +230,7 @@ TEST(TensorflowModuleTest, Success_Predict) { ASSERT_FALSE(response.output().empty()); ASSERT_EQ(response.output(), kPcvrResponse); ASSERT_FALSE(response.metrics().empty()); - EXPECT_EQ(response.metrics().size(), 4); + EXPECT_EQ(response.metrics().size(), 6); } TEST(TensorflowModuleTest, Success_Predict_ValidateMetrics) { @@ -249,10 +249,10 @@ TEST(TensorflowModuleTest, Success_Predict_ValidateMetrics) { ASSERT_FALSE(response.output().empty()); ASSERT_EQ(response.output(), kPcvrResponse); ASSERT_FALSE(response.metrics().empty()); - EXPECT_EQ(response.metrics().size(), 4); + EXPECT_EQ(response.metrics().size(), 6); CheckMetric(response.metrics(), "kInferenceRequestCount", 1); CheckMetric(response.metrics(), "kInferenceRequestSize", 1435); - CheckMetric(response.metrics(), "kInferenceResponseSize", 288); + CheckMetric(response.metrics(), "kInferenceResponseSize", 415); auto it = response.metrics().find("kInferenceRequestDuration"); ASSERT_NE(it, response.metrics().end()) @@ -278,37 +278,7 @@ TEST(TensorflowModuleTest, Success_PredictWithConsentedRequest) { ASSERT_FALSE(response.output().empty()); ASSERT_EQ(response.output(), kPcvrResponse); ASSERT_FALSE(response.metrics().empty()); - EXPECT_EQ(response.metrics().size(), 4); -} - -TEST(TensorflowModuleTest, Success_ResetModels_NoModel) { - InferenceSidecarRuntimeConfig config; - config.set_model_reset_probability(1.0); - std::unique_ptr tensorflow_module = - ModuleInterface::Create(config); - tensorflow_module->ResetModels(); -} - -TEST(TensorflowModuleTest, Success_ResetModels) { - InferenceSidecarRuntimeConfig config; - config.set_model_reset_probability(1.0); - std::unique_ptr tensorflow_module = - ModuleInterface::Create(config); - RegisterModelRequest register_request; - ASSERT_TRUE(PopulateRegisterModelRequest(kModel1Dir, register_request).ok()); - ASSERT_TRUE(tensorflow_module->RegisterModel(register_request).ok()); - - PredictRequest predict_request; - predict_request.set_input(kPcvrJsonRequest); - absl::StatusOr predict_status = tensorflow_module->Predict(predict_request); - ASSERT_TRUE(predict_status.ok()); - PredictResponse response = predict_status.value(); - ASSERT_FALSE(response.output().empty()); - ASSERT_EQ(response.output(), kPcvrResponse); - ASSERT_FALSE(response.metrics().empty()); - EXPECT_EQ(response.metrics().size(), 4); - - tensorflow_module->ResetModels(); + EXPECT_EQ(response.metrics().size(), 6); } constexpr char kPcvrJsonRequestBatchSize2[] = R"json({ @@ -395,7 +365,160 @@ TEST(TensorflowModuleTest, Success_PredictBatchSize2) { "0\",\"tensor_shape\":[2,1],\"data_type\":\"FLOAT\",\"tensor_" "content\":[0.019116630777716638,0.1847093403339386]}]}]}"); ASSERT_FALSE(response.metrics().empty()); - EXPECT_EQ(response.metrics().size(), 4); + EXPECT_EQ(response.metrics().size(), 6); +} + +TEST(TensorflowModuleTest, Success_RegisterModelWithWarmUpData) { + InferenceSidecarRuntimeConfig config; + std::unique_ptr tensorflow_module = + ModuleInterface::Create(config); + RegisterModelRequest register_request; + ASSERT_TRUE(PopulateRegisterModelRequest(kModel1Dir, register_request).ok()); + register_request.set_warm_up_batch_request_json(kPcvrJsonRequestBatchSize2); + ASSERT_TRUE(tensorflow_module->RegisterModel(register_request).ok()); +} + +constexpr char kPcvrJsonRequestBatchSizeWithWrongPath[] = R"json({ + "request" : [{ + "model_path" : "./benchmark_models/non_exist_path", + "tensors" : [ + { + "tensor_name": "serving_default_double1:0", + "data_type": "DOUBLE", + "tensor_shape": [ + 2, 10 + ], + "tensor_content": ["0.32", "0.12", "0.98", "0.32", "0.12", "0.98", "0.32", "0.12", "0.98", "0.11", "0.32", "0.12", "0.98", "0.32", "0.12", "0.98", "0.32", "0.12", "0.98", "0.11"] + }, + { + "tensor_name": "serving_default_double2:0", + "data_type": "DOUBLE", + "tensor_shape": [ + 2, 10 + ], + "tensor_content": ["0.32", "0.12", "0.98", "0.32", "0.12", "0.98", "0.32", "0.12", "0.98", "0.11", "0.32", "0.12", "0.98", "0.32", "0.12", "0.98", "0.32", "0.12", "0.98", "0.11"] + }, + { + "tensor_name": "serving_default_int_input1:0", + "data_type": "INT64", + "tensor_shape": [ + 2, 1 + ], + "tensor_content": ["7", "3"] + }, + { + "tensor_name": "serving_default_int_input2:0", + "data_type": "INT64", + "tensor_shape": [ + 2, 1 + ], + "tensor_content": ["7", "3"] + }, + { + "tensor_name": "serving_default_int_input3:0", + "data_type": "INT64", + "tensor_shape": [ + 2, 1 + ], + "tensor_content": ["7", "3"] + }, + { + "tensor_name": "serving_default_int_input4:0", + "data_type": "INT64", + "tensor_shape": [ + 2, 1 + ], + "tensor_content": ["7", "3"] + }, + { + "tensor_name": "serving_default_int_input5:0", + "data_type": "INT64", + "tensor_shape": [ + 2, 1 + ], + "tensor_content": ["7", "3"] + } + ] +}] + })json"; + +constexpr char kPcvrJsonRequestBatchSizeWithWrongTensor[] = R"json({ + "request" : [{ + "model_path" : "./benchmark_models/pcvr", + "tensors" : [ + { + "tensor_name": "serving_default_double1:0", + "data_type": "DOUBLE", + "tensor_shape": [ + 2, 10 + ], + "tensor_content": ["0.32", "0.12", "0.98", "0.32", "0.12", "0.98", "0.32", "0.12", "0.98", "0.11", "0.32", "0.12", "0.98", "0.32", "0.12", "0.98", "0.32", "0.12", "0.98", "0.11"] + }, + { + "tensor_name": "serving_default_double2:0", + "data_type": "DOUBLE", + "tensor_shape": [ + 2, 10 + ], + "tensor_content": ["0.32", "0.12", "0.98", "0.32", "0.12", "0.98", "0.32", "0.12", "0.98", "0.11", "0.32", "0.12", "0.98", "0.32", "0.12", "0.98", "0.32", "0.12", "0.98", "0.11"] + }, + { + "tensor_name": "serving_default_int_input1:0", + "data_type": "INT64", + "tensor_shape": [ + 2, 1 + ], + "tensor_content": ["7", "3"] + }, + { + "tensor_name": "serving_default_int_input2:0", + "data_type": "INT64", + "tensor_shape": [ + 2, 1 + ], + "tensor_content": ["7", "3"] + }, + { + "tensor_name": "serving_default_int_input3:0", + "data_type": "INT64", + "tensor_shape": [ + 2, 1 + ], + "tensor_content": ["7", "3"] + }, + { + "tensor_name": "serving_default_int_input4:0", + "data_type": "INT64", + "tensor_shape": [ + 2, 1 + ], + "tensor_content": ["7", "3"] + } + ] +}] + })json"; + +TEST(TensorflowModuleTest, Failure_RegisterModelWithInvalidWarmUpData) { + InferenceSidecarRuntimeConfig config; + std::unique_ptr tensorflow_module = + ModuleInterface::Create(config); + RegisterModelRequest register_request_wrong_path; + ASSERT_TRUE( + PopulateRegisterModelRequest(kModel1Dir, register_request_wrong_path) + .ok()); + register_request_wrong_path.set_warm_up_batch_request_json( + kPcvrJsonRequestBatchSizeWithWrongPath); + ASSERT_FALSE( + tensorflow_module->RegisterModel(register_request_wrong_path).ok()); + + RegisterModelRequest register_request_wrong_tensor; + ASSERT_TRUE( + PopulateRegisterModelRequest(kModel1Dir, register_request_wrong_tensor) + .ok()); + register_request_wrong_tensor.set_warm_up_batch_request_json( + kPcvrJsonRequestBatchSizeWithWrongTensor); + ASSERT_FALSE( + tensorflow_module->RegisterModel(register_request_wrong_tensor).ok()); } constexpr char kPcvrJsonRequestMissingTensorName[] = R"json({ @@ -660,7 +783,7 @@ TEST(TensorflowModuleTest, Success_PredictWith2Models) { "0\",\"tensor_shape\":[2,1],\"data_type\":\"FLOAT\",\"tensor_" "content\":[0.14649735391139985,0.2522672712802887]}]}]}"); ASSERT_FALSE(response.metrics().empty()); - EXPECT_EQ(response.metrics().size(), 4); + EXPECT_EQ(response.metrics().size(), 6); } constexpr char kPcvrJsonRequestWith1ModelVariedSize[] = R"json({ @@ -815,7 +938,7 @@ TEST(TensorflowModuleTest, "0\",\"tensor_shape\":[1,1],\"data_type\":\"FLOAT\",\"tensor_" "content\":[0.010360434651374817]}]}]}"); ASSERT_FALSE(response.metrics().empty()); - EXPECT_EQ(response.metrics().size(), 4); + EXPECT_EQ(response.metrics().size(), 6); } constexpr char kPcvrJsonRequestEmbeddingModel[] = R"json({ @@ -910,7 +1033,7 @@ TEST(TensorflowModuleTest, Success_PredictEmbed) { "{\"tensor_name\":\"StatefulPartitionedCall:1\",\"tensor_shape\":[1,1]," "\"data_type\":\"INT32\",\"tensor_content\":[0]}")); ASSERT_FALSE(response.metrics().empty()); - EXPECT_EQ(response.metrics().size(), 4); + EXPECT_EQ(response.metrics().size(), 6); } constexpr char kMixedValidInvalidBatchJsonRequest[] = R"json({ diff --git a/services/inference_sidecar/modules/tensorflow_v2_14_0/tools/BUILD b/services/inference_sidecar/modules/tensorflow_v2_14_0/tools/BUILD new file mode 100644 index 00000000..d16db80b --- /dev/null +++ b/services/inference_sidecar/modules/tensorflow_v2_14_0/tools/BUILD @@ -0,0 +1,55 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_cc//cc:defs.bzl", "cc_binary") + +package(default_visibility = ["//visibility:public"]) + +cc_binary( + name = "list_ops", + srcs = ["list_ops.cc"], + deps = [ + "@org_tensorflow//tensorflow/cc:cc_ops", + "@org_tensorflow//tensorflow/cc:ops", + "@org_tensorflow//tensorflow/core:core_cpu", + "@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:protos_all_cc", + ], +) + +cc_binary( + name = "freeze_model", + srcs = ["freeze_model.cc"], + deps = [ + "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/flags:parse", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@google_privacysandbox_servers_common//src/util/status_macro:status_macros", + "@org_tensorflow//tensorflow/cc:cc_ops", + "@org_tensorflow//tensorflow/cc:client_session", + "@org_tensorflow//tensorflow/cc:ops", + "@org_tensorflow//tensorflow/cc/saved_model:constants", + "@org_tensorflow//tensorflow/cc/saved_model:loader", + "@org_tensorflow//tensorflow/cc/saved_model:signature_constants", + "@org_tensorflow//tensorflow/cc/saved_model:tag_constants", + "@org_tensorflow//tensorflow/cc/tools:freeze_saved_model", + "@org_tensorflow//tensorflow/core:core_cpu", + "@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:protos_all_cc", + ], +) diff --git a/services/inference_sidecar/modules/tensorflow_v2_14_0/tools/freeze_model.cc b/services/inference_sidecar/modules/tensorflow_v2_14_0/tools/freeze_model.cc new file mode 100644 index 00000000..9a6fed5c --- /dev/null +++ b/services/inference_sidecar/modules/tensorflow_v2_14_0/tools/freeze_model.cc @@ -0,0 +1,93 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Freezes a given TensorFlow model. +// Usage: ./builders/tools/bazel-debian run //tools:freeze_model -- --model_path +// /src/workspace/ + +#include +#include +#include +#include + +#include "absl/flags/flag.h" +#include "absl/flags/parse.h" +#include "absl/log/absl_log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "src/util/status_macro/status_macros.h" +#include "tensorflow/cc/client/client_session.h" +#include "tensorflow/cc/saved_model/loader.h" +#include "tensorflow/cc/saved_model/tag_constants.h" +#include "tensorflow/cc/tools/freeze_saved_model.h" +#include "tensorflow/core/framework/graph.pb.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/platform/init_main.h" +#include "tensorflow/core/protobuf/meta_graph.pb.h" +#include "tensorflow/core/public/session.h" +#include "tensorflow/core/public/version.h" + +ABSL_FLAG(std::string, model_path, "", "Path to the TensorFlow model."); + +namespace { + +absl::Status FreezeSavedModel(tensorflow::SessionOptions& session_options, + tensorflow::SavedModelBundle& model_bundle) { + tensorflow::GraphDef frozen_graph_def; + std::unordered_set dummy_inputs, dummy_outputs; + PS_RETURN_IF_ERROR(tensorflow::FreezeSavedModel( + model_bundle, &frozen_graph_def, &dummy_inputs, &dummy_outputs)); + + std::unique_ptr frozen_session( + tensorflow::NewSession(session_options)); + PS_RETURN_IF_ERROR(frozen_session->Create(frozen_graph_def)); + + // Updates the model in place. + model_bundle.session = std::move(frozen_session); + *model_bundle.meta_graph_def.mutable_graph_def() = + std::move(frozen_graph_def); + + return absl::OkStatus(); +} + +} // namespace + +int main(int argc, char** argv) { + absl::ParseCommandLine(argc, argv); + + ABSL_LOG(INFO) << "TensorFlow Version: " << TF_VERSION_STRING << std::endl; + ABSL_LOG(INFO) << "TensorFlow GraphDef Version: " << TF_GRAPH_DEF_VERSION + << std::endl; + + tensorflow::SessionOptions session_options; + const std::unordered_set tags = {"serve"}; + auto model_bundle = std::make_unique(); + if (auto status = tensorflow::LoadSavedModel(session_options, {}, + absl::GetFlag(FLAGS_model_path), + tags, model_bundle.get()); + !status.ok()) { + ABSL_LOG(ERROR) << "Error loading model: " << status; + return 1; + } + ABSL_LOG(INFO) << "The model is successfully loaded."; + + if (auto status = FreezeSavedModel(session_options, *model_bundle); + !status.ok()) { + ABSL_LOG(ERROR) << "Error freezing model: " << status; + return 1; + } + ABSL_LOG(INFO) << "The model is successfully frozen."; + return 0; +} diff --git a/services/inference_sidecar/modules/tensorflow_v2_14_0/tools/list_ops.cc b/services/inference_sidecar/modules/tensorflow_v2_14_0/tools/list_ops.cc new file mode 100644 index 00000000..af08635f --- /dev/null +++ b/services/inference_sidecar/modules/tensorflow_v2_14_0/tools/list_ops.cc @@ -0,0 +1,40 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Lists all registered TensorFlow operations. +// Usage: ./builders/tools/bazel-debian run //tools:list_ops + +#include +#include + +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/platform/init_main.h" +#include "tensorflow/core/public/version.h" + +int main(int argc, char** argv) { + tensorflow::port::InitMain(argv[0], &argc, &argv); + + std::cout << "TensorFlow Version: " << TF_VERSION_STRING << std::endl; + std::cout << "TensorFlow GraphDef Version: " << TF_GRAPH_DEF_VERSION + << std::endl; + + std::vector op_list; + tensorflow::OpRegistry::Global()->GetRegisteredOps(&op_list); + + std::cout << "Ops Name, is_stateful" << std::endl; + for (const auto& op_def : op_list) { + std::cout << op_def.name() << ", " << op_def.is_stateful() << std::endl; + } + return 0; +} diff --git a/services/seller_frontend_service/benchmarking/BUILD b/services/seller_frontend_service/benchmarking/BUILD index 85107cc7..8c21c03c 100644 --- a/services/seller_frontend_service/benchmarking/BUILD +++ b/services/seller_frontend_service/benchmarking/BUILD @@ -28,6 +28,7 @@ cc_binary( "//services/common/encryption:mock_crypto_client_wrapper", "//services/common/test:mocks", "//services/common/test:random", + "//services/common/test/utils:test_init", "//services/seller_frontend_service", "//services/seller_frontend_service/util:select_ad_reactor_test_utils", "@google_benchmark//:benchmark", diff --git a/services/seller_frontend_service/benchmarking/select_ad_reactor_benchmarks.cc b/services/seller_frontend_service/benchmarking/select_ad_reactor_benchmarks.cc index de601812..b422853b 100644 --- a/services/seller_frontend_service/benchmarking/select_ad_reactor_benchmarks.cc +++ b/services/seller_frontend_service/benchmarking/select_ad_reactor_benchmarks.cc @@ -20,6 +20,7 @@ #include "services/common/metric/server_definition.h" #include "services/common/reporters/async_reporter.h" #include "services/common/test/random.h" +#include "services/common/test/utils/test_init.h" #include "services/seller_frontend_service/select_ad_reactor_web.h" #include "services/seller_frontend_service/util/select_ad_reactor_test_utils.h" @@ -333,6 +334,7 @@ void ScoringSignalsProviderStub::Get( } static void BM_PerformDebugReporting(benchmark::State& state) { + CommonTestInit(); ProtectedAuctionInput protected_auction_input = MakeARandomProtectedAuctionInput(); protected_auction_input.set_enable_debug_reporting(true); @@ -358,6 +360,7 @@ static void BM_PerformDebugReporting(benchmark::State& state) { TrustedServersConfigClient config_client = CreateConfig(); config_client.SetOverride("", CONSENTED_DEBUG_TOKEN); config_client.SetOverride(kFalse, ENABLE_PROTECTED_APP_SIGNALS); + config_client.SetOverride(kFalse, ENABLE_CHAFFING); ClientRegistry clients{scoring_provider, scoring_client, buyer_clients, @@ -384,6 +387,7 @@ static void BM_PerformDebugReporting(benchmark::State& state) { BENCHMARK(BM_PerformDebugReporting); static void BM_PerformCurrencyCheckingAndFiltering(benchmark::State& state) { + CommonTestInit(); ProtectedAuctionInput protected_auction_input = MakeARandomProtectedAuctionInput(); SelectAdRequest request = MakeARandomSelectAdRequest( @@ -410,6 +414,7 @@ static void BM_PerformCurrencyCheckingAndFiltering(benchmark::State& state) { TrustedServersConfigClient config_client = CreateConfig(); config_client.SetOverride("", CONSENTED_DEBUG_TOKEN); config_client.SetOverride(kTrue, ENABLE_PROTECTED_APP_SIGNALS); + config_client.SetOverride(kFalse, ENABLE_CHAFFING); ClientRegistry clients{scoring_provider, scoring_client, buyer_clients, diff --git a/services/seller_frontend_service/private_aggregation/BUILD b/services/seller_frontend_service/private_aggregation/BUILD new file mode 100644 index 00000000..5ae61dda --- /dev/null +++ b/services/seller_frontend_service/private_aggregation/BUILD @@ -0,0 +1,52 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "private_aggregation_helper", + srcs = [ + "private_aggregation_helper.cc", + ], + hdrs = [ + "private_aggregation_helper.h", + ], + deps = [ + "//api:bidding_auction_servers_cc_proto", + "//services/common/loggers:request_log_context", + "//services/common/private_aggregation:private_aggregation_post_auction_util", + "//services/common/util:reporting_util", + "//services/seller_frontend_service/data:seller_frontend_data", + "@google_privacysandbox_servers_common//src/logger:request_context_impl", + ], +) + +cc_test( + name = "private_aggregation_helper_test", + size = "small", + srcs = [ + "private_aggregation_helper_test.cc", + ], + deps = [ + ":private_aggregation_helper", + "//services/common/private_aggregation:private_aggregation_test_util", + "//services/common/test/utils:test_init", + "//services/seller_frontend_service/data:seller_frontend_data", + "@com_google_absl//absl/numeric:int128", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/services/seller_frontend_service/private_aggregation/private_aggregation_helper.cc b/services/seller_frontend_service/private_aggregation/private_aggregation_helper.cc new file mode 100644 index 00000000..f3b3c420 --- /dev/null +++ b/services/seller_frontend_service/private_aggregation/private_aggregation_helper.cc @@ -0,0 +1,172 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "services/seller_frontend_service/private_aggregation/private_aggregation_helper.h" + +#include +#include +#include +#include + +#include "services/common/loggers/request_log_context.h" +#include "services/common/private_aggregation/private_aggregation_post_auction_util.h" +#include "services/common/util/reporting_util.h" +#include "src/logger/request_context_logger.h" + +namespace privacy_sandbox::bidding_auction_servers { + +namespace { + +// Filters the referenced AdWithBid parameter to select win, custom, and +// always contributions. Edits the passed AdWithBid input parameter. +void FilterWinContributions(AdWithBid& ad_with_bid) { + std::vector winning_contributions; + for (auto& contribution : + *ad_with_bid.mutable_private_aggregation_contributions()) { + if (contribution.event().event_type() == EventType::EVENT_TYPE_WIN || + contribution.event().event_type() == EventType::EVENT_TYPE_CUSTOM || + contribution.event().event_type() == EventType::EVENT_TYPE_ALWAYS) { + winning_contributions.push_back(std::move(contribution)); + } + } + ad_with_bid.clear_private_aggregation_contributions(); + + for (auto& win_contribution : winning_contributions) { + *ad_with_bid.add_private_aggregation_contributions() = + std::move(win_contribution); + } +} + +// Filters the referenced AdWithBid parameter to select loss and always +// contributions. Edits the passed AdWithBid input parameter. +void FilterLossContributions(AdWithBid& ad_with_bid) { + std::vector losing_contributions; + for (auto& contribution : + *ad_with_bid.mutable_private_aggregation_contributions()) { + if (contribution.event().event_type() == EventType::EVENT_TYPE_LOSS || + contribution.event().event_type() == EventType::EVENT_TYPE_ALWAYS) { + losing_contributions.push_back(std::move(contribution)); + } + } + + ad_with_bid.clear_private_aggregation_contributions(); + + for (auto& loss_contribution : losing_contributions) { + *ad_with_bid.add_private_aggregation_contributions() = + std::move(loss_contribution); + } +} + +BaseValues GetBaseValues(ScoreAdsResponse::AdScore& high_score) { + BaseValues base_values; + float highest_scoring_other_bid = 0.0; + if (high_score.ig_owner_highest_scoring_other_bids_map().size() > 0) { + auto iterator = + high_score.ig_owner_highest_scoring_other_bids_map().begin(); + if (!iterator->second.values().empty()) { + highest_scoring_other_bid = + iterator->second.values().Get(0).number_value(); + } + } + base_values.highest_scoring_other_bid = highest_scoring_other_bid; + + if (high_score.incoming_bid_in_seller_currency() > 0) { + base_values.winning_bid = high_score.incoming_bid_in_seller_currency(); + } else { + base_values.winning_bid = high_score.buyer_bid(); + } + return base_values; +} + +std::vector GetProcessedAndFilteredContributions( + AdWithBid& ad_with_bid, BaseValues& base_values) { + std::vector processed_filtered_contributions; + for (PrivateAggregateContribution& contribution : + *ad_with_bid.mutable_private_aggregation_contributions()) { + absl::StatusOr signal_value_status = + GetPrivateAggregationValuePostAuction(base_values, + contribution.value()); + absl::StatusOr signal_bucket_status = + GetPrivateAggregationBucketPostAuction(base_values, + contribution.bucket()); + if (signal_value_status.ok() && signal_bucket_status.ok()) { + if (contribution.value().has_extended_value()) { + contribution.mutable_value()->clear_extended_value(); + } + contribution.mutable_value()->set_int_value(signal_value_status.value()); + if (contribution.bucket().has_signal_bucket()) { + contribution.mutable_bucket()->clear_signal_bucket(); + } + *contribution.mutable_bucket()->mutable_bucket_128_bit() = + signal_bucket_status.value(); + contribution.clear_event(); + processed_filtered_contributions.push_back(std::move(contribution)); + } else { + PS_VLOG(4) << "Private Aggregation Contribution dropped."; + } + } + return processed_filtered_contributions; +} + +} // namespace + +void HandlePrivateAggregationContributions( + ScoreAdsResponse::AdScore& high_score, + BuyerBidsResponseMap& shared_buyer_bids_map) { + for (auto& [ig_owner, get_bid_response] : shared_buyer_bids_map) { + PrivateAggregateReportingResponse reporting_response; + reporting_response.set_adtech_origin(ig_owner); + + for (auto& ad_with_bid : *get_bid_response->mutable_bids()) { + if (ad_with_bid.private_aggregation_contributions().empty()) { + continue; + } + const std::string& ig_name = ad_with_bid.interest_group_name(); + + BaseValues base_values = GetBaseValues(high_score); + + // Filtering. + if (high_score.interest_group_owner() == ig_owner && + ig_name == high_score.interest_group_name()) { + FilterWinContributions(ad_with_bid); + } else { + FilterLossContributions(ad_with_bid); + + // Iterate over ad_rejection_reasons to find the appropriate rejection + // reason. + for (const auto& rejection : high_score.ad_rejection_reasons()) { + if (rejection.interest_group_owner() == ig_owner && + rejection.interest_group_name() == ig_name) { + base_values.reject_reason = rejection.rejection_reason(); + break; + } + } + } + if (ad_with_bid.private_aggregation_contributions().empty()) { + continue; + } + + std::vector + processed_filtered_contributions = + GetProcessedAndFilteredContributions(ad_with_bid, base_values); + + for (PrivateAggregateContribution& contribution : + processed_filtered_contributions) { + *reporting_response.add_contributions() = std::move(contribution); + } + } + *high_score.add_top_level_contributions() = std::move(reporting_response); + } +} + +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/seller_frontend_service/private_aggregation/private_aggregation_helper.h b/services/seller_frontend_service/private_aggregation/private_aggregation_helper.h new file mode 100644 index 00000000..dce7d8a8 --- /dev/null +++ b/services/seller_frontend_service/private_aggregation/private_aggregation_helper.h @@ -0,0 +1,32 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#ifndef SERVICES_SELLER_FRONTEND_SERVICE_PRIVATE_AGGREGATION_PRIVATE_AGGREGATION_HELPER_H_ +#define SERVICES_SELLER_FRONTEND_SERVICE_PRIVATE_AGGREGATION_PRIVATE_AGGREGATION_HELPER_H_ + +#include +#include + +#include "api/bidding_auction_servers.pb.h" +#include "services/seller_frontend_service/data/scoring_signals.h" + +namespace privacy_sandbox::bidding_auction_servers { + +// Handles private aggregation contributions filtering and post processing. +void HandlePrivateAggregationContributions( + ScoreAdsResponse::AdScore& high_score, + BuyerBidsResponseMap& shared_buyer_bids_map); + +} // namespace privacy_sandbox::bidding_auction_servers + +#endif // SERVICES_SELLER_FRONTEND_SERVICE_PRIVATE_AGGREGATION_PRIVATE_AGGREGATION_HELPER_H_ diff --git a/services/seller_frontend_service/private_aggregation/private_aggregation_helper_test.cc b/services/seller_frontend_service/private_aggregation/private_aggregation_helper_test.cc new file mode 100644 index 00000000..7c655121 --- /dev/null +++ b/services/seller_frontend_service/private_aggregation/private_aggregation_helper_test.cc @@ -0,0 +1,122 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "services/seller_frontend_service/private_aggregation/private_aggregation_helper.h" + +#include + +#include +#include +#include + +#include "absl/numeric/int128.h" +#include "google/protobuf/util/message_differencer.h" +#include "gtest/gtest.h" +#include "services/common/private_aggregation/private_aggregation_test_util.h" +#include "services/seller_frontend_service/data/scoring_signals.h" + +namespace privacy_sandbox::bidding_auction_servers { +namespace { + +constexpr absl::string_view kTestIgNameWin = "testIgNameWin"; +constexpr absl::string_view kTestIgNameLoss = "testIgNameLoss"; +constexpr absl::string_view kTestIgOwner = "testIgOwner"; + +TEST(HandlePrivateAggregationContributionsTest, + DISABLED_FiltersAndPostProcessesContributions) { + ScoreAdsResponse::AdScore high_score; + high_score.set_interest_group_name(kTestIgNameWin); + high_score.set_buyer_bid(1.0); + HighestScoringOtherBidsMap ig_owner_highest_scoring_other_bids_map = + GetHighestScoringOtherBidsMap(kTestIgOwner); + high_score.mutable_ig_owner_highest_scoring_other_bids_map()->insert( + ig_owner_highest_scoring_other_bids_map.begin(), + ig_owner_highest_scoring_other_bids_map.end()); + + BuyerBidsResponseMap shared_buyer_bids_map; + GetBidsResponse::GetBidsRawResponse buyer_bids_raw_response = + GetBidsRawResponseWithPrivateAggregationContributions(kTestIgNameWin, + kTestIgNameLoss); + auto raw_response = std::make_unique(); + std::unique_ptr buyer_bids_ptr = + std::make_unique( + buyer_bids_raw_response); + + shared_buyer_bids_map.try_emplace( + static_cast>(kTestIgOwner), + std::move(buyer_bids_ptr)); + + HandlePrivateAggregationContributions(high_score, shared_buyer_bids_map); + + // Filling expected Bucket128Bit testFinalBucketFromSignal using values from + // test util. + auto ig_owner_it = + high_score.ig_owner_highest_scoring_other_bids_map().find(kTestIgOwner); + const auto& other_bids = ig_owner_it->second; + float base_value_numerical_value = other_bids.values().Get(0).number_value(); + absl::uint128 offset_value = + absl::MakeUint128(static_cast(112233445566778899ULL), + static_cast(12345678901234567890ULL)); + absl::uint128 final_value = absl::MakeUint128( + 0, static_cast(base_value_numerical_value * 2.0)); + final_value -= offset_value; + Bucket128Bit testFinalBucketFromSignal; + testFinalBucketFromSignal.add_bucket_128_bits( + absl::Uint128Low64(final_value)); + testFinalBucketFromSignal.add_bucket_128_bits( + absl::Uint128High64(final_value)); + Bucket128Bit testFinalBucketFrom128Bit = GetTestBucket128Bit(); + + // Filling an expected AdScore object with values to match high_score after + // HandlePrivateAggregationContributions is called. + ScoreAdsResponse::AdScore expected_adscore; + PrivateAggregateReportingResponse expected_response; + expected_response.set_adtech_origin(kTestIgOwner); + PrivateAggregateContribution contribution0; + contribution0.mutable_value()->set_int_value(11); + *contribution0.mutable_bucket()->mutable_bucket_128_bit() = + testFinalBucketFromSignal; + PrivateAggregateContribution contribution1; + contribution1.mutable_value()->set_int_value(10); + *contribution1.mutable_bucket()->mutable_bucket_128_bit() = + testFinalBucketFrom128Bit; + PrivateAggregateContribution contribution2; + contribution2.mutable_value()->set_int_value(11); + *contribution2.mutable_bucket()->mutable_bucket_128_bit() = + testFinalBucketFromSignal; + PrivateAggregateContribution contribution3; + contribution3.mutable_value()->set_int_value(10); + *contribution3.mutable_bucket()->mutable_bucket_128_bit() = + testFinalBucketFrom128Bit; + *expected_response.add_contributions() = std::move(contribution0); + *expected_response.add_contributions() = std::move(contribution1); + *expected_response.add_contributions() = std::move(contribution2); + *expected_response.add_contributions() = std::move(contribution3); + *expected_adscore.add_top_level_contributions() = + std::move(expected_response); + expected_adscore.set_interest_group_name(kTestIgNameWin); + expected_adscore.set_buyer_bid(1.0); + expected_adscore.mutable_ig_owner_highest_scoring_other_bids_map()->insert( + ig_owner_highest_scoring_other_bids_map.begin(), + ig_owner_highest_scoring_other_bids_map.end()); + + google::protobuf::util::MessageDifferencer diff; + std::string diff_output; + diff.ReportDifferencesToString(&diff_output); + EXPECT_TRUE(diff.Compare(high_score, expected_adscore)) << diff_output; +} + +} // namespace + +} // namespace privacy_sandbox::bidding_auction_servers diff --git a/services/seller_frontend_service/select_ad_reactor.cc b/services/seller_frontend_service/select_ad_reactor.cc index 7627dd25..00c42185 100644 --- a/services/seller_frontend_service/select_ad_reactor.cc +++ b/services/seller_frontend_service/select_ad_reactor.cc @@ -521,8 +521,8 @@ void SelectAdReactor::Execute() { LogIfError(metric_context_->LogHistogram( (int)request_->auction_config().ByteSizeLong())); - PS_VLOG(kEncrypted, log_context_) << "Encrypted SelectAdRequest:\n" - << request_->ShortDebugString(); + PS_VLOG(kEncrypted, log_context_) + << "Encrypted SelectAdRequest exported in EventMessage"; log_context_.SetEventMessageField(*request_); PS_VLOG(kPlain, log_context_) @@ -539,14 +539,10 @@ void SelectAdReactor::Execute() { std::visit( [this](auto& input) { log_context_.SetEventMessageField(input); - for (auto& each_buyer_input : *input.mutable_buyer_input()) { - each_buyer_input.second = "encoded buyer input erased"; - } PS_VLOG(kPlain, log_context_) << (is_protected_auction_request_ ? "ProtectedAuctionInput" : "ProtectedAudienceInput") - << ":\n" - << input.ShortDebugString(); + << " exported in EventMessage"; }, protected_auction_input_); MayLogBuyerInput(); @@ -1139,7 +1135,7 @@ void SelectAdReactor::FinishWithStatus(const grpc::Status& status) { static_cast((absl::Now() - start_) / absl::Milliseconds(1)))); } benchmarking_logger_->End(); - log_context_.ExportEventMessage(); + log_context_.ExportEventMessage(/*if_export_consented=*/true); Finish(status); } diff --git a/services/seller_frontend_service/select_ad_reactor_app.cc b/services/seller_frontend_service/select_ad_reactor_app.cc index 2c5e38ce..c996ec3e 100644 --- a/services/seller_frontend_service/select_ad_reactor_app.cc +++ b/services/seller_frontend_service/select_ad_reactor_app.cc @@ -90,8 +90,7 @@ absl::StatusOr SelectAdReactorForApp::GetNonEncryptedResponse( protected_auction_input_, request_->auction_config().top_level_seller()); } - PS_VLOG(kPlain, log_context_) << "AuctionResult:\n" - << auction_result.ShortDebugString(); + PS_VLOG(kPlain, log_context_) << "AuctionResult exported in EventMessage"; log_context_.SetEventMessageField(auction_result); // Serialized the data to bytes array. diff --git a/services/seller_frontend_service/select_ad_reactor_web.cc b/services/seller_frontend_service/select_ad_reactor_web.cc index dd45b50a..424c56d7 100644 --- a/services/seller_frontend_service/select_ad_reactor_web.cc +++ b/services/seller_frontend_service/select_ad_reactor_web.cc @@ -77,7 +77,7 @@ absl::StatusOr SelectAdReactorForWeb::GetNonEncryptedResponse( auto result = CborDecodeAuctionResultToProto(encoded_data); if (result.ok()) { log_context_.SetEventMessageField(*result); - return result->DebugString(); + return std::string("exported in EventMessage"); } else { return result.status().ToString(); } @@ -91,7 +91,7 @@ absl::StatusOr SelectAdReactorForWeb::GetNonEncryptedResponse( request_->auction_config().top_level_seller(), high_score, GetBiddingGroups(shared_buyer_bids_map_, *buyer_inputs_), error, error_handler)); - PS_VLOG(kPlain, log_context_) << "AuctionResult:\n" << (decode_lambda()); + PS_VLOG(kPlain, log_context_) << "AuctionResult: " << (decode_lambda()); } else if (auction_scope_ == AuctionScope::AUCTION_SCOPE_SERVER_COMPONENT_MULTI_SELLER) { // If this is server component auction, serialize as proto. @@ -110,8 +110,7 @@ absl::StatusOr SelectAdReactorForWeb::GetNonEncryptedResponse( // Serialized the data to bytes array. encoded_data = auction_result.SerializeAsString(); - PS_VLOG(kPlain, log_context_) << "AuctionResult:\n" - << auction_result.ShortDebugString(); + PS_VLOG(kPlain, log_context_) << "AuctionResult exported in EventMessage"; log_context_.SetEventMessageField(auction_result); } else { // SINGLE_SELLER or SERVER_TOP_LEVEL Auction diff --git a/services/seller_frontend_service/seller_frontend_main.cc b/services/seller_frontend_service/seller_frontend_main.cc index 93f195fc..005a65d9 100644 --- a/services/seller_frontend_service/seller_frontend_main.cc +++ b/services/seller_frontend_service/seller_frontend_main.cc @@ -66,10 +66,10 @@ ABSL_FLAG(std::optional, key_value_signals_host, std::nullopt, ABSL_FLAG(std::optional, buyer_server_hosts, std::nullopt, "Comma seperated list of domain addresses of the BuyerFrontEnd " "services for getting bids."); -ABSL_FLAG(std::optional, enable_buyer_compression, true, - "Flag to enable buyer client compression. True by default."); -ABSL_FLAG(std::optional, enable_auction_compression, true, - "Flag to enable auction client compression. True by default."); +ABSL_FLAG(std::optional, enable_buyer_compression, std::nullopt, + "Flag to enable buyer client compression. Turned off by default."); +ABSL_FLAG(std::optional, enable_auction_compression, std::nullopt, + "Flag to enable auction client compression. Turned off by default."); ABSL_FLAG(std::optional, enable_seller_frontend_benchmarking, std::nullopt, "Flag to enable benchmarking."); ABSL_FLAG( diff --git a/services/seller_frontend_service/util/proto_mapping_util.cc b/services/seller_frontend_service/util/proto_mapping_util.cc index b5c76328..6192b76e 100644 --- a/services/seller_frontend_service/util/proto_mapping_util.cc +++ b/services/seller_frontend_service/util/proto_mapping_util.cc @@ -77,6 +77,10 @@ AuctionResult MapAdScoreToAuctionResult( ->try_emplace(event, url); } SetReportingUrls(high_score->win_reporting_urls(), auction_result); + if (high_score->top_level_contributions().size() > 0) { + *auction_result.mutable_top_level_contributions() = + high_score->top_level_contributions(); + } *auction_result.mutable_ad_component_render_urls() = high_score->component_renders(); auction_result.set_ad_type(high_score->ad_type()); @@ -134,12 +138,11 @@ absl::StatusOr PackageAuctionResultForWeb( return absl::Status(serialized_data.status().code(), error_msg); } else { PS_VLOG(kPlain, log_context) - << "AuctionResult:\n" - << [&](absl::string_view encoded_data) { + << "AuctionResult: " << [&](absl::string_view encoded_data) { auto result = CborDecodeAuctionResultToProto(encoded_data); if (result.ok()) { log_context.SetEventMessageField(*result); - return result->DebugString(); + return std::string("exported in EventMessage"); } else { return result.status().ToString(); } @@ -159,7 +162,7 @@ absl::StatusOr PackageAuctionResultForApp( // Map to AuctionResult proto and serialized to bytes array. AuctionResult result = MapAdScoreToAuctionResult(high_score, error); - PS_VLOG(kPlain, log_context) << "AuctionResult:\n" << result.DebugString(); + PS_VLOG(kPlain, log_context) << "AuctionResult exported in EventMessage"; log_context.SetEventMessageField(result); return PackageAuctionResultCiphertext(result.SerializeAsString(), decrypted_request); diff --git a/services/seller_frontend_service/util/proto_mapping_util_test.cc b/services/seller_frontend_service/util/proto_mapping_util_test.cc index 3d950b08..e31d0e5d 100644 --- a/services/seller_frontend_service/util/proto_mapping_util_test.cc +++ b/services/seller_frontend_service/util/proto_mapping_util_test.cc @@ -51,6 +51,46 @@ AuctionResult::Error MakeAnError() { return error; } +PrivateAggregateContribution CreateTestPAggContribution( + EventType event_type, absl::string_view event_name = "") { + // Create the example SignalValue + SignalValue signal_value; + signal_value.set_base_value(BASE_VALUE_WINNING_BID); // Set the base_value + signal_value.set_offset(10); // Set some offset value + signal_value.set_scale(1.5); // Set scale factor + + // Create the example SignalBucket + SignalBucket signal_bucket; + signal_bucket.set_base_value( + BASE_VALUE_HIGHEST_SCORING_OTHER_BID); // Set the base_value + signal_bucket.set_scale(2.0); // Set scale factor + + // Create the BucketOffset + BucketOffset bucket_offset; + bucket_offset.add_value(12345678901234567890ULL); // Add 64-bit values + bucket_offset.add_value(12345678901234567890ULL); // Add another 64-bit value + bucket_offset.set_is_negative(true); // Set the is_negative flag + + // Set the BucketOffset in SignalBucket + *signal_bucket.mutable_offset() = bucket_offset; + + // Create the PrivateAggregationValue with SignalValue + PrivateAggregationValue private_aggregation_value; + *private_aggregation_value.mutable_extended_value() = signal_value; + + // Create the PrivateAggregationBucket with SignalBucket + PrivateAggregationBucket private_aggregation_bucket; + *private_aggregation_bucket.mutable_signal_bucket() = signal_bucket; + + // Create a PrivateAggregateContribution and set its fields + PrivateAggregateContribution contribution; + *contribution.mutable_bucket() = private_aggregation_bucket; + *contribution.mutable_value() = private_aggregation_value; + contribution.mutable_event()->set_event_type(event_type); + contribution.mutable_event()->set_event_name(event_name); + return contribution; +} + AuctionResult MapBasicScoreFieldsToAuctionResult( const ScoreAdsResponse::AdScore& high_score) { AuctionResult auction_result; @@ -109,6 +149,16 @@ AuctionResult MapBasicScoreFieldsToAuctionResult( return auction_result; } +void MapBasicScoreFieldsToAuctionResultForPrivateAggregation( + ScoreAdsResponse::AdScore& high_score, AuctionResult& auction_result) { + PrivateAggregateContribution contribution = + CreateTestPAggContribution(EVENT_TYPE_WIN); + PrivateAggregateReportingResponse pagg_response; + *pagg_response.add_contributions() = contribution; + *auction_result.add_top_level_contributions() = pagg_response; + *high_score.add_top_level_contributions() = std::move(pagg_response); +} + AuctionResult MapBasicErrorFieldsToAuctionResult( const AuctionResult::Error& error) { AuctionResult auction_result; @@ -169,6 +219,18 @@ TEST_F(AdScoreToAuctionResultTest, MapsAllFieldsForSingleSellerAuction) { EXPECT_THAT(expected, EqualsProto(output)); } +TEST_F(AdScoreToAuctionResultTest, + MapsAllFieldsForSingleSellerAuctionWithPAAPIContributions) { + AuctionResult expected = MapBasicScoreFieldsToAuctionResult(valid_score_); + MapBasicScoreFieldsToAuctionResultForPrivateAggregation(valid_score_, + expected); + AuctionResult output = AdScoreToAuctionResult( + valid_score_, /*maybe_bidding_groups=*/std::nullopt, + /*error=*/std::nullopt, AuctionScope::AUCTION_SCOPE_SINGLE_SELLER, + empty_seller_, empty_audience_); + EXPECT_THAT(expected, EqualsProto(output)); +} + TEST_F(AdScoreToAuctionResultTest, MapsAllFieldsForTopLevelAuction) { AuctionResult expected = MapBasicScoreFieldsToAuctionResult(valid_score_); AuctionResult output = AdScoreToAuctionResult( diff --git a/third_party/container_deps.bzl b/third_party/container_deps.bzl index 8620e41d..60f7e25a 100644 --- a/third_party/container_deps.bzl +++ b/third_party/container_deps.bzl @@ -14,13 +14,20 @@ load("@google_privacysandbox_servers_common//third_party:container_deps.bzl", common_container_deps = "container_deps") load("@io_bazel_rules_docker//container:container.bzl", "container_pull") -load("@rules_oci//oci:pull.bzl", "oci_pull") def container_deps(): common_container_deps() - # GCP SFE still needs the docker images due to compatibility issues with container_flatten rule. - docker_images = { + images = { + "aws-lambda-python": { + "arch_hashes": { + # 3.9.2022.09.27.12 + "amd64": "c5d5475944755926a3caa6aac4486632f5fed11d531e6437b42dd48718725f29", + "arm64": "d4d6d56eae30e4f74c6aa617043e2018142b2c4aafa02468653799c891bf86cf", + }, + "registry": "public.ecr.aws", + "repository": "lambda/python", + }, "envoy-distroless": { "arch_hashes": { # v1.23.1 @@ -39,17 +46,14 @@ def container_deps(): "registry": "gcr.io", "repository": "distroless/cc-debian11", }, - } - - images = { - "aws-lambda-python": { + "runtime-debian-slim": { "arch_hashes": { - # 3.9.2022.09.27.12 - "amd64": "c5d5475944755926a3caa6aac4486632f5fed11d531e6437b42dd48718725f29", - "arm64": "d4d6d56eae30e4f74c6aa617043e2018142b2c4aafa02468653799c891bf86cf", + # stable-20221004-slim + "amd64": "a4912461baca94ca557af4e779857867a25e0215d157d2dc04f148811e7877f8", + "arm64": "af9a018b749427a53fded449bd1fbb2cbdc7077d8922b7ebb7bd6478ed40d8e7", }, - "registry": "public.ecr.aws", - "repository": "lambda/python", + "registry": "docker.io", + "repository": "library/debian", }, } @@ -60,17 +64,6 @@ def container_deps(): registry = image["registry"], repository = image["repository"], ) - for img_name, image in docker_images.items() - for arch, hash in image["arch_hashes"].items() - ] - - [ - oci_pull( - name = img_name + "-" + arch, - digest = "sha256:" + hash, - registry = image["registry"], - repository = image["repository"], - ) for img_name, image in images.items() for arch, hash in image["arch_hashes"].items() ] diff --git a/tools/debug/start_auction b/tools/debug/start_auction index 59e5f78a..5aeff445 100755 --- a/tools/debug/start_auction +++ b/tools/debug/start_auction @@ -51,7 +51,7 @@ export SERVER_START_CMD=$(cat << END --enable_protected_audience="true" \ --auction_tcmalloc_background_release_rate_bytes_per_second=4096 \ --auction_tcmalloc_max_total_thread_cache_bytes=10737418240 \ ---enable_protected_app_signals="false" && exit +--enable_protected_app_signals="true" && exit END ) diff --git a/tools/debug/start_bfe b/tools/debug/start_bfe index 6e90089e..dd7f8e1a 100755 --- a/tools/debug/start_bfe +++ b/tools/debug/start_bfe @@ -39,7 +39,7 @@ export SERVER_START_CMD=$(cat << END --enable_protected_audience="true" \ --bfe_tcmalloc_background_release_rate_bytes_per_second=4096 \ --bfe_tcmalloc_max_total_thread_cache_bytes=10737418240 \ ---enable_protected_app_signals="false" && exit +--enable_protected_app_signals="true" && exit END ) diff --git a/tools/debug/start_bidding b/tools/debug/start_bidding index 9771986d..972ab191 100755 --- a/tools/debug/start_bidding +++ b/tools/debug/start_bidding @@ -139,7 +139,7 @@ export SERVER_START_CMD=$(cat << END "urlFetchTimeoutMs": 30000, }' \ --buyer_code_fetch_config='${BUYER_CODE_FETCH_CONFIG}' \ ---enable_protected_app_signals="false" \ +--enable_protected_app_signals="true" \ --enable_otel_based_logging="true" \ --consented_debug_token="test_token" \ --enable_protected_audience="true" \ diff --git a/tools/debug/start_sfe b/tools/debug/start_sfe index 1825d751..6ddf9a69 100755 --- a/tools/debug/start_sfe +++ b/tools/debug/start_sfe @@ -47,7 +47,7 @@ export SERVER_START_CMD=$(cat << END --enable_protected_audience="true" \ --sfe_tcmalloc_background_release_rate_bytes_per_second=4096 \ --sfe_tcmalloc_max_total_thread_cache_bytes=10737418240 \ ---enable_protected_app_signals="false" && exit +--enable_protected_app_signals="true" && exit END ) diff --git a/tools/secure_invoke/BUILD b/tools/secure_invoke/BUILD index 442da1f6..f7b6a490 100644 --- a/tools/secure_invoke/BUILD +++ b/tools/secure_invoke/BUILD @@ -52,6 +52,7 @@ cc_binary( deps = [ ":secure_invoke_lib", "//api:bidding_auction_servers_cc_grpc_proto", + "//services/common/test/utils:test_init", "//tools/secure_invoke/payload_generator:payload_packaging_lib", "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/flags:parse", @@ -73,6 +74,7 @@ cc_binary( ], deps = [ ":secure_invoke_lib", + "//services/common/test/utils:test_init", "//tools/secure_invoke/payload_generator:payload_packaging_lib", "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/flags:parse", diff --git a/tools/secure_invoke/secure_invoke.cc b/tools/secure_invoke/secure_invoke.cc index e628eb67..e6afc219 100644 --- a/tools/secure_invoke/secure_invoke.cc +++ b/tools/secure_invoke/secure_invoke.cc @@ -23,6 +23,7 @@ #include "absl/synchronization/notification.h" #include "api/bidding_auction_servers.grpc.pb.h" #include "services/common/test/utils/ohttp_utils.h" +#include "services/common/test/utils/test_init.h" #include "src/encryption/key_fetcher/key_fetcher_utils.h" #include "tools/secure_invoke/flags.h" #include "tools/secure_invoke/payload_generator/payload_packaging.h" @@ -40,6 +41,7 @@ using ::privacy_sandbox::bidding_auction_servers::SelectAdRequest; int main(int argc, char** argv) { absl::ParseCommandLine(argc, argv); + privacy_sandbox::bidding_auction_servers::CommonTestInit(); // Set log level to 0 and to be shown on terminal. std::string input_file = absl::GetFlag(FLAGS_input_file); std::string json_input_str = absl::GetFlag(FLAGS_json_input_str); diff --git a/version.txt b/version.txt index e0af1238..0c89fc92 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -3.11.0 \ No newline at end of file +4.0.0 \ No newline at end of file