From 128919af58ed6796ef3fa1b1d7c297a89690133a Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Tue, 24 Sep 2024 10:49:09 +0200 Subject: [PATCH] [Mirror] Fix network fees (#111) This PR mirrors the implementation of https://github.com/cowprotocol/solver-rewards/pull/400 It should be adapted when the original implementation if changed. It needs to be merged, hotfixed into prod, and data for the accounting week starting on 2024-09-17 needs to be resynced. Otherwise the dashboard for the next payment will be out of sync with solver-rewards. --- src/sql/orderbook/barn_batch_rewards.sql | 142 +++++++++++++++------- src/sql/orderbook/prod_batch_rewards.sql | 142 +++++++++++++++------- src/sync/order_rewards.py | 4 +- tests/integration/test_fetch_orderbook.py | 10 +- 4 files changed, 199 insertions(+), 99 deletions(-) diff --git a/src/sql/orderbook/barn_batch_rewards.sql b/src/sql/orderbook/barn_batch_rewards.sql index 817d421..6d34950 100644 --- a/src/sql/orderbook/barn_batch_rewards.sql +++ b/src/sql/orderbook/barn_batch_rewards.sql @@ -7,7 +7,6 @@ WITH observed_settlements AS ( -- settlement_observations effective_gas_price * gas_used AS execution_cost, surplus, - fee, s.auction_id FROM settlement_observations so @@ -35,30 +34,52 @@ auction_participation as ( GROUP BY ss.auction_id ), --- protocol fees: +-- order data +order_data AS ( + SELECT + uid, + sell_token, + buy_token, + sell_amount, + buy_amount, + kind, + app_data + FROM orders + UNION ALL + SELECT + uid, + sell_token, + buy_token, + sell_amount, + buy_amount, + kind, + app_data + FROM jit_orders +), +-- additional trade data order_surplus AS ( SELECT ss.winner as solver, s.auction_id, s.tx_hash, t.order_uid, - o.sell_token, - o.buy_token, + od.sell_token, + od.buy_token, t.sell_amount, -- the total amount the user sends t.buy_amount, -- the total amount the user receives oe.surplus_fee as observed_fee, -- the total discrepancy between what the user sends and what they would have send if they traded at clearing price - o.kind, + od.kind, CASE - WHEN o.kind = 'sell' THEN t.buy_amount - t.sell_amount * o.buy_amount / (o.sell_amount + o.fee_amount) - WHEN o.kind = 'buy' THEN t.buy_amount * (o.sell_amount + o.fee_amount) / o.buy_amount - t.sell_amount + WHEN od.kind = 'sell' THEN t.buy_amount - t.sell_amount * od.buy_amount / od.sell_amount + WHEN od.kind = 'buy' THEN t.buy_amount * od.sell_amount / od.buy_amount - t.sell_amount END AS surplus, CASE - WHEN o.kind = 'sell' THEN t.buy_amount - t.sell_amount * (oq.buy_amount - oq.buy_amount / oq.sell_amount * oq.gas_amount * oq.gas_price / oq.sell_token_price) / oq.sell_amount - WHEN o.kind = 'buy' THEN t.buy_amount * (oq.sell_amount + oq.gas_amount * oq.gas_price / oq.sell_token_price) / oq.buy_amount - t.sell_amount + WHEN od.kind = 'sell' THEN t.buy_amount - t.sell_amount * (oq.buy_amount - oq.buy_amount / oq.sell_amount * oq.gas_amount * oq.gas_price / oq.sell_token_price) / oq.sell_amount + WHEN od.kind = 'buy' THEN t.buy_amount * (oq.sell_amount + oq.gas_amount * oq.gas_price / oq.sell_token_price) / oq.buy_amount - t.sell_amount END AS price_improvement, CASE - WHEN o.kind = 'sell' THEN o.buy_token - WHEN o.kind = 'buy' THEN o.sell_token + WHEN od.kind = 'sell' THEN od.buy_token + WHEN od.kind = 'buy' THEN od.sell_token END AS surplus_token, ad.full_app_data as app_data FROM @@ -67,19 +88,20 @@ order_surplus AS ( ON s.auction_id = ss.auction_id JOIN trades t -- contains traded amounts ON s.block_number = t.block_number -- log_index cannot be checked, does not work correctly with multiple auctions on the same block - JOIN orders o -- contains tokens and limit amounts - ON t.order_uid = o.uid + JOIN order_data od -- contains tokens and limit amounts + ON t.order_uid = od.uid JOIN order_execution oe -- contains surplus fee ON t.order_uid = oe.order_uid AND s.auction_id = oe.auction_id LEFT OUTER JOIN order_quotes oq -- contains quote amounts - ON o.uid = oq.order_uid + ON od.uid = oq.order_uid LEFT OUTER JOIN app_data ad -- contains full app data - on o.app_data = ad.contract_app_data + on od.app_data = ad.contract_app_data WHERE ss.block_deadline > {{start_block}} AND ss.block_deadline <= {{end_block}} ), +-- protocol fees: fee_policies_first_proxy as ( select auction_id, @@ -211,7 +233,7 @@ order_surplus_intermediate as ( partner_fee_recipient from order_protocol_fee_first ), -order_protocol_fee as ( +order_protocol_fee as materialized ( SELECT os.auction_id, os.solver, @@ -287,13 +309,33 @@ order_protocol_fee as ( ON os.auction_id = fp.auction_id AND os.order_uid = fp.order_uid ), -order_protocol_fee_prices AS ( +price_data AS ( SELECT - opf.auction_id, - opf.solver, - opf.tx_hash, - opf.order_uid, - opf.surplus, + os.auction_id, + os.order_uid, + ap_surplus.price / pow(10, 18) as surplus_token_native_price, + ap_protocol.price / pow(10, 18) as protocol_fee_token_native_price, + ap_sell.price / pow(10, 18) as network_fee_token_native_price + FROM + order_surplus AS os + LEFT OUTER JOIN auction_prices ap_sell -- contains price: sell token + ON os.auction_id = ap_sell.auction_id + AND os.sell_token = ap_sell.token + LEFT OUTER JOIN auction_prices ap_surplus -- contains price: surplus token + ON os.auction_id = ap_surplus.auction_id + AND os.surplus_token = ap_surplus.token + LEFT OUTER JOIN auction_prices ap_protocol -- contains price: protocol fee token + ON os.auction_id = ap_protocol.auction_id + AND os.surplus_token = ap_protocol.token +), +combined_order_data AS ( + SELECT + os.auction_id, + os.solver, + os.tx_hash, + os.order_uid, + os.surplus, + os.surplus_token, opf.protocol_fee, opf.protocol_fee_token, CASE @@ -303,33 +345,40 @@ order_protocol_fee_prices AS ( opf.partner_fee, opf.partner_fee_recipient, CASE - WHEN opf.sell_token != opf.protocol_fee_token THEN (opf.sell_amount - opf.observed_fee) / opf.buy_amount * opf.protocol_fee - ELSE opf.protocol_fee - END AS network_fee_correction, - opf.sell_token as network_fee_token, - ap_surplus.price / pow(10, 18) as surplus_token_native_price, - ap_protocol.price / pow(10, 18) as protocol_fee_token_native_price, - ap_sell.price / pow(10, 18) as network_fee_token_native_price + WHEN os.sell_token != os.surplus_token THEN os.observed_fee - (os.sell_amount - os.observed_fee) / os.buy_amount * coalesce(opf.protocol_fee, 0) + ELSE os.observed_fee - coalesce(opf.protocol_fee, 0) + END AS network_fee, + os.sell_token as network_fee_token, + surplus_token_native_price, + protocol_fee_token_native_price, + network_fee_token_native_price FROM - order_protocol_fee as opf - JOIN auction_prices ap_sell -- contains price: sell token - ON opf.auction_id = ap_sell.auction_id - AND opf.sell_token = ap_sell.token - JOIN auction_prices ap_surplus -- contains price: surplus token - ON opf.auction_id = ap_surplus.auction_id - AND opf.surplus_token = ap_surplus.token - JOIN auction_prices ap_protocol -- contains price: protocol fee token - ON opf.auction_id = ap_protocol.auction_id - AND opf.protocol_fee_token = ap_protocol.token + order_surplus AS os + LEFT OUTER JOIN order_protocol_fee as opf + ON os.auction_id = opf.auction_id + AND os.order_uid = opf.order_uid + JOIN price_data pd + ON os.auction_id = pd.auction_id + AND os.order_uid = pd.order_uid ), batch_protocol_fees AS ( SELECT solver, tx_hash, - sum(protocol_fee * protocol_fee_token_native_price) as protocol_fee, - sum(network_fee_correction * network_fee_token_native_price) as network_fee_correction + sum(protocol_fee * protocol_fee_token_native_price) as protocol_fee + FROM + combined_order_data + group by + solver, + tx_hash +), +batch_network_fees AS ( + SELECT + solver, + tx_hash, + sum(network_fee * network_fee_token_native_price) as network_fee FROM - order_protocol_fee_prices + combined_order_data group by solver, tx_hash @@ -346,7 +395,6 @@ reward_data AS ( block_deadline, coalesce(execution_cost, 0) as execution_cost, coalesce(surplus, 0) as surplus, - coalesce(fee, 0) as fee, -- scores winning_score, case @@ -360,9 +408,9 @@ reward_data AS ( -- protocol_fees coalesce(cast(protocol_fee as numeric(78, 0)), 0) as protocol_fee, coalesce( - cast(network_fee_correction as numeric(78, 0)), + cast(network_fee as numeric(78, 0)), 0 - ) as network_fee_correction + ) as network_fee FROM settlement_scores ss -- If there are reported scores, @@ -371,6 +419,7 @@ reward_data AS ( -- outer joins made in order to capture non-existent settlements. LEFT OUTER JOIN observed_settlements os ON os.auction_id = ss.auction_id LEFT OUTER JOIN batch_protocol_fees bpf ON bpf.tx_hash = os.tx_hash + LEFT OUTER JOIN batch_network_fees bnf ON bnf.tx_hash = os.tx_hash ), reward_per_auction as ( SELECT @@ -382,7 +431,7 @@ reward_per_auction as ( execution_cost, surplus, protocol_fee, -- the protocol fee - fee - network_fee_correction as network_fee, -- the network fee + network_fee, -- the network fee observed_score - reference_score as uncapped_payment, -- Capped Reward = CLAMP_[-E, E + exec_cost](uncapped_reward_eth) LEAST( @@ -417,3 +466,4 @@ SELECT participating_solvers FROM reward_per_auction +ORDER BY block_deadline diff --git a/src/sql/orderbook/prod_batch_rewards.sql b/src/sql/orderbook/prod_batch_rewards.sql index 817d421..6d34950 100644 --- a/src/sql/orderbook/prod_batch_rewards.sql +++ b/src/sql/orderbook/prod_batch_rewards.sql @@ -7,7 +7,6 @@ WITH observed_settlements AS ( -- settlement_observations effective_gas_price * gas_used AS execution_cost, surplus, - fee, s.auction_id FROM settlement_observations so @@ -35,30 +34,52 @@ auction_participation as ( GROUP BY ss.auction_id ), --- protocol fees: +-- order data +order_data AS ( + SELECT + uid, + sell_token, + buy_token, + sell_amount, + buy_amount, + kind, + app_data + FROM orders + UNION ALL + SELECT + uid, + sell_token, + buy_token, + sell_amount, + buy_amount, + kind, + app_data + FROM jit_orders +), +-- additional trade data order_surplus AS ( SELECT ss.winner as solver, s.auction_id, s.tx_hash, t.order_uid, - o.sell_token, - o.buy_token, + od.sell_token, + od.buy_token, t.sell_amount, -- the total amount the user sends t.buy_amount, -- the total amount the user receives oe.surplus_fee as observed_fee, -- the total discrepancy between what the user sends and what they would have send if they traded at clearing price - o.kind, + od.kind, CASE - WHEN o.kind = 'sell' THEN t.buy_amount - t.sell_amount * o.buy_amount / (o.sell_amount + o.fee_amount) - WHEN o.kind = 'buy' THEN t.buy_amount * (o.sell_amount + o.fee_amount) / o.buy_amount - t.sell_amount + WHEN od.kind = 'sell' THEN t.buy_amount - t.sell_amount * od.buy_amount / od.sell_amount + WHEN od.kind = 'buy' THEN t.buy_amount * od.sell_amount / od.buy_amount - t.sell_amount END AS surplus, CASE - WHEN o.kind = 'sell' THEN t.buy_amount - t.sell_amount * (oq.buy_amount - oq.buy_amount / oq.sell_amount * oq.gas_amount * oq.gas_price / oq.sell_token_price) / oq.sell_amount - WHEN o.kind = 'buy' THEN t.buy_amount * (oq.sell_amount + oq.gas_amount * oq.gas_price / oq.sell_token_price) / oq.buy_amount - t.sell_amount + WHEN od.kind = 'sell' THEN t.buy_amount - t.sell_amount * (oq.buy_amount - oq.buy_amount / oq.sell_amount * oq.gas_amount * oq.gas_price / oq.sell_token_price) / oq.sell_amount + WHEN od.kind = 'buy' THEN t.buy_amount * (oq.sell_amount + oq.gas_amount * oq.gas_price / oq.sell_token_price) / oq.buy_amount - t.sell_amount END AS price_improvement, CASE - WHEN o.kind = 'sell' THEN o.buy_token - WHEN o.kind = 'buy' THEN o.sell_token + WHEN od.kind = 'sell' THEN od.buy_token + WHEN od.kind = 'buy' THEN od.sell_token END AS surplus_token, ad.full_app_data as app_data FROM @@ -67,19 +88,20 @@ order_surplus AS ( ON s.auction_id = ss.auction_id JOIN trades t -- contains traded amounts ON s.block_number = t.block_number -- log_index cannot be checked, does not work correctly with multiple auctions on the same block - JOIN orders o -- contains tokens and limit amounts - ON t.order_uid = o.uid + JOIN order_data od -- contains tokens and limit amounts + ON t.order_uid = od.uid JOIN order_execution oe -- contains surplus fee ON t.order_uid = oe.order_uid AND s.auction_id = oe.auction_id LEFT OUTER JOIN order_quotes oq -- contains quote amounts - ON o.uid = oq.order_uid + ON od.uid = oq.order_uid LEFT OUTER JOIN app_data ad -- contains full app data - on o.app_data = ad.contract_app_data + on od.app_data = ad.contract_app_data WHERE ss.block_deadline > {{start_block}} AND ss.block_deadline <= {{end_block}} ), +-- protocol fees: fee_policies_first_proxy as ( select auction_id, @@ -211,7 +233,7 @@ order_surplus_intermediate as ( partner_fee_recipient from order_protocol_fee_first ), -order_protocol_fee as ( +order_protocol_fee as materialized ( SELECT os.auction_id, os.solver, @@ -287,13 +309,33 @@ order_protocol_fee as ( ON os.auction_id = fp.auction_id AND os.order_uid = fp.order_uid ), -order_protocol_fee_prices AS ( +price_data AS ( SELECT - opf.auction_id, - opf.solver, - opf.tx_hash, - opf.order_uid, - opf.surplus, + os.auction_id, + os.order_uid, + ap_surplus.price / pow(10, 18) as surplus_token_native_price, + ap_protocol.price / pow(10, 18) as protocol_fee_token_native_price, + ap_sell.price / pow(10, 18) as network_fee_token_native_price + FROM + order_surplus AS os + LEFT OUTER JOIN auction_prices ap_sell -- contains price: sell token + ON os.auction_id = ap_sell.auction_id + AND os.sell_token = ap_sell.token + LEFT OUTER JOIN auction_prices ap_surplus -- contains price: surplus token + ON os.auction_id = ap_surplus.auction_id + AND os.surplus_token = ap_surplus.token + LEFT OUTER JOIN auction_prices ap_protocol -- contains price: protocol fee token + ON os.auction_id = ap_protocol.auction_id + AND os.surplus_token = ap_protocol.token +), +combined_order_data AS ( + SELECT + os.auction_id, + os.solver, + os.tx_hash, + os.order_uid, + os.surplus, + os.surplus_token, opf.protocol_fee, opf.protocol_fee_token, CASE @@ -303,33 +345,40 @@ order_protocol_fee_prices AS ( opf.partner_fee, opf.partner_fee_recipient, CASE - WHEN opf.sell_token != opf.protocol_fee_token THEN (opf.sell_amount - opf.observed_fee) / opf.buy_amount * opf.protocol_fee - ELSE opf.protocol_fee - END AS network_fee_correction, - opf.sell_token as network_fee_token, - ap_surplus.price / pow(10, 18) as surplus_token_native_price, - ap_protocol.price / pow(10, 18) as protocol_fee_token_native_price, - ap_sell.price / pow(10, 18) as network_fee_token_native_price + WHEN os.sell_token != os.surplus_token THEN os.observed_fee - (os.sell_amount - os.observed_fee) / os.buy_amount * coalesce(opf.protocol_fee, 0) + ELSE os.observed_fee - coalesce(opf.protocol_fee, 0) + END AS network_fee, + os.sell_token as network_fee_token, + surplus_token_native_price, + protocol_fee_token_native_price, + network_fee_token_native_price FROM - order_protocol_fee as opf - JOIN auction_prices ap_sell -- contains price: sell token - ON opf.auction_id = ap_sell.auction_id - AND opf.sell_token = ap_sell.token - JOIN auction_prices ap_surplus -- contains price: surplus token - ON opf.auction_id = ap_surplus.auction_id - AND opf.surplus_token = ap_surplus.token - JOIN auction_prices ap_protocol -- contains price: protocol fee token - ON opf.auction_id = ap_protocol.auction_id - AND opf.protocol_fee_token = ap_protocol.token + order_surplus AS os + LEFT OUTER JOIN order_protocol_fee as opf + ON os.auction_id = opf.auction_id + AND os.order_uid = opf.order_uid + JOIN price_data pd + ON os.auction_id = pd.auction_id + AND os.order_uid = pd.order_uid ), batch_protocol_fees AS ( SELECT solver, tx_hash, - sum(protocol_fee * protocol_fee_token_native_price) as protocol_fee, - sum(network_fee_correction * network_fee_token_native_price) as network_fee_correction + sum(protocol_fee * protocol_fee_token_native_price) as protocol_fee + FROM + combined_order_data + group by + solver, + tx_hash +), +batch_network_fees AS ( + SELECT + solver, + tx_hash, + sum(network_fee * network_fee_token_native_price) as network_fee FROM - order_protocol_fee_prices + combined_order_data group by solver, tx_hash @@ -346,7 +395,6 @@ reward_data AS ( block_deadline, coalesce(execution_cost, 0) as execution_cost, coalesce(surplus, 0) as surplus, - coalesce(fee, 0) as fee, -- scores winning_score, case @@ -360,9 +408,9 @@ reward_data AS ( -- protocol_fees coalesce(cast(protocol_fee as numeric(78, 0)), 0) as protocol_fee, coalesce( - cast(network_fee_correction as numeric(78, 0)), + cast(network_fee as numeric(78, 0)), 0 - ) as network_fee_correction + ) as network_fee FROM settlement_scores ss -- If there are reported scores, @@ -371,6 +419,7 @@ reward_data AS ( -- outer joins made in order to capture non-existent settlements. LEFT OUTER JOIN observed_settlements os ON os.auction_id = ss.auction_id LEFT OUTER JOIN batch_protocol_fees bpf ON bpf.tx_hash = os.tx_hash + LEFT OUTER JOIN batch_network_fees bnf ON bnf.tx_hash = os.tx_hash ), reward_per_auction as ( SELECT @@ -382,7 +431,7 @@ reward_per_auction as ( execution_cost, surplus, protocol_fee, -- the protocol fee - fee - network_fee_correction as network_fee, -- the network fee + network_fee, -- the network fee observed_score - reference_score as uncapped_payment, -- Capped Reward = CLAMP_[-E, E + exec_cost](uncapped_reward_eth) LEAST( @@ -417,3 +466,4 @@ SELECT participating_solvers FROM reward_per_auction +ORDER BY block_deadline diff --git a/src/sync/order_rewards.py b/src/sync/order_rewards.py index 5c2e259..5c9a30e 100644 --- a/src/sync/order_rewards.py +++ b/src/sync/order_rewards.py @@ -20,7 +20,7 @@ class OrderbookDataHandler( RecordHandler -): # pylint:disable=too-few-public-methods,too-many-arguments +): # pylint:disable=too-few-public-methods,too-many-arguments,too-many-positional-arguments """ This class is responsible for consuming new dune records and missing values from previous runs it attempts to fetch content for them and filters them into "found" and "not found" as necessary @@ -103,7 +103,7 @@ def sync_batch_rewards( ) -def sync_orderbook_data( # pylint:disable=too-many-arguments +def sync_orderbook_data( # pylint:disable=too-many-arguments,too-many-positional-arguments aws: AWSClient, block_range: BlockRange, config: SyncConfig, diff --git a/tests/integration/test_fetch_orderbook.py b/tests/integration/test_fetch_orderbook.py index 6475dd5..928abf0 100644 --- a/tests/integration/test_fetch_orderbook.py +++ b/tests/integration/test_fetch_orderbook.py @@ -243,13 +243,13 @@ def test_get_batch_rewards(self): "0", ], "network_fee": [ - "7715157183718636", - "18639716210784841", + "7715157183718640", + "18639716210784800", "0", "0", - "15011842152833827", - "21801311054855324", - "8912931423922369", + "15011842152833800", + "21801311054855300", + "8912931423922370", ], "uncapped_payment_eth": [ "405658366132340",