Compare trading vs prediction accuracy (#544)

prediction_market_agent_tooling/markets/omen/data_models.py

@@ -24,6 +24,7 @@
     ResolvedBet,
 )
 from prediction_market_agent_tooling.tools.utils import (
+    BPS_CONSTANT,
     DatetimeUTC,
     check_not_none,
     should_not_happen,
@@ -502,7 +503,7 @@ class OmenBet(BaseModel):
     feeAmount: Wei
     outcomeIndex: int
     outcomeTokensTraded: Wei
-    transactionHash: HexAddress
+    transactionHash: HexBytes
     fpmm: OmenMarket
 
     @property
@@ -785,6 +786,14 @@ class ContractPrediction(BaseModel):
     tx_hashes: list[HexBytes] = Field(..., alias="txHashes")
     estimated_probability_bps: int = Field(..., alias="estimatedProbabilityBps")
 
+    @property
+    def estimated_probability(self) -> Probability:
+        return Probability(self.estimated_probability_bps / BPS_CONSTANT)
+
+    @property
+    def boolean_outcome(self) -> bool:
+        return self.estimated_probability > 0.5
+
     @computed_field  # type: ignore[prop-decorator] # Mypy issue: https://github.com/python/mypy/issues/14461
     @property
     def publisher_checksummed(self) -> ChecksumAddress:

prediction_market_agent_tooling/markets/omen/omen.py

@@ -78,6 +78,7 @@
 from prediction_market_agent_tooling.tools.hexbytes_custom import HexBytes
 from prediction_market_agent_tooling.tools.ipfs.ipfs_handler import IPFSHandler
 from prediction_market_agent_tooling.tools.utils import (
+    BPS_CONSTANT,
     DatetimeUTC,
     calculate_sell_amount_in_collateral,
     check_not_none,
@@ -448,7 +449,7 @@ def store_trades(
             publisher=keys.public_key,
             ipfs_hash=ipfs_hash_decoded,
             tx_hashes=tx_hashes,
-            estimated_probability_bps=int(traded_market.answer.p_yes * 10000),
+            estimated_probability_bps=int(traded_market.answer.p_yes * BPS_CONSTANT),
         )
         tx_receipt = OmenAgentResultMappingContract().add_prediction(
             api_keys=keys,

prediction_market_agent_tooling/markets/omen/omen_subgraph_handler.py

@@ -908,3 +908,17 @@ def get_agent_results_for_market(
         if not items:
             return []
         return [ContractPrediction.model_validate(i) for i in items]
+
+    def get_agent_results_for_bet(self, bet: OmenBet) -> ContractPrediction | None:
+        results = [
+            result
+            for result in self.get_agent_results_for_market(bet.fpmm.id)
+            if bet.transactionHash in result.tx_hashes
+        ]
+
+        if not results:
+            return None
+        elif len(results) > 1:
+            raise RuntimeError("Multiple results found for a single bet.")
+
+        return results[0]

prediction_market_agent_tooling/tools/utils.py

@@ -28,6 +28,8 @@
 # For consistent results, also include seed for models that supports it.
 LLM_SEED = 0
 
+BPS_CONSTANT = 10000
+
 
 def check_not_none(
     value: Optional[T],

scripts/compare_trading_vs_prediction_accuracy.py

@@ -0,0 +1,44 @@
+import typer
+from web3 import Web3
+
+from prediction_market_agent_tooling.markets.omen.omen import OmenSubgraphHandler
+
+
+def main(public_key: str) -> None:
+    public_key_checksummed = Web3.to_checksum_address(public_key)
+
+    all_bets = [
+        bet
+        for bet in OmenSubgraphHandler().get_bets(
+            better_address=public_key_checksummed,
+            filter_by_answer_finalized_not_null=True,
+        )
+        if bet.fpmm.is_resolved_with_valid_answer
+    ]
+    correct_bets = [
+        bet for bet in all_bets if bet.boolean_outcome == bet.fpmm.boolean_outcome
+    ]
+
+    all_bets_with_results = []
+    all_bets_without_results = []
+    correct_results = []
+    for bet in all_bets:
+        result = OmenSubgraphHandler().get_agent_results_for_bet(bet)
+        if result is None:
+            all_bets_without_results.append(bet)
+            continue
+        all_bets_with_results.append(bet)
+        if result.boolean_outcome == bet.fpmm.boolean_outcome:
+            correct_results.append(result)
+
+    print("N bets:", len(all_bets))
+    print("Bet accuracy:", len(correct_bets) / len(all_bets) if all_bets else None)
+    print(
+        "Prediction accuracy:",
+        len(correct_results) / len(all_bets_with_results) if all_bets else None,
+    )
+    print("N bets without results:", len(all_bets_without_results))
+
+
+if __name__ == "__main__":
+    typer.run(main)