zcash_client_sqlite: Track all transparent spends.

zcash_client_backend/src/data_api.rs

@@ -566,7 +566,7 @@ pub struct SpendableNotes<NoteRef> {
 
 /// A request for transaction data enhancement, spentness check, or discovery
 /// of spends from a given transparent address within a specific block range.
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum TransactionDataRequest {
     /// Information about the chain's view of a transaction is requested.
     ///

zcash_client_sqlite/src/lib.rs

@@ -52,8 +52,8 @@
         scanning::{ScanPriority, ScanRange},
         Account, AccountBirthday, AccountPurpose, AccountSource, BlockMetadata,
         DecryptedTransaction, InputSource, NullifierQuery, ScannedBlock, SeedRelevance,
-        SentTransaction, SpendableNotes, TransactionDataRequest, WalletCommitmentTrees, WalletRead,
-        WalletSummary, WalletWrite, SAPLING_SHARD_HEIGHT,
+        SentTransaction, SpendableNotes, TransactionDataRequest, TransactionStatus,
+        WalletCommitmentTrees, WalletRead, WalletSummary, WalletWrite, SAPLING_SHARD_HEIGHT,
     },
     keys::{
         AddressGenerationError, UnifiedAddressRequest, UnifiedFullViewingKey, UnifiedSpendingKey,
@@ -63,6 +63,7 @@
     PoolType, ShieldedProtocol, TransferType,
 };
 use zcash_keys::address::Receiver;
+use zcash_keys::encoding::AddressCodec;
 use zcash_primitives::{
     block::BlockHash,
     consensus::{self, BlockHeight},
@@ -85,7 +86,6 @@
 #[cfg(feature = "transparent-inputs")]
 use {
     zcash_client_backend::wallet::TransparentAddressMetadata,
-    zcash_keys::encoding::AddressCodec,
     zcash_primitives::{legacy::TransparentAddress, transaction::components::OutPoint},
 };
 
@@ -1180,6 +1180,10 @@
     ) -> Result<(), Self::Error> {
         self.transactionally(|wdb| {
             let tx_ref = wallet::put_tx_data(wdb.conn.0, d_tx.tx(), None, None, None)?;
+            if let Some(height) = d_tx.mined_height() {
+                wallet::set_transaction_status(wdb.conn.0, d_tx.tx().txid(), TransactionStatus::Mined(height))?;
+            }
+
             let funding_accounts = wallet::get_funding_accounts(wdb.conn.0, d_tx.tx())?;
 
             // TODO(#1305): Correctly track accounts that fund each transaction output.
@@ -1403,6 +1407,13 @@
                     .enumerate()
                 {
                     if let Some(address) = txout.recipient_address() {
+                        debug!(
+                            "{:?} output {} has recipient {}",
+                            d_tx.tx().txid(),
+                            output_index,
+                            address.encode(&wdb.params)
+                        );
+
                         // The transaction is not necessarily mined yet, but we want to record
                         // that an output to the address was seen in this tx anyway. This will
                         // advance the gap regardless of whether it is mined, but an output in
@@ -1417,6 +1428,12 @@
                             &wdb.params,
                             &address
                         )? {
+                            debug!(
+                                "{:?} output {} belongs to account {:?}",
+                                d_tx.tx().txid(),
+                                output_index,
+                                account_id
+                            );
                             put_transparent_output(
                                 wdb.conn.0,
                                 &wdb.params,
@@ -1445,6 +1462,11 @@
                                 tx_ref,
                                 output_index.try_into().unwrap()
                             )?;
+                        } else {
+                            debug!(
+                                "Address {} is not recognized as belonging to any of our accounts.",
+                                address.encode(&wdb.params)
+                            );
                         }
 
                         // If a transaction we observe contains spends from our wallet, we will
@@ -1493,6 +1515,12 @@
                                 queue_status_retrieval = true;
                             }
                         }
+                    } else {
+                        warn!(
+                            "Unable to determine recipient address for tx {:?} output {}",
+                            d_tx.tx().txid(),
+                            output_index
+                        );
                     }
                 }
             }

zcash_client_sqlite/src/wallet.rs

@@ -2613,7 +2613,7 @@ pub(crate) fn put_sent_output<P: consensus::Parameters>(
             :to_address, :to_account_id, :value, :memo)
         ON CONFLICT (tx, output_pool, output_index) DO UPDATE
         SET from_account_id = :from_account_id,
-            to_address = :to_address,
+            to_address = IFNULL(to_address, :to_address),
             to_account_id = IFNULL(to_account_id, :to_account_id),
             value = :value,
             memo = IFNULL(:memo, memo)",

zcash_client_sqlite/src/wallet/db.rs

@@ -365,6 +365,28 @@ CREATE TABLE "transparent_received_output_spends" (
     UNIQUE (transparent_received_output_id, transaction_id)
 )"#;
 
+/// A cache of the relationship between a transaction and the prevout data of its
+/// transparent inputs.
+///
+/// This table is used in out-of-order wallet recovery to cache the information about
+/// what transaction(s) spend each transparent outpoint, so that if an output belonging
+/// to the wallet is detected after the transaction that spends it has been processed,
+/// the spend can also be recorded as part of the process of adding the output to
+/// [`TABLE_TRANSPARENT_RECEIVED_OUTPUTS`].
+pub(super) const TABLE_TRANSPARENT_SPEND_MAP: &str = r#"
+CREATE TABLE transparent_spend_map (
+    spending_transaction_id INTEGER NOT NULL,
+    prevout_txid BLOB NOT NULL,
+    prevout_output_index INTEGER NOT NULL,
+    FOREIGN KEY (spending_transaction_id) REFERENCES transactions(id_tx)
+    -- NOTE: We can't create a unique constraint on just (prevout_txid, prevout_output_index) 
+    -- because the same output may be attempted to be spent in multiple transactions, even 
+    -- though only one will ever be mined.
+    CONSTRAINT transparent_spend_map_unique UNIQUE (
+        spending_transaction_id, prevout_txid, prevout_output_index
+    )
+)"#;
+
 /// Stores the outputs of transactions created by the wallet.
 ///
 /// Unlike with outputs received by the wallet, we store sent outputs for all pools in

zcash_client_sqlite/src/wallet/init.rs

@@ -406,6 +406,7 @@ mod tests {
             db::TABLE_TRANSACTIONS,
             db::TABLE_TRANSPARENT_RECEIVED_OUTPUT_SPENDS,
             db::TABLE_TRANSPARENT_RECEIVED_OUTPUTS,
+            db::TABLE_TRANSPARENT_SPEND_MAP,
             db::TABLE_TRANSPARENT_SPEND_SEARCH_QUEUE,
             db::TABLE_TX_LOCATOR_MAP,
             db::TABLE_TX_RETRIEVAL_QUEUE,

zcash_client_sqlite/src/wallet/init/migrations/tx_retrieval_queue.rs

@@ -48,6 +48,19 @@ impl RusqliteMigration for Migration {
                 output_index INTEGER NOT NULL,
                 FOREIGN KEY (transaction_id) REFERENCES transactions(id_tx),
                 CONSTRAINT value_received_height UNIQUE (transaction_id, output_index)
+            );
+
+            CREATE TABLE transparent_spend_map (
+                spending_transaction_id INTEGER NOT NULL,
+                prevout_txid BLOB NOT NULL,
+                prevout_output_index INTEGER NOT NULL,
+                FOREIGN KEY (spending_transaction_id) REFERENCES transactions(id_tx)
+                -- NOTE: We can't create a unique constraint on just (prevout_txid, prevout_output_index) 
+                -- because the same output may be attempted to be spent in multiple transactions, even 
+                -- though only one will ever be mined.
+                CONSTRAINT transparent_spend_map_unique UNIQUE (
+                    spending_transaction_id, prevout_txid, prevout_output_index
+                )
             );",
         )?;
 
@@ -67,7 +80,8 @@ impl RusqliteMigration for Migration {
 
     fn down(&self, transaction: &Transaction) -> Result<(), WalletMigrationError> {
         transaction.execute_batch(
-            "DROP TABLE transparent_spend_search_queue;
+            "DROP TABLE transparent_spend_map;
+             DROP TABLE transparent_spend_search_queue;
              ALTER TABLE transactions DROP COLUMN target_height;
              DROP TABLE tx_retrieval_queue;",
         )?;

zcash_client_sqlite/src/wallet/transparent.rs

@@ -427,25 +427,31 @@
 }
 
 /// Marks the given UTXO as having been spent.
+///
+/// Returns `true` if the UTXO was known to the wallet.
 pub(crate) fn mark_transparent_utxo_spent(
     conn: &rusqlite::Connection,
-    tx_ref: TxRef,
+    spent_in_tx: TxRef,
     outpoint: &OutPoint,
-) -> Result<(), SqliteClientError> {
+) -> Result<bool, SqliteClientError> {
+    let spend_params = named_params![
+        ":spent_in_tx": spent_in_tx.0,
+        ":prevout_txid": outpoint.hash().as_ref(),
+        ":prevout_idx": outpoint.n(),
+    ];
     let mut stmt_mark_transparent_utxo_spent = conn.prepare_cached(
         "INSERT INTO transparent_received_output_spends (transparent_received_output_id, transaction_id)
          SELECT txo.id, :spent_in_tx
          FROM transparent_received_outputs txo
          JOIN transactions t ON t.id_tx = txo.transaction_id
          WHERE t.txid = :prevout_txid
          AND txo.output_index = :prevout_idx
-         ON CONFLICT (transparent_received_output_id, transaction_id) DO NOTHING",
+         ON CONFLICT (transparent_received_output_id, transaction_id)
+         -- The following UPDATE is effectively a no-op, but we perform it anyway so that the
+         -- number of affected rows can be used to determine whether a record existed.
+         DO UPDATE SET transaction_id = :spent_in_tx",
     )?;
-    stmt_mark_transparent_utxo_spent.execute(named_params![
-        ":spent_in_tx": tx_ref.0,
-        ":prevout_txid": outpoint.hash().as_ref(),
-        ":prevout_idx": outpoint.n(),
-    ])?;
+    let affected_rows = stmt_mark_transparent_utxo_spent.execute(spend_params)?;
 
     // Since we know that the output is spent, we no longer need to search for
     // it to find out if it has been spent.
@@ -461,7 +467,24 @@
         ":prevout_idx": outpoint.n(),
     ])?;
 
-    Ok(())
+    // If no rows were affected, we know that we don't actually have the output in
+    // `transparent_received_outputs` yet, so we have to record the output as spent
+    // so that when we eventually detect the output, we can create the spend record.
+    if affected_rows == 0 {
+        conn.execute(
+            "INSERT INTO transparent_spend_map (
+                spending_transaction_id,
+                prevout_txid,
+                prevout_output_index
+            )
+            VALUES (:spent_in_tx, :prevout_txid, :prevout_idx)
+            ON CONFLICT (spending_transaction_id, prevout_txid, prevout_output_index)
+            DO NOTHING",
+            spend_params,
+        )?;
+    }
+
+    Ok(affected_rows > 0)
 }
 
 /// Adds the given received UTXO to the datastore.
@@ -728,6 +751,29 @@
 
     let utxo_id = stmt_upsert_transparent_output
         .query_row(sql_args, |row| row.get::<_, i64>(0).map(UtxoId))?;
+
+    // If we have a record of the output already having been spent, then mark it as spent using the
+    // stored reference to the spending transaction.
+    let spending_tx_ref = conn
+        .query_row(
+            "SELECT ts.spending_transaction_id
+             FROM transparent_spend_map ts
+             JOIN transactions t ON t.id_tx = ts.spending_transaction_id
+             WHERE ts.prevout_txid = :prevout_txid
+             AND ts.prevout_output_index = :prevout_idx
+             ORDER BY t.block NULLS LAST LIMIT 1",
+            named_params![
+                ":prevout_txid": outpoint.txid().as_ref(),
+                ":prevout_idx": outpoint.n()
+            ],
+            |row| row.get::<_, i64>(0).map(TxRef),
+        )
+        .optional()?;
+
+    if let Some(spending_transaction_id) = spending_tx_ref {
+        mark_transparent_utxo_spent(conn, spending_transaction_id, outpoint)?;
+    }
+
     Ok(utxo_id)
 }