indexer: optimize idx.VoteList, etc performance

vocdoni · Sep 12, 2024 · df37077 · df37077
1 parent 43b6518
commit df37077
Show file tree

Hide file tree

Showing 5 changed files with 106 additions and 62 deletions.
diff --git a/vochain/indexer/queries/account.sql b/vochain/indexer/queries/account.sql
@@ -4,23 +4,28 @@ REPLACE INTO accounts (
 ) VALUES (?, ?, ?);
 
 -- name: SearchAccounts :many
-WITH results AS (
-  SELECT *
-  FROM accounts
-  WHERE (
-    (
-    sqlc.arg(account_id_substr) = ''
-    OR (LENGTH(sqlc.arg(account_id_substr)) = 40 AND LOWER(HEX(account)) = LOWER(sqlc.arg(account_id_substr)))
-    OR (LENGTH(sqlc.arg(account_id_substr)) < 40 AND INSTR(LOWER(HEX(account)), LOWER(sqlc.arg(account_id_substr))) > 0)
-    -- TODO: consider keeping an account_hex column for faster searches
-    )
+SELECT *
+FROM accounts
+WHERE (
+  (
+  sqlc.arg(account_id_substr) = ''
+  OR (LENGTH(sqlc.arg(account_id_substr)) = 40 AND LOWER(HEX(account)) = LOWER(sqlc.arg(account_id_substr)))
+  OR (LENGTH(sqlc.arg(account_id_substr)) < 40 AND INSTR(LOWER(HEX(account)), LOWER(sqlc.arg(account_id_substr))) > 0)
+  -- TODO: consider keeping an account_hex column for faster searches
   )
 )
-SELECT *, COUNT(*) OVER() AS total_count
-FROM results
 ORDER BY balance DESC
 LIMIT sqlc.arg(limit)
 OFFSET sqlc.arg(offset);
 
 -- name: CountAccounts :one
-SELECT COUNT(*) FROM accounts;
+SELECT COUNT(*)
+FROM accounts;
+WHERE (
+  (
+  sqlc.arg(account_id_substr) = ''
+  OR (LENGTH(sqlc.arg(account_id_substr)) = 40 AND LOWER(HEX(account)) = LOWER(sqlc.arg(account_id_substr)))
+  OR (LENGTH(sqlc.arg(account_id_substr)) < 40 AND INSTR(LOWER(HEX(account)), LOWER(sqlc.arg(account_id_substr))) > 0)
+  -- TODO: consider keeping an account_hex column for faster searches
+  )
+);
diff --git a/vochain/indexer/queries/token_fees.sql b/vochain/indexer/queries/token_fees.sql
@@ -7,18 +7,23 @@ INSERT INTO token_fees (
 	?, ?, ?
 );
 
+-- name: CountTokenFees :one
+SELECT COUNT(*)
+FROM token_fees
+WHERE (
+  (sqlc.arg(from_account) = '' OR LOWER(HEX(from_account)) = LOWER(sqlc.arg(from_account)))
+  AND (sqlc.arg(tx_type) = '' OR LOWER(tx_type) = LOWER(sqlc.arg(tx_type)))
+  AND (sqlc.arg(reference) = '' OR LOWER(reference) = LOWER(sqlc.arg(reference)))
+);
+
 -- name: SearchTokenFees :many
-WITH results AS (
-  SELECT *
-  FROM token_fees
-  WHERE (
-    (sqlc.arg(from_account) = '' OR LOWER(HEX(from_account)) = LOWER(sqlc.arg(from_account)))
-    AND (sqlc.arg(tx_type) = '' OR LOWER(tx_type) = LOWER(sqlc.arg(tx_type)))
-    AND (sqlc.arg(reference) = '' OR LOWER(reference) = LOWER(sqlc.arg(reference)))
-  )
+SELECT *
+FROM token_fees
+WHERE (
+  (sqlc.arg(from_account) = '' OR LOWER(HEX(from_account)) = LOWER(sqlc.arg(from_account)))
+  AND (sqlc.arg(tx_type) = '' OR LOWER(tx_type) = LOWER(sqlc.arg(tx_type)))
+  AND (sqlc.arg(reference) = '' OR LOWER(reference) = LOWER(sqlc.arg(reference)))
 )
-SELECT *, COUNT(*) OVER() AS total_count
-FROM results
 ORDER BY spend_time DESC
 LIMIT sqlc.arg(limit)
 OFFSET sqlc.arg(offset);
diff --git a/vochain/indexer/queries/token_transfers.sql b/vochain/indexer/queries/token_transfers.sql
@@ -12,21 +12,29 @@ SELECT * FROM token_transfers
 WHERE tx_hash = ?
 LIMIT 1;
 
+-- name: CountTokenTransfers :one
+SELECT COUNT(*)
+FROM token_transfers
+WHERE (
+	(sqlc.arg(from_or_to_account) = ''
+		OR LOWER(HEX(from_account)) = LOWER(sqlc.arg(from_or_to_account))
+		OR LOWER(HEX(to_account)) = LOWER(sqlc.arg(from_or_to_account))
+	)
+	AND (sqlc.arg(from_account) = '' OR LOWER(HEX(from_account)) = LOWER(sqlc.arg(from_account)))
+	AND (sqlc.arg(to_account) = '' OR LOWER(HEX(to_account)) = LOWER(sqlc.arg(to_account)))
+);
+
 -- name: SearchTokenTransfers :many
-WITH results AS (
-  SELECT *
-  FROM token_transfers
-  WHERE (
-    (sqlc.arg(from_or_to_account) = '' OR (
-		LOWER(HEX(from_account)) = LOWER(sqlc.arg(from_or_to_account))
+SELECT *
+FROM token_transfers
+WHERE (
+	(sqlc.arg(from_or_to_account) = ''
+		OR LOWER(HEX(from_account)) = LOWER(sqlc.arg(from_or_to_account))
 		OR LOWER(HEX(to_account)) = LOWER(sqlc.arg(from_or_to_account))
-	))
-    AND (sqlc.arg(from_account) = '' OR LOWER(HEX(from_account)) = LOWER(sqlc.arg(from_account)))
-    AND (sqlc.arg(to_account) = '' OR LOWER(HEX(to_account)) = LOWER(sqlc.arg(to_account)))
-  )
+	)
+	AND (sqlc.arg(from_account) = '' OR LOWER(HEX(from_account)) = LOWER(sqlc.arg(from_account)))
+	AND (sqlc.arg(to_account) = '' OR LOWER(HEX(to_account)) = LOWER(sqlc.arg(to_account)))
 )
-SELECT *, COUNT(*) OVER() AS total_count
-FROM results
 ORDER BY transfer_time DESC
 LIMIT sqlc.arg(limit)
 OFFSET sqlc.arg(offset);

diff --git a/vochain/indexer/queries/transactions.sql b/vochain/indexer/queries/transactions.sql
@@ -20,7 +20,20 @@ WHERE hash = ?
 LIMIT 1;
 
 -- name: CountTransactions :one
-SELECT COUNT(*) FROM transactions;
+SELECT COUNT(*)
+FROM transactions
+WHERE (
+  (sqlc.arg(block_height) = 0 OR block_height = sqlc.arg(block_height))
+  AND (sqlc.arg(tx_type) = '' OR LOWER(type) = LOWER(sqlc.arg(tx_type)))
+  AND (sqlc.arg(tx_subtype) = '' OR LOWER(subtype) = LOWER(sqlc.arg(tx_subtype)))
+  AND (sqlc.arg(tx_signer) = '' OR LOWER(HEX(signer)) = LOWER(sqlc.arg(tx_signer)))
+  AND (
+    sqlc.arg(hash_substr) = ''
+    OR (LENGTH(sqlc.arg(hash_substr)) = 64 AND LOWER(HEX(hash)) = LOWER(sqlc.arg(hash_substr)))
+    OR (LENGTH(sqlc.arg(hash_substr)) < 64 AND INSTR(LOWER(HEX(hash)), LOWER(sqlc.arg(hash_substr))) > 0)
+    -- TODO: consider keeping an hash_hex column for faster searches
+  )
+);
 
 -- name: CountTransactionsByHeight :one
 SELECT COUNT(*) FROM transactions
@@ -32,7 +45,7 @@ WHERE block_height = ? AND block_index = ?
 LIMIT 1;
 
 -- name: SearchTransactions :many
-SELECT *, COUNT(*) OVER() AS total_count
+SELECT *
 FROM transactions
 WHERE
   (sqlc.arg(block_height) = 0 OR block_height = sqlc.arg(block_height))

diff --git a/vochain/indexer/queries/votes.sql b/vochain/indexer/queries/votes.sql
@@ -20,34 +20,47 @@ WHERE v.nullifier = ?
 LIMIT 1;
 
 -- name: CountVotes :one
-SELECT COUNT(*) FROM votes;
+SELECT COUNT(*)
+FROM votes
+WHERE (
+	LENGTH(sqlc.arg(process_id_substr)) <= 64 -- if passed arg is longer, then just abort the query
+	AND (
+		sqlc.arg(process_id_substr) = ''
+		OR (LENGTH(sqlc.arg(process_id_substr)) = 64 AND LOWER(HEX(process_id)) = LOWER(sqlc.arg(process_id_substr)))
+		OR (LENGTH(sqlc.arg(process_id_substr)) < 64 AND INSTR(LOWER(HEX(process_id)), LOWER(sqlc.arg(process_id_substr))) > 0)
+		-- TODO: consider keeping an process_id_hex column for faster searches
+	)
+	AND LENGTH(sqlc.arg(nullifier_substr)) <= 64 -- if passed arg is longer, then just abort the query
+	AND (
+		sqlc.arg(nullifier_substr) = ''
+		OR (LENGTH(sqlc.arg(nullifier_substr)) = 64 AND LOWER(HEX(nullifier)) = LOWER(sqlc.arg(nullifier_substr)))
+		OR (LENGTH(sqlc.arg(nullifier_substr)) < 64 AND INSTR(LOWER(HEX(nullifier)), LOWER(sqlc.arg(nullifier_substr))) > 0)
+		-- TODO: consider keeping an nullifier_hex column for faster searches
+	)
+);
 
 -- name: SearchVotes :many
-WITH results AS (
-	SELECT v.*, t.hash
-	FROM votes AS v
-	LEFT JOIN transactions AS t
-		ON v.block_height = t.block_height
-		AND v.block_index = t.block_index
-	WHERE (
-		LENGTH(sqlc.arg(process_id_substr)) <= 64 -- if passed arg is longer, then just abort the query
-		AND (
-			sqlc.arg(process_id_substr) = ''
-			OR (LENGTH(sqlc.arg(process_id_substr)) = 64 AND LOWER(HEX(process_id)) = LOWER(sqlc.arg(process_id_substr)))
-			OR (LENGTH(sqlc.arg(process_id_substr)) < 64 AND INSTR(LOWER(HEX(process_id)), LOWER(sqlc.arg(process_id_substr))) > 0)
-			-- TODO: consider keeping an process_id_hex column for faster searches
-		)
-		AND LENGTH(sqlc.arg(nullifier_substr)) <= 64 -- if passed arg is longer, then just abort the query
-		AND (
-			sqlc.arg(nullifier_substr) = ''
-			OR (LENGTH(sqlc.arg(nullifier_substr)) = 64 AND LOWER(HEX(nullifier)) = LOWER(sqlc.arg(nullifier_substr)))
-			OR (LENGTH(sqlc.arg(nullifier_substr)) < 64 AND INSTR(LOWER(HEX(nullifier)), LOWER(sqlc.arg(nullifier_substr))) > 0)
-			-- TODO: consider keeping an nullifier_hex column for faster searches
-		)
+SELECT v.*, t.hash
+FROM votes AS v
+LEFT JOIN transactions AS t
+	ON v.block_height = t.block_height
+	AND v.block_index = t.block_index
+WHERE (
+	LENGTH(sqlc.arg(process_id_substr)) <= 64 -- if passed arg is longer, then just abort the query
+	AND (
+		sqlc.arg(process_id_substr) = ''
+		OR (LENGTH(sqlc.arg(process_id_substr)) = 64 AND LOWER(HEX(process_id)) = LOWER(sqlc.arg(process_id_substr)))
+		OR (LENGTH(sqlc.arg(process_id_substr)) < 64 AND INSTR(LOWER(HEX(process_id)), LOWER(sqlc.arg(process_id_substr))) > 0)
+		-- TODO: consider keeping an process_id_hex column for faster searches
+	)
+	AND LENGTH(sqlc.arg(nullifier_substr)) <= 64 -- if passed arg is longer, then just abort the query
+	AND (
+		sqlc.arg(nullifier_substr) = ''
+		OR (LENGTH(sqlc.arg(nullifier_substr)) = 64 AND LOWER(HEX(nullifier)) = LOWER(sqlc.arg(nullifier_substr)))
+		OR (LENGTH(sqlc.arg(nullifier_substr)) < 64 AND INSTR(LOWER(HEX(nullifier)), LOWER(sqlc.arg(nullifier_substr))) > 0)
+		-- TODO: consider keeping an nullifier_hex column for faster searches
 	)
 )
-SELECT *, COUNT(*) OVER() AS total_count
-FROM results
-ORDER BY block_height DESC, nullifier ASC
+ORDER BY v.block_height DESC, v.nullifier ASC
 LIMIT sqlc.arg(limit)
 OFFSET sqlc.arg(offset);