feat: Selectable db query performance logging (#476)

* feat: make tracing subscriber reloadable * feat: add InspectionSettings type * feat: add 'GET health/inspection' endpoint * fix: simplify mutability for inspection settings * feat: add inspectionable query method in state fix: use async mutex for db inspection settings * feat: add inspectable query and query_one methods for EventDb fix: refactor db settings into EventDb * feat(wip): substitute tokio::Client::query and query_one calls use EventDb::query and EventDb::query_one methods * feat: all endpoint queries can be inspected * fix: update project dictionary * fix: implement poem_openapi param traits on query arguments * feat: add EventDb::query and EventDb::modify methods * feat: use EventDb::modify for INSERT, UPDATE, DELETE operations * feat: add stringzilla to crate dependencies * feat: add function to determine if a statement is a query operation * chore: update project dictionary * fix: improve is_query_stmt function * fix: inspect statement to query or modify the DB * fix: update openapi response types, add missing docs * fix: update flutter generated code * try * fix * try * try * try * set deep query inspection to false * refactor DeepQueryInspectionFlag * wip * refactor * split OpenAI definition of LogLevel and DeepQueryInspectionFlag structs from the original one. * fix * wip * fix: only print response data when its code is 200 * chore: cleanup debugging code --------- Co-authored-by: Steven Johnson <[email protected]> Co-authored-by: Mr-Leshiy <[email protected]>
input-output-hk · May 30, 2024 · 4e1af1b · 4e1af1b
1 parent c8427e5
commit 4e1af1b
Show file tree

Hide file tree

Showing 31 changed files with 710 additions and 181 deletions.
diff --git a/.config/dictionaries/project.dic b/.config/dictionaries/project.dic
@@ -137,6 +137,7 @@ pytest
 rapidoc
 redoc
 Replayability
+reloadable
 repr
 reqwest
 rfwtxt
@@ -159,6 +160,7 @@ slotno
 sqlfluff
 Stefano
 stevenj
+stringzilla
 Subkey
 subosito
 SYSROOT

diff --git a/catalyst-gateway/Cargo.toml b/catalyst-gateway/Cargo.toml
@@ -50,6 +50,7 @@ cddl = "0.9.2"
 ciborium = "0.2"
 pallas = { git = "https://github.com/input-output-hk/catalyst-pallas.git", rev = "709acb19c52c6b789279ecc4bc8793b5d8b5abe9", version = "0.25.0" }
 cardano-chain-follower = { git = "https://github.com/input-output-hk/hermes.git", version="0.0.1" }
+stringzilla = "3.8.4"
 
 [workspace.lints.rust]
 warnings = "deny"

diff --git a/catalyst-gateway/bin/Cargo.toml b/catalyst-gateway/bin/Cargo.toml
@@ -24,7 +24,7 @@ tokio-postgres = { workspace = true, features = [
 ] }
 clap = { workspace = true, features = ["derive", "env"] }
 tracing = { workspace = true, features = ["log"] }
-tracing-subscriber = { workspace = true, features = ["fmt", "json", "time"] }
+tracing-subscriber = { workspace = true, features = ["fmt", "json", "registry", "std", "time"] }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
 tokio = { workspace = true, features = ["rt", "macros", "rt-multi-thread"] }
@@ -68,3 +68,4 @@ handlebars = { workspace = true }
 cddl = { workspace = true }
 ciborium = { workspace = true }
 ed25519-dalek = "2.1.1"
+stringzilla = { workspace = true }
diff --git a/catalyst-gateway/bin/src/cli.rs b/catalyst-gateway/bin/src/cli.rs
@@ -38,13 +38,16 @@ impl Cli {
     pub(crate) async fn exec(self) -> anyhow::Result<()> {
         match self {
             Self::Run(settings) => {
-                logger::init(settings.log_level)?;
+                let logger_handle = logger::init(settings.log_level);
 
                 // Unique machine id
                 let machine_id = settings.follower_settings.machine_uid;
 
-                let state = Arc::new(State::new(Some(settings.database_url)).await?);
+                let state = Arc::new(State::new(Some(settings.database_url), logger_handle).await?);
                 let event_db = state.event_db();
+                event_db
+                    .modify_deep_query(settings.deep_query_inspection.into())
+                    .await;
 
                 tokio::spawn(async move {
                     match service::run(&settings.docs_settings, state.clone()).await {

diff --git a/catalyst-gateway/bin/src/event_db/cardano/chain_state/mod.rs b/catalyst-gateway/bin/src/event_db/cardano/chain_state/mod.rs
@@ -91,17 +91,14 @@ impl EventDB {
         &self, slot_no: SlotNumber, network: Network, epoch_no: EpochNumber, block_time: DateTime,
         block_hash: BlockHash,
     ) -> anyhow::Result<()> {
-        let conn = self.pool.get().await?;
-
-        let _rows = conn
-            .query(INSERT_SLOT_INDEX_SQL, &[
-                &slot_no,
-                &network.to_string(),
-                &epoch_no,
-                &block_time,
-                &block_hash,
-            ])
-            .await?;
+        self.modify(INSERT_SLOT_INDEX_SQL, &[
+            &slot_no,
+            &network.to_string(),
+            &epoch_no,
+            &block_time,
+            &block_hash,
+        ])
+        .await?;
 
         Ok(())
     }
@@ -110,9 +107,7 @@ impl EventDB {
     pub(crate) async fn get_slot_info(
         &self, date_time: DateTime, network: Network, query_type: SlotInfoQueryType,
     ) -> anyhow::Result<(SlotNumber, BlockHash, DateTime)> {
-        let conn = self.pool.get().await?;
-
-        let rows = conn
+        let rows = self
             .query(&query_type.get_sql_query()?, &[
                 &network.to_string(),
                 &date_time,
@@ -131,9 +126,7 @@ impl EventDB {
     pub(crate) async fn last_updated_state(
         &self, network: Network,
     ) -> anyhow::Result<(SlotNumber, BlockHash, DateTime)> {
-        let conn = self.pool.get().await?;
-
-        let rows = conn
+        let rows = self
             .query(SELECT_UPDATE_STATE_SQL, &[&network.to_string()])
             .await?;
 
@@ -152,27 +145,24 @@ impl EventDB {
         &self, last_updated: DateTime, slot_no: SlotNumber, block_hash: BlockHash,
         network: Network, machine_id: &MachineId,
     ) -> anyhow::Result<()> {
-        let conn = self.pool.get().await?;
-
         // Rollback or update
         let update = true;
 
         let network_id: u64 = network.into();
 
         // An insert only happens once when there is no update metadata available
         // All future additions are just updates on ended, slot_no and block_hash
-        let _rows = conn
-            .query(INSERT_UPDATE_STATE_SQL, &[
-                &i64::try_from(network_id)?,
-                &last_updated,
-                &last_updated,
-                &machine_id,
-                &slot_no,
-                &network.to_string(),
-                &block_hash,
-                &update,
-            ])
-            .await?;
+        self.modify(INSERT_UPDATE_STATE_SQL, &[
+            &i64::try_from(network_id)?,
+            &last_updated,
+            &last_updated,
+            &machine_id,
+            &slot_no,
+            &network.to_string(),
+            &block_hash,
+            &update,
+        ])
+        .await?;
 
         Ok(())
     }

diff --git a/catalyst-gateway/bin/src/event_db/cardano/cip36_registration/mod.rs b/catalyst-gateway/bin/src/event_db/cardano/cip36_registration/mod.rs
@@ -46,8 +46,6 @@ impl EventDB {
         voting_info: Option<PublicVotingInfo>, payment_address: Option<PaymentAddress>,
         metadata_cip36: Option<MetadataCip36>, nonce: Option<Nonce>, errors_report: ErrorReport,
     ) -> anyhow::Result<()> {
-        let conn = self.pool.get().await?;
-
         // for the catalyst we dont support multiple delegations
         let multiple_delegations = voting_info.as_ref().is_some_and(|voting_info| {
             if let PublicVotingInfo::Delegated(delegations) = voting_info {
@@ -76,18 +74,17 @@ impl EventDB {
             && nonce.is_some()
             && errors_report.is_empty();
 
-        let _rows = conn
-            .query(INSERT_VOTER_REGISTRATION_SQL, &[
-                &tx_id,
-                &stake_credential,
-                &encoded_voting_info,
-                &payment_address,
-                &nonce,
-                &metadata_cip36,
-                &json!(&errors_report),
-                &is_valid,
-            ])
-            .await?;
+        self.modify(INSERT_VOTER_REGISTRATION_SQL, &[
+            &tx_id,
+            &stake_credential,
+            &encoded_voting_info,
+            &payment_address,
+            &nonce,
+            &metadata_cip36,
+            &json!(&errors_report),
+            &is_valid,
+        ])
+        .await?;
 
         Ok(())
     }
@@ -131,9 +128,7 @@ impl EventDB {
     pub(crate) async fn get_registration_info(
         &self, stake_credential: StakeCredential, network: Network, slot_num: SlotNumber,
     ) -> anyhow::Result<(TxId, PaymentAddress, PublicVotingInfo, Nonce)> {
-        let conn = self.pool.get().await?;
-
-        let rows = conn
+        let rows = self
             .query(SELECT_VOTER_REGISTRATION_SQL, &[
                 &stake_credential,
                 &network.to_string(),

diff --git a/catalyst-gateway/bin/src/event_db/cardano/config/mod.rs b/catalyst-gateway/bin/src/event_db/cardano/config/mod.rs
@@ -55,12 +55,10 @@ const SELECT_CONFIG_SQL: &str = include_str!("select_config.sql");
 impl EventDB {
     /// Config query
     pub(crate) async fn get_follower_config(&self) -> anyhow::Result<Vec<FollowerConfig>> {
-        let conn = self.pool.get().await?;
-
         let id = "cardano";
         let id2 = "follower";
 
-        let rows = conn.query(SELECT_CONFIG_SQL, &[&id, &id2]).await?;
+        let rows = self.query(SELECT_CONFIG_SQL, &[&id, &id2]).await?;
 
         let mut follower_configs = Vec::new();
         for row in rows {

diff --git a/catalyst-gateway/bin/src/event_db/cardano/utxo/mod.rs b/catalyst-gateway/bin/src/event_db/cardano/utxo/mod.rs
@@ -24,8 +24,6 @@ const UPDATE_UTXO_SQL: &str = include_str!("update_utxo.sql");
 impl EventDB {
     /// Index utxo data
     pub(crate) async fn index_utxo_data(&self, tx: &MultiEraTx<'_>) -> anyhow::Result<()> {
-        let conn = self.pool.get().await?;
-
         let tx_hash = tx.hash();
 
         // index outputs
@@ -41,29 +39,27 @@ impl EventDB {
             };
             let stake_credential = stake_address.map(|val| val.payload().as_hash().to_vec());
 
-            let _rows = conn
-                .query(INSERT_UTXO_SQL, &[
-                    &i32::try_from(index)?,
-                    &tx_hash.as_slice(),
-                    &i64::try_from(tx_out.lovelace_amount())?,
-                    &stake_credential,
-                    &assets,
-                ])
-                .await?;
+            self.modify(INSERT_UTXO_SQL, &[
+                &i32::try_from(index)?,
+                &tx_hash.as_slice(),
+                &i64::try_from(tx_out.lovelace_amount())?,
+                &stake_credential,
+                &assets,
+            ])
+            .await?;
         }
         // update outputs with inputs
         for tx_in in tx.inputs() {
             let output = tx_in.output_ref();
             let output_tx_hash = output.hash();
             let out_index = output.index();
 
-            let _rows = conn
-                .query(UPDATE_UTXO_SQL, &[
-                    &tx_hash.as_slice(),
-                    &output_tx_hash.as_slice(),
-                    &i32::try_from(out_index)?,
-                ])
-                .await?;
+            self.modify(UPDATE_UTXO_SQL, &[
+                &tx_hash.as_slice(),
+                &output_tx_hash.as_slice(),
+                &i32::try_from(out_index)?,
+            ])
+            .await?;
         }
 
         Ok(())
@@ -73,15 +69,12 @@ impl EventDB {
     pub(crate) async fn index_txn_data(
         &self, tx_id: &[u8], slot_no: SlotNumber, network: Network,
     ) -> anyhow::Result<()> {
-        let conn = self.pool.get().await?;
-
-        let _rows = conn
-            .query(INSERT_TXN_INDEX_SQL, &[
-                &tx_id,
-                &slot_no,
-                &network.to_string(),
-            ])
-            .await?;
+        self.modify(INSERT_TXN_INDEX_SQL, &[
+            &tx_id,
+            &slot_no,
+            &network.to_string(),
+        ])
+        .await?;
 
         Ok(())
     }
@@ -90,9 +83,7 @@ impl EventDB {
     pub(crate) async fn total_utxo_amount(
         &self, stake_credential: StakeCredential, network: Network, slot_num: SlotNumber,
     ) -> anyhow::Result<(StakeAmount, SlotNumber)> {
-        let conn = self.pool.get().await?;
-
-        let row = conn
+        let row = self
             .query_one(SELECT_TOTAL_UTXO_AMOUNT_SQL, &[
                 &stake_credential,
                 &network.to_string(),

diff --git a/catalyst-gateway/bin/src/event_db/legacy/queries/event/ballot.rs b/catalyst-gateway/bin/src/event_db/legacy/queries/event/ballot.rs
@@ -53,9 +53,7 @@ impl EventDB {
     pub(crate) async fn get_ballot(
         &self, event: EventId, objective: ObjectiveId, proposal: ProposalId,
     ) -> anyhow::Result<Ballot> {
-        let conn = self.pool.get().await?;
-
-        let rows = conn
+        let rows = self
             .query(Self::BALLOT_VOTE_OPTIONS_QUERY, &[
                 &event.0,
                 &objective.0,
@@ -65,7 +63,7 @@ impl EventDB {
         let row = rows.first().ok_or(NotFoundError)?;
         let choices = row.try_get("objective")?;
 
-        let rows = conn
+        let rows = self
             .query(Self::BALLOT_VOTE_PLANS_QUERY, &[
                 &event.0,
                 &objective.0,
@@ -96,9 +94,7 @@ impl EventDB {
     pub(crate) async fn get_objective_ballots(
         &self, event: EventId, objective: ObjectiveId,
     ) -> anyhow::Result<Vec<ProposalBallot>> {
-        let conn = self.pool.get().await?;
-
-        let rows = conn
+        let rows = self
             .query(Self::BALLOTS_VOTE_OPTIONS_PER_OBJECTIVE_QUERY, &[
                 &event.0,
                 &objective.0,
@@ -110,7 +106,7 @@ impl EventDB {
             let choices = row.try_get("objective")?;
             let proposal_id = ProposalId(row.try_get("proposal_id")?);
 
-            let rows = conn
+            let rows = self
                 .query(Self::BALLOT_VOTE_PLANS_QUERY, &[
                     &event.0,
                     &objective.0,
@@ -146,9 +142,7 @@ impl EventDB {
     pub(crate) async fn get_event_ballots(
         &self, event: EventId,
     ) -> anyhow::Result<Vec<ObjectiveBallots>> {
-        let conn = self.pool.get().await?;
-
-        let rows = conn
+        let rows = self
             .query(Self::BALLOTS_VOTE_OPTIONS_PER_EVENT_QUERY, &[&event.0])
             .await?;
         let mut ballots = HashMap::<ObjectiveId, Vec<ProposalBallot>>::new();
@@ -157,7 +151,7 @@ impl EventDB {
             let proposal_id = ProposalId(row.try_get("proposal_id")?);
             let objective_id = ObjectiveId(row.try_get("objective_id")?);
 
-            let rows = conn
+            let rows = self
                 .query(Self::BALLOT_VOTE_PLANS_QUERY, &[
                     &event.0,
                     &objective_id.0,

diff --git a/catalyst-gateway/bin/src/event_db/legacy/queries/event/mod.rs b/catalyst-gateway/bin/src/event_db/legacy/queries/event/mod.rs
@@ -45,9 +45,7 @@ impl EventDB {
     pub(crate) async fn get_events(
         &self, limit: Option<i64>, offset: Option<i64>,
     ) -> anyhow::Result<Vec<EventSummary>> {
-        let conn = self.pool.get().await?;
-
-        let rows = conn
+        let rows = self
             .query(Self::EVENTS_QUERY, &[&limit, &offset.unwrap_or(0)])
             .await?;
 
@@ -77,9 +75,7 @@ impl EventDB {
     /// Get event query
     #[allow(dead_code)]
     pub(crate) async fn get_event(&self, event: EventId) -> anyhow::Result<Event> {
-        let conn = self.pool.get().await?;
-
-        let rows = conn.query(Self::EVENT_QUERY, &[&event.0]).await?;
+        let rows = self.query(Self::EVENT_QUERY, &[&event.0]).await?;
         let row = rows.first().ok_or(NotFoundError)?;
 
         let ends = row
@@ -133,7 +129,7 @@ impl EventDB {
                 .map(|val| val.and_local_timezone(Utc).unwrap()),
         };
 
-        let rows = conn.query(Self::EVENT_GOALS_QUERY, &[&event.0]).await?;
+        let rows = self.query(Self::EVENT_GOALS_QUERY, &[&event.0]).await?;
         let mut goals = Vec::new();
         for row in rows {
             goals.push(EventGoal {