Skip to content

Commit

Permalink
add postgresql index and use IN instead of many OR (#4670)
Browse files Browse the repository at this point in the history
  • Loading branch information
@trinity-1686a
trinity-1686a authored Mar 5, 2024
1 parent 718f666 commit a65e269
Show file tree
Hide file tree
Showing 4 changed files with 23 additions and 25 deletions.
1 change: 1 addition & 0 deletions quickwit/quickwit-metastore/migrations/postgresql/16_create-index-split-uid.down.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
DROP INDEX IF EXISTS splits_index_uid_idx;
1 change: 1 addition & 0 deletions quickwit/quickwit-metastore/migrations/postgresql/16_create-index-split-uid.up.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
CREATE INDEX IF NOT EXISTS splits_index_uid_idx ON splits USING HASH(index_uid);
34 changes: 17 additions & 17 deletions quickwit/quickwit-metastore/src/metastore/postgres/metastore.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1667,7 +1667,7 @@ mod tests {
assert_eq!(
sql.to_string(PostgresQueryBuilder),
format!(
r#"SELECT * FROM "splits" WHERE "index_uid" = '{index_uid}' AND "split_state" IN ('Staged')"#
r#"SELECT * FROM "splits" WHERE "index_uid" IN ('{index_uid}') AND "split_state" IN ('Staged')"#
)
);

Expand All @@ -1681,7 +1681,7 @@ mod tests {
assert_eq!(
sql.to_string(PostgresQueryBuilder),
format!(
r#"SELECT * FROM "splits" WHERE "index_uid" = '{index_uid}' AND "split_state" IN ('Published')"#
r#"SELECT * FROM "splits" WHERE "index_uid" IN ('{index_uid}') AND "split_state" IN ('Published')"#
)
);

Expand All @@ -1694,7 +1694,7 @@ mod tests {
assert_eq!(
sql.to_string(PostgresQueryBuilder),
format!(
r#"SELECT * FROM "splits" WHERE "index_uid" = '{index_uid}' AND "split_state" IN ('Published', 'MarkedForDeletion')"#
r#"SELECT * FROM "splits" WHERE "index_uid" IN ('{index_uid}') AND "split_state" IN ('Published', 'MarkedForDeletion')"#
)
);

Expand All @@ -1706,7 +1706,7 @@ mod tests {
assert_eq!(
sql.to_string(PostgresQueryBuilder),
format!(
r#"SELECT * FROM "splits" WHERE "index_uid" = '{index_uid}' AND "update_timestamp" < TO_TIMESTAMP(51)"#
r#"SELECT * FROM "splits" WHERE "index_uid" IN ('{index_uid}') AND "update_timestamp" < TO_TIMESTAMP(51)"#
)
);

Expand All @@ -1718,7 +1718,7 @@ mod tests {
assert_eq!(
sql.to_string(PostgresQueryBuilder),
format!(
r#"SELECT * FROM "splits" WHERE "index_uid" = '{index_uid}' AND "create_timestamp" <= TO_TIMESTAMP(55)"#
r#"SELECT * FROM "splits" WHERE "index_uid" IN ('{index_uid}') AND "create_timestamp" <= TO_TIMESTAMP(55)"#
)
);

Expand All @@ -1733,7 +1733,7 @@ mod tests {
assert_eq!(
sql.to_string(PostgresQueryBuilder),
format!(
r#"SELECT * FROM "splits" WHERE "index_uid" = '{index_uid}' AND ("maturity_timestamp" = TO_TIMESTAMP(0) OR "maturity_timestamp" <= TO_TIMESTAMP(55))"#
r#"SELECT * FROM "splits" WHERE "index_uid" IN ('{index_uid}') AND ("maturity_timestamp" = TO_TIMESTAMP(0) OR "maturity_timestamp" <= TO_TIMESTAMP(55))"#
)
);

Expand All @@ -1746,7 +1746,7 @@ mod tests {
assert_eq!(
sql.to_string(PostgresQueryBuilder),
format!(
r#"SELECT * FROM "splits" WHERE "index_uid" = '{index_uid}' AND "maturity_timestamp" > TO_TIMESTAMP(55)"#
r#"SELECT * FROM "splits" WHERE "index_uid" IN ('{index_uid}') AND "maturity_timestamp" > TO_TIMESTAMP(55)"#
)
);

Expand All @@ -1758,7 +1758,7 @@ mod tests {
assert_eq!(
sql.to_string(PostgresQueryBuilder),
format!(
r#"SELECT * FROM "splits" WHERE "index_uid" = '{index_uid}' AND "delete_opstamp" >= 4"#
r#"SELECT * FROM "splits" WHERE "index_uid" IN ('{index_uid}') AND "delete_opstamp" >= 4"#
)
);

Expand All @@ -1770,7 +1770,7 @@ mod tests {
assert_eq!(
sql.to_string(PostgresQueryBuilder),
format!(
r#"SELECT * FROM "splits" WHERE "index_uid" = '{index_uid}' AND ("time_range_end" > 45 OR "time_range_end" IS NULL)"#
r#"SELECT * FROM "splits" WHERE "index_uid" IN ('{index_uid}') AND ("time_range_end" > 45 OR "time_range_end" IS NULL)"#
)
);

Expand All @@ -1782,7 +1782,7 @@ mod tests {
assert_eq!(
sql.to_string(PostgresQueryBuilder),
format!(
r#"SELECT * FROM "splits" WHERE "index_uid" = '{index_uid}' AND ("time_range_start" < 45 OR "time_range_start" IS NULL)"#
r#"SELECT * FROM "splits" WHERE "index_uid" IN ('{index_uid}') AND ("time_range_start" < 45 OR "time_range_start" IS NULL)"#
)
);

Expand All @@ -1799,7 +1799,7 @@ mod tests {
assert_eq!(
sql.to_string(PostgresQueryBuilder),
format!(
r#"SELECT * FROM "splits" WHERE "index_uid" = '{index_uid}' AND (NOT ($$tag-2$$ = ANY(tags)))"#
r#"SELECT * FROM "splits" WHERE "index_uid" IN ('{index_uid}') AND (NOT ($$tag-2$$ = ANY(tags)))"#
)
);

Expand All @@ -1812,7 +1812,7 @@ mod tests {
assert_eq!(
sql.to_string(PostgresQueryBuilder),
format!(
r#"SELECT * FROM "splits" WHERE "index_uid" = '{index_uid}' ORDER BY "split_id" ASC OFFSET 4"#
r#"SELECT * FROM "splits" WHERE "index_uid" IN ('{index_uid}') ORDER BY "split_id" ASC OFFSET 4"#
)
);
}
Expand All @@ -1830,7 +1830,7 @@ mod tests {
assert_eq!(
sql.to_string(PostgresQueryBuilder),
format!(
r#"SELECT * FROM "splits" WHERE "index_uid" = '{index_uid}' AND ("time_range_end" > 0 OR "time_range_end" IS NULL) AND ("time_range_start" < 40 OR "time_range_start" IS NULL)"#
r#"SELECT * FROM "splits" WHERE "index_uid" IN ('{index_uid}') AND ("time_range_end" > 0 OR "time_range_end" IS NULL) AND ("time_range_start" < 40 OR "time_range_start" IS NULL)"#
)
);

Expand All @@ -1844,7 +1844,7 @@ mod tests {
assert_eq!(
sql.to_string(PostgresQueryBuilder),
format!(
r#"SELECT * FROM "splits" WHERE "index_uid" = '{index_uid}' AND ("time_range_end" > 45 OR "time_range_end" IS NULL) AND "delete_opstamp" > 0"#
r#"SELECT * FROM "splits" WHERE "index_uid" IN ('{index_uid}') AND ("time_range_end" > 45 OR "time_range_end" IS NULL) AND "delete_opstamp" > 0"#
)
);

Expand All @@ -1858,7 +1858,7 @@ mod tests {
assert_eq!(
sql.to_string(PostgresQueryBuilder),
format!(
r#"SELECT * FROM "splits" WHERE "index_uid" = '{index_uid}' AND "update_timestamp" < TO_TIMESTAMP(51) AND "create_timestamp" <= TO_TIMESTAMP(63)"#
r#"SELECT * FROM "splits" WHERE "index_uid" IN ('{index_uid}') AND "update_timestamp" < TO_TIMESTAMP(51) AND "create_timestamp" <= TO_TIMESTAMP(63)"#
)
);

Expand All @@ -1875,7 +1875,7 @@ mod tests {
assert_eq!(
sql.to_string(PostgresQueryBuilder),
format!(
r#"SELECT * FROM "splits" WHERE "index_uid" = '{index_uid}' AND ($$tag-1$$ = ANY(tags)) AND ("time_range_end" > 90 OR "time_range_end" IS NULL)"#
r#"SELECT * FROM "splits" WHERE "index_uid" IN ('{index_uid}') AND ($$tag-1$$ = ANY(tags)) AND ("time_range_end" > 90 OR "time_range_end" IS NULL)"#
)
);

Expand All @@ -1890,7 +1890,7 @@ mod tests {
assert_eq!(
sql.to_string(PostgresQueryBuilder),
format!(
r#"SELECT * FROM "splits" WHERE "index_uid" = '{index_uid}' OR "index_uid" = '{index_uid_2}'"#
r#"SELECT * FROM "splits" WHERE "index_uid" IN ('{index_uid}', '{index_uid_2}')"#
)
);
}
Expand Down
12 changes: 4 additions & 8 deletions quickwit/quickwit-metastore/src/metastore/postgres/utils.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ use std::time::Duration;

use quickwit_common::uri::Uri;
use quickwit_proto::metastore::{MetastoreError, MetastoreResult};
use sea_query::{any, Cond, Expr, Func, Order, SelectStatement};
use sea_query::{any, Expr, Func, Order, SelectStatement};
use sqlx::postgres::{PgConnectOptions, PgPoolOptions};
use sqlx::{ConnectOptions, Pool, Postgres};
use tracing::error;
Expand Down Expand Up @@ -93,13 +93,9 @@ pub(super) fn append_range_filters<V: Display>(
pub(super) fn append_query_filters(sql: &mut SelectStatement, query: &ListSplitsQuery) {
// Note: `ListSplitsQuery` builder enforces a non empty `index_uids` list.

let or_condition = query
.index_uids
.iter()
.fold(Cond::any(), |cond, index_uid| {
cond.add(Expr::col(Splits::IndexUid).eq(Expr::val(index_uid.to_string())))
});
sql.cond_where(or_condition);
sql.cond_where(
Expr::col(Splits::IndexUid).is_in(query.index_uids.iter().map(|val| val.to_string())),
);

if !query.split_states.is_empty() {
sql.cond_where(
Expand Down

0 comments on commit a65e269

Please sign in to comment.