From 5a876592211a6f315945bfafe16b974f78ce9c41 Mon Sep 17 00:00:00 2001 From: "David H. Irving" Date: Tue, 27 Aug 2024 17:20:08 -0700 Subject: [PATCH] Decrease page size for queries Decrease query driver's default page size. Queries were consuming excessive memory on Butler server (>100MB per in-flight query), and lowering this value does not seem to make queries any slower. --- python/lsst/daf/butler/direct_query_driver/_driver.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/lsst/daf/butler/direct_query_driver/_driver.py b/python/lsst/daf/butler/direct_query_driver/_driver.py index fbb19b7a0f..cada3e6767 100644 --- a/python/lsst/daf/butler/direct_query_driver/_driver.py +++ b/python/lsst/daf/butler/direct_query_driver/_driver.py @@ -134,7 +134,10 @@ def __init__( dimension_record_cache: DimensionRecordCache, default_collections: Iterable[str], default_data_id: DataCoordinate, - raw_page_size: int = 10000, + # Increasing raw_page_size increases memory usage for queries on + # Butler server, so if you increase this you may need to increase the + # memory allocation for the server in Phalanx as well. + raw_page_size: int = 2000, constant_rows_limit: int = 1000, postprocessing_filter_factor: int = 10, ):