From d249b51829a375d54c7072968edc6eb1763ffb76 Mon Sep 17 00:00:00 2001 From: "David H. Irving" Date: Tue, 27 Aug 2024 17:20:08 -0700 Subject: [PATCH] Decrease page size for queries Decrease query driver's default page size. Queries were consuming excessive memory on Butler server, and lowering this value does not seem to make queries any slower. This reduces memory consumption for a dataset query with dimension records from ~120MB to ~30MB per in-flight query. --- python/lsst/daf/butler/direct_query_driver/_driver.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/lsst/daf/butler/direct_query_driver/_driver.py b/python/lsst/daf/butler/direct_query_driver/_driver.py index fbb19b7a0f..cada3e6767 100644 --- a/python/lsst/daf/butler/direct_query_driver/_driver.py +++ b/python/lsst/daf/butler/direct_query_driver/_driver.py @@ -134,7 +134,10 @@ def __init__( dimension_record_cache: DimensionRecordCache, default_collections: Iterable[str], default_data_id: DataCoordinate, - raw_page_size: int = 10000, + # Increasing raw_page_size increases memory usage for queries on + # Butler server, so if you increase this you may need to increase the + # memory allocation for the server in Phalanx as well. + raw_page_size: int = 2000, constant_rows_limit: int = 1000, postprocessing_filter_factor: int = 10, ):