From d760438f5989d80b18a4bebf74f8752ffc00d02a Mon Sep 17 00:00:00 2001 From: Tim Jenness Date: Fri, 12 Jul 2024 17:15:29 -0700 Subject: [PATCH] Allow butler ingest-files to disable file attribute tracking --- python/lsst/daf/butler/cli/cmd/commands.py | 2 ++ python/lsst/daf/butler/cli/opt/options.py | 9 +++++++++ python/lsst/daf/butler/script/ingest_files.py | 7 ++++++- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/python/lsst/daf/butler/cli/cmd/commands.py b/python/lsst/daf/butler/cli/cmd/commands.py index 0ebaeb56ce..1339588d3b 100644 --- a/python/lsst/daf/butler/cli/cmd/commands.py +++ b/python/lsst/daf/butler/cli/cmd/commands.py @@ -54,6 +54,7 @@ query_datasets_options, register_dataset_types_option, repo_argument, + track_file_attrs_option, transfer_dimensions_option, transfer_option, verbose_option, @@ -725,6 +726,7 @@ def collection_chain(**kwargs: Any) -> None: help="For relative paths in the table file, specify a prefix to use. The default is to" " use the current working directory.", ) +@track_file_attrs_option() @transfer_option() def ingest_files(**kwargs: Any) -> None: """Ingest files from table file. diff --git a/python/lsst/daf/butler/cli/opt/options.py b/python/lsst/daf/butler/cli/opt/options.py index d4abe3e713..115d52bcc6 100644 --- a/python/lsst/daf/butler/cli/opt/options.py +++ b/python/lsst/daf/butler/cli/opt/options.py @@ -54,6 +54,7 @@ "order_by_option", "limit_option", "offset_option", + "track_file_attrs_option", ) from functools import partial @@ -313,3 +314,11 @@ def _config_split(*args: Any) -> dict[str, str]: type=int, default=0, ) + +track_file_attrs_option = MWOptionDecorator( + "--track-file-attrs/--no-track-file-attrs", + default=True, + help="Indicate to the datastore whether file attributes such as file size" + " or checksum should be tracked or not. Whether this parameter is honored" + " depends on the specific datastore implementation.", +) diff --git a/python/lsst/daf/butler/script/ingest_files.py b/python/lsst/daf/butler/script/ingest_files.py index dc168e5843..93dd90b73c 100644 --- a/python/lsst/daf/butler/script/ingest_files.py +++ b/python/lsst/daf/butler/script/ingest_files.py @@ -57,6 +57,7 @@ def ingest_files( id_generation_mode: str = "UNIQUE", prefix: str | None = None, transfer: str = "auto", + track_file_attrs: bool = True, ) -> None: """Ingest files from a table. @@ -89,6 +90,10 @@ def ingest_files( is to use the current working directory. transfer : `str`, optional Transfer mode to use for ingest. + track_file_attrs : `bool`, optional + Control whether file attributes such as the size or checksum should + be tracked by the datastore. Whether this parameter is honored + depends on the specific datastore implementation. """ # Check that the formatter can be imported -- validate this as soon # as possible before we read a potentially large table file. @@ -122,7 +127,7 @@ def ingest_files( table, common_data_id, datasetType, run, formatter, prefix, id_gen_mode ) - butler.ingest(*datasets, transfer=transfer) + butler.ingest(*datasets, transfer=transfer, record_validation_info=track_file_attrs) def extract_datasets_from_table(