Skip to content

Commit

Permalink
Allow butler ingest-files to disable file attribute tracking
Browse files Browse the repository at this point in the history
  • Loading branch information
timj committed Jul 13, 2024
1 parent ce18b02 commit d760438
Show file tree
Hide file tree
Showing 3 changed files with 17 additions and 1 deletion.
2 changes: 2 additions & 0 deletions python/lsst/daf/butler/cli/cmd/commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
query_datasets_options,
register_dataset_types_option,
repo_argument,
track_file_attrs_option,
transfer_dimensions_option,
transfer_option,
verbose_option,
Expand Down Expand Up @@ -725,6 +726,7 @@ def collection_chain(**kwargs: Any) -> None:
help="For relative paths in the table file, specify a prefix to use. The default is to"
" use the current working directory.",
)
@track_file_attrs_option()
@transfer_option()
def ingest_files(**kwargs: Any) -> None:
"""Ingest files from table file.
Expand Down
9 changes: 9 additions & 0 deletions python/lsst/daf/butler/cli/opt/options.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
"order_by_option",
"limit_option",
"offset_option",
"track_file_attrs_option",
)

from functools import partial
Expand Down Expand Up @@ -313,3 +314,11 @@ def _config_split(*args: Any) -> dict[str, str]:
type=int,
default=0,
)

track_file_attrs_option = MWOptionDecorator(
"--track-file-attrs/--no-track-file-attrs",
default=True,
help="Indicate to the datastore whether file attributes such as file size"
" or checksum should be tracked or not. Whether this parameter is honored"
" depends on the specific datastore implementation.",
)
7 changes: 6 additions & 1 deletion python/lsst/daf/butler/script/ingest_files.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ def ingest_files(
id_generation_mode: str = "UNIQUE",
prefix: str | None = None,
transfer: str = "auto",
track_file_attrs: bool = True,
) -> None:
"""Ingest files from a table.
Expand Down Expand Up @@ -89,6 +90,10 @@ def ingest_files(
is to use the current working directory.
transfer : `str`, optional
Transfer mode to use for ingest.
track_file_attrs : `bool`, optional
Control whether file attributes such as the size or checksum should
be tracked by the datastore. Whether this parameter is honored
depends on the specific datastore implementation.
"""
# Check that the formatter can be imported -- validate this as soon
# as possible before we read a potentially large table file.
Expand Down Expand Up @@ -122,7 +127,7 @@ def ingest_files(
table, common_data_id, datasetType, run, formatter, prefix, id_gen_mode
)

butler.ingest(*datasets, transfer=transfer)
butler.ingest(*datasets, transfer=transfer, record_validation_info=track_file_attrs)


def extract_datasets_from_table(
Expand Down

0 comments on commit d760438

Please sign in to comment.