Skip to content

Commit

Permalink
Change query-datasets command-line to use the butler it creates
Browse files Browse the repository at this point in the history
Without this change the chained datastore created and populated
is not used in the actual chaining test and so the tests are
not testing what it is stated to test.
  • Loading branch information
timj committed Sep 6, 2024
1 parent e14eae1 commit 1dbd433
Show file tree
Hide file tree
Showing 2 changed files with 18 additions and 15 deletions.
1 change: 1 addition & 0 deletions tests/config/basic/posixDatastore2.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,4 @@ datastore:
StructuredDataPickle: lsst.daf.butler.formatters.pickle.PickleFormatter
ThingOne: lsst.daf.butler.formatters.pickle.PickleFormatter
MetricsConversion: lsst.daf.butler.formatters.json.JsonFormatter
StructuredDataDataTest: lsst.daf.butler.tests.testFormatters.MetricsExampleDataFormatter
32 changes: 17 additions & 15 deletions tests/test_cliCmdQueryDatasets.py
Original file line number Diff line number Diff line change
Expand Up @@ -157,8 +157,8 @@ def _queryDatasets(
find_first=find_first,
show_uri=show_uri,
limit=limit,
repo=repo,
order_by=order_by,
butler=repo,
)
return list(query.getTables())

Expand All @@ -174,9 +174,11 @@ def testChained(self):
self.repoDir, configFile=os.path.join(TESTDIR, "config/basic/butler-chained.yaml")
)

tables = self._queryDatasets(repo=self.repoDir, show_uri=True)
tables = self._queryDatasets(repo=testRepo.butler, show_uri=True)

# Want second datastore root.
# Want second datastore root since in-memory is ephemeral and
# all the relevant datasets are stored in the second as well as third
# datastore.
roots = testRepo.butler.get_datastore_roots()
datastore_root = roots[testRepo.butler.get_datastore_names()[1]]

Expand All @@ -190,7 +192,7 @@ def testShowURI(self):
"""Test for expected output with show_uri=True."""
testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)

tables = self._queryDatasets(repo=self.repoDir, show_uri=True)
tables = self._queryDatasets(repo=testRepo.butler, show_uri=True)

roots = testRepo.butler.get_datastore_roots()
datastore_root = list(roots.values())[0]
Expand All @@ -201,9 +203,9 @@ def testShowURI(self):

def testNoShowURI(self):
"""Test for expected output without show_uri (default is False)."""
_ = MetricTestRepo(self.repoDir, configFile=self.configFile)
testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)

tables = self._queryDatasets(repo=self.repoDir)
tables = self._queryDatasets(repo=testRepo.butler)

expectedTables = (
AstropyTable(
Expand All @@ -223,9 +225,9 @@ def testWhere(self):
"""Test using the where clause to reduce the number of rows returned by
queryDatasets.
"""
_ = MetricTestRepo(self.repoDir, configFile=self.configFile)
testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)

tables = self._queryDatasets(repo=self.repoDir, where="instrument='DummyCamComp' AND visit=423")
tables = self._queryDatasets(repo=testRepo.butler, where="instrument='DummyCamComp' AND visit=423")

expectedTables = (
AstropyTable(
Expand All @@ -237,7 +239,7 @@ def testWhere(self):
self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)

with self.assertRaises(RuntimeError):
self._queryDatasets(repo=self.repoDir, collections="*", find_first=True)
self._queryDatasets(repo=testRepo.butler, collections="*", find_first=True)

def testGlobDatasetType(self):
"""Test specifying dataset type."""
Expand All @@ -262,7 +264,7 @@ def testGlobDatasetType(self):
testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType)

# verify the new dataset type increases the number of tables found:
tables = self._queryDatasets(repo=self.repoDir)
tables = self._queryDatasets(repo=testRepo.butler)

expectedTables = (
AstropyTable(
Expand All @@ -285,10 +287,10 @@ def testGlobDatasetType(self):
def test_limit_order(self):
"""Test limit and ordering."""
# Create and register an additional DatasetType
MetricTestRepo(self.repoDir, configFile=self.configFile)
testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)

with self.assertLogs("lsst.daf.butler.script.queryDatasets", level="WARNING") as cm:
tables = self._queryDatasets(repo=self.repoDir, limit=-1, order_by=("visit"))
tables = self._queryDatasets(repo=testRepo.butler, limit=-1, order_by=("visit"))

self.assertIn("increase this limit", cm.output[0])

Expand All @@ -301,7 +303,7 @@ def test_limit_order(self):
self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)

with self.assertLogs("lsst.daf.butler.script.queryDatasets", level="WARNING") as cm:
tables = self._queryDatasets(repo=self.repoDir, limit=-1, order_by=("-visit"))
tables = self._queryDatasets(repo=testRepo.butler, limit=-1, order_by=("-visit"))
self.assertIn("increase this limit", cm.output[0])

expectedTables = [
Expand All @@ -322,7 +324,7 @@ def testFindFirstAndCollections(self):
testRepo.addDataset(run="foo", dataId={"instrument": "DummyCamComp", "visit": 424})

# Verify that without find-first, duplicate datasets are returned
tables = self._queryDatasets(repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True)
tables = self._queryDatasets(repo=testRepo.butler, collections=["foo", "ingest/run"], show_uri=True)

# The test should be running with a single FileDatastore.
roots = testRepo.butler.get_datastore_roots()
Expand Down Expand Up @@ -465,7 +467,7 @@ def testFindFirstAndCollections(self):
# Verify that with find first the duplicate dataset is eliminated and
# the more recent dataset is returned.
tables = self._queryDatasets(
repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True, find_first=True
repo=testRepo.butler, collections=["foo", "ingest/run"], show_uri=True, find_first=True
)

expectedTables = (
Expand Down

0 comments on commit 1dbd433

Please sign in to comment.