From 1dbd43320c2ed1caf3fe4fa11a3eb3d1786bd801 Mon Sep 17 00:00:00 2001 From: Tim Jenness Date: Fri, 6 Sep 2024 10:11:15 -0700 Subject: [PATCH] Change query-datasets command-line to use the butler it creates Without this change the chained datastore created and populated is not used in the actual chaining test and so the tests are not testing what it is stated to test. --- tests/config/basic/posixDatastore2.yaml | 1 + tests/test_cliCmdQueryDatasets.py | 32 +++++++++++++------------ 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/tests/config/basic/posixDatastore2.yaml b/tests/config/basic/posixDatastore2.yaml index c496ec4572..2830339f10 100644 --- a/tests/config/basic/posixDatastore2.yaml +++ b/tests/config/basic/posixDatastore2.yaml @@ -23,3 +23,4 @@ datastore: StructuredDataPickle: lsst.daf.butler.formatters.pickle.PickleFormatter ThingOne: lsst.daf.butler.formatters.pickle.PickleFormatter MetricsConversion: lsst.daf.butler.formatters.json.JsonFormatter + StructuredDataDataTest: lsst.daf.butler.tests.testFormatters.MetricsExampleDataFormatter diff --git a/tests/test_cliCmdQueryDatasets.py b/tests/test_cliCmdQueryDatasets.py index 8ec35ed6e8..3072671e4e 100644 --- a/tests/test_cliCmdQueryDatasets.py +++ b/tests/test_cliCmdQueryDatasets.py @@ -157,8 +157,8 @@ def _queryDatasets( find_first=find_first, show_uri=show_uri, limit=limit, - repo=repo, order_by=order_by, + butler=repo, ) return list(query.getTables()) @@ -174,9 +174,11 @@ def testChained(self): self.repoDir, configFile=os.path.join(TESTDIR, "config/basic/butler-chained.yaml") ) - tables = self._queryDatasets(repo=self.repoDir, show_uri=True) + tables = self._queryDatasets(repo=testRepo.butler, show_uri=True) - # Want second datastore root. + # Want second datastore root since in-memory is ephemeral and + # all the relevant datasets are stored in the second as well as third + # datastore. roots = testRepo.butler.get_datastore_roots() datastore_root = roots[testRepo.butler.get_datastore_names()[1]] @@ -190,7 +192,7 @@ def testShowURI(self): """Test for expected output with show_uri=True.""" testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) - tables = self._queryDatasets(repo=self.repoDir, show_uri=True) + tables = self._queryDatasets(repo=testRepo.butler, show_uri=True) roots = testRepo.butler.get_datastore_roots() datastore_root = list(roots.values())[0] @@ -201,9 +203,9 @@ def testShowURI(self): def testNoShowURI(self): """Test for expected output without show_uri (default is False).""" - _ = MetricTestRepo(self.repoDir, configFile=self.configFile) + testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) - tables = self._queryDatasets(repo=self.repoDir) + tables = self._queryDatasets(repo=testRepo.butler) expectedTables = ( AstropyTable( @@ -223,9 +225,9 @@ def testWhere(self): """Test using the where clause to reduce the number of rows returned by queryDatasets. """ - _ = MetricTestRepo(self.repoDir, configFile=self.configFile) + testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) - tables = self._queryDatasets(repo=self.repoDir, where="instrument='DummyCamComp' AND visit=423") + tables = self._queryDatasets(repo=testRepo.butler, where="instrument='DummyCamComp' AND visit=423") expectedTables = ( AstropyTable( @@ -237,7 +239,7 @@ def testWhere(self): self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) with self.assertRaises(RuntimeError): - self._queryDatasets(repo=self.repoDir, collections="*", find_first=True) + self._queryDatasets(repo=testRepo.butler, collections="*", find_first=True) def testGlobDatasetType(self): """Test specifying dataset type.""" @@ -262,7 +264,7 @@ def testGlobDatasetType(self): testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType) # verify the new dataset type increases the number of tables found: - tables = self._queryDatasets(repo=self.repoDir) + tables = self._queryDatasets(repo=testRepo.butler) expectedTables = ( AstropyTable( @@ -285,10 +287,10 @@ def testGlobDatasetType(self): def test_limit_order(self): """Test limit and ordering.""" # Create and register an additional DatasetType - MetricTestRepo(self.repoDir, configFile=self.configFile) + testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) with self.assertLogs("lsst.daf.butler.script.queryDatasets", level="WARNING") as cm: - tables = self._queryDatasets(repo=self.repoDir, limit=-1, order_by=("visit")) + tables = self._queryDatasets(repo=testRepo.butler, limit=-1, order_by=("visit")) self.assertIn("increase this limit", cm.output[0]) @@ -301,7 +303,7 @@ def test_limit_order(self): self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) with self.assertLogs("lsst.daf.butler.script.queryDatasets", level="WARNING") as cm: - tables = self._queryDatasets(repo=self.repoDir, limit=-1, order_by=("-visit")) + tables = self._queryDatasets(repo=testRepo.butler, limit=-1, order_by=("-visit")) self.assertIn("increase this limit", cm.output[0]) expectedTables = [ @@ -322,7 +324,7 @@ def testFindFirstAndCollections(self): testRepo.addDataset(run="foo", dataId={"instrument": "DummyCamComp", "visit": 424}) # Verify that without find-first, duplicate datasets are returned - tables = self._queryDatasets(repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True) + tables = self._queryDatasets(repo=testRepo.butler, collections=["foo", "ingest/run"], show_uri=True) # The test should be running with a single FileDatastore. roots = testRepo.butler.get_datastore_roots() @@ -465,7 +467,7 @@ def testFindFirstAndCollections(self): # Verify that with find first the duplicate dataset is eliminated and # the more recent dataset is returned. tables = self._queryDatasets( - repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True, find_first=True + repo=testRepo.butler, collections=["foo", "ingest/run"], show_uri=True, find_first=True ) expectedTables = (