Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/main' into 188-the-frequency-in-…
Browse files Browse the repository at this point in the history
…the-intake-catalog-does-not-account-for-snapshot-data
  • Loading branch information
marc-white committed Sep 30, 2024
2 parents 12f0ddf + 06a9d42 commit 61ff278
Show file tree
Hide file tree
Showing 16 changed files with 579 additions and 42 deletions.
8 changes: 4 additions & 4 deletions config/cmip5.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@ translator: Cmip5Translator

sources:

- metadata_yaml: /g/data/tm70/intake/metadata/cmip5_al33/metadata.yaml
- metadata_yaml: /g/data/xp65/admin/intake/metadata/cmip5_al33/metadata.yaml
path:
- /g/data/dk92/catalog/v2/esm/cmip5-al33/catalog.json
- /g/data/al33/catalog/v2/esm/catalog.json

- metadata_yaml: /g/data/tm70/intake/metadata/cmip5_rr3/metadata.yaml
- metadata_yaml: /g/data/xp65/admin/intake/metadata/cmip5_rr3/metadata.yaml
path:
- /g/data/dk92/catalog/v2/esm/cmip5-rr3/catalog.json
- /g/data/rr3/catalog/v2/esm/catalog.json
8 changes: 4 additions & 4 deletions config/cmip6.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@ translator: Cmip6Translator

sources:

- metadata_yaml: /g/data/tm70/intake/metadata/cmip6_fs38/metadata.yaml
- metadata_yaml: /g/data/xp65/admin/intake/metadata/cmip6_fs38/metadata.yaml
path:
- /g/data/dk92/catalog/v2/esm/cmip6-fs38/catalog.json
- /g/data/fs38/catalog/v2/esm/catalog.json

- metadata_yaml: /g/data/tm70/intake/metadata/cmip6_oi10/metadata.yaml
- metadata_yaml: /g/data/xp65/admin/intake/metadata/cmip6_oi10/metadata.yaml
path:
- /g/data/dk92/catalog/v2/esm/cmip6-oi10/catalog.json
- /g/data/oi10/catalog/v2/esm/catalog.json
52 changes: 52 additions & 0 deletions config/experiments/cmip5-al33/metadata.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
# Metadata recovered 2024-09-25
# {'contact': 'NCI',
# 'created': None,
# 'description': 'Replicated CMIP5-era datasets catalogued by NCI',
# 'email': '[email protected]',
# 'experiment_uuid': '658c95cc-c299-450c-82a1-b2b2308f7c6e',
# 'keywords': ['cmip'],
# 'license': None,
# 'long_description': 'All CMIP5-era replicated data contained under the project al33. All file versions present are in the listing. Maintained By: NCI Contact: [email protected] References: https://pcmdi.llnl.gov/mips/cmip5/',
# 'model': ['CMIP5'],
# 'name': 'cmip5_al33',
# 'nominal_resolution': [None],
# 'notes': 'null',
# 'parent_experiment': None,
# 'reference': None,
# 'related_experiments': [None],
# 'url': 'https://geonetwork.nci.org.au/geonetwork/srv/eng/catalog.search#/metadata/f9489_5106_5649_5038',
# 'version': None,
# 'catalog_dir': ''}

name: cmip5_al33
experiment_uuid: 658c95cc-c299-450c-82a1-b2b2308f7c6e
description: Replicated CMIP5-era datasets catalogued by NCI
long_description: >-
All CMIP5-era replicated data contained under the project al33.
All file versions present are in the listing.
Maintained By: NCI
Contact: [email protected]
References: https://pcmdi.llnl.gov/mips/cmip5/
model:
- CMIP5
realm:
- atmos
- ocean
- aerosol
- ocnBgchem
- land
- seaIce
- landIce
- landonly
version:
contact: NCI
email: [email protected]
created:
reference:
license:
url: https://geonetwork.nci.org.au/geonetwork/srv/eng/catalog.search#/metadata/f9489_5106_5649_5038
parent_experiment:
related_experiments:
-
notes:
- cmip
60 changes: 60 additions & 0 deletions config/experiments/cmip5-rr3/metadata.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
# Metadata recovered 2024-09-25
# {'contact': 'NCI',
# 'created': None,
# 'description': 'Australian CMIP5-era datasets catalogued by NCI',
# 'email': '[email protected]',
# 'experiment_uuid': '473d0c44-ab66-458c-b32e-1e1774175853',
# 'keywords': ['cmip'],
# 'license': None,
# 'long_description': 'All CMIP5-era Australian published data contained under the project rr3. All file versions present are in the listing. Maintained By: NCI Contact: [email protected] References: https://pcmdi.llnl.gov/mips/cmip5/',
# 'model': ['CMIP5'],
# 'name': 'cmip5_rr3',
# 'nominal_resolution': [None],
# 'notes': 'null',
# 'parent_experiment': None,
# 'reference': None,
# 'related_experiments': [None],
# 'url': 'https://geonetwork.nci.org.au/geonetwork/srv/eng/catalog.search#/metadata/f7448_2157_9857_1076',
# 'version': None,
# 'catalog_dir': ''}
#
name: cmip5_rr3
experiment_uuid: 473d0c44-ab66-458c-b32e-1e1774175853
description: Australian CMIP5-era datasets catalogued by NCI
long_description: >-
All CMIP5-era Australian published data contained under the project rr3.
All file versions present are in the listing.
Maintained By: NCI
Contact: [email protected]
References: https://pcmdi.llnl.gov/mips/cmip5/
realm:
- atmos
- ocean
- seaIce
- landIce
- land
- aerosol
model:
- CMIP5
frequency:
- 1hr
- mon
- day
- 6hr
- sem
- 3hr
- fx
nominal_resolution:
-
version:
contact: NCI
email: [email protected]
created:
reference:
license:
url: https://geonetwork.nci.org.au/geonetwork/srv/eng/catalog.search#/metadata/f7448_2157_9857_1076
related_experiments:
-
notes:
keywords:
- cmip
54 changes: 54 additions & 0 deletions config/experiments/cmip6-fs38/metadata.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
name: cmip6_fs38
experiment_uuid: dfdeb421-5c56-4d58-a0b2-04b717e5cff7
description: Australian CMIP6-era datasets catalogued by NCI
long_description: >-
All CMIP6-era Australian published data contained under the project fs38.
All file versions present are in the listing.
Maintained By: NCI
Contact: [email protected]
References: https://pcmdi.llnl.gov/CMIP6/
model:
- CMIP6
realm:
- ocnBgchem
- land
- ocean
- atmos
- landIce
- seaIce
- aerosol
nominal_resolution:
-
version:
contact: NCI
email: [email protected]
created:
reference:
license:
url: https://geonetwork.nci.org.au/geonetwork/srv/eng/catalog.search#/metadata/f3154_9976_7262_7595
parent_experiment:
related_experiments:
-
notes:
keywords:
- cmip

# Metadata recovered 2024-09-25
# {'contact': 'NCI',
# 'created': None,
# 'description': 'Australian CMIP6-era datasets catalogued by NCI',
# 'email': '[email protected]',
# 'experiment_uuid': 'dfdeb421-5c56-4d58-a0b2-04b717e5cff7',
# 'keywords': ['cmip'],
# 'license': None,
# 'long_description': 'All CMIP6-era Australian published data contained under the project fs38. All file versions present are in the listing. Maintained By: NCI Contact: [email protected] References: https://pcmdi.llnl.gov/CMIP6/',
# 'model': ['CMIP6'],
# 'name': 'cmip6_fs38',
# 'nominal_resolution': [None],
# 'notes': 'null',
# 'parent_experiment': None,
# 'reference': None,
# 'related_experiments': [None],
# 'url': 'https://geonetwork.nci.org.au/geonetwork/srv/eng/catalog.search#/metadata/f3154_9976_7262_7595',
# 'version': None,
# 'catalog_dir': ''}
61 changes: 61 additions & 0 deletions config/experiments/cmip6-oi10/metadata.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
name: cmip6_oi10
experiment_uuid: b05038ca-8c78-4ca6-a914-ae33dd9abffe
description: Replicated CMIP6-era datasets catalogued by NCI
long_description: >-
All CMIP6-era replicated data contained under the project oi10.
All file versions present are in the listing.
Maintained By: NCI
Contact: [email protected]
References: https://pcmdi.llnl.gov/CMIP6/
model:
- CMIP6
realm:
- atmos
- ocean
- land
- seaIce
- aerosol
- landIce
- ocnBgchem
- seaIce ocean
- atmos land
- ocnBgChem
- landIce land
- seaice
- ocean seaIce
- atmos atmosChem
nominal_resolution:
-
version:
contact: NCI
email: [email protected]
created:
reference:
license:
url: https://geonetwork.nci.org.au/geonetwork/srv/eng/catalog.search#/metadata/f5194_5909_8003_9216
parent_experiment:
related_experiments:
-
notes:
keywords:
- cmip

# Metadata recovered 2024-09-25
# {'contact': 'NCI',
# 'created': None,
# 'description': 'Replicated CMIP6-era datasets catalogued by NCI',
# 'email': '[email protected]',
# 'experiment_uuid': 'b05038ca-8c78-4ca6-a914-ae33dd9abffe',
# 'keywords': ['cmip'],
# 'license': None,
# 'long_description': 'All CMIP6-era replicated data contained under the project oi10. All file versions present are in the listing. Maintained By: NCI Contact: [email protected] References: https://pcmdi.llnl.gov/CMIP6/',
# 'model': ['CMIP6'],
# 'name': 'cmip6_oi10',
# 'nominal_resolution': [None],
# 'notes': 'null',
# 'parent_experiment': None,
# 'reference': None,
# 'related_experiments': [None],
# 'url': 'https://geonetwork.nci.org.au/geonetwork/srv/eng/catalog.search#/metadata/f5194_5909_8003_9216',
# 'version': None,
# 'catalog_dir': ''}
6 changes: 5 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,11 @@ requires = [
where = ["src"]

[tool.setuptools.package-data]
access_nri_intake = ["data/catalog.yaml"]
access_nri_intake = [
"data/catalog.yaml",
"data/metadata_schema_experiment.json",
"data/metadata_schema_file.json",
]

[tool.versioneer]
VCS = "git"
Expand Down
4 changes: 1 addition & 3 deletions src/access_nri_intake/catalog/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,9 @@
NAME_COLUMN = "name"
TRANSLATOR_GROUPBY_COLUMNS = ["model", "realm", "frequency"]

SCHEMA_URL = "https://raw.githubusercontent.com/ACCESS-NRI/schema/e9055da95093ec2faa555c090fc5af17923d1566/au.org.access-nri/model/output/experiment-metadata/1-0-2.json"
SCHEMA_HASH = "ecb72c1adde3679896ceeca96aa6500d07ea2e05810155ec7a5dc301593c1dc7"

EXP_JSONSCHEMA, CATALOG_JSONSCHEMA = get_jsonschema(
url=SCHEMA_URL, known_hash=SCHEMA_HASH, required=CORE_COLUMNS
metadata_file="data/metadata_schema_experiment.json", required=CORE_COLUMNS
)

COLUMNS_WITH_ITERABLES = [
Expand Down
5 changes: 2 additions & 3 deletions src/access_nri_intake/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from .catalog import EXP_JSONSCHEMA, translators
from .catalog.manager import CatalogManager
from .source import builders
from .utils import _can_be_array, load_metadata_yaml
from .utils import _can_be_array, get_catalog_fp, load_metadata_yaml


class MetadataCheckError(Exception):
Expand Down Expand Up @@ -214,9 +214,8 @@ def _get_project(path):
# Save the catalog
cm.save()

_here = os.path.abspath(os.path.dirname(__file__))
if update:
with open(os.path.join(_here, "data", "catalog.yaml"), "w") as fobj:
with get_catalog_fp().open(mode="w") as fobj:
yaml.dump(yaml_dict, fobj)


Expand Down
7 changes: 3 additions & 4 deletions src/access_nri_intake/data/__init__.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
# Copyright 2023 ACCESS-NRI and contributors. See the top-level COPYRIGHT file for details.
# SPDX-License-Identifier: Apache-2.0

import os

import intake

_here = os.path.abspath(os.path.dirname(__file__))
data = intake.open_catalog(os.path.join(_here, "catalog.yaml")).access_nri
from access_nri_intake.utils import get_catalog_fp

data = intake.open_catalog(get_catalog_fp()).access_nri
Loading

0 comments on commit 61ff278

Please sign in to comment.