DO NOT MERGE: Update Runtime to work with Berkeley schema (meta PR)

.env.example

@@ -40,4 +40,9 @@ NERSC_USERNAME=replaceme
 ORCID_NMDC_CLIENT_ID=replaceme
 ORCID_NMDC_CLIENT_SECRET=replaceme
 
+# Base URL (without a trailing slash) at which the Runtime can access an instance of ORCID.
+# Note: For the production instance of ORCID, use: https://orcid.org (default)
+#       For the sandbox instance of ORCID, use: https://sandbox.orcid.org
+ORCID_BASE_URL=https://orcid.org
+
 INFO_BANNER_INNERHTML='Announcement: Something important is about to happen. If you have questions, please contact <a href="mailto:[email protected]">[email protected]</a>.'

README.md

@@ -1,5 +1,30 @@
 A runtime system for NMDC data management and orchestration.
 
+<!-- TODO: Remove this note once we've merged the `berkeley` branch into the `main` branch. -->
+> [!NOTE]
+> You are on a Berkeley-related branch (i.e. a branch related to getting the Runtime to work with the Berkeley schema).
+
+<!-- TODO: Remove this section once we've merged the `berkeley` branch into the `main` branch. -->
+## Berkeley-related branches
+
+### Introduction
+
+The `nmdc-runtime` repository contains a branch named `main` and a branch named `berkeley` (among other branches).
+
+The `main` branch is a permanent branch. It is used the same way as in many Git repositories: it is the base branch of most PRs. Its contents get [built and] deployed to the production environment.
+
+The `berkeley` branch is a long-living (but not permanent) branch. It is the base branch of PRs that introduce changes that make the Runtime work with the Berkeley schema. Its contents get [built and] deployed to the "Berkeley environment."
+
+### Merging the `main` branch into the `berkeley` branch
+
+We want to avoid being in a situation where the `berkeley` branch is lacking features/fixes that have been introduced into the `main` branch; especially when the absence of those features/fixes is affecting NMDC team members working on the Berkeley Schema Roll Out.
+
+To that end, we've instituted the following policy around merging the `main` branch into the `berkeley` branch:
+
+1. Any Runtime maintainer can merge `main` into `berkeley` at any time, provided they address any merge conflicts that arise. We expect there to always be at least one merge conflict: the `nmdc-schema` version in the `main.in` file.
+2. Those merges can happen multiple times a day (no limit).
+3. When those merges happen, the person performing the merge will announce the merge in the `#nmdc-runtime` Slack channel. The announcement can be made either before or after merging. Although not all team members are on that Slack channel, we think enough _are_ that—should someone else report an issue shortly thereafter—news of the recent merge will make its way to them. The announcement can be a normal message (i.e. tagging people in it is optional).
+
 ## Service Status
 
 http://nmdcstatus.polyneme.xyz/

docs/nb/bulk_validation_referential_integrity_check.ipynb

@@ -37,7 +37,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "localhost:27018\n"
+      "mongodb://localhost:27018\n"
      ]
     }
    ],
@@ -93,7 +93,7 @@
     {
      "data": {
       "text/plain": [
-       "'10.7.0'"
+       "'11.0.0rc22'"
       ]
      },
      "execution_count": 3,
@@ -126,8 +126,8 @@
     "from tqdm.notebook import tqdm\n",
     "\n",
     "from nmdc_runtime.api.core.util import pick\n",
-    "from nmdc_runtime.api.db.mongo import get_mongo_db, nmdc_schema_collection_names, get_collection_names_from_schema\n",
-    "from nmdc_runtime.util import collection_name_to_class_names, nmdc_schema_view, nmdc_database_collection_instance_class_names, get_nmdc_jsonschema_dict\n",
+    "from nmdc_runtime.api.db.mongo import get_mongo_db, get_nonempty_nmdc_schema_collection_names, get_collection_names_from_schema\n",
+    "from nmdc_runtime.util import collection_name_to_class_names, populated_schema_collection_names_with_id_field, nmdc_schema_view, nmdc_database_collection_instance_class_names, get_nmdc_jsonschema_dict\n",
     "from nmdc_schema.nmdc import Database as NMDCDatabase \n",
     "from nmdc_schema.get_nmdc_view import ViewGetter\n",
     "\n",
@@ -156,9 +156,18 @@
    "execution_count": 5,
    "id": "1d76b70e-4412-4b17-9db9-322ac791859a",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'study_set', 'workflow_execution_set', 'material_processing_set', 'instrument_set', 'data_object_set', 'configuration_set', 'biosample_set', 'functional_annotation_agg', 'calibration_set', 'processed_sample_set', 'field_research_site_set', 'data_generation_set'}\n"
+     ]
+    }
+   ],
    "source": [
-    "collection_names = sorted(nmdc_schema_collection_names(mdb))"
+    "collection_names = get_nonempty_nmdc_schema_collection_names(mdb)\n",
+    "print(collection_names)"
    ]
   },
   {
@@ -279,20 +288,28 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "95817da9de0a4934b5e3683f2f81893e",
+       "model_id": "6c88577a3a9342808d3bbc0e3707a95a",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
-       "  0%|          | 0/9601505 [00:00<?, ?it/s]"
+       "  0%|          | 0/2351449 [00:00<?, ?it/s]"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'bad_type': [], 'no_type': [], 'bad_slot': [], 'is_null': []}\n"
+     ]
     }
    ],
    "source": [
-    "errors = collect_errors(note_doc_field_errors)"
+    "errors = collect_errors(note_doc_field_errors)\n",
+    "print(errors)"
    ]
   },
   {
@@ -363,12 +380,12 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "0a5d8aabad3a43448826525e77820b76",
+       "model_id": "f63a4ce942bc4278b3e99a5a87b0155c",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
-       "  0%|          | 0/9601505 [00:00<?, ?it/s]"
+       "  0%|          | 0/2351449 [00:00<?, ?it/s]"
       ]
      },
      "metadata": {},
@@ -486,7 +503,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 35,
+   "execution_count": 15,
    "id": "103d70b6-24ab-41bd-8b7f-d2faaa028bdf",
    "metadata": {
     "scrolled": true
@@ -495,12 +512,12 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "dc9d77f96c9548c4adf28e124c99d8bf",
+       "model_id": "c8c75b6bd622470f9ded8e3813fc1d64",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
-       "  0%|          | 0/9601505 [00:00<?, ?it/s]"
+       "  0%|          | 0/3039449 [00:00<?, ?it/s]"
       ]
      },
      "metadata": {},
@@ -559,17 +576,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 36,
+   "execution_count": 16,
    "id": "e01450d1-3369-4fc5-80be-9787e00a6597",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "(0, 3157)"
+       "(5, 45604)"
       ]
      },
-     "execution_count": 36,
+     "execution_count": 16,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -591,17 +608,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 37,
+   "execution_count": 17,
    "id": "afd25543-1cb3-4887-9aba-0086d4b998a6",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "set()"
+       "{'nmdc:dobj-11-cvcxxr53', 'nmdc:dobj-11-fg28a080', 'nmdc:dobj-11-gxgpbv06'}"
       ]
      },
-     "execution_count": 37,
+     "execution_count": 17,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -612,17 +629,41 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 38,
+   "execution_count": 18,
    "id": "a25857f4-e26e-4896-9e5f-607e7b4bb07c",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "[]"
+       "[{'id': 'nmdc:wfmgan-11-w1d6gy98.1',\n",
+       "  'id_is_nmdc_id': True,\n",
+       "  'field': 'has_input',\n",
+       "  'value': 'nmdc:dobj-11-cvcxxr53',\n",
+       "  'slot_range': 'NamedThing'},\n",
+       " {'id': 'nmdc:wfmgan-11-fmymf551.1',\n",
+       "  'id_is_nmdc_id': True,\n",
+       "  'field': 'has_input',\n",
+       "  'value': 'nmdc:dobj-11-fg28a080',\n",
+       "  'slot_range': 'NamedThing'},\n",
+       " {'id': 'nmdc:wfmgan-11-3nkefn97.1',\n",
+       "  'id_is_nmdc_id': True,\n",
+       "  'field': 'has_input',\n",
+       "  'value': 'nmdc:dobj-11-gxgpbv06',\n",
+       "  'slot_range': 'NamedThing'},\n",
+       " {'id': 'nmdc:wfmgan-11-fmymf551.1',\n",
+       "  'id_is_nmdc_id': True,\n",
+       "  'field': 'has_input',\n",
+       "  'value': 'nmdc:dobj-11-fg28a080',\n",
+       "  'slot_range': 'NamedThing'},\n",
+       " {'id': 'nmdc:wfmgan-11-3nkefn97.1',\n",
+       "  'id_is_nmdc_id': True,\n",
+       "  'field': 'has_input',\n",
+       "  'value': 'nmdc:dobj-11-gxgpbv06',\n",
+       "  'slot_range': 'NamedThing'}]"
       ]
      },
-     "execution_count": 38,
+     "execution_count": 18,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -641,16 +682,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 39,
+   "execution_count": 19,
    "id": "33516e3c-f10d-4c30-942b-0d01d06082f9",
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "{'id': 'nmdc:dobj-11-1epz0d53', 'id_is_nmdc_id': True, 'field': 'was_generated_by', 'value': 'nmdc:omprc-11-sxze4w22', 'slot_range': 'Activity'}\n",
-      "{'id': 'nmdc:libprp-11-f6kv1904', 'id_is_nmdc_id': True, 'field': 'has_input', 'value': 'nmdc:procsm-11-v5sykd35', 'slot_range': 'Biosample'}\n"
+      "{'id': 'nmdc:dobj-11-xt088e26', 'id_is_nmdc_id': True, 'field': 'was_generated_by', 'value': 'nmdc:omprc-11-ymxzx274', 'slot_range': 'WorkflowExecution'}\n"
      ]
     }
    ],
@@ -673,22 +713,26 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 41,
+   "execution_count": 20,
    "id": "29ec7e82-d079-4525-bd7b-d770fd69d788",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "{'_id': ObjectId('66a8f648b3ed1b2200050335'),\n",
+       "{'_id': ObjectId('66edad78007ef07eb670a09d'),\n",
        " 'id': 'nmdc:omprc-11-sxze4w22',\n",
        " 'has_input': ['nmdc:bsm-11-978cs285'],\n",
        " 'has_output': ['nmdc:dobj-11-1epz0d53'],\n",
-       " 'part_of': ['nmdc:sty-11-28tm5d36'],\n",
-       " 'type': ['OmicsProcessing', 'PlannedProcess', 'NamedThing']}"
+       " 'associated_studies': ['nmdc:sty-11-28tm5d36'],\n",
+       " 'instrument_used': ['nmdc:inst-14-mwrrj632'],\n",
+       " 'type': ['MassSpectrometry',\n",
+       "  'DataGeneration',\n",
+       "  'PlannedProcess',\n",
+       "  'NamedThing']}"
       ]
      },
-     "execution_count": 41,
+     "execution_count": 20,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -700,19 +744,19 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 40,
+   "execution_count": 21,
    "id": "802290e0-58dd-4fbd-835a-c9928006819d",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "{'_id': ObjectId('66a8f648b3ed1b2200051041'),\n",
+       "{'_id': ObjectId('66edad78007ef07eb67078c8'),\n",
        " 'id': 'nmdc:procsm-11-v5sykd35',\n",
        " 'type': ['ProcessedSample', 'MaterialEntity', 'NamedThing']}"
       ]
      },
-     "execution_count": 40,
+     "execution_count": 21,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -725,9 +769,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "nmdc-runtime",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
-   "name": "nmdc-runtime"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
@@ -739,7 +783,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.12"
+   "version": "3.10.14"
   }
  },
  "nbformat": 4,

nmdc_runtime/api/core/auth.py

@@ -21,20 +21,39 @@
 from starlette.requests import Request
 from starlette.status import HTTP_400_BAD_REQUEST, HTTP_401_UNAUTHORIZED
 
+ORCID_PRODUCTION_BASE_URL = "https://orcid.org"
+
 SECRET_KEY = os.getenv("JWT_SECRET_KEY")
 ALGORITHM = "HS256"
 ORCID_NMDC_CLIENT_ID = os.getenv("ORCID_NMDC_CLIENT_ID")
 ORCID_NMDC_CLIENT_SECRET = os.getenv("ORCID_NMDC_CLIENT_SECRET")
-
-# https://orcid.org/.well-known/openid-configuration
-# XXX do we want to live-load this?
-ORCID_JWK = {  # https://orcid.org/oauth/jwks
+ORCID_BASE_URL = os.getenv("ORCID_BASE_URL", default=ORCID_PRODUCTION_BASE_URL)
+
+# Define the JSON Web Key Set (JWKS) for ORCID.
+#
+# Note: The URL from which we got this dictionary is: https://orcid.org/oauth/jwks
+#       We got _that_ URL from the dictionary at: https://orcid.org/.well-known/openid-configuration
+#
+# TODO: Consider _live-loading_ this dictionary from the Internet.
+#
+ORCID_JWK = {
     "e": "AQAB",
     "kid": "production-orcid-org-7hdmdswarosg3gjujo8agwtazgkp1ojs",
     "kty": "RSA",
     "n": "jxTIntA7YvdfnYkLSN4wk__E2zf_wbb0SV_HLHFvh6a9ENVRD1_rHK0EijlBzikb-1rgDQihJETcgBLsMoZVQqGj8fDUUuxnVHsuGav_bf41PA7E_58HXKPrB2C0cON41f7K3o9TStKpVJOSXBrRWURmNQ64qnSSryn1nCxMzXpaw7VUo409ohybbvN6ngxVy4QR2NCC7Fr0QVdtapxD7zdlwx6lEwGemuqs_oG5oDtrRuRgeOHmRps2R6gG5oc-JqVMrVRv6F9h4ja3UgxCDBQjOVT1BFPWmMHnHCsVYLqbbXkZUfvP2sO1dJiYd_zrQhi-FtNth9qrLLv3gkgtwQ",
     "use": "sig",
 }
+# If the application is using a _non-production_ ORCID environment, overwrite
+# the "kid" and "n" values with those from the sandbox ORCID environment.
+#
+# Source: https://sandbox.orcid.org/oauth/jwks
+#
+if ORCID_BASE_URL != ORCID_PRODUCTION_BASE_URL:
+    ORCID_JWK["kid"] = "sandbox-orcid-org-3hpgosl3b6lapenh1ewsgdob3fawepoj"
+    ORCID_JWK["n"] = (
+        "pl-jp-kTAGf6BZUrWIYUJTvqqMVd4iAnoLS6vve-KNV0q8TxKvMre7oi9IulDcqTuJ1alHrZAIVlgrgFn88MKirZuTqHG6LCtEsr7qGD9XyVcz64oXrb9vx4FO9tLNQxvdnIWCIwyPAYWtPMHMSSD5oEVUtVL_5IaxfCJvU-FchdHiwfxvXMWmA-i3mcEEe9zggag2vUPPIqUwbPVUFNj2hE7UsZbasuIToEMFRZqSB6juc9zv6PEUueQ5hAJCEylTkzMwyBMibrt04TmtZk2w9DfKJR91555s2ZMstX4G_su1_FqQ6p9vgcuLQ6tCtrW77tta-Rw7McF_tyPmvnhQ"
+    )
+
 ORCID_JWS_VERITY_ALGORITHM = "RS256"