update requirements

.flake8

@@ -4,4 +4,5 @@ extend-ignore = E203, W503
 exclude =
     .git
     __pycache__
-    setup.py
+    setup.py
+    .venv

.github/workflows/cicd.yml

@@ -32,7 +32,7 @@ jobs:
 
       - name: Get image tag
         id: get_image_tag
-        run: 
+        run:
           case "${GITHUB_REF}" in
             *tags*)
               echo "tag=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT ;
@@ -57,7 +57,7 @@ jobs:
       - build_and_publish
     steps:
       - uses: actions/checkout@v3
-      
+
       - name: Log in with Azure
         uses: azure/login@v1
         with:
@@ -86,4 +86,4 @@ jobs:
           ARM_CLIENT_ID: ${{ fromJSON(secrets.SECURE_AZURE_CREDENTIALS).clientId }}
           ARM_SUBSCRIPTION_ID: ${{ fromJSON(secrets.SECURE_AZURE_CREDENTIALS).subscriptionId }}
           ARM_TENANT_ID: ${{ fromJSON(secrets.SECURE_AZURE_CREDENTIALS).tenantId }}
-          ARM_USE_OIDC: true
+          ARM_USE_OIDC: true

deployment/README.md

@@ -10,6 +10,8 @@ The logic for the deployment workflow is encapsulated in the [bin/deploy](bin/de
 scripts/console --deploy
 ```
 
+To have access to the remote backend terraform state, the identity (App Registration in CI, or local corp credential if local) will need to have the `Storage Blob Data Owner` role on the `pctesttfstate` storage account.
+
 ## Manual resources
 
 ### Deployment secrets Key Vault
@@ -40,25 +42,27 @@ Container Registry repo where you published your local images:
 - `ACR_TILER_REPO`
 - `IMAGE_TAG`
 
-__Note:__ Remember to bring down your resources after testing with `terraform destroy`!
+**Note:** Remember to bring down your resources after testing with `terraform destroy`!
 
 ## Loading configuration data
 
 Configuration data is stored in Azure Storage Tables. Use the `pcapis` command line interface that is installed with the `pccommon` package to load data. For example:
 
+```console
+>  az login # Use an account that has "Storage Table Data Contributor" on the account
+>  pcapis load -t collection --account pctapissatyasa  --table collectionconfig --file pccommon/tests/data-files/collection_config.json
 ```
->  pcapis load -t collection --sas "${SAS_TOKEN}" --account pctapissatyasa  --table collectionconfig --file pccommon/tests/data-files/collection_config.json
-```
+
 To dump a single collection config, use:
 
-```
->  pcapis dump -t collection --sas "${SAS_TOKEN}" --account pctapissatyasa  --table collectionconfig --id naip
+```console
+>  pcapis dump -t collection  --account pctapissatyasa  --table collectionconfig --id naip
 ```
 
 For container configs, you must also specify the container account name used as the Partition Key:
 
-```
->  pcapis dump -t collection --sas "${SAS_TOKEN}" --account pctapissatyasa  --table containerconfig --id naip --container-account naipeuwest
+```console
+>  pcapis dump -t collection --account pctapissatyasa  --table containerconfig --id naip --container-account naipeuwest
 ```
 
 Using the `load` command on a single dump file for either config will update the single row.

deployment/bin/deploy

@@ -14,6 +14,7 @@ function usage() {
 Deploys the project infrastructure.
 
 -t TERRAFORM_DIR: The terraform directory. Required.
+-y: Auto approve the terraform changes.
 --plan: Only run Terraform plan.
 --skip-tf: Skips Terraform apply. Will still gather terraform output
 "
@@ -37,6 +38,10 @@ while [[ "$#" -gt 0 ]]; do case $1 in
         PLAN_ONLY=1
         shift
         ;;
+    -y)
+        AUTO_APPROVE=-auto-approve
+        shift
+        ;;
     --help)
         usage
         exit 0
@@ -49,10 +54,29 @@ while [[ "$#" -gt 0 ]]; do case $1 in
         ;;
     esac done
 
+# Always disable shared access keys on script exit
+trap disable_shared_access_keys EXIT
+
 ###################################
 # Check and configure environment #
 ###################################
 
+# Enable shared access keys on storage accounts that must have properties read
+# [storage_account]=resource_group
+declare -A SAK_STORAGE_ACCOUNTS
+SAK_STORAGE_ACCOUNTS=(
+  ["pctapisstagingsa"]="pct-apis-westeurope-staging_rg"
+  ["pcfilestest"]="pc-test-manual-resources"
+)
+
+# Add client IP to firewall for storage accounts that must have properties read
+# [storage_account]=resource_group
+declare -A FW_STORAGE_ACCOUNTS
+FW_STORAGE_ACCOUNTS=(
+    ["pctesttfstate"]="pc-test-manual-resources"
+    ["pctapisstagingsa"]="pct-apis-westeurope-staging_rg"
+)
+
 if [[ -z ${TERRAFORM_DIR} ]]; then
     echo "Must pass in TERRAFORM_DIR with -t"
     exit 1
@@ -73,15 +97,21 @@ setup_env
 echo "===== Running Deploy ====="
 echo "IMAGE_TAG: ${IMAGE_TAG}"
 
+if [ -z "$ARM_CLIENT_ID" ]; then
+  export ARM_CLIENT_ID=$(az account show --query user.name -o tsv)
+  echo "Using Azure CLI auth with username: ${ARM_CLIENT_ID}"
+fi
+
+
 # ---------------------------------------------------
 
 if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
 
     #########################
-    # Add IP to KV firewall #
+    # Add IP to firewalls #
     #########################
 
-    bin/kv_add_ip
+    add_ip_to_firewalls
 
     #####################
     # Deploy Terraform  #
@@ -91,14 +121,17 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
 
     if [[ "${SKIP_TF}" != 1 ]]; then
         echo "Deploying infrastructure with Terraform..."
+
+        enable_shared_access_keys
+
         terraform init --upgrade
 
         if [ "${PLAN_ONLY}" ]; then
             terraform plan
             exit 0
         fi
 
-        terraform apply -auto-approve
+        terraform apply "$AUTO_APPROVE"
     fi
 
     # Gather terraform output
@@ -107,10 +140,10 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
     popd
 
     ##############################
-    # Remove IP from KV firewall #
+    # Remove IP from firewalls #
     ##############################
 
-    bin/kv_rmv_ip
+    remove_ip_from_firewalls
 
     ############################
     # Render Helm chart values #
@@ -142,7 +175,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
         --kube-context "${KUBE_CONTEXT}" \
         --wait \
         --timeout 2m0s \
-        -f ${DEPLOY_VALUES_FILE}
+        -f ${DEPLOY_VALUES_FILE} \
 
     echo "================"
     echo "==== Tiler ====="
@@ -154,7 +187,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
         --kube-context "${KUBE_CONTEXT}" \
         --wait \
         --timeout 2m0s \
-        -f ${DEPLOY_VALUES_FILE}
+        -f ${DEPLOY_VALUES_FILE} \
 
     echo "=================="
     echo "==== Ingress ====="

deployment/bin/lib

@@ -131,3 +131,96 @@ function get_cidr_range() {
     IFS='.' read -r -a ip_parts <<< "$runnerIpAddress"
     echo "${ip_parts[0]}.${ip_parts[1]}.0.0/16"
 }
+
+function disable_shared_access_keys() {
+    echo "Disabling shared access key on storage account..."
+
+    for SAK_STORAGE_ACCOUNT in "${!SAK_STORAGE_ACCOUNTS[@]}"; do
+        SAK_RESOURCE_GROUP=${SAK_STORAGE_ACCOUNTS[$SAK_STORAGE_ACCOUNT]}
+
+        az storage account update \
+            --name ${SAK_STORAGE_ACCOUNT} \
+            --resource-group ${SAK_RESOURCE_GROUP} \
+            --allow-shared-key-access false \
+            --subscription ${ARM_SUBSCRIPTION_ID} \
+            --output none
+
+        if [ $? -ne 0 ]; then
+            echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+            echo "WARNING: Failed to turn off shared key access on the storage account."
+            echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+            exit 2
+        fi
+    done
+}
+
+function enable_shared_access_keys() {
+    # Terraform isn't able to read all resources from a storage account if shared key access is disabled
+    # so while we're deploying, we need to enable it. Since we haven't run TF yet, we don't have the name of the account
+    # so they are hardcoded here. This is a temporary workaround until this is resolved
+    # https://github.com/hashicorp/terraform-provider-azurerm/issues/25218
+
+    echo "Enabling shared key access for storage accounts..."
+    for SAK_STORAGE_ACCOUNT in "${!SAK_STORAGE_ACCOUNTS[@]}"; do
+        SAK_RESOURCE_GROUP=${SAK_STORAGE_ACCOUNTS[$SAK_STORAGE_ACCOUNT]}
+
+        echo "   - ${SAK_RESOURCE_GROUP}.${SAK_STORAGE_ACCOUNT}"
+        az storage account update \
+            --name ${SAK_STORAGE_ACCOUNT} \
+            --resource-group ${SAK_RESOURCE_GROUP} \
+            --allow-shared-key-access true \
+            --subscription ${ARM_SUBSCRIPTION_ID} \
+            --output none
+    done
+
+    sleep 10
+}
+
+function add_ip_to_firewalls() {
+    cidr=$(get_cidr_range)
+
+    echo "Adding IP $cidr to Key Vault firewall allow list..."
+    az keyvault network-rule add \
+        -g "${KEY_VAULT_RESOURCE_GROUP_NAME}"  \
+        -n "${KEY_VAULT_NAME}" \
+        --ip-address "$cidr" \
+        --subscription "${ARM_SUBSCRIPTION_ID}" \
+        --output none
+
+    # Also add the IP to the terraform state storage account
+    for FW_STORAGE_ACCOUNT in "${!FW_STORAGE_ACCOUNTS[@]}"; do
+        FW_RESOURCE_GROUP=${FW_STORAGE_ACCOUNTS[$FW_STORAGE_ACCOUNT]}
+        echo "Adding IP $cidr to ${FW_STORAGE_ACCOUNT} Storage firewall allow list..."
+        az storage account network-rule add \
+            -g "${FW_RESOURCE_GROUP}" \
+            -n "${FW_STORAGE_ACCOUNT}" \
+            --ip-address "$cidr" \
+            --subscription "${ARM_SUBSCRIPTION_ID}" \
+            --output none
+    done
+
+    sleep 10
+}
+
+function remove_ip_from_firewalls() {
+    cidr=$(get_cidr_range)
+
+    echo "Removing IP $cidr from Key Vault firewall allow list..."
+    az keyvault network-rule remove \
+        -g ${KEY_VAULT_RESOURCE_GROUP_NAME}  \
+        -n ${KEY_VAULT_NAME} \
+        --ip-address $cidr \
+        --subscription ${ARM_SUBSCRIPTION_ID} \
+        --output none
+
+    for FW_STORAGE_ACCOUNT in "${!FW_STORAGE_ACCOUNTS[@]}"; do
+        FW_RESOURCE_GROUP=${FW_STORAGE_ACCOUNTS[$FW_STORAGE_ACCOUNT]}
+        echo "Removing IP $cidr from ${FW_STORAGE_ACCOUNT} Storage firewall allow list..."
+        az storage account network-rule remove \
+            -g ${FW_RESOURCE_GROUP} \
+            -n ${FW_STORAGE_ACCOUNT} \
+            --ip-address $cidr \
+            --subscription ${ARM_SUBSCRIPTION_ID} \
+            --output none
+    done
+}

deployment/docker-compose.yml

@@ -8,11 +8,11 @@ services:
     environment:
       - ACR_STAC_REPO=${ACR_STAC_REPO:-pccomponentstest.azurecr.io/planetary-computer-apis/stac}
       - ACR_TILER_REPO=${ACR_TILER_REPO:-pccomponentstest.azurecr.io/planetary-computer-apis/tiler}
-      - IMAGE_TAG
+      - IMAGE_TAG=${IMAGE_TAG:-latest}
       - GIT_COMMIT
 
       - ARM_SUBSCRIPTION_ID=${ARM_SUBSCRIPTION_ID:-a84a690d-585b-4c7c-80d9-851a48af5a50}
-      - ARM_TENANT_ID
+      - ARM_TENANT_ID=${ARM_TENANT_ID:-72f988bf-86f1-41af-91ab-2d7cd011db47}
       - ARM_CLIENT_ID
       - ARM_USE_OIDC
       - ARM_OIDC_TOKEN