diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0964d452..34bb207a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -222,19 +222,18 @@ publish_container: - /deployment-tool/deploy.sh --deployment-plan ${CLOUD} --action create --output-folder env_metadata/${CLOUD}_${TF_VAR_DOMAIN}/${VERSION_PATH} # Collecting deployment metadata - grep "\-\-\-" /deployment-tool/plans/${CLOUD}/user_data.tpl -A 150 > env_metadata/${CLOUD}_${TF_VAR_DOMAIN}/${VERSION_PATH}/f5-bigip-runtime-declaration.yaml - - MGMT_IP=$(cat env_metadata/${CLOUD}_${TF_VAR_DOMAIN}/${VERSION_PATH}/deployment_info.json | jq .instances[].mgmt_address -r) - - USERNAME=$(cat env_metadata/${CLOUD}_${TF_VAR_DOMAIN}/${VERSION_PATH}/deployment_info.json | jq .instances[].admin_username -r) - - PASSWORD=$(cat env_metadata/${CLOUD}_${TF_VAR_DOMAIN}/${VERSION_PATH}/deployment_info.json | jq .instances[].admin_password -r) - - DEPLOYMENT_ID=$(cat env_metadata/${CLOUD}_${TF_VAR_DOMAIN}/${VERSION_PATH}/deployment_info.json | jq .deploymentId -r) - - SECRET_ID=$(cat env_metadata/${CLOUD}_${TF_VAR_DOMAIN}/${VERSION_PATH}/deployment_info.json | jq .secret_id -r) - - NAME=$(cat package.json | jq .name -r) - - VERSION=$(cat package.json | jq -r ".version") - - RELEASE=$(cat package.json | jq -r ".release") - # dhcp must be disalbe to workaround DO bug https://github.com/F5Networks/f5-declarative-onboarding/issues/129 - - sshpass -p $PASSWORD ssh -o StrictHostKeyChecking=no $USERNAME@$MGMT_IP "modify sys global-settings mgmt-dhcp disabled" - - sshpass -p $PASSWORD ssh -o StrictHostKeyChecking=no $USERNAME@$MGMT_IP "save sys config" - # end of workaround - # Installing RPM and executing + - export MGMT_IP=$(cat env_metadata/${CLOUD}_${TF_VAR_DOMAIN}/${VERSION_PATH}/deployment_info.json | jq .instances[].mgmt_address -r) + - export USERNAME=$(cat env_metadata/${CLOUD}_${TF_VAR_DOMAIN}/${VERSION_PATH}/deployment_info.json | jq .instances[].admin_username -r) + - export PASSWORD=$(cat env_metadata/${CLOUD}_${TF_VAR_DOMAIN}/${VERSION_PATH}/deployment_info.json | jq .instances[].admin_password -r) + - export DEPLOYMENT_ID=$(cat env_metadata/${CLOUD}_${TF_VAR_DOMAIN}/${VERSION_PATH}/deployment_info.json | jq .deploymentId -r) + - export SECRET_ID=$(cat env_metadata/${CLOUD}_${TF_VAR_DOMAIN}/${VERSION_PATH}/deployment_info.json | jq .secret_id -r) + - export NAME=$(cat package.json | jq .name -r) + - export VERSION=$(cat package.json | jq -r ".version") + - export RELEASE=$(cat package.json | jq -r ".release") + - echo "Verifying that BIGIP is ready to accept commands" + - bash tests/scripts/verify_bash_available.sh + - if [[ $? != 0 ]]; then exit 1; fi + - echo "Installing RPM and executing" - sshpass -p $PASSWORD scp -o StrictHostKeyChecking=no dist/f5-bigip-runtime-init-$VERSION-$RELEASE.gz.run $USERNAME@$MGMT_IP:/var/tmp/ # Workaround: copying over declaration to support bigip version with no user-data support - sed "s/\${deployment_id}/$DEPLOYMENT_ID/g" env_metadata/${CLOUD}_${TF_VAR_DOMAIN}/${VERSION_PATH}/f5-bigip-runtime-declaration.yaml > env_metadata/${CLOUD}_${TF_VAR_DOMAIN}/${VERSION_PATH}/f5-bigip-runtime-declaration_replaced01.yaml diff --git a/README.md b/README.md index 7ebf25aa..efa5290f 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ - [Azure (Terraform) snippet](#azure-terraform-snippet) - [AWS (Terraform) snippet](#aws-terraform-snippet) - [GCP (Terraform) snippet](#gcp-terraform-snippet) + - [Runtime parameters](#runtime-parameters) - [Private Environments](#private-environments) - [Troubleshooting](#troubleshooting) - [F5 Automation Toolchain Components](#f5-automation-toolchain-components) @@ -57,7 +58,7 @@ resulting in a complete overlay deployment tool for configuring a BIG-IP instanc From a high level overview, using this tool involves three steps: -- Step 1: Download OR render inline a runtime-init configuration file (runtime-init-conf.yaml). +- Step 1: Download OR render inline a Runtime Init configuration file (runtime-init-conf.yaml). ```sh curl -o /config/cloud/runtime-init-conf.yaml https://my-source-host/my-repo/bigip-configs/0.0.1/runtime-init-conf.yaml ``` @@ -65,7 +66,7 @@ From a high level overview, using this tool involves three steps: - Step 2: Download and install F5 BIG-IP Runtime Init using the self-extracting installer: ```sh - curl -o /tmp/f5-bigip-runtime-init-1.1.0-1.gz.run https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.1.0/dist/f5-bigip-runtime-init-1.1.0-1.gz.run && bash /tmp/f5-bigip-runtime-init-1.1.0-1.gz.run -- '--cloud azure' + curl -o /tmp/f5-bigip-runtime-init-1.2.0-1.gz.run https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.2.0/dist/f5-bigip-runtime-init-1.2.0-1.gz.run && bash /tmp/f5-bigip-runtime-init-1.2.0-1.gz.run -- '--cloud azure' ``` See [installer](#installer) details and [downloads](#downloads) below. @@ -119,15 +120,17 @@ F5 BIG-IP Runtime Init has been tested and validated with the following versions The F5 BIG-IP Runtime Init configuration consists of the following attributes: -| Attribute | Default Value | Required | Description | +| Attribute | Default Value | Required | Description | | --- | --- | --- | --- | -| extension_packages | none | No | List of URLs to download and install iControl LX extension packages before onboarding. | -| extension_services | none | No | List of declarations to to configure. | -| runtime_parameters | none | No | List of runtime parameters to gather. | -| pre_onboard_enabled | none | No | List of commands to run before sending iControl LX declarations. | -| post_onboard_enabled | none | No | List of commands to run after sending iControl LX declarations. | +| pre_onboard_enabled | none | No | List of commands to run that do not check if BIG-IP and MCPD are up and running. However, execution before BIG-IP is ready depends on cloud agent/download times/etc. | +| runtime_parameters | none | No | List of runtime parameters to gather. | +| bigip_ready_enabled | none | No | List of commands to run after BIG-IP and MCPD are up and running. Example: tmsh commands, misc optimizations, etc. | +| extension_packages | none | No | List of URLs to download and install iControl LX extension packages before onboarding. | +| extension_services | none | No | List of declarations to configure. | +| post_onboard_enabled | none | No | List of commands to run after sending iControl LX declarations. | | post_hook | none | No | Webhook to send upon completion. | + ### Configuration Examples and Schema Documentation See [SCHEMA.md](https://github.com/F5Networks/f5-bigip-runtime-init/blob/main/SCHEMA.md) for complete schema documentation and configuration examples. @@ -144,7 +147,7 @@ The self extracting installer accepts the following parameters: ex: ``` - curl https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.1.0/dist/f5-bigip-runtime-init-1.1.0-1.gz.run -o f5-bigip-runtime-init-1.1.0-1.gz.run && bash f5-bigip-runtime-init-1.1.0-1.gz.run -- '--cloud aws' + curl https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.2.0/dist/f5-bigip-runtime-init-1.2.0-1.gz.run -o f5-bigip-runtime-init-1.2.0-1.gz.run && bash f5-bigip-runtime-init-1.2.0-1.gz.run -- '--cloud aws' ``` See [Private Environments](#private-environments) section below. @@ -155,32 +158,32 @@ Self-extracting installer, RPMs, and file hashes are available from the followin | Cloud | Type | Location | | --- | --- | --- | -| All | Self-extracting installer | https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.1.0/dist/f5-bigip-runtime-init-1.1.0-1.gz.run | -| All | SHA256 | https://github.com/f5networks/f5-bigip-runtime-init/releases/download/1.1.0/f5-bigip-runtime-init-1.1.0-1.gz.run.sha256 | -| All | RPM | https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.1.0/dist/rpms/f5-bigip-runtime-init-all-1.1.0-1-signed.noarch.rpm | -| All | SHA256 | https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.1.0/dist/rpms/f5-bigip-runtime-init-all-1.1.0-1-signed.noarch.rpm.sha256 | -| AWS | RPM | https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.1.0/dist/rpms/f5-bigip-runtime-init-aws-1.1.0-1-signed.noarch.rpm | -| AWS | SHA256 | https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.1.0/dist/rpms/f5-bigip-runtime-init-aws-1.1.0-1-signed.noarch.rpm.sha256 | -| Azure | RPM | https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.1.0/dist/rpms/f5-bigip-runtime-init-azure-1.1.0-1-signed.noarch.rpm | -| Azure | SHA256 | https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.1.0/dist/rpms/f5-bigip-runtime-init-azure-1.1.0-1-signed.noarch.rpm.sha256 | -| GCP | RPM | https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.1.0/dist/rpms/f5-bigip-runtime-init-gcp-1.1.0-1-signed.noarch.rpm | -| GCP | SHA256 | https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.1.0/dist/rpms/f5-bigip-runtime-init-gcp-1.1.0-1-signed.noarch.rpm.sha256 | -| None | RPM | https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.1.0/dist/rpms/f5-bigip-runtime-init-base-1.1.0-1-signed.noarch.rpm | -| None | SHA256 | https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.1.0/dist/rpms/f5-bigip-runtime-init-base-1.1.0-1-signed.noarch.rpm.sha256 | +| All | Self-extracting installer | https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.2.0/dist/f5-bigip-runtime-init-1.2.0-1.gz.run | +| All | SHA256 | https://github.com/f5networks/f5-bigip-runtime-init/releases/download/1.2.0/f5-bigip-runtime-init-1.2.0-1.gz.run.sha256 | +| All | RPM | https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.2.0/dist/rpms/f5-bigip-runtime-init-all-1.2.0-1-signed.noarch.rpm | +| All | SHA256 | https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.2.0/dist/rpms/f5-bigip-runtime-init-all-1.2.0-1-signed.noarch.rpm.sha256 | +| AWS | RPM | https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.2.0/dist/rpms/f5-bigip-runtime-init-aws-1.2.0-1-signed.noarch.rpm | +| AWS | SHA256 | https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.2.0/dist/rpms/f5-bigip-runtime-init-aws-1.2.0-1-signed.noarch.rpm.sha256 | +| Azure | RPM | https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.2.0/dist/rpms/f5-bigip-runtime-init-azure-1.2.0-1-signed.noarch.rpm | +| Azure | SHA256 | https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.2.0/dist/rpms/f5-bigip-runtime-init-azure-1.2.0-1-signed.noarch.rpm.sha256 | +| GCP | RPM | https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.2.0/dist/rpms/f5-bigip-runtime-init-gcp-1.2.0-1-signed.noarch.rpm | +| GCP | SHA256 | https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.2.0/dist/rpms/f5-bigip-runtime-init-gcp-1.2.0-1-signed.noarch.rpm.sha256 | +| None | RPM | https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.2.0/dist/rpms/f5-bigip-runtime-init-base-1.2.0-1-signed.noarch.rpm | +| None | SHA256 | https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.2.0/dist/rpms/f5-bigip-runtime-init-base-1.2.0-1-signed.noarch.rpm.sha256 | ## Usage Examples ### Azure (ARM Template) Virtual Machine extension snippet #### Download F5 BIG-IP Runtime Config from URL ```json -"commandToExecute": "concat('mkdir -p /config/cloud; mkdir -p /var/log/cloud/azure; cp $(ls -v | tail -n1)/runtime-init-conf.yaml /config/cloud/runtime-init-conf.yaml; curl -L https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.1.0/dist/f5-bigip-runtime-init-1.1.0-1.gz.run -o f5-bigip-runtime-init-1.1.0-1.gz.run && bash f5-bigip-runtime-init-1.1.0-1.gz.run -- ', variables('singleQuote'), '--cloud azure', variables('singleQuote'), ' 2>&1')", +"commandToExecute": "concat('mkdir -p /config/cloud; mkdir -p /var/log/cloud/azure; cp $(ls -v | tail -n1)/runtime-init-conf.yaml /config/cloud/runtime-init-conf.yaml; curl -L https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.2.0/dist/f5-bigip-runtime-init-1.2.0-1.gz.run -o f5-bigip-runtime-init-1.2.0-1.gz.run && bash f5-bigip-runtime-init-1.2.0-1.gz.run -- ', variables('singleQuote'), '--cloud azure', variables('singleQuote'), ' 2>&1')", "fileUris": [ "https://example.com/runtime-init-conf.yaml" ] ``` #### Use inline F5 BIG-IP Runtime Config ```json -"commandToExecute": "[concat('mkdir -p /config/cloud; mkdir -p /var/log/cloud/azure; echo -e ', variables('singleQuote'), parameters('runtimeConfig'), variables('singleQuote'), ' > /config/cloud/runtime-init-conf.yaml; curl -L https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.1.0/dist/f5-bigip-runtime-init-1.1.0-1.gz.run -o f5-bigip-runtime-init-1.1.0-1.gz.run; bash f5-bigip-runtime-init-1.1.0-1.gz.run -- ', variables('singleQuote'), '--cloud azure', variables('singleQuote'), ' 2>&1; f5-bigip-runtime-init --config-file /config/cloud/runtime-init-conf.yaml 2>&1')]" +"commandToExecute": "[concat('mkdir -p /config/cloud; mkdir -p /var/log/cloud/azure; echo -e ', variables('singleQuote'), parameters('runtimeConfig'), variables('singleQuote'), ' > /config/cloud/runtime-init-conf.yaml; curl -L https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.2.0/dist/f5-bigip-runtime-init-1.2.0-1.gz.run -o f5-bigip-runtime-init-1.2.0-1.gz.run; bash f5-bigip-runtime-init-1.2.0-1.gz.run -- ', variables('singleQuote'), '--cloud azure', variables('singleQuote'), ' 2>&1; f5-bigip-runtime-init --config-file /config/cloud/runtime-init-conf.yaml 2>&1')]" ``` ### Terraform @@ -279,20 +282,26 @@ runtime_parameters: type: KeyVault vaultUrl: https://my-keyvault.vault.azure.net secretId: ${secret_id} + field: password pre_onboard_enabled: - name: provision_rest type: inline commands: - /usr/bin/setdb provision.extramb 500 - /usr/bin/setdb restjavad.useextramb true +bigip_ready_enabled: + - name: set_message_size + type: inline + commands: + - '/usr/bin/curl -s -f -u admin: -H "Content-Type: application/json" -d ''{"maxMessageBodySize":134217728}'' -X POST http://localhost:8100/mgmt/shared/server/messaging/settings/8100 | jq .' extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 - extensionType: ts - extensionVersion: 1.15.0 + extensionVersion: 1.16.0 extension_services: service_operations: - extensionType: do @@ -303,10 +312,9 @@ extension_services: value: https://raw.githubusercontent.com/F5Networks/f5-bigip-runtime-init/main/examples/declarations/as3.json - EOF -curl https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.1.0/dist/f5-bigip-runtime-init-1.1.0-1.gz.run -o f5-bigip-runtime-init-1.1.0-1.gz.run && bash f5-bigip-runtime-init-1.1.0-1.gz.run -- '--cloud azure' +curl https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.2.0/dist/f5-bigip-runtime-init-1.2.0-1.gz.run -o f5-bigip-runtime-init-1.2.0-1.gz.run && bash f5-bigip-runtime-init-1.2.0-1.gz.run -- '--cloud azure' f5-bigip-runtime-init --config-file /config/cloud/runtime-init-conf.yaml ``` @@ -326,9 +334,10 @@ runtime_parameters: type: KeyVault vaultUrl: https://my-keyvault.vault.azure.net secretId: mySecret01 + field: password ``` -When BIG-IP is launched, Runtime Init will fetch the **value** for the secret named```mySecret01``` from the native vault and set the runtime variable ``ADMIN_PASS``. Any declarations containing ```{{{ ADMIN_PASS }}}``` (ex. do.json, as3.json templates formatted with mustache) will be populated with the secret **value** (ex. the admin password). +When BIG-IP is launched, Runtime Init will fetch the **value** for the secret named```mySecret01``` from the native vault and set the runtime variable ``ADMIN_PASS``. Any declarations containing ```{{{ ADMIN_PASS }}}``` (ex. do.json, as3.json templates formatted with mustache) will be populated with the secret **value** (ex. the admin password). ***field*** provides field name to which this secret is map to and it instructs Runtime Init to masks the secret value in any logging outputs. #### AWS (Terraform) snippet @@ -388,11 +397,11 @@ pre_onboard_enabled: extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 - extensionType: ts - extensionVersion: 1.15.0 + extensionVersion: 1.16.0 extension_services: service_operations: - extensionType: do @@ -405,13 +414,21 @@ extension_services: EOF -curl https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.1.0/dist/f5-bigip-runtime-init-1.1.0-1.gz.run -o f5-bigip-runtime-init-1.1.0-1.gz.run && bash f5-bigip-runtime-init-1.1.0-1.gz.run -- '--cloud aws' +curl https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.2.0/dist/f5-bigip-runtime-init-1.2.0-1.gz.run -o f5-bigip-runtime-init-1.2.0-1.gz.run && bash f5-bigip-runtime-init-1.2.0-1.gz.run -- '--cloud aws' f5-bigip-runtime-init --config-file /config/cloud/runtime-init-conf.yaml ``` -NOTE: ```--cloud aws``` is passed to the installer to specify the environment - +NOTES: + - ```--cloud aws``` is passed to the installer to specify the environment + - when extension package includes ```extensionUrl``` field, ```extensionVersion``` is not required; however, ```extensionVersion``` is required when package defined without ```extensionUrl``` + ```yaml + extension_packages: + install_operations: + - extensionType: as3 + extensionUrl: https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.24.0/f5-appsvcs-3.24.0-5.noarch.rpm + ``` + The terraform variable that is templatized is ```${secret_id}``` which will be rendered by terraform before sending to the instance's ```user_data``` parameter. Ex. the rendered ```user_data``` finally sent to BIG-IP will contain the actual name of secret 'mySecret01' to gather at runtime: ex. @@ -526,11 +543,11 @@ pre_onboard_enabled: extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 - extensionType: ts - extensionVersion: 1.15.0 + extensionVersion: 1.16.0 extension_services: service_operations: - extensionType: do @@ -543,13 +560,59 @@ extension_services: EOF -curl https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.1.0/dist/f5-bigip-runtime-init-1.1.0-1.gz.run -o f5-bigip-runtime-init-1.1.0-1.gz.run && bash f5-bigip-runtime-init-1.1.0-1.gz.run -- '--cloud gcp' +curl https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v1.2.0/dist/f5-bigip-runtime-init-1.2.0-1.gz.run -o f5-bigip-runtime-init-1.2.0-1.gz.run && bash f5-bigip-runtime-init-1.2.0-1.gz.run -- '--cloud gcp' f5-bigip-runtime-init --config-file /config/cloud/runtime-init-conf.yaml ``` NOTE: ```--cloud gcp``` is passed to the installer to specify the environment +## Runtime parameters + +runtime_parameters allows to defined list of parameters and these parameters can be used for substituting tokens defined within declarations. There are a few types of parameters: + + * secret - fetches secret from Secret Vault + ```yaml + runtime_parameters: + - name: ADMIN_PASS + type: secret + secretProvider: + environment: azure + type: KeyVault + vaultUrl: https://my-keyvault.vault.azure.net + secretId: mySecret01 + ``` + * metadata - fetches common pre-defined metadata from the Metadata Service + ```yaml + runtime_parameters: + - name: MGMT_ROUTE + type: metadata + metadataProvider: + environment: aws + type: network + field: subnet-ipv4-cidr-block + index: 0 + ``` + * static - defines a static value. Example below will replace AVAILABILITY_ZONE token with "us-west-2a" string + ```yaml + runtime_parameters: + - name: AVAILABILITY_ZONE + type: static + value: us-west-2a + ``` + * url - defines url to fetch a runtime parameter (ex. custom metadata). This parameter allows to provide HTTP headers as well as JMESPath query for querying JSON document/response. The headers and query fields are optional. + ```yaml + runtime_parameters: + - name: REGION + type: url + value: http://169.254.169.254/latest/dynamic/instance-identity/document + query: region + headers: + - name: Content-Type + value: json + - name: User-Agent + value: some-user-agent + ``` ## Private Environments By default, this tool makes calls to the Internet to download a GPG key [here](https://f5-cft.s3.amazonaws.com/f5-bigip-runtime-init/gpg.key) to verify RPM signatures, find the latest Automation Tool Chain packages and send usage data. To disable calls to the Internet, you can use the examples below: @@ -558,12 +621,12 @@ By default, this tool makes calls to the Internet to download a GPG key [here](h Example (secure) of hosting the gpg key locally and disabling checking for latest Automation Tool Chain packages. ``` - curl https://myprivatehost/f5-bigip-runtime-init-1.1.0-1.gz.run -o f5-bigip-runtime-init-1.1.0-1.gz.run && bash f5-bigip-runtime-init-1.1.0-1.gz.run -- '--cloud aws --key https://mylocalhost/gpg.key --skip-toolchain-metadata-sync' + curl https://myprivatehost/f5-bigip-runtime-init-1.2.0-1.gz.run -o f5-bigip-runtime-init-1.2.0-1.gz.run && bash f5-bigip-runtime-init-1.2.0-1.gz.run -- '--cloud aws --key https://mylocalhost/gpg.key --skip-toolchain-metadata-sync' ``` Example (thisisinsecure) of skipping downloading the GPG key and checking for latest Automation Tool Chain packages, using a local copy of the metadata instead. ``` -curl https://myprivatehost/f5-bigip-runtime-init-1.1.0-1.gz.run -o f5-bigip-runtime-init-1.1.0-1.gz.run -o f5-bigip-runtime-init-1.1.0-1.gz.run && bash f5-bigip-runtime-init-1.1.0-1.gz.run -- '--cloud aws --skip-verify --skip-toolchain-metadata-sync' +curl https://myprivatehost/f5-bigip-runtime-init-1.2.0-1.gz.run -o f5-bigip-runtime-init-1.2.0-1.gz.run -o f5-bigip-runtime-init-1.2.0-1.gz.run && bash f5-bigip-runtime-init-1.2.0-1.gz.run -- '--cloud aws --skip-verify --skip-toolchain-metadata-sync' ``` #### Disable Calls from the Command @@ -608,7 +671,50 @@ The following enviroment variables can be used for setting logging options: {"message":"this is a json message","level":"info","timestamp":"2020-08-04T00:22:28.069Z"} ``` -Example of how to set the log level using an environment variable: ```export F5_BIGIP_RUNTIME_INIT_LOG_LEVEL=silly && bash /var/tmp/f5-bigip-runtime-init-1.1.0-1.gz.run -- '--cloud ${CLOUD}'``` +Example of how to set the log level using an environment variable: ```export F5_BIGIP_RUNTIME_INIT_LOG_LEVEL=silly && bash /var/tmp/f5-bigip-runtime-init-1.2.0-1.gz.run -- '--cloud ${CLOUD}'``` + + +By default, runtime will mask out (i.e. "********") the following common fields when logging: +```json + [ + "password", + "localPassword", + "remotePassword", + "bigIqPassword", + "bigIpPassword", + "passphrase", + "cookiePassphrase", + "certificate", + "privateKey", + "ciphertext", + "protected", + "secret", + "sharedSecret", + "secretAccessKey", + "apiAccessKey", + "encodedCredentials", + "encodedToken", + "oldPassword", + "newPassword", + "bindPassword", + "checkBindPassword", + "md5SignaturePassphrase" + ] +``` +However, it is possible to extend this list by providing additional metadata (***field***) for the Secret object: + +```yaml + runtime_parameters: + - name: MY_SECRET + type: secret + secretProvider: + environment: azure + type: KeyVault + vaultUrl: https://my-keyvault.vault.azure.net + secretId: mySecret01 + field: newCustomSecretField +``` +This example shows how to instruct Runtime Init to mask out the value for ```newCustomSecretField```. #### Send output to log file and serial console @@ -657,4 +763,4 @@ Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations -under the License. \ No newline at end of file +under the License. diff --git a/SCHEMA.md b/SCHEMA.md index ecb98516..d4887bab 100644 --- a/SCHEMA.md +++ b/SCHEMA.md @@ -26,6 +26,7 @@ Type: `array` 1. _"static"_ 2. _"secret"_ 3. _"metadata"_ + 4. _"url"_ - value - Type: `string` - path: #/items/properties/value @@ -73,6 +74,13 @@ Type: `array` 1. _"https://my-keyvault.vault.azure.net"_ 2. _"https://my-keyvault.vault.usgovcloudapi.net"_ - The value must match this pattern: `(https?://(.+?\.)?vault\.(azure|usgovcloudapi)\.net(/[A-Za-z0-9\-\._~:/\?#\[\]@!$&'\(\)\*\+,;\=]*)?)` + - field + - _field name to which secret value is mapped to_ + - Type: `string` + - path: #/items/properties/secretProvider/properties/field + - Example values: + 1. _"bigiqPassword"_ + 2. _"regKey"_ - metadataProvider - Type: `object` - path: #/items/properties/metadataProvider @@ -107,6 +115,14 @@ Type: `array` 1. _"0"_ 2. _"1"_ 3. _"2"_ + - query + - Type: `string` + - path: #/items/properties/query + - Example values: + 1. _"region"_ + - headers + - Type: `array` + - path: #/items/properties/headers ### runtime_parameters: Configuration Examples @@ -281,7 +297,7 @@ remote_exec: *** ### pre_onboard_enabled: Schema -_Used to specify commands which will be executed before extension package operations._ +_Used to specify commands which will be executed before extension package operations before BIG-IP is ready._ Type: `array` @@ -359,6 +375,82 @@ remote_exec: commands: - 'https://the-delivery-location.com/remote_pre_onboard.sh' +``` +*** +### bigip_ready_enabled: Schema + +_Used to specify commands which will be executed before extension package operations after BIG-IP and MCPD are up and running._ + +Type: `array` + +path: # + + - **_Items_** + - Type: `object` + - path: #/items + - **_Properties_** + - name + - Type: `string` + - path: #/items/properties/name + - Example values: + 1. _"my_preonboard_command"_ + 2. _"example_local_exec"_ + 3. _"provision_rest"_ + - type + - Type: `string` + - path: #/items/properties/type + - The value is restricted to the following: + 1. _"inline"_ + 2. _"file"_ + 3. _"url"_ + - command + - Type: `array` + - path: #/items/properties/command + - **_Items_** + - Type: `string` + - path: #/items/properties/command/items + - Example values: + 1. _"tmsh create net vlan external interfaces replace-all-with { 1.1 }"_ + 2. _"tmsh create sys folder /LOCAL_ONLY device-group none traffic-group traffic-group-local-only"_ + 3. _"tmsh save sys config"_ + - verifyTls + - _For enabling secure site verification_ + - Type: `boolean` + - path: #/items/properties/verifyTls + - Example values: + 1. _true_ + 2. _false_ + + +### bigip_ready_enabled: Configuration Examples + +```yaml +inline: + description: Runs commands specified inline + bigip_ready_enabled: + - name: set_message_size + type: inline + commands: + - >- + /usr/bin/curl -s -f -u admin: -H "Content-Type: application/json" -d + '{"maxMessageBodySize":134217728}' -X POST + http://localhost:8100/mgmt/shared/server/messaging/settings/8100 | jq + . +local_exec: + description: Runs commands from a local file + bigip_ready_enabled: + - name: example_local_exec + type: file + commands: + - /tmp/bigip_ready_enabled.sh +remote_exec: + description: Runs commands from a URL + bigip_ready_enabled: + - name: example_remote_exec + type: url + commands: + - 'https://the-delivery-location.com/bigip_ready_enabled.sh' + ``` *** ### extension_packages: Schema @@ -432,46 +524,46 @@ default: extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 versioned: description: Installs packages using specific versions extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 hashed: description: Verifies and installs packages using specified hashes extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 - extensionHash: 536eccb9dbf40aeabd31e64da8c5354b57d893286ddc6c075ecc9273fcca10a1 + extensionVersion: 1.17.0 + extensionHash: a359aa8aa14dc565146d4ccc413f169f1e8d02689559c5e4a652f91609a55fbb - extensionType: as3 - extensionVersion: 3.23.0 - extensionHash: de615341b91beaed59195dceefc122932580d517600afce1ba8d3770dfe42d28 + extensionVersion: 3.24.0 + extensionHash: eaef5c650da84f593d17939e3bb5d649097f60a6d302d6abc06951f2b4a986d9 url: description: Installs packages from custom locations extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 extensionUrl: >- - https://github.com/F5Networks/f5-declarative-onboarding/releases/download/v1.16.0/f5-declarative-onboarding-1.16.0-8.noarch.rpm + https://github.com/F5Networks/f5-declarative-onboarding/releases/download/v1.17.0/f5-declarative-onboarding-1.17.0-3.noarch.rpm - extensionType: as3 - extensionVersion: 3.23.0 - extensionUrl: 'file:///var/config/rest/downloads/f5-appsvcs-3.23.0-5.noarch.rpm' + extensionVersion: 3.24.0 + extensionUrl: 'file:///var/config/rest/downloads/f5-appsvcs-3.24.0-5.noarch.rpm' ilx: description: Installs a custom iLX package extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 - extensionType: ilx extensionUrl: >- file:///var/config/rest/downloads/f5-appsvcs-templates-1.1.0-1.noarch.rpm @@ -640,14 +732,23 @@ example_1: commands: - /usr/bin/setdb provision.extramb 500 - /usr/bin/setdb restjavad.useextramb true + bigip_ready_enabled: + - name: set_message_size + type: inline + commands: + - >- + /usr/bin/curl -s -f -u admin: -H "Content-Type: application/json" -d + '{"maxMessageBodySize":134217728}' -X POST + http://localhost:8100/mgmt/shared/server/messaging/settings/8100 | + jq . extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 - extensionHash: 536eccb9dbf40aeabd31e64da8c5354b57d893286ddc6c075ecc9273fcca10a1 + extensionVersion: 1.17.0 + extensionHash: 484b34ecbbe6b97a3b7d3c547a6d9c234c9fc99766d07bed4384298437327f23 - extensionType: as3 - extensionVersion: 3.23.0 - extensionHash: de615341b91beaed59195dceefc122932580d517600afce1ba8d3770dfe42d28 + extensionVersion: 3.24.0 + extensionHash: 3dbb3e2cb170d2fc9ead746aa2df37518c0f38876debe33eec95cf928f91a329 extension_services: service_operations: - extensionType: as3 @@ -669,9 +770,9 @@ example_2: install_operations: - extensionType: do extensionUrl: >- - file:///var/config/rest/downloads/f5-declarative-onboarding-1.16.0-8.noarch.rpm - extensionHash: 536eccb9dbf40aeabd31e64da8c5354b57d893286ddc6c075ecc9273fcca10a1 - extensionVersion: 1.16.0 + file:///var/config/rest/downloads/f5-declarative-onboarding-1.17.0-3.noarch.rpm + extensionHash: a359aa8aa14dc565146d4ccc413f169f1e8d02689559c5e4a652f91609a55fbb + extensionVersion: 1.17.0 - extensionType: ilx extensionUrl: 'file:///var/config/rest/downloads/myIlxApp.rpm' extensionVersion: 1.1.0 @@ -704,9 +805,9 @@ example_3: extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 extension_services: service_operations: - extensionType: do @@ -745,9 +846,9 @@ example_4: extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 extension_services: service_operations: - extensionType: do @@ -782,9 +883,9 @@ example_5: extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 extension_services: service_operations: - extensionType: do @@ -792,8 +893,9 @@ example_5: value: 'file:///examples/declarations/example_5_do.json' example_6: description: >- - Replaces variables used within DO declaration with properties from instance - metadata to configure hostname and self IP addresses on BIGIP device. + Replaces variables used within DO and AS3 declarations with properties from + instance metadata to configure hostname, self IP addresses and pool members + on BIGIP device. runtime_config: runtime_parameters: - name: HOST_NAME @@ -823,6 +925,15 @@ example_6: type: network field: subnet-ipv4-cidr-block index: 1 + - name: REGION + type: url + value: 'http://169.254.169.254/latest/dynamic/instance-identity/document' + query: region + headers: + - name: Content-type + value: json + - name: User-Agent + value: bigip-ve pre_onboard_enabled: - name: provision_rest type: inline @@ -832,14 +943,17 @@ example_6: extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 extension_services: service_operations: - extensionType: do type: url value: 'file:///examples/declarations/example_6_do.json' + - extensionType: as3 + type: url + value: 'file:///examples/declarations/example_7_as3.json' example_7: description: Installs AS3 and DO and uses an inline AS3 declaration to setup the BIG-IP. runtime_config: @@ -852,9 +966,9 @@ example_7: extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 extension_services: service_operations: - extensionType: as3 @@ -907,9 +1021,9 @@ example_8: extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 extension_services: service_operations: - extensionType: do @@ -1014,9 +1128,9 @@ example_9: extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 example_10: description: Sending a customized webhook on completion. runtime_config: @@ -1029,9 +1143,9 @@ example_10: extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 post_hook: - name: example_webhook type: webhook @@ -1085,13 +1199,13 @@ example_11: extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 verifyTls: false extensionUrl: >- - https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.23.0/f5-appsvcs-3.23.0-5.noarch.rpm - extensionHash: de615341b91beaed59195dceefc122932580d517600afce1ba8d3770dfe42d28 + https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.24.0/f5-appsvcs-3.24.0-5.noarch.rpm + extensionHash: bc21ff6460e65610c50d8eb358155098f79a10371e69c166764735a38a3523cd - extensionType: ilx extensionUrl: >- file:///var/config/rest/downloads/f5-appsvcs-templates-1.1.0-1.noarch.rpm @@ -1144,9 +1258,9 @@ example_12: extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 extension_services: service_operations: - extensionType: do diff --git a/examples/config/bigip_ready_enabled.yaml b/examples/config/bigip_ready_enabled.yaml new file mode 100644 index 00000000..3f292c86 --- /dev/null +++ b/examples/config/bigip_ready_enabled.yaml @@ -0,0 +1,21 @@ +inline: + description: Runs commands specified inline + bigip_ready_enabled: + - name: set_message_size + type: inline + commands: + - '/usr/bin/curl -s -f -u admin: -H "Content-Type: application/json" -d ''{"maxMessageBodySize":134217728}'' -X POST http://localhost:8100/mgmt/shared/server/messaging/settings/8100 | jq .' +local_exec: + description: Runs commands from a local file + bigip_ready_enabled: + - name: example_local_exec + type: file + commands: + - /tmp/bigip_ready_enabled.sh +remote_exec: + description: Runs commands from a URL + bigip_ready_enabled: + - name: example_remote_exec + type: url + commands: + - https://the-delivery-location.com/bigip_ready_enabled.sh diff --git a/examples/config/cloud_config_install_only.json b/examples/config/cloud_config_install_only.json index 06ecfde3..3f6d9ded 100644 --- a/examples/config/cloud_config_install_only.json +++ b/examples/config/cloud_config_install_only.json @@ -21,15 +21,15 @@ "install_operations": [ { "extensionType": "do", - "extensionVersion": "1.16.0", - "extensionUrl": "file:///var/config/rest/downloads/f5-declarative-onboarding-1.16.0-8.noarch.rpm", - "extensionHash": "536eccb9dbf40aeabd31e64da8c5354b57d893286ddc6c075ecc9273fcca10a1" + "extensionVersion": "1.17.0", + "extensionUrl": "file:///var/config/rest/downloads/f5-declarative-onboarding-1.17.0-3.noarch.rpm", + "extensionHash": "a359aa8aa14dc565146d4ccc413f169f1e8d02689559c5e4a652f91609a55fbb" }, { "extensionType": "as3", - "extensionVersion": "3.23.0", - "extensionUrl": "https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.23.0/f5-appsvcs-3.23.0-5.noarch.rpm", - "extensionHash": "de615341b91beaed59195dceefc122932580d517600afce1ba8d3770dfe42d28" + "extensionVersion": "3.24.0", + "extensionUrl": "https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.24.0/f5-appsvcs-3.24.0-5.noarch.rpm", + "extensionHash": "df786fc755c5de6f3fcc47638caf3db4c071fcd9cf37855de78fd7e25e5117b4" }, { "extensionType": "ilx", diff --git a/examples/config/cloud_config_install_only.yaml b/examples/config/cloud_config_install_only.yaml index 44d9ba64..3f6d9ded 100644 --- a/examples/config/cloud_config_install_only.yaml +++ b/examples/config/cloud_config_install_only.yaml @@ -1,24 +1,41 @@ -runtime_parameters: [] -pre_onboard_enabled: - - name: provision_rest - type: inline - commands: - - /usr/bin/setdb provision.extramb 500 - - /usr/bin/setdb restjavad.useextramb true - - name: provision_modules - type: inline - commands: - - echo 'sys provision asm { level nominal }' >> bigip_base.conf -extension_packages: - install_operations: - - extensionType: do - extensionVersion: 1.16.0 - extensionUrl: file:///var/config/rest/downloads/f5-declarative-onboarding-1.16.0-8.noarch.rpm - extensionHash: 536eccb9dbf40aeabd31e64da8c5354b57d893286ddc6c075ecc9273fcca10a1 - - extensionType: as3 - extensionVersion: 3.23.0 - extensionUrl: https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.23.0/f5-appsvcs-3.23.0-5.noarch.rpm - extensionHash: de615341b91beaed59195dceefc122932580d517600afce1ba8d3770dfe42d28 - - extensionType: ilx - extensionUrl: file:///var/config/rest/downloads/f5-appsvcs-templates-1.1.0-1.noarch.rpm - extensionVerificationEndpoint: /mgmt/shared/fast/info +{ + "runtime_parameters": [], + "pre_onboard_enabled": [ + { + "name": "provision_rest", + "type": "inline", + "commands": [ + "/usr/bin/setdb provision.extramb 500", + "/usr/bin/setdb restjavad.useextramb true" + ] + }, + { + "name": "provision_modules", + "type": "inline", + "commands": [ + "echo 'sys provision asm { level nominal }' >> bigip_base.conf" + ] + } + ], + "extension_packages": { + "install_operations": [ + { + "extensionType": "do", + "extensionVersion": "1.17.0", + "extensionUrl": "file:///var/config/rest/downloads/f5-declarative-onboarding-1.17.0-3.noarch.rpm", + "extensionHash": "a359aa8aa14dc565146d4ccc413f169f1e8d02689559c5e4a652f91609a55fbb" + }, + { + "extensionType": "as3", + "extensionVersion": "3.24.0", + "extensionUrl": "https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.24.0/f5-appsvcs-3.24.0-5.noarch.rpm", + "extensionHash": "df786fc755c5de6f3fcc47638caf3db4c071fcd9cf37855de78fd7e25e5117b4" + }, + { + "extensionType": "ilx", + "extensionUrl": "file:///var/config/rest/downloads/f5-appsvcs-templates-1.1.0-1.noarch.rpm", + "extensionVerificationEndpoint": "/mgmt/shared/fast/info" + } + ] + } +} diff --git a/examples/config/cloud_config_local.json b/examples/config/cloud_config_local.json index c8aedc8d..0cc2cd33 100644 --- a/examples/config/cloud_config_local.json +++ b/examples/config/cloud_config_local.json @@ -1,109 +1,111 @@ { - "runtime_parameters": [ - { - "name": "AZURE_SERVICE_PRINCIPAL", - "type": "secret", - "secretProvider": { - "environment": "azure", - "type": "KeyVault", - "vaultUrl": "https://my-keyvault.vault.azure.net", - "secretId": "my_azure_admin_secret", - "version": "6e86876be4ce46a49ec578dfda897593" - } - }, - { - "name": "GCP_ROOT_PASSWORD", - "type": "secret", - "secretProvider": { - "environment": "gcp", - "type": "SecretManager", - "secretId": "my-secret-id-02", - "version": "latest" - } - }, - { - "name": "AWS_ADMIN_PASSWORD", - "type": "secret", - "secretProvider": { - "environment": "aws", - "type": "SecretsManager", - "secretId": "my_azure_root_secret", - "version": "AWSCURRENT" - } - }, - { - "name": "HOST_NAME", - "type": "metadata", - "metadataProvider": { - "environment": "azure", - "type": "compute", - "field": "name" - } - }, + "runtime_parameters": [ + { + "name": "AZURE_SERVICE_PRINCIPAL", + "type": "secret", + "secretProvider": { + "environment": "azure", + "type": "KeyVault", + "vaultUrl": "https://my-keyvault.vault.azure.net", + "secretId": "my_azure_admin_secret", + "version": "6e86876be4ce46a49ec578dfda897593" + } + }, + { + "name": "GCP_ROOT_PASSWORD", + "type": "secret", + "secretProvider": { + "environment": "gcp", + "type": "SecretManager", + "secretId": "my-secret-id-02", + "version": "latest" + } + }, + { + "name": "AWS_ADMIN_PASSWORD", + "type": "secret", + "secretProvider": { + "environment": "aws", + "type": "SecretsManager", + "secretId": "my_azure_root_secret", + "version": "AWSCURRENT" + } + }, + { + "name": "HOST_NAME", + "type": "metadata", + "metadataProvider": { + "environment": "azure", + "type": "compute", + "field": "name" + } + }, + { + "name": "SELF_IP_INTERNAL", + "type": "metadata", + "metadataProvider": { + "environment": "azure", + "type": "network", + "field": "ipv4", + "index": 1 + } + }, + { + "name": "SELF_IP_EXTERNAL", + "type": "metadata", + "metadataProvider": { + "environment": "azure", + "type": "network", + "field": "ipv4", + "index": 2 + } + } + ], + "pre_onboard_enabled": [ + { + "name": "provision_rest", + "type": "inline", + "commands": [ + "/usr/bin/setdb provision.extramb 500", + "/usr/bin/setdb restjavad.useextramb true" + ] + }, + { + "name": "provision_modules", + "type": "inline", + "commands": [ + "echo 'sys provision asm { level nominal }' >> bigip_base.conf" + ] + } + ], + "post_onboard_enabled": [], + "extension_packages": { + "install_operations": [ { - "name": "SELF_IP_INTERNAL", - "type": "metadata", - "metadataProvider": { - "environment": "azure", - "type": "network", - "field": "ipv4", - "index": 1 - } + "extensionType": "do", + "extensionVersion": "1.17.0", + "extensionHash": "a359aa8aa14dc565146d4ccc413f169f1e8d02689559c5e4a652f91609a55fbb", + "extensionUrl": "https://github.com/F5Networks/f5-declarative-onboarding/releases/download/v1.17.0/f5-declarative-onboarding-1.17.0-3.noarch.rpm" }, { - "name": "SELF_IP_EXTERNAL", - "type": "metadata", - "metadataProvider": { - "environment": "azure", - "type": "network", - "field": "ipv4", - "index": 2 - } + "extensionType": "as3", + "extensionVersion": "3.24.0", + "extensionUrl": "https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.24.0/f5-appsvcs-3.24.0-5.noarch.rpm" } - ], - "pre_onboard_enabled": [ + ] + }, + "extension_services": { + "service_operations": [ { - "name": "provision_rest", - "type": "inline", - "commands": [ - "/usr/bin/setdb provision.extramb 500", - "/usr/bin/setdb restjavad.useextramb true" - ] + "extensionType": "do", + "type": "url", + "value": "https://cdn.f5.com/product/cloudsolutions/templates/f5-azure-arm-templates/examples/modules/bigip/autoscale_do.json" }, { - "name": "provision_modules", - "type": "inline", - "commands": [ - "echo 'sys provision asm { level nominal }' >> bigip_base.conf" - ] + "extensionType": "as3", + "type": "url", + "value": "file:///var/tmp/as3_example.json" } - ], - "post_onboard_enabled": [], - "extension_packages": { - "install_operations": [ - { - "extensionType": "do", - "extensionVersion": "1.16.0", - "extensionHash": "536eccb9dbf40aeabd31e64da8c5354b57d893286ddc6c075ecc9273fcca10a1" - }, - { - "extensionType": "as3", - "extensionVersion": "3.23.0" - } - ] - }, - "extension_services": { - "service_operations": [ - { - "extensionType": "do", - "type": "url", - "value": "https://cdn.f5.com/product/cloudsolutions/templates/f5-azure-arm-templates/examples/modules/bigip/autoscale_do.json" - }, - { - "extensionType": "as3", - "type": "url", - "value": "file:///var/tmp/as3_example.json" - } - ] - } -} \ No newline at end of file + ] + } +} diff --git a/examples/config/cloud_config_local.yaml b/examples/config/cloud_config_local.yaml index fc612be9..76241e7a 100644 --- a/examples/config/cloud_config_local.yaml +++ b/examples/config/cloud_config_local.yaml @@ -1,46 +1,46 @@ runtime_parameters: -- name: AZURE_SERVICE_PRINCIPAL - type: secret - secretProvider: - environment: azure - type: KeyVault - vaultUrl: https://my-keyvault.vault.azure.net - secretId: my_azure_admin_secret - version: 6e86876be4ce46a49ec578dfda897593 -- name: GCP_ROOT_PASSWORD - type: secret - secretProvider: - environment: gcp - type: SecretManager - secretId: my-secret-id-02 - version: latest -- name: AWS_ADMIN_PASSWORD - type: secret - secretProvider: - environment: aws - type: SecretsManager - secretId: my_azure_root_secret - version: AWSCURRENT -- name: HOST_NAME - type: metadata - metadataProvider: - environment: azure - type: compute - field: name -- name: SELF_IP_INTERNAL - type: metadata - metadataProvider: - environment: azure - type: network - field: ipv4 - index: 1 -- name: SELF_IP_EXTERNAL - type: metadata - metadataProvider: - environment: azure - type: network - field: ipv4 - index: 2 + - name: AZURE_SERVICE_PRINCIPAL + type: secret + secretProvider: + environment: azure + type: KeyVault + vaultUrl: https://my-keyvault.vault.azure.net + secretId: my_azure_admin_secret + version: 6e86876be4ce46a49ec578dfda897593 + - name: GCP_ROOT_PASSWORD + type: secret + secretProvider: + environment: gcp + type: SecretManager + secretId: my-secret-id-02 + version: latest + - name: AWS_ADMIN_PASSWORD + type: secret + secretProvider: + environment: aws + type: SecretsManager + secretId: my_azure_root_secret + version: AWSCURRENT + - name: HOST_NAME + type: metadata + metadataProvider: + environment: azure + type: compute + field: name + - name: SELF_IP_INTERNAL + type: metadata + metadataProvider: + environment: azure + type: network + field: ipv4 + index: 1 + - name: SELF_IP_EXTERNAL + type: metadata + metadataProvider: + environment: azure + type: network + field: ipv4 + index: 2 pre_onboard_enabled: - name: provision_rest type: inline @@ -54,16 +54,16 @@ pre_onboard_enabled: post_onboard_enabled: [] extension_packages: install_operations: - - extensionType: do - extensionVersion: 1.16.0 - extensionHash: 536eccb9dbf40aeabd31e64da8c5354b57d893286ddc6c075ecc9273fcca10a1 - - extensionType: as3 - extensionVersion: 3.23.0 + - extensionType: do + extensionVersion: 1.17.0 + extensionHash: a359aa8aa14dc565146d4ccc413f169f1e8d02689559c5e4a652f91609a55fbb + - extensionType: as3 + extensionVersion: 3.24.0 extension_services: service_operations: - - extensionType: do - type: url - value: https://cdn.f5.com/product/cloudsolutions/templates/f5-azure-arm-templates/examples/modules/bigip/autoscale_do.json - - extensionType: as3 - type: url - value: file:///var/tmp/as3_example.json + - extensionType: do + type: url + value: https://cdn.f5.com/product/cloudsolutions/templates/f5-azure-arm-templates/examples/modules/bigip/autoscale_do.json + - extensionType: as3 + type: url + value: file:///var/tmp/as3_example.json diff --git a/examples/config/cloud_config_local_w_as3.json b/examples/config/cloud_config_local_w_as3.json index b2b43658..f3e669d4 100644 --- a/examples/config/cloud_config_local_w_as3.json +++ b/examples/config/cloud_config_local_w_as3.json @@ -1,148 +1,150 @@ { - "runtime_parameters": [ - { - "name": "AZURE_SERVICE_PRINCIPAL", - "type": "secret", - "secretProvider": { - "environment": "azure", - "type": "KeyVault", - "vaultUrl": "https://my-keyvault.vault.azure.net", - "secretId": "my_azure_admin_secret", - "version": "6e86876be4ce46a49ec578dfda897593" - } - }, - { - "name": "GCP_ROOT_PASSWORD", - "type": "secret", - "secretProvider": { - "environment": "gcp", - "type": "SecretManager", - "secretId": "my-secret-id-02", - "version": "latest" - } - }, - { - "name": "AWS_ADMIN_PASSWORD", - "type": "secret", - "secretProvider": { - "environment": "aws", - "type": "SecretsManager", - "secretId": "my_azure_root_secret", - "version": "AWSCURRENT" - } - }, - { - "name": "HOST_NAME", - "type": "metadata", - "metadataProvider": { - "environment": "azure", - "type": "compute", - "field": "name" - } - }, + "runtime_parameters": [ + { + "name": "AZURE_SERVICE_PRINCIPAL", + "type": "secret", + "secretProvider": { + "environment": "azure", + "type": "KeyVault", + "vaultUrl": "https://my-keyvault.vault.azure.net", + "secretId": "my_azure_admin_secret", + "version": "6e86876be4ce46a49ec578dfda897593" + } + }, + { + "name": "GCP_ROOT_PASSWORD", + "type": "secret", + "secretProvider": { + "environment": "gcp", + "type": "SecretManager", + "secretId": "my-secret-id-02", + "version": "latest" + } + }, + { + "name": "AWS_ADMIN_PASSWORD", + "type": "secret", + "secretProvider": { + "environment": "aws", + "type": "SecretsManager", + "secretId": "my_azure_root_secret", + "version": "AWSCURRENT" + } + }, + { + "name": "HOST_NAME", + "type": "metadata", + "metadataProvider": { + "environment": "azure", + "type": "compute", + "field": "name" + } + }, + { + "name": "SELF_IP_INTERNAL", + "type": "metadata", + "metadataProvider": { + "environment": "azure", + "type": "network", + "field": "ipv4", + "index": 1 + } + }, + { + "name": "SELF_IP_EXTERNAL", + "type": "metadata", + "metadataProvider": { + "environment": "azure", + "type": "network", + "field": "ipv4", + "index": 2 + } + } + ], + "pre_onboard_enabled": [ + { + "name": "provision_rest", + "type": "inline", + "commands": [ + "/usr/bin/setdb provision.extramb 500", + "/usr/bin/setdb restjavad.useextramb true" + ] + }, + { + "name": "provision_modules", + "type": "inline", + "commands": [ + "echo 'sys provision asm { level nominal }' >> bigip_base.conf" + ] + } + ], + "post_onboard_enabled": [], + "extension_packages": { + "install_operations": [ { - "name": "SELF_IP_INTERNAL", - "type": "metadata", - "metadataProvider": { - "environment": "azure", - "type": "network", - "field": "ipv4", - "index": 1 - } + "extensionType": "do", + "extensionVersion": "1.17.0", + "extensionHash": "a359aa8aa14dc565146d4ccc413f169f1e8d02689559c5e4a652f91609a55fbb", + "extensionUrl": "https://github.com/F5Networks/f5-declarative-onboarding/releases/download/v1.17.0/f5-declarative-onboarding-1.17.0-3.noarch.rpm" }, { - "name": "SELF_IP_EXTERNAL", - "type": "metadata", - "metadataProvider": { - "environment": "azure", - "type": "network", - "field": "ipv4", - "index": 2 - } + "extensionType": "as3", + "extensionVersion": "3.24.0", + "extensionUrl": "https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.24.0/f5-appsvcs-3.24.0-5.noarch.rpm" } - ], - "pre_onboard_enabled": [ + ] + }, + "extension_services": { + "service_operations": [ { - "name": "provision_rest", - "type": "inline", - "commands": [ - "/usr/bin/setdb provision.extramb 500", - "/usr/bin/setdb restjavad.useextramb true" - ] + "extensionType": "do", + "type": "url", + "value": "https://cdn.f5.com/product/cloudsolutions/templates/f5-azure-arm-templates/examples/modules/bigip/autoscale_do.json" }, { - "name": "provision_modules", - "type": "inline", - "commands": [ - "echo 'sys provision asm { level nominal }' >> bigip_base.conf" - ] - } - ], - "post_onboard_enabled": [], - "extension_packages": { - "install_operations": [ - { - "extensionType": "do", - "extensionVersion": "1.16.0", - "extensionHash": "536eccb9dbf40aeabd31e64da8c5354b57d893286ddc6c075ecc9273fcca10a1" - }, - { - "extensionType": "as3", - "extensionVersion": "3.23.0" - } - ] - }, - "extension_services": { - "service_operations": [ - { - "extensionType": "do", - "type": "url", - "value": "https://cdn.f5.com/product/cloudsolutions/templates/f5-azure-arm-templates/examples/modules/bigip/autoscale_do.json" - }, - { - "extensionType": "as3", - "type": "inline", - "value": { - "class": "AS3", - "action": "deploy", - "persist": true, - "declaration": { - "class": "ADC", - "schemaVersion": "3.0.0", - "label": "Sample 1", - "remark": "Simple HTTP Service with Round-Robin Load Balancing", - "Sample_01": { - "class": "Tenant", - "A1": { - "class": "Application", - "template": "http", - "serviceMain": { - "class": "Service_HTTP", - "virtualAddresses": [ - "10.0.1.10" - ], - "pool": "web_pool" - }, - "web_pool": { - "class": "Pool", - "monitors": [ - "http" - ], - "members": [ - { - "servicePort": 80, - "serverAddresses": [ - "192.0.1.10", - "192.0.1.11" - ] - } - ] - } - } - } - } + "extensionType": "as3", + "type": "inline", + "value": { + "class": "AS3", + "action": "deploy", + "persist": true, + "declaration": { + "class": "ADC", + "schemaVersion": "3.0.0", + "label": "Sample 1", + "remark": "Simple HTTP Service with Round-Robin Load Balancing", + "Sample_01": { + "class": "Tenant", + "A1": { + "class": "Application", + "template": "http", + "serviceMain": { + "class": "Service_HTTP", + "virtualAddresses": [ + "10.0.1.10" + ], + "pool": "web_pool" + }, + "web_pool": { + "class": "Pool", + "monitors": [ + "http" + ], + "members": [ + { + "servicePort": 80, + "serverAddresses": [ + "192.0.1.10", + "192.0.1.11" + ] + } + ] + } + } } - } - ] - } -} \ No newline at end of file + } + } + } + ] + } +} diff --git a/examples/config/cloud_config_local_w_as3.yaml b/examples/config/cloud_config_local_w_as3.yaml index cab77369..b93336b5 100644 --- a/examples/config/cloud_config_local_w_as3.yaml +++ b/examples/config/cloud_config_local_w_as3.yaml @@ -1,46 +1,46 @@ runtime_parameters: -- name: AZURE_SERVICE_PRINCIPAL - type: secret - secretProvider: - environment: azure - type: KeyVault - vaultUrl: https://my-keyvault.vault.azure.net - secretId: my_azure_admin_secret - version: 6e86876be4ce46a49ec578dfda897593 -- name: GCP_ROOT_PASSWORD - type: secret - secretProvider: - environment: gcp - type: SecretManager - secretId: my-secret-id-02 - version: latest -- name: AWS_ADMIN_PASSWORD - type: secret - secretProvider: - environment: aws - type: SecretsManager - secretId: my_azure_root_secret - version: AWSCURRENT -- name: HOST_NAME - type: metadata - metadataProvider: - environment: azure - type: compute - field: name -- name: SELF_IP_INTERNAL - type: metadata - metadataProvider: - environment: azure - type: network - field: ipv4 - index: 1 -- name: SELF_IP_EXTERNAL - type: metadata - metadataProvider: - environment: azure - type: network - field: ipv4 - index: 2 + - name: AZURE_SERVICE_PRINCIPAL + type: secret + secretProvider: + environment: azure + type: KeyVault + vaultUrl: https://my-keyvault.vault.azure.net + secretId: my_azure_admin_secret + version: 6e86876be4ce46a49ec578dfda897593 + - name: GCP_ROOT_PASSWORD + type: secret + secretProvider: + environment: gcp + type: SecretManager + secretId: my-secret-id-02 + version: latest + - name: AWS_ADMIN_PASSWORD + type: secret + secretProvider: + environment: aws + type: SecretsManager + secretId: my_azure_root_secret + version: AWSCURRENT + - name: HOST_NAME + type: metadata + metadataProvider: + environment: azure + type: compute + field: name + - name: SELF_IP_INTERNAL + type: metadata + metadataProvider: + environment: azure + type: network + field: ipv4 + index: 1 + - name: SELF_IP_EXTERNAL + type: metadata + metadataProvider: + environment: azure + type: network + field: ipv4 + index: 2 pre_onboard_enabled: - name: provision_rest type: inline @@ -54,43 +54,43 @@ pre_onboard_enabled: post_onboard_enabled: [] extension_packages: install_operations: - - extensionType: do - extensionVersion: 1.16.0 - extensionHash: 536eccb9dbf40aeabd31e64da8c5354b57d893286ddc6c075ecc9273fcca10a1 - - extensionType: as3 - extensionVersion: 3.23.0 + - extensionType: do + extensionVersion: 1.17.0 + extensionHash: a359aa8aa14dc565146d4ccc413f169f1e8d02689559c5e4a652f91609a55fbb + - extensionType: as3 + extensionVersion: 3.24.0 extension_services: service_operations: - - extensionType: do - type: url - value: https://cdn.f5.com/product/cloudsolutions/templates/f5-azure-arm-templates/examples/modules/bigip/autoscale_do.json - - extensionType: as3 - type: inline - value: - class: AS3 - action: deploy - persist: true - declaration: - class: ADC - schemaVersion: 3.0.0 - label: Sample 1 - remark: Simple HTTP Service with Round-Robin Load Balancing - Sample_01: - class: Tenant - A1: - class: Application - template: http - serviceMain: - class: Service_HTTP - virtualAddresses: - - 10.0.1.10 - pool: web_pool - web_pool: - class: Pool - monitors: - - http - members: - - servicePort: 80 - serverAddresses: - - 192.0.1.10 - - 192.0.1.11 + - extensionType: do + type: url + value: https://cdn.f5.com/product/cloudsolutions/templates/f5-azure-arm-templates/examples/modules/bigip/autoscale_do.json + - extensionType: as3 + type: inline + value: + class: AS3 + action: deploy + persist: true + declaration: + class: ADC + schemaVersion: 3.0.0 + label: Sample 1 + remark: Simple HTTP Service with Round-Robin Load Balancing + Sample_01: + class: Tenant + A1: + class: Application + template: http + serviceMain: + class: Service_HTTP + virtualAddresses: + - 10.0.1.10 + pool: web_pool + web_pool: + class: Pool + monitors: + - http + members: + - servicePort: 80 + serverAddresses: + - 192.0.1.10 + - 192.0.1.11 diff --git a/examples/config/cloud_config_local_w_as3_w_runtime_param.yaml b/examples/config/cloud_config_local_w_as3_w_runtime_param.yaml index 28dfb2ee..1b8bc601 100644 --- a/examples/config/cloud_config_local_w_as3_w_runtime_param.yaml +++ b/examples/config/cloud_config_local_w_as3_w_runtime_param.yaml @@ -1,49 +1,49 @@ runtime_parameters: -- name: AZURE_SERVICE_PRINCIPAL - type: secret - secretProvider: - environment: azure - type: KeyVault - vaultUrl: https://my-keyvault.vault.azure.net - secretId: my_azure_admin_secret - version: 6e86876be4ce46a49ec578dfda897593 -- name: LABEL_NAME - type: static - value: Sample 1 -- name: GCP_ROOT_PASSWORD - type: secret - secretProvider: - environment: gcp - type: SecretManager - secretId: my-secret-id-02 - version: latest -- name: AWS_ADMIN_PASSWORD - type: secret - secretProvider: - environment: aws - type: SecretsManager - secretId: my_azure_root_secret - version: AWSCURRENT -- name: HOST_NAME - type: metadata - metadataProvider: - environment: azure - type: compute - field: name -- name: SELF_IP_INTERNAL - type: metadata - metadataProvider: - environment: azure - type: network - field: ipv4 - index: 1 -- name: SELF_IP_EXTERNAL - type: metadata - metadataProvider: - environment: azure - type: network - field: ipv4 - index: 2 + - name: AZURE_SERVICE_PRINCIPAL + type: secret + secretProvider: + environment: azure + type: KeyVault + vaultUrl: https://my-keyvault.vault.azure.net + secretId: my_azure_admin_secret + version: 6e86876be4ce46a49ec578dfda897593 + - name: LABEL_NAME + type: static + value: Sample 1 + - name: GCP_ROOT_PASSWORD + type: secret + secretProvider: + environment: gcp + type: SecretManager + secretId: my-secret-id-02 + version: latest + - name: AWS_ADMIN_PASSWORD + type: secret + secretProvider: + environment: aws + type: SecretsManager + secretId: my_azure_root_secret + version: AWSCURRENT + - name: HOST_NAME + type: metadata + metadataProvider: + environment: azure + type: compute + field: name + - name: SELF_IP_INTERNAL + type: metadata + metadataProvider: + environment: azure + type: network + field: ipv4 + index: 1 + - name: SELF_IP_EXTERNAL + type: metadata + metadataProvider: + environment: azure + type: network + field: ipv4 + index: 2 pre_onboard_enabled: - name: provision_rest type: inline @@ -57,43 +57,43 @@ pre_onboard_enabled: post_onboard_enabled: [] extension_packages: install_operations: - - extensionType: do - extensionVersion: 1.16.0 - extensionHash: 536eccb9dbf40aeabd31e64da8c5354b57d893286ddc6c075ecc9273fcca10a1 - - extensionType: as3 - extensionVersion: 3.23.0 + - extensionType: do + extensionVersion: 1.17.0 + extensionHash: a359aa8aa14dc565146d4ccc413f169f1e8d02689559c5e4a652f91609a55fbb + - extensionType: as3 + extensionVersion: 3.24.0 extension_services: service_operations: - - extensionType: do - type: url - value: https://cdn.f5.com/product/cloudsolutions/templates/f5-azure-arm-templates/examples/modules/bigip/autoscale_do.json - - extensionType: as3 - type: inline - value: - class: AS3 - action: deploy - persist: true - declaration: - class: ADC - schemaVersion: 3.0.0 - label: "{{{ LABEL_NAME }}}" - remark: Simple HTTP Service with Round-Robin Load Balancing - Sample_01: - class: Tenant - A1: - class: Application - template: http - serviceMain: - class: Service_HTTP - virtualAddresses: - - 10.0.1.10 - pool: web_pool - web_pool: - class: Pool - monitors: - - http - members: - - servicePort: 80 - serverAddresses: - - 192.0.1.10 - - 192.0.1.11 + - extensionType: do + type: url + value: https://cdn.f5.com/product/cloudsolutions/templates/f5-azure-arm-templates/examples/modules/bigip/autoscale_do.json + - extensionType: as3 + type: inline + value: + class: AS3 + action: deploy + persist: true + declaration: + class: ADC + schemaVersion: 3.0.0 + label: "{{{ LABEL_NAME }}}" + remark: Simple HTTP Service with Round-Robin Load Balancing + Sample_01: + class: Tenant + A1: + class: Application + template: http + serviceMain: + class: Service_HTTP + virtualAddresses: + - 10.0.1.10 + pool: web_pool + web_pool: + class: Pool + monitors: + - http + members: + - servicePort: 80 + serverAddresses: + - 192.0.1.10 + - 192.0.1.11 diff --git a/examples/config/cloud_config_local_w_do.yaml b/examples/config/cloud_config_local_w_do.yaml index cea577aa..8addfa7d 100644 --- a/examples/config/cloud_config_local_w_do.yaml +++ b/examples/config/cloud_config_local_w_do.yaml @@ -31,50 +31,50 @@ pre_onboard_enabled: - echo 'sys provision asm { level nominal }' >> bigip_base.conf extension_packages: install_operations: - - extensionType: do - extensionVersion: 1.16.0 + - extensionType: do + extensionVersion: 1.17.0 extension_services: service_operations: - - extensionType: do - type: inline - value: - schemaVersion: 1.0.0 - class: Device - async: true - label: my BIG-IP declaration for declarative onboarding - Common: - class: Tenant - hostname: "{{{ HOST_NAME }}}.local" - internal: - class: VLAN - tag: 4093 - mtu: 1500 - interfaces: - - name: '1.2' - tagged: true - internal-self: - class: SelfIp - address: "{{{ SELF_IP_INTERNAL }}}" - vlan: internal - allowService: default - trafficGroup: traffic-group-local-only - external: - class: VLAN - tag: 4094 - mtu: 1500 - interfaces: - - name: '1.1' - tagged: true - external-self: - class: SelfIp - address: "{{{ SELF_IP_EXTERNAL }}}" - vlan: external - allowService: none - trafficGroup: traffic-group-local-only - provision: - class: Provision - ltm: nominal - gtm: minimum + - extensionType: do + type: inline + value: + schemaVersion: 1.0.0 + class: Device + async: true + label: my BIG-IP declaration for declarative onboarding + Common: + class: Tenant + hostname: "{{{ HOST_NAME }}}.local" + internal: + class: VLAN + tag: 4093 + mtu: 1500 + interfaces: + - name: '1.2' + tagged: true + internal-self: + class: SelfIp + address: "{{{ SELF_IP_INTERNAL }}}" + vlan: internal + allowService: default + trafficGroup: traffic-group-local-only + external: + class: VLAN + tag: 4094 + mtu: 1500 + interfaces: + - name: '1.1' + tagged: true + external-self: + class: SelfIp + address: "{{{ SELF_IP_EXTERNAL }}}" + vlan: external + allowService: none + trafficGroup: traffic-group-local-only + provision: + class: Provision + ltm: nominal + gtm: minimum post_hook: - name: example_webhook type: webhook diff --git a/examples/config/cloud_config_secret.yaml b/examples/config/cloud_config_secret.yaml index c7cfdd0d..f9c259b7 100644 --- a/examples/config/cloud_config_secret.yaml +++ b/examples/config/cloud_config_secret.yaml @@ -1,7 +1,7 @@ runtime_parameters: - name: AZURE_SERVICE_PRINCIPAL type: secret - secretProvider: + secretProvider: type: KeyVault environment: azure vault: my-keyvault.vault.azure.net @@ -18,16 +18,16 @@ pre_onboard_enabled: - echo 'sys provision asm { level nominal }' >> bigip_base.conf post_onboard_enabled: [] extension_packages: - install_operations: - - extensionType: do - extensionVersion: 1.16.0 - - extensionType: as3 - extensionVersion: 3.23.0 + install_operations: + - extensionType: do + extensionVersion: 1.17.0 + - extensionType: as3 + extensionVersion: 3.24.0 extension_services: - service_operations: - - extensionType: do - type: url - value: https://cdn.f5.com/product/cloudsolutions/f5-azure-arm-templates/examples/modules/autoscale_bigip/do.json - - extensionType: as3 - type: url - value: https://cdn.f5.com/product/cloudsolutions/f5-azure-arm-templates/examples/modules/autoscale_bigip/as3.json \ No newline at end of file + service_operations: + - extensionType: do + type: url + value: https://cdn.f5.com/product/cloudsolutions/f5-azure-arm-templates/examples/modules/autoscale_bigip/do.json + - extensionType: as3 + type: url + value: https://cdn.f5.com/product/cloudsolutions/f5-azure-arm-templates/examples/modules/autoscale_bigip/as3.json diff --git a/examples/config/complete_examples.yaml b/examples/config/complete_examples.yaml index a1ebca5c..9a73e287 100644 --- a/examples/config/complete_examples.yaml +++ b/examples/config/complete_examples.yaml @@ -14,7 +14,7 @@ example_5: description: Renders secret referenced within DO declaration to configure the admin password on a BIG-IP device in GCP. runtime_config: example_6: - description: Replaces variables used within DO declaration with properties from instance metadata to configure hostname and self IP addresses on BIGIP device. + description: Replaces variables used within DO and AS3 declarations with properties from instance metadata to configure hostname, self IP addresses and pool members on BIGIP device. runtime_config: example_7: description: Installs AS3 and DO and uses an inline AS3 declaration to setup the BIG-IP. diff --git a/examples/config/example_1.yaml b/examples/config/example_1.yaml index e09f9fbb..d200c285 100644 --- a/examples/config/example_1.yaml +++ b/examples/config/example_1.yaml @@ -4,14 +4,19 @@ pre_onboard_enabled: commands: - /usr/bin/setdb provision.extramb 500 - /usr/bin/setdb restjavad.useextramb true +bigip_ready_enabled: + - name: set_message_size + type: inline + commands: + - '/usr/bin/curl -s -f -u admin: -H "Content-Type: application/json" -d ''{"maxMessageBodySize":134217728}'' -X POST http://localhost:8100/mgmt/shared/server/messaging/settings/8100 | jq .' extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 - extensionHash: 536eccb9dbf40aeabd31e64da8c5354b57d893286ddc6c075ecc9273fcca10a1 + extensionVersion: 1.17.0 + extensionHash: 484b34ecbbe6b97a3b7d3c547a6d9c234c9fc99766d07bed4384298437327f23 - extensionType: as3 - extensionVersion: 3.23.0 - extensionHash: de615341b91beaed59195dceefc122932580d517600afce1ba8d3770dfe42d28 + extensionVersion: 3.24.0 + extensionHash: 3dbb3e2cb170d2fc9ead746aa2df37518c0f38876debe33eec95cf928f91a329 extension_services: service_operations: - extensionType: as3 diff --git a/examples/config/example_10.yaml b/examples/config/example_10.yaml index 34544c9d..f5ffaf27 100644 --- a/examples/config/example_10.yaml +++ b/examples/config/example_10.yaml @@ -7,9 +7,9 @@ pre_onboard_enabled: extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 post_hook: - name: example_webhook type: webhook diff --git a/examples/config/example_11.yaml b/examples/config/example_11.yaml index aa84f100..22225612 100644 --- a/examples/config/example_11.yaml +++ b/examples/config/example_11.yaml @@ -1,60 +1,18 @@ -pre_onboard_enabled: - - name: example_remote_exec - type: url - commands: - - https://ak-metadata-package-poc.s3.amazonaws.com/remote_pre_onboard.sh - - name: provision_rest - type: inline - commands: - - /usr/bin/setdb provision.extramb 500 - - /usr/bin/setdb restjavad.useextramb true -post_onboard_enabled: - - name: example_inline_command - type: inline - commands: - - touch /tmp/post_onboard_script.sh - - chmod 777 /tmp/post_onboard_script.sh - - echo "touch /tmp/created_by_autogenerated_post_local" > /tmp/post_onboard_script.sh - - name: example_local_exec - type: file - commands: - - /tmp/post_onboard_script.sh - - name: example_remote_exec - type: url - commands: - - https://ak-metadata-package-poc.s3.amazonaws.com/remote_post_onboard.sh - verifyTls: false - - name: example_remote_exec - type: url - commands: - - https://ak-metadata-package-poc.s3.amazonaws.com/remote_post_onboard.sh -extension_packages: - install_operations: - - extensionType: do - extensionVersion: 1.16.0 - - extensionType: as3 - extensionVersion: 3.23.0 - verifyTls: false - extensionUrl: https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.23.0/f5-appsvcs-3.23.0-5.noarch.rpm - extensionHash: de615341b91beaed59195dceefc122932580d517600afce1ba8d3770dfe42d28 - - extensionType: ilx - extensionUrl: file:///var/config/rest/downloads/f5-appsvcs-templates-1.1.0-1.noarch.rpm - extensionVersion: 1.1.0 - extensionVerificationEndpoint: /mgmt/shared/fast/info -extension_services: - service_operations: - - extensionType: do - type: url - value: https://cdn.f5.com/product/cloudsolutions/declarations/autoscale-waf/autoscale_do_payg.json - verifyTls: false - - extensionType: as3 - type: url - value: https://cdn.f5.com/product/cloudsolutions/templates/f5-azure-arm-templates/examples/modules/bigip/autoscale_as3.json -post_hook: - - name: example_webhook - type: webhook - verifyTls: false - url: https://postman-echo.com/post - properties: - optionalKey1: optional_value1 - optionalKey2: optional_value2 +pre_onboard_enabled: [{name: "example_remote_exec", type: "url", commands: ["https://ak-metadata-package-poc.s3.amazonaws.com/remote_pre_onboard.sh"]}, + {name: "provision_rest", type: "inline", commands: ["/usr/bin/setdb provision.extramb\ + \ 500", "/usr/bin/setdb restjavad.useextramb true"]}] +post_onboard_enabled: [{name: "example_inline_command", type: "inline", commands: [ + "touch /tmp/post_onboard_script.sh", "chmod 777 /tmp/post_onboard_script.sh", + "echo \"touch /tmp/created_by_autogenerated_post_local\" > /tmp/post_onboard_script.sh"]}, + {name: "example_local_exec", type: "file", commands: ["/tmp/post_onboard_script.sh"]}, + {name: "example_remote_exec", type: "url", commands: ["https://ak-metadata-package-poc.s3.amazonaws.com/remote_post_onboard.sh"], + verifyTls: false}, {name: "example_remote_exec", type: "url", commands: ["https://ak-metadata-package-poc.s3.amazonaws.com/remote_post_onboard.sh"]}] +extension_packages: {install_operations: [{extensionType: "do", extensionVersion: "1.17.0"}, + {extensionType: "as3", extensionVersion: "3.24.0", verifyTls: false, extensionUrl: "https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.24.0/f5-appsvcs-3.24.0-5.noarch.rpm", + extensionHash: "bc21ff6460e65610c50d8eb358155098f79a10371e69c166764735a38a3523cd"}, + {extensionType: "ilx", extensionUrl: "file:///var/config/rest/downloads/f5-appsvcs-templates-1.1.0-1.noarch.rpm", + extensionVersion: "1.1.0", extensionVerificationEndpoint: "/mgmt/shared/fast/info"}]} +extension_services: {service_operations: [{extensionType: "do", type: "url", value: "https://cdn.f5.com/product/cloudsolutions/declarations/autoscale-waf/autoscale_do_payg.json", + verifyTls: false}, {extensionType: "as3", type: "url", value: "https://cdn.f5.com/product/cloudsolutions/templates/f5-azure-arm-templates/examples/modules/bigip/autoscale_as3.json"}]} +post_hook: [{name: "example_webhook", type: "webhook", verifyTls: false, url: "https://postman-echo.com/post", + properties: {optionalKey1: "optional_value1", optionalKey2: "optional_value2"}}] diff --git a/examples/config/example_12.yaml b/examples/config/example_12.yaml index bb7d473e..bf6a0be1 100644 --- a/examples/config/example_12.yaml +++ b/examples/config/example_12.yaml @@ -21,9 +21,9 @@ pre_onboard_enabled: extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 extension_services: service_operations: - extensionType: do diff --git a/examples/config/example_2.yaml b/examples/config/example_2.yaml index 34bc8c34..ceb7d34a 100644 --- a/examples/config/example_2.yaml +++ b/examples/config/example_2.yaml @@ -1,22 +1,38 @@ -pre_onboard_enabled: - - name: provision_rest - type: inline - commands: - - /usr/bin/setdb provision.extramb 500 - - /usr/bin/setdb restjavad.useextramb true -extension_packages: - install_operations: - - extensionType: do - extensionUrl: file:///var/config/rest/downloads/f5-declarative-onboarding-1.16.0-8.noarch.rpm - extensionHash: 536eccb9dbf40aeabd31e64da8c5354b57d893286ddc6c075ecc9273fcca10a1 - extensionVersion: 1.16.0 - - extensionType: ilx - extensionUrl: file:///var/config/rest/downloads/myIlxApp.rpm - extensionVersion: 1.1.0 - extensionVerificationEndpoint: /mgmt/shared/myIlxApp/info - extensionHash: de615341b91beaed59195dceefc122932580d517600afce1ba8d3770dfe42d28 -extension_services: - service_operations: - - extensionType: do - type: url - value: file:///var/config/rest/downloads/do.json +{ + "pre_onboard_enabled": [ + { + "name": "provision_rest", + "type": "inline", + "commands": [ + "/usr/bin/setdb provision.extramb 500", + "/usr/bin/setdb restjavad.useextramb true" + ] + } + ], + "extension_packages": { + "install_operations": [ + { + "extensionType": "do", + "extensionUrl": "file:///var/config/rest/downloads/f5-declarative-onboarding-1.17.0-3.noarch.rpm", + "extensionHash": "a359aa8aa14dc565146d4ccc413f169f1e8d02689559c5e4a652f91609a55fbb", + "extensionVersion": "1.17.0" + }, + { + "extensionType": "ilx", + "extensionUrl": "file:///var/config/rest/downloads/myIlxApp.rpm", + "extensionVersion": "1.1.0", + "extensionVerificationEndpoint": "/mgmt/shared/myIlxApp/info", + "extensionHash": "de615341b91beaed59195dceefc122932580d517600afce1ba8d3770dfe42d28" + } + ] + }, + "extension_services": { + "service_operations": [ + { + "extensionType": "do", + "type": "url", + "value": "file:///var/config/rest/downloads/do.json" + } + ] + } +} diff --git a/examples/config/example_3.yaml b/examples/config/example_3.yaml index f180bbcd..290ffdd0 100644 --- a/examples/config/example_3.yaml +++ b/examples/config/example_3.yaml @@ -15,9 +15,9 @@ pre_onboard_enabled: extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 extension_services: service_operations: - extensionType: do diff --git a/examples/config/example_4.yaml b/examples/config/example_4.yaml index 221af0f7..342d6d4d 100644 --- a/examples/config/example_4.yaml +++ b/examples/config/example_4.yaml @@ -22,9 +22,9 @@ pre_onboard_enabled: extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 extension_services: service_operations: - extensionType: do diff --git a/examples/config/example_5.yaml b/examples/config/example_5.yaml index 69157497..2b2a23bc 100644 --- a/examples/config/example_5.yaml +++ b/examples/config/example_5.yaml @@ -22,9 +22,9 @@ pre_onboard_enabled: extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 extension_services: service_operations: - extensionType: do diff --git a/examples/config/example_6.yaml b/examples/config/example_6.yaml index 3ad5637d..3a740d06 100644 --- a/examples/config/example_6.yaml +++ b/examples/config/example_6.yaml @@ -26,6 +26,15 @@ runtime_parameters: type: network field: subnet-ipv4-cidr-block index: 1 + - name: REGION + type: url + value: http://169.254.169.254/latest/dynamic/instance-identity/document + query: region + headers: + - name: Content-type + value: json + - name: User-Agent + value: bigip-ve pre_onboard_enabled: - name: provision_rest type: inline @@ -35,11 +44,14 @@ pre_onboard_enabled: extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 extension_services: service_operations: - extensionType: do type: url value: file:///examples/declarations/example_6_do.json + - extensionType: as3 + type: url + value: file:///examples/declarations/example_7_as3.json diff --git a/examples/config/example_7.yaml b/examples/config/example_7.yaml index 24083cca..d6501d37 100644 --- a/examples/config/example_7.yaml +++ b/examples/config/example_7.yaml @@ -7,9 +7,9 @@ pre_onboard_enabled: extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 extension_services: service_operations: - extensionType: as3 diff --git a/examples/config/example_8.yaml b/examples/config/example_8.yaml index 9f1d47d1..623ad381 100644 --- a/examples/config/example_8.yaml +++ b/examples/config/example_8.yaml @@ -14,9 +14,9 @@ pre_onboard_enabled: extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 extension_services: service_operations: - extensionType: do diff --git a/examples/config/example_9.yaml b/examples/config/example_9.yaml index 2ac1b8ad..9ef92275 100644 --- a/examples/config/example_9.yaml +++ b/examples/config/example_9.yaml @@ -36,6 +36,6 @@ post_onboard_enabled: extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 diff --git a/examples/config/extension_packages_default.yaml b/examples/config/extension_packages_default.yaml index c08bc1cd..a2e4f21c 100644 --- a/examples/config/extension_packages_default.yaml +++ b/examples/config/extension_packages_default.yaml @@ -2,6 +2,6 @@ description: Installs packages from metadata using latest versions extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 diff --git a/examples/config/extension_packages_hashed.yaml b/examples/config/extension_packages_hashed.yaml index 0f4c3694..04be0937 100644 --- a/examples/config/extension_packages_hashed.yaml +++ b/examples/config/extension_packages_hashed.yaml @@ -2,8 +2,8 @@ description: Verifies and installs packages using specified hashes extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 - extensionHash: 536eccb9dbf40aeabd31e64da8c5354b57d893286ddc6c075ecc9273fcca10a1 + extensionVersion: 1.17.0 + extensionHash: a359aa8aa14dc565146d4ccc413f169f1e8d02689559c5e4a652f91609a55fbb - extensionType: as3 - extensionVersion: 3.23.0 - extensionHash: de615341b91beaed59195dceefc122932580d517600afce1ba8d3770dfe42d28 + extensionVersion: 3.24.0 + extensionHash: eaef5c650da84f593d17939e3bb5d649097f60a6d302d6abc06951f2b4a986d9 diff --git a/examples/config/extension_packages_ilx.yaml b/examples/config/extension_packages_ilx.yaml index 74f8c79c..d8aee8a4 100644 --- a/examples/config/extension_packages_ilx.yaml +++ b/examples/config/extension_packages_ilx.yaml @@ -2,9 +2,9 @@ description: Installs a custom iLX package extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 - extensionType: ilx extensionUrl: file:///var/config/rest/downloads/f5-appsvcs-templates-1.1.0-1.noarch.rpm extensionVersion: 1.1.0 diff --git a/examples/config/extension_packages_url.yaml b/examples/config/extension_packages_url.yaml index 621e7126..1fc5dbe1 100644 --- a/examples/config/extension_packages_url.yaml +++ b/examples/config/extension_packages_url.yaml @@ -1,9 +1,17 @@ -description: Installs packages from custom locations -extension_packages: - install_operations: - - extensionType: do - extensionVersion: 1.16.0 - extensionUrl: https://github.com/F5Networks/f5-declarative-onboarding/releases/download/v1.16.0/f5-declarative-onboarding-1.16.0-8.noarch.rpm - - extensionType: as3 - extensionVersion: 3.23.0 - extensionUrl: file:///var/config/rest/downloads/f5-appsvcs-3.23.0-5.noarch.rpm +{ + "description": "Installs packages from custom locations", + "extension_packages": { + "install_operations": [ + { + "extensionType": "do", + "extensionVersion": "1.17.0", + "extensionUrl": "https://github.com/F5Networks/f5-declarative-onboarding/releases/download/v1.17.0/f5-declarative-onboarding-1.17.0-3.noarch.rpm" + }, + { + "extensionType": "as3", + "extensionVersion": "3.24.0", + "extensionUrl": "file:///var/config/rest/downloads/f5-appsvcs-3.24.0-5.noarch.rpm" + } + ] + } +} diff --git a/examples/config/extension_packages_versioned.yaml b/examples/config/extension_packages_versioned.yaml index da480e21..bc509836 100644 --- a/examples/config/extension_packages_versioned.yaml +++ b/examples/config/extension_packages_versioned.yaml @@ -2,6 +2,6 @@ description: Installs packages using specific versions extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 diff --git a/examples/config/readme_snippet_01.yaml b/examples/config/readme_snippet_01.yaml index 59ec65c3..ff65b185 100644 --- a/examples/config/readme_snippet_01.yaml +++ b/examples/config/readme_snippet_01.yaml @@ -6,20 +6,26 @@ runtime_parameters: type: KeyVault vaultUrl: https://my-keyvault.vault.azure.net secretId: ${secret_id} + field: password pre_onboard_enabled: - name: provision_rest type: inline commands: - /usr/bin/setdb provision.extramb 500 - /usr/bin/setdb restjavad.useextramb true +bigip_ready_enabled: + - name: set_message_size + type: inline + commands: + - '/usr/bin/curl -s -f -u admin: -H "Content-Type: application/json" -d ''{"maxMessageBodySize":134217728}'' -X POST http://localhost:8100/mgmt/shared/server/messaging/settings/8100 | jq .' extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 - extensionType: ts - extensionVersion: 1.15.0 + extensionVersion: 1.16.0 extension_services: service_operations: - extensionType: do @@ -28,4 +34,3 @@ extension_services: - extensionType: as3 type: url value: https://raw.githubusercontent.com/F5Networks/f5-bigip-runtime-init/main/examples/declarations/as3.json - diff --git a/examples/config/readme_snippet_02.yaml b/examples/config/readme_snippet_02.yaml index 9ad2bd1b..2f49ee9b 100644 --- a/examples/config/readme_snippet_02.yaml +++ b/examples/config/readme_snippet_02.yaml @@ -15,11 +15,11 @@ pre_onboard_enabled: extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 - extensionType: ts - extensionVersion: 1.15.0 + extensionVersion: 1.16.0 extension_services: service_operations: - extensionType: do diff --git a/examples/config/readme_snippet_03.yaml b/examples/config/readme_snippet_03.yaml index 07581708..9bf00b79 100644 --- a/examples/config/readme_snippet_03.yaml +++ b/examples/config/readme_snippet_03.yaml @@ -15,11 +15,11 @@ pre_onboard_enabled: extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 - extensionType: ts - extensionVersion: 1.15.0 + extensionVersion: 1.16.0 extension_services: service_operations: - extensionType: do diff --git a/examples/declarations/example_6_as3.json b/examples/declarations/example_6_as3.json new file mode 100644 index 00000000..5b5dc669 --- /dev/null +++ b/examples/declarations/example_6_as3.json @@ -0,0 +1,75 @@ +{ + "remark":"ASM_VS1", + "schemaVersion":"3.0.0", + "label":"ASM_VS1", + "class":"ADC", + "Sample_app_sec_Tenant":{ + "class":"Tenant", + "Shared":{ + "class":"Application", + "template":"shared", + "shared_pool":{ + "class":"Pool", + "remark":"Service 1 shared pool", + "monitors":[ + "http" + ], + "members":[ + { + "servicePort": 80, + "addressDiscovery": "aws", + "region": "{{{REGION}}}", + "updateInterval": 20, + "tagKey": "aws:cloudformation:logical-id", + "tagValue": "appAutoscaleGroup", + "addressRealm": "private" + } + ] + } + }, + "HTTPS_Service":{ + "template":"https", + "class":"Application", + "serviceMain":{ + "virtualAddresses":[ + "0.0.0.0" + ], + "snat":"auto", + "policyWAF":{ + "use":"WAFPolicy" + }, + "pool":"/Sample_app_sec_Tenant/Shared/shared_pool", + "serverTLS":{ + "bigip":"/Common/clientssl" + }, + "redirect80":false, + "class":"Service_HTTPS" + }, + "WAFPolicy":{ + "ignoreChanges":true, + "url":"https://raw.githubusercontent.com/f5devcentral/f5-asm-policy-templates/master/owasp_ready_template/owasp-no-auto-tune-v1.1.xml", + "class":"WAF_Policy" + } + }, + "HTTP_Service":{ + "template":"http", + "class":"Application", + "serviceMain":{ + "class":"Service_HTTP", + "virtualAddresses":[ + "0.0.0.0" + ], + "snat":"auto", + "policyWAF":{ + "use":"WAFPolicy" + }, + "pool":"/Sample_app_sec_Tenant/Shared/shared_pool" + }, + "WAFPolicy":{ + "ignoreChanges":true, + "url":"https://raw.githubusercontent.com/f5devcentral/f5-asm-policy-templates/master/owasp_ready_template/owasp-no-auto-tune-v1.1.xml", + "class":"WAF_Policy" + } + } + } +} diff --git a/package-lock.json b/package-lock.json index 68cbba79..9a10b8e7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "f5-bigip-runtime-init", - "version": "1.1.0", + "version": "1.2.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index eed53f8d..349bb083 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "f5-bigip-runtime-init", - "version": "1.1.0", + "version": "1.2.0", "release": "1", "description": "Cloud Onboarder for F5 BIG-IP", "author": "F5 Cloud Solutions", @@ -40,7 +40,8 @@ "mustache": "^4.0.0", "request": "^2.88.2", "uuid": "^8.2.0", - "winston": "^3.3.3" + "winston": "^3.3.3", + "jmespath": "^0.15.0" }, "dependencyMap": { "common": { @@ -51,7 +52,8 @@ "@f5devcentral/f5-teem": "^1.4.6", "js-yaml": "^3.13.1", "mustache": "^4.0.0", - "request": "^2.88.0" + "request": "^2.88.0", + "jmespath": "^0.15.0" }, "gcp": { "google-auth-library": "^5.6.1", diff --git a/plans/aws/main.tf b/plans/aws/main.tf index 83851773..71fbd457 100644 --- a/plans/aws/main.tf +++ b/plans/aws/main.tf @@ -298,10 +298,6 @@ resource "aws_instance" "vm0" { iam_instance_profile = "${aws_iam_instance_profile.instance_profile.name}" user_data = "${data.template_file.user_data_vm0.rendered}" tags = merge(var.global_tags, {Name="runtime-init-vm0-${module.utils.env_prefix}"}) - # Wait until the instance is in a running state - provisioner "local-exec" { - command = "aws ec2 wait instance-status-ok --instance-ids ${aws_instance.vm0.id} --region ${var.AWS_DEFAULT_REGION}" - } depends_on = [null_resource.delay] } diff --git a/plans/aws/user_data.tpl b/plans/aws/user_data.tpl index 80d94928..63cb3788 100644 --- a/plans/aws/user_data.tpl +++ b/plans/aws/user_data.tpl @@ -30,6 +30,7 @@ runtime_parameters: type: SecretsManager secretId: ${secret_id} version: AWSCURRENT + field: password - name: HOST_NAME type: metadata metadataProvider: @@ -57,6 +58,15 @@ runtime_parameters: type: network field: subnet-ipv4-cidr-block index: 0 + - name: REGION + type: url + value: http://169.254.169.254/latest/dynamic/instance-identity/document + query: region + headers: + - name: Content-Type + value: json + - name: User-Agent + value: func-test pre_onboard_enabled: - name: provision_modules type: inline @@ -67,6 +77,7 @@ pre_onboard_enabled: commands: - /usr/bin/setdb provision.extramb 500 - /usr/bin/setdb restjavad.useextramb true +bigip_ready_enabled: - name: example_inline_command type: inline commands: @@ -99,11 +110,10 @@ post_onboard_enabled: extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 - extensionUrl: https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.23.0/f5-appsvcs-3.23.0-5.noarch.rpm - extensionHash: de615341b91beaed59195dceefc122932580d517600afce1ba8d3770dfe42d28 + extensionUrl: https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.24.0/f5-appsvcs-3.24.0-5.noarch.rpm + extensionHash: df786fc755c5de6f3fcc47638caf3db4c071fcd9cf37855de78fd7e25e5117b4 - extensionType: ilx extensionUrl: https://github.com/f5networks/f5-appsvcs-templates/releases/download/v1.1.0/f5-appsvcs-templates-1.1.0-1.noarch.rpm extensionVerificationEndpoint: /mgmt/shared/fast/info @@ -115,4 +125,4 @@ extension_services: value: https://cdn.f5.com/product/cloudsolutions/templates/f5-aws-cloudformation/examples/modules/failover_bigip/do.json - extensionType: as3 type: url - value: https://cdn.f5.com/product/cloudsolutions/templates/f5-azure-arm-templates/examples/modules/bigip/autoscale_as3.json + value: https://f5-cft.s3.amazonaws.com/autoscale_as3_aws.json diff --git a/plans/azure/user_data.tpl b/plans/azure/user_data.tpl index 9e31237f..9a6660a0 100644 --- a/plans/azure/user_data.tpl +++ b/plans/azure/user_data.tpl @@ -47,6 +47,7 @@ pre_onboard_enabled: commands: - /usr/bin/setdb provision.extramb 500 - /usr/bin/setdb restjavad.useextramb true +bigip_ready_enabled: - name: example_inline_command type: inline commands: @@ -84,12 +85,12 @@ post_onboard_enabled: extension_packages: install_operations: - extensionType: do - extensionVersion: 1.16.0 + extensionVersion: 1.17.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 verifyTls: false - extensionUrl: https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.23.0/f5-appsvcs-3.23.0-5.noarch.rpm - extensionHash: de615341b91beaed59195dceefc122932580d517600afce1ba8d3770dfe42d28 + extensionUrl: https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.24.0/f5-appsvcs-3.24.0-5.noarch.rpm + extensionHash: df786fc755c5de6f3fcc47638caf3db4c071fcd9cf37855de78fd7e25e5117b4 - extensionType: ilx extensionUrl: file:///var/lib/cloud/icontrollx_installs/f5-appsvcs-templates-1.1.0-1.noarch.rpm extensionVerificationEndpoint: /mgmt/shared/fast/info diff --git a/plans/base/user_data.tpl b/plans/base/user_data.tpl index 8b69b13e..73e3ca24 100644 --- a/plans/base/user_data.tpl +++ b/plans/base/user_data.tpl @@ -128,10 +128,10 @@ post_onboard_enabled: extension_packages: install_operations: - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 verifyTls: false - extensionUrl: https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.23.0/f5-appsvcs-3.23.0-5.noarch.rpm - extensionHash: de615341b91beaed59195dceefc122932580d517600afce1ba8d3770dfe42d28 + extensionUrl: https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.24.0/f5-appsvcs-3.24.0-5.noarch.rpm + extensionHash: df786fc755c5de6f3fcc47638caf3db4c071fcd9cf37855de78fd7e25e5117b4 - extensionType: ilx extensionUrl: file:///var/lib/cloud/icontrollx_installs/f5-appsvcs-templates-1.1.0-1.noarch.rpm extensionVerificationEndpoint: /mgmt/shared/fast/info diff --git a/plans/gcp/user_data.tpl b/plans/gcp/user_data.tpl index cf46ed5f..d4c7168e 100644 --- a/plans/gcp/user_data.tpl +++ b/plans/gcp/user_data.tpl @@ -109,7 +109,7 @@ runtime_parameters: type: network field: ip index: 0 -pre_onboard_enabled: +bigip_ready_enabled: - name: provision_modules type: inline commands: @@ -119,6 +119,7 @@ pre_onboard_enabled: commands: - /usr/bin/setdb provision.extramb 500 - /usr/bin/setdb restjavad.useextramb true +pre_onboard_enabled: - name: example_inline_command type: inline commands: @@ -158,10 +159,10 @@ extension_packages: - extensionType: do extensionVersion: 1.16.0 - extensionType: as3 - extensionVersion: 3.23.0 + extensionVersion: 3.24.0 verifyTls: false - extensionUrl: https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.23.0/f5-appsvcs-3.23.0-5.noarch.rpm - extensionHash: de615341b91beaed59195dceefc122932580d517600afce1ba8d3770dfe42d28 + extensionUrl: https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.24.0/f5-appsvcs-3.24.0-5.noarch.rpm + extensionHash: df786fc755c5de6f3fcc47638caf3db4c071fcd9cf37855de78fd7e25e5117b4 - extensionType: ilx extensionUrl: file:///var/lib/cloud/icontrollx_installs/f5-appsvcs-templates-1.1.0-1.noarch.rpm extensionVerificationEndpoint: /mgmt/shared/fast/info diff --git a/scripts/README_template.md b/scripts/README_template.md index 7d1d6b35..639a489c 100755 --- a/scripts/README_template.md +++ b/scripts/README_template.md @@ -25,6 +25,7 @@ - [Azure (Terraform) snippet](#azure-terraform-snippet) - [AWS (Terraform) snippet](#aws-terraform-snippet) - [GCP (Terraform) snippet](#gcp-terraform-snippet) + - [Runtime parameters](#runtime-parameters) - [Private Environments](#private-environments) - [Troubleshooting](#troubleshooting) - [F5 Automation Toolchain Components](#f5-automation-toolchain-components) @@ -57,7 +58,7 @@ resulting in a complete overlay deployment tool for configuring a BIG-IP instanc From a high level overview, using this tool involves three steps: -- Step 1: Download OR render inline a runtime-init configuration file (runtime-init-conf.yaml). +- Step 1: Download OR render inline a Runtime Init configuration file (runtime-init-conf.yaml). ```sh curl -o /config/cloud/runtime-init-conf.yaml https://my-source-host/my-repo/bigip-configs/0.0.1/runtime-init-conf.yaml ``` @@ -119,15 +120,17 @@ F5 BIG-IP Runtime Init has been tested and validated with the following versions The F5 BIG-IP Runtime Init configuration consists of the following attributes: -| Attribute | Default Value | Required | Description | +| Attribute | Default Value | Required | Description | | --- | --- | --- | --- | -| extension_packages | none | No | List of URLs to download and install iControl LX extension packages before onboarding. | -| extension_services | none | No | List of declarations to to configure. | -| runtime_parameters | none | No | List of runtime parameters to gather. | -| pre_onboard_enabled | none | No | List of commands to run before sending iControl LX declarations. | -| post_onboard_enabled | none | No | List of commands to run after sending iControl LX declarations. | +| pre_onboard_enabled | none | No | List of commands to run that do not check if BIG-IP and MCPD are up and running. However, execution before BIG-IP is ready depends on cloud agent/download times/etc. | +| runtime_parameters | none | No | List of runtime parameters to gather. | +| bigip_ready_enabled | none | No | List of commands to run after BIG-IP and MCPD are up and running. Example: tmsh commands, misc optimizations, etc. | +| extension_packages | none | No | List of URLs to download and install iControl LX extension packages before onboarding. | +| extension_services | none | No | List of declarations to configure. | +| post_onboard_enabled | none | No | List of commands to run after sending iControl LX declarations. | | post_hook | none | No | Webhook to send upon completion. | + ### Configuration Examples and Schema Documentation See [SCHEMA.md](https://github.com/F5Networks/f5-bigip-runtime-init/blob/main/SCHEMA.md) for complete schema documentation and configuration examples. @@ -295,9 +298,10 @@ runtime_parameters: type: KeyVault vaultUrl: https://my-keyvault.vault.azure.net secretId: mySecret01 + field: password ``` -When BIG-IP is launched, Runtime Init will fetch the **value** for the secret named```mySecret01``` from the native vault and set the runtime variable ``ADMIN_PASS``. Any declarations containing ```{{{ ADMIN_PASS }}}``` (ex. do.json, as3.json templates formatted with mustache) will be populated with the secret **value** (ex. the admin password). +When BIG-IP is launched, Runtime Init will fetch the **value** for the secret named```mySecret01``` from the native vault and set the runtime variable ``ADMIN_PASS``. Any declarations containing ```{{{ ADMIN_PASS }}}``` (ex. do.json, as3.json templates formatted with mustache) will be populated with the secret **value** (ex. the admin password). ***field*** provides field name to which this secret is map to and it instructs Runtime Init to masks the secret value in any logging outputs. #### AWS (Terraform) snippet @@ -349,8 +353,16 @@ curl https://cdn.f5.com/product/cloudsolutions/f5-bigip-runtime-init/v{{ RELEASE f5-bigip-runtime-init --config-file /config/cloud/runtime-init-conf.yaml ``` -NOTE: ```--cloud aws``` is passed to the installer to specify the environment - +NOTES: + - ```--cloud aws``` is passed to the installer to specify the environment + - when extension package includes ```extensionUrl``` field, ```extensionVersion``` is not required; however, ```extensionVersion``` is required when package defined without ```extensionUrl``` + ```yaml + extension_packages: + install_operations: + - extensionType: as3 + extensionUrl: https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.24.0/f5-appsvcs-3.24.0-5.noarch.rpm + ``` + The terraform variable that is templatized is ```${secret_id}``` which will be rendered by terraform before sending to the instance's ```user_data``` parameter. Ex. the rendered ```user_data``` finally sent to BIG-IP will contain the actual name of secret 'mySecret01' to gather at runtime: ex. @@ -459,6 +471,52 @@ f5-bigip-runtime-init --config-file /config/cloud/runtime-init-conf.yaml NOTE: ```--cloud gcp``` is passed to the installer to specify the environment +## Runtime parameters + +runtime_parameters allows to defined list of parameters and these parameters can be used for substituting tokens defined within declarations. There are a few types of parameters: + + * secret - fetches secret from Secret Vault + ```yaml + runtime_parameters: + - name: ADMIN_PASS + type: secret + secretProvider: + environment: azure + type: KeyVault + vaultUrl: https://my-keyvault.vault.azure.net + secretId: mySecret01 + ``` + * metadata - fetches common pre-defined metadata from the Metadata Service + ```yaml + runtime_parameters: + - name: MGMT_ROUTE + type: metadata + metadataProvider: + environment: aws + type: network + field: subnet-ipv4-cidr-block + index: 0 + ``` + * static - defines a static value. Example below will replace AVAILABILITY_ZONE token with "us-west-2a" string + ```yaml + runtime_parameters: + - name: AVAILABILITY_ZONE + type: static + value: us-west-2a + ``` + * url - defines url to fetch a runtime parameter (ex. custom metadata). This parameter allows to provide HTTP headers as well as JMESPath query for querying JSON document/response. The headers and query fields are optional. + ```yaml + runtime_parameters: + - name: REGION + type: url + value: http://169.254.169.254/latest/dynamic/instance-identity/document + query: region + headers: + - name: Content-Type + value: json + - name: User-Agent + value: some-user-agent + ``` ## Private Environments By default, this tool makes calls to the Internet to download a GPG key [here](https://f5-cft.s3.amazonaws.com/f5-bigip-runtime-init/gpg.key) to verify RPM signatures, find the latest Automation Tool Chain packages and send usage data. To disable calls to the Internet, you can use the examples below: @@ -519,6 +577,49 @@ The following enviroment variables can be used for setting logging options: Example of how to set the log level using an environment variable: ```export F5_BIGIP_RUNTIME_INIT_LOG_LEVEL=silly && bash /var/tmp/f5-bigip-runtime-init-{{ RELEASE_VERSION }}-{{ RELEASE_BUILD }}.gz.run -- '--cloud ${CLOUD}'``` + +By default, runtime will mask out (i.e. "********") the following common fields when logging: +```json + [ + "password", + "localPassword", + "remotePassword", + "bigIqPassword", + "bigIpPassword", + "passphrase", + "cookiePassphrase", + "certificate", + "privateKey", + "ciphertext", + "protected", + "secret", + "sharedSecret", + "secretAccessKey", + "apiAccessKey", + "encodedCredentials", + "encodedToken", + "oldPassword", + "newPassword", + "bindPassword", + "checkBindPassword", + "md5SignaturePassphrase" + ] +``` +However, it is possible to extend this list by providing additional metadata (***field***) for the Secret object: + +```yaml + runtime_parameters: + - name: MY_SECRET + type: secret + secretProvider: + environment: azure + type: KeyVault + vaultUrl: https://my-keyvault.vault.azure.net + secretId: mySecret01 + field: newCustomSecretField +``` +This example shows how to instruct Runtime Init to mask out the value for ```newCustomSecretField```. + #### Send output to log file and serial console Add the following to the beginning of user data to log startup events to a local file/serial console. See the simple [example](https://github.com/F5Networks/f5-bigip-runtime-init/blob/main/examples/simple/terraform/startup-script.tpl) for more information. @@ -566,4 +667,4 @@ Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations -under the License. \ No newline at end of file +under the License. diff --git a/scripts/link_checker/package.json b/scripts/link_checker/package.json old mode 100644 new mode 100755 diff --git a/scripts/ssh_to_instance.sh b/scripts/ssh_to_instance.sh new file mode 100755 index 00000000..59bd9c54 --- /dev/null +++ b/scripts/ssh_to_instance.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# usage: ./ssh_to_instance.sh + +INSTANCE_TO_USE=${1:-primary} +PATH_TO_DEPLOYMENT_INFO='deployment_info.json' + +INSTANCES=$(cat $PATH_TO_DEPLOYMENT_INFO | jq .instances -r) +USERNAME=$(echo $INSTANCES | jq '.[0] | .admin_username' -r) +PASSWORD=$(echo $INSTANCES | jq '.[0] | .admin_password' -r) +HOST=$(echo $INSTANCES | jq '.[0] | .mgmt_address' -r) +sshpass -p $PASSWORD ssh -o "StrictHostKeyChecking no" ${USERNAME}@${HOST} diff --git a/src/cli.ts b/src/cli.ts index 993717c9..c2af0f11 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -32,6 +32,7 @@ import { TelemetryClient } from './lib/telemetry/telemetryClient'; const logger = Logger.getLogger(); const executionResults = {}; let telemetryClient; +let statusCode = 0; export async function cli(): Promise { /* eslint-disable no-await-in-loop */ @@ -73,6 +74,16 @@ export async function cli(): Promise { } logger.info('Successfully validated declaration'); + const resolver = new ResolverClient(); + + // pre onboard + // run before install operations in case they require out-of-band changes + const preOnboardEnabled = config.pre_onboard_enabled || []; + if (preOnboardEnabled.length) { + logger.info('Executing custom pre_onboard_enabled commands'); + await resolver.resolveOnboardActions(preOnboardEnabled); + } + // create management client const mgmtClient = new ManagementClient(); // perform ready check @@ -80,22 +91,22 @@ export async function cli(): Promise { telemetryClient = new TelemetryClient( mgmtClient ); - // resolve runtime parameters - const resolver = new ResolverClient(); + logger.info('Resolving parameters'); const resolvedRuntimeParams = config.runtime_parameters !== undefined ? await resolver.resolveRuntimeParameters(config.runtime_parameters): undefined; - // pre onboard - // run before install operations in case they require out-of-band changes - const preOnboardEnabled = config.pre_onboard_enabled || []; - if (preOnboardEnabled.length) { - logger.info('Executing custom pre-onboard commands'); - await resolver.resolveOnboardActions(preOnboardEnabled); + const bigipReadyEnabled = config.bigip_ready_enabled || []; + if (bigipReadyEnabled.length) { + logger.info('Executing custom bigip_ready_enabled commands'); + await resolver.resolveOnboardActions(bigipReadyEnabled); } + await mgmtClient.isReady(); + // perform install operations const extensionPackages = config.extension_packages || {}; const installOperations = extensionPackages.install_operations || []; + const extensionsVersions = {}; if (installOperations.length) { logger.info('Executing install operations.'); for (let i = 0; i < installOperations.length; i += 1) { @@ -104,6 +115,7 @@ export async function cli(): Promise { installOperations[i].extensionType, installOperations[i] ); + extensionsVersions[installOperations[i].extensionType] = installOperations[i].extensionVersion ? installOperations[i].extensionVersion: 'unknown'; const response = await toolchainClient.package.isInstalled(); if (response.isInstalled) { logger.silly('package is already installed'); @@ -129,7 +141,7 @@ export async function cli(): Promise { mgmtClient, serviceOperations[i].extensionType, { - version: serviceOperations[i].extensionVersion + extensionVersion: extensionsVersions[serviceOperations[i].extensionType] ? extensionsVersions[serviceOperations[i].extensionType]: 'unknown' } ); // if the config is already an object, then use it as such, @@ -157,9 +169,10 @@ export async function cli(): Promise { } // post onboard + await mgmtClient.isReady(); const postOnboardEnabled = config.post_onboard_enabled || []; if (postOnboardEnabled.length) { - logger.info('Executing custom post-onboard commands'); + logger.info('Executing custom post_onboard_enabled commands'); await resolver.resolveOnboardActions(postOnboardEnabled); } @@ -190,13 +203,20 @@ export async function cli(): Promise { return 'All operations finished successfully'; } +function exit(): void { + setTimeout(() => { + process.exit(statusCode); + }, 2000); +} + cli() .then((message) => { logger.info(message); - process.exit(); + exit(); }) .catch((err) => { logger.error(err.message); + statusCode = 1; if (!program.skipTelemetry) { executionResults['endTime'] = (new Date()).toISOString(); executionResults['result'] = 'FAILURE'; @@ -212,12 +232,14 @@ cli() return telemetryClient.report(telemetryClient.createTelemetryData()); }).then(() => { logger.info('F5 Teem report was successfully sent for failure case.'); - logger.error(executionResults['resultSummary']); - process.exit(1); + logger.info(executionResults['resultSummary']); + exit(); }) .catch((err2) => { logger.error(err2.message); - process.exit(1); + exit(); }); + } else { + exit(); } }); diff --git a/src/constants.ts b/src/constants.ts index 22e81c88..1467e1f4 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -22,12 +22,17 @@ export const ENVIRONMENT_KEY_NAME= 'environment'; export const HTTP_STATUS_CODES = { ACCEPTED: 202, - OK: 200 + OK: 200, + UNPROCESSABLE: 422, + INTERNALS: 500 }; export const RETRY = { + LONG_COUNT: 150, DEFAULT_COUNT: 100, - DELAY_IN_MS: 10000 + DELAY_IN_MS: 10000, + SHORT_COUNT: 20, + SHORT_DELAY_IN_MS: 1000 }; export const ENV_VARS = { @@ -42,11 +47,27 @@ export const LOGGER = { DEFAULT_LOG_TO_JSON: false, FIELDS_TO_HIDE: [ "password", + "localPassword", + "remotePassword", + "bigIqPassword", + "bigIpPassword", "passphrase", + "cookiePassphrase", "certificate", "privateKey", "ciphertext", - "protected" + "protected", + "secret", + "sharedSecret", + "secretAccessKey", + "apiAccessKey", + "encodedCredentials", + "encodedToken", + "oldPassword", + "newPassword", + "bindPassword", + "checkBindPassword", + "md5SignaturePassphrase" ] }; diff --git a/src/lib/bigip/toolchain/toolChainClient.ts b/src/lib/bigip/toolchain/toolChainClient.ts index 5f18f5b0..fe0ea1aa 100644 --- a/src/lib/bigip/toolchain/toolChainClient.ts +++ b/src/lib/bigip/toolchain/toolChainClient.ts @@ -305,21 +305,34 @@ class PackageClient { } else { utils.verifyDirectory(constants.TMP_DIR); tmpFile = `${constants.TMP_DIR}/${downloadPackageName}`; - await utils.downloadToFile(downloadUrl, tmpFile, { - verifyTls: this._metadataClient._getVerifyTls() - }); + logger.silly(`Downloading file: ${downloadUrl}`); + await utils.retrier( + utils.downloadToFile, + [downloadUrl, tmpFile, { verifyTls: this._metadataClient._getVerifyTls() }], + { + thisContext: this, + maxRetries: constants.RETRY.SHORT_COUNT, + retryInterval: constants.RETRY.SHORT_DELAY_IN_MS + }); } - - // verify package + // verify package integrity if (this._metadataClient.getComponentHash()) { - utils.verifyHash(tmpFile, this._metadataClient.getComponentHash()); + if(!utils.verifyHash(tmpFile, this._metadataClient.getComponentHash())) { + return Promise.reject(new Error(`Installation of ${this.component} failed because RPM hash is not valid`)); + } } - // upload to target if (urlObject.pathname.indexOf(constants.DOWNLOADS_DIR) === -1) { - await this._uploadRpm(tmpFile); + logger.silly(`Uploading RPM.`); + await utils.retrier( + this._uploadRpm, + [tmpFile], + { + thisContext: this, + maxRetries: constants.RETRY.SHORT_COUNT, + retryInterval: constants.RETRY.SHORT_DELAY_IN_MS + }); } - // install on target await this._installRpm(`${constants.DOWNLOADS_DIR}/${downloadPackageName}`); @@ -431,8 +444,10 @@ class ServiceClient { if (taskResponse.code === constants.HTTP_STATUS_CODES.OK) { return Promise.resolve(taskResponse); + } else if (taskResponse.code === constants.HTTP_STATUS_CODES.UNPROCESSABLE || taskResponse.code >= constants.HTTP_STATUS_CODES.INTERNALS) { + return Promise.resolve(taskResponse); } - return Promise.reject(new Error(`Task state has not passed: ${taskResponse.code}`)); + return Promise.reject(`Task is still in progress; status code: ${taskResponse.code}`); } /** @@ -448,11 +463,17 @@ class ServiceClient { * @returns Task response */ async _waitForTask(taskUri: string): Promise { - await utils.retrier(this._checkTaskState, [taskUri], { + const taskResponse = await utils.retrier(this._checkTaskState, [taskUri], { thisContext: this, maxRetries: this.maxRetries, retryInterval: this.retryInterval }); + if (taskResponse.code === undefined) { + throw new Error(`Task response does not include status code: ${taskResponse}`); + } + if (taskResponse.code !== constants.HTTP_STATUS_CODES.OK) { + throw new Error(`Task with error: ${JSON.stringify(taskResponse)}`); + } } /** @@ -506,7 +527,6 @@ class ServiceClient { const config = options.config; logger.info(`Creating - ${this.component} ${this.version} ${utils.stringify(config)}`); - const response = await utils.makeRequest( `${this.uriPrefix}${this._metadataClient.getConfigurationEndpoint().endpoint}`, { @@ -518,12 +538,16 @@ class ServiceClient { verifyTls: this._metadataClient._getVerifyTls() } ); - if (response.code === constants.HTTP_STATUS_CODES.ACCEPTED) { await this._waitForTask(response.body.selfLink.split('https://localhost')[1]); } - return response; + if (response.code === constants.HTTP_STATUS_CODES.ACCEPTED || response.code === constants.HTTP_STATUS_CODES.OK) { + return response; + } else { + logger.warn(`Task creation failed; response code: ${response.code}`); + return Promise.reject(new Error(utils.stringify(response.body))); + } } } diff --git a/src/lib/bigip/toolchain/toolchain_metadata.json b/src/lib/bigip/toolchain/toolchain_metadata.json index 0dda2200..0104568d 100644 --- a/src/lib/bigip/toolchain/toolchain_metadata.json +++ b/src/lib/bigip/toolchain/toolchain_metadata.json @@ -23,10 +23,15 @@ } }, "versions": { + "1.17.0": { + "downloadUrl": "https://github.com/F5Networks/f5-declarative-onboarding/releases/download/v1.17.0/f5-declarative-onboarding-1.17.0-3.noarch.rpm", + "packageName": "f5-declarative-onboarding-1.17.0-3.noarch", + "latest": true + }, "1.16.0": { "downloadUrl": "https://github.com/F5Networks/f5-declarative-onboarding/releases/download/v1.16.0/f5-declarative-onboarding-1.16.0-8.noarch.rpm", "packageName": "f5-declarative-onboarding-1.16.0-8.noarch", - "latest": true + "latest": false }, "1.15.0": { "downloadUrl": "https://github.com/F5Networks/f5-declarative-onboarding/releases/download/v1.15.0/f5-declarative-onboarding-1.15.0-3.noarch.rpm", @@ -129,10 +134,15 @@ } }, "versions": { + "3.24.0": { + "downloadUrl": "https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.24.0/f5-appsvcs-3.24.0-5.noarch.rpm", + "packageName": "f5-appsvcs-3.24.0-5.noarch", + "latest": true + }, "3.23.0": { "downloadUrl": "https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.23.0/f5-appsvcs-3.23.0-5.noarch.rpm", "packageName": "f5-appsvcs-3.23.0-5.noarch", - "latest": true + "latest": false }, "3.22.1": { "downloadUrl": "https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.22.1/f5-appsvcs-3.22.1-1.noarch.rpm", @@ -278,11 +288,6 @@ "downloadUrl": "https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.3.0/f5-appsvcs-3.3.0-6.noarch.rpm", "packageName": "f5-appsvcs-3.3.0-6.noarch", "latest": false - }, - "3.2.0": { - "downloadUrl": "https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.2.0/f5-appsvcs-3.2.0-7.noarch.rpm", - "packageName": "f5-appsvcs-3.2.0-7.noarch", - "latest": false } }, "componentDependencies": { @@ -315,10 +320,15 @@ } }, "versions": { + "1.16.0": { + "downloadUrl": "https://github.com/F5Networks/f5-telemetry-streaming/releases/download/v1.16.0/f5-telemetry-1.16.0-4.noarch.rpm", + "packageName": "f5-telemetry-1.16.0-4.noarch", + "latest": true + }, "1.15.0": { "downloadUrl": "https://github.com/F5Networks/f5-telemetry-streaming/releases/download/v1.15.0/f5-telemetry-1.15.0-4.noarch.rpm", "packageName": "f5-telemetry-1.15.0-4.noarch", - "latest": true + "latest": false }, "1.14.0": { "downloadUrl": "https://github.com/F5Networks/f5-telemetry-streaming/releases/download/v1.14.0/f5-telemetry-1.14.0-2.noarch.rpm", @@ -433,10 +443,20 @@ } }, "versions": { + "1.6.1": { + "downloadUrl": "https://github.com/F5Networks/f5-cloud-failover-extension/releases/download/v1.6.1/f5-cloud-failover-1.6.1-1.noarch.rpm", + "packageName": "f5-cloud-failover-1.6.1-1.noarch", + "latest": true + }, + "1.6.0": { + "downloadUrl": "https://github.com/F5Networks/f5-cloud-failover-extension/releases/download/v1.6.0/f5-cloud-failover-1.6.0-0.noarch.rpm", + "packageName": "f5-cloud-failover-1.6.0-0.noarch", + "latest": false + }, "1.5.0": { "downloadUrl": "https://github.com/F5Networks/f5-cloud-failover-extension/releases/download/v1.5.0/f5-cloud-failover-1.5.0-0.noarch.rpm", "packageName": "f5-cloud-failover-1.5.0-0.noarch", - "latest": true + "latest": false }, "1.4.0": { "downloadUrl": "https://github.com/F5Networks/f5-cloud-failover-extension/releases/download/v1.4.0/f5-cloud-failover-1.4.0-0.noarch.rpm", @@ -481,10 +501,15 @@ } }, "versions": { + "1.5.0": { + "downloadUrl": "https://github.com/F5Networks/f5-appsvcs-templates/releases/download/v1.5.0/f5-appsvcs-templates-1.5.0-1.noarch.rpm", + "packageName": "f5-appsvcs-templates-1.5.0-1.noarch", + "latest": true + }, "1.4.0": { "downloadUrl": "https://github.com/F5Networks/f5-appsvcs-templates/releases/download/v1.4.0/f5-appsvcs-templates-1.4.0-1.noarch.rpm", "packageName": "f5-appsvcs-templates-1.4.0-1.noarch", - "latest": true + "latest": false }, "1.3.0": { "downloadUrl": "https://github.com/F5Networks/f5-appsvcs-templates/releases/download/v1.3.0/f5-appsvcs-templates-1.3.0-1.noarch.rpm", diff --git a/src/lib/cloud/aws/cloudClient.ts b/src/lib/cloud/aws/cloudClient.ts index 32d43857..ef77ace1 100644 --- a/src/lib/cloud/aws/cloudClient.ts +++ b/src/lib/cloud/aws/cloudClient.ts @@ -123,7 +123,7 @@ export class AwsCloudClient extends AbstractCloudClient { if (!type) { throw new Error('AWS Cloud Client metadata type is missing'); } - + if (type !== 'compute' && type !== 'network') { throw new Error('AWS Cloud Client metadata type is unknown. Must be one of [ compute, network ]'); } @@ -134,27 +134,42 @@ export class AwsCloudClient extends AbstractCloudClient { if (type === 'compute'){ return this._getInstanceCompute(field) .catch(err => Promise.reject(err)); - } + } if (type === 'network') { /** grab big-ip interface info */ - const interfaceResponse = await utils.makeRequest( - `http://localhost:8100/mgmt/tm/net/interface`, - { - method: 'GET', - headers: { - Authorization: `Basic ${utils.base64('encode', 'admin:admin')}` - }, - verifyTls: false + let mac; + let retries = 0; + const timer = ms => new Promise(res => setTimeout(res, ms)); + while (true) { + const interfaceResponse = await utils.makeRequest( + `http://localhost:8100/mgmt/tm/net/interface`, + { + method: 'GET', + headers: { + Authorization: `Basic ${utils.base64('encode', 'admin:admin')}` + }, + verifyTls: false + } + ); + logger.silly(`Interface Response: ${utils.stringify(interfaceResponse)}`); + const interfaceName = index === 0 ? 'mgmt' : `1.${index}`; + logger.info('Interface:' + interfaceName ); + const interfaces = interfaceResponse.body.items; + const filtered = where(interfaces, { "name": interfaceName }); + logger.silly(`filtered: ${utils.stringify(filtered)}`); + mac = filtered[0]["macAddress"]; + if (mac !== 'none') { + logger.info('MAC address found for ' + interfaceName + ': ' + mac); + break; + } else { + retries++; + if (retries > constants.RETRY.DEFAULT_COUNT) { + return Promise.reject(new Error(`Failed to fetch MAC address for BIGIP interface ${interfaceName}.`)) + } + logger.info(`MAC adddress is not populated on ${interfaceName} BIGIP interface. Trying to re-fecth interface data. Left attempts: ${constants.RETRY.DEFAULT_COUNT - retries}`); + await timer(constants.RETRY.DELAY_IN_MS); } - ); - logger.debug('Interface Response:' + interfaceResponse); - const interfaceName = index === 0 ? 'mgmt' : `1.${index}`; - logger.info('Interface:' + interfaceName ); - const interfaces = interfaceResponse.body.items; - const filtered = where(interfaces, { "name": interfaceName }); - logger.debug('filtered:' + filtered); - const mac = filtered[0]["macAddress"]; - logger.info('MAC address found for ' + interfaceName + ': ' + mac); + } /** manipulate returned data into ip/mask when field eq local-ipv4s */ if (field === 'local-ipv4s') { const getInstanceNetwork = await this._getInstanceNetwork(field, type, mac) @@ -167,7 +182,7 @@ export class AwsCloudClient extends AbstractCloudClient { const ipMask = primaryIp + cidr.match(/(\/([0-9]{1,2}))/g); logger.info('ip and mask for ' + mac + ': ' + ipMask); return ipMask; - /** manipulate data to form gateway address when field eq subnet-ipv4-cidr-block */ + /** manipulate data to form gateway address when field eq subnet-ipv4-cidr-block */ } else if (field === 'subnet-ipv4-cidr-block') { const cidr = await this._getInstanceNetwork('subnet-ipv4-cidr-block', type, mac) .catch(err => Promise.reject(err)); @@ -209,7 +224,7 @@ export class AwsCloudClient extends AbstractCloudClient { * Gets instance compute type metadata * See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-categories.html for available fields * Type compute supports all categories with base uri + /field - * + * * @returns - A Promise that will be resolved with metadata for the supplied field or * rejected if an error occurs */ @@ -231,7 +246,7 @@ export class AwsCloudClient extends AbstractCloudClient { * Gets instance network type metadata * See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-categories.html for available fields * Type network supports all categories with base uri + /network/interfaces/macs// where is determined by $index - * + * * @returns - A Promise that will be resolved with metadata for the supplied network field or * rejected if an error occurs */ diff --git a/src/lib/logger.ts b/src/lib/logger.ts index 5b419974..15c4f7d3 100644 --- a/src/lib/logger.ts +++ b/src/lib/logger.ts @@ -141,7 +141,7 @@ export default class Logger{ winston.format.printf(info => { let message = info.message ? info.message : ''; for (const fieldName of constants.LOGGER.FIELDS_TO_HIDE) { - message = message.replace(new RegExp(`"${fieldName}":.[^"]+`), `"${fieldName}":"********"`); + message = message.replace(new RegExp(`"${fieldName}":.[^"]+`), `"${fieldName}":"********`); } return `${info.timestamp} [${process.pid}]: ${info.level}: ${message}` }) diff --git a/src/lib/resolver/resolverClient.ts b/src/lib/resolver/resolverClient.ts index dea15c55..84ef3d40 100644 --- a/src/lib/resolver/resolverClient.ts +++ b/src/lib/resolver/resolverClient.ts @@ -17,6 +17,7 @@ 'use strict'; +import * as jmesPath from 'jmespath'; import Logger from '../logger'; import { getCloudProvider } from '../cloud/cloudFactory'; import * as utils from '../utils'; @@ -69,6 +70,11 @@ export class ResolverClient { parameters[i].name, this._resolveMetadata(parameters[i]) )); + } else if (parameters[i].type === 'url') { + promises.push(this._resolveHelper( + parameters[i].name, + this._resolveUrl(parameters[i]) + )); } else { throw new Error('Runtime parameter type is unknown. Must be one of [ secret, static ]'); } @@ -158,6 +164,9 @@ export class ResolverClient { { logger } ); await _cloudClient.init(); + if (secretMetadata.secretProvider.field !== undefined) { + constants.LOGGER.FIELDS_TO_HIDE.push(secretMetadata.secretProvider.field); + } const secretValue = await _cloudClient.getSecret( secretMetadata.secretProvider.secretId, secretMetadata.secretProvider @@ -185,4 +194,30 @@ export class ResolverClient { ); return metadataValue; } + + async _resolveUrl(metadata): Promise { + const options: { + method: string; + headers?: any; + } = { + method: 'GET', + headers: {} + }; + if (metadata.headers !== undefined && metadata.headers.length > 0) { + metadata.headers.forEach((header) => { + options.headers[header.name] = header.value; + }); + } + const response = await utils.makeRequest(metadata.value, options); + if ( metadata.query !== undefined ) { + try { + const searchResult = jmesPath.search(response.body, metadata.query); + return searchResult; + } catch (error) { + throw new Error(`Error caught while searching json using jmesPath; Json Document: ${JSON.stringify(response.body)} - Query: ${metadata.query} - Error ${error.message}`); + } + } else { + return response.body; + } + } } diff --git a/src/lib/utils.ts b/src/lib/utils.ts index 66da694a..a32b46a6 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -37,7 +37,11 @@ const logger = Logger.getLogger(); * @returns stringified data */ export function stringify(data: object): string { - return JSON.stringify(data); + try { + return JSON.stringify(data); + } catch { + return data.toString(); + } } /** @@ -70,6 +74,8 @@ export async function downloadToFile(url: string, file: string, options): Promis .catch(err => Promise.reject(err)); } + + /** * Verifies that directory exists or create directory * @@ -138,7 +144,7 @@ export async function retrier( thisContext?: any; // eslint-disable-line @typescript-eslint/no-explicit-any maxRetries?: number; retryInterval?: number; - }): Promise { + }): Promise { // eslint-disable-line @typescript-eslint/no-explicit-any options = options || {}; const thisContext = options.thisContext || this; @@ -154,6 +160,7 @@ export async function retrier( response = await func.apply(thisContext, args); } catch (err) { error = err; + logger.silly(`${err}`); } if (error === null) { @@ -161,10 +168,12 @@ export async function retrier( } else { await new Promise(resolve => setTimeout(resolve, retryInterval)); i += 1; + logger.silly(`Retrying... Attempts left: ${retryCount - i}`); } } if (error !== null) { + logger.silly(`All ${retryCount} retry attempts failed. Terminating execution.`); return Promise.reject(error); } return response; diff --git a/src/schema/base_schema.json b/src/schema/base_schema.json index e60707bb..d416b54d 100644 --- a/src/schema/base_schema.json +++ b/src/schema/base_schema.json @@ -16,7 +16,7 @@ }, "type": { "type": "string", - "enum": ["static", "secret", "metadata"] + "enum": ["static", "secret", "metadata", "url"] }, "value": { "type": "string", @@ -48,6 +48,11 @@ "type": "string", "pattern": "(https?://(.+?\\.)?vault\\.(azure|usgovcloudapi)\\.net(/[A-Za-z0-9\\-\\._~:/\\?#\\[\\]@!$&'\\(\\)\\*\\+,;\\=]*)?)", "examples": ["https://my-keyvault.vault.azure.net", "https://my-keyvault.vault.usgovcloudapi.net"] + }, + "field": { + "description": "field name to which secret value is mapped to", + "type": "string", + "examples": ["bigiqPassword", "regKey"] } }, "allOf": [ @@ -107,12 +112,23 @@ ], "required": ["environment", "type"], "additionalProperties": false + }, + "query": { + "type": "string", + "examples": ["region"] + }, + "headers": { + "type": "array" } }, "allOf": [ { "if": { "properties": { "type": { "const": "static" } } }, "then": { "required": ["value"] } + }, + { + "if": { "properties": { "type": { "const": "url" } } }, + "then": { "required": ["value"] } } ], "required": ["name", "type"], @@ -150,7 +166,7 @@ } }, "pre_onboard_enabled": { - "description": "Used to specify commands which will be executed before extension package operations.", + "description": "Used to specify commands which will be executed before extension package operations before BIG-IP is ready.", "type": "array", "items": { "type": "object", @@ -178,6 +194,35 @@ } } }, + "bigip_ready_enabled": { + "description": "Used to specify commands which will be executed before extension package operations after BIG-IP and MCPD are up and running.", + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "examples": ["my_preonboard_command", "example_local_exec", "provision_rest"] + }, + "type": { + "type": "string", + "enum": ["inline", "file", "url"] + }, + "command": { + "type": "array", + "items" : { + "type": "string", + "examples": [ "tmsh create net vlan external interfaces replace-all-with { 1.1 }", "tmsh create sys folder /LOCAL_ONLY device-group none traffic-group traffic-group-local-only", "tmsh save sys config" ] + } + }, + "verifyTls": { + "description": "For enabling secure site verification", + "type": "boolean", + "examples": ["true", "false"] + } + } + } + }, "extension_packages": { "description": "Used to specify Automation Toolchain packages to be installed on device.", "type": "object", @@ -218,8 +263,11 @@ "allOf": [ { "if": { "properties": { "extensionType": { "const": "ilx" } } }, - "then": { "required": ["extensionUrl", "extensionVerificationEndpoint", "extensionVersion"] }, - "else": { "required": ["extensionVersion"] } + "then": { "required": ["extensionUrl", "extensionVerificationEndpoint"] } + }, + { + "if": { "properties": { "extensionUrl": { "const": null } } }, + "then": { "required": ["extensionVersion"] } } ], "required": ["extensionType"], diff --git a/tests/functional/providers/aws/tests.js b/tests/functional/providers/aws/tests.js index 1949ba18..e3278df2 100644 --- a/tests/functional/providers/aws/tests.js +++ b/tests/functional/providers/aws/tests.js @@ -24,6 +24,7 @@ describe('Provider: AWS', () => { let testNewAuthData; let testHostName; let testMgmtIp; + let as3Declaration; before(function () { this.timeout(80000); @@ -52,13 +53,25 @@ describe('Provider: AWS', () => { testHostName = response.hostname; }) .then(() => { - const mgmtUri = '/mgmt/tm/sys/management-ip'; + const mgmtUri = '/mgmt/tm/sys/management-ip'; // getting mgmt-ip return funcUtils.getBigipApi(firstDut.ip, firstDut.port, testNewAuthData.token, mgmtUri); }) .then((response) => { testMgmtIp = response.items[0].name.substring(0, response.items[0].name.length -3).replace(/[.]/g,'-') + return Promise.resolve(); }) + .then(() => { + return funcUtils.getInstalledDeclaration( + firstDut.ip, + firstDut.port, + testNewAuthData.token, + 'as3' + ) + }) + .then((data) => { + as3Declaration = data + }) .catch(err => Promise.reject(err)); }); @@ -83,4 +96,9 @@ describe('Provider: AWS', () => { it('should confirm hostname was updated using DO', () => { assert.strictEqual(testHostName, 'ip-' + testMgmtIp + '.' + env.region + '.compute.internal'); }); + + it ('should validate that runtime init resolve region correctly', () => { + assert.ok(!JSON.stringify(as3Declaration).includes('{{{REGION}}}')) + }); + }); diff --git a/tests/functional/systemTests.js b/tests/functional/systemTests.js index 542e0402..dffe7c2b 100644 --- a/tests/functional/systemTests.js +++ b/tests/functional/systemTests.js @@ -84,11 +84,13 @@ describe('System tests', () => { it('should verify AS3 installed', () => { const declaredAs3Version = postedBigIpRunTimeInitDeclaration.extension_packages.install_operations.filter(item => item.extensionType === 'as3')[0].extensionVersion; - const declaredPackageName = availablePackages.components.as3.versions[declaredAs3Version].packageName; - + if (declaredAs3Version !== undefined) { + const declaredPackageName = availablePackages.components.as3.versions[declaredAs3Version].packageName; + assert.ok(installedPackages.filter(item => item.packageName === declaredPackageName).length > 0); + assert.ok(installedPackages.filter(item => item.version === declaredAs3Version).length > 0); + } assert.ok(installedPackages.filter(item => item.name === 'f5-appsvcs').length > 0); - assert.ok(installedPackages.filter(item => item.packageName === declaredPackageName).length > 0); - assert.ok(installedPackages.filter(item => item.version === declaredAs3Version).length > 0); + }); it('should verify DO installed', () => { diff --git a/tests/scripts/verify_bash_available.sh b/tests/scripts/verify_bash_available.sh new file mode 100755 index 00000000..19668fc2 --- /dev/null +++ b/tests/scripts/verify_bash_available.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +checks=0 +exitCode=1 +while [ $checks -lt 120 ]; +do + response=$(sshpass -p ${PASSWORD} ssh -o StrictHostKeyChecking=no -o ConnectionAttempts=120 ${USERNAME}@${MGMT_IP} "bash --version") + if [[ ! -z $response ]]; then + echo 'BIGIP is ready to accept bash commands. Continue installation...' + exitCode=0 + break + else + echo 'BIGIP is not ready to accept bash commands' + exitCode=1 + ((checks=checks+1)) + fi + done + +echo "verify_bash_availability: exitCode: $exitCode" +exit $exitCode diff --git a/tests/unit/bigip/toolchain/toolChainClientTests.ts b/tests/unit/bigip/toolchain/toolChainClientTests.ts index 5e17548d..b1006301 100644 --- a/tests/unit/bigip/toolchain/toolChainClientTests.ts +++ b/tests/unit/bigip/toolchain/toolChainClientTests.ts @@ -21,6 +21,7 @@ import assert from 'assert'; import mock from 'mock-fs'; import nock from 'nock'; +import * as constants from '../../../../src/constants'; import { ManagementClient } from '../../../../src/lib/bigip/managementClient'; import { ToolChainClient } from '../../../../src/lib/bigip/toolchain/toolChainClient'; import * as installedPackages from '../../payloads/bigip_mgmt_shared_installed_packages.json'; @@ -182,7 +183,6 @@ describe('BIG-IP Package Client', () => { .catch(err => Promise.reject(err)); }); - it('should validate isInsatlled method when package installed but not required update', () => { const mgmtClient = new ManagementClient(standardMgmtOptions); const toolChainOptions = { @@ -244,6 +244,32 @@ describe('BIG-IP Package Client', () => { .catch(err => Promise.reject(err)); }); + it('should validate install failure with downloadToFile', () => { + const mgmtClient = new ManagementClient(standardMgmtOptions); + const toolChainClient = new ToolChainClient(mgmtClient, 'as3', standardToolchainOptions); + const packageClient = toolChainClient.package; + constants.RETRY.SHORT_COUNT = 5; + nock('https://github.com') + .get('/F5Networks/f5-appsvcs-extension/releases/download/v3.17.0/f5-appsvcs-3.17.0-3.noarch.rpm') + .times(constants.RETRY.SHORT_COUNT) + .replyWithError('ECONNRESET'); + mock({ + '/var/lib/cloud/icontrollx_installs': { + 'f5-appsvcs-3.17.0-3.noarch.rpm': '1' + } + }); + return packageClient.install() + .then(() => { + assert.fail(); + nock.cleanAll(); + }) + .catch((err) => { + assert.ok(err.message.indexOf('ECONNRESET') !== -1 ); + constants.RETRY.SHORT_COUNT = 50; + nock.cleanAll(); + }); + }).timeout(300000); + it('should validate uninstall method', () => { const mgmtClient = new ManagementClient(standardMgmtOptions); const toolChainClient = new ToolChainClient(mgmtClient, 'as3', standardToolchainOptions); @@ -259,7 +285,44 @@ describe('BIG-IP Package Client', () => { }); - + it('should validate install failure with via URL', () => { + const mgmtClient = new ManagementClient(standardMgmtOptions); + const toolChainClient = new ToolChainClient(mgmtClient, 'as3', standardToolchainOptions); + const packageClient = toolChainClient.package; + nock('http://localhost:8100') + .post('/mgmt/shared/iapp/package-management-tasks') + .reply(200, { + id: "1" + }); + nock('http://localhost:8100') + .get('/mgmt/shared/iapp/package-management-tasks/1') + .reply(200, { + id: '1', + status: 'FINISHED' + }); + nock('http://localhost:8100') + .get('/mgmt/shared/file-transfer/uploads/') + .reply(200); + nock('http://localhost:8100') + .post('/mgmt/shared/file-transfer/uploads/f5-appsvcs-3.17.0-3.noarch.rpm') + .times(30) + .reply(200, { + id: '1' + }); + mock({ + '/var/lib/cloud/icontrollx_installs': { + 'f5-appsvcs-3.17.0-3.noarch.rpm': '1' + } + }); + return packageClient.install() + .then((response) => { + assert.strictEqual(response.component, 'as3'); + assert.strictEqual(response.version, '3.17.0'); + assert.ok(response.installed); + nock.cleanAll(); + }) + .catch(err => Promise.reject(err)); + }).timeout(300000); it('should validate install done via URL', () => { const mgmtClient = new ManagementClient(standardMgmtOptions); @@ -300,6 +363,7 @@ describe('BIG-IP Package Client', () => { .catch(err => Promise.reject(err)); }).timeout(300000); + it('should validate install done via FILE', () => { const mgmtClient = new ManagementClient(standardMgmtOptions); const toolChainClient = new ToolChainClient(mgmtClient, 'as3', ilxToolchainOptions); @@ -333,22 +397,27 @@ describe('BIG-IP Package Client', () => { const mgmtClient = new ManagementClient(standardMgmtOptions); const toolChainClient = new ToolChainClient(mgmtClient, 'as3', standardToolchainOptions); const packageClient = toolChainClient.package; + constants.RETRY.SHORT_COUNT = 5; nock('http://localhost:8100') .post('/mgmt/shared/iapp/package-management-tasks') + .times(constants.RETRY.SHORT_COUNT) .reply(200, { id: "1" }); nock('http://localhost:8100') .get('/mgmt/shared/iapp/package-management-tasks/1') + .times(constants.RETRY.SHORT_COUNT) .reply(200, { id: '1', status: 'FAILED' }); nock('http://localhost:8100') .get('/mgmt/shared/file-transfer/uploads/') + .times(constants.RETRY.SHORT_COUNT) .reply(200); nock('http://localhost:8100') .post('/mgmt/shared/file-transfer/uploads/f5-appsvcs-3.17.0-3.noarch.rpm') + .times(constants.RETRY.SHORT_COUNT) .times(30) .reply(200, { id: '1' @@ -361,6 +430,53 @@ describe('BIG-IP Package Client', () => { return packageClient.install() .catch((err) => { assert.ok(err.message.includes('RPM installation failed')); + constants.RETRY.SHORT_COUNT = 50; + nock.cleanAll(); + }); + }).timeout(300000); + + it('should validate install failure when package hash is not matching', () => { + const standardToolchainOptions = { + extensionVersion: '3.17.0', + extensionHash: 'wrongHashValueHere', + maxRetries: 3, + retryInterval: 2500, + verifyTls: true + }; + const mgmtClient = new ManagementClient(standardMgmtOptions); + const toolChainClient = new ToolChainClient(mgmtClient, 'as3', standardToolchainOptions); + const packageClient = toolChainClient.package; + nock('http://localhost:8100') + .post('/mgmt/shared/iapp/package-management-tasks') + .reply(200, { + id: "1" + }); + nock('http://localhost:8100') + .get('/mgmt/shared/iapp/package-management-tasks/1') + .reply(200, { + id: '1', + status: 'FINISHED' + }); + nock('http://localhost:8100') + .get('/mgmt/shared/file-transfer/uploads/') + .reply(200); + nock('http://localhost:8100') + .post('/mgmt/shared/file-transfer/uploads/f5-appsvcs-3.17.0-3.noarch.rpm') + .times(30) + .reply(200, { + id: '1' + }); + mock({ + '/var/lib/cloud/icontrollx_installs': { + 'f5-appsvcs-3.17.0-3.noarch.rpm': '1' + } + }); + return packageClient.install() + .then(() => { + assert.fail(); + }) + .catch((err) => { + assert.ok(err.message.indexOf('failed because RPM hash is not valid') !== -1); nock.cleanAll(); }); }).timeout(300000); @@ -466,6 +582,86 @@ describe('BIG-IP Service Client', () => { }) .catch(err => Promise.reject(err)); }); + + + it('should validate create method when task is unprocessable', () => { + const mgmtClient = new ManagementClient(standardMgmtOptions); + const toolChainClient = new ToolChainClient(mgmtClient, 'as3', standardToolchainOptions); + const serviceClient = toolChainClient.service; + nock('http://localhost:8100') + .post('/mgmt/shared/appsvcs/declare') + .reply(202, { + selfLink: 'https://localhost/tasks/myTask/1', + }); + nock('http://localhost:8100') + .get('/tasks/myTask/1') + .reply(422); + return serviceClient.create() + .then(() => { + assert.fail() + }) + .catch((err) => { + assert.ok(err.message.indexOf('Task with error') !== -1); + }); + }); + + it('should validate create method when internal error for task', () => { + const mgmtClient = new ManagementClient(standardMgmtOptions); + const toolChainClient = new ToolChainClient(mgmtClient, 'as3', standardToolchainOptions); + const serviceClient = toolChainClient.service; + nock('http://localhost:8100') + .post('/mgmt/shared/appsvcs/declare') + .reply(202, { + selfLink: 'https://localhost/tasks/myTask/1', + }); + nock('http://localhost:8100') + .get('/tasks/myTask/1') + .reply(503); + return serviceClient.create() + .then(() => { + assert.fail() + }) + .catch((err) => { + assert.ok(err.message.indexOf('Task with error') !== -1); + }); + }); + + it('should validate create method with retries', () => { + const mgmtClient = new ManagementClient(standardMgmtOptions); + const toolChainClient = new ToolChainClient(mgmtClient, 'as3', standardToolchainOptions); + const serviceClient = toolChainClient.service; + nock('http://localhost:8100') + .post('/mgmt/shared/appsvcs/declare') + .reply(202, { + selfLink: 'https://localhost/tasks/myTask/1', + }); + nock('http://localhost:8100') + .get('/tasks/myTask/1') + .reply(202) + .get('/tasks/myTask/1') + .reply(200); + return serviceClient.create() + .then((resp) => { + assert.strictEqual(resp.code, 202); + }) + .catch(err => Promise.reject(err)); + }); + + it('should validate create method promise rejection on 400s', () => { + const mgmtClient = new ManagementClient(standardMgmtOptions); + const toolChainClient = new ToolChainClient(mgmtClient, 'as3', standardToolchainOptions); + const serviceClient = toolChainClient.service; + nock('http://localhost:8100') + .post('/mgmt/shared/appsvcs/declare') + .reply(400, 'This is test error message'); + return serviceClient.create() + .then(() => { + assert.fail(); + }) + .catch((err) => { + assert.ok(err.message.indexOf('This is test error message') !== -1); + }); + }); }); describe('BIG-IP Toolchain Client', () => { diff --git a/tests/unit/resolver/resolverClientTests.ts b/tests/unit/resolver/resolverClientTests.ts index eff3e50e..fd7d4baa 100644 --- a/tests/unit/resolver/resolverClientTests.ts +++ b/tests/unit/resolver/resolverClientTests.ts @@ -11,9 +11,11 @@ /* eslint-disable global-require */ import assert from 'assert'; import sinon from 'sinon'; +import nock from 'nock'; import { ResolverClient } from '../../../src/lib/resolver/resolverClient'; sinon.stub(process, 'env').value({ F5_BIGIP_RUNTIME_INIT_LOG_LEVEL: 'info' }); + describe('Resolver Client', () => { let runtimeParameters; let onboardActions; @@ -25,6 +27,27 @@ describe('Resolver Client', () => { }); beforeEach(() => { + nock.cleanAll(); + nock('http://169.254.169.254') + .get('/latest/dynamic/instance-identity/document') + .reply(200, { + "accountId" : "0000000001", + "architecture" : "x86_64", + "availabilityZone" : "us-west-2a", + "billingProducts" : null, + "devpayProductCodes" : null, + "marketplaceProductCodes" : [ "asdasdasfavzxcz" ], + "imageId" : "ami-000001", + "instanceId" : "i-0a43ae03d7f8e8f42", + "instanceType" : "m5.xlarge", + "kernelId" : null, + "pendingTime" : "2020-11-19T21:20:26Z", + "privateIp" : "10.0.0.165", + "ramdiskId" : null, + "region" : "us-west-2", + "version" : "2017-09-30" + }); + onboardActions = [ { name: "test_inline_command", @@ -64,6 +87,13 @@ describe('Resolver Client', () => { secretId: 'secert-document' } }, + { + name: 'REGION', + type: 'url', + value: 'http://169.254.169.254/latest/dynamic/instance-identity/document', + query: 'region', + headers: [ { name: 'Content-Type', value: 'json'}] + }, { name: 'AZURE_PASS', type: 'secret', @@ -72,6 +102,7 @@ describe('Resolver Client', () => { environment: 'azure', version: '6e86876be4ce46a49ec578dfda897593', secretId: 'this-secret', + field: 'sensitiveFieldName', debug: true } }, @@ -123,10 +154,11 @@ describe('Resolver Client', () => { }); return resolver.resolveRuntimeParameters(runtimeParameters) .then((results) => { - assert.ok(Object.keys(results).length === 3); + assert.ok(Object.keys(results).length === 4); assert.strictEqual(results.SOME_NAME, 'SOME VALUE'); assert.strictEqual(results.AWS_PASS, 'StrongPassword2010+'); assert.strictEqual(results.AZURE_PASS, 'StrongPassword2010+'); + assert.strictEqual(results.REGION, 'us-west-2'); }); }); @@ -136,7 +168,7 @@ describe('Resolver Client', () => { resolver._resolveMetadata = sinon.stub().resolves('ru65wrde-vm0'); return resolver.resolveRuntimeParameters(runtimeParameters) .then((results) => { - assert.ok(Object.keys(results).length === 3); + assert.ok(Object.keys(results).length === 4); assert.strictEqual(results.SOME_NAME, 'SOME VALUE'); assert.strictEqual(results.AZURE_HOST_NAME, 'ru65wrde-vm0'); }); @@ -154,7 +186,7 @@ describe('Resolver Client', () => { }); return resolver.resolveRuntimeParameters(runtimeParameters) .then((results) => { - assert.ok(Object.keys(results).length === 3); + assert.ok(Object.keys(results).length === 4); assert.strictEqual(results.SOME_NAME, 'SOME VALUE'); assert.strictEqual(results.AZURE_SELF_IP, '10.0.1.4/24'); }); @@ -164,6 +196,7 @@ describe('Resolver Client', () => { const resolver = new ResolverClient(); resolver._resolveSecret = sinon.stub().resolves(''); resolver._resolveMetadata = sinon.stub().resolves(''); + resolver._resolveUrl = sinon.stub().resolves(''); return resolver.resolveRuntimeParameters(runtimeParameters) .then((results) => { assert.ok(Object.keys(results).length === 1); @@ -257,4 +290,23 @@ describe('Resolver Client', () => { assert.ok(error.message.includes('Unexpected onboard action type')) }); }); + + it('should validate _resolveUrl throw error when invalida JSON', () => { + const resolver = new ResolverClient(); + nock.cleanAll(); + runtimeParameters = [ + { + name: 'REGION', + type: 'url', + value: 'http://169.254.169.254/my-test' + } + ]; + nock('http://169.254.169.254') + .get('/my-test') + .reply(200, 'us-west'); + return resolver.resolveRuntimeParameters(runtimeParameters) + .then((results) => assert.strictEqual(results.REGION, 'us-west')) + .catch(() => assert.ok(false)) + }); + }); diff --git a/tests/unit/utilsTests.ts b/tests/unit/utilsTests.ts index 3f9d2ad0..6eed798a 100644 --- a/tests/unit/utilsTests.ts +++ b/tests/unit/utilsTests.ts @@ -241,7 +241,7 @@ describe('Util', () => { .get('/awesome_file.txt') .replyWithError('Not found'); return util.downloadToFile( 'https://fakedomain.com/awesome_file.txt', 'test_file.txt') - .catch(err => assert.ok(err.message.includes('Not found'))); + .catch((err) => assert.ok(err.message.includes('Not found'))); }); }) });