diff --git a/Gopkg.lock b/Gopkg.lock index 3dd3cb59b2c9..5001e6aa3024 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -399,11 +399,11 @@ [[projects]] branch = "master" - digest = "1:f42d23c4286880873a951329d5a6002fcb9ee9f7d6c5fd8263ea53df2391acee" + digest = "1:4d0724515d22c3964ef7bf671c989c45eefb81d71be09c1d8ab74acbdbb2ca94" name = "github.com/knative/test-infra" packages = ["."] pruneopts = "T" - revision = "e7ddaaf7b3c285a538c6dfe061d59b73ace965cf" + revision = "f710a703baf3ac7e5e9005693fb1c8d61c4eccbb" [[projects]] digest = "1:56dbf15e091bf7926cb33a57cb6bdfc658fc6d3498d2f76f10a97ce7856f1fde" diff --git a/hack/README.md b/hack/README.md index 4903430ccc33..be111c963bad 100644 --- a/hack/README.md +++ b/hack/README.md @@ -5,6 +5,7 @@ This directory contains several scripts useful in the development process of Kna * `boilerplate/add-boilerplate.sh` Adds license boilerplate to *txt* or *go* files in a directory, recursively. * `deploy.sh` Deploys Knative Serving to an [environment](environments.md). * `diagnose-me.sh` Performs several diagnostic checks on the running Kubernetes cluster, for debugging. +* `generate-yamls.sh` Builds all the YAMLs that Knative publishes. * `release.sh` Creates a new [release](release.md) of Knative Serving. * `update-codegen.sh` Updates auto-generated client libraries. * `update-deps.sh` Updates Go dependencies. diff --git a/hack/generate-yamls.sh b/hack/generate-yamls.sh index c73cba465992..9caf224a99a6 100755 --- a/hack/generate-yamls.sh +++ b/hack/generate-yamls.sh @@ -14,47 +14,75 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This script builds all the YAMLs that Knative publishes. It may be varied +# This script builds all the YAMLs that Knative publishes. It may be varied # between different branches, of what it does, but the following usage must # be observed: # # generate-yamls.sh # repo-root-dir the root directory of the repository. -# generated-yaml-list an output file that contains the list of all -# YAML file. The first file listed must be our +# generated-yaml-list an output file that will contain the list of all +# YAML files. The first file listed must be our # manifest that contains all images to be tagged. -# -# Different version of release.sh should be able to call this script with -# such assumption so that the publishing/tagging steps can evolve differently -# than how the YAMLs are built. -REPO_ROOT_DIR=$1 -GENERATED_YAML_LIST=$2 - -# istio.yaml file to upload -# We publish our own istio.yaml, so users don't need to use helm -readonly ISTIO_CRD_YAML=./third_party/istio-1.0.2/istio-crds.yaml -readonly ISTIO_YAML=./third_party/istio-1.0.2/istio.yaml -readonly ISTIO_LEAN_YAML=./third_party/istio-1.0.2/istio-lean.yaml - -readonly BUILD_YAML=build.yaml -readonly SERVING_YAML=serving.yaml -readonly MONITORING_YAML=monitoring.yaml -readonly MONITORING_METRIC_PROMETHEUS_YAML=monitoring-metrics-prometheus.yaml -readonly MONITORING_TRACE_ZIPKIN_YAML=monitoring-tracing-zipkin.yaml -readonly MONITORING_TRACE_ZIPKIN_IN_MEM_YAML=monitoring-tracing-zipkin-in-mem.yaml -readonly MONITORING_LOG_ELASTICSEARCH_YAML=monitoring-logs-elasticsearch.yaml - -# Build the release + +# Different versions of our scripts should be able to call this script with +# such assumption so that the test/publishing/tagging steps can evolve +# differently than how the YAMLs are built. + +# The following environment variables affect the behavior of this script: +# * `$KO_FLAGS` Any extra flags that will be passed to ko. +# * `$YAML_OUTPUT_DIR` Where to put the generated YAML files, otherwise a +# random temporary directory will be created. **All existing YAML files in +# this directory will be deleted.** +# * `$KO_DOCKER_REPO` If not set, use ko.local as the registry. + +set -o errexit +set -o pipefail + +readonly YAML_REPO_ROOT=${1:?"First argument must be the repo root dir"} +readonly YAML_LIST_FILE=${2:?"Second argument must be the output file"} + +# Location of istio YAMLs +readonly ISTIO_CRD_YAML=${YAML_REPO_ROOT}/third_party/istio-1.0.2/istio-crds.yaml +readonly ISTIO_YAML=${YAML_REPO_ROOT}/third_party/istio-1.0.2/istio.yaml +readonly ISTIO_LEAN_YAML=${YAML_REPO_ROOT}/third_party/istio-1.0.2/istio-lean.yaml + +# Set output directory +if [[ -z "${YAML_OUTPUT_DIR:-}" ]]; then + readonly YAML_OUTPUT_DIR="$(mktemp -d)" +fi +rm -fr ${YAML_OUTPUT_DIR}/*.yaml + +# Generated Knative component YAML files +readonly BUILD_YAML=${YAML_OUTPUT_DIR}/build.yaml +readonly SERVING_YAML=${YAML_OUTPUT_DIR}/serving.yaml +readonly MONITORING_YAML=${YAML_OUTPUT_DIR}/monitoring.yaml +readonly MONITORING_METRIC_PROMETHEUS_YAML=${YAML_OUTPUT_DIR}/monitoring-metrics-prometheus.yaml +readonly MONITORING_TRACE_ZIPKIN_YAML=${YAML_OUTPUT_DIR}/monitoring-tracing-zipkin.yaml +readonly MONITORING_TRACE_ZIPKIN_IN_MEM_YAML=${YAML_OUTPUT_DIR}/monitoring-tracing-zipkin-in-mem.yaml +readonly MONITORING_LOG_ELASTICSEARCH_YAML=${YAML_OUTPUT_DIR}/monitoring-logs-elasticsearch.yaml + +# Generated Knative "bundled" YAML files +readonly RELEASE_YAML=${YAML_OUTPUT_DIR}/release.yaml +readonly RELEASE_LITE_YAML=${YAML_OUTPUT_DIR}/release-lite.yaml +readonly RELEASE_NO_MON_YAML=${YAML_OUTPUT_DIR}/release-no-mon.yaml + +# Flags for all ko commands +readonly KO_YAML_FLAGS="-P ${KO_FLAGS}" + +: ${KO_DOCKER_REPO:="ko.local"} +export KO_DOCKER_REPO + +cd "${YAML_REPO_ROOT}" echo "Copying Build release" -cp "${REPO_ROOT_DIR}/third_party/config/build/release.yaml" "${BUILD_YAML}" +cp "third_party/config/build/release.yaml" "${BUILD_YAML}" echo "Building Knative Serving" -ko resolve ${KO_FLAGS} -f config/ > "${SERVING_YAML}" +ko resolve ${KO_YAML_FLAGS} -f config/ > "${SERVING_YAML}" echo "Building Monitoring & Logging" # Use ko to concatenate them all together. -ko resolve ${KO_FLAGS} -R -f config/monitoring/100-namespace.yaml \ +ko resolve ${KO_YAML_FLAGS} -R -f config/monitoring/100-namespace.yaml \ -f third_party/config/monitoring/logging/elasticsearch \ -f config/monitoring/logging/elasticsearch \ -f third_party/config/monitoring/metrics/prometheus \ @@ -62,49 +90,45 @@ ko resolve ${KO_FLAGS} -R -f config/monitoring/100-namespace.yaml \ -f config/monitoring/tracing/zipkin > "${MONITORING_YAML}" # Metrics via Prometheus & Grafana -ko resolve ${KO_FLAGS} -R -f config/monitoring/100-namespace.yaml \ +ko resolve ${KO_YAML_FLAGS} -R -f config/monitoring/100-namespace.yaml \ -f third_party/config/monitoring/metrics/prometheus \ -f config/monitoring/metrics/prometheus > "${MONITORING_METRIC_PROMETHEUS_YAML}" # Logs via ElasticSearch, Fluentd & Kibana -ko resolve ${KO_FLAGS} -R -f config/monitoring/100-namespace.yaml \ +ko resolve ${KO_YAML_FLAGS} -R -f config/monitoring/100-namespace.yaml \ -f third_party/config/monitoring/logging/elasticsearch \ -f config/monitoring/logging/elasticsearch > "${MONITORING_LOG_ELASTICSEARCH_YAML}" # Traces via Zipkin when ElasticSearch is installed -ko resolve ${KO_FLAGS} -R -f config/monitoring/tracing/zipkin > "${MONITORING_TRACE_ZIPKIN_YAML}" +ko resolve ${KO_YAML_FLAGS} -R -f config/monitoring/tracing/zipkin > "${MONITORING_TRACE_ZIPKIN_YAML}" # Traces via Zipkin in Memory when ElasticSearch is not installed -ko resolve ${KO_FLAGS} -R -f config/monitoring/tracing/zipkin-in-mem >> "${MONITORING_TRACE_ZIPKIN_IN_MEM_YAML}" - -echo "Building Release Bundles." +ko resolve ${KO_YAML_FLAGS} -R -f config/monitoring/tracing/zipkin-in-mem >> "${MONITORING_TRACE_ZIPKIN_IN_MEM_YAML}" -# These are the "bundled" yaml files that we publish. -# Local generated yaml file. -readonly RELEASE_YAML=release.yaml -# Local generated lite yaml file. -readonly LITE_YAML=release-lite.yaml -# Local generated yaml file without the logging and monitoring components. -readonly NO_MON_YAML=release-no-mon.yaml +echo "Building Release bundles" # NO_MON is just build and serving -cp "${BUILD_YAML}" "${NO_MON_YAML}" -echo "---" >> "${NO_MON_YAML}" -cat "${SERVING_YAML}" >> "${NO_MON_YAML}" -echo "---" >> "${NO_MON_YAML}" +cp "${BUILD_YAML}" "${RELEASE_NO_MON_YAML}" +echo "---" >> "${RELEASE_NO_MON_YAML}" +cat "${SERVING_YAML}" >> "${RELEASE_NO_MON_YAML}" +echo "---" >> "${RELEASE_NO_MON_YAML}" # LITE is NO_MON plus "lean" monitoring -cp "${NO_MON_YAML}" "${LITE_YAML}" -echo "---" >> "${LITE_YAML}" -cat "${MONITORING_METRIC_PROMETHEUS_YAML}" >> "${LITE_YAML}" -echo "---" >> "${LITE_YAML}" +cp "${RELEASE_NO_MON_YAML}" "${RELEASE_LITE_YAML}" +echo "---" >> "${RELEASE_LITE_YAML}" +cat "${MONITORING_METRIC_PROMETHEUS_YAML}" >> "${RELEASE_LITE_YAML}" +echo "---" >> "${RELEASE_LITE_YAML}" # RELEASE is NO_MON plus full monitoring -cp "${NO_MON_YAML}" "${RELEASE_YAML}" +cp "${RELEASE_NO_MON_YAML}" "${RELEASE_YAML}" echo "---" >> "${RELEASE_YAML}" cat "${MONITORING_YAML}" >> "${RELEASE_YAML}" echo "---" >> "${RELEASE_YAML}" -readonly YAMLS_TO_PUBLISH="${RELEASE_YAML} ${LITE_YAML} ${NO_MON_YAML} ${SERVING_YAML} ${BUILD_YAML} ${MONITORING_YAML} ${MONITORING_METRIC_PROMETHEUS_YAML} ${MONITORING_LOG_ELASTICSEARCH_YAML} ${MONITORING_TRACE_ZIPKIN_YAML} ${MONITORING_TRACE_ZIPKIN_IN_MEM_YAML} ${ISTIO_CRD_YAML} ${ISTIO_YAML} ${ISTIO_LEAN_YAML}" +echo "All manifests generated" + +# List generated YAML files -echo $YAMLS_TO_PUBLISH | sed "s/ /\n/g" > $GENERATED_YAML_LIST +ls -1 ${RELEASE_YAML} > ${YAML_LIST_FILE} +ls -1 ${YAML_OUTPUT_DIR}/*.yaml | grep -v ${RELEASE_YAML} >> ${YAML_LIST_FILE} +ls -1 ${ISTIO_CRD_YAML} ${ISTIO_YAML} ${ISTIO_LEAN_YAML} >> ${YAML_LIST_FILE} diff --git a/hack/release.sh b/hack/release.sh index bcf3ad012711..60b69a419722 100755 --- a/hack/release.sh +++ b/hack/release.sh @@ -22,6 +22,13 @@ source $(dirname $0)/../vendor/github.com/knative/test-infra/scripts/release.sh readonly SERVING_RELEASE_GCS readonly SERVING_RELEASE_GCR +# Set the repository +export KO_DOCKER_REPO="${SERVING_RELEASE_GCR}" +# Build should not try to deploy anything, use a bogus value for cluster. +export K8S_CLUSTER_OVERRIDE=CLUSTER_NOT_SET +export K8S_USER_OVERRIDE=USER_NOT_SET +export DOCKER_REPO_OVERRIDE=DOCKER_NOT_SET + # Script entry point initialize $@ @@ -33,44 +40,37 @@ run_validation_tests ./test/presubmit-tests.sh banner "Building the release" -# Set the repository -export KO_DOCKER_REPO="${SERVING_RELEASE_GCR}" -# Build should not try to deploy anything, use a bogus value for cluster. -export K8S_CLUSTER_OVERRIDE=CLUSTER_NOT_SET -export K8S_USER_OVERRIDE=USER_NOT_SET -export DOCKER_REPO_OVERRIDE=DOCKER_NOT_SET - +echo "- Destination GCR: ${KO_DOCKER_REPO}" if (( PUBLISH_RELEASE )); then - echo "- Destination GCR: ${SERVING_RELEASE_GCR}" echo "- Destination GCS: ${SERVING_RELEASE_GCS}" fi # Build the release -# -# Run this generate-yamls.sh script, which should be versioned with the -# branch since the detail of building may change over time. -YAML_LIST="generated-yamls.txt" -$(dirname $0)/generate-yamls.sh "${REPO_ROOT_DIR}" "${YAML_LIST}" || abort "Cannot build the release." -YAMLS_TO_PUBLISH=$(cat "${YAML_LIST}") -RELEASE_YAML=$(head -n1 "${YAML_LIST}") - -echo "Tagging referenced images with ${TAG}." +# Run `generate-yamls.sh`, which should be versioned with the +# branch since the detail of building may change over time. +readonly YAML_LIST="$(mktemp)" +$(dirname $0)/generate-yamls.sh "${REPO_ROOT_DIR}" "${YAML_LIST}" +readonly YAMLS_TO_PUBLISH=$(cat "${YAML_LIST}" | tr '\n' ' ') +readonly RELEASE_YAML="$(head -n1 ${YAML_LIST})" tag_images_in_yaml "${RELEASE_YAML}" "${SERVING_RELEASE_GCR}" "${TAG}" echo "New release built successfully" if (( ! PUBLISH_RELEASE )); then - exit 0 + # Copy the generated YAML files to the repo root dir. + cp ${YAMLS_TO_PUBLISH} ${REPO_ROOT_DIR} + exit 0 fi # Publish the release +# We publish our own istio.yaml, so users don't need to use helm for yaml in ${YAMLS_TO_PUBLISH}; do echo "Publishing ${yaml}" publish_yaml "${yaml}" "${SERVING_RELEASE_GCS}" "${TAG}" done -branch_release "Knative Serving" ${YAMLS_TO_PUBLISH} +branch_release "Knative Serving" "${YAMLS_TO_PUBLISH}" echo "New release published successfully" diff --git a/test/cluster.sh b/test/cluster.sh index 210bf74b47db..53d22013c92e 100644 --- a/test/cluster.sh +++ b/test/cluster.sh @@ -18,19 +18,39 @@ source $(dirname $0)/../vendor/github.com/knative/test-infra/scripts/e2e-tests.sh -# Location of istio for the test cluster -readonly ISTIO_YAML=./third_party/istio-1.0.2/istio.yaml +CLUSTER_SH_CREATED_MANIFESTS=0 -function create_istio() { - echo ">> Bringing up Istio" - kubectl apply -f ${ISTIO_YAML} +# Create all manifests required to install Knative Serving. +function create_manifests() { + # Don't generate twice. + (( CLUSTER_SH_CREATED_MANIFESTS )) && return 0 + local YAML_LIST="$(mktemp)" + # Generate manifests, capture environment variables pointing to the YAML files. + local FULL_OUTPUT="$( \ + source $(dirname $0)/../hack/generate-yamls.sh ${REPO_ROOT_DIR} ${YAML_LIST} ; \ + set | grep _YAML=/)" + local LOG_OUTPUT="$(echo "${FULL_OUTPUT}" | grep -v _YAML=/)" + local ENV_OUTPUT="$(echo "${FULL_OUTPUT}" | grep '^[_0-9A-Z]\+_YAML=/')" + [[ -z "${LOG_OUTPUT}" || -z "${ENV_OUTPUT}" ]] && fail_test "Error generating manifests" + # Only import the environment variables pointing to the YAML files. + echo "${LOG_OUTPUT}" + echo -e "Generated manifests:\n${ENV_OUTPUT}" + eval "${ENV_OUTPUT}" + CLUSTER_SH_CREATED_MANIFESTS=1 } -function create_serving() { +# Installs Knative Serving in the current cluster, and waits for it to be ready. +function install_knative_serving() { + export KO_DOCKER_REPO=${DOCKER_REPO_OVERRIDE} + create_manifests + echo ">> Bringing up Istio" + kubectl apply -f "${ISTIO_CRD_YAML}" + kubectl apply -f "${ISTIO_YAML}" + echo ">> Bringing up Serving" - # We still need this for at least one e2e test - kubectl apply -f third_party/config/build/release.yaml - ko apply -f config/ + # TODO(#2122): Use RELEASE_YAML once we have monitoring e2e. + kubectl apply -f "${RELEASE_NO_MON_YAML}" + # Due to the lack of Status in Istio, we have to ignore failures in initial requests. # # However, since network configurations may reach different ingress pods at slightly @@ -43,69 +63,26 @@ function create_serving() { # TODO(tcnghia): remove this when https://github.com/istio/istio/issues/882 is fixed. echo ">> Patching Istio" kubectl patch hpa -n istio-system knative-ingressgateway --patch '{"spec": {"maxReplicas": 1}}' -} -function create_test_resources() { echo ">> Creating test resources (test/config/)" ko apply -f test/config/ -} - -function create_monitoring() { - echo ">> Bringing up Monitoring" - kubectl apply -R -f config/monitoring/100-namespace.yaml \ - -f third_party/config/monitoring/logging/elasticsearch \ - -f config/monitoring/logging/elasticsearch \ - -f third_party/config/monitoring/metrics/prometheus \ - -f config/monitoring/metrics/prometheus \ - -f config/monitoring/tracing/zipkin -} - -function create_everything() { - export KO_DOCKER_REPO=${DOCKER_REPO_OVERRIDE} - create_istio - create_serving - create_test_resources - # TODO(#2122): Re-enable once we have monitoring e2e. - # create_monitoring -} - -function delete_istio() { - echo ">> Bringing down Istio" - kubectl delete --ignore-not-found=true -f ${ISTIO_YAML} - kubectl delete clusterrolebinding cluster-admin-binding -} -function delete_serving() { - echo ">> Bringing down Serving" - ko delete --ignore-not-found=true -f config/ - kubectl delete --ignore-not-found=true -f third_party/config/build/release.yaml + wait_until_pods_running knative-serving || fail_test "Knative Serving is not up" + wait_until_pods_running istio-system || fail_test "Istio system is not up" + wait_until_service_has_external_ip istio-system knative-ingressgateway || fail_test "Ingress has no external IP" } -function delete_test_resources() { +# Uninstalls Knative Serving from the current cluster. +function uninstall_knative_serving() { + create_manifests echo ">> Removing test resources (test/config/)" ko delete --ignore-not-found=true -f test/config/ -} - -function delete_monitoring() { - echo ">> Bringing down Monitoring" - kubectl delete --ignore-not-found=true -f config/monitoring/100-namespace.yaml \ - -f third_party/config/monitoring/logging/elasticsearch \ - -f config/monitoring/logging/elasticsearch \ - -f third_party/config/monitoring/metrics/prometheus \ - -f config/monitoring/metrics/prometheus \ - -f config/monitoring/tracing/zipkin -} -function delete_everything() { - # TODO(#2122): Re-enable once we have monitoring e2e. - # delete_monitoring - delete_test_resources - delete_serving - delete_istio -} + echo ">> Bringing down Serving" + # TODO(#2122): Use RELEASE_YAML once we have monitoring e2e. + ko delete --ignore-not-found=true -f "${RELEASE_NO_MON_YAML}" -function wait_until_cluster_up() { - wait_until_pods_running knative-serving || fail_test "Knative Serving is not up" - wait_until_pods_running istio-system || fail_test "Istio system is not up" - wait_until_service_has_external_ip istio-system knative-ingressgateway || fail_test "Ingress has no external IP" + echo ">> Bringing down Istio" + kubectl delete --ignore-not-found=true -f ${ISTIO_YAML} + kubectl delete --ignore-not-found=true clusterrolebinding cluster-admin-binding } diff --git a/test/e2e-tests.sh b/test/e2e-tests.sh index 3fd08c599c91..960afbf0748e 100755 --- a/test/e2e-tests.sh +++ b/test/e2e-tests.sh @@ -53,27 +53,26 @@ function publish_test_images() { # Deletes everything created on the cluster including all knative and istio components. function teardown() { - delete_everything + uninstall_knative_serving } # Script entry point. initialize $@ +header "Setting up environment" + # Fail fast during setup. set -o errexit set -o pipefail -header "Setting up environment" -create_everything +install_knative_serving publish_test_images # Handle test failures ourselves, so we can dump useful info. set +o errexit set +o pipefail -wait_until_cluster_up - # Run the tests header "Running tests" diff --git a/test/performance-tests.sh b/test/performance-tests.sh index 2e55c0287cc2..13165d86ed32 100755 --- a/test/performance-tests.sh +++ b/test/performance-tests.sh @@ -37,7 +37,7 @@ function perf_tests() { sleep 1m local ip=$(kubectl get svc knative-ingressgateway -n istio-system -o jsonpath="{.status.loadBalancer.ingress[*].ip}") local host=$(kubectl get route observed-concurrency -o jsonpath="{.status.domain}") - + wait_until_routable "$ip" "$host" || return 1 wrk -t 1 -c "$1" -d "$2" -s "${PERF_DIR}/reporter.lua" --latency -H "Host: ${host}" "http://${ip}/?timeout=1000" } @@ -46,7 +46,7 @@ function perf_tests() { function teardown() { # Delete the service now that the test is done kubectl delete --ignore-not-found=true -f ${TEST_APP_YAML} - delete_everything + uninstall_knative_serving } # Fail fast during setup. @@ -56,16 +56,15 @@ set -o pipefail header "Setting up environment" initialize $@ -create_everything +install_knative_serving -wait_until_cluster_up ko apply -f ${TEST_APP_YAML} # Handle test failures ourselves, so we can dump useful info. set +o errexit set +o pipefail -# Run the test with concurrency=5 and for 60s duration. +# Run the test with concurrency=5 and for 60s duration. # Need to export concurrency var as it is required by the parser. export concurrency=5 perf_tests "${concurrency}" 60s diff --git a/vendor/github.com/knative/test-infra/scripts/README.md b/vendor/github.com/knative/test-infra/scripts/README.md index cf175d663a54..083f2353293a 100644 --- a/vendor/github.com/knative/test-infra/scripts/README.md +++ b/vendor/github.com/knative/test-infra/scripts/README.md @@ -3,6 +3,62 @@ This directory contains helper scripts used by Prow test jobs, as well and local development scripts. +## Using the `presubmit-tests.sh` helper script + +This is a helper script to run the presubmit tests. To use it: + +1. Source this script. + +1. Define the functions `build_tests()` and `unit_tests()`. They should run all +tests (i.e., not fail fast), and return 0 if all passed, 1 if a failure +occurred. The environment variables `RUN_BUILD_TESTS`, `RUN_UNIT_TESTS` and +`RUN_INTEGRATION_TESTS` are set to 0 (false) or 1 (true) accordingly. If +`--emit-metrics` is passed, `EMIT_METRICS` will be set to 1. + +1. [optional] Define the function `integration_tests()`, just like the previous +ones. If you don't define this function, the default action for running the +integration tests is to call the `./test/e2e-tests.sh` script (passing the +`--emit-metrics` flag if necessary). + +1. [optional] Define the functions `pre_integration_tests()` or +`post_integration_tests()`. These functions will be called before or after the +integration tests (either your custom one or the default action) and will cause +the test to fail if they don't return success. + +1. Call the `main()` function passing `$@` (without quotes). + +Running the script without parameters, or with the `--all-tests` flag causes +all tests to be executed, in the right order (i.e., build, then unit, then +integration tests). + +Use the flags `--build-tests`, `--unit-tests` and `--integration-tests` to run +a specific set of tests. The flag `--emit-metrics` is used to emit metrics when +running the tests, and is automatically handled by the default action (see +above). + +### Sample presubmit test script + +```bash +source vendor/github.com/knative/test-infra/scripts/presubmit-tests.sh + +function build_tests() { + go build . +} + +function unit_tests() { + report_go_test . +} + +function pre_integration_tests() { + echo "Cleaning up before integration tests" + rm -fr ./staging-area +} + +# We use the default integration test runner. + +main $@ +``` + ## Using the `e2e-tests.sh` helper script This is a helper script for Knative E2E test scripts. To use it: @@ -16,6 +72,12 @@ resources. called when a test fails, and can dump extra information about the current state of the cluster (tipically using `kubectl`). +1. [optional] Write the `parse_flags()` function. It will be called whenever an +unrecognized flag is passed to the script, allowing you to define your own flags. +The function must return 0 if the flag is unrecognized, or the number of items +to skip in the command line if the flag was parsed successfully. For example, +return 1 for a simple flag, and 2 for a flag with a parameter. + 1. Call the `initialize()` function passing `$@` (without quotes). 1. Write logic for the end-to-end tests. Run all go tests using `go_test_e2e()` @@ -41,7 +103,9 @@ tests against the cluster. ### Sample end-to-end test script -This script will test that the latest Knative Serving nightly release works. +This script will test that the latest Knative Serving nightly release works. It +defines a special flag (`--no-knative-wait`) that causes the script not to +wait for Knative Serving to be up before running the tests. ```bash source vendor/github.com/knative/test-infra/scripts/e2e-tests.sh @@ -50,11 +114,23 @@ function teardown() { echo "TODO: tear down test resources" } +function parse_flags() { + if [[ "$1" == "--no-knative-wait" ]]; then + WAIT_FOR_KNATIVE=0 + return 1 + fi + return 0 +} + +WAIT_FOR_KNATIVE=1 + initialize $@ start_latest_knative_serving -wait_until_pods_running knative-serving || fail_test "Knative Serving is not up" +if (( WAIT_FOR_KNATIVE )); then + wait_until_pods_running knative-serving || fail_test "Knative Serving is not up" +fi # TODO: use go_test_e2e to run the tests. kubectl get pods || fail_test diff --git a/vendor/github.com/knative/test-infra/scripts/e2e-tests.sh b/vendor/github.com/knative/test-infra/scripts/e2e-tests.sh index 75541e23e223..e276b7ba1173 100755 --- a/vendor/github.com/knative/test-infra/scripts/e2e-tests.sh +++ b/vendor/github.com/knative/test-infra/scripts/e2e-tests.sh @@ -49,7 +49,7 @@ readonly TEST_RESULT_FILE=/tmp/${E2E_BASE_NAME}-e2e-result function teardown_test_resources() { header "Tearing down test environment" # Free resources in GCP project. - if (( ! USING_EXISTING_CLUSTER )) && [[ "$(type -t teardown)" == "function" ]]; then + if (( ! USING_EXISTING_CLUSTER )) && function_exists teardown; then teardown fi @@ -72,12 +72,14 @@ function fail_test() { exit 1 } -# Run the given E2E tests (must be tagged as such). +# Run the given E2E tests. Assume tests are tagged e2e, unless `-tags=XXX` is passed. # Parameters: $1..$n - any go test flags, then directories containing the tests to run. function go_test_e2e() { - local options="" - (( EMIT_METRICS )) && options="-emitmetrics" - report_go_test -v -tags=e2e -count=1 $@ ${options} + local test_options="" + local go_options="" + (( EMIT_METRICS )) && test_options="-emitmetrics" + [[ ! " $@" == *" -tags="* ]] && go_options="-tags=e2e" + report_go_test -v -count=1 ${go_options} $@ ${test_options} } # Download the k8s binaries required by kubetest. @@ -132,7 +134,7 @@ function dump_cluster_state() { kubectl get services --all-namespaces echo ">>> Events:" kubectl get events --all-namespaces - [[ "$(type -t dump_extra_cluster_state)" == "function" ]] && dump_extra_cluster_state + function_exists dump_extra_cluster_state && dump_extra_cluster_state echo "***************************************" echo "*** TEST FAILED ***" echo "*** End of information dump ***" @@ -184,7 +186,8 @@ function create_test_cluster() { # Don't fail test for kubetest, as it might incorrectly report test failure # if teardown fails (for details, see success() below) set +o errexit - kubetest "${CLUSTER_CREATION_ARGS[@]}" \ + run_go_tool k8s.io/test-infra/kubetest \ + kubetest "${CLUSTER_CREATION_ARGS[@]}" \ --up \ --down \ --extract local \ @@ -248,7 +251,7 @@ function setup_test_cluster() { trap teardown_test_resources EXIT - if (( USING_EXISTING_CLUSTER )) && [[ "$(type -t teardown)" == "function" ]]; then + if (( USING_EXISTING_CLUSTER )) && function_exists teardown; then echo "Deleting any previous SUT instance" teardown fi @@ -289,7 +292,19 @@ function initialize() { readonly E2E_SCRIPT cd ${REPO_ROOT_DIR} - for parameter in $@; do + while [[ $# -ne 0 ]]; do + local parameter=$1 + # Try parsing flag as a custom one. + if function_exists parse_flags; then + parse_flags $@ + local skip=$? + if [[ ${skip} -ne 0 ]]; then + # Skip parsed flag (and possibly argument) and continue + shift ${skip} + continue + fi + fi + # Try parsing flag as a standard one. case $parameter in --run-tests) RUN_TESTS=1 ;; --emit-metrics) EMIT_METRICS=1 ;; diff --git a/vendor/github.com/knative/test-infra/scripts/library.sh b/vendor/github.com/knative/test-infra/scripts/library.sh index 38d464d6b134..95acac29a585 100755 --- a/vendor/github.com/knative/test-infra/scripts/library.sh +++ b/vendor/github.com/knative/test-infra/scripts/library.sh @@ -23,6 +23,7 @@ readonly SERVING_GKE_VERSION=latest readonly SERVING_GKE_IMAGE=cos # Public images and yaml files. +readonly KNATIVE_ISTIO_CRD_YAML=https://storage.googleapis.com/knative-releases/serving/latest/istio-crds.yaml readonly KNATIVE_ISTIO_YAML=https://storage.googleapis.com/knative-releases/serving/latest/istio.yaml readonly KNATIVE_SERVING_RELEASE=https://storage.googleapis.com/knative-releases/serving/latest/release.yaml readonly KNATIVE_BUILD_RELEASE=https://storage.googleapis.com/knative-releases/build/latest/release.yaml @@ -66,6 +67,11 @@ function warning() { make_banner "!" "$1" } +# Checks whether the given function exists. +function function_exists() { + [[ "$(type -t $1)" == "function" ]] +} + # Remove ALL images in the given GCR repository. # Parameters: $1 - GCR repository. function delete_gcr_images() { @@ -172,7 +178,7 @@ function wait_until_routable() { return 1 } -# Returns the name of the pod of the given app. +# Returns the name of the first pod of the given app. # Parameters: $1 - app name. # $2 - namespace (optional). function get_app_pod() { @@ -181,6 +187,15 @@ function get_app_pod() { kubectl get pods ${namespace} --selector=app=$1 --output=jsonpath="{.items[0].metadata.name}" } +# Returns the name of all pods of the given app. +# Parameters: $1 - app name. +# $2 - namespace (optional). +function get_app_pods() { + local namespace="" + [[ -n $2 ]] && namespace="-n $2" + kubectl get pods ${namespace} --selector=app=$1 --output=jsonpath="{.items[*].metadata.name}" +} + # Sets the given user as cluster admin. # Parameters: $1 - user # $2 - cluster name @@ -330,6 +345,7 @@ function report_go_test() { function start_latest_knative_serving() { header "Starting Knative Serving" subheader "Installing Istio" + kubectl apply -f ${KNATIVE_ISTIO_CRD_YAML} || return 1 kubectl apply -f ${KNATIVE_ISTIO_YAML} || return 1 wait_until_pods_running istio-system || return 1 kubectl label namespace default istio-injection=enabled || return 1 @@ -351,16 +367,18 @@ function start_latest_knative_build() { } # Run a go tool, installing it first if necessary. -# Parameters: $1 - tool to run. -# $2 - tool package for go get. +# Parameters: $1 - tool package/dir for go get/install. +# $2 - tool to run. # $3..$n - parameters passed to the tool. function run_go_tool() { - local tool=$1 + local tool=$2 if [[ -z "$(which ${tool})" ]]; then - go get -u $2 + local action=get + [[ $1 =~ ^[\./].* ]] && action=install + go ${action} $1 fi shift 2 - ${tool} $@ + ${tool} "$@" } # Run dep-collector to update licenses. @@ -370,7 +388,7 @@ function update_licenses() { cd ${REPO_ROOT_DIR} || return 1 local dst=$1 shift - run_go_tool dep-collector github.com/mattmoor/dep-collector $@ > ./${dst} + run_go_tool ./vendor/github.com/knative/test-infra/tools/dep-collector dep-collector $@ > ./${dst} } # Run dep-collector to check for forbidden liceses. @@ -379,7 +397,7 @@ function check_licenses() { # Fetch the google/licenseclassifier for its license db go get -u github.com/google/licenseclassifier # Check that we don't have any forbidden licenses in our images. - run_go_tool dep-collector github.com/mattmoor/dep-collector -check $@ + run_go_tool ./vendor/github.com/knative/test-infra/tools/dep-collector dep-collector -check $@ } # Run the given linter on the given files, checking it exists first. diff --git a/vendor/github.com/knative/test-infra/scripts/presubmit-tests.sh b/vendor/github.com/knative/test-infra/scripts/presubmit-tests.sh index 74d3f5585e7f..db78f3d0ca12 100755 --- a/vendor/github.com/knative/test-infra/scripts/presubmit-tests.sh +++ b/vendor/github.com/knative/test-infra/scripts/presubmit-tests.sh @@ -14,21 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This is a helper script to run the presubmit tests. To use it: -# 1. Source this script. -# 2. Define the functions build_tests(), unit_tests() and -# integration_tests(). They should run all tests (i.e., not fail -# fast), and return 0 if all passed, 1 if a failure occurred. -# The environment variables RUN_BUILD_TESTS, RUN_UNIT_TESTS and -# RUN_INTEGRATION_TESTS are set to 0 (false) or 1 (true) accordingly. -# If --emit-metrics is passed, EMIT_METRICS will be set to 1. -# 3. Call the main() function passing $@ (without quotes). -# -# Running the script without parameters, or with the --all-tests -# flag, causes all tests to be executed, in the right order. -# Use the flags --build-tests, --unit-tests and --integration-tests -# to run a specific set of tests. The flag --emit-metrics is used -# to emit metrics when running the tests. +# This is a helper script for Knative presubmit test scripts. +# See README.md for instructions on how to use it. source $(dirname ${BASH_SOURCE})/library.sh @@ -121,15 +108,29 @@ function main() { # Tests to be performed, in the right order if --all-tests is passed. - local result=0 + local failed=0 if (( RUN_BUILD_TESTS )); then - build_tests || result=1 + build_tests || failed=1 fi if (( RUN_UNIT_TESTS )); then - unit_tests || result=1 + unit_tests || failed=1 fi if (( RUN_INTEGRATION_TESTS )); then - integration_tests || result=1 + if function_exists pre_integration_tests; then + pre_integration_tests || failed=1 + fi + if (( ! failed )); then + if function_exists integration_tests; then + integration_tests || failed=1 + else + local options="" + (( EMIT_METRICS )) && options="--emit-metrics" + ./test/e2e-tests.sh ${options} || failed=1 + fi + fi + if (( ! failed )) && function_exists post_integration_tests; then + post_integration_tests || failed=1 + fi fi - exit ${result} + exit ${failed} } diff --git a/vendor/github.com/knative/test-infra/scripts/release.sh b/vendor/github.com/knative/test-infra/scripts/release.sh index 3ddf02922bed..3e41bdda9d10 100755 --- a/vendor/github.com/knative/test-infra/scripts/release.sh +++ b/vendor/github.com/knative/test-infra/scripts/release.sh @@ -68,7 +68,7 @@ function parse_flags() { RELEASE_VERSION="" RELEASE_NOTES="" RELEASE_BRANCH="" - KO_FLAGS="-P -L" + KO_FLAGS="-P" cd ${REPO_ROOT_DIR} while [[ $# -ne 0 ]]; do local parameter=$1 @@ -76,16 +76,8 @@ function parse_flags() { --skip-tests) SKIP_TESTS=1 ;; --tag-release) TAG_RELEASE=1 ;; --notag-release) TAG_RELEASE=0 ;; - --publish) - PUBLISH_RELEASE=1 - # Remove -L from ko flags - KO_FLAGS="${KO_FLAGS/-L}" - ;; - --nopublish) - PUBLISH_RELEASE=0 - # Add -L to ko flags - KO_FLAGS="-L ${KO_FLAGS}" - ;; + --publish) PUBLISH_RELEASE=1 ;; + --nopublish) PUBLISH_RELEASE=0 ;; --version) shift [[ $# -ge 1 ]] || abort "missing version after --version" @@ -109,6 +101,12 @@ function parse_flags() { shift done + # Update KO_DOCKER_REPO and KO_FLAGS if we're not publishing. + if (( ! PUBLISH_RELEASE )); then + KO_DOCKER_REPO="ko.local" + KO_FLAGS="-L ${KO_FLAGS}" + fi + if (( TAG_RELEASE )); then # Currently we're not considering the tags in refs/tags namespace. commit=$(git describe --always --dirty) @@ -138,7 +136,10 @@ function run_validation_tests() { if (( ! SKIP_TESTS )); then banner "Running release validation tests" # Run tests. - $1 + if ! $1; then + banner "Release validation tests failed, aborting" + exit 1 + fi fi } @@ -147,7 +148,7 @@ function initialize() { parse_flags $@ # Checkout specific branch, if necessary if (( BRANCH_RELEASE )); then - git checkout --track upstream/${RELEASE_BRANCH} || abort "cannot checkout branch ${RELEASE_BRANCH}" + git checkout upstream/${RELEASE_BRANCH} || abort "cannot checkout branch ${RELEASE_BRANCH}" fi } @@ -157,18 +158,23 @@ function initialize() { function branch_release() { (( BRANCH_RELEASE )) || return 0 local title="$1 release ${TAG}" - local yamls="$2" - local attach="--attach=${yamls// / --attach=}" + local attachments=() local description="$(mktemp)" + local attachments_dir="$(mktemp -d)" + # Copy each YAML to a separate dir + for yaml in $2; do + cp ${yaml} ${attachments_dir}/ + attachments+=("--attach=${yaml}#$(basename ${yaml})") + done echo -e "${title}\n" > ${description} if [[ -n "${RELEASE_NOTES}" ]]; then cat ${RELEASE_NOTES} >> ${description} fi git tag -a ${TAG} -m "${title}" git push $(git remote get-url upstream) tag ${TAG} - run_go_tool hub github.com/github/hub release create \ + run_go_tool github.com/github/hub hub release create \ --prerelease \ - ${attach} \ + ${attachments[@]} \ --file=${description} \ --commitish=${RELEASE_BRANCH} \ ${TAG}