[RELEASE 0.2] Cherry pick #2382 (#2419)

From #2382

Now `cluster.sh` will use the same artifacts generated by `release.sh` to test Knative Serving. This will avoid surprises in the future due to having slightly different manifests for Serving when testing or releasing.

Also fixes #2354.

hack/README.md

@@ -5,6 +5,7 @@ This directory contains several scripts useful in the development process of Kna
 * `boilerplate/add-boilerplate.sh` Adds license boilerplate to *txt* or *go* files in a directory, recursively.
 * `deploy.sh` Deploys Knative Serving to an [environment](environments.md).
 * `diagnose-me.sh` Performs several diagnostic checks on the running Kubernetes cluster, for debugging.
+* `generate-yamls.sh` Builds all the YAMLs that Knative publishes.
 * `release.sh` Creates a new [release](release.md) of Knative Serving.
 * `update-codegen.sh` Updates auto-generated client libraries.
 * `update-deps.sh` Updates Go dependencies.

hack/generate-yamls.sh

@@ -14,97 +14,121 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# This script builds all the YAMLs that Knative publishes.  It may be varied
+# This script builds all the YAMLs that Knative publishes. It may be varied
 # between different branches, of what it does, but the following usage must
 # be observed:
 #
 # generate-yamls.sh  <repo-root-dir> <generated-yaml-list>
 #     repo-root-dir         the root directory of the repository.
-#     generated-yaml-list   an output file that contains the list of all
-#                           YAML file.  The first file listed must be our
+#     generated-yaml-list   an output file that will contain the list of all
+#                           YAML files. The first file listed must be our
 #                           manifest that contains all images to be tagged.
-#
-# Different version of release.sh should be able to call this script with
-# such assumption so that the publishing/tagging steps can evolve differently
-# than how the YAMLs are built.
-REPO_ROOT_DIR=$1
-GENERATED_YAML_LIST=$2
-
-# istio.yaml file to upload
-# We publish our own istio.yaml, so users don't need to use helm
-readonly ISTIO_CRD_YAML=./third_party/istio-1.0.2/istio-crds.yaml
-readonly ISTIO_YAML=./third_party/istio-1.0.2/istio.yaml
-readonly ISTIO_LEAN_YAML=./third_party/istio-1.0.2/istio-lean.yaml
-
-readonly BUILD_YAML=build.yaml
-readonly SERVING_YAML=serving.yaml
-readonly MONITORING_YAML=monitoring.yaml
-readonly MONITORING_METRIC_PROMETHEUS_YAML=monitoring-metrics-prometheus.yaml
-readonly MONITORING_TRACE_ZIPKIN_YAML=monitoring-tracing-zipkin.yaml
-readonly MONITORING_TRACE_ZIPKIN_IN_MEM_YAML=monitoring-tracing-zipkin-in-mem.yaml
-readonly MONITORING_LOG_ELASTICSEARCH_YAML=monitoring-logs-elasticsearch.yaml
-
-# Build the release
+
+# Different versions of our scripts should be able to call this script with
+# such assumption so that the test/publishing/tagging steps can evolve
+# differently than how the YAMLs are built.
+
+# The following environment variables affect the behavior of this script:
+# * `$KO_FLAGS` Any extra flags that will be passed to ko.
+# * `$YAML_OUTPUT_DIR` Where to put the generated YAML files, otherwise a
+#   random temporary directory will be created. **All existing YAML files in
+#   this directory will be deleted.**
+# * `$KO_DOCKER_REPO` If not set, use ko.local as the registry.
+
+set -o errexit
+set -o pipefail
+
+readonly YAML_REPO_ROOT=${1:?"First argument must be the repo root dir"}
+readonly YAML_LIST_FILE=${2:?"Second argument must be the output file"}
+
+# Location of istio YAMLs
+readonly ISTIO_CRD_YAML=${YAML_REPO_ROOT}/third_party/istio-1.0.2/istio-crds.yaml
+readonly ISTIO_YAML=${YAML_REPO_ROOT}/third_party/istio-1.0.2/istio.yaml
+readonly ISTIO_LEAN_YAML=${YAML_REPO_ROOT}/third_party/istio-1.0.2/istio-lean.yaml
+
+# Set output directory
+if [[ -z "${YAML_OUTPUT_DIR:-}" ]]; then
+  readonly YAML_OUTPUT_DIR="$(mktemp -d)"
+fi
+rm -fr ${YAML_OUTPUT_DIR}/*.yaml
+
+# Generated Knative component YAML files
+readonly BUILD_YAML=${YAML_OUTPUT_DIR}/build.yaml
+readonly SERVING_YAML=${YAML_OUTPUT_DIR}/serving.yaml
+readonly MONITORING_YAML=${YAML_OUTPUT_DIR}/monitoring.yaml
+readonly MONITORING_METRIC_PROMETHEUS_YAML=${YAML_OUTPUT_DIR}/monitoring-metrics-prometheus.yaml
+readonly MONITORING_TRACE_ZIPKIN_YAML=${YAML_OUTPUT_DIR}/monitoring-tracing-zipkin.yaml
+readonly MONITORING_TRACE_ZIPKIN_IN_MEM_YAML=${YAML_OUTPUT_DIR}/monitoring-tracing-zipkin-in-mem.yaml
+readonly MONITORING_LOG_ELASTICSEARCH_YAML=${YAML_OUTPUT_DIR}/monitoring-logs-elasticsearch.yaml
+
+# Generated Knative "bundled" YAML files
+readonly RELEASE_YAML=${YAML_OUTPUT_DIR}/release.yaml
+readonly RELEASE_LITE_YAML=${YAML_OUTPUT_DIR}/release-lite.yaml
+readonly RELEASE_NO_MON_YAML=${YAML_OUTPUT_DIR}/release-no-mon.yaml
+
+# Flags for all ko commands
+readonly KO_YAML_FLAGS="-P ${KO_FLAGS}"
+
+: ${KO_DOCKER_REPO:="ko.local"}
+export KO_DOCKER_REPO
+
+cd "${YAML_REPO_ROOT}"
 
 echo "Copying Build release"
-cp "${REPO_ROOT_DIR}/third_party/config/build/release.yaml" "${BUILD_YAML}"
+cp "third_party/config/build/release.yaml" "${BUILD_YAML}"
 
 echo "Building Knative Serving"
-ko resolve ${KO_FLAGS} -f config/ > "${SERVING_YAML}"
+ko resolve ${KO_YAML_FLAGS} -f config/ > "${SERVING_YAML}"
 
 echo "Building Monitoring & Logging"
 # Use ko to concatenate them all together.
-ko resolve ${KO_FLAGS} -R -f config/monitoring/100-namespace.yaml \
+ko resolve ${KO_YAML_FLAGS} -R -f config/monitoring/100-namespace.yaml \
     -f third_party/config/monitoring/logging/elasticsearch \
     -f config/monitoring/logging/elasticsearch \
     -f third_party/config/monitoring/metrics/prometheus \
     -f config/monitoring/metrics/prometheus \
     -f config/monitoring/tracing/zipkin > "${MONITORING_YAML}"
 
 # Metrics via Prometheus & Grafana
-ko resolve ${KO_FLAGS} -R -f config/monitoring/100-namespace.yaml \
+ko resolve ${KO_YAML_FLAGS} -R -f config/monitoring/100-namespace.yaml \
     -f third_party/config/monitoring/metrics/prometheus \
     -f config/monitoring/metrics/prometheus > "${MONITORING_METRIC_PROMETHEUS_YAML}"
 
 # Logs via ElasticSearch, Fluentd & Kibana
-ko resolve ${KO_FLAGS} -R -f config/monitoring/100-namespace.yaml \
+ko resolve ${KO_YAML_FLAGS} -R -f config/monitoring/100-namespace.yaml \
     -f third_party/config/monitoring/logging/elasticsearch \
     -f config/monitoring/logging/elasticsearch > "${MONITORING_LOG_ELASTICSEARCH_YAML}"
 
 # Traces via Zipkin when ElasticSearch is installed
-ko resolve ${KO_FLAGS} -R -f config/monitoring/tracing/zipkin > "${MONITORING_TRACE_ZIPKIN_YAML}"
+ko resolve ${KO_YAML_FLAGS} -R -f config/monitoring/tracing/zipkin > "${MONITORING_TRACE_ZIPKIN_YAML}"
 
 # Traces via Zipkin in Memory when ElasticSearch is not installed
-ko resolve ${KO_FLAGS} -R -f config/monitoring/tracing/zipkin-in-mem >> "${MONITORING_TRACE_ZIPKIN_IN_MEM_YAML}"
-
-echo "Building Release Bundles."
+ko resolve ${KO_YAML_FLAGS} -R -f config/monitoring/tracing/zipkin-in-mem >> "${MONITORING_TRACE_ZIPKIN_IN_MEM_YAML}"
 
-# These are the "bundled" yaml files that we publish.
-# Local generated yaml file.
-readonly RELEASE_YAML=release.yaml
-# Local generated lite yaml file.
-readonly LITE_YAML=release-lite.yaml
-# Local generated yaml file without the logging and monitoring components.
-readonly NO_MON_YAML=release-no-mon.yaml
+echo "Building Release bundles"
 
 # NO_MON is just build and serving
-cp "${BUILD_YAML}" "${NO_MON_YAML}"
-echo "---" >> "${NO_MON_YAML}"
-cat "${SERVING_YAML}" >> "${NO_MON_YAML}"
-echo "---" >> "${NO_MON_YAML}"
+cp "${BUILD_YAML}" "${RELEASE_NO_MON_YAML}"
+echo "---" >> "${RELEASE_NO_MON_YAML}"
+cat "${SERVING_YAML}" >> "${RELEASE_NO_MON_YAML}"
+echo "---" >> "${RELEASE_NO_MON_YAML}"
 
 # LITE is NO_MON plus "lean" monitoring
-cp "${NO_MON_YAML}" "${LITE_YAML}"
-echo "---" >> "${LITE_YAML}"
-cat "${MONITORING_METRIC_PROMETHEUS_YAML}" >> "${LITE_YAML}"
-echo "---" >> "${LITE_YAML}"
+cp "${RELEASE_NO_MON_YAML}" "${RELEASE_LITE_YAML}"
+echo "---" >> "${RELEASE_LITE_YAML}"
+cat "${MONITORING_METRIC_PROMETHEUS_YAML}" >> "${RELEASE_LITE_YAML}"
+echo "---" >> "${RELEASE_LITE_YAML}"
 
 # RELEASE is NO_MON plus full monitoring
-cp "${NO_MON_YAML}" "${RELEASE_YAML}"
+cp "${RELEASE_NO_MON_YAML}" "${RELEASE_YAML}"
 echo "---" >> "${RELEASE_YAML}"
 cat "${MONITORING_YAML}" >> "${RELEASE_YAML}"
 echo "---" >> "${RELEASE_YAML}"
 
-readonly YAMLS_TO_PUBLISH="${RELEASE_YAML} ${LITE_YAML} ${NO_MON_YAML} ${SERVING_YAML} ${BUILD_YAML} ${MONITORING_YAML} ${MONITORING_METRIC_PROMETHEUS_YAML} ${MONITORING_LOG_ELASTICSEARCH_YAML} ${MONITORING_TRACE_ZIPKIN_YAML} ${MONITORING_TRACE_ZIPKIN_IN_MEM_YAML} ${ISTIO_CRD_YAML} ${ISTIO_YAML} ${ISTIO_LEAN_YAML}"
+echo "All manifests generated"
+
+# List generated YAML files
 
-echo $YAMLS_TO_PUBLISH | sed "s/ /\n/g" > $GENERATED_YAML_LIST
+ls -1 ${RELEASE_YAML} > ${YAML_LIST_FILE}
+ls -1 ${YAML_OUTPUT_DIR}/*.yaml | grep -v ${RELEASE_YAML} >> ${YAML_LIST_FILE}
+ls -1 ${ISTIO_CRD_YAML} ${ISTIO_YAML} ${ISTIO_LEAN_YAML} >> ${YAML_LIST_FILE}

hack/release.sh

@@ -22,6 +22,13 @@ source $(dirname $0)/../vendor/github.com/knative/test-infra/scripts/release.sh
 readonly SERVING_RELEASE_GCS
 readonly SERVING_RELEASE_GCR
 
+# Set the repository
+export KO_DOCKER_REPO="${SERVING_RELEASE_GCR}"
+# Build should not try to deploy anything, use a bogus value for cluster.
+export K8S_CLUSTER_OVERRIDE=CLUSTER_NOT_SET
+export K8S_USER_OVERRIDE=USER_NOT_SET
+export DOCKER_REPO_OVERRIDE=DOCKER_NOT_SET
+
 # Script entry point
 
 initialize $@
@@ -33,44 +40,37 @@ run_validation_tests ./test/presubmit-tests.sh
 
 banner "Building the release"
 
-# Set the repository
-export KO_DOCKER_REPO="${SERVING_RELEASE_GCR}"
-# Build should not try to deploy anything, use a bogus value for cluster.
-export K8S_CLUSTER_OVERRIDE=CLUSTER_NOT_SET
-export K8S_USER_OVERRIDE=USER_NOT_SET
-export DOCKER_REPO_OVERRIDE=DOCKER_NOT_SET
-
+echo "- Destination GCR: ${KO_DOCKER_REPO}"
 if (( PUBLISH_RELEASE )); then
-  echo "- Destination GCR: ${SERVING_RELEASE_GCR}"
   echo "- Destination GCS: ${SERVING_RELEASE_GCS}"
 fi
 
 # Build the release
-#
-# Run this generate-yamls.sh script, which should be versioned with the
-# branch since the detail of building may change over time.
-YAML_LIST="generated-yamls.txt"
 
-$(dirname $0)/generate-yamls.sh "${REPO_ROOT_DIR}" "${YAML_LIST}" || abort "Cannot build the release."
-YAMLS_TO_PUBLISH=$(cat "${YAML_LIST}")
-RELEASE_YAML=$(head -n1 "${YAML_LIST}")
-
-echo "Tagging referenced images with ${TAG}."
+# Run `generate-yamls.sh`, which should be versioned with the
+# branch since the detail of building may change over time.
+readonly YAML_LIST="$(mktemp)"
+$(dirname $0)/generate-yamls.sh "${REPO_ROOT_DIR}" "${YAML_LIST}"
+readonly YAMLS_TO_PUBLISH=$(cat "${YAML_LIST}" | tr '\n' ' ')
+readonly RELEASE_YAML="$(head -n1 ${YAML_LIST})"
 
 tag_images_in_yaml "${RELEASE_YAML}" "${SERVING_RELEASE_GCR}" "${TAG}"
 
 echo "New release built successfully"
 
 if (( ! PUBLISH_RELEASE )); then
- exit 0
+  # Copy the generated YAML files to the repo root dir.
+  cp ${YAMLS_TO_PUBLISH} ${REPO_ROOT_DIR}
+  exit 0
 fi
 
 # Publish the release
+# We publish our own istio.yaml, so users don't need to use helm
 for yaml in ${YAMLS_TO_PUBLISH}; do
   echo "Publishing ${yaml}"
   publish_yaml "${yaml}" "${SERVING_RELEASE_GCS}" "${TAG}"
 done
 
-branch_release "Knative Serving" ${YAMLS_TO_PUBLISH}
+branch_release "Knative Serving" "${YAMLS_TO_PUBLISH}"
 
 echo "New release published successfully"

test/cluster.sh

@@ -18,19 +18,39 @@
 
 source $(dirname $0)/../vendor/github.com/knative/test-infra/scripts/e2e-tests.sh
 
-# Location of istio for the test cluster
-readonly ISTIO_YAML=./third_party/istio-1.0.2/istio.yaml
+CLUSTER_SH_CREATED_MANIFESTS=0
 
-function create_istio() {
-  echo ">> Bringing up Istio"
-  kubectl apply -f ${ISTIO_YAML}
+# Create all manifests required to install Knative Serving.
+function create_manifests() {
+  # Don't generate twice.
+  (( CLUSTER_SH_CREATED_MANIFESTS )) && return 0
+  local YAML_LIST="$(mktemp)"
+  # Generate manifests, capture environment variables pointing to the YAML files.
+  local FULL_OUTPUT="$( \
+      source $(dirname $0)/../hack/generate-yamls.sh ${REPO_ROOT_DIR} ${YAML_LIST} ; \
+      set | grep _YAML=/)"
+  local LOG_OUTPUT="$(echo "${FULL_OUTPUT}" | grep -v _YAML=/)"
+  local ENV_OUTPUT="$(echo "${FULL_OUTPUT}" | grep '^[_0-9A-Z]\+_YAML=/')"
+  [[ -z "${LOG_OUTPUT}" || -z "${ENV_OUTPUT}" ]] && fail_test "Error generating manifests"
+  # Only import the environment variables pointing to the YAML files.
+  echo "${LOG_OUTPUT}"
+  echo -e "Generated manifests:\n${ENV_OUTPUT}"
+  eval "${ENV_OUTPUT}"
+  CLUSTER_SH_CREATED_MANIFESTS=1
 }
 
-function create_serving() {
+# Installs Knative Serving in the current cluster, and waits for it to be ready.
+function install_knative_serving() {
+  export KO_DOCKER_REPO=${DOCKER_REPO_OVERRIDE}
+  create_manifests
+  echo ">> Bringing up Istio"
+  kubectl apply -f "${ISTIO_CRD_YAML}"
+  kubectl apply -f "${ISTIO_YAML}"
+
   echo ">> Bringing up Serving"
-  # We still need this for at least one e2e test
-  kubectl apply -f third_party/config/build/release.yaml
-  ko apply -f config/
+  # TODO(#2122): Use RELEASE_YAML once we have monitoring e2e.
+  kubectl apply -f "${RELEASE_NO_MON_YAML}"
+
   # Due to the lack of Status in Istio, we have to ignore failures in initial requests.
   #
   # However, since network configurations may reach different ingress pods at slightly
@@ -43,69 +63,26 @@ function create_serving() {
   # TODO(tcnghia): remove this when https://github.com/istio/istio/issues/882 is fixed.
   echo ">> Patching Istio"
   kubectl patch hpa -n istio-system knative-ingressgateway --patch '{"spec": {"maxReplicas": 1}}'
-}
 
-function create_test_resources() {
   echo ">> Creating test resources (test/config/)"
   ko apply -f test/config/
-}
-
-function create_monitoring() {
-  echo ">> Bringing up Monitoring"
-  kubectl apply -R -f config/monitoring/100-namespace.yaml \
-    -f third_party/config/monitoring/logging/elasticsearch \
-    -f config/monitoring/logging/elasticsearch \
-    -f third_party/config/monitoring/metrics/prometheus \
-    -f config/monitoring/metrics/prometheus \
-    -f config/monitoring/tracing/zipkin
-}
-
-function create_everything() {
-  export KO_DOCKER_REPO=${DOCKER_REPO_OVERRIDE}
-  create_istio
-  create_serving
-  create_test_resources
-  # TODO(#2122): Re-enable once we have monitoring e2e.
-  # create_monitoring
-}
-
-function delete_istio() {
-  echo ">> Bringing down Istio"
-  kubectl delete --ignore-not-found=true -f ${ISTIO_YAML}
-  kubectl delete clusterrolebinding cluster-admin-binding
-}
 
-function delete_serving() {
-  echo ">> Bringing down Serving"
-  ko delete --ignore-not-found=true -f config/
-  kubectl delete --ignore-not-found=true -f third_party/config/build/release.yaml
+  wait_until_pods_running knative-serving || fail_test "Knative Serving is not up"
+  wait_until_pods_running istio-system || fail_test "Istio system is not up"
+  wait_until_service_has_external_ip istio-system knative-ingressgateway || fail_test "Ingress has no external IP"
 }
 
-function delete_test_resources() {
+# Uninstalls Knative Serving from the current cluster.
+function uninstall_knative_serving() {
+  create_manifests
   echo ">> Removing test resources (test/config/)"
   ko delete --ignore-not-found=true -f test/config/
-}
-
-function delete_monitoring() {
-  echo ">> Bringing down Monitoring"
-  kubectl delete --ignore-not-found=true -f config/monitoring/100-namespace.yaml \
-    -f third_party/config/monitoring/logging/elasticsearch \
-    -f config/monitoring/logging/elasticsearch \
-    -f third_party/config/monitoring/metrics/prometheus \
-    -f config/monitoring/metrics/prometheus \
-    -f config/monitoring/tracing/zipkin
-}
 
-function delete_everything() {
-  # TODO(#2122): Re-enable once we have monitoring e2e.
-  # delete_monitoring
-  delete_test_resources
-  delete_serving
-  delete_istio
-}
+  echo ">> Bringing down Serving"
+  # TODO(#2122): Use RELEASE_YAML once we have monitoring e2e.
+  ko delete --ignore-not-found=true -f "${RELEASE_NO_MON_YAML}"
 
-function wait_until_cluster_up() {
-  wait_until_pods_running knative-serving || fail_test "Knative Serving is not up"
-  wait_until_pods_running istio-system || fail_test "Istio system is not up"
-  wait_until_service_has_external_ip istio-system knative-ingressgateway || fail_test "Ingress has no external IP"
+  echo ">> Bringing down Istio"
+  kubectl delete --ignore-not-found=true -f ${ISTIO_YAML}
+  kubectl delete --ignore-not-found=true clusterrolebinding cluster-admin-binding
 }