Generate log files for raw toolchain builds (#10662)

toolkit/scripts/toolchain.mk

@@ -17,6 +17,8 @@ toolchain_from_repos = $(toolchain_build_dir)/repo_rpms
 toolchain_logs_dir = $(LOGS_DIR)/toolchain
 toolchain_downloads_logs_dir = $(toolchain_logs_dir)/downloads
 toolchain_rehydrate_logs_dir = $(toolchain_logs_dir)/rehydrate
+toolchain_raw_logs_dir = $(toolchain_logs_dir)/raw
+toolchain_official_logs_dir = $(toolchain_logs_dir)/official
 toolchain_downloads_manifest = $(toolchain_downloads_logs_dir)/download_manifest.txt
 toolchain_log_tail_length = 20
 populated_toolchain_chroot = $(toolchain_build_dir)/populated_toolchain
@@ -155,13 +157,18 @@ hydrate-toolchain:
 # out/toolchain/toolchain_from_container.tar.gz
 $(raw_toolchain): $(toolchain_files)
 	@echo "Building raw toolchain"
+	rm -rf $(toolchain_raw_logs_dir) && mkdir -p $(toolchain_raw_logs_dir)
 	cd $(SCRIPTS_DIR)/toolchain && \
 		./create_toolchain_in_container.sh \
 			$(BUILD_DIR) \
 			$(SPECS_DIR) \
 			$(SOURCE_URL) \
 			$(INCREMENTAL_TOOLCHAIN) \
-			$(ARCHIVE_TOOL)
+			$(ARCHIVE_TOOL) \
+			$(toolchain_raw_logs_dir) 2>&1 | tee $(toolchain_raw_logs_dir)/create_toolchain_in_container_full.log; \
+		if [ $${PIPESTATUS[0]} -ne 0 ]; then \
+			$(call print_error, create_toolchain_in_container.sh failed); \
+		fi
 
 # This target establishes a cache of toolchain RPMs for partially rehydrating the toolchain from package repos.
 # $(toolchain_from_repos) is a staging folder for these RPMs. We use the toolchain manifest to get a list of
@@ -207,6 +214,7 @@ $(final_toolchain): $(no_repo_acl) $(raw_toolchain) $(toolchain_rpms_rehydrated)
 	# Clean the existing chroot if not doing an incremental build
 	$(if $(filter y,$(INCREMENTAL_TOOLCHAIN)),,$(SCRIPTS_DIR)/safeunmount.sh "$(populated_toolchain_chroot)" || $(call print_error,failed to clean mounts for toolchain build))
 	$(if $(filter y,$(INCREMENTAL_TOOLCHAIN)),,rm -rf $(populated_toolchain_chroot))
+	rm -rf $(toolchain_official_logs_dir) && mkdir -p $(toolchain_official_logs_dir)
 	cd $(SCRIPTS_DIR)/toolchain && \
 		./build_mariner_toolchain.sh \
 			"$(DIST_TAG)" \
@@ -224,7 +232,10 @@ $(final_toolchain): $(no_repo_acl) $(raw_toolchain) $(toolchain_rpms_rehydrated)
 			"$(toolchain_from_repos)" \
 			"$(TOOLCHAIN_MANIFEST)" \
 			"$(go-bldtracker)" \
-			"$(TIMESTAMP_DIR)/build_mariner_toolchain.jsonl" && \
+			"$(TIMESTAMP_DIR)/build_mariner_toolchain.jsonl" 2>&1 | tee $(toolchain_official_logs_dir)/build_official_rpms.log; \
+		if [ $${PIPESTATUS[0]} -ne 0 ]; then \
+			$(call print_error, build_mariner_toolchain.sh failed); \
+		fi && \
 	$(if $(filter y,$(UPDATE_TOOLCHAIN_LIST)), ls -1 $(toolchain_build_dir)/built_rpms_all > $(MANIFESTS_DIR)/package/toolchain_$(build_arch).txt && ) \
 	touch $@
 

toolkit/scripts/toolchain/create_toolchain_in_container.sh

@@ -10,6 +10,12 @@ MARINER_SPECS_DIR=$2
 MARINER_SOURCE_URL=$3
 INCREMENTAL_TOOLCHAIN=$4
 ARCHIVE_TOOL=$5
+LOG_DIR=$6
+
+if [ -z "$MARINER_BUILD_DIR" ] || [ -z "$MARINER_SPECS_DIR" ] || [ -z "$MARINER_SOURCE_URL" ] || [ -z "$INCREMENTAL_TOOLCHAIN" ] || [ -z "$ARCHIVE_TOOL" ] || [ -z "$LOG_DIR" ]; then
+    echo "Usage: $0 <MARINER_BUILD_DIR> <MARINER_SPECS_DIR> <MARINER_SOURCE_URL> <INCREMENTAL_TOOLCHAIN> <ARCHIVE_TOOL> <LOG_DIR>"
+    exit 1
+fi
 
 # Grab an identity for the raw toolchain components so we can avoid rebuilding it if it hasn't changed
 files_to_watch=(    "./create_toolchain_in_container.sh" \
@@ -85,7 +91,10 @@ pushd $MARINER_BUILD_DIR/toolchain
 # Pull out the populated toolchain from the container
 docker rm -f marinertoolchain-container-temp 2>/dev/null || true
 temporary_toolchain_container=$(docker create --name marinertoolchain-container-temp marinertoolchain_populated:${sha_component_tag})
+rm -rf ${LOG_DIR}/logs
+mkdir -p ${LOG_DIR}/logs
 docker cp "${temporary_toolchain_container}":/temptoolchain/lfs .
+docker cp "${temporary_toolchain_container}":/temptoolchain/lfs/logs ${LOG_DIR}
 docker rm marinertoolchain-container-temp
 
 rm -rf ./populated_toolchain
@@ -105,4 +114,10 @@ rm -vf ./container/*.patch
 rm -vf ./container/.bashrc
 rm -vf ./container/toolchain-local-wget-list
 
+# Ensure the populated toolchain was created successfully
+if [ ! -f ${LOG_DIR}/logs/status_building_in_chroot_complete ]; then
+    echo "Error: Raw toolchain container build failed, check logs in ${LOG_DIR} for details"
+    exit 1
+fi
+
 echo Raw toolchain build complete