From 48c122c06fe5b60a6d85a342f3fde7952fc829c4 Mon Sep 17 00:00:00 2001 From: Josh Cooper Date: Thu, 18 Jan 2024 15:52:37 -0800 Subject: [PATCH] (PA-4154) Add post-suite step to archive logs --- acceptance/config/aio/options.rb | 5 +- .../teardown/common/099_Archive_Logs.rb | 107 ++++++++++++++++++ 2 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 acceptance/teardown/common/099_Archive_Logs.rb diff --git a/acceptance/config/aio/options.rb b/acceptance/config/aio/options.rb index b45b945259f..d2d69edfd05 100644 --- a/acceptance/config/aio/options.rb +++ b/acceptance/config/aio/options.rb @@ -4,5 +4,8 @@ 'use-service' => true, # use service scripts to start/stop stuff 'puppetservice' => 'puppetserver', 'puppetserver-confdir' => '/etc/puppetlabs/puppetserver/conf.d', - 'puppetserver-config' => '/etc/puppetlabs/puppetserver/conf.d/puppetserver.conf' + 'puppetserver-config' => '/etc/puppetlabs/puppetserver/conf.d/puppetserver.conf', + :post_suite => [ + 'teardown/common/099_Archive_Logs.rb', + ], } diff --git a/acceptance/teardown/common/099_Archive_Logs.rb b/acceptance/teardown/common/099_Archive_Logs.rb new file mode 100644 index 00000000000..d940e2bea2e --- /dev/null +++ b/acceptance/teardown/common/099_Archive_Logs.rb @@ -0,0 +1,107 @@ +require 'date' + +def file_glob(host, path) + result = on(host, "ls #{path}", :acceptable_exit_codes => [0, 2]) + return [] if result.exit_code != 0 + return result.stdout.strip.split("\n") +end + +# This test is prefixed with zzz so it will hopefully run last. +test_name 'Backup puppet logs and app data on all hosts' do + today = Date.today().to_s + # truncate the job name so it only has the name-y part and no parameters + job_name = (ENV['JOB_NAME'] || 'unknown_jenkins_job') + .sub(/[A-Z0-9_]+=.*$/, '') + .gsub(/[\/,.]/, '_')[0..200] + archive_name = "#{job_name}__#{ENV['BUILD_ID']}__#{today}__sut-files.tgz" + archive_root = "SUT_#{today}" + + hosts.each do |host| + step("Capturing log errors for #{host}") do + case host[:platform] + when /windows/ + # on Windows, all of the desired data (including logs) is in the data dir + puppetlabs_data = 'C:/ProgramData/PuppetLabs' + archive_file_from(host, puppetlabs_data, {}, archive_root, archive_name) + + # Note: Windows `ls` uses absolute paths for all matches when an absolute path is supplied. + tempdir = 'C:/Windows/TEMP' + file_glob(host, File.join(tempdir, 'install-puppet-*.log')).each do |install_log| + archive_file_from(host, install_log, {}, archive_root, archive_name) + end + file_glob(host, File.join(tempdir, 'puppet-*-installer.log')).each do |install_log| + archive_file_from(host, install_log, {}, archive_root, archive_name) + end + else + puppetlabs_logdir = '/var/log/puppetlabs' + grep_for_alerts = if host[:platform] =~ /solaris/ + "egrep -i 'warn|error|fatal'" + elsif host[:platform] =~ /aix/ + "grep -iE -B5 -A10 'warn|error|fatal'" + else + "grep -i -B5 -A10 'warn\\|error\\|fatal'" + end + + ## If there are any PL logs, try to echo all warning, error, and fatal + ## messages from all PL logs to the job's output + on(host, <<-GREP_FOR_ALERTS, :accept_all_exit_codes => true ) + if [ -d #{puppetlabs_logdir} ] && [ -n "$(find #{puppetlabs_logdir} -name '*.log*')" ]; then + for log in $(find #{puppetlabs_logdir} -name '*.log*'); do + # grep /dev/null only to get grep to print filenames, since -H is not in POSIX spec for grep + #{grep_for_alerts} $log /dev/null; + echo "" + done + fi + GREP_FOR_ALERTS + + step("Archiving logs for #{host} into #{archive_name} (muzzling everything but :warn or higher beaker logs...)") do + ## turn the logger off to avoid getting hundreds of lines of scp progress output + previous_level = @logger.log_level + @logger.log_level = :warn + + pxp_cache = '/opt/puppetlabs/pxp-agent/spool' + puppetlabs_data = '/etc/puppetlabs' + + version_lookup_result = on(host, "cat /opt/puppetlabs/puppet/VERSION", :accept_all_exit_codes => true) + + # If we can't find a VERSION file, chances are puppet wasn't + # installed and these paths aren't present. Beaker's + # archive_file_from() will fail if it can't find the file, and we + # want to proceed... + if version_lookup_result.exit_code == 0 + agent_version = version_lookup_result.output.strip + archive_file_from(host, pxp_cache, {}, archive_root, archive_name) unless version_is_less(agent_version, "1.3.2") + archive_file_from(host, puppetlabs_data, {}, archive_root, archive_name) + archive_file_from(host, puppetlabs_logdir, {}, archive_root, archive_name) + end + + syslog_dir = '/var/log' + syslog_name = 'messages' + if host[:platform] =~ /ubuntu|debian/ + syslog_name = 'syslog' + elsif host[:platform] =~ /solaris/ + syslog_dir = '/var/adm' + # Next few lines are for debugging POOLER-200, once that is resolved this can be removed + @logger.log_level = previous_level + on(host, 'egrep -i \'reboot after panic\' /var/adm/messages', :acceptable_exit_codes => [0,1,2]) + @logger.log_level = :warn + elsif host[:platform] =~ /osx/ + syslog_name = "system.log" + elsif host[:platform] =~ /fedora/ + on(host, "journalctl --no-pager > /var/log/messages") + elsif host[:platform] =~ /aix/ + on(host, "alog -o -t console > /var/log/messages") + end + + syslog_path = File.join(syslog_dir, syslog_name) + if host.file_exist?(syslog_path) + archive_file_from(host, syslog_path, {}, archive_root, archive_name) + end + + ## turn the logger back on in case someone else wants to log things + @logger.log_level = previous_level + end + end + end + end +end