From b71305aaa44c640c3c749211f139f73affa7e3c3 Mon Sep 17 00:00:00 2001 From: tmaeno Date: Wed, 3 May 2023 10:16:45 +0200 Subject: [PATCH] pbook in batch mode --- ChangeLog.txt | 3 ++ pandaclient/AthenaUtils.py | 50 ++++++++++++++------------------ pandaclient/MiscUtils.py | 31 ++++++++++---------- pandaclient/PandaToolsPkgInfo.py | 2 +- 4 files changed, 41 insertions(+), 45 deletions(-) diff --git a/ChangeLog.txt b/ChangeLog.txt index 2b867bea..91371642 100644 --- a/ChangeLog.txt +++ b/ChangeLog.txt @@ -1,5 +1,8 @@ ** Release Notes +1.5.50 + * to recreate cpack tarball for subsequent file appending as gzip -d is not enough + 1.5.49 * not to ignore tar failures diff --git a/pandaclient/AthenaUtils.py b/pandaclient/AthenaUtils.py index 9f698b4e..4c59d784 100644 --- a/pandaclient/AthenaUtils.py +++ b/pandaclient/AthenaUtils.py @@ -3,7 +3,7 @@ import sys from . import MiscUtils -from .MiscUtils import commands_get_output, commands_get_status_output_with_env, commands_get_output_with_env, commands_failOnNonZeroExitStatus +from .MiscUtils import commands_get_output, commands_get_output_with_env, commands_fail_on_non_zero_exit_status from . import PLogger from . import Client @@ -796,10 +796,9 @@ def archiveFiles(_workArea,_packages,_archiveFullName): if verbose: print(relPath) - commands_failOnNonZeroExitStatus( + commands_fail_on_non_zero_exit_status( comStr, EC_Archive, - verboseCmd=False, verboseOutputCmd=verbose, - logger=tmpLog, errorLogMsg='tarball creation failed') + logger=tmpLog, error_log_msg='tarball creation failed') continue # else if dereferenceSymLinks: @@ -808,12 +807,11 @@ def archiveFiles(_workArea,_packages,_archiveFullName): comStr = "tar rf '%s' '%s/%s' --exclude '%s'" % (_archiveFullName,pack,item,excludePattern) if verbose: - print("%s/%s" % (pack,item)) + print("%s/%s" % (pack, item)) - commands_failOnNonZeroExitStatus( + commands_fail_on_non_zero_exit_status( comStr, EC_Archive, - verboseCmd=False, verboseOutputCmd=verbose, - logger=tmpLog, errorLogMsg='tarball creation failed') + logger=tmpLog, error_log_msg='tarball creation failed') # back to previous dir os.chdir(_curdir) @@ -880,10 +878,9 @@ def archiveFiles(_workArea,_packages,_archiveFullName): if verbose: print(relPath) - commands_failOnNonZeroExitStatus( + commands_fail_on_non_zero_exit_status( comStr, EC_Archive, - verboseCmd=verbose, verboseOutputCmd=verbose, - logger=tmpLog, errorLogMsg='tarball creation failed') + logger=tmpLog, error_log_msg='tarball creation failed') # back to current dir os.chdir(currentDir) # return @@ -934,10 +931,9 @@ def getJobOs(dir,files): if verbose: print(relPath) - commands_failOnNonZeroExitStatus( + commands_fail_on_non_zero_exit_status( comStr, EC_Archive, - verboseCmd=False, verboseOutputCmd=verbose, - logger=tmpLog, errorLogMsg='tarball creation failed') + logger=tmpLog, error_log_msg='tarball creation failed') # return return archiveName,archiveFullName @@ -1046,20 +1042,18 @@ def getCMTFiles(dir,files): if verbose: print(file) - commands_failOnNonZeroExitStatus( + commands_fail_on_non_zero_exit_status( comStr, EC_Archive, - verboseCmd=False, verboseOutputCmd=verbose, - logger=tmpLog, errorLogMsg='tarball creation failed') + logger=tmpLog, error_log_msg='tarball creation failed') # append groupArea to sources if groupArea != '' and (not nobuild): os.chdir(tmpDir) if os.path.exists(groupFileName): comStr = "tar -rh '%s' -f '%s'" % (groupFileName,archiveFullName) - commands_failOnNonZeroExitStatus( + commands_fail_on_non_zero_exit_status( comStr, EC_Archive, - verboseCmd=False, verboseOutputCmd=True, - logger=tmpLog, errorLogMsg='tarball creation failed') + logger=tmpLog, error_log_msg='tarball creation failed') commands_get_output('rm -rf %s' % groupFullName) @@ -1090,10 +1084,10 @@ def archiveWithCpack(withSource,tmpDir,verbose): comStr += '-D CPACK_PACKAGE_VERSION_MINOR="" -D CPACK_PACKAGE_VERSION_PATCH="" ' comStr += '-D CPACK_PACKAGE_DESCRIPTION="" ' - commands_failOnNonZeroExitStatus( + commands_fail_on_non_zero_exit_status( comStr, EC_Config, - verboseCmd=False, verboseOutputCmd=verbose, - logger=tmpLog, logMsg=comStr, errorLogMsg='cpack failed') + verbose_cmd=verbose, verbose_output=verbose, + logger=tmpLog, error_log_msg='cpack failed') else: use_cpack = False @@ -1103,15 +1097,15 @@ def archiveWithCpack(withSource,tmpDir,verbose): archiveFullName += '.tar' os.chdir(tmpDir) if use_cpack: - # decompress the tar.gz archive created by cpack to .tar archive using gzip - comStr = 'gzip -d {0}.gz && rm -rf _CPack_Packages'.format(archiveName) + # recreate tar to allow appending other files in the subsequent steps, as gzip decompress is not enough + comStr = 'tar xfz {0}.gz; tar cf {0} usr > /dev/null 2>&1; rm -rf usr _CPack_Packages {0}.gz'.format( + archiveName) else: comStr = 'tar cf {0} -T /dev/null > /dev/null 2>&1'.format(archiveName) - commands_failOnNonZeroExitStatus( + commands_fail_on_non_zero_exit_status( comStr, EC_Archive, - verboseCmd=False, verboseOutputCmd=False, - logger=tmpLog, errorLogMsg='tarball creation failed') + logger=tmpLog, error_log_msg='tarball creation failed') os.chdir(_curdir) return archiveName, archiveFullName diff --git a/pandaclient/MiscUtils.py b/pandaclient/MiscUtils.py index 9adb0698..f86958d8 100644 --- a/pandaclient/MiscUtils.py +++ b/pandaclient/MiscUtils.py @@ -153,34 +153,33 @@ def commands_get_status_output(com): def commands_get_output(com): return commands_get_status_output(com)[1] -def commands_failOnNonZeroExitStatus( - com, errorStatusOnFailure, - verboseCmd=False,verboseOutputCmd=False, - logger=None,logMsg="",errorLogMsg=""): - - # add log message if logger and log message have been provided - if logger is not None and logMsg != "": - logger.debug(logMsg) + +def commands_fail_on_non_zero_exit_status( + com, error_status_on_failure, + verbose_cmd=False, verbose_output=False, + logger=None, error_log_msg=""): # print command if verbose - if verboseCmd: + if verbose_cmd: print(com) # execute command, get status code and message printed by the command - status,data = commands_get_status_output(com) + status, data = commands_get_status_output(com) # fail for non zero exit status if status != 0: + if not verbose_cmd: + print(com) # print error message before failing print(data) # report error message if logger and log message have been provided - if logger is not None and errorLogMsg != "": - logger.error(errorLogMsg) + if logger and error_log_msg: + logger.error(error_log_msg) - if type(errorStatusOnFailure) == int: + if type(error_status_on_failure) == int: # use error status provided to the function - sys.exit(errorStatusOnFailure) - elif errorStatusOnFailure == "sameAsStatus": + sys.exit(error_status_on_failure) + elif error_status_on_failure == "sameAsStatus": # use error status exit code returned # by the execution of the command sys.exit(status) @@ -189,7 +188,7 @@ def commands_failOnNonZeroExitStatus( sys.exit(1) # print command output message if verbose - if verboseOutputCmd and data != "": + if verbose_output and data: print(data) return status,data diff --git a/pandaclient/PandaToolsPkgInfo.py b/pandaclient/PandaToolsPkgInfo.py index 069d7805..6cb4bfbf 100644 --- a/pandaclient/PandaToolsPkgInfo.py +++ b/pandaclient/PandaToolsPkgInfo.py @@ -1 +1 @@ -release_version = "1.5.49" +release_version = "1.5.50"