pbook in batch mode

ChangeLog.txt

@@ -1,5 +1,8 @@
 ** Release Notes
 
+1.5.50
+  * to recreate cpack tarball for subsequent file appending as gzip -d is not enough
+
 1.5.49
   * not to ignore tar failures
 

pandaclient/AthenaUtils.py

@@ -3,7 +3,7 @@
 import sys
 
 from . import MiscUtils
-from .MiscUtils import commands_get_output, commands_get_status_output_with_env, commands_get_output_with_env, commands_failOnNonZeroExitStatus
+from .MiscUtils import commands_get_output, commands_get_output_with_env, commands_fail_on_non_zero_exit_status
 from . import PLogger
 from . import Client
 
@@ -796,10 +796,9 @@ def archiveFiles(_workArea,_packages,_archiveFullName):
                             if verbose:
                                 print(relPath)
 
-                            commands_failOnNonZeroExitStatus(
+                            commands_fail_on_non_zero_exit_status(
                                 comStr, EC_Archive,
-                                verboseCmd=False, verboseOutputCmd=verbose,
-                                logger=tmpLog, errorLogMsg='tarball creation failed')
+                                logger=tmpLog, error_log_msg='tarball creation failed')
                     continue
                 # else
                 if dereferenceSymLinks:
@@ -808,12 +807,11 @@ def archiveFiles(_workArea,_packages,_archiveFullName):
                     comStr = "tar rf '%s' '%s/%s' --exclude '%s'" % (_archiveFullName,pack,item,excludePattern)
 
                 if verbose:
-                    print("%s/%s" % (pack,item))
+                    print("%s/%s" % (pack, item))
 
-                commands_failOnNonZeroExitStatus(
+                commands_fail_on_non_zero_exit_status(
                     comStr, EC_Archive,
-                    verboseCmd=False, verboseOutputCmd=verbose,
-                    logger=tmpLog, errorLogMsg='tarball creation failed')
+                    logger=tmpLog, error_log_msg='tarball creation failed')
         # back to previous dir
         os.chdir(_curdir)
 
@@ -880,10 +878,9 @@ def archiveFiles(_workArea,_packages,_archiveFullName):
             if verbose:
                 print(relPath)
 
-            commands_failOnNonZeroExitStatus(
+            commands_fail_on_non_zero_exit_status(
                 comStr, EC_Archive,
-                verboseCmd=verbose, verboseOutputCmd=verbose,
-                logger=tmpLog, errorLogMsg='tarball creation failed')
+                logger=tmpLog, error_log_msg='tarball creation failed')
     # back to current dir
     os.chdir(currentDir)
     # return
@@ -934,10 +931,9 @@ def getJobOs(dir,files):
         if verbose:
             print(relPath)
 
-        commands_failOnNonZeroExitStatus(
+        commands_fail_on_non_zero_exit_status(
             comStr, EC_Archive,
-            verboseCmd=False, verboseOutputCmd=verbose,
-            logger=tmpLog, errorLogMsg='tarball creation failed')
+            logger=tmpLog, error_log_msg='tarball creation failed')
 
     # return
     return archiveName,archiveFullName
@@ -1046,20 +1042,18 @@ def getCMTFiles(dir,files):
                     if verbose:
                         print(file)
 
-                    commands_failOnNonZeroExitStatus(
+                    commands_fail_on_non_zero_exit_status(
                         comStr, EC_Archive,
-                        verboseCmd=False, verboseOutputCmd=verbose,
-                        logger=tmpLog, errorLogMsg='tarball creation failed')
+                        logger=tmpLog, error_log_msg='tarball creation failed')
 
     # append groupArea to sources
     if groupArea != '' and (not nobuild):
         os.chdir(tmpDir)
         if os.path.exists(groupFileName):
             comStr = "tar -rh '%s' -f '%s'" % (groupFileName,archiveFullName)
-            commands_failOnNonZeroExitStatus(
+            commands_fail_on_non_zero_exit_status(
                 comStr, EC_Archive,
-                verboseCmd=False, verboseOutputCmd=True,
-                logger=tmpLog, errorLogMsg='tarball creation failed')
+                logger=tmpLog, error_log_msg='tarball creation failed')
 
             commands_get_output('rm -rf %s' % groupFullName)
 
@@ -1090,10 +1084,10 @@ def archiveWithCpack(withSource,tmpDir,verbose):
         comStr += '-D CPACK_PACKAGE_VERSION_MINOR="" -D CPACK_PACKAGE_VERSION_PATCH="" '
         comStr += '-D CPACK_PACKAGE_DESCRIPTION="" '
 
-        commands_failOnNonZeroExitStatus(
+        commands_fail_on_non_zero_exit_status(
           comStr, EC_Config,
-          verboseCmd=False, verboseOutputCmd=verbose,
-          logger=tmpLog, logMsg=comStr, errorLogMsg='cpack failed')
+          verbose_cmd=verbose, verbose_output=verbose,
+          logger=tmpLog, error_log_msg='cpack failed')
 
     else:
         use_cpack = False
@@ -1103,15 +1097,15 @@ def archiveWithCpack(withSource,tmpDir,verbose):
     archiveFullName += '.tar'
     os.chdir(tmpDir)
     if use_cpack:
-        # decompress the tar.gz archive created by cpack to .tar archive using gzip 
-        comStr = 'gzip -d {0}.gz && rm -rf _CPack_Packages'.format(archiveName)
+        # recreate tar to allow appending other files in the subsequent steps, as gzip decompress is not enough
+        comStr = 'tar xfz {0}.gz; tar cf {0} usr > /dev/null 2>&1; rm -rf usr _CPack_Packages {0}.gz'.format(
+            archiveName)
     else:
         comStr = 'tar cf {0} -T /dev/null > /dev/null 2>&1'.format(archiveName)
 
-    commands_failOnNonZeroExitStatus(
+    commands_fail_on_non_zero_exit_status(
         comStr, EC_Archive,
-        verboseCmd=False, verboseOutputCmd=False,
-        logger=tmpLog, errorLogMsg='tarball creation failed')
+        logger=tmpLog, error_log_msg='tarball creation failed')
 
     os.chdir(_curdir)
     return archiveName, archiveFullName

pandaclient/MiscUtils.py

@@ -153,34 +153,33 @@ def commands_get_status_output(com):
 def commands_get_output(com):
     return commands_get_status_output(com)[1]
 
-def commands_failOnNonZeroExitStatus(
-    com, errorStatusOnFailure,
-    verboseCmd=False,verboseOutputCmd=False,
-    logger=None,logMsg="",errorLogMsg=""):
-
-    # add log message if logger and log message have been provided 
-    if logger is not None and logMsg != "":
-        logger.debug(logMsg)
+
+def commands_fail_on_non_zero_exit_status(
+        com, error_status_on_failure,
+        verbose_cmd=False, verbose_output=False,
+        logger=None, error_log_msg=""):
 
     # print command if verbose
-    if verboseCmd:
+    if verbose_cmd:
         print(com)
 
     # execute command, get status code and message printed by the command 
-    status,data = commands_get_status_output(com)
+    status, data = commands_get_status_output(com)
 
     # fail for non zero exit status 
     if status != 0:
+        if not verbose_cmd:
+            print(com)
         # print error message before failing
         print(data)
         # report error message if logger and log message have been provided 
-        if logger is not None and errorLogMsg != "":
-            logger.error(errorLogMsg)
+        if logger and error_log_msg:
+            logger.error(error_log_msg)
 
-        if type(errorStatusOnFailure) == int:
+        if type(error_status_on_failure) == int:
             # use error status provided to the function
-            sys.exit(errorStatusOnFailure)
-        elif errorStatusOnFailure == "sameAsStatus":
+            sys.exit(error_status_on_failure)
+        elif error_status_on_failure == "sameAsStatus":
             # use error status exit code returned 
             # by the execution of the command
             sys.exit(status)
@@ -189,7 +188,7 @@ def commands_failOnNonZeroExitStatus(
             sys.exit(1)
 
     # print command output message if verbose
-    if verboseOutputCmd and data != "":
+    if verbose_output and data:
         print(data)
 
     return status,data

pandaclient/PandaToolsPkgInfo.py

@@ -1 +1 @@
-release_version = "1.5.49"
+release_version = "1.5.50"