Merge remote-tracking branch 'origin/main' into database-private-mapped

orchestration-service/job_status.py

@@ -57,6 +57,7 @@ class JobStatus:
         initialized None
     `actual_integrity_hash` hash once last block has been reached
         initialized to None
+    `error_message` error message reported back on failure
     The class in inialized from an array, whose elements have a property `replay_slice_id`
     """
     def __init__(self, config):
@@ -67,6 +68,7 @@ def __init__(self, config):
         self.start_time = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
         self.end_time = None
         self.actual_integrity_hash = None
+        self.error_message = None
 
     def __repr__(self):
         return (f"JobStatus(job_id={self.job_id}, "
@@ -80,7 +82,8 @@ def __repr__(self):
                 f"last_block_processed={self.last_block_processed}, "
                 f"start_time={self.start_time}, end_time={self.end_time}, "
                 f"expected_integrity_hash={self.slice_config.expected_integrity_hash}, "
-                f"actual_integrity_hash={self.actual_integrity_hash})")
+                f"actual_integrity_hash={self.actual_integrity_hash}), "
+                f"error_message={self.error_message}")
 
     def __str__(self):
         """converts job object to string"""
@@ -95,7 +98,8 @@ def __str__(self):
                 f"last_block_processed={self.last_block_processed}, "
                 f"start_time={self.start_time}, end_time={self.end_time}, "
                 f"expected_integrity_hash={self.slice_config.expected_integrity_hash}, "
-                f"actual_integrity_hash={self.actual_integrity_hash}")
+                f"actual_integrity_hash={self.actual_integrity_hash}, "
+                f"error_message={self.error_message}")
 
     def as_dict(self):
         """converts job object to a dictionary"""
@@ -113,6 +117,7 @@ def as_dict(self):
         this_dict['end_time'] = self.end_time
         this_dict['expected_integrity_hash'] = self.slice_config.expected_integrity_hash
         this_dict['actual_integrity_hash'] = self.actual_integrity_hash
+        this_dict['error_message'] = self.error_message
         return this_dict
 
 
@@ -178,11 +183,12 @@ def set_job(self, data):
             self.jobs[jobid].start_time = data['start_time']
         if 'actual_integrity_hash' in data:
             self.jobs[jobid].actual_integrity_hash = data['actual_integrity_hash']
+        if 'error_message' in data:
+            self.jobs[jobid].error_message = data['error_message']
 
         # success
         return True
 
-
     def set_job_from_json(self, status_as_json, jobid):
         """sets jobs data from json, calls set_job(), return bool for success"""
         data = json.loads(status_as_json)

replay-client/job_operations.py

@@ -152,6 +152,15 @@ def update_job_status(base_url, max_tries, job_id, status):
     """Fetch a job (GET) by id; update status to provided value"""
     return proccess_job_update(base_url, max_tries, job_id, {'status':status})
 
+def update_error_message(base_url, max_tries, job_id, error_message):
+    """Fetch a job (GET) by id; set status to error and set error message"""
+    status = "ERROR"
+    error_object = {
+        'status':status,
+        'error_message': error_message
+    }
+    return proccess_job_update(base_url, max_tries, job_id, error_object)
+
 def update_job_progress(base_url, max_tries, job_id, block_processed):
     """Fetch a job (GET) by id; update status to provided value"""
     fields_to_update = {
@@ -193,6 +202,9 @@ def set_job_completed(base_url, max_tries, job_id, last_block_processed, end_tim
     parser.add_argument('--status',
         type=str,
         help='status to set job')
+    parser.add_argument('--error-message',
+        type=str,
+        help='error message')
     parser.add_argument('--block-processed',
         type=str,
         help='last block processed')
@@ -206,7 +218,7 @@ def set_job_completed(base_url, max_tries, job_id, last_block_processed, end_tim
     args = parser.parse_args()
 
     # validate argument values
-    if args.operation in ['update-status', 'update-progress', 'complete'] and args.job_id is None:
+    if args.operation in ['update-status', 'update-error', 'update-progress', 'complete'] and args.job_id is None:
         sys.exit(f"Error job_id must be specifid for operation {args.operation}")
     if args.max_tries < 1:
         sys.exit("Error max-tries must be greater then zero")
@@ -229,6 +241,11 @@ def set_job_completed(base_url, max_tries, job_id, last_block_processed, end_tim
             args.max_tries,
             args.job_id,
             args.status)
+    elif args.operation == "update-error":
+        job_message = update_error_message(url,
+            args.max_tries,
+            args.job_id,
+            args.error_message)
     elif args.operation == "update-progress":
         job_message = update_job_progress(url,
             args.max_tries,

replay-client/start-nodeos-run-replay.sh

@@ -37,6 +37,7 @@ else
 fi
 
 function trap_exit() {
+  ERROR_MSG=${1:-"NA"}
   if [ -n "${BACKGROUND_STATUS_PID}" ]; then
     kill "${BACKGROUND_STATUS_PID}"
   fi
@@ -45,15 +46,15 @@ function trap_exit() {
   fi
   [ -f "$LOCK_FILE" ] && rm "$LOCK_FILE"
   if [ -n "${JOBID}" ]; then
-    python3 "${REPLAY_CLIENT_DIR:?}"/job_operations.py --host ${ORCH_IP} --port ${ORCH_PORT} --operation update-status --status "ERROR" --job-id ${JOBID}
+    python3 "${REPLAY_CLIENT_DIR:?}"/job_operations.py --host ${ORCH_IP} --port ${ORCH_PORT} --operation update-error --error-message "$ERROR_MSG" --job-id ${JOBID}
   fi
   echo "Caught signal or detected error exiting"
   exit 127
 }
 
 ## set status to error if we exit on signal ##
-trap trap_exit INT
-trap trap_exit TERM
+trap 'trap_exit SIGINT' INT
+trap 'trap_exit SIGTERM' TERM
 
 ##################
 # 1) performs file setup: create dirs, get snapshot to load
@@ -66,7 +67,7 @@ TUID=$(id -ur)
 ## must not be root to run ##
 if [ "$TUID" -eq 0 ]; then
   echo "Trying to run as root user exiting"
-  trap_exit
+  trap_exit "Cannot run as root user"
 fi
 
 ## cleanup previous runs ##
@@ -76,7 +77,7 @@ fi
 volsize=$(df -B 1073741824 /data | awk 'NR==2 {print $4}')
 if [ ${volsize:-0} -lt 40 ]; then
   echo "/data volume does not exist or does not have 40Gb free space"
-  trap_exit
+  trap_exit "/data volume does not exist or does not have 40Gb free space"
 fi
 
 ## directory setup ##
@@ -91,7 +92,7 @@ python3 "${REPLAY_CLIENT_DIR:?}"/job_operations.py --host ${ORCH_IP} --port ${OR
 # if json result is empty failed to aquire job
 if [[ ! -e "/tmp/job.conf.json" || ! -s "/tmp/job.conf.json" ]]; then
   echo "Failed to aquire job"
-  trap_exit
+  trap_exit "Failed to aquire job"
 fi
 echo "Received job details processing..."
 
@@ -125,15 +126,15 @@ if [ $STORAGE_TYPE = "s3" ]; then
   fi
 else
   echo "Unknown snapshot type ${STORAGE_TYPE}"
-  trap_exit
+  trap_exit "Unknown snapshot type ${STORAGE_TYPE}"
 fi
 
 # restore blocks.log from cloud storage
 echo "Restoring Blocks.log from Cloud Storage"
 "${REPLAY_CLIENT_DIR:?}"/manage_blocks_log.sh "$NODEOS_DIR" $START_BLOCK $END_BLOCK "${SNAPSHOT_PATH}"
 if [ $? -ne 0 ]; then
   echo "Failed to restore blocks.log"
-  trap_exit
+  trap_exit "Failed to restore blocks.log"
 fi
 
 ## when start block 0 no snapshot to process ##
@@ -143,7 +144,7 @@ if [ $START_BLOCK -gt 0 ] && [ -f "${NODEOS_DIR}"/snapshot/snapshot.bin.zst ]; t
   # sometimes compression format is bad error out on failure
   if [ $? -ne 0 ]; then
     echo "Failed to unzip snapshot"
-    trap_exit
+    trap_exit "Failed to unzip snapshot"
   fi
 fi
 

webcontent/detail.html

@@ -48,6 +48,9 @@ <h3 id="detail-config-heading">Configuration</h3>
     jobDetailList += '<li><strong>Current Block<strong>: <p>'+jobObj.last_block_processed+'</p><li>'
     jobDetailList += '<li><strong>Status<strong>: <lable class="badge '
       +getBadgeType(jobObj.status)+'">'+jobObj.status+'</lable><li>'
+    if (jobObj.status === "ERROR" && jobObj.error_message !== "NA") {
+      jobDetailList += '<li><strong>Error Message<strong>: <p>'+jobObj.error_message+'</p><li>'
+    }
     jobDetailList += '<li><strong>Actual Integrity Hash<strong>: <p>'+jobObj.actual_integrity_hash+'</p><li>'
     jobDetailList += '<li><strong>Expected Integrity Hash<strong>: <p>'+jobObj.expected_integrity_hash+'</p><li>'
     jobDetailList += '</ul>'