Refactoring & Added PagerDuty incident creation on job failure

webapp/src/main/java/com/box/l10n/mojito/entity/ScheduledJob.java

@@ -13,9 +13,9 @@
 import java.util.Date;
 import org.hibernate.envers.Audited;
 
+@Audited
 @Entity
 @Table(name = "scheduled_job")
-@Audited
 public class ScheduledJob {
   @Id private String id;
 

webapp/src/main/java/com/box/l10n/mojito/rest/scheduledjob/ScheduledJobWS.java

@@ -8,6 +8,8 @@
 import java.util.List;
 import java.util.Optional;
 import java.util.stream.Collectors;
+import org.quartz.JobExecutionContext;
+import org.quartz.JobKey;
 import org.quartz.SchedulerException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -22,6 +24,11 @@
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.server.ResponseStatusException;
 
+/**
+ * Webservice for enabling/disabling and triggering scheduled jobs.
+ *
+ * @author mattwilshire
+ */
 @RestController
 @ConditionalOnProperty(value = "l10n.scheduledJobs.enabled", havingValue = "true")
 public class ScheduledJobWS {
@@ -59,11 +66,33 @@ public ResponseEntity<ScheduledJobResponse> triggerJob(@PathVariable String id)
           .body(new ScheduledJobResponse(ScheduledJobResponse.Status.ERROR, "Job not found"));
 
     ScheduledJob scheduledJob = optionalScheduledJob.get();
+    JobKey jobKey = scheduledJobManager.getJobKey(scheduledJob);
+
     try {
-      return scheduledJobManager.triggerJob(scheduledJob);
+      // Is the job currently running ?
+      // Ignore the trigger request and tell the user it is currently running
+      for (JobExecutionContext jobExecutionContext :
+          scheduledJobManager.getScheduler().getCurrentlyExecutingJobs()) {
+        if (jobExecutionContext.getJobDetail().getKey().equals(jobKey)) {
+          return ResponseEntity.status(HttpStatus.CONFLICT)
+              .body(
+                  new ScheduledJobResponse(
+                      ScheduledJobResponse.Status.ERROR,
+                      "Trigger ignored, job is currently running"));
+        }
+      }
+
+      if (!scheduledJobManager.getScheduler().checkExists(jobKey))
+        return ResponseEntity.status(HttpStatus.NOT_FOUND)
+            .body(new ScheduledJobResponse(ScheduledJobResponse.Status.ERROR, "Job doesn't exist"));
+      scheduledJobManager.getScheduler().triggerJob(jobKey);
+      return ResponseEntity.status(HttpStatus.OK)
+          .body(new ScheduledJobResponse(ScheduledJobResponse.Status.SUCCESS, "Job triggered"));
     } catch (SchedulerException e) {
-      throw new ResponseStatusException(
-          HttpStatus.INTERNAL_SERVER_ERROR, "Job with id: " + id + " could not be triggered");
+      logger.error(
+          "Error triggering job manually, job: {}", jobKey.getName() + ":" + jobKey.getGroup(), e);
+      return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
+          .body(new ScheduledJobResponse(ScheduledJobResponse.Status.ERROR, e.getMessage()));
     }
   }
 
@@ -76,8 +105,21 @@ public ResponseEntity<ScheduledJobResponse> toggleJob(@PathVariable String id) {
           .body(new ScheduledJobResponse(ScheduledJobResponse.Status.ERROR, "Job not found"));
 
     ScheduledJob scheduledJob = optionalScheduledJob.get();
+    JobKey jobKey = scheduledJobManager.getJobKey(scheduledJob);
+
     try {
-      return scheduledJobManager.toggleJob(scheduledJob);
+      if (!scheduledJobManager.getScheduler().checkExists(jobKey))
+        return ResponseEntity.status(HttpStatus.NOT_FOUND)
+            .body(new ScheduledJobResponse(ScheduledJobResponse.Status.ERROR, "Job doesn't exist"));
+
+      scheduledJob.setEnabled(!scheduledJob.getEnabled());
+      scheduledJobRepository.save(scheduledJob);
+
+      return ResponseEntity.status(HttpStatus.OK)
+          .body(
+              new ScheduledJobResponse(
+                  ScheduledJobResponse.Status.SUCCESS,
+                  "Job " + (scheduledJob.getEnabled() ? "enabled" : "disabled")));
     } catch (SchedulerException e) {
       throw new ResponseStatusException(
           HttpStatus.INTERNAL_SERVER_ERROR, "Job with id: " + id + " could not be disabled");

webapp/src/main/java/com/box/l10n/mojito/service/scheduledjob/ScheduledJobListener.java

@@ -10,6 +10,14 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+/**
+ * Listener that listens for Quartz job events, this listener is attached to the 'scheduledJobs'
+ * scheduler and handles setting the job status, start date and end date for pre- and post-execution
+ * of the job. Scheduled jobs implement the IScheduledJob interface which allows the job to receive
+ * failure and success notifications from the listener.
+ *
+ * @author mattwilshire
+ */
 public class ScheduledJobListener extends JobListenerSupport {
 
   static Logger logger = LoggerFactory.getLogger(ScheduledJobListener.class);
@@ -32,32 +40,50 @@ public String getName() {
     return "ScheduledJobListener";
   }
 
+  /** The job is about to be executed, set the status and start date */
   @Override
   @Transactional
   public void jobToBeExecuted(JobExecutionContext context) {
     ScheduledJob scheduledJob =
         scheduledJobRepository.findByJobKey(context.getJobDetail().getKey());
 
+    logger.debug(
+        "Preparing to execute job {} for repository {}",
+        scheduledJob.getJobType().getEnum(),
+        scheduledJob.getRepository().getName());
+
     scheduledJob.setJobStatus(
         scheduledJobStatusRepository.findByEnum(ScheduledJobStatus.IN_PROGRESS));
     scheduledJob.setStartDate(new Date());
     scheduledJob.setEndDate(null);
 
-    // TODO: Try catch, PD NOTIFICATION
+    // This had a deadlock due to the audited table being updated by other jobs are the same time,
+    // the rev and revend columns are incremental meaning a lock is needed to increment the next
+    // row.
     deadlockRetryTemplate.execute(
         c -> {
           scheduledJobRepository.save(scheduledJob);
           return null;
         });
+
+    logger.debug(
+        "Job {} for repository {} is now in progress.",
+        scheduledJob.getJobType().getEnum(),
+        scheduledJob.getRepository().getName());
   }
 
+  /** The job finished execution, if an error occurred jobException will not be null */
   @Override
   @Transactional
   public void jobWasExecuted(JobExecutionContext context, JobExecutionException jobException) {
     ScheduledJob scheduledJob =
         scheduledJobRepository.findByJobKey(context.getJobDetail().getKey());
-    scheduledJob.setEndDate(new Date());
+    logger.debug(
+        "Handling post execution for job {} for repository {}",
+        scheduledJob.getJobType().getEnum(),
+        scheduledJob.getRepository().getName());
 
+    scheduledJob.setEndDate(new Date());
     IScheduledJob jobInstance = (IScheduledJob) context.getJobInstance();
 
     scheduledJob.setJobStatus(
@@ -71,11 +97,15 @@ public void jobWasExecuted(JobExecutionContext context, JobExecutionException jo
       jobInstance.onFailure(context, jobException);
     }
 
-    // TODO: Try catch, PD NOTIFICATION
     deadlockRetryTemplate.execute(
         c -> {
           scheduledJobRepository.save(scheduledJob);
           return null;
         });
+
+    logger.debug(
+        "Saved results for job {} for repository {}",
+        scheduledJob.getJobType().getEnum(),
+        scheduledJob.getRepository().getName());
   }
 }