From 348a15d8cf9f24a6f90838ad60b1a0ea4dc708b5 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 22 Oct 2024 10:16:34 +0100 Subject: [PATCH] Refactoring & Added PagerDuty incident creation on job failure --- .../box/l10n/mojito/entity/ScheduledJob.java | 2 +- .../rest/scheduledjob/ScheduledJobWS.java | 50 ++++- .../scheduledjob/ScheduledJobListener.java | 36 +++- .../scheduledjob/ScheduledJobManager.java | 194 ++++++++---------- .../ScheduledJobTriggerListener.java | 11 +- .../jobs/ScheduledThirdPartySync.java | 61 +++++- 6 files changed, 226 insertions(+), 128 deletions(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/entity/ScheduledJob.java b/webapp/src/main/java/com/box/l10n/mojito/entity/ScheduledJob.java index 972621c37..5e60e635f 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/entity/ScheduledJob.java +++ b/webapp/src/main/java/com/box/l10n/mojito/entity/ScheduledJob.java @@ -13,9 +13,9 @@ import java.util.Date; import org.hibernate.envers.Audited; +@Audited @Entity @Table(name = "scheduled_job") -@Audited public class ScheduledJob { @Id private String id; diff --git a/webapp/src/main/java/com/box/l10n/mojito/rest/scheduledjob/ScheduledJobWS.java b/webapp/src/main/java/com/box/l10n/mojito/rest/scheduledjob/ScheduledJobWS.java index 4a68a5e4d..ffb7211c6 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/rest/scheduledjob/ScheduledJobWS.java +++ b/webapp/src/main/java/com/box/l10n/mojito/rest/scheduledjob/ScheduledJobWS.java @@ -8,6 +8,8 @@ import java.util.List; import java.util.Optional; import java.util.stream.Collectors; +import org.quartz.JobExecutionContext; +import org.quartz.JobKey; import org.quartz.SchedulerException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,6 +24,11 @@ import org.springframework.web.bind.annotation.RestController; import org.springframework.web.server.ResponseStatusException; +/** + * Webservice for enabling/disabling and triggering scheduled jobs. + * + * @author mattwilshire + */ @RestController @ConditionalOnProperty(value = "l10n.scheduledJobs.enabled", havingValue = "true") public class ScheduledJobWS { @@ -59,11 +66,33 @@ public ResponseEntity triggerJob(@PathVariable String id) .body(new ScheduledJobResponse(ScheduledJobResponse.Status.ERROR, "Job not found")); ScheduledJob scheduledJob = optionalScheduledJob.get(); + JobKey jobKey = scheduledJobManager.getJobKey(scheduledJob); + try { - return scheduledJobManager.triggerJob(scheduledJob); + // Is the job currently running ? + // Ignore the trigger request and tell the user it is currently running + for (JobExecutionContext jobExecutionContext : + scheduledJobManager.getScheduler().getCurrentlyExecutingJobs()) { + if (jobExecutionContext.getJobDetail().getKey().equals(jobKey)) { + return ResponseEntity.status(HttpStatus.CONFLICT) + .body( + new ScheduledJobResponse( + ScheduledJobResponse.Status.ERROR, + "Trigger ignored, job is currently running")); + } + } + + if (!scheduledJobManager.getScheduler().checkExists(jobKey)) + return ResponseEntity.status(HttpStatus.NOT_FOUND) + .body(new ScheduledJobResponse(ScheduledJobResponse.Status.ERROR, "Job doesn't exist")); + scheduledJobManager.getScheduler().triggerJob(jobKey); + return ResponseEntity.status(HttpStatus.OK) + .body(new ScheduledJobResponse(ScheduledJobResponse.Status.SUCCESS, "Job triggered")); } catch (SchedulerException e) { - throw new ResponseStatusException( - HttpStatus.INTERNAL_SERVER_ERROR, "Job with id: " + id + " could not be triggered"); + logger.error( + "Error triggering job manually, job: {}", jobKey.getName() + ":" + jobKey.getGroup(), e); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) + .body(new ScheduledJobResponse(ScheduledJobResponse.Status.ERROR, e.getMessage())); } } @@ -76,8 +105,21 @@ public ResponseEntity toggleJob(@PathVariable String id) { .body(new ScheduledJobResponse(ScheduledJobResponse.Status.ERROR, "Job not found")); ScheduledJob scheduledJob = optionalScheduledJob.get(); + JobKey jobKey = scheduledJobManager.getJobKey(scheduledJob); + try { - return scheduledJobManager.toggleJob(scheduledJob); + if (!scheduledJobManager.getScheduler().checkExists(jobKey)) + return ResponseEntity.status(HttpStatus.NOT_FOUND) + .body(new ScheduledJobResponse(ScheduledJobResponse.Status.ERROR, "Job doesn't exist")); + + scheduledJob.setEnabled(!scheduledJob.getEnabled()); + scheduledJobRepository.save(scheduledJob); + + return ResponseEntity.status(HttpStatus.OK) + .body( + new ScheduledJobResponse( + ScheduledJobResponse.Status.SUCCESS, + "Job " + (scheduledJob.getEnabled() ? "enabled" : "disabled"))); } catch (SchedulerException e) { throw new ResponseStatusException( HttpStatus.INTERNAL_SERVER_ERROR, "Job with id: " + id + " could not be disabled"); diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/scheduledjob/ScheduledJobListener.java b/webapp/src/main/java/com/box/l10n/mojito/service/scheduledjob/ScheduledJobListener.java index f341d96dd..9e58ce16b 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/scheduledjob/ScheduledJobListener.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/scheduledjob/ScheduledJobListener.java @@ -10,6 +10,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Listener that listens for Quartz job events, this listener is attached to the 'scheduledJobs' + * scheduler and handles setting the job status, start date and end date for pre- and post-execution + * of the job. Scheduled jobs implement the IScheduledJob interface which allows the job to receive + * failure and success notifications from the listener. + * + * @author mattwilshire + */ public class ScheduledJobListener extends JobListenerSupport { static Logger logger = LoggerFactory.getLogger(ScheduledJobListener.class); @@ -32,32 +40,50 @@ public String getName() { return "ScheduledJobListener"; } + /** The job is about to be executed, set the status and start date */ @Override @Transactional public void jobToBeExecuted(JobExecutionContext context) { ScheduledJob scheduledJob = scheduledJobRepository.findByJobKey(context.getJobDetail().getKey()); + logger.debug( + "Preparing to execute job {} for repository {}", + scheduledJob.getJobType().getEnum(), + scheduledJob.getRepository().getName()); + scheduledJob.setJobStatus( scheduledJobStatusRepository.findByEnum(ScheduledJobStatus.IN_PROGRESS)); scheduledJob.setStartDate(new Date()); scheduledJob.setEndDate(null); - // TODO: Try catch, PD NOTIFICATION + // This had a deadlock due to the audited table being updated by other jobs are the same time, + // the rev and revend columns are incremental meaning a lock is needed to increment the next + // row. deadlockRetryTemplate.execute( c -> { scheduledJobRepository.save(scheduledJob); return null; }); + + logger.debug( + "Job {} for repository {} is now in progress.", + scheduledJob.getJobType().getEnum(), + scheduledJob.getRepository().getName()); } + /** The job finished execution, if an error occurred jobException will not be null */ @Override @Transactional public void jobWasExecuted(JobExecutionContext context, JobExecutionException jobException) { ScheduledJob scheduledJob = scheduledJobRepository.findByJobKey(context.getJobDetail().getKey()); - scheduledJob.setEndDate(new Date()); + logger.debug( + "Handling post execution for job {} for repository {}", + scheduledJob.getJobType().getEnum(), + scheduledJob.getRepository().getName()); + scheduledJob.setEndDate(new Date()); IScheduledJob jobInstance = (IScheduledJob) context.getJobInstance(); scheduledJob.setJobStatus( @@ -71,11 +97,15 @@ public void jobWasExecuted(JobExecutionContext context, JobExecutionException jo jobInstance.onFailure(context, jobException); } - // TODO: Try catch, PD NOTIFICATION deadlockRetryTemplate.execute( c -> { scheduledJobRepository.save(scheduledJob); return null; }); + + logger.debug( + "Saved results for job {} for repository {}", + scheduledJob.getJobType().getEnum(), + scheduledJob.getRepository().getName()); } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/scheduledjob/ScheduledJobManager.java b/webapp/src/main/java/com/box/l10n/mojito/service/scheduledjob/ScheduledJobManager.java index 54f2bfafc..6cfb5e548 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/scheduledjob/ScheduledJobManager.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/scheduledjob/ScheduledJobManager.java @@ -9,13 +9,13 @@ import com.box.l10n.mojito.service.scheduledjob.jobs.ScheduledThirdPartySyncProperties; import com.box.l10n.mojito.service.thirdparty.ThirdPartySyncJobConfig; import com.box.l10n.mojito.service.thirdparty.ThirdPartySyncJobsConfig; +import com.google.common.base.Strings; import jakarta.annotation.PostConstruct; import java.util.ArrayList; import java.util.List; import org.quartz.CronScheduleBuilder; import org.quartz.JobBuilder; import org.quartz.JobDetail; -import org.quartz.JobExecutionContext; import org.quartz.JobKey; import org.quartz.Scheduler; import org.quartz.SchedulerException; @@ -30,10 +30,14 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Configuration; import org.springframework.dao.DataIntegrityViolationException; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Component; +/** + * Component for scheduling jobs inside the scheduled jobs table. Currently, jobs are pulled from + * the application.properties and pushed to the scheduled_job table. + * + * @author mattwilshire + */ @Configuration @Component @ConditionalOnProperty(value = "l10n.scheduledJobs.enabled", havingValue = "true") @@ -57,30 +61,96 @@ public class ScheduledJobManager { @PostConstruct public void init() throws ClassNotFoundException, SchedulerException { logger.info("Scheduled Job Manager started."); - - // Attach Job and Trigger listeners on the scheduler + // Add Job Listener getScheduler() .getListenerManager() .addJobListener( new ScheduledJobListener( scheduledJobRepository, scheduledJobStatusRepository, deadlockRetryTemplate)); + // Add Trigger Listener getScheduler() .getListenerManager() .addTriggerListener(new ScheduledJobTriggerListener(scheduledJobRepository)); - // Loop through app properties jobs and push them to DB + pushJobsToDB(); + cleanQuartzJobs(); + scheduleAllJobs(); + } + + /** Schedule all the jobs in the scheduled_job table with their cron expression. */ + public void scheduleAllJobs() throws ClassNotFoundException, SchedulerException { + List scheduledJobs = scheduledJobRepository.findAll(); + + for (ScheduledJob scheduledJob : scheduledJobs) { + JobKey jobKey = getJobKey(scheduledJob); + TriggerKey triggerKey = getTriggerKey(scheduledJob); + + // Retrieve job class from enum value e.g. THIRD_PARTY_SYNC -> ScheduledThirdPartySync + Class jobType = + loadJobClass(scheduledJob.getJobType().getEnum().getJobClassName()); + + JobDetail job = JobBuilder.newJob(jobType).withIdentity(jobKey).build(); + Trigger trigger = buildTrigger(jobKey, scheduledJob.getCron(), triggerKey); + + if (getScheduler().checkExists(jobKey)) { + // The cron could have changed, reschedule the job + getScheduler().rescheduleJob(triggerKey, trigger); + } else { + getScheduler().scheduleJob(job, trigger); + } + + scheduledJob.setJobStatus( + scheduledJobStatusRepository.findByEnum(ScheduledJobStatus.SCHEDULED)); + scheduledJob.setStartDate(null); + scheduledJob.setEndDate(null); + scheduledJobRepository.save(scheduledJob); + } + } + + /** Push the jobs defined in the application.properties to the jobs table. */ + public void pushJobsToDB() { for (ThirdPartySyncJobConfig syncJobConfig : thirdPartySyncJobsConfig.getThirdPartySyncJobs().values()) { - if (syncJobConfig.getUuid().isEmpty() || syncJobConfig.getCron().isEmpty()) { + if (Strings.isNullOrEmpty(syncJobConfig.getUuid()) + || Strings.isNullOrEmpty(syncJobConfig.getCron())) { logger.debug( "UUID or cron expression not defined for repository {}, skipping this third party sync job.", syncJobConfig.getRepository()); continue; } uuidPool.add(syncJobConfig.getUuid()); - pushJobToDB(syncJobConfig); + + ScheduledJob scheduledJob = + scheduledJobRepository.findByIdAndJobType( + syncJobConfig.getUuid(), ScheduledJobType.THIRD_PARTY_SYNC); + + if (scheduledJob == null) { + scheduledJob = new ScheduledJob(); + } + + scheduledJob.setId(syncJobConfig.getUuid()); + scheduledJob.setCron(syncJobConfig.getCron()); + scheduledJob.setRepository(repositoryRepository.findByName(syncJobConfig.getRepository())); + scheduledJob.setJobStatus( + scheduledJobStatusRepository.findByEnum(ScheduledJobStatus.SCHEDULED)); + scheduledJob.setJobType( + scheduledJobTypeRepository.findByEnum(ScheduledJobType.THIRD_PARTY_SYNC)); + scheduledJob.setProperties(getScheduledThirdPartySyncProperties(syncJobConfig)); + + try { + scheduledJobRepository.save(scheduledJob); + } catch (DataIntegrityViolationException e) { + // Attempted to insert another scheduled job into the table with the same repo and job type, + // this can happen in a clustered quartz environment, don't need to display the error. + } } + } + /** + * Remove jobs defined under this custom scheduler that are not listed in the application + * properties but are present in the DB table. + */ + public void cleanQuartzJobs() throws SchedulerException { // Clean Quartz jobs that don't exist in the UUID pool logger.info("Performing Quartz scheduled jobs clean up"); for (JobKey jobKey : getScheduler().getJobKeys(GroupMatcher.anyGroup())) { @@ -100,22 +170,11 @@ public void init() throws ClassNotFoundException, SchedulerException { "Removed job with id: '{}' as it is no longer in the data source.", jobKey.getName()); } } - - scheduleAllJobs(); } - public void pushJobToDB(ThirdPartySyncJobConfig jobConfig) { - // v1 pull jobs from application.properties and push to the DB - ScheduledJob scheduledJob = new ScheduledJob(); - scheduledJob.setId(jobConfig.getUuid()); - scheduledJob.setCron(jobConfig.getCron()); - scheduledJob.setRepository(repositoryRepository.findByName(jobConfig.getRepository())); - scheduledJob.setJobStatus( - scheduledJobStatusRepository.findByEnum(ScheduledJobStatus.SCHEDULED)); - - scheduledJob.setJobType( - scheduledJobTypeRepository.findByEnum(ScheduledJobType.THIRD_PARTY_SYNC)); - + // v1 + private ScheduledThirdPartySyncProperties getScheduledThirdPartySyncProperties( + ThirdPartySyncJobConfig jobConfig) { ScheduledThirdPartySyncProperties thirdPartySyncProperties = new ScheduledThirdPartySyncProperties(); thirdPartySyncProperties.setThirdPartyProjectId(jobConfig.getThirdPartyProjectId()); @@ -127,45 +186,7 @@ public void pushJobToDB(ThirdPartySyncJobConfig jobConfig) { thirdPartySyncProperties.setIncludeTextUnitsWithPattern( jobConfig.getIncludeTextUnitsWithPattern()); thirdPartySyncProperties.setOptions(jobConfig.getOptions()); - - scheduledJob.setProperties(thirdPartySyncProperties); - - try { - scheduledJobRepository.save(scheduledJob); - } catch (DataIntegrityViolationException e) { - // Attempted to insert another scheduled job into the table with the same repo and job type, - // this can happen in a clustered quartz environment, don't need to display the error. - } - } - - public void scheduleAllJobs() throws ClassNotFoundException, SchedulerException { - List scheduledJobs = scheduledJobRepository.findAll(); - - for (ScheduledJob scheduledJob : scheduledJobs) { - JobKey jobKey = getJobKey(scheduledJob); - TriggerKey triggerKey = getTriggerKey(scheduledJob); - - // Retrieve job class from enum value e.g. THIRD_PARTY_SYNC -> ScheduledThirdPartySync - Class jobType = - loadJobClass(scheduledJob.getJobType().getEnum().getJobClassName()); - - JobDetail job = JobBuilder.newJob(jobType).withIdentity(jobKey).build(); - - Trigger trigger = buildTrigger(jobKey, scheduledJob.getCron(), triggerKey); - - if (getScheduler().checkExists(jobKey)) { - // The cron could have changed, reschedule the job - getScheduler().rescheduleJob(triggerKey, trigger); - } else { - getScheduler().scheduleJob(job, trigger); - } - - scheduledJob.setJobStatus( - scheduledJobStatusRepository.findByEnum(ScheduledJobStatus.SCHEDULED)); - scheduledJob.setStartDate(null); - scheduledJob.setEndDate(null); - scheduledJobRepository.save(scheduledJob); - } + return thirdPartySyncProperties; } public Trigger buildTrigger(JobKey jobKey, String cronExpression, TriggerKey triggerKey) { @@ -181,7 +202,7 @@ public Trigger buildTrigger(JobKey jobKey, String cronExpression, TriggerKey tri .build(); } - private JobKey getJobKey(ScheduledJob scheduledJob) { + public JobKey getJobKey(ScheduledJob scheduledJob) { // name = UUID // group = THIRD_PARTY_SYNC return new JobKey(scheduledJob.getId(), scheduledJob.getJobType().getEnum().toString()); @@ -198,55 +219,6 @@ public Scheduler getScheduler() { return schedulerManager.getScheduler(schedulerName); } - public ResponseEntity triggerJob(ScheduledJob scheduledJob) - throws SchedulerException { - JobKey jobKey = getJobKey(scheduledJob); - - // Is the job currently running ? - // Ignore the trigger request and tell the user it is currently running - for (JobExecutionContext jobExecutionContext : getScheduler().getCurrentlyExecutingJobs()) { - if (jobExecutionContext.getJobDetail().getKey().equals(jobKey)) { - return ResponseEntity.status(HttpStatus.CONFLICT) - .body( - new ScheduledJobResponse( - ScheduledJobResponse.Status.ERROR, - "Trigger ignored, job is currently running")); - } - } - - try { - if (!getScheduler().checkExists(jobKey)) - return ResponseEntity.status(HttpStatus.NOT_FOUND) - .body(new ScheduledJobResponse(ScheduledJobResponse.Status.ERROR, "Job doesn't exist")); - getScheduler().triggerJob(jobKey); - return ResponseEntity.status(HttpStatus.OK) - .body(new ScheduledJobResponse(ScheduledJobResponse.Status.SUCCESS, "Job triggered")); - } catch (SchedulerException e) { - logger.error( - "Error triggering job manually, job: {}", jobKey.getName() + ":" + jobKey.getGroup(), e); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) - .body(new ScheduledJobResponse(ScheduledJobResponse.Status.ERROR, e.getMessage())); - } - } - - public ResponseEntity toggleJob(ScheduledJob scheduledJob) - throws SchedulerException { - JobKey jobKey = getJobKey(scheduledJob); - - if (!getScheduler().checkExists(jobKey)) - return ResponseEntity.status(HttpStatus.NOT_FOUND) - .body(new ScheduledJobResponse(ScheduledJobResponse.Status.ERROR, "Job doesn't exist")); - - scheduledJob.setEnabled(!scheduledJob.getEnabled()); - scheduledJobRepository.save(scheduledJob); - - return ResponseEntity.status(HttpStatus.OK) - .body( - new ScheduledJobResponse( - ScheduledJobResponse.Status.SUCCESS, - "Job " + (scheduledJob.getEnabled() ? "enabled" : "disabled"))); - } - public Class loadJobClass(String className) throws ClassNotFoundException { Class clazz = Class.forName(className); diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/scheduledjob/ScheduledJobTriggerListener.java b/webapp/src/main/java/com/box/l10n/mojito/service/scheduledjob/ScheduledJobTriggerListener.java index 1fd36fcdd..84e1de106 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/scheduledjob/ScheduledJobTriggerListener.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/scheduledjob/ScheduledJobTriggerListener.java @@ -7,6 +7,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Listener that receives events about jobs that are about to be triggered under the 'scheduledJobs' + * scheduler, if the job to be triggered is disabled the job is vetoed out of execution. Pausing the + * jobs execution exists in Quartz but upon using it, the job loses track of its schedule when + * re-enabled, or it doesn't schedule at all, using the trigger listener solves this. + * + * @author mattwilshire + */ public class ScheduledJobTriggerListener extends TriggerListenerSupport { static Logger logger = LoggerFactory.getLogger(ScheduledJobTriggerListener.class); @@ -25,15 +33,14 @@ public String getName() { @Override public void triggerMisfired(Trigger trigger) { super.triggerMisfired(trigger); - // Scheduled job misfired, most likely need to allocate more threads ScheduledJob job = scheduledJobRepository.findByJobKey(trigger.getJobKey()); logger.warn("TRIGGER MISFIRE FOR {} | {}", job.getRepository().getName(), job.getJobType()); } @Override public boolean vetoJobExecution(Trigger trigger, JobExecutionContext context) { + // If the job is disabled, don't execute ScheduledJob job = scheduledJobRepository.findByJobKey(trigger.getJobKey()); - return !job.getEnabled(); } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/scheduledjob/jobs/ScheduledThirdPartySync.java b/webapp/src/main/java/com/box/l10n/mojito/service/scheduledjob/jobs/ScheduledThirdPartySync.java index e179b3c01..2b1bbc9da 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/scheduledjob/jobs/ScheduledThirdPartySync.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/scheduledjob/jobs/ScheduledThirdPartySync.java @@ -1,12 +1,17 @@ package com.box.l10n.mojito.service.scheduledjob.jobs; import com.box.l10n.mojito.entity.ScheduledJob; +import com.box.l10n.mojito.pagerduty.PagerDutyException; +import com.box.l10n.mojito.pagerduty.PagerDutyIntegrationService; +import com.box.l10n.mojito.pagerduty.PagerDutyPayload; import com.box.l10n.mojito.quartz.QuartzPollableTaskScheduler; import com.box.l10n.mojito.service.pollableTask.PollableFuture; import com.box.l10n.mojito.service.scheduledjob.IScheduledJob; import com.box.l10n.mojito.service.scheduledjob.ScheduledJobRepository; import com.box.l10n.mojito.service.thirdparty.ThirdPartySyncJob; import com.box.l10n.mojito.service.thirdparty.ThirdPartySyncJobInput; +import com.box.l10n.mojito.utils.ServerConfig; +import com.google.common.collect.ImmutableMap; import org.quartz.DisallowConcurrentExecution; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; @@ -14,7 +19,13 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.springframework.web.util.UriComponentsBuilder; +/** + * Scheduled job that performs the third party sync for a repository off a cron scheduled. + * + * @author mattwilshire + */ @Component @DisallowConcurrentExecution public class ScheduledThirdPartySync implements IScheduledJob { @@ -23,8 +34,9 @@ public class ScheduledThirdPartySync implements IScheduledJob { @Autowired private QuartzPollableTaskScheduler quartzPollableTaskScheduler; @Autowired private ScheduledJobRepository scheduledJobRepository; + @Autowired private PagerDutyIntegrationService pagerDutyIntegrationService; - public static int runAmount = 1; + @Autowired ServerConfig serverConfig; private ScheduledJob scheduledJob; private ScheduledThirdPartySyncProperties scheduledJobProperties; @@ -35,16 +47,12 @@ public void execute(JobExecutionContext jobExecutionContext) throws JobExecution // Fetch the scheduled job and cast the properties scheduledJob = scheduledJobRepository.findByJobKey(jobExecutionContext.getJobDetail().getKey()); scheduledJobProperties = (ScheduledThirdPartySyncProperties) scheduledJob.getProperties(); + pollableTaskId = 1080L; + // if (1 == 1) throw new JobExecutionException("ON PURPOSE"); logger.info( "Third party sync for repository {} has started.", scheduledJob.getRepository().getName()); - if (runAmount == 1) { - return; - } - - runAmount++; - // Create ThirdPartySyncInput from scheduled job and properties ThirdPartySyncJobInput thirdPartySyncJobInput = new ThirdPartySyncJobInput(scheduledJob, scheduledJobProperties); @@ -65,6 +73,18 @@ public void execute(JobExecutionContext jobExecutionContext) throws JobExecution public void onSuccess(JobExecutionContext context) { logger.info( "Third-Party Sync succeeded for repository {}.", scheduledJob.getRepository().getName()); + + // Resolve PD incident + pagerDutyIntegrationService + .getDefaultPagerDutyClient() + .ifPresent( + pd -> { + try { + pd.resolveIncident(scheduledJob.getId()); + } catch (PagerDutyException e) { + // TODO: + } + }); } @Override @@ -73,5 +93,32 @@ public void onFailure(JobExecutionContext context, JobExecutionException jobExce "Third-Party Sync for repository {} has failed. Pollable Task ID: {}", scheduledJob.getRepository().getName(), pollableTaskId); + + // Trigger PD incident + pagerDutyIntegrationService + .getDefaultPagerDutyClient() + .ifPresent( + pd -> { + String pollableTaskUrl = + UriComponentsBuilder.fromHttpUrl(serverConfig.getUrl()) + .path("api/pollableTasks/" + pollableTaskId.toString()) + .build() + .toUriString(); + + PagerDutyPayload payload = + new PagerDutyPayload( + "Mojito | Third Party Sync failed for '" + + scheduledJob.getRepository().getName() + + "'", + "Mojito", + PagerDutyPayload.Severity.CRITICAL, + ImmutableMap.of("Pollable Task URL", pollableTaskUrl)); + + try { + pd.triggerIncident(scheduledJob.getId(), payload); + } catch (PagerDutyException e) { + // TODO: + } + }); } }