From 66fb228eb9af94476f50bfc4f52789cb756beb8b Mon Sep 17 00:00:00 2001 From: Mark Allen <3417310+maallen@users.noreply.github.com> Date: Fri, 18 Aug 2023 16:31:05 +0100 Subject: [PATCH] Initial multi scheduler created --- .../box/l10n/mojito/quartz/QuartzConfig.java | 21 +++--- .../quartz/QuartzPollableTaskScheduler.java | 19 ++++-- .../box/l10n/mojito/quartz/QuartzQueue.java | 17 +++++ .../mojito/quartz/QuartzSchedulerConfig.java | 65 +++++++++++++++---- .../mojito/quartz/QuartzSchedulerManager.java | 34 ++++++++++ .../box/l10n/mojito/quartz/QuartzService.java | 6 +- 6 files changed, 136 insertions(+), 26 deletions(-) create mode 100644 webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzQueue.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzSchedulerManager.java diff --git a/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzConfig.java b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzConfig.java index f6c33ac8a8..4f99d5ac80 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzConfig.java +++ b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzConfig.java @@ -25,7 +25,7 @@ public class QuartzConfig { public static final String DYNAMIC_GROUP_NAME = "DYNAMIC"; - @Autowired Scheduler scheduler; + @Autowired List schedulers; @Autowired(required = false) List triggers = new ArrayList<>(); @@ -41,22 +41,25 @@ public class QuartzConfig { * @throws SchedulerException */ @PostConstruct - void startScheduler() throws SchedulerException { + void startSchedulers() throws SchedulerException { Properties quartzProps = quartzPropertiesConfig.getQuartzProperties(); removeOutdatedJobs(); if (Boolean.parseBoolean(quartzProps.getProperty("org.quartz.scheduler.enabled", "true"))) { - logger.info("Starting scheduler"); - scheduler.startDelayed(2); + logger.info("Starting schedulers"); + for (Scheduler scheduler : schedulers) { + scheduler.startDelayed(2); + } } } void removeOutdatedJobs() throws SchedulerException { - scheduler.unscheduleJobs(new ArrayList(getOutdatedTriggerKeys())); - scheduler.deleteJobs(new ArrayList(getOutdatedJobKeys())); + for (Scheduler scheduler : schedulers) { + scheduler.unscheduleJobs(new ArrayList(getOutdatedTriggerKeys(scheduler))); + scheduler.deleteJobs(new ArrayList(getOutdatedJobKeys(scheduler))); + } } - Set getOutdatedJobKeys() throws SchedulerException { - + Set getOutdatedJobKeys(Scheduler scheduler) throws SchedulerException { Set jobKeys = scheduler.getJobKeys(GroupMatcher.jobGroupEquals(Scheduler.DEFAULT_GROUP)); Set newJobKeys = new HashSet<>(); @@ -72,7 +75,7 @@ Set getOutdatedJobKeys() throws SchedulerException { return jobKeys; } - Set getOutdatedTriggerKeys() throws SchedulerException { + Set getOutdatedTriggerKeys(Scheduler scheduler) throws SchedulerException { Set triggerKeys = scheduler.getTriggerKeys(GroupMatcher.triggerGroupEquals(Scheduler.DEFAULT_GROUP)); diff --git a/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzPollableTaskScheduler.java b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzPollableTaskScheduler.java index 7f0d30b440..9fb035d8d8 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzPollableTaskScheduler.java +++ b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzPollableTaskScheduler.java @@ -1,6 +1,7 @@ package com.box.l10n.mojito.quartz; import static com.box.l10n.mojito.quartz.QuartzConfig.DYNAMIC_GROUP_NAME; +import static com.box.l10n.mojito.quartz.QuartzQueue.DEFAULT; import com.box.l10n.mojito.entity.PollableTask; import com.box.l10n.mojito.json.ObjectMapper; @@ -29,7 +30,7 @@ public class QuartzPollableTaskScheduler { /** logger */ static Logger logger = LoggerFactory.getLogger(QuartzPollableTaskScheduler.class); - @Autowired Scheduler scheduler; + @Autowired QuartzSchedulerManager schedulerManager; @Autowired PollableTaskService pollableTaskService; @@ -41,8 +42,7 @@ public PollableFuture scheduleJob( Class> clazz, I input) { QuartzJobInfo quartzJobInfo = QuartzJobInfo.newBuilder(clazz).withInput(input).withMessage(clazz.getSimpleName()).build(); - - return scheduleJob(quartzJobInfo); + return scheduleJob(quartzJobInfo, DEFAULT); } public PollableFuture scheduleJobWithCustomTimeout( @@ -54,7 +54,11 @@ public PollableFuture scheduleJobWithCustomTimeout( .withMessage(clazz.getSimpleName()) .build(); - return scheduleJob(quartzJobInfo); + return scheduleJob(quartzJobInfo, DEFAULT); + } + + public PollableFuture scheduleJob(QuartzJobInfo quartzJobInfo) { + return scheduleJob(quartzJobInfo, DEFAULT); } /** @@ -75,7 +79,12 @@ public PollableFuture scheduleJobWithCustomTimeout( * @param * @return */ - public PollableFuture scheduleJob(QuartzJobInfo quartzJobInfo) { + public PollableFuture scheduleJob( + QuartzJobInfo quartzJobInfo, QuartzQueue quartzQueue) { + + Scheduler scheduler = schedulerManager.getScheduler(quartzQueue); + + logger.info("Scheduling job on queue: {}", quartzQueue.getDescription()); String pollableTaskName = getPollableTaskName(quartzJobInfo.getClazz()); diff --git a/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzQueue.java b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzQueue.java new file mode 100644 index 0000000000..00c8d2ee01 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzQueue.java @@ -0,0 +1,17 @@ +package com.box.l10n.mojito.quartz; + +public enum QuartzQueue { + LOW_PRIORITY("Low priority"), + DEFAULT("Default"), + HIGH_PRIORITY("High priority"); + + private String description; + + QuartzQueue(String description) { + this.description = description; + } + + public String getDescription() { + return description; + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzSchedulerConfig.java b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzSchedulerConfig.java index 10154a372c..3aa569d29c 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzSchedulerConfig.java +++ b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzSchedulerConfig.java @@ -5,11 +5,11 @@ import java.util.List; import java.util.Properties; import javax.sql.DataSource; -import org.quartz.SchedulerException; import org.quartz.Trigger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -50,22 +50,51 @@ public class QuartzSchedulerConfig { * removed. * * @return - * @throws SchedulerException */ - @Bean - public SchedulerFactoryBean scheduler() throws SchedulerException { - logger.info("Create SchedulerFactoryBean"); + // TODO (maallen): Update config to read multiple schedulers config from app.properties so each + // scheduler can have it's own configured thread pool. + @Bean(name = "defaultScheduler") + public SchedulerFactoryBean defaultScheduler( + @Qualifier("defaultJobFactory") SpringBeanJobFactory jobFactory) { + + logger.info("Create default Scheduler"); Properties quartzProperties = quartzPropertiesConfig.getQuartzProperties(); + quartzProperties.put("org.quartz.scheduler.instanceName", "defaultScheduler"); - SchedulerFactoryBean schedulerFactory = new SchedulerFactoryBean(); + SchedulerFactoryBean factory = getSchedulerFactory(quartzProperties, jobFactory); + factory.setTriggers(triggers.toArray(new Trigger[] {})); + return factory; + } + + @Bean(name = "lowPriorityScheduler") + public SchedulerFactoryBean lowPriorityScheduler( + @Qualifier("lowPriorityJobFactory") SpringBeanJobFactory jobFactory) { + + logger.info("Create Low Priority Scheduler"); + + Properties quartzProperties = quartzPropertiesConfig.getQuartzProperties(); + quartzProperties.put("org.quartz.scheduler.instanceName", "lowPriorityScheduler"); + + return getSchedulerFactory(quartzProperties, jobFactory); + } + + @Bean(name = "highPriorityScheduler") + public SchedulerFactoryBean highPriorityScheduler( + @Qualifier("highPriorityJobFactory") SpringBeanJobFactory jobFactory) { + logger.info("Create High Priority Scheduler"); + Properties quartzProperties = quartzPropertiesConfig.getQuartzProperties(); + quartzProperties.put("org.quartz.scheduler.instanceName", "highPriorityScheduler"); + return getSchedulerFactory(quartzProperties, jobFactory); + } - String dataSource = quartzProperties.getProperty("org.quartz.jobStore.dataSource"); + private SchedulerFactoryBean getSchedulerFactory( + Properties quartzProperties, SpringBeanJobFactory springBeanJobFactory) { + SchedulerFactoryBean schedulerFactory = new SchedulerFactoryBean(); schedulerFactory.setQuartzProperties(quartzProperties); - schedulerFactory.setJobFactory(springBeanJobFactory()); + schedulerFactory.setJobFactory(springBeanJobFactory); schedulerFactory.setOverwriteExistingJobs(true); - schedulerFactory.setTriggers(triggers.toArray(new Trigger[] {})); schedulerFactory.setAutoStartup(false); if (quartzMetricsReportingJobListener != null) { @@ -75,8 +104,22 @@ public SchedulerFactoryBean scheduler() throws SchedulerException { return schedulerFactory; } - @Bean - public SpringBeanJobFactory springBeanJobFactory() { + @Bean(name = "lowPriorityJobFactory") + public SpringBeanJobFactory lowPrioritySpringBeanJobFactory() { + AutoWiringSpringBeanJobFactory jobFactory = new AutoWiringSpringBeanJobFactory(); + jobFactory.setApplicationContext(applicationContext); + return jobFactory; + } + + @Bean(name = "defaultJobFactory") + public SpringBeanJobFactory defaultSpringBeanJobFactory() { + AutoWiringSpringBeanJobFactory jobFactory = new AutoWiringSpringBeanJobFactory(); + jobFactory.setApplicationContext(applicationContext); + return jobFactory; + } + + @Bean(name = "highPriorityJobFactory") + public SpringBeanJobFactory highPrioritySpringBeanJobFactory() { AutoWiringSpringBeanJobFactory jobFactory = new AutoWiringSpringBeanJobFactory(); jobFactory.setApplicationContext(applicationContext); return jobFactory; diff --git a/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzSchedulerManager.java b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzSchedulerManager.java new file mode 100644 index 0000000000..0508d4dffa --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzSchedulerManager.java @@ -0,0 +1,34 @@ +package com.box.l10n.mojito.quartz; + +import org.quartz.Scheduler; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; + +@Component +public class QuartzSchedulerManager { + + @Autowired + @Qualifier("lowPriorityScheduler") + Scheduler lowPriorityScheduler; + + @Autowired + @Qualifier("defaultScheduler") + Scheduler defaultScheduler; + + @Autowired + @Qualifier("highPriorityScheduler") + Scheduler highPriorityScheduler; + + public Scheduler getScheduler(QuartzQueue quartzQueue) { + switch (quartzQueue) { + case LOW_PRIORITY: + return lowPriorityScheduler; + case HIGH_PRIORITY: + return highPriorityScheduler; + case DEFAULT: + default: + return defaultScheduler; + } + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzService.java b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzService.java index a9b73cf914..1e857527fd 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzService.java @@ -13,6 +13,7 @@ import org.quartz.impl.matchers.GroupMatcher; import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; @Service @@ -21,8 +22,11 @@ public class QuartzService { /** logger */ static Logger logger = getLogger(QuartzService.class); - @Autowired Scheduler scheduler; + @Autowired + @Qualifier("lowPriorityScheduler") + Scheduler scheduler; + // TODO(mallen): Add handling for an injected list of Schedulers instead. public List getDynamicJobs() throws SchedulerException { Set jobKeys = scheduler.getJobKeys(GroupMatcher.jobGroupEquals(DYNAMIC_GROUP_NAME)); return jobKeys.stream().map(jobKey -> jobKey.getName()).collect(Collectors.toList());