From b69705c8f879de4aadb45e9903f652858776f0ca Mon Sep 17 00:00:00 2001 From: Ludovic DEHON Date: Fri, 6 May 2022 12:04:44 +0200 Subject: [PATCH 01/14] feat(core): add a `lateMaximumDelay` on Schedule to skip late executions --- .../core/models/triggers/types/Schedule.java | 58 +++++++++++++++++-- .../io/kestra/core/validations/Schedule.java | 11 ++++ .../core/validations/ValidationFactory.java | 19 ++++++ .../models/triggers/types/ScheduleTest.java | 27 +++++++++ .../kestra/core/validations/ScheduleTest.java | 36 ++++++++++++ 5 files changed, 145 insertions(+), 6 deletions(-) create mode 100644 core/src/main/java/io/kestra/core/validations/Schedule.java create mode 100644 core/src/test/java/io/kestra/core/validations/ScheduleTest.java diff --git a/core/src/main/java/io/kestra/core/models/triggers/types/Schedule.java b/core/src/main/java/io/kestra/core/models/triggers/types/Schedule.java index 38e630a9ef..3acc354c29 100644 --- a/core/src/main/java/io/kestra/core/models/triggers/types/Schedule.java +++ b/core/src/main/java/io/kestra/core/models/triggers/types/Schedule.java @@ -41,6 +41,7 @@ @EqualsAndHashCode @Getter @NoArgsConstructor +@io.kestra.core.validations.Schedule @Schema( title = "Schedule a flow based on cron date", description = "Kestra is able to trigger flow based on Schedule (aka the time). If you need to wait another system " + @@ -116,13 +117,21 @@ public class Schedule extends AbstractTrigger implements PollingTriggerInterface @PluginProperty(dynamic = true) private Map inputs; + @Schema( + title = "The maximum late delay accepted", + description = "If the schedule didn't start after this delay, the execution will be skip." + ) + private Duration lateMaximumDelay; + + @Getter(AccessLevel.NONE) + private transient ExecutionTime executionTime; + @Override public ZonedDateTime nextEvaluationDate(ConditionContext conditionContext, Optional last) { ExecutionTime executionTime = this.executionTime(); // previous present & scheduleConditions if (last.isPresent() && this.scheduleConditions != null) { - Optional next = this.truePreviousNextDateWithCondition( executionTime, conditionContext, @@ -152,7 +161,12 @@ public ZonedDateTime nextEvaluationDate(ConditionContext conditionContext, Optio public Optional evaluate(ConditionContext conditionContext, TriggerContext context) throws Exception { RunContext runContext = conditionContext.getRunContext(); ExecutionTime executionTime = this.executionTime(); - Output output = this.output(executionTime, context.getDate()).orElse(null); + ZonedDateTime previousDate = context.getDate(); + + Output output = this.output(executionTime, previousDate).orElse(null); + + // if max delay reach, we calculate a new date + output = this.handleMaxDelay(output); if (output == null || output.getDate() == null) { return Optional.empty(); @@ -161,7 +175,7 @@ public Optional evaluate(ConditionContext conditionContext, TriggerCo ZonedDateTime next = output.getDate(); // we try at the exact time / standard behaviour - boolean isReady = next.compareTo(context.getDate()) == 0; + boolean isReady = next.compareTo(previousDate) == 0; // in case on cron expression changed, the next date will never match, so we allow past operation to start boolean isLate = next.compareTo(ZonedDateTime.now().minus(Duration.ofMinutes(1))) < 0; @@ -175,6 +189,8 @@ public Optional evaluate(ConditionContext conditionContext, TriggerCo return Optional.empty(); } + + // inject outputs variables for scheduleCondition conditionContext = conditionContext(conditionContext, output); @@ -244,10 +260,14 @@ private ConditionContext conditionContext(ConditionContext conditionContext, Out )); } - private ExecutionTime executionTime() { - Cron parse = CRON_PARSER.parse(this.cron); + private synchronized ExecutionTime executionTime() { + if (this.executionTime == null) { + Cron parse = CRON_PARSER.parse(this.cron); + + this.executionTime = ExecutionTime.forCron(parse); + } - return ExecutionTime.forCron(parse); + return this.executionTime; } private Optional computeNextEvaluationDate(ExecutionTime executionTime, ZonedDateTime date) { @@ -299,6 +319,32 @@ private Optional truePreviousNextDateWithCondition(ExecutionTime return Optional.empty(); } + private Output handleMaxDelay(Output output) { + if (output == null) { + return null; + } + + if (this.lateMaximumDelay == null) { + return output; + } + + while ( + (output.getDate().getYear() < ZonedDateTime.now().getYear() + 10) || + (output.getDate().getYear() > ZonedDateTime.now().getYear() - 10) + ) { + if (output.getDate().plus(this.lateMaximumDelay).compareTo(ZonedDateTime.now()) < 0) { + output = this.output(executionTime, output.getNext()).orElse(null); + if (output == null) { + return null; + } + } else { + return output; + } + } + + return output; + } + private boolean validateScheduleCondition(ConditionContext conditionContext) { if (scheduleConditions != null) { ConditionService conditionService = conditionContext.getRunContext().getApplicationContext().getBean(ConditionService.class); diff --git a/core/src/main/java/io/kestra/core/validations/Schedule.java b/core/src/main/java/io/kestra/core/validations/Schedule.java new file mode 100644 index 0000000000..4ec54c3be6 --- /dev/null +++ b/core/src/main/java/io/kestra/core/validations/Schedule.java @@ -0,0 +1,11 @@ +package io.kestra.core.validations; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import javax.validation.Constraint; + +@Retention(RetentionPolicy.RUNTIME) +@Constraint(validatedBy = { }) +public @interface Schedule { + String message() default "invalid schedule ({validatedValue})"; +} \ No newline at end of file diff --git a/core/src/main/java/io/kestra/core/validations/ValidationFactory.java b/core/src/main/java/io/kestra/core/validations/ValidationFactory.java index 8e26af2b08..476d28461e 100644 --- a/core/src/main/java/io/kestra/core/validations/ValidationFactory.java +++ b/core/src/main/java/io/kestra/core/validations/ValidationFactory.java @@ -58,6 +58,25 @@ ConstraintValidator cronExpressionValidator() { }; } + + @Singleton + ConstraintValidator scheduleValidator() { + return (value, annotationMetadata, context) -> { + if (value == null) { + return true; + } + + if (value.getBackfill() != null && value.getBackfill().getStart() != null && value.getLateMaximumDelay() != null) { + context.messageTemplate("invalid schedule: backfill and lateMaximumDelay are incompatible options"); + + return false; + } + + return true; + }; + } + + @Singleton ConstraintValidator jsonStringValidator() { return (value, annotationMetadata, context) -> { diff --git a/core/src/test/java/io/kestra/core/models/triggers/types/ScheduleTest.java b/core/src/test/java/io/kestra/core/models/triggers/types/ScheduleTest.java index f312bf7b6c..ac8801dcbe 100644 --- a/core/src/test/java/io/kestra/core/models/triggers/types/ScheduleTest.java +++ b/core/src/test/java/io/kestra/core/models/triggers/types/ScheduleTest.java @@ -301,6 +301,33 @@ void conditionsWithBackfill() throws Exception { } } + + @SuppressWarnings("unchecked") + @Test + void lateMaximumDelay() throws Exception { + Schedule trigger = Schedule.builder() + .cron("* * * * *") + .lateMaximumDelay(Duration.ofMinutes(5)) + .build(); + + ZonedDateTime date = ZonedDateTime.now().minusMinutes(15); + ZonedDateTime expected = ZonedDateTime.now().minusMinutes(4) + .withSecond(0) + .truncatedTo(ChronoUnit.SECONDS); + + Optional evaluate = trigger.evaluate( + conditionContext(), + TriggerContext.builder() + .date(date) + .build() + ); + + assertThat(evaluate.isPresent(), is(true)); + var vars = (Map) evaluate.get().getVariables().get("schedule"); + assertThat(dateFromVars(vars.get("date"), date), is(expected)); + + } + private ConditionContext conditionContext() { return ConditionContext.builder() .runContext(runContextFactory.of()) diff --git a/core/src/test/java/io/kestra/core/validations/ScheduleTest.java b/core/src/test/java/io/kestra/core/validations/ScheduleTest.java new file mode 100644 index 0000000000..9759d3d281 --- /dev/null +++ b/core/src/test/java/io/kestra/core/validations/ScheduleTest.java @@ -0,0 +1,36 @@ +package io.kestra.core.validations; + +import io.kestra.core.models.triggers.types.Schedule; +import io.kestra.core.models.validations.ModelValidator; +import io.kestra.core.utils.IdUtils; +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import jakarta.inject.Inject; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.time.ZonedDateTime; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; + +@MicronautTest +class ScheduleTest { + @Inject + private ModelValidator modelValidator; + + @Test + void cronValidation() { + Schedule build = Schedule.builder() + .id(IdUtils.create()) + .type(Schedule.class.getName()) + .cron("* * * * *") + .backfill(Schedule.ScheduleBackfill.builder().start(ZonedDateTime.now()).build()) + .lateMaximumDelay(Duration.ofSeconds(10)) + .build(); + + + assertThat(modelValidator.isValid(build).isPresent(), is(true)); + assertThat(modelValidator.isValid(build).get().getMessage(), containsString("backfill and lateMaximumDelay are incompatible options")); + } +} From 870ae257e230bd2f46db5c84e9b5bb75d9ade5b8 Mon Sep 17 00:00:00 2001 From: Ludovic DEHON Date: Mon, 9 May 2022 16:27:04 +0200 Subject: [PATCH 02/14] fix(docker): missing venv package for python on full image --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9dd93101d1..2314c0808e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -131,7 +131,7 @@ jobs: packages: "" - name: "-full" plugins: io.kestra.storage:storage-azure:LATEST io.kestra.storage:storage-gcs:LATEST io.kestra.storage:storage-minio:LATEST io.kestra.plugin:plugin-aws:LATEST io.kestra.plugin:plugin-azure:LATEST io.kestra.plugin:plugin-cassandra:LATEST io.kestra.plugin:plugin-compress:LATEST io.kestra.plugin:plugin-crypto:LATEST io.kestra.plugin:plugin-dbt:LATEST io.kestra.plugin:plugin-debezium-mysql:LATEST io.kestra.plugin:plugin-debezium-postgres:LATEST io.kestra.plugin:plugin-debezium-sqlserver:LATEST io.kestra.plugin:plugin-elasticsearch:LATEST io.kestra.plugin:plugin-fs:LATEST io.kestra.plugin:plugin-gcp:LATEST io.kestra.plugin:plugin-googleworkspace:LATEST io.kestra.plugin:plugin-jdbc-clickhouse:LATEST io.kestra.plugin:plugin-jdbc-mysql:LATEST io.kestra.plugin:plugin-jdbc-oracle:LATEST io.kestra.plugin:plugin-jdbc-postgres:LATEST io.kestra.plugin:plugin-jdbc-redshift:LATEST io.kestra.plugin:plugin-jdbc-snowflake:LATEST io.kestra.plugin:plugin-jdbc-sqlserver:LATEST io.kestra.plugin:plugin-jdbc-vertica:LATEST io.kestra.plugin:plugin-jdbc-vectorwise:LATEST io.kestra.plugin:plugin-kafka:LATEST io.kestra.plugin:plugin-kubernetes:LATEST io.kestra.plugin:plugin-mongodb:LATEST io.kestra.plugin:plugin-mqtt:LATEST io.kestra.plugin:plugin-notifications:LATEST io.kestra.plugin:plugin-script-groovy:LATEST io.kestra.plugin:plugin-script-jython:LATEST io.kestra.plugin:plugin-script-nashorn:LATEST io.kestra.plugin:plugin-serdes:LATEST io.kestra.plugin:plugin-singer:LATEST io.kestra.plugin:plugin-spark:LATEST - packages: python3-pip python3-wheel python3-setuptools python3-virtualenv python-is-python3 nodejs curl wait-for-it zip unzip + packages: python3 python3-venv python-is-python3 nodejs curl wait-for-it zip unzip steps: - uses: actions/checkout@v2 From 90e91d5337b16f8da3dca67b7268db386287f9c5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 May 2022 05:22:51 +0000 Subject: [PATCH 03/14] chore(deps): bump org.gradle.test-retry from 1.3.1 to 1.4.0 Bumps org.gradle.test-retry from 1.3.1 to 1.4.0. --- updated-dependencies: - dependency-name: org.gradle.test-retry dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 26fbcf0b1e..b4e1f6ddf5 100644 --- a/build.gradle +++ b/build.gradle @@ -8,7 +8,7 @@ plugins { // test id 'com.adarshr.test-logger' version '3.2.0' - id 'org.gradle.test-retry' version '1.3.1' + id 'org.gradle.test-retry' version '1.4.0' // helper id "com.github.ben-manes.versions" version "0.42.0" From 2940885acf06260f44b1f4553bc430b0163b058a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 May 2022 05:21:30 +0000 Subject: [PATCH 04/14] chore(deps): bump docker/login-action from 1 to 2 Bumps [docker/login-action](https://github.com/docker/login-action) from 1 to 2. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v1...v2) --- updated-dependencies: - dependency-name: docker/login-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2314c0808e..5fefa3cb45 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -176,7 +176,7 @@ jobs: # Docker Login - name: Login to DockerHub if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/tags/v') - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} From a97ef6bf41132b4d3c7525a88376ee6cc50a39c7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 May 2022 05:21:28 +0000 Subject: [PATCH 05/14] chore(deps): bump docker/setup-qemu-action from 1 to 2 Bumps [docker/setup-qemu-action](https://github.com/docker/setup-qemu-action) from 1 to 2. - [Release notes](https://github.com/docker/setup-qemu-action/releases) - [Commits](https://github.com/docker/setup-qemu-action/compare/v1...v2) --- updated-dependencies: - dependency-name: docker/setup-qemu-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5fefa3cb45..923bc53b6a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -167,7 +167,7 @@ jobs: # Docker setup - name: Set up QEMU if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/tags/v') - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/tags/v') From b4453cbf1385330091a33aada6c1e669465a1760 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 May 2022 05:21:25 +0000 Subject: [PATCH 06/14] chore(deps): bump docker/build-push-action from 2 to 3 Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 2 to 3. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 923bc53b6a..a937c474af 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -184,7 +184,7 @@ jobs: # Docker Build and push - name: Push to Docker Hub if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/tags/v') - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: context: . push: true From 1f815ef359edf3b6b5427c45f92be6eb0b0e0c58 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 May 2022 05:21:22 +0000 Subject: [PATCH 07/14] chore(deps): bump docker/setup-buildx-action from 1 to 2 Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 1 to 2. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v1...v2) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a937c474af..50aec9e92d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -171,7 +171,7 @@ jobs: - name: Set up Docker Buildx if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/tags/v') - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 # Docker Login - name: Login to DockerHub From 1c768f9aaaa9aa466d0d736646a7e418fc229e86 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Apr 2022 05:39:09 +0000 Subject: [PATCH 08/14] chore(deps): bump github/codeql-action from 1 to 2 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 1 to 2. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v1...v2) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 3ab7a82809..a2f403eb0b 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -40,7 +40,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -58,7 +58,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v2 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -72,4 +72,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v2 From bd49a53e072381f777c85a02f05f2116a5ddf777 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 22 Apr 2022 05:22:03 +0000 Subject: [PATCH 09/14] chore(deps): bump mockito-junit-jupiter from 4.4.0 to 4.5.1 Bumps [mockito-junit-jupiter](https://github.com/mockito/mockito) from 4.4.0 to 4.5.1. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v4.4.0...v4.5.1) --- updated-dependencies: - dependency-name: org.mockito:mockito-junit-jupiter dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- core/build.gradle | 2 +- indexer-kafka-elasticsearch/build.gradle | 2 +- repository-elasticsearch/build.gradle | 2 +- runner-kafka/build.gradle | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/build.gradle b/core/build.gradle index 20f52ad8af..82fb34f5c5 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -43,7 +43,7 @@ dependencies { testImplementation project(':runner-memory') testImplementation project(':storage-local') - testImplementation 'org.mockito:mockito-junit-jupiter:4.4.0' + testImplementation 'org.mockito:mockito-junit-jupiter:4.5.1' testImplementation "io.micronaut:micronaut-http-client" testImplementation "io.micronaut.rxjava2:micronaut-rxjava2-http-client" testImplementation "io.micronaut:micronaut-http-server-netty" diff --git a/indexer-kafka-elasticsearch/build.gradle b/indexer-kafka-elasticsearch/build.gradle index e4968523af..cf2777107d 100644 --- a/indexer-kafka-elasticsearch/build.gradle +++ b/indexer-kafka-elasticsearch/build.gradle @@ -11,5 +11,5 @@ dependencies { implementation group: "org.apache.kafka", name: "kafka-clients", version: kafkaVersion implementation group: 'net.jodah', name: 'failsafe', version: '2.4.4' - testImplementation 'org.mockito:mockito-junit-jupiter:4.4.0' + testImplementation 'org.mockito:mockito-junit-jupiter:4.5.1' } diff --git a/repository-elasticsearch/build.gradle b/repository-elasticsearch/build.gradle index 6901e3d06f..2628786c9f 100644 --- a/repository-elasticsearch/build.gradle +++ b/repository-elasticsearch/build.gradle @@ -9,5 +9,5 @@ dependencies { testImplementation project(':core').sourceSets.test.output testImplementation project(':runner-memory') - testImplementation 'org.mockito:mockito-junit-jupiter:4.4.0' + testImplementation 'org.mockito:mockito-junit-jupiter:4.5.1' } diff --git a/runner-kafka/build.gradle b/runner-kafka/build.gradle index e45158e131..e7e882187e 100644 --- a/runner-kafka/build.gradle +++ b/runner-kafka/build.gradle @@ -16,5 +16,5 @@ dependencies { testImplementation group: 'org.apache.kafka', name: 'kafka-streams-test-utils', version: kafkaVersion - testImplementation 'org.mockito:mockito-junit-jupiter:4.4.0' + testImplementation 'org.mockito:mockito-junit-jupiter:4.5.1' } From ce9d32d11ab5ecbdec7fdb53ceda17fc2efb7420 Mon Sep 17 00:00:00 2001 From: Ludovic DEHON Date: Fri, 13 May 2022 15:06:39 +0200 Subject: [PATCH 10/14] fix(core): end event in sse controller is never send --- .../io/kestra/webserver/controllers/ExecutionController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webserver/src/main/java/io/kestra/webserver/controllers/ExecutionController.java b/webserver/src/main/java/io/kestra/webserver/controllers/ExecutionController.java index 75f6bd991e..a17be46aba 100644 --- a/webserver/src/main/java/io/kestra/webserver/controllers/ExecutionController.java +++ b/webserver/src/main/java/io/kestra/webserver/controllers/ExecutionController.java @@ -505,7 +505,7 @@ public Flowable> follow( emitter.onNext(Event.of(current).id("progress")); - if (this.isStopFollow(flow, execution)) { + if (this.isStopFollow(flow, current)) { emitter.onNext(Event.of(current).id("end")); emitter.onComplete(); } From c911f7dcf730077782cbe95ffbc383d37b98c345 Mon Sep 17 00:00:00 2001 From: Ludovic DEHON Date: Fri, 13 May 2022 22:59:31 +0200 Subject: [PATCH 11/14] fix(ui): search with no result on flow raise an error --- ui/src/components/flows/Flows.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/components/flows/Flows.vue b/ui/src/components/flows/Flows.vue index 73ce5b2bbc..cbcaaa3277 100644 --- a/ui/src/components/flows/Flows.vue +++ b/ui/src/components/flows/Flows.vue @@ -290,7 +290,7 @@ }) }, rowClasses(flow) { - return flow.disabled ? ["disabled"] : []; + return flow && flow.disabled ? ["disabled"] : []; } } }; From 8e60a0db6cf426c00df9b4c9c37903d5af8ddd28 Mon Sep 17 00:00:00 2001 From: Ludovic DEHON Date: Fri, 13 May 2022 23:09:07 +0200 Subject: [PATCH 12/14] chore(docs): move community to slack --- .github/ISSUE_TEMPLATE/bug.yml | 2 +- .github/ISSUE_TEMPLATE/config.yml | 4 ++-- README.md | 6 +++--- ui/src/override/components/Menu.vue | 10 +++++----- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 8978294a72..e8c0056f3c 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -6,7 +6,7 @@ body: value: | Thanks for reporting an issue, please review the task list below before submitting the issue. Your issue report will be closed if the issue is incomplete and the below tasks not completed. - NOTE: If you are unsure about something and the issue is more of a question a better place to ask questions is on [Github Discussions](https://github.com/kestra-io/kestra/discussions) or [Discord](https://discord.gg/NMG39WKGth). + NOTE: If you are unsure about something and the issue is more of a question a better place to ask questions is on [Github Discussions](https://github.com/kestra-io/kestra/discussions) or [Slack](https://join.slack.com/t/kestra-io/shared_invite/zt-193shv281-rK9QOEfZC2_vEbDO7Uxtbw). - type: textarea attributes: label: Expected Behavior diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index eb7a47ca44..dbf76ca375 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -3,5 +3,5 @@ contact_links: url: https://github.com/kestra-io/kestra/discussions about: Ask questions about Kestra on Github - name: Chat - url: https://discord.gg/NMG39WKGth - about: Chat with us on Discord. \ No newline at end of file + url: https://join.slack.com/t/kestra-io/shared_invite/zt-193shv281-rK9QOEfZC2_vEbDO7Uxtbw + about: Chat with us on Slack. \ No newline at end of file diff --git a/README.md b/README.md index e12a541f40..a079397455 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Docker pull Artifact Hub Kestra infinitely scalable orchestration and scheduling platform - Discord + Slack Github discussions Twitter Code Cov @@ -29,7 +29,7 @@ Website • Twitter • Linked In • - Discord • + Slack • Documentation

@@ -245,7 +245,7 @@ Join our community if you need help, want to chat or have any other questions fo - [GitHub](https://github.com/kestra-io/kestra/discussions) - Discussion forums and updates from the Kestra team - [Twitter](https://twitter.com/kestra_io) - For all the latest Kestra news -- [Discord](https://discord.gg/NMG39WKGth) - Join the conversation! Get all the latest updates and chat to the devs +- [Slack](https://join.slack.com/t/kestra-io/shared_invite/zt-193shv281-rK9QOEfZC2_vEbDO7Uxtbw) - Join the conversation! Get all the latest updates and chat to the devs ## Roadmap diff --git a/ui/src/override/components/Menu.vue b/ui/src/override/components/Menu.vue index e3ce4e6c1e..585562f235 100644 --- a/ui/src/override/components/Menu.vue +++ b/ui/src/override/components/Menu.vue @@ -36,7 +36,7 @@ import BookMultipleOutline from "vue-material-design-icons/BookMultipleOutline"; import FileCodeOutline from "vue-material-design-icons/FileCodeOutline"; import GoogleCirclesExtended from "vue-material-design-icons/GoogleCirclesExtended"; - import Discord from "vue-material-design-icons/Discord"; + import Slack from "vue-material-design-icons/Slack"; import Github from "vue-material-design-icons/Github"; import CogOutline from "vue-material-design-icons/CogOutline"; import {mapState} from "vuex"; @@ -49,7 +49,7 @@ Vue.component("DocumentationMenuIcon", BookMultipleOutline); Vue.component("DocumentationDeveloperMenuIcon", FileCodeOutline); Vue.component("DocumentationPluginsMenuIcon", GoogleCirclesExtended); - Vue.component("Discord", Discord); + Vue.component("Slack", Slack); Vue.component("Github", Github); Vue.component("SettingMenuIcon", CogOutline); @@ -158,10 +158,10 @@ }, }, { - href: "https://discord.gg/NMG39WKGth", - title: "Discord", + href: "https://join.slack.com/t/kestra-io/shared_invite/zt-193shv281-rK9QOEfZC2_vEbDO7Uxtbw", + title: "Slack", icon: { - element: "Discord", + element: "Slack", class: "menu-icon" }, external: true From 18ccc98581c73303334c8c6b46f752ffaf9ded87 Mon Sep 17 00:00:00 2001 From: Ludovic DEHON Date: Thu, 26 May 2022 22:39:18 +0200 Subject: [PATCH 13/14] feat(task): allow to use system-site-packages for python venv creation close #615 --- .../main/java/io/kestra/core/tasks/scripts/AbstractPython.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/io/kestra/core/tasks/scripts/AbstractPython.java b/core/src/main/java/io/kestra/core/tasks/scripts/AbstractPython.java index ad3ec343dc..0773e26d43 100644 --- a/core/src/main/java/io/kestra/core/tasks/scripts/AbstractPython.java +++ b/core/src/main/java/io/kestra/core/tasks/scripts/AbstractPython.java @@ -111,7 +111,7 @@ protected String virtualEnvCommand(RunContext runContext, List requireme } renderer.addAll(Arrays.asList( - this.pythonPath + " -m venv " + workingDirectory + " > /dev/null", + this.pythonPath + " -m venv --system-site-packages " + workingDirectory + " > /dev/null", "./bin/pip install pip --upgrade > /dev/null", requirementsAsString )); From 40efdcb4e084ca7787025b562a151ec4a3d09039 Mon Sep 17 00:00:00 2001 From: Ludovic DEHON Date: Thu, 2 Jun 2022 16:29:17 +0200 Subject: [PATCH 14/14] feat(core): support LocalDate & LocalDateTime on pebble date --- .../core/runners/pebble/AbstractDate.java | 25 ++++++++++++++++--- .../pebble/filters/DateFilterTest.java | 19 ++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/io/kestra/core/runners/pebble/AbstractDate.java b/core/src/main/java/io/kestra/core/runners/pebble/AbstractDate.java index 9ea248c3b7..6a2cd2cff8 100644 --- a/core/src/main/java/io/kestra/core/runners/pebble/AbstractDate.java +++ b/core/src/main/java/io/kestra/core/runners/pebble/AbstractDate.java @@ -6,6 +6,7 @@ import java.time.*; import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import java.time.format.FormatStyle; import java.util.*; @@ -102,10 +103,26 @@ protected static ZonedDateTime convert(Object value, ZoneId zoneId, String exist return Instant.ofEpochSecond((Long) value).atZone(zoneId); } - if (existingFormat != null) { - return ZonedDateTime.parse((String) value, formatter(existingFormat)); + try { + if (existingFormat != null) { + return ZonedDateTime.parse((String) value, formatter(existingFormat)); + } else { + return ZonedDateTime.parse((String) value); + } + } catch (DateTimeParseException e) { + try { + if (existingFormat != null) { + return LocalDateTime.parse((String) value, formatter(existingFormat)).atZone(zoneId); + } else { + return LocalDateTime.parse((String) value).atZone(zoneId); + } + } catch (DateTimeParseException e2) { + if (existingFormat != null) { + return LocalDate.parse((String) value, formatter(existingFormat)).atStartOfDay().atZone(zoneId); + } else { + return LocalDate.parse((String) value).atStartOfDay().atZone(zoneId); + } + } } - - return ZonedDateTime.parse((String) value); } } diff --git a/core/src/test/java/io/kestra/core/runners/pebble/filters/DateFilterTest.java b/core/src/test/java/io/kestra/core/runners/pebble/filters/DateFilterTest.java index 498e655fb4..aaae90d1f7 100644 --- a/core/src/test/java/io/kestra/core/runners/pebble/filters/DateFilterTest.java +++ b/core/src/test/java/io/kestra/core/runners/pebble/filters/DateFilterTest.java @@ -10,6 +10,8 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.Date; +import java.util.Map; + import jakarta.inject.Inject; import static org.hamcrest.MatcherAssert.assertThat; @@ -49,6 +51,23 @@ void dateFormat() throws IllegalVariableEvaluationException { )); } + @Test + void dateStringFormat() throws IllegalVariableEvaluationException { + String render = variableRenderer.render( + "{{ \"July 24, 2001\" | date(\"yyyy-MM-dd\", existingFormat=\"MMMM dd, yyyy\") }}\n" + + "{{ \"2013-09-08T17:19:12+02:00\" | date(timeZone=\"Europe/Paris\") }}\n" + + "{{ \"2013-09-08T17:19:12\" | date(timeZone=\"Europe/Paris\") }}\n" + + "{{ \"2013-09-08\" | date(timeZone=\"Europe/Paris\") }}\n", + Map.of() + ); + + assertThat(render, is("2001-07-24\n" + + "2013-09-08T17:19:12.000000+02:00\n" + + "2013-09-08T17:19:12.000000+02:00\n" + + "2013-09-08T00:00:00.000000+02:00\n" + )); + } + @Test void timestamp() throws IllegalVariableEvaluationException { String render = variableRenderer.render(