diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 0ce531ea6c..179521c435 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -37,7 +37,7 @@ jobs:
architecture: 'x64'
- uses: actions/setup-node@v3
with:
- node-version: '12'
+ node-version: '14'
check-latest: true
# Services
@@ -120,6 +120,7 @@ jobs:
- name: Create GitHub release
uses: "marvinpinto/action-automatic-releases@latest"
if: startsWith(github.ref, 'refs/tags/v')
+ continue-on-error: true
with:
repo_token: "${{ secrets.GITHUB_TOKEN }}"
prerelease: false
@@ -138,10 +139,10 @@ jobs:
plugins: ""
packages: ""
- name: "-full"
- plugins: io.kestra.storage:storage-azure:LATEST io.kestra.storage:storage-gcs:LATEST io.kestra.storage:storage-minio:LATEST io.kestra.plugin:plugin-aws:LATEST io.kestra.plugin:plugin-azure:LATEST io.kestra.plugin:plugin-powerbi:LATEST io.kestra.plugin:plugin-pulsar:LATEST io.kestra.plugin:plugin-cassandra:LATEST io.kestra.plugin:plugin-compress:LATEST io.kestra.plugin:plugin-crypto:LATEST io.kestra.plugin:plugin-dbt:LATEST io.kestra.plugin:plugin-debezium-mysql:LATEST io.kestra.plugin:plugin-debezium-postgres:LATEST io.kestra.plugin:plugin-debezium-sqlserver:LATEST io.kestra.plugin:plugin-elasticsearch:LATEST io.kestra.plugin:plugin-fs:LATEST io.kestra.plugin:plugin-gcp:LATEST io.kestra.plugin:plugin-googleworkspace:LATEST io.kestra.plugin:plugin-jdbc-clickhouse:LATEST io.kestra.plugin:plugin-jdbc-duckdb:LATEST io.kestra.plugin:plugin-jdbc-mysql:LATEST io.kestra.plugin:plugin-jdbc-oracle:LATEST io.kestra.plugin:plugin-jdbc-pinot:LATEST io.kestra.plugin:plugin-jdbc-postgres:LATEST io.kestra.plugin:plugin-jdbc-redshift:LATEST io.kestra.plugin:plugin-jdbc-rockset:LATEST io.kestra.plugin:plugin-jdbc-snowflake:LATEST io.kestra.plugin:plugin-jdbc-sqlserver:LATEST io.kestra.plugin:plugin-jdbc-trino:LATEST io.kestra.plugin:plugin-jdbc-vertica:LATEST io.kestra.plugin:plugin-jdbc-vectorwise:LATEST io.kestra.plugin:plugin-kafka:LATEST io.kestra.plugin:plugin-kubernetes:LATEST io.kestra.plugin:plugin-mongodb:LATEST io.kestra.plugin:plugin-mqtt:LATEST io.kestra.plugin:plugin-notifications:LATEST io.kestra.plugin:plugin-script-groovy:LATEST io.kestra.plugin:plugin-script-jython:LATEST io.kestra.plugin:plugin-script-nashorn:LATEST io.kestra.plugin:plugin-serdes:LATEST io.kestra.plugin:plugin-servicenow:LATEST io.kestra.plugin:plugin-singer:LATEST io.kestra.plugin:plugin-soda:LATEST io.kestra.plugin:plugin-spark:LATEST io.kestra.plugin:plugin-tika:LATEST
+ plugins: io.kestra.storage:storage-azure:LATEST io.kestra.storage:storage-gcs:LATEST io.kestra.storage:storage-minio:LATEST io.kestra.plugin:plugin-airbyte:LATEST io.kestra.plugin:plugin-aws:LATEST io.kestra.plugin:plugin-azure:LATEST io.kestra.plugin:plugin-powerbi:LATEST io.kestra.plugin:plugin-pulsar:LATEST io.kestra.plugin:plugin-cassandra:LATEST io.kestra.plugin:plugin-compress:LATEST io.kestra.plugin:plugin-crypto:LATEST io.kestra.plugin:plugin-dbt:LATEST io.kestra.plugin:plugin-debezium-mysql:LATEST io.kestra.plugin:plugin-debezium-postgres:LATEST io.kestra.plugin:plugin-debezium-sqlserver:LATEST io.kestra.plugin:plugin-elasticsearch:LATEST io.kestra.plugin:plugin-fivetran:LATEST io.kestra.plugin:plugin-fs:LATEST io.kestra.plugin:plugin-gcp:LATEST io.kestra.plugin:plugin-googleworkspace:LATEST io.kestra.plugin:plugin-jdbc-clickhouse:LATEST io.kestra.plugin:plugin-jdbc-duckdb:LATEST io.kestra.plugin:plugin-jdbc-mysql:LATEST io.kestra.plugin:plugin-jdbc-oracle:LATEST io.kestra.plugin:plugin-jdbc-pinot:LATEST io.kestra.plugin:plugin-jdbc-postgres:LATEST io.kestra.plugin:plugin-jdbc-redshift:LATEST io.kestra.plugin:plugin-jdbc-rockset:LATEST io.kestra.plugin:plugin-jdbc-snowflake:LATEST io.kestra.plugin:plugin-jdbc-sqlserver:LATEST io.kestra.plugin:plugin-jdbc-trino:LATEST io.kestra.plugin:plugin-jdbc-vertica:LATEST io.kestra.plugin:plugin-jdbc-vectorwise:LATEST io.kestra.plugin:plugin-kafka:LATEST io.kestra.plugin:plugin-kubernetes:LATEST io.kestra.plugin:plugin-mongodb:LATEST io.kestra.plugin:plugin-mqtt:LATEST io.kestra.plugin:plugin-notifications:LATEST io.kestra.plugin:plugin-script-groovy:LATEST io.kestra.plugin:plugin-script-jython:LATEST io.kestra.plugin:plugin-script-nashorn:LATEST io.kestra.plugin:plugin-serdes:LATEST io.kestra.plugin:plugin-servicenow:LATEST io.kestra.plugin:plugin-singer:LATEST io.kestra.plugin:plugin-soda:LATEST io.kestra.plugin:plugin-spark:LATEST io.kestra.plugin:plugin-tika:LATEST
packages: python3 python3-venv python-is-python3 nodejs curl zip unzip
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
# Artifact
- name: Download executable
@@ -161,15 +162,15 @@ jobs:
TAG=${GITHUB_REF#refs/*/}
if [[ $TAG = "master" ]]
then
- echo ::set-output name=tag::latest
- echo ::set-output name=plugins::${{ matrix.image.plugins }}
+ echo "tag=latest" >> $GITHUB_OUTPUT
+ echo "plugins=${{ matrix.image.plugins }}" >> $GITHUB_OUTPUT
elif [[ $TAG == v* ]]
then
- echo ::set-output name=tag::${TAG}
- echo ::set-output name=plugins::${{ matrix.image.plugins }}
+ echo "tag=${TAG}" >> $GITHUB_OUTPUT
+ echo "plugins=${{ matrix.image.plugins }}" >> $GITHUB_OUTPUT
else
- echo ::set-output name=tag::${TAG}
- echo ::set-output name=plugins::--repositories=https://s01.oss.sonatype.org/content/repositories/snapshots ${{ matrix.image.plugins }}
+ echo "tag=${TAG}" >> $GITHUB_OUTPUT
+ echo "plugins=--repositories=https://s01.oss.sonatype.org/content/repositories/snapshots" >> $GITHUB_OUTPUT
fi
# Docker setup
@@ -208,7 +209,7 @@ jobs:
needs: check
if: github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/tags/v')
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
# Caches
- name: Gradle cache
@@ -287,6 +288,6 @@ jobs:
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}
- channel: '#git'
- name: GitHub Actions
+ username: GitHub Actions
icon_emoji: ':github-actions:'
+ channel: 'C02DQ1A7JLR'
diff --git a/Dockerfile b/Dockerfile
index cf665a2cae..920f3672f4 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -11,7 +11,7 @@ RUN apt-get update -y && \
if [ -n "${APT_PACKAGES}" ]; then apt-get install -y --no-install-recommends ${APT_PACKAGES}; fi && \
apt-get clean && \
rm -rf /var/lib/apt/lists/* /var/tmp/* /tmp/* && \
- if [ -n "${KESTRA_PLUGINS}" ]; then /app/kestra plugins install ${KESTRA_PLUGINS}; rm -rf /tmp/*; fi
+ if [ -n "${KESTRA_PLUGINS}" ]; then /app/kestra plugins install ${KESTRA_PLUGINS} && rm -rf /tmp/*; fi
RUN groupadd kestra && \
useradd -m -g kestra kestra && \
diff --git a/README.md b/README.md
index 69d7d1cd6a..f824e5014f 100644
--- a/README.md
+++ b/README.md
@@ -164,27 +164,32 @@ Kestra is built on [plugin systems](https://kestra.io/plugins/). You can find yo
+ Airbyte |
Amazon S3 |
Avro |
- Azure Blob Storage |
+ Azure Blob Storage |
Bash |
Big Query |
- CSV |
+ CSV |
Cassandra |
ClickHouse |
- DBT |
+ DBT |
Debezium MYSQL |
Debezium Postgres |
- Debezium Microsoft SQL Server |
+ Debezium Microsoft SQL Server |
+ DuckDb |
ElasticSearch |
+
+
+ Fivetran |
Email |
FTP |
diff --git a/build.gradle b/build.gradle
index f85d306b01..7a1c066776 100644
--- a/build.gradle
+++ b/build.gradle
@@ -8,21 +8,21 @@ plugins {
// test
id 'com.adarshr.test-logger' version '3.2.0'
- id 'org.gradle.test-retry' version '1.4.0'
+ id 'org.gradle.test-retry' version '1.4.1'
// helper
- id "com.github.ben-manes.versions" version "0.42.0"
+ id "com.github.ben-manes.versions" version "0.43.0"
// front
id 'org.siouan.frontend-jdk11' version '6.0.0' apply false
// release
id "io.github.gradle-nexus.publish-plugin" version "1.1.0"
- id 'net.researchgate.release' version '3.0.1'
+ id 'net.researchgate.release' version '3.0.2'
id "com.gorylenko.gradle-git-properties" version "2.4.1"
id 'signing'
id 'ru.vyarus.pom' version '2.2.1' apply false
- id 'ru.vyarus.github-info' version '1.3.0' apply false
+ id 'ru.vyarus.github-info' version '1.4.0' apply false
}
idea {
@@ -70,7 +70,8 @@ allprojects {
resolutionStrategy {
force("org.apache.kafka:kafka-clients:" + kafkaVersion)
force("org.apache.kafka:kafka-streams:" + kafkaVersion)
- force("org.jooq:jooq:3.16.9")
+ force("org.jooq:jooq:3.16.10")
+ force("org.slf4j:slf4j-api:1.7.36")
}
}
@@ -97,10 +98,11 @@ allprojects {
api "io.micronaut.rxjava2:micronaut-rxjava2-http-client"
// logs
- implementation "org.slf4j:slf4j-api:1.7.36"
- implementation "ch.qos.logback:logback-classic:1.2.11"
- implementation group: 'org.apache.logging.log4j', name: 'log4j-to-slf4j', version: '2.18.0'
+ implementation "org.slf4j:slf4j-api"
+ implementation "ch.qos.logback:logback-classic"
+ implementation group: 'org.apache.logging.log4j', name: 'log4j-to-slf4j', version: '2.19.0'
implementation group: 'org.slf4j', name: 'jul-to-slf4j', version: '1.7.36'
+ implementation group: 'org.slf4j', name: 'jcl-over-slf4j', version: '1.7.36'
// jackson
implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind'
@@ -110,7 +112,7 @@ allprojects {
// kestra
implementation group: 'com.devskiller.friendly-id', name: 'friendly-id', version: '1.1.0'
- implementation 'com.github.jknack:handlebars:4.3.0'
+ implementation 'com.github.jknack:handlebars:4.3.1'
implementation group: 'net.thisptr', name: 'jackson-jq', version: '1.0.0-preview.20220705'
// exposed utils
diff --git a/core/build.gradle b/core/build.gradle
index 1c7670df1a..f63481a589 100644
--- a/core/build.gradle
+++ b/core/build.gradle
@@ -13,15 +13,15 @@ processResources.dependsOn copyGradleProperties
dependencies {
// log
- implementation group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.11'
+ implementation group: 'ch.qos.logback', name: 'logback-classic'
// serializers
implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-ion'
// utils
implementation group: 'net.jodah', name: 'failsafe', version: '2.4.4'
- implementation 'com.github.oshi:oshi-core:6.2.2'
- implementation 'io.pebbletemplates:pebble:3.1.5'
+ implementation 'com.github.oshi:oshi-core:6.3.0'
+ implementation 'io.pebbletemplates:pebble:3.1.6'
// scheduler
implementation group: 'com.cronutils', name: 'cron-utils', version: '9.2.0'
@@ -33,17 +33,17 @@ dependencies {
implementation 'com.github.docker-java:docker-java-transport-httpclient5:3.2.13'
// schema
- implementation group: 'com.github.victools', name: 'jsonschema-generator', version: '4.26.0'
- implementation group: 'com.github.victools', name: 'jsonschema-module-javax-validation', version: '4.26.0'
- implementation group: 'com.github.victools', name: 'jsonschema-module-jackson', version: '4.26.0'
- implementation group: 'com.github.victools', name: 'jsonschema-module-swagger-2', version: '4.26.0'
+ implementation group: 'com.github.victools', name: 'jsonschema-generator', version: '4.27.0'
+ implementation group: 'com.github.victools', name: 'jsonschema-module-javax-validation', version: '4.27.0'
+ implementation group: 'com.github.victools', name: 'jsonschema-module-jackson', version: '4.27.0'
+ implementation group: 'com.github.victools', name: 'jsonschema-module-swagger-2', version: '4.27.0'
// test
testImplementation project(':repository-memory')
testImplementation project(':runner-memory')
testImplementation project(':storage-local')
- testImplementation 'org.mockito:mockito-junit-jupiter:4.7.0'
+ testImplementation 'org.mockito:mockito-junit-jupiter:4.8.1'
testImplementation "io.micronaut:micronaut-http-client"
testImplementation "io.micronaut.rxjava2:micronaut-rxjava2-http-client"
testImplementation "io.micronaut:micronaut-http-server-netty"
diff --git a/core/src/main/java/io/kestra/core/models/flows/Input.java b/core/src/main/java/io/kestra/core/models/flows/Input.java
index 1fcffef135..78728c03d1 100644
--- a/core/src/main/java/io/kestra/core/models/flows/Input.java
+++ b/core/src/main/java/io/kestra/core/models/flows/Input.java
@@ -20,7 +20,7 @@
public class Input {
@NotNull
@NotBlank
- @Pattern(regexp="[a-zA-Z0-9_-]+")
+ @Pattern(regexp="[.a-zA-Z0-9_-]+")
String name;
@NotBlank
@@ -47,5 +47,6 @@ public enum Type {
DURATION,
FILE,
JSON,
+ URI,
}
}
diff --git a/core/src/main/java/io/kestra/core/models/triggers/types/Schedule.java b/core/src/main/java/io/kestra/core/models/triggers/types/Schedule.java
index f78b55ae11..d4553e86df 100644
--- a/core/src/main/java/io/kestra/core/models/triggers/types/Schedule.java
+++ b/core/src/main/java/io/kestra/core/models/triggers/types/Schedule.java
@@ -129,7 +129,8 @@ public class Schedule extends AbstractTrigger implements PollingTriggerInterface
@Schema(
title = "The time zone id to use for evaluate cron. Default value is the server default zone id."
)
- @PluginProperty(dynamic = true)
+ @PluginProperty(dynamic = false)
+ @Builder.Default
private String timezone = ZoneId.systemDefault().toString();
@Schema(
@@ -192,7 +193,9 @@ public ZonedDateTime nextEvaluationDate(ConditionContext conditionContext, Optio
// no previous present but backfill
if (backfill != null && backfill.getStart() != null) {
- return backfill.getStart();
+ return this.timezone != null ?
+ backfill.getStart().withZoneSameLocal(ZoneId.of(this.timezone)) :
+ backfill.getStart();
}
// no previous present & no backfill, just provide now
diff --git a/core/src/main/java/io/kestra/core/runners/RunnerUtils.java b/core/src/main/java/io/kestra/core/runners/RunnerUtils.java
index 3e914d1781..74596b26f7 100644
--- a/core/src/main/java/io/kestra/core/runners/RunnerUtils.java
+++ b/core/src/main/java/io/kestra/core/runners/RunnerUtils.java
@@ -31,19 +31,20 @@
import java.time.LocalDate;
import java.time.LocalTime;
import java.time.format.DateTimeParseException;
-import java.util.AbstractMap;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
+import java.util.*;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.function.Predicate;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import java.util.stream.Collectors;
@Singleton
public class RunnerUtils {
+ public static final Pattern URI_PATTERN = Pattern.compile("^[a-z]+:\\/\\/(?:www\\.)?[-a-zA-Z0-9@:%._\\+~#=]{1,256}\\.[a-zA-Z0-9()]{1,6}\\b(?:[-a-zA-Z0-9()@:%_\\+.~#?&\\/=]*)$");
+
@Inject
@Named(QueueFactoryInterface.EXECUTION_NAMED)
protected QueueInterface executionQueue;
@@ -99,7 +100,7 @@ public Map typedInputs(Flow flow, Execution execution, Map results = flow
.getInputs()
.stream()
.map((Function>>) input -> {
@@ -211,6 +212,17 @@ public Map typedInputs(Flow flow, Execution execution, Map(
+ input.getName(),
+ current
+ ));
+ } else {
+ throw new MissingRequiredInput("Invalid URI format for '" + input.getName() + "' for '" + current + "'");
+ }
+
default:
throw new MissingRequiredInput("Invalid input type '" + input.getType() + "' for '" + input.getName() + "'");
}
@@ -218,6 +230,26 @@ public Map typedInputs(Flow flow, Execution execution, Map handleNestedInputs(Map inputs) {
+ Map result = new TreeMap<>();
+
+ for (Map.Entry entry : inputs.entrySet()) {
+ String[] f = entry.getKey().split("\\.");
+ Map t = result;
+ int i = 0;
+ for (int m = f.length - 1; i < m; ++i) {
+ t = (Map) t.computeIfAbsent(f[i], k -> new TreeMap<>());
+ }
+
+ t.put(f[i], entry.getValue());
+ }
+
+ return result;
}
public Execution runOne(String namespace, String flowId) throws TimeoutException {
diff --git a/core/src/main/java/io/kestra/core/tasks/scripts/AbstractBash.java b/core/src/main/java/io/kestra/core/tasks/scripts/AbstractBash.java
index d92bcecc65..5757addd7e 100644
--- a/core/src/main/java/io/kestra/core/tasks/scripts/AbstractBash.java
+++ b/core/src/main/java/io/kestra/core/tasks/scripts/AbstractBash.java
@@ -116,15 +116,14 @@ abstract public class AbstractBash extends Task {
@Schema(
title = "Input files are extra files supplied by user that make it simpler organize code.",
- description = "Describe a files map that will be written and usable in execution context. In python execution " +
- "context is in a temp folder, for bash scripts, you can reach files using a workingDir variable " +
- "like 'source {{workingDir}}/myfile.sh' "
+ description = "Describe a files map (that can be a map or a json string) that will be written and usable in execution context. " +
+ "You can reach files using a workingDir variable like 'source {{workingDir}}/myfile.sh' "
)
@PluginProperty(
additionalProperties = String.class,
dynamic = true
)
- protected Map inputFiles;
+ protected Object inputFiles;
@Schema(
title = "Additional environnements variable to add for current process."
@@ -151,7 +150,7 @@ abstract public class AbstractBash extends Task {
protected transient Map additionalVars = new HashMap<>();
protected Map finalInputFiles(RunContext runContext) throws IOException, IllegalVariableEvaluationException {
- return this.inputFiles != null ? new HashMap<>(this.inputFiles) : new HashMap<>();
+ return this.inputFiles != null ? new HashMap<>(BashService.transformInputFiles(runContext, this.inputFiles)) : new HashMap<>();
}
protected Map finalEnv() throws IOException {
@@ -426,5 +425,23 @@ public static class DockerOptions {
@PluginProperty(dynamic = false)
@Builder.Default
protected Boolean pullImage = true;
+
+ @Schema(
+ title = "A list of request for devices to be sent to device drivers"
+ )
+ @PluginProperty(dynamic = false)
+ protected List deviceRequests;
+
+ @SuperBuilder
+ @NoArgsConstructor
+ @Getter
+ @Introspected
+ public static class DeviceRequest {
+ private String driver;
+ private Integer count;
+ private List deviceIds;
+ private List> capabilities;
+ private Map options;
+ }
}
}
diff --git a/core/src/main/java/io/kestra/core/tasks/scripts/BashService.java b/core/src/main/java/io/kestra/core/tasks/scripts/BashService.java
index bb9459cc63..c1d43bb2bf 100644
--- a/core/src/main/java/io/kestra/core/tasks/scripts/BashService.java
+++ b/core/src/main/java/io/kestra/core/tasks/scripts/BashService.java
@@ -1,6 +1,7 @@
package io.kestra.core.tasks.scripts;
import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.kestra.core.exceptions.IllegalVariableEvaluationException;
import io.kestra.core.runners.RunContext;
@@ -17,11 +18,13 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import javax.validation.constraints.NotNull;
+
import static io.kestra.core.utils.Rethrow.throwConsumer;
abstract public class BashService {
protected static final ObjectMapper MAPPER = JacksonMapper.ofJson();
- private static final Pattern PATTERN = Pattern.compile("^::(\\{.*\\})::$");
+ private static final Pattern PATTERN = Pattern.compile("^::(\\{.*})::$");
public static List finalCommandsWithInterpreter(
String interpreter,
@@ -100,6 +103,23 @@ private static void validFilename(String s) {
}
}
+ public static Map transformInputFiles(RunContext runContext, @NotNull Object inputFiles) throws IllegalVariableEvaluationException, JsonProcessingException {
+ if (inputFiles instanceof Map) {
+ //noinspection unchecked
+ return (Map) inputFiles;
+ } else if (inputFiles instanceof String) {
+ final TypeReference