Skip to content

Commit

Permalink
chore(version): update to version 'v0.5.2'.
Browse files Browse the repository at this point in the history
  • Loading branch information
@tchiotludo
tchiotludo committed Oct 28, 2022
2 parents 638695e + a78309a commit c4c78d8
Show file tree
Hide file tree
Showing 61 changed files with 2,039 additions and 1,244 deletions.
25 changes: 13 additions & 12 deletions .github/workflows/main.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ jobs:
architecture: 'x64'
- uses: actions/setup-node@v3
with:
node-version: '12'
node-version: '14'
check-latest: true

# Services
Expand Down Expand Up @@ -120,6 +120,7 @@ jobs:
- name: Create GitHub release
uses: "marvinpinto/action-automatic-releases@latest"
if: startsWith(github.ref, 'refs/tags/v')
continue-on-error: true
with:
repo_token: "${{ secrets.GITHUB_TOKEN }}"
prerelease: false
Expand All @@ -138,10 +139,10 @@ jobs:
plugins: ""
packages: ""
- name: "-full"
plugins: io.kestra.storage:storage-azure:LATEST io.kestra.storage:storage-gcs:LATEST io.kestra.storage:storage-minio:LATEST io.kestra.plugin:plugin-aws:LATEST io.kestra.plugin:plugin-azure:LATEST io.kestra.plugin:plugin-powerbi:LATEST io.kestra.plugin:plugin-pulsar:LATEST io.kestra.plugin:plugin-cassandra:LATEST io.kestra.plugin:plugin-compress:LATEST io.kestra.plugin:plugin-crypto:LATEST io.kestra.plugin:plugin-dbt:LATEST io.kestra.plugin:plugin-debezium-mysql:LATEST io.kestra.plugin:plugin-debezium-postgres:LATEST io.kestra.plugin:plugin-debezium-sqlserver:LATEST io.kestra.plugin:plugin-elasticsearch:LATEST io.kestra.plugin:plugin-fs:LATEST io.kestra.plugin:plugin-gcp:LATEST io.kestra.plugin:plugin-googleworkspace:LATEST io.kestra.plugin:plugin-jdbc-clickhouse:LATEST io.kestra.plugin:plugin-jdbc-duckdb:LATEST io.kestra.plugin:plugin-jdbc-mysql:LATEST io.kestra.plugin:plugin-jdbc-oracle:LATEST io.kestra.plugin:plugin-jdbc-pinot:LATEST io.kestra.plugin:plugin-jdbc-postgres:LATEST io.kestra.plugin:plugin-jdbc-redshift:LATEST io.kestra.plugin:plugin-jdbc-rockset:LATEST io.kestra.plugin:plugin-jdbc-snowflake:LATEST io.kestra.plugin:plugin-jdbc-sqlserver:LATEST io.kestra.plugin:plugin-jdbc-trino:LATEST io.kestra.plugin:plugin-jdbc-vertica:LATEST io.kestra.plugin:plugin-jdbc-vectorwise:LATEST io.kestra.plugin:plugin-kafka:LATEST io.kestra.plugin:plugin-kubernetes:LATEST io.kestra.plugin:plugin-mongodb:LATEST io.kestra.plugin:plugin-mqtt:LATEST io.kestra.plugin:plugin-notifications:LATEST io.kestra.plugin:plugin-script-groovy:LATEST io.kestra.plugin:plugin-script-jython:LATEST io.kestra.plugin:plugin-script-nashorn:LATEST io.kestra.plugin:plugin-serdes:LATEST io.kestra.plugin:plugin-servicenow:LATEST io.kestra.plugin:plugin-singer:LATEST io.kestra.plugin:plugin-soda:LATEST io.kestra.plugin:plugin-spark:LATEST io.kestra.plugin:plugin-tika:LATEST
plugins: io.kestra.storage:storage-azure:LATEST io.kestra.storage:storage-gcs:LATEST io.kestra.storage:storage-minio:LATEST io.kestra.plugin:plugin-airbyte:LATEST io.kestra.plugin:plugin-aws:LATEST io.kestra.plugin:plugin-azure:LATEST io.kestra.plugin:plugin-powerbi:LATEST io.kestra.plugin:plugin-pulsar:LATEST io.kestra.plugin:plugin-cassandra:LATEST io.kestra.plugin:plugin-compress:LATEST io.kestra.plugin:plugin-crypto:LATEST io.kestra.plugin:plugin-dbt:LATEST io.kestra.plugin:plugin-debezium-mysql:LATEST io.kestra.plugin:plugin-debezium-postgres:LATEST io.kestra.plugin:plugin-debezium-sqlserver:LATEST io.kestra.plugin:plugin-elasticsearch:LATEST io.kestra.plugin:plugin-fivetran:LATEST io.kestra.plugin:plugin-fs:LATEST io.kestra.plugin:plugin-gcp:LATEST io.kestra.plugin:plugin-googleworkspace:LATEST io.kestra.plugin:plugin-jdbc-clickhouse:LATEST io.kestra.plugin:plugin-jdbc-duckdb:LATEST io.kestra.plugin:plugin-jdbc-mysql:LATEST io.kestra.plugin:plugin-jdbc-oracle:LATEST io.kestra.plugin:plugin-jdbc-pinot:LATEST io.kestra.plugin:plugin-jdbc-postgres:LATEST io.kestra.plugin:plugin-jdbc-redshift:LATEST io.kestra.plugin:plugin-jdbc-rockset:LATEST io.kestra.plugin:plugin-jdbc-snowflake:LATEST io.kestra.plugin:plugin-jdbc-sqlserver:LATEST io.kestra.plugin:plugin-jdbc-trino:LATEST io.kestra.plugin:plugin-jdbc-vertica:LATEST io.kestra.plugin:plugin-jdbc-vectorwise:LATEST io.kestra.plugin:plugin-kafka:LATEST io.kestra.plugin:plugin-kubernetes:LATEST io.kestra.plugin:plugin-mongodb:LATEST io.kestra.plugin:plugin-mqtt:LATEST io.kestra.plugin:plugin-notifications:LATEST io.kestra.plugin:plugin-script-groovy:LATEST io.kestra.plugin:plugin-script-jython:LATEST io.kestra.plugin:plugin-script-nashorn:LATEST io.kestra.plugin:plugin-serdes:LATEST io.kestra.plugin:plugin-servicenow:LATEST io.kestra.plugin:plugin-singer:LATEST io.kestra.plugin:plugin-soda:LATEST io.kestra.plugin:plugin-spark:LATEST io.kestra.plugin:plugin-tika:LATEST
packages: python3 python3-venv python-is-python3 nodejs curl zip unzip
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3

# Artifact
- name: Download executable
Expand All @@ -161,15 +162,15 @@ jobs:
TAG=${GITHUB_REF#refs/*/}
if [[ $TAG = "master" ]]
then
echo ::set-output name=tag::latest
echo ::set-output name=plugins::${{ matrix.image.plugins }}
echo "tag=latest" >> $GITHUB_OUTPUT
echo "plugins=${{ matrix.image.plugins }}" >> $GITHUB_OUTPUT
elif [[ $TAG == v* ]]
then
echo ::set-output name=tag::${TAG}
echo ::set-output name=plugins::${{ matrix.image.plugins }}
echo "tag=${TAG}" >> $GITHUB_OUTPUT
echo "plugins=${{ matrix.image.plugins }}" >> $GITHUB_OUTPUT
else
echo ::set-output name=tag::${TAG}
echo ::set-output name=plugins::--repositories=https://s01.oss.sonatype.org/content/repositories/snapshots ${{ matrix.image.plugins }}
echo "tag=${TAG}" >> $GITHUB_OUTPUT
echo "plugins=--repositories=https://s01.oss.sonatype.org/content/repositories/snapshots" >> $GITHUB_OUTPUT
fi
# Docker setup
Expand Down Expand Up @@ -208,7 +209,7 @@ jobs:
needs: check
if: github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/tags/v')
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3

# Caches
- name: Gradle cache
Expand Down Expand Up @@ -287,6 +288,6 @@ jobs:
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}
channel: '#git'
name: GitHub Actions
username: GitHub Actions
icon_emoji: ':github-actions:'
channel: 'C02DQ1A7JLR'
2 changes: 1 addition & 1 deletion Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ RUN apt-get update -y && \
if [ -n "${APT_PACKAGES}" ]; then apt-get install -y --no-install-recommends ${APT_PACKAGES}; fi && \
apt-get clean && \
rm -rf /var/lib/apt/lists/* /var/tmp/* /tmp/* && \
if [ -n "${KESTRA_PLUGINS}" ]; then /app/kestra plugins install ${KESTRA_PLUGINS}; rm -rf /tmp/*; fi
if [ -n "${KESTRA_PLUGINS}" ]; then /app/kestra plugins install ${KESTRA_PLUGINS} && rm -rf /tmp/*; fi

RUN groupadd kestra && \
useradd -m -g kestra kestra && \
Expand Down
13 changes: 9 additions & 4 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,27 +164,32 @@ Kestra is built on [plugin systems](https://kestra.io/plugins/). You can find yo

<table>
<tr>
<td><a href="https://kestra.io/plugins/plugin-airbyte">Airbyte</a></td>
<td><a href="https://kestra.io/plugins/plugin-aws#s3">Amazon S3</a></td>
<td><a href="https://kestra.io/plugins/plugin-serdes#avro">Avro</a></td>
<td><a href="https://kestra.io/plugins/plugin-azure/#storage-blob">Azure Blob Storage</a></td>
</tr>
<tr>
<td><a href="https://kestra.io/plugins/plugin-azure/#storage-blob">Azure Blob Storage</a></td>
<td><a href="https://kestra.io/plugins/core/tasks/scripts/io.kestra.core.tasks.scripts.Bash">Bash</a></td>
<td><a href="https://kestra.io/plugins/plugin-gcp#bigquery">Big Query</a></td>
<td><a href="https://kestra.io/plugins/plugin-serdes#csv">CSV</a></td>
</tr>
<tr>
<td><a href="https://kestra.io/plugins/plugin-serdes#csv">CSV</a></td>
<td><a href="https://kestra.io/plugins/plugin-cassandra">Cassandra</a></td>
<td><a href="https://kestra.io/plugins/plugin-jdbc-clickhouse">ClickHouse</a></td>
<td><a href="https://kestra.io/plugins/plugin-dbt">DBT</a></td>
</tr>
<tr>
<td><a href="https://kestra.io/plugins/plugin-dbt">DBT</a></td>
<td><a href="https://kestra.io/plugins/plugin-debezium-mysql">Debezium MYSQL</a></td>
<td><a href="https://kestra.io/plugins/plugin-debezium-postgres">Debezium Postgres</a></td>
<td><a href="https://kestra.io/plugins/plugin-debezium-sqlserver">Debezium Microsoft SQL Server</a></td>
</tr>
<tr>
<td><a href="https://kestra.io/plugins/plugin-debezium-sqlserver">Debezium Microsoft SQL Server</a></td>
<td><a href="https://kestra.io/plugins/plugin-jdbc-duckdb">DuckDb</a></td>
<td><a href="https://kestra.io/plugins/plugin-elasticsearch">ElasticSearch</a></td>
</tr>
<tr>
<td><a href="https://kestra.io/plugins/plugin-fivetran">Fivetran</a></td>
<td><a href="https://kestra.io/plugins/plugin-notifications#mail">Email</a></td>
<td><a href="https://kestra.io/plugins/plugin-fs#ftp">FTP</a></td>
</tr>
Expand Down
20 changes: 11 additions & 9 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,21 @@ plugins {

// test
id 'com.adarshr.test-logger' version '3.2.0'
id 'org.gradle.test-retry' version '1.4.0'
id 'org.gradle.test-retry' version '1.4.1'

// helper
id "com.github.ben-manes.versions" version "0.42.0"
id "com.github.ben-manes.versions" version "0.43.0"

// front
id 'org.siouan.frontend-jdk11' version '6.0.0' apply false

// release
id "io.github.gradle-nexus.publish-plugin" version "1.1.0"
id 'net.researchgate.release' version '3.0.1'
id 'net.researchgate.release' version '3.0.2'
id "com.gorylenko.gradle-git-properties" version "2.4.1"
id 'signing'
id 'ru.vyarus.pom' version '2.2.1' apply false
id 'ru.vyarus.github-info' version '1.3.0' apply false
id 'ru.vyarus.github-info' version '1.4.0' apply false
}

idea {
Expand Down Expand Up @@ -70,7 +70,8 @@ allprojects {
resolutionStrategy {
force("org.apache.kafka:kafka-clients:" + kafkaVersion)
force("org.apache.kafka:kafka-streams:" + kafkaVersion)
force("org.jooq:jooq:3.16.9")
force("org.jooq:jooq:3.16.10")
force("org.slf4j:slf4j-api:1.7.36")
}
}

Expand All @@ -97,10 +98,11 @@ allprojects {
api "io.micronaut.rxjava2:micronaut-rxjava2-http-client"

// logs
implementation "org.slf4j:slf4j-api:1.7.36"
implementation "ch.qos.logback:logback-classic:1.2.11"
implementation group: 'org.apache.logging.log4j', name: 'log4j-to-slf4j', version: '2.18.0'
implementation "org.slf4j:slf4j-api"
implementation "ch.qos.logback:logback-classic"
implementation group: 'org.apache.logging.log4j', name: 'log4j-to-slf4j', version: '2.19.0'
implementation group: 'org.slf4j', name: 'jul-to-slf4j', version: '1.7.36'
implementation group: 'org.slf4j', name: 'jcl-over-slf4j', version: '1.7.36'

// jackson
implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind'
Expand All @@ -110,7 +112,7 @@ allprojects {

// kestra
implementation group: 'com.devskiller.friendly-id', name: 'friendly-id', version: '1.1.0'
implementation 'com.github.jknack:handlebars:4.3.0'
implementation 'com.github.jknack:handlebars:4.3.1'
implementation group: 'net.thisptr', name: 'jackson-jq', version: '1.0.0-preview.20220705'

// exposed utils
Expand Down
16 changes: 8 additions & 8 deletions core/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,15 @@ processResources.dependsOn copyGradleProperties

dependencies {
// log
implementation group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.11'
implementation group: 'ch.qos.logback', name: 'logback-classic'

// serializers
implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-ion'

// utils
implementation group: 'net.jodah', name: 'failsafe', version: '2.4.4'
implementation 'com.github.oshi:oshi-core:6.2.2'
implementation 'io.pebbletemplates:pebble:3.1.5'
implementation 'com.github.oshi:oshi-core:6.3.0'
implementation 'io.pebbletemplates:pebble:3.1.6'

// scheduler
implementation group: 'com.cronutils', name: 'cron-utils', version: '9.2.0'
Expand All @@ -33,17 +33,17 @@ dependencies {
implementation 'com.github.docker-java:docker-java-transport-httpclient5:3.2.13'

// schema
implementation group: 'com.github.victools', name: 'jsonschema-generator', version: '4.26.0'
implementation group: 'com.github.victools', name: 'jsonschema-module-javax-validation', version: '4.26.0'
implementation group: 'com.github.victools', name: 'jsonschema-module-jackson', version: '4.26.0'
implementation group: 'com.github.victools', name: 'jsonschema-module-swagger-2', version: '4.26.0'
implementation group: 'com.github.victools', name: 'jsonschema-generator', version: '4.27.0'
implementation group: 'com.github.victools', name: 'jsonschema-module-javax-validation', version: '4.27.0'
implementation group: 'com.github.victools', name: 'jsonschema-module-jackson', version: '4.27.0'
implementation group: 'com.github.victools', name: 'jsonschema-module-swagger-2', version: '4.27.0'

// test
testImplementation project(':repository-memory')
testImplementation project(':runner-memory')
testImplementation project(':storage-local')

testImplementation 'org.mockito:mockito-junit-jupiter:4.7.0'
testImplementation 'org.mockito:mockito-junit-jupiter:4.8.1'
testImplementation "io.micronaut:micronaut-http-client"
testImplementation "io.micronaut.rxjava2:micronaut-rxjava2-http-client"
testImplementation "io.micronaut:micronaut-http-server-netty"
Expand Down
3 changes: 2 additions & 1 deletion core/src/main/java/io/kestra/core/models/flows/Input.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
public class Input {
@NotNull
@NotBlank
@Pattern(regexp="[a-zA-Z0-9_-]+")
@Pattern(regexp="[.a-zA-Z0-9_-]+")
String name;

@NotBlank
Expand All @@ -47,5 +47,6 @@ public enum Type {
DURATION,
FILE,
JSON,
URI,
}
}
7 changes: 5 additions & 2 deletions core/src/main/java/io/kestra/core/models/triggers/types/Schedule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,8 @@ public class Schedule extends AbstractTrigger implements PollingTriggerInterface
@Schema(
title = "The time zone id to use for evaluate cron. Default value is the server default zone id."
)
@PluginProperty(dynamic = true)
@PluginProperty(dynamic = false)
@Builder.Default
private String timezone = ZoneId.systemDefault().toString();

@Schema(
Expand Down Expand Up @@ -192,7 +193,9 @@ public ZonedDateTime nextEvaluationDate(ConditionContext conditionContext, Optio

// no previous present but backfill
if (backfill != null && backfill.getStart() != null) {
return backfill.getStart();
return this.timezone != null ?
backfill.getStart().withZoneSameLocal(ZoneId.of(this.timezone)) :
backfill.getStart();
}

// no previous present & no backfill, just provide now
Expand Down
42 changes: 37 additions & 5 deletions core/src/main/java/io/kestra/core/runners/RunnerUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,19 +31,20 @@
import java.time.LocalDate;
import java.time.LocalTime;
import java.time.format.DateTimeParseException;
import java.util.AbstractMap;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.*;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

@Singleton
public class RunnerUtils {
public static final Pattern URI_PATTERN = Pattern.compile("^[a-z]+:\\/\\/(?:www\\.)?[-a-zA-Z0-9@:%._\\+~#=]{1,256}\\.[a-zA-Z0-9()]{1,6}\\b(?:[-a-zA-Z0-9()@:%_\\+.~#?&\\/=]*)$");

@Inject
@Named(QueueFactoryInterface.EXECUTION_NAMED)
protected QueueInterface<Execution> executionQueue;
Expand Down Expand Up @@ -99,7 +100,7 @@ public Map<String, Object> typedInputs(Flow flow, Execution execution, Map<Strin
return ImmutableMap.of();
}

return flow
Map<String, Object> results = flow
.getInputs()
.stream()
.map((Function<Input, Optional<AbstractMap.SimpleEntry<String, Object>>>) input -> {
Expand Down Expand Up @@ -211,13 +212,44 @@ public Map<String, Object> typedInputs(Flow flow, Execution execution, Map<Strin
throw new MissingRequiredInput("Invalid JSON format for '" + input.getName() + "' for '" + current + "' with error " + e.getMessage(), e);
}

case URI:
Matcher matcher = URI_PATTERN.matcher(current);
if (matcher.matches()) {
return Optional.of(new AbstractMap.SimpleEntry<>(
input.getName(),
current
));
} else {
throw new MissingRequiredInput("Invalid URI format for '" + input.getName() + "' for '" + current + "'");
}

default:
throw new MissingRequiredInput("Invalid input type '" + input.getType() + "' for '" + input.getName() + "'");
}
})
.filter(Optional::isPresent)
.map(Optional::get)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));

return handleNestedInputs(results);
}

@SuppressWarnings("unchecked")
private Map<String, Object> handleNestedInputs(Map<String, Object> inputs) {
Map<String, Object> result = new TreeMap<>();

for (Map.Entry<String, Object> entry : inputs.entrySet()) {
String[] f = entry.getKey().split("\\.");
Map<String, Object> t = result;
int i = 0;
for (int m = f.length - 1; i < m; ++i) {
t = (Map<String, Object>) t.computeIfAbsent(f[i], k -> new TreeMap<>());
}

t.put(f[i], entry.getValue());
}

return result;
}

public Execution runOne(String namespace, String flowId) throws TimeoutException {
Expand Down
Loading

0 comments on commit c4c78d8

Please sign in to comment.