diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 88a0ecfdff..833151861a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -139,7 +139,7 @@ jobs: plugins: "" packages: "" - name: "-full" - plugins: io.kestra.storage:storage-azure:LATEST io.kestra.storage:storage-gcs:LATEST io.kestra.storage:storage-minio:LATEST io.kestra.plugin:plugin-airbyte:LATEST io.kestra.plugin:plugin-amqp:LATEST io.kestra.plugin:plugin-aws:LATEST io.kestra.plugin:plugin-azure:LATEST io.kestra.plugin:plugin-powerbi:LATEST io.kestra.plugin:plugin-pulsar:LATEST io.kestra.plugin:plugin-cassandra:LATEST io.kestra.plugin:plugin-compress:LATEST io.kestra.plugin:plugin-couchbase:LATEST io.kestra.plugin:plugin-crypto:LATEST io.kestra.plugin:plugin-dbt:LATEST io.kestra.plugin:plugin-debezium-mysql:LATEST io.kestra.plugin:plugin-debezium-postgres:LATEST io.kestra.plugin:plugin-debezium-sqlserver:LATEST io.kestra.plugin:plugin-elasticsearch:LATEST io.kestra.plugin:plugin-fivetran:LATEST io.kestra.plugin:plugin-fs:LATEST io.kestra.plugin:plugin-gcp:LATEST io.kestra.plugin:plugin-git:LATEST io.kestra.plugin:plugin-googleworkspace:LATEST io.kestra.plugin:plugin-jdbc-clickhouse:LATEST io.kestra.plugin:plugin-jdbc-duckdb:LATEST io.kestra.plugin:plugin-jdbc-mysql:LATEST io.kestra.plugin:plugin-nats:LATEST io.kestra.plugin:plugin-jdbc-oracle:LATEST io.kestra.plugin:plugin-jdbc-pinot:LATEST io.kestra.plugin:plugin-jdbc-postgres:LATEST io.kestra.plugin:plugin-jdbc-redshift:LATEST io.kestra.plugin:plugin-jdbc-rockset:LATEST io.kestra.plugin:plugin-jdbc-snowflake:LATEST io.kestra.plugin:plugin-jdbc-sqlserver:LATEST io.kestra.plugin:plugin-jdbc-trino:LATEST io.kestra.plugin:plugin-jdbc-vertica:LATEST io.kestra.plugin:plugin-jdbc-vectorwise:LATEST io.kestra.plugin:plugin-kafka:LATEST io.kestra.plugin:plugin-kubernetes:LATEST io.kestra.plugin:plugin-mongodb:LATEST io.kestra.plugin:plugin-mqtt:LATEST io.kestra.plugin:plugin-neo4j:LATEST io.kestra.plugin:plugin-notifications:LATEST io.kestra.plugin:plugin-openai:LATEST io.kestra.plugin:plugin-redis:LATEST io.kestra.plugin:plugin-script-groovy:LATEST io.kestra.plugin:plugin-script-jython:LATEST io.kestra.plugin:plugin-script-nashorn:LATEST io.kestra.plugin:plugin-script-node:LATEST io.kestra.plugin:plugin-script-powershell:LATEST io.kestra.plugin:plugin-script-python:LATEST io.kestra.plugin:plugin-script-r:LATEST io.kestra.plugin:plugin-script-shell:LATEST io.kestra.plugin:plugin-serdes:LATEST io.kestra.plugin:plugin-servicenow:LATEST io.kestra.plugin:plugin-singer:LATEST io.kestra.plugin:plugin-soda:LATEST io.kestra.plugin:plugin-spark:LATEST io.kestra.plugin:plugin-tika:LATEST + plugins: io.kestra.storage:storage-azure:LATEST io.kestra.storage:storage-gcs:LATEST io.kestra.storage:storage-minio:LATEST io.kestra.plugin:plugin-airbyte:LATEST io.kestra.plugin:plugin-amqp:LATEST io.kestra.plugin:plugin-aws:LATEST io.kestra.plugin:plugin-azure:LATEST io.kestra.plugin:plugin-powerbi:LATEST io.kestra.plugin:plugin-pulsar:LATEST io.kestra.plugin:plugin-cassandra:LATEST io.kestra.plugin:plugin-compress:LATEST io.kestra.plugin:plugin-couchbase:LATEST io.kestra.plugin:plugin-crypto:LATEST io.kestra.plugin:plugin-databricks:LATEST io.kestra.plugin:plugin-dbt:LATEST io.kestra.plugin:plugin-debezium-mysql:LATEST io.kestra.plugin:plugin-debezium-postgres:LATEST io.kestra.plugin:plugin-debezium-sqlserver:LATEST io.kestra.plugin:plugin-elasticsearch:LATEST io.kestra.plugin:plugin-fivetran:LATEST io.kestra.plugin:plugin-fs:LATEST io.kestra.plugin:plugin-gcp:LATEST io.kestra.plugin:plugin-git:LATEST io.kestra.plugin:plugin-googleworkspace:LATEST io.kestra.plugin:plugin-hightouch:LATEST io.kestra.plugin:plugin-jdbc-clickhouse:LATEST io.kestra.plugin:plugin-jdbc-duckdb:LATEST io.kestra.plugin:plugin-jdbc-mysql:LATEST io.kestra.plugin:plugin-nats:LATEST io.kestra.plugin:plugin-jdbc-oracle:LATEST io.kestra.plugin:plugin-jdbc-pinot:LATEST io.kestra.plugin:plugin-jdbc-postgres:LATEST io.kestra.plugin:plugin-jdbc-redshift:LATEST io.kestra.plugin:plugin-jdbc-rockset:LATEST io.kestra.plugin:plugin-jdbc-snowflake:LATEST io.kestra.plugin:plugin-jdbc-sqlserver:LATEST io.kestra.plugin:plugin-jdbc-trino:LATEST io.kestra.plugin:plugin-jdbc-vertica:LATEST io.kestra.plugin:plugin-jdbc-vectorwise:LATEST io.kestra.plugin:plugin-kafka:LATEST io.kestra.plugin:plugin-kubernetes:LATEST io.kestra.plugin:plugin-mongodb:LATEST io.kestra.plugin:plugin-mqtt:LATEST io.kestra.plugin:plugin-neo4j:LATEST io.kestra.plugin:plugin-notifications:LATEST io.kestra.plugin:plugin-openai:LATEST io.kestra.plugin:plugin-redis:LATEST io.kestra.plugin:plugin-script-groovy:LATEST io.kestra.plugin:plugin-script-jython:LATEST io.kestra.plugin:plugin-script-nashorn:LATEST io.kestra.plugin:plugin-script-julia:LATEST io.kestra.plugin:plugin-script-node:LATEST io.kestra.plugin:plugin-script-powershell:LATEST io.kestra.plugin:plugin-script-python:LATEST io.kestra.plugin:plugin-script-r:LATEST io.kestra.plugin:plugin-script-shell:LATEST io.kestra.plugin:plugin-serdes:LATEST io.kestra.plugin:plugin-servicenow:LATEST io.kestra.plugin:plugin-singer:LATEST io.kestra.plugin:plugin-soda:LATEST io.kestra.plugin:plugin-spark:LATEST io.kestra.plugin:plugin-tika:LATEST io.kestra.plugin:plugin-malloy:LATEST packages: python3 python3-venv python-is-python3 nodejs npm curl zip unzip steps: - uses: actions/checkout@v3 @@ -278,48 +278,6 @@ jobs: echo ${SONATYPE_GPG_FILE} | base64 -d > ~/.gradle/secring.gpg ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository --no-daemon --priority=normal - pip: - name: Publish to Pip - runs-on: ubuntu-latest - needs: check - if: startsWith(github.ref, 'refs/tags/v') - steps: - - uses: actions/checkout@v3 - - - name: Set up Python 3.10 - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - - name: Copy gradle properties - run: | - cp gradle.properties core/src/main/resources/scripts/python/gradle.properties - cp README.md core/src/main/resources/scripts/python/README.md - - - name: Install pypa/build - working-directory: core/src/main/resources/scripts/python - run: >- - python -m - pip install - build - --user - - - name: Build a binary wheel and a source tarball - working-directory: core/src/main/resources/scripts/python - run: >- - python -m - build - --sdist - --wheel - --outdir dist/ - . - - - name: Publish distribution to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.PYPI_API_TOKEN }} - packages-dir: core/src/main/resources/scripts/python - end: runs-on: ubuntu-latest needs: diff --git a/.github/workflows/sonar.yml b/.github/workflows/sonar.yml new file mode 100644 index 0000000000..feff199d12 --- /dev/null +++ b/.github/workflows/sonar.yml @@ -0,0 +1,45 @@ +name: SonarCloud +on: + push: + branches: + - develop + pull_request: + types: [opened, synchronize, reopened] + +jobs: + build: + name: Build and analyze + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: 17 + + - name: Cache SonarCloud packages + uses: actions/cache@v3 + with: + path: ~/.sonar/cache + key: ${{ runner.os }}-sonar + restore-keys: ${{ runner.os }}-sonar + + - name: Cache Gradle packages + uses: actions/cache@v3 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle*.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Build and analyze + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + run: ./gradlew build jacoco sonar --info \ No newline at end of file diff --git a/.gitignore b/.gitignore index fba2503617..a12045e00c 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ ui/.env.*.local webserver/src/main/resources/ui yarn.lock ui/coverage +ui/stats.html ### Docker /.env @@ -43,11 +44,5 @@ docker/app/kestra docker/app/confs/*.yml docker/app/secrets/*.yml -### Python -core/src/main/resources/scripts/python/src/kestra.egg-info -core/src/main/resources/scripts/python/dist -core/src/main/resources/scripts/python/gradle.properties -core/src/main/resources/scripts/python/README.md - ### Build core/src/main/resources/gradle.properties \ No newline at end of file diff --git a/README.md b/README.md index a203bb5b3f..c255a5f22f 100644 --- a/README.md +++ b/README.md @@ -1,42 +1,36 @@

- Kestra workflow orchestrator + Kestra workflow orchestrator

- Event-driven declarative orchestrator to simplify data operations
+ Event-Driven Declarative Orchestrator

- License - Commits-per-month - Github star - Last Version - Docker pull - Artifact Hub - Kestra infinitely scalable orchestration and scheduling platform - Slack - Github discussions - Twitter - Code Cov - Github Actions + Last Version + License + Github star
+Kestra infinitely scalable orchestration and scheduling platform +Slack

- Website • - Twitter • - Linked In • - Slack • - Documentation + twitter   + linkedin   +youtube  


- -

modern data orchestration and scheduling platform

- +

+ + Get started in 4 minutes with Kestra + +

+

"Click on the image to get started in 4 minutes with Kestra."

## Live Demo @@ -113,17 +107,17 @@ tasks: concurrent: 3 tasks: - id: task1 - type: io.kestra.core.tasks.scripts.Bash + type: io.kestra.plugin.scripts.shell.Commands commands: - 'echo "running {{task.id}}"' - 'sleep 10' - id: task2 - type: io.kestra.core.tasks.scripts.Bash + type: io.kestra.plugin.scripts.shell.Commands commands: - 'echo "running {{task.id}}"' - 'sleep 10' - id: task3 - type: io.kestra.core.tasks.scripts.Bash + type: io.kestra.plugin.scripts.shell.Commands commands: - 'echo "running {{task.id}}"' - 'sleep 10' @@ -194,10 +188,10 @@ For more information: - Follow the [getting started tutorial](https://kestra.io/docs/getting-started/). - Read the [documentation](https://kestra.io/docs/) to understand how to: - - [Develop your flows](https://kestra.io/docs/developer-guide/) - - [Deploy Kestra](https://kestra.io/docs/administrator-guide/) - - Use our [Terraform provider](https://kestra.io/docs/terraform/) to deploy your flows - - Develop your [own plugins](https://kestra.io/docs/plugin-developer-guide/). + - [Develop your flows](https://kestra.io/docs/developer-guide/) + - [Deploy Kestra](https://kestra.io/docs/administrator-guide/) + - Use our [Terraform provider](https://kestra.io/docs/terraform/) to deploy your flows + - Develop your [own plugins](https://kestra.io/docs/plugin-developer-guide/). @@ -212,98 +206,143 @@ Here are some examples of the available plugins: - + + + + + + + + + + - + + + + + + + + + + + + + + + + + + - - + - + - - - + + + + + + + + - + + + + + + - + + + + + + + + + + + - + - + - - + + - + - - - + + + - - - + + + + - - + - + - + + - + + - - - - - + + + + -
AirbyteAirbyte CloudAirbyte OSSAmazon Athena
Amazon CLIAmazon DynamoDbAmazon Redshift
Amazon S3AvroAmazon SNSAmazon SQS
AMQPApache AvroApache Cassandra
Apache KafkaApache PinotApache Parquet
Apache PulsarApache SparkApache Tika
Azure Batch Azure Blob StorageBashBig QueryAzure Blob Table
CSVCassandra ClickHouseCompression
DBTDebezium MYSQLDebezium PostgresCouchbaseDatabricksdbt cloud
dbt core Debezium Microsoft SQL ServerDebezium MYSQL
Debezium Postgres DuckDb ElasticSearch
Fivetran EmailFivetran FTP
FTPSGitGoogle Big Query
Google Pub/Sub Google Cloud StorageGoogle DriveGoogle DataProc
Google FirestoreGoogle CliGoogle Vertex AI
Google Kubernetes EnginesGoogle Drive Google Sheets
Groovy HttpJSON
JSONJulia JythonKafkaKubernetes
KubernetesMQTT Microsoft SQL ServerMicrosoft TeamsMongoDb
MongoDbMQTT MySQL Nashorn
NodeOpen PGPOracleNATSNeo4jNode
ParquetApache PinotPostgresOpenAIOpen PGPOracle
PostgreSQL Power BIApache PulsarPythonPowerShell
RedshiftPython RocksetSFTPRScript
SFTP ServiceNow SingerSlack
ShellSlack SnowflakeSodaSpark
TikaTrinoVectorwiseSodaSSHTelegram
Trino XMLVertex AI Vertica
@@ -316,8 +355,8 @@ This list is growing quickly and we welcome contributions. If you need help or have any questions, reach out using one of the following channels: -- [GitHub discussions](https://github.com/kestra-io/kestra/discussions) - useful to start a conversation that is not a bug or feature request. - [Slack](https://kestra.io/slack) - join the community and get the latest updates. +- [GitHub discussions](https://github.com/kestra-io/kestra/discussions) - useful to start a conversation that is not a bug or feature request. - [Twitter](https://twitter.com/kestra_io) - to follow up with the latest updates. diff --git a/build.gradle b/build.gradle index 3e4832e055..f1842a6396 100644 --- a/build.gradle +++ b/build.gradle @@ -8,6 +8,7 @@ plugins { // test id 'com.adarshr.test-logger' version '3.2.0' + id "org.sonarqube" version "4.2.1.3168" // helper id "com.github.ben-manes.versions" version "0.47.0" @@ -69,7 +70,7 @@ allprojects { // configurations.all { resolutionStrategy { - force("org.jooq:jooq:3.16.10") + force("org.jooq:jooq:3.17.14") force("org.slf4j:slf4j-api:1.7.36") // ugly bug on google cloud plugins @@ -78,6 +79,21 @@ allprojects { // ugly bug for elastic plugins force("org.apache.httpcomponents:httpclient:4.5.13") + + // ugly bug on crypto plugin + force('org.bouncycastle:bcprov-jdk15on:1.70') + force('org.bouncycastle:bcpg-jdk15on:1.70') + + // ugly bug for jackson + force("com.fasterxml.jackson.core:jackson-bom:" + jacksonVersion) + force("com.fasterxml.jackson.core:jackson-core:" + jacksonVersion) + force("com.fasterxml.jackson.core:jackson-databind:" + jacksonVersion) + force("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:" + jacksonVersion) + force("com.fasterxml.jackson.module:jackson-module-parameter-names:" + jacksonVersion) + force("com.fasterxml.jackson.datatype:jackson-datatype-guava:" + jacksonVersion) + force("com.fasterxml.jackson.core:jackson-annotations:" + jacksonVersion) + force("com.fasterxml.jackson.dataformat:jackson-dataformat-smile:" + jacksonVersion) + force("com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:" + jacksonVersion) } } @@ -317,6 +333,17 @@ task jacoco(type: JacocoReport) { } } +/**********************************************************************************************************************\ + * Sonar + **********************************************************************************************************************/ +sonar { + properties { + property "sonar.projectKey", "kestra-io_kestra" + property "sonar.organization", "kestra-io" + property "sonar.host.url", "https://sonarcloud.io" + } +} + /**********************************************************************************************************************\ * Standalone **********************************************************************************************************************/ diff --git a/cli/src/main/java/io/kestra/cli/commands/servers/ExecutorCommand.java b/cli/src/main/java/io/kestra/cli/commands/servers/ExecutorCommand.java index fe93652b74..941177e6dc 100644 --- a/cli/src/main/java/io/kestra/cli/commands/servers/ExecutorCommand.java +++ b/cli/src/main/java/io/kestra/cli/commands/servers/ExecutorCommand.java @@ -3,12 +3,15 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.models.ServerType; import io.kestra.core.runners.ExecutorInterface; +import io.kestra.core.services.SkipExecutionService; import io.kestra.core.utils.Await; import io.micronaut.context.ApplicationContext; import jakarta.inject.Inject; import lombok.extern.slf4j.Slf4j; import picocli.CommandLine; +import java.util.Collections; +import java.util.List; import java.util.Map; @CommandLine.Command( @@ -20,6 +23,12 @@ public class ExecutorCommand extends AbstractServerCommand { @Inject private ApplicationContext applicationContext; + @Inject + private SkipExecutionService skipExecutionService; + + @CommandLine.Option(names = {"--skip-executions"}, split=",", description = "a list of execution identifiers to skip, separated by a coma; for troubleshooting purpose only") + private List skipExecutions = Collections.emptyList(); + @SuppressWarnings("unused") public static Map propertiesOverrides() { return ImmutableMap.of( @@ -29,6 +38,8 @@ public static Map propertiesOverrides() { @Override public Integer call() throws Exception { + this.skipExecutionService.setSkipExecutions(skipExecutions); + super.call(); ExecutorInterface executorService = applicationContext.getBean(ExecutorInterface.class); diff --git a/cli/src/main/java/io/kestra/cli/commands/servers/StandAloneCommand.java b/cli/src/main/java/io/kestra/cli/commands/servers/StandAloneCommand.java index 3f581514fc..35165f2bc4 100644 --- a/cli/src/main/java/io/kestra/cli/commands/servers/StandAloneCommand.java +++ b/cli/src/main/java/io/kestra/cli/commands/servers/StandAloneCommand.java @@ -4,6 +4,7 @@ import io.kestra.core.models.ServerType; import io.kestra.core.repositories.LocalFlowRepositoryLoader; import io.kestra.core.runners.StandAloneRunner; +import io.kestra.core.services.SkipExecutionService; import io.kestra.core.utils.Await; import io.micronaut.context.ApplicationContext; import jakarta.inject.Inject; @@ -12,6 +13,8 @@ import java.io.File; import java.io.IOException; +import java.util.Collections; +import java.util.List; import java.util.Map; @CommandLine.Command( @@ -26,12 +29,18 @@ public class StandAloneCommand extends AbstractServerCommand { @Inject private ApplicationContext applicationContext; + @Inject + private SkipExecutionService skipExecutionService; + @CommandLine.Option(names = {"-f", "--flow-path"}, description = "the flow path containing flow to inject at startup (when running with a memory flow repository)") private File flowPath; @CommandLine.Option(names = {"--worker-thread"}, description = "the number of worker thread") private Integer workerThread; + @CommandLine.Option(names = {"--skip-executions"}, split=",", description = "a list of execution identifiers to skip, separated by a coma; for troubleshooting purpose only") + private List skipExecutions = Collections.emptyList(); + @SuppressWarnings("unused") public static Map propertiesOverrides() { return ImmutableMap.of( @@ -41,6 +50,8 @@ public static Map propertiesOverrides() { @Override public Integer call() throws Exception { + this.skipExecutionService.setSkipExecutions(skipExecutions); + super.call(); if (flowPath != null) { diff --git a/cli/src/main/java/io/kestra/cli/commands/sys/ReindexCommand.java b/cli/src/main/java/io/kestra/cli/commands/sys/ReindexCommand.java new file mode 100644 index 0000000000..708012572d --- /dev/null +++ b/cli/src/main/java/io/kestra/cli/commands/sys/ReindexCommand.java @@ -0,0 +1,51 @@ +package io.kestra.cli.commands.sys; + +import io.kestra.cli.AbstractCommand; +import io.kestra.cli.App; +import io.kestra.core.models.flows.FlowWithSource; +import io.kestra.core.repositories.FlowRepositoryInterface; +import io.kestra.core.services.TaskDefaultService; +import io.micronaut.configuration.picocli.PicocliRunner; +import io.micronaut.context.ApplicationContext; +import jakarta.inject.Inject; +import lombok.extern.slf4j.Slf4j; +import picocli.CommandLine; + +import java.util.List; + +@CommandLine.Command( + name = "reindex", + description = "reindex all records of a type: read them from the database then update them", + mixinStandardHelpOptions = true, + subcommands = { + RestoreQueueCommand.class, + FlowListenersRestoreCommand.class + } +) +@Slf4j +public class ReindexCommand extends AbstractCommand { + @Inject + private ApplicationContext applicationContext; + + @CommandLine.Option(names = {"-t", "--type"}, description = "The type of the records to reindex, only 'flow' is supported for now.") + private String type; + + @Override + public Integer call() throws Exception { + super.call(); + + if ("flow".equals(type)) { + FlowRepositoryInterface flowRepository = applicationContext.getBean(FlowRepositoryInterface.class); + + List flows = flowRepository.findWithSource(null, null, null); + flows.forEach(flow -> flowRepository.update(flow.toFlow(), flow.toFlow(), flow.getSource(), flow.toFlow())); + + stdOut("Successfully reindex " + flows.size() + " flow(s)."); + } + else { + throw new IllegalArgumentException("Reindexing type '" + type + "' is not supported"); + } + + return 0; + } +} diff --git a/cli/src/main/java/io/kestra/cli/commands/sys/RestoreQueueCommand.java b/cli/src/main/java/io/kestra/cli/commands/sys/RestoreQueueCommand.java index 6889151c52..27f8ef7b64 100644 --- a/cli/src/main/java/io/kestra/cli/commands/sys/RestoreQueueCommand.java +++ b/cli/src/main/java/io/kestra/cli/commands/sys/RestoreQueueCommand.java @@ -2,6 +2,7 @@ import io.kestra.cli.AbstractCommand; import io.kestra.cli.services.RestoreQueueService; +import io.kestra.core.repositories.TemplateRepositoryInterface; import io.micronaut.context.ApplicationContext; import jakarta.inject.Inject; import lombok.extern.slf4j.Slf4j; @@ -46,7 +47,7 @@ public Integer call() throws Exception { } // templates - if (!this.noTemplates) { + if (!this.noTemplates && applicationContext.containsBean(TemplateRepositoryInterface.class)) { int size = restoreQueueService.templates(noRecreate); stdOut("Successfully send {0} templates", size); } diff --git a/cli/src/main/java/io/kestra/cli/commands/sys/SysCommand.java b/cli/src/main/java/io/kestra/cli/commands/sys/SysCommand.java index 92e977fd6a..ee92c2ddb6 100644 --- a/cli/src/main/java/io/kestra/cli/commands/sys/SysCommand.java +++ b/cli/src/main/java/io/kestra/cli/commands/sys/SysCommand.java @@ -12,7 +12,8 @@ mixinStandardHelpOptions = true, subcommands = { RestoreQueueCommand.class, - FlowListenersRestoreCommand.class + FlowListenersRestoreCommand.class, + ReindexCommand.class } ) @Slf4j diff --git a/cli/src/main/java/io/kestra/cli/commands/templates/TemplateCommand.java b/cli/src/main/java/io/kestra/cli/commands/templates/TemplateCommand.java index b65362b031..8685ed08df 100644 --- a/cli/src/main/java/io/kestra/cli/commands/templates/TemplateCommand.java +++ b/cli/src/main/java/io/kestra/cli/commands/templates/TemplateCommand.java @@ -3,6 +3,7 @@ import io.kestra.cli.AbstractCommand; import io.kestra.cli.App; import io.kestra.cli.commands.templates.namespaces.TemplateNamespaceCommand; +import io.kestra.core.models.templates.TemplateEnabled; import io.micronaut.configuration.picocli.PicocliRunner; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -19,6 +20,7 @@ } ) @Slf4j +@TemplateEnabled public class TemplateCommand extends AbstractCommand { @SneakyThrows @Override diff --git a/cli/src/main/java/io/kestra/cli/commands/templates/TemplateExportCommand.java b/cli/src/main/java/io/kestra/cli/commands/templates/TemplateExportCommand.java index 68d7f471e6..6a688a2f40 100644 --- a/cli/src/main/java/io/kestra/cli/commands/templates/TemplateExportCommand.java +++ b/cli/src/main/java/io/kestra/cli/commands/templates/TemplateExportCommand.java @@ -1,6 +1,7 @@ package io.kestra.cli.commands.templates; import io.kestra.cli.AbstractApiCommand; +import io.kestra.core.models.templates.TemplateEnabled; import io.micronaut.http.HttpRequest; import io.micronaut.http.HttpResponse; import io.micronaut.http.MediaType; @@ -19,6 +20,7 @@ mixinStandardHelpOptions = true ) @Slf4j +@TemplateEnabled public class TemplateExportCommand extends AbstractApiCommand { private static final String DEFAULT_FILE_NAME = "templates.zip"; diff --git a/cli/src/main/java/io/kestra/cli/commands/templates/TemplateValidateCommand.java b/cli/src/main/java/io/kestra/cli/commands/templates/TemplateValidateCommand.java index 069ef7ce4f..8b12e6d654 100644 --- a/cli/src/main/java/io/kestra/cli/commands/templates/TemplateValidateCommand.java +++ b/cli/src/main/java/io/kestra/cli/commands/templates/TemplateValidateCommand.java @@ -2,6 +2,7 @@ import io.kestra.cli.AbstractValidateCommand; import io.kestra.core.models.templates.Template; +import io.kestra.core.models.templates.TemplateEnabled; import io.kestra.core.models.validations.ModelValidator; import io.kestra.core.serializers.YamlFlowParser; import jakarta.inject.Inject; @@ -11,6 +12,7 @@ name = "validate", description = "validate a template" ) +@TemplateEnabled public class TemplateValidateCommand extends AbstractValidateCommand { @Inject private YamlFlowParser yamlFlowParser; diff --git a/cli/src/main/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceCommand.java b/cli/src/main/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceCommand.java index 8e9afb423e..7edebd902a 100644 --- a/cli/src/main/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceCommand.java +++ b/cli/src/main/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceCommand.java @@ -1,6 +1,7 @@ package io.kestra.cli.commands.templates.namespaces; import io.kestra.cli.AbstractCommand; +import io.kestra.core.models.templates.TemplateEnabled; import lombok.extern.slf4j.Slf4j; import picocli.CommandLine; @@ -13,6 +14,7 @@ } ) @Slf4j +@TemplateEnabled public class TemplateNamespaceCommand extends AbstractCommand { } diff --git a/cli/src/main/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceUpdateCommand.java b/cli/src/main/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceUpdateCommand.java index a5c9126323..630e032ecd 100644 --- a/cli/src/main/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceUpdateCommand.java +++ b/cli/src/main/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceUpdateCommand.java @@ -3,6 +3,7 @@ import io.kestra.cli.commands.AbstractServiceNamespaceUpdateCommand; import io.kestra.cli.commands.templates.TemplateValidateCommand; import io.kestra.core.models.templates.Template; +import io.kestra.core.models.templates.TemplateEnabled; import io.kestra.core.serializers.YamlFlowParser; import io.micronaut.core.type.Argument; import io.micronaut.http.HttpRequest; @@ -13,10 +14,10 @@ import lombok.extern.slf4j.Slf4j; import picocli.CommandLine; -import javax.validation.ConstraintViolationException; import java.nio.file.Files; import java.util.List; import java.util.stream.Collectors; +import javax.validation.ConstraintViolationException; @CommandLine.Command( name = "update", @@ -24,6 +25,7 @@ mixinStandardHelpOptions = true ) @Slf4j +@TemplateEnabled public class TemplateNamespaceUpdateCommand extends AbstractServiceNamespaceUpdateCommand { @Inject public YamlFlowParser yamlFlowParser; diff --git a/cli/src/test/java/io/kestra/cli/commands/sys/FlowListenersRestoreCommandTest.java b/cli/src/test/java/io/kestra/cli/commands/sys/FlowListenersRestoreCommandTest.java index 55ead53e72..80556b642e 100644 --- a/cli/src/test/java/io/kestra/cli/commands/sys/FlowListenersRestoreCommandTest.java +++ b/cli/src/test/java/io/kestra/cli/commands/sys/FlowListenersRestoreCommandTest.java @@ -6,9 +6,9 @@ import io.micronaut.context.ApplicationContext; import io.micronaut.context.env.Environment; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; import io.kestra.core.contexts.KestraClassLoader; import io.kestra.core.repositories.FlowRepositoryInterface; +import org.junitpioneer.jupiter.RetryingTest; import java.io.ByteArrayOutputStream; import java.io.PrintStream; @@ -26,7 +26,7 @@ static void init() { } } - @Test + @RetryingTest(5) void run() throws InterruptedException { final int COUNT = 5; diff --git a/cli/src/test/java/io/kestra/cli/commands/sys/ReindexCommandTest.java b/cli/src/test/java/io/kestra/cli/commands/sys/ReindexCommandTest.java new file mode 100644 index 0000000000..fa3f8a1093 --- /dev/null +++ b/cli/src/test/java/io/kestra/cli/commands/sys/ReindexCommandTest.java @@ -0,0 +1,51 @@ +package io.kestra.cli.commands.sys; + +import io.kestra.cli.commands.flows.namespaces.FlowNamespaceUpdateCommand; +import io.micronaut.configuration.picocli.PicocliRunner; +import io.micronaut.context.ApplicationContext; +import io.micronaut.context.env.Environment; +import io.micronaut.runtime.server.EmbeddedServer; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.net.URL; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.StringContains.containsString; + +class ReindexCommandTest { + @Test + void reindexFlow() { + URL directory = ReindexCommandTest.class.getClassLoader().getResource("flows"); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + System.setOut(new PrintStream(out)); + + try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) { + EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class); + embeddedServer.start(); + + // we use the update command to add flows to extract + String[] updateArgs = { + "--server", + embeddedServer.getURL().toString(), + "--user", + "myuser:pass:word", + "io.kestra.cli", + directory.getPath(), + }; + PicocliRunner.call(FlowNamespaceUpdateCommand.class, ctx, updateArgs); + assertThat(out.toString(), containsString("3 flow(s)")); + + // then we reindex them + String[] reindexArgs = { + "--type", + "flow", + }; + Integer call = PicocliRunner.call(ReindexCommand.class, ctx, reindexArgs); + assertThat(call, is(0)); + assertThat(out.toString(), containsString("Successfully reindex 3 flow(s).")); + } + } +} \ No newline at end of file diff --git a/cli/src/test/java/io/kestra/cli/commands/templates/TemplateExportCommandTest.java b/cli/src/test/java/io/kestra/cli/commands/templates/TemplateExportCommandTest.java index a6bde6d2af..34b1d53d4d 100644 --- a/cli/src/test/java/io/kestra/cli/commands/templates/TemplateExportCommandTest.java +++ b/cli/src/test/java/io/kestra/cli/commands/templates/TemplateExportCommandTest.java @@ -12,6 +12,7 @@ import java.io.IOException; import java.io.PrintStream; import java.net.URL; +import java.util.Map; import java.util.zip.ZipFile; import static org.hamcrest.MatcherAssert.assertThat; @@ -25,7 +26,7 @@ void run() throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); System.setOut(new PrintStream(out)); - try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) { + try (ApplicationContext ctx = ApplicationContext.run(Map.of("kestra.templates.enabled", "true"), Environment.CLI, Environment.TEST)) { EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class); embeddedServer.start(); diff --git a/cli/src/test/java/io/kestra/cli/commands/templates/TemplateValidateCommandTest.java b/cli/src/test/java/io/kestra/cli/commands/templates/TemplateValidateCommandTest.java index af6fdc240c..0662594030 100644 --- a/cli/src/test/java/io/kestra/cli/commands/templates/TemplateValidateCommandTest.java +++ b/cli/src/test/java/io/kestra/cli/commands/templates/TemplateValidateCommandTest.java @@ -9,6 +9,7 @@ import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.net.URL; +import java.util.Map; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.Is.is; @@ -21,7 +22,7 @@ void runLocal() { ByteArrayOutputStream out = new ByteArrayOutputStream(); System.setErr(new PrintStream(out)); - try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) { + try (ApplicationContext ctx = ApplicationContext.run(Map.of("kestra.templates.enabled", "true"), Environment.CLI, Environment.TEST)) { String[] args = { "--local", directory.getPath() @@ -40,7 +41,7 @@ void runServer() { ByteArrayOutputStream out = new ByteArrayOutputStream(); System.setErr(new PrintStream(out)); - try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) { + try (ApplicationContext ctx = ApplicationContext.run(Map.of("kestra.templates.enabled", "true"), Environment.CLI, Environment.TEST)) { EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class); embeddedServer.start(); diff --git a/cli/src/test/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceUpdateCommandTest.java b/cli/src/test/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceUpdateCommandTest.java index 050df79c37..7ce8b0b7ff 100644 --- a/cli/src/test/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceUpdateCommandTest.java +++ b/cli/src/test/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceUpdateCommandTest.java @@ -9,6 +9,7 @@ import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.net.URL; +import java.util.Map; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.StringContains.containsString; @@ -20,7 +21,7 @@ void run() { ByteArrayOutputStream out = new ByteArrayOutputStream(); System.setOut(new PrintStream(out)); - try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) { + try (ApplicationContext ctx = ApplicationContext.run(Map.of("kestra.templates.enabled", "true"), Environment.CLI, Environment.TEST)) { EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class); embeddedServer.start(); @@ -46,7 +47,7 @@ void invalid() { ByteArrayOutputStream out = new ByteArrayOutputStream(); System.setErr(new PrintStream(out)); - try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) { + try (ApplicationContext ctx = ApplicationContext.run(Map.of("kestra.templates.enabled", "true"), Environment.CLI, Environment.TEST)) { EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class); embeddedServer.start(); @@ -76,7 +77,7 @@ void runNoDelete() { ByteArrayOutputStream out = new ByteArrayOutputStream(); System.setOut(new PrintStream(out)); - try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) { + try (ApplicationContext ctx = ApplicationContext.run(Map.of("kestra.templates.enabled", "true"), Environment.CLI, Environment.TEST)) { EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class); embeddedServer.start(); diff --git a/cli/src/test/resources/templates/template-2.yml b/cli/src/test/resources/templates/template-2.yml index a4f5777a95..b72bf36b2b 100644 --- a/cli/src/test/resources/templates/template-2.yml +++ b/cli/src/test/resources/templates/template-2.yml @@ -2,7 +2,6 @@ id: template-2 namespace: io.kestra.tests tasks: - id: "bash" - type: "io.kestra.core.tasks.scripts.Bash" - commands: - - 'echo "The current execution is : {{execution.id}}"' + type: io.kestra.core.tasks.log.Log + message: "The current execution is : {{execution.id}}" diff --git a/cli/src/test/resources/templates/templatesSubFolder/template-3.yml b/cli/src/test/resources/templates/templatesSubFolder/template-3.yml index d41b80a441..7b631c6e4c 100644 --- a/cli/src/test/resources/templates/templatesSubFolder/template-3.yml +++ b/cli/src/test/resources/templates/templatesSubFolder/template-3.yml @@ -2,7 +2,6 @@ id: template-3 namespace: io.kestra.tests tasks: - id: "bash" - type: "io.kestra.core.tasks.scripts.Bash" - commands: - - 'echo "The current execution is : {{execution.id}}"' + type: "io.kestra.core.tasks.log.Log" + message: "The current execution is : {{execution.id}}" diff --git a/core/build.gradle b/core/build.gradle index cdff57aab5..02d60d9419 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -26,12 +26,6 @@ dependencies { // scheduler implementation group: 'com.cronutils', name: 'cron-utils', version: '9.2.1' - // process - implementation ('com.github.docker-java:docker-java:3.3.1') { - exclude group: 'com.github.docker-java', module: 'docker-java-transport-jersey' - } - implementation 'com.github.docker-java:docker-java-transport-httpclient5:3.3.1' - // schema implementation group: 'com.github.victools', name: 'jsonschema-generator', version: '4.31.1' implementation group: 'com.github.victools', name: 'jsonschema-module-javax-validation', version: '4.31.1' diff --git a/core/src/main/java/io/kestra/core/docs/JsonSchemaGenerator.java b/core/src/main/java/io/kestra/core/docs/JsonSchemaGenerator.java index 5e381441cc..3d90b70cb7 100644 --- a/core/src/main/java/io/kestra/core/docs/JsonSchemaGenerator.java +++ b/core/src/main/java/io/kestra/core/docs/JsonSchemaGenerator.java @@ -23,6 +23,7 @@ import io.kestra.core.models.tasks.Output; import io.kestra.core.models.tasks.Task; import io.kestra.core.models.triggers.AbstractTrigger; +import io.kestra.core.plugins.RegisteredPlugin; import io.kestra.core.serializers.JacksonMapper; import io.kestra.core.services.PluginService; import io.micronaut.core.annotation.Nullable; @@ -304,29 +305,25 @@ public CustomDefinition provideCustomSchemaDefinition(ResolvedType javaType, Sch TypeContext typeContext = context.getTypeContext(); if (declaredType.getErasedType() == Task.class) { - return pluginService - .allPlugins() + return getRegisteredPlugins() .stream() .flatMap(registeredPlugin -> registeredPlugin.getTasks().stream()) .map(clz -> typeContext.resolveSubtype(declaredType, clz)) .collect(Collectors.toList()); } else if (declaredType.getErasedType() == AbstractTrigger.class) { - return pluginService - .allPlugins() + return getRegisteredPlugins() .stream() .flatMap(registeredPlugin -> registeredPlugin.getTriggers().stream()) .map(clz -> typeContext.resolveSubtype(declaredType, clz)) .collect(Collectors.toList()); } else if (declaredType.getErasedType() == Condition.class) { - return pluginService - .allPlugins() + return getRegisteredPlugins() .stream() .flatMap(registeredPlugin -> registeredPlugin.getConditions().stream()) .map(clz -> typeContext.resolveSubtype(declaredType, clz)) .collect(Collectors.toList()); } else if (declaredType.getErasedType() == ScheduleCondition.class) { - return pluginService - .allPlugins() + return getRegisteredPlugins() .stream() .flatMap(registeredPlugin -> registeredPlugin.getConditions().stream()) .filter(ScheduleCondition.class::isAssignableFrom) @@ -404,6 +401,11 @@ public CustomDefinition provideCustomSchemaDefinition(ResolvedType javaType, Sch } } + protected List getRegisteredPlugins() { + return pluginService + .allPlugins(); + } + private boolean defaultInAllOf(JsonNode property) { if (property.has("allOf")) { for (Iterator it = property.get("allOf").elements(); it.hasNext(); ) { @@ -424,10 +426,10 @@ protected Map generate(Class cls, @Nullable Cla this.build(builder,false); - // base is passed, we don't return base properties + // we don't return base properties unless specified with @PluginProperty builder .forFields() - .withIgnoreCheck(fieldScope -> base != null && fieldScope.getDeclaringType().getTypeName().equals(base.getName())); + .withIgnoreCheck(fieldScope -> base != null && fieldScope.getAnnotation(PluginProperty.class) == null && fieldScope.getDeclaringType().getTypeName().equals(base.getName())); SchemaGeneratorConfig schemaGeneratorConfig = builder.build(); diff --git a/core/src/main/java/io/kestra/core/models/Label.java b/core/src/main/java/io/kestra/core/models/Label.java index 290c2631da..91418c3299 100644 --- a/core/src/main/java/io/kestra/core/models/Label.java +++ b/core/src/main/java/io/kestra/core/models/Label.java @@ -1,3 +1,5 @@ package io.kestra.core.models; -public record Label(String key, String value) {} +import javax.validation.constraints.NotNull; + +public record Label(@NotNull String key, @NotNull String value) {} diff --git a/core/src/main/java/io/kestra/core/models/conditions/types/MultipleCondition.java b/core/src/main/java/io/kestra/core/models/conditions/types/MultipleCondition.java index 13d92d4767..212a1d7454 100644 --- a/core/src/main/java/io/kestra/core/models/conditions/types/MultipleCondition.java +++ b/core/src/main/java/io/kestra/core/models/conditions/types/MultipleCondition.java @@ -47,13 +47,12 @@ title = "A flow that is waiting for 2 flows that is successful in 1 days", code = { "triggers:", - " - id: flow-status-conditions", - " type: io.kestra.core.models.conditions.types.ExecutionStatusCondition", - " in:", - " - SUCCESS", " - id: multiple-listen-flow", " type: io.kestra.core.models.triggers.types.Flow", " conditions:", + " - type: io.kestra.core.models.conditions.types.ExecutionStatusCondition", + " in:", + " - SUCCESS", " - id: multiple", " type: io.kestra.core.models.conditions.types.MultipleCondition", " window: P1D", diff --git a/core/src/main/java/io/kestra/core/models/executions/AbstractMetricEntry.java b/core/src/main/java/io/kestra/core/models/executions/AbstractMetricEntry.java index c7647246b9..181bcf2a6d 100644 --- a/core/src/main/java/io/kestra/core/models/executions/AbstractMetricEntry.java +++ b/core/src/main/java/io/kestra/core/models/executions/AbstractMetricEntry.java @@ -13,7 +13,9 @@ import lombok.ToString; import java.time.Instant; +import java.util.Collection; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.validation.constraints.NotNull; @@ -63,9 +65,9 @@ private static Map tagsAsMap(String... keyValues) { protected String[] tagsAsArray(Map others) { return Stream.concat( - this.tags.entrySet().stream(), - others.entrySet().stream() - ) + Optional.ofNullable(this.tags).map(Map::entrySet).stream().flatMap(Collection::stream), + others.entrySet().stream() + ) .flatMap(e -> Stream.of(e.getKey(), e.getValue())) .collect(Collectors.toList()) .toArray(String[]::new); diff --git a/core/src/main/java/io/kestra/core/models/flows/Flow.java b/core/src/main/java/io/kestra/core/models/flows/Flow.java index e0a5237adf..bde1b57026 100644 --- a/core/src/main/java/io/kestra/core/models/flows/Flow.java +++ b/core/src/main/java/io/kestra/core/models/flows/Flow.java @@ -3,10 +3,13 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.fasterxml.jackson.databind.introspect.AnnotatedMember; import com.fasterxml.jackson.databind.introspect.JacksonAnnotationIntrospector; import io.kestra.core.exceptions.InternalException; import io.kestra.core.models.DeletedInterface; +import io.kestra.core.models.Label; import io.kestra.core.models.executions.Execution; import io.kestra.core.models.listeners.Listener; import io.kestra.core.models.tasks.FlowableTask; @@ -14,6 +17,8 @@ import io.kestra.core.models.triggers.AbstractTrigger; import io.kestra.core.models.validations.ManualConstraintViolation; import io.kestra.core.serializers.JacksonMapper; +import io.kestra.core.serializers.ListOrMapOfLabelDeserializer; +import io.kestra.core.serializers.ListOrMapOfLabelSerializer; import io.kestra.core.services.FlowService; import io.kestra.core.validations.FlowValidation; import io.micronaut.core.annotation.Introspected; @@ -63,7 +68,10 @@ public boolean hasIgnoreMarker(final AnnotatedMember m) { String description; - Map labels; + @JsonSerialize(using = ListOrMapOfLabelSerializer.class) + @JsonDeserialize(using = ListOrMapOfLabelDeserializer.class) + List