diff --git a/element-template-generator/uniquet/README.md b/element-template-generator/uniquet/README.md new file mode 100644 index 0000000000..2f0452fb81 --- /dev/null +++ b/element-template-generator/uniquet/README.md @@ -0,0 +1,37 @@ +# Uniquet + +`uniquet` goes through a github repository checking for all different versions of all elements-template and provides a single file containing references to all versions + +## Installation + +Currently, the only way to install `uniquet` is to build it from source. + +### Local build + +To build `uniquet` locally, check out the repository and build the project with Maven: + +```shell +mvn install -pl element-template-generator/uniquet -am +``` + +This will build the `uniquet` module and all its dependencies. + +Navigate to the `element-template-generator/uniquet/target/appassembler` directory. +The executable `uniquet` script is located in the `bin` directory. The compiled Java code required for +running `uniquet` is located in the `repo` directory. Make sure to copy both directories to your application installation directory. + +Executables for Windows and Unix systems are provided (`.bat` and Shell scripts, respectively). + +## Usage + +`uniquet` has to be run at root of a git repository, It will crawl through every files inside `element-templates` directories for every commit and compile latest version of each of them in a single file + +### Examples + +`--branch` or `-b` is the branch one wants to start from. not required, default `main` +`--destination` or `-d` is the location where the file will be generated. +`--git-repository` or `-g` is the location of the git repository. + +```shell +uniquet --destination ~/Desktop/singlefile.json --branch main +``` diff --git a/element-template-generator/uniquet/pom.xml b/element-template-generator/uniquet/pom.xml new file mode 100644 index 0000000000..63d3efd9f3 --- /dev/null +++ b/element-template-generator/uniquet/pom.xml @@ -0,0 +1,104 @@ + + + 4.0.0 + + + io.camunda.connector + connector-parent + ../../parent/pom.xml + 8.6.0-SNAPSHOT + + + Uniquet + Generate a single ET file + uniquet + jar + + + 21 + 21 + 4.7.6 + UTF-8 + + + + + info.picocli + picocli + ${version.picocli} + + + + io.camunda.connector + jackson-datatype-feel + + + + org.eclipse.jgit + org.eclipse.jgit + 6.10.0.202406032230-r + + + + org.slf4j + slf4j-nop + + + + org.junit.jupiter + junit-jupiter + test + + + commons-codec + commons-codec + test + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + + info.picocli + picocli-codegen + 4.7.6 + + + + -Aproject=${project.groupId}/${project.artifactId} + + + + + org.codehaus.mojo + appassembler-maven-plugin + 2.1.0 + + + package + + assemble + + + + + + + io.camunda.connector.uniquet.Main + uniquet + + + + + + + \ No newline at end of file diff --git a/element-template-generator/uniquet/src/main/java/io/camunda/connector/uniquet/Main.java b/element-template-generator/uniquet/src/main/java/io/camunda/connector/uniquet/Main.java new file mode 100644 index 0000000000..5268cdbc2a --- /dev/null +++ b/element-template-generator/uniquet/src/main/java/io/camunda/connector/uniquet/Main.java @@ -0,0 +1,30 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information regarding copyright + * ownership. Camunda licenses this file to you under the Apache License, + * Version 2.0; you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.camunda.connector.uniquet; + +import io.camunda.connector.uniquet.command.UniquetCommand; +import picocli.CommandLine; + +public class Main { + public static void main(String[] args) { + int exitCode = + new CommandLine(new UniquetCommand()) + .setUnmatchedOptionsArePositionalParams(true) + .execute(args); + System.exit(exitCode); + } +} diff --git a/element-template-generator/uniquet/src/main/java/io/camunda/connector/uniquet/command/UniquetCommand.java b/element-template-generator/uniquet/src/main/java/io/camunda/connector/uniquet/command/UniquetCommand.java new file mode 100644 index 0000000000..7d27ebe924 --- /dev/null +++ b/element-template-generator/uniquet/src/main/java/io/camunda/connector/uniquet/command/UniquetCommand.java @@ -0,0 +1,51 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information regarding copyright + * ownership. Camunda licenses this file to you under the Apache License, + * Version 2.0; you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.camunda.connector.uniquet.command; + +import io.camunda.connector.uniquet.core.GitCrawler; +import java.util.concurrent.Callable; +import picocli.CommandLine; + +public class UniquetCommand implements Callable { + + @CommandLine.Option( + names = {"-b", "--branch"}, + defaultValue = "main") + private String branch; + + @CommandLine.Option( + names = {"-d", "--destination"}, + required = true) + private String pathDestination; + + @CommandLine.Option( + names = {"-g", "--git-directory"}, + defaultValue = "") + private String gitDirectory; + + @Override + public Integer call() { + try { + GitCrawler gitCrawler = GitCrawler.create(gitDirectory); + gitCrawler.crawl(branch).persist(pathDestination).close(); + } catch (RuntimeException e) { + System.err.println(e.getMessage()); + return 1; + } + return 0; + } +} diff --git a/element-template-generator/uniquet/src/main/java/io/camunda/connector/uniquet/core/ElementTemplateIterator.java b/element-template-generator/uniquet/src/main/java/io/camunda/connector/uniquet/core/ElementTemplateIterator.java new file mode 100644 index 0000000000..416236bab5 --- /dev/null +++ b/element-template-generator/uniquet/src/main/java/io/camunda/connector/uniquet/core/ElementTemplateIterator.java @@ -0,0 +1,110 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information regarding copyright + * ownership. Camunda licenses this file to you under the Apache License, + * Version 2.0; you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.camunda.connector.uniquet.core; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.camunda.connector.uniquet.dto.ElementTemplate; +import io.camunda.connector.uniquet.dto.ElementTemplateFile; +import java.io.IOException; +import java.util.Iterator; +import java.util.Objects; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.treewalk.TreeWalk; +import org.eclipse.jgit.treewalk.filter.PathFilter; + +public class ElementTemplateIterator implements Iterator { + + private final RevCommit commit; + private final ObjectMapper objectMapper; + private final Repository repository; + private final TreeWalk initialWalk; + private ElementTemplateFile elementTemplate; + private TreeWalk currentWalk; + private String currentFolderBeingAnalyzed; + + public ElementTemplateIterator(Repository repository, RevCommit commit) { + this.commit = commit; + this.repository = repository; + this.objectMapper = new ObjectMapper(); + try { + TreeWalk treeWalk = new TreeWalk(repository); + treeWalk.addTree(this.commit.getTree()); + treeWalk.setRecursive(false); + treeWalk.setFilter(PathFilter.create("connectors")); + this.initialWalk = treeWalk; + this.initialWalk.next(); + } catch (IOException e) { + throw new RuntimeException(e); + } + this.elementTemplate = this.prepareNext(); + } + + @Override + public boolean hasNext() { + return this.elementTemplate != null; + } + + @Override + public ElementTemplateFile next() { + ElementTemplateFile elementTemplate = this.elementTemplate; + this.elementTemplate = this.prepareNext(); + return elementTemplate; + } + + private ElementTemplateFile prepareNext() { + try { + do { + if (this.initialWalk.isSubtree() + && !this.initialWalk.getPathString().endsWith("element-templates")) { + this.initialWalk.enterSubtree(); + } else if (this.initialWalk.getPathString().endsWith("element-templates")) { + if (!Objects.equals(this.currentFolderBeingAnalyzed, this.initialWalk.getPathString())) { + this.currentFolderBeingAnalyzed = this.initialWalk.getPathString(); + this.currentWalk = new TreeWalk(repository); + this.currentWalk.addTree(this.commit.getTree()); + this.currentWalk.setRecursive(true); + this.currentWalk.setFilter(PathFilter.create(this.currentFolderBeingAnalyzed)); + } + while (this.currentWalk.next()) { + if (!this.currentWalk.getPathString().endsWith(".json")) continue; + if (this.currentWalk.getPathString().contains("/hybrid/")) continue; + ObjectId objectId = this.currentWalk.getObjectId(0); + ObjectLoader loader = this.repository.open(objectId); + byte[] bytes = loader.getBytes(); + try { + return new ElementTemplateFile( + objectMapper.readValue(bytes, ElementTemplate.class), + this.currentWalk.getPathString()); + } catch (IOException e) { + System.err.println( + "Error while reading element-template: " + + this.currentWalk.getPathString() + + ". Commit is: " + + commit.getName()); + } + } + } + } while (this.initialWalk.next()); + } catch (Exception e) { + throw new RuntimeException(e); + } + return null; + } +} diff --git a/element-template-generator/uniquet/src/main/java/io/camunda/connector/uniquet/core/GitCrawler.java b/element-template-generator/uniquet/src/main/java/io/camunda/connector/uniquet/core/GitCrawler.java new file mode 100644 index 0000000000..86171418e5 --- /dev/null +++ b/element-template-generator/uniquet/src/main/java/io/camunda/connector/uniquet/core/GitCrawler.java @@ -0,0 +1,125 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information regarding copyright + * ownership. Camunda licenses this file to you under the Apache License, + * Version 2.0; you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.camunda.connector.uniquet.core; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.camunda.connector.uniquet.dto.OutputElementTemplate; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.storage.file.FileRepositoryBuilder; + +public class GitCrawler { + + private static final String RAW_GITHUB_LINK = + "https://raw.githubusercontent.com/camunda/connectors/%s/%s"; + private final Map> result = new HashMap<>(); + private final Repository repository; + + public GitCrawler(Repository repository) { + this.repository = repository; + } + + public static GitCrawler create(String gitDirectory) { + if (!gitDirectory.endsWith(File.separator) && !gitDirectory.isEmpty()) { + gitDirectory = gitDirectory + File.separator; + } + try { + Repository repository = FileRepositoryBuilder.create(new File(gitDirectory + ".git")); + return new GitCrawler(repository); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public Map> getResult() { + return result; + } + + public GitCrawler crawl(String branch) { + try { + RevWalk walk = new RevWalk(repository); + ObjectId mainBranch = repository.resolve("refs/heads/%s".formatted(branch)); + walk.markStart(walk.parseCommit(mainBranch)); + for (RevCommit commit : walk) { + this.analyzeCommit(commit); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + private void analyzeCommit(RevCommit commit) { + new ElementTemplateIterator(repository, commit) + .forEachRemaining( + elementTemplateFile -> { + if (result.containsKey(elementTemplateFile.elementTemplate().id())) { + result + .get(elementTemplateFile.elementTemplate().id()) + .putIfAbsent( + elementTemplateFile.elementTemplate().version(), + RAW_GITHUB_LINK.formatted(commit.getName(), elementTemplateFile.path())); + } else { + Map version = new HashMap<>(); + version.put( + elementTemplateFile.elementTemplate().version(), + RAW_GITHUB_LINK.formatted(commit.getName(), elementTemplateFile.path())); + result.put(elementTemplateFile.elementTemplate().id(), version); + } + }); + } + + public GitCrawler persist(String location) { + + try (FileWriter myWriter = new FileWriter(location)) { + myWriter.write(new ObjectMapper().writeValueAsString(fromMap(this.result))); + } catch (Exception e) { + throw new RuntimeException(e); + } + return this; + } + + private Map> fromMap( + Map> result) { + return result.entrySet().stream() + .map( + stringMapEntry -> + Map.entry( + stringMapEntry.getKey(), + stringMapEntry.getValue().entrySet().stream() + .map( + integerStringEntry -> + new OutputElementTemplate( + integerStringEntry.getKey(), integerStringEntry.getValue())) + .sorted((o1, o2) -> o2.version() - o1.version()) + .toList())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + public void close() { + repository.close(); + } +} diff --git a/element-template-generator/uniquet/src/main/java/io/camunda/connector/uniquet/dto/ElementTemplate.java b/element-template-generator/uniquet/src/main/java/io/camunda/connector/uniquet/dto/ElementTemplate.java new file mode 100644 index 0000000000..5ecd5072f9 --- /dev/null +++ b/element-template-generator/uniquet/src/main/java/io/camunda/connector/uniquet/dto/ElementTemplate.java @@ -0,0 +1,24 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information regarding copyright + * ownership. Camunda licenses this file to you under the Apache License, + * Version 2.0; you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.camunda.connector.uniquet.dto; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonIgnoreProperties(ignoreUnknown = true) +public record ElementTemplate( + @JsonProperty(required = true) String id, @JsonProperty(required = true) Integer version) {} diff --git a/element-template-generator/uniquet/src/main/java/io/camunda/connector/uniquet/dto/ElementTemplateFile.java b/element-template-generator/uniquet/src/main/java/io/camunda/connector/uniquet/dto/ElementTemplateFile.java new file mode 100644 index 0000000000..19c7fa33a6 --- /dev/null +++ b/element-template-generator/uniquet/src/main/java/io/camunda/connector/uniquet/dto/ElementTemplateFile.java @@ -0,0 +1,19 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information regarding copyright + * ownership. Camunda licenses this file to you under the Apache License, + * Version 2.0; you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.camunda.connector.uniquet.dto; + +public record ElementTemplateFile(ElementTemplate elementTemplate, String path) {} diff --git a/element-template-generator/uniquet/src/main/java/io/camunda/connector/uniquet/dto/OutputElementTemplate.java b/element-template-generator/uniquet/src/main/java/io/camunda/connector/uniquet/dto/OutputElementTemplate.java new file mode 100644 index 0000000000..fa64ec7992 --- /dev/null +++ b/element-template-generator/uniquet/src/main/java/io/camunda/connector/uniquet/dto/OutputElementTemplate.java @@ -0,0 +1,19 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information regarding copyright + * ownership. Camunda licenses this file to you under the Apache License, + * Version 2.0; you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.camunda.connector.uniquet.dto; + +public record OutputElementTemplate(Integer version, String ref) {} diff --git a/element-template-generator/uniquet/src/test/java/io/camunda/connector/core/GitCrawlerTest.java b/element-template-generator/uniquet/src/test/java/io/camunda/connector/core/GitCrawlerTest.java new file mode 100644 index 0000000000..814ddc2e52 --- /dev/null +++ b/element-template-generator/uniquet/src/test/java/io/camunda/connector/core/GitCrawlerTest.java @@ -0,0 +1,84 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information regarding copyright + * ownership. Camunda licenses this file to you under the Apache License, + * Version 2.0; you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.camunda.connector.core; + +import io.camunda.connector.uniquet.core.GitCrawler; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Map; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.revwalk.RevCommit; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class GitCrawlerTest { + + private static Git git; + private static Path newFilePath; + + @BeforeAll + public static void setUp(@TempDir Path tempDir) throws GitAPIException, IOException { + git = Git.init().setDirectory(tempDir.toFile()).call(); + newFilePath = tempDir.resolve("element-templates/test.json"); + Files.createDirectories(newFilePath.getParent()); + } + + @AfterAll + public static void tearDown() { + git.getRepository().close(); + } + + @Test + void crawl() throws IOException, GitAPIException { + + // commit 1 + String content1 = Files.readString(Path.of("src/test/resources/commit1.json")); + Files.write(newFilePath, content1.getBytes(), StandardOpenOption.CREATE); + git.add().addFilepattern(".").call(); + RevCommit revCommit1 = git.commit().setSign(false).setMessage("Initial commit").call(); + + // commit 2 + String content2 = + Files.readString(Path.of("src/test/resources/commit2_version_unchanged.json")); + Files.write(newFilePath, content2.getBytes(), StandardOpenOption.CREATE); + git.add().addFilepattern(".").call(); + RevCommit revCommit2 = git.commit().setSign(false).setMessage("commit 2").call(); + + // commit 3 + String content3 = Files.readString(Path.of("src/test/resources/commit3_version_changed.json")); + Files.write(newFilePath, content3.getBytes(), StandardOpenOption.CREATE); + git.add().addFilepattern(".").call(); + RevCommit revCommit3 = git.commit().setSign(false).setMessage("commit 3").call(); + + Map> map = + GitCrawler.create(git.getRepository().getDirectory().getParentFile().getPath()) + .crawl("master") + .getResult(); + + // Verification that the version 2 is the last commit + Assertions.assertTrue(map.get("test").get(2).contains(revCommit3.getName())); + // Verification that the version 1 is the last commit containing version 1 + Assertions.assertTrue(map.get("test").get(1).contains(revCommit2.getName())); + Assertions.assertFalse(map.get("test").get(1).contains(revCommit1.getName())); + } +} diff --git a/element-template-generator/uniquet/src/test/resources/commit1.json b/element-template-generator/uniquet/src/test/resources/commit1.json new file mode 100644 index 0000000000..8a389a1dac --- /dev/null +++ b/element-template-generator/uniquet/src/test/resources/commit1.json @@ -0,0 +1,4 @@ +{ + "version":1, + "id":"test" +} \ No newline at end of file diff --git a/element-template-generator/uniquet/src/test/resources/commit2_version_unchanged.json b/element-template-generator/uniquet/src/test/resources/commit2_version_unchanged.json new file mode 100644 index 0000000000..05a0fae157 --- /dev/null +++ b/element-template-generator/uniquet/src/test/resources/commit2_version_unchanged.json @@ -0,0 +1,5 @@ +{ + "version":1, + "id":"test", + "addition" : "somehting" +} \ No newline at end of file diff --git a/element-template-generator/uniquet/src/test/resources/commit3_version_changed.json b/element-template-generator/uniquet/src/test/resources/commit3_version_changed.json new file mode 100644 index 0000000000..4138755078 --- /dev/null +++ b/element-template-generator/uniquet/src/test/resources/commit3_version_changed.json @@ -0,0 +1,5 @@ +{ + "version":2, + "id":"test", + "addition" : "somehting" +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 69dd6b3495..71bc22a772 100644 --- a/pom.xml +++ b/pom.xml @@ -26,6 +26,7 @@ element-template-generator/openapi-parser element-template-generator/postman-collections-parser element-template-generator/congen-cli + element-template-generator/uniquet secret-providers/gcp-secret-provider connector-runtime/connector-runtime-core connector-runtime/connector-runtime-spring