Test cdap

.github/workflows/e2e.yml

@@ -0,0 +1,103 @@
+# Copyright © 2021 Cask Data, Inc.
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may not
+#  use this file except in compliance with the License. You may obtain a copy of
+#  the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations under
+#  the License.
+
+# This workflow will build a Java project with Maven
+# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
+# Note: Any changes to this workflow would be used only after merging into develop
+name: Build e2e tests
+
+on:
+  push:
+    branches: [ develop ]
+  pull_request:
+    branches: [ develop]
+    types: [opened, synchronize, reopened, labeled]
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: self-hosted
+    # We allow builds:
+    # 1) When triggered manually
+    # 2) When it's a merge into a branch
+    # 3) For PRs that are labeled as build and
+    #  - It's a code change
+    #  - A build label was just added
+    # A bit complex but prevents builds when other labels are manipulated
+    if: >
+      github.event_name == 'workflow_dispatch'
+      || github.event_name == 'push'
+      || (contains(github.event.pull_request.labels.*.name, 'build')
+          && (github.event.action != 'labeled' || github.event.label.name == 'build')
+          )
+    strategy:
+      matrix:
+        tests: [ cdap-e2e-tests ]
+      fail-fast: false
+    steps:
+      # Pinned 1.0.0 version
+      - uses: actions/checkout@v3
+        with:
+          path: plugin
+
+      - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50
+        # Pinned version 2.11.1
+        if: github.event_name != 'workflow_dispatch' && github.event_name != 'push'
+        id: filter
+        with:
+          working-directory: plugin
+          filters: |
+            e2e-test:
+              - '**/e2e-test/**'
+
+      - name: Checkout e2e test repo
+        uses: actions/checkout@v3
+        with:
+          repository: Vipinofficial11/cdap-e2e-tests
+          path: e2e
+          ref: addPluginUploadSkipFlag
+
+      - name: Cache
+        uses: actions/cache@v3
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ github.workflow }}-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-${{ github.workflow }}
+
+      - name: Run required e2e tests
+        if: github.event_name != 'workflow_dispatch' && github.event_name != 'push' && steps.filter.outputs.e2e-test == 'false'
+        run: python3 e2e/src/main/scripts/run_e2e_test.py --module ${{ matrix.tests }} --testRunner TestRunnerRequired.java --skipPluginUpload yes
+
+      - name: Run all e2e tests
+        if: github.event_name == 'workflow_dispatch' || github.event_name == 'push' || steps.filter.outputs.e2e-test == 'true'
+        run: python3 e2e/src/main/scripts/run_e2e_test.py --module ${{ matrix.tests }} --skipPluginUpload yes
+
+      - name: Upload report
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: Cucumber report - ${{ matrix.tests }}
+          path: ./plugin/${{ matrix.tests }}/target/cucumber-reports
+
+      - name: Upload debug files
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: Debug files - ${{ matrix.tests }}
+          path: ./**/target/e2e-debug
+
+      - name: Upload files to GCS
+        uses: google-github-actions/upload-cloud-storage@v0
+        if: always()
+        with:
+          path: ./plugin/target/cucumber-reports
+          destination: e2e-tests-cucumber-reports/${{ github.event.repository.name }}/${{ github.ref }}

.gitignore

@@ -55,3 +55,5 @@ derby.log
 
 # generated by docs build
 *.pyc
+
+#/cdap-ui

cdap-common/src/main/java/io/cdap/cdap/common/security/KeyStores.java

@@ -17,29 +17,22 @@
 package io.cdap.cdap.common.security;
 
 import com.google.common.hash.Hashing;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
+
+import java.io.*;
 import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.security.GeneralSecurityException;
-import java.security.KeyPair;
-import java.security.KeyPairGenerator;
-import java.security.KeyStore;
-import java.security.KeyStoreException;
-import java.security.NoSuchAlgorithmException;
-import java.security.PrivateKey;
-import java.security.SecureRandom;
+import java.nio.file.Paths;
+import java.security.*;
 import java.security.cert.Certificate;
 import java.security.cert.CertificateException;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.Enumeration;
-import java.util.List;
+import java.util.*;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
+
+import com.jcraft.jsch.KeyPairPKCS8;
+import io.netty.handler.ssl.PemPrivateKey;
 import org.apache.twill.filesystem.Location;
 import org.bouncycastle.asn1.x500.X500Name;
 import org.bouncycastle.asn1.x509.AlgorithmIdentifier;
@@ -49,6 +42,7 @@
 import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
 import org.bouncycastle.crypto.params.AsymmetricKeyParameter;
 import org.bouncycastle.crypto.util.PrivateKeyFactory;
+import org.bouncycastle.jcajce.provider.keystore.PKCS12;
 import org.bouncycastle.jce.provider.BouncyCastleProvider;
 import org.bouncycastle.openssl.PEMEncryptedKeyPair;
 import org.bouncycastle.openssl.PEMKeyPair;
@@ -126,6 +120,13 @@ public static KeyStore generatedCertKeyStore(int validityDays, String password)
     }
   }
 
+  public static void main(String[] args) throws KeyStoreException, IOException, UnrecoverableKeyException, NoSuchAlgorithmException {
+    Path path = Paths.get("cdap-common/src/main/java/io/cdap/cdap/common/security/certificate.pem");
+    KeyStore key = createKeyStore(path, "");
+
+    System.out.println(key.size());
+  }
+
   /**
    * Creates a key store from the given PEM file.
    *
@@ -148,6 +149,9 @@ public static KeyStore createKeyStore(Path certificatePath, String password) {
         Files.newBufferedReader(certificatePath, StandardCharsets.ISO_8859_1))) {
       Object obj = parser.readObject();
       while (obj != null) {
+
+        System.out.println(obj);
+
         // Decrypt the key block if it is encrypted
         if (obj instanceof PEMEncryptedKeyPair) {
           obj = ((PEMEncryptedKeyPair) obj).decryptKeyPair(new BcPEMDecryptorProvider(passPhase));
@@ -168,11 +172,20 @@ public static KeyStore createKeyStore(Path certificatePath, String password) {
         throw new RuntimeException("Missing private key from file " + certificatePath);
       }
 
+
       KeyStore keyStore = KeyStore.getInstance(SSL_KEYSTORE_TYPE);
       keyStore.load(null, passPhase);
+
+      System.out.println("private key : " + privateKey);
+      System.out.println("Certificate : " + Arrays.toString(certificates.toArray(new Certificate[0])));
+
       keyStore.setKeyEntry(CERT_ALIAS, privateKey, passPhase,
           certificates.toArray(new Certificate[0]));
+
+
       return keyStore;
+
+
     } catch (IOException | CertificateException | NoSuchAlgorithmException | KeyStoreException e) {
       throw new RuntimeException("Failed to create keystore from PEM file " + certificatePath, e);
     }

cdap-common/src/test/java/io/cdap/cdap/common/security/KeyStoresTest.java

@@ -17,7 +17,10 @@
 package io.cdap.cdap.common.security;
 
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.FileWriter;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.security.Key;
 import java.security.KeyStore;
 import java.security.KeyStoreException;
@@ -92,7 +95,6 @@ public void testPEMToKeyStore() throws Exception {
       // Generate a keystore and write out PEM blocks
       KeyStore keystore = KeyStores.generatedCertKeyStore(KeyStores.VALIDITY, password);
       Key key = keystore.getKey(KeyStores.CERT_ALIAS, password.toCharArray());
-
       File pemFile = writePEMFile(TEMP_FOLDER.newFile(), keystore, KeyStores.CERT_ALIAS, password);
 
       // Create a keystore from the PEM file
@@ -103,6 +105,34 @@ public void testPEMToKeyStore() throws Exception {
     }
   }
 
+  @Test
+  public void createPEMFile() throws Exception {
+    // Write out PEM file. First without password for the key, then with password
+    for (String password : new String[] { "", "1234" }) {
+      // Generate a keystore and write out PEM blocks
+      KeyStore keystore = KeyStores.generatedCertKeyStore(KeyStores.VALIDITY, password);
+      Key key = keystore.getKey(KeyStores.CERT_ALIAS, password.toCharArray());
+      Path path = Paths.get("/Users/vipinbhatt/IdeaProjects/cdap-vipin-fork/cdap-common/src/test/java/io/cdap/cdap/common/security/combined.pem");
+      File file = new File(path.toUri());
+
+      // Override the existing file content.
+      File pemFile = writePEMFile(file, keystore, KeyStores.CERT_ALIAS, password);
+
+      KeyStore keystore2 = KeyStores.createKeyStore(pemFile.toPath(), password);
+
+      String keystorePath = "/Users/vipinbhatt/IdeaProjects/cdap-vipin-fork/cdap-common/src/test/java/io/cdap/cdap/common/security/keystore.jks"; // You can create an empty file with .jks extension.
+      FileOutputStream fos = new FileOutputStream(keystorePath);
+      keystore2.store(fos, password.toCharArray());
+//
+
+//      System.out.println("Key from cert" + keystore2.getKey(KeyStores.CERT_ALIAS, password.toCharArray()));
+//
+//      Assert.assertEquals(key, keystore2.getKey(KeyStores.CERT_ALIAS, password.toCharArray()));
+
+    }
+  }
+
+
   /**
    * Writes a private key and certificate pair from a KeyStore to the given PEM file.
    */

cdap-common/src/test/java/io/cdap/cdap/common/security/combined.pem

@@ -0,0 +1,48 @@
+-----BEGIN RSA PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: AES-128-CBC,91E4CC2711FA9DEFB217C97413544436
+
+uLgVSgPqEjpdJUJspPiBAj8G8pJ3Lm8jxNWs9JXCJQA1B0omEqlgFlRdW5TDe/s6
+ezB1Twdqbu/4GzlDC52E16LKkpZYz6o5KmIzuB1sm5w2ymHhBMNZFKwSLTbEynSL
+dZU7kzGhDDUs0BUr2itUZ/1CNfRAVRfRZvPjErHnPkDVGHaN0LEf3S5XRAGFp+kX
+iNofqj9NbFc8J5BaUOZ0b7z/S3C61qkanB4KGTKynzRgJ+YjCfXa9PLOPvlCW3YH
+elQmJaTmvySzXq34iy22ZC5YmYLRIo3dvHRxrW/zp8Uu/mrHV2DzLjTAekSp0hgj
+3LehTWaYTDAP8HgBlJU3ETaHcclCzcnrXuW/e0ShC61lPtp3t6frwkUMxc1IxYbl
+hWFTsUQdt/XzmwJ/nfebl7P/aQTtMw56MpLKRdvXmm1q+oEfUPlvrYB5JXrRNIHc
+zHMtykFby+Ib82k21DI0BPwCmmPM0gytxWCdbt/5xp4zBo4FYhbq0vZrUmgGlXLq
+rczJPdZwln/LCRD4CdyZxdyEoNSI+wtaUrA0GDJ1PWDW8p8u4VtDfOgh5zAKHuvA
+1im+J8w7GQe5mUlDaBeir+cHGPgMg5Ri3Dn2t3L2vq6/O6GEIqFCQQnapQ7wMSH3
+cG06b0Kw9fRrCtzz6giwmnxVlPZ9hMzSVEjH1yYMqfy2hSsOMSXFzfc9NUD470kY
+knv4pShFhTV+iontd3mq5XQqTZYmuPclHKAzMUaDs0NlQ1rMdSzQL9fKIZKDZSDc
+Hm5SSTVh6/giw9liOiuUQ/K78Qu2aAbntjlCwO1ATG1ZhV31+xgbThk+cBsQA876
+4pxuAnAK5vb3dfFmrn9RuSOyua+mviQ2JH8wFeyeJMF37TlbH7PlP1cN1EKehH9p
+V70KqsGmuspaFYLF7epmYpVKB7gs/IFgir35mIcweCDp4YZz6sl7KXLbPNi8DcAc
+mzbdW/+SD9ifWyKh+NDEXdvpk/vSkLDBGQy2s1AmBAlwSsjrIEuOdtZoGkAyc4F5
+ytSNOi7KtXFCASqA2ChCVfcePCpDv20jHhE7Iz/zrquuvItea7MFLcbXVV4TgWFa
+LuCCKLhdlcN65nESIsYp2kbCU9TxX/yNg4jnRKRB2qNOCoNK/cFS+ijLGOjuV0+h
+G1ao6EP3sgfOq1xw7XGxQmmFXFC5VJFWuEcPhBjheECD1OZyJuu1WmNOgmja/ljK
+V+MMHyLOzj6Q5qRNiEJ2TW+Vjb7zDqn3/734RpR3nIS6qpdIDoNC0URVQTPsUpeP
+ekhJp9FtdBZLjELjnuZYM6hbf4bHQaGyaPskPbhFiELGsCk27IgVsLPvy/s8FRhb
+e0ONtSqbxT59jY1GnvgQwAzavu85pjiTtl7ZI1R/lEo2cntHEspcB2AG8D6rblwf
+oMAjZNDvYejfT6s+JUcF5U5q7Kzh5xrocEEGT8qEkezMC1gkSNEPvwfN9JClamCV
+9aN0PsevVuGQIQnJJHn1+vBPHguVc/M+GI9kgzI/sWvjoDpFniOQN9ciG24Qt6je
+gy3lPCJUgix1tljFR7bc59t52q+/XovBTFSxRFjxQ88qMs+r6HXvhMotAksQNQhv
+-----END RSA PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIIC4TCCAcmgAwIBAgIJANtANShY2EXsMA0GCSqGSIb3DQEBBQUAMDAxDTALBgNV
+BAMMBENEQVAxEjAQBgNVBAcMCVBhbG8gQWx0bzELMAkGA1UEBhMCVVMwHhcNMjMx
+MTE1MTQ0OTI1WhcNMjYwODEwMTQ0OTI1WjAwMQ0wCwYDVQQDDARDREFQMRIwEAYD
+VQQHDAlQYWxvIEFsdG8xCzAJBgNVBAYTAlVTMIIBIjANBgkqhkiG9w0BAQEFAAOC
+AQ8AMIIBCgKCAQEAv2UGGX/1iZgIYSrj5680Cq9Dx1ipfjZE8NouMP9dPM2N54R5
+Tk19dBs5Se4e4V+4ziHrRWu5YwYd2nhfyDqQ0ZX7BqM+eiW3dMvT/EuUT8enEFmX
+BKLemVIUFYplg2T0ipYYgUtVltOyYZ6914HBGh1qORuFAEKe+lTjOmqbp7cLXge7
+2uiCTe/wArxTzhL6vRUYiGzcIwcJyBDZ/TgDlQVDP7oNpC2RUulCWFOu2sUqwebn
+xb7panChE/UVE6rf8bZPVB7VXRfPikL14VaRaK4XTAuC+V98YR26ir+vAGtFpQBv
+mn27cpEeDsiT3tGgrUFjh4LlV8a6BdWjhV/ZdwIDAQABMA0GCSqGSIb3DQEBBQUA
+A4IBAQALlI9VrAgfJ74m3Oqg06Q5E2MW2RP0vdM9s2gUo0hWBGCAVtVJsr1WCgQC
+v1CdwWYaJQEgSjFeK4x/fY4F1DWm8fEKsX9Rgfylwe1DieCLYy/FIEXaw94O3c3H
+g2AR72llI3i6Na5t1JlZvWJAeSZCbA9sLO/JsogU7WLpITtt01Fj1IguZH0Qir8p
+MbMOacFLJTEAk4hhw2fqGERRJt3LNKECL69GhTYff8kloOA+ISMdlqZyygiba3Bl
+ad+Y4ZgwhoTF6GU2LAouPnExDunf38g5rCPP9m0hh2WpeYFONEqTsc6Pg3b4cLux
+WbDBy0lE/7WvBduXfUzVvVZuOFJ/
+-----END CERTIFICATE-----

cdap-e2e-tests/pom.xml

@@ -69,7 +69,7 @@
               <execution>
                 <goals>
                   <goal>integration-test</goal>
-                  <goal>verify</goal>
+<!--                  <goal>verify</goal>-->
                 </goals>
               </execution>
             </executions>

cdap-e2e-tests/src/e2e-test/features/systemadmin/SysAdminDesignTime.feature

@@ -0,0 +1,71 @@
+#
+# Copyright © 2023 Cask Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+#
+
+@Sysadmin
+Feature: Sysadmin - Validate system admin page design time scenarios
+
+  @Sysadmin1
+  Scenario:Validate user is able to create new system preferences and able to delete the added system preferences successfully
+    Given Open Datafusion Project to configure pipeline
+    When Open "System Admin" menu
+    Then Click on the Configuration link on the System admin page
+    Then Select "systemPreferences" option from Configuration page
+    Then Click on edit system preferences
+    Then Set system preferences with key: "keyValue" and value: "systemPreferences1"
+    Then Click on the Save & Close preferences button
+    Then Select "systemPreferences" option from Configuration page
+    Then Click on edit system preferences
+    Then Delete the preferences
+    Then Click on the Save & Close preferences button
+    Then Verify the system admin page is navigated successfully
+
+  Scenario:Validate user is able to add multiple system preferences inside system admin successfully
+    Given Open Datafusion Project to configure pipeline
+    When Open "System Admin" menu
+    Then Click on the Configuration link on the System admin page
+    Then Select "systemPreferences" option from Configuration page
+    Then Click on edit system preferences
+    Then Set system preferences with key: "keyValue" and value: "systemPreferences2"
+    Then Click on the Save & Close preferences button
+    Then Click on edit system preferences
+    Then Delete the preferences
+    Then Delete the preferences
+    Then Click on the Save & Close preferences button
+    Then Verify the system admin page is navigated successfully
+
+  Scenario:Validate user is able to successfully reload system artifacts using reload
+    Given Open Datafusion Project to configure pipeline
+    When Open "System Admin" menu
+    Then Click on the Configuration link on the System admin page
+    Then Click on Reload System Artifacts from the System admin page
+    Then Click on Reload button on popup to reload the System Artifacts successfully
+    Then Verify the system admin page is navigated successfully
+
+  Scenario:Validate user is able to open compute profile page and create a compute profile for selected provisioner
+    Given Open Datafusion Project to configure pipeline
+    When Open "System Admin" menu
+    Then Click on the Configuration link on the System admin page
+    Then Click on the Compute Profile from the System admin page
+    Then Click on create compute profile button
+    Then Select a provisioner: "remoteHadoopProvisioner" for the compute profile
+    Then Verify the Create a Profile page is loaded for selected provisioner
+    Then Enter input plugin property: "profileLabel" with value: "validProfile"
+    Then Enter textarea plugin property: "profileDescription" with value: "validDescription"
+    Then Enter input plugin property: "host" with value: "testHost"
+    Then Enter input plugin property: "user" with value: "testUser"
+    Then Enter textarea plugin property: "sshKey" with value: "testSSHKey"
+    Then Click on: "Create" button in the properties
+    Then Verify the created compute profile: "validProfile" is displayed in system compute profile list