wrangler e2e tests

wrangler-transform/src/e2e-test/features/Wrangler/DataTypeParsers.feature

@@ -0,0 +1,71 @@
+# Copyright © 2023 Cask Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+@Wrangler
+Feature:  Wrangler - Run time scenarios
+
+  @BQ_SOURCE_TS_TEST @BQ_SINK_TEST
+  Scenario: To verify User is able to run a pipeline using parse timestamp directive
+    Given Open Datafusion Project to configure pipeline
+    Then Click on the Plus Green Button to import the pipelines
+    Then Select the file for importing the pipeline for the plugin "Directive_parse_Timestamp"
+    Then Navigate to the properties page of plugin: "BigQueryTable"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "dataset" with value: "dataset"
+    Then Replace input plugin property: "table" with value: "bqSourceTable"
+    Then Click on the Get Schema button
+    Then Click on the Validate button
+    Then Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "BigQuery2"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "table" with value: "bqTargetTable"
+    Then Replace input plugin property: "dataset" with value: "dataset"
+    Then Click on the Validate button
+    Then Close the Plugin Properties page
+    Then Rename the pipeline
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Wait till pipeline is in running state
+    Then Open and capture logs
+    Then Verify the pipeline status is "Succeeded"
+    Then Close the pipeline logs
+    Then Validate The Data From BQ To BQ With Actual And Expected File for: "ExpectedDirective_parse_Timestamp"
+
+
+  @BQ_SOURCE_DATETIME_TEST @BQ_SINK_TEST
+  Scenario: To verify User is able to run a pipeline using parse datetime directive
+    Given Open Datafusion Project to configure pipeline
+    Then Click on the Plus Green Button to import the pipelines
+    Then Select the file for importing the pipeline for the plugin "Directive_parse_Datetime"
+    Then Navigate to the properties page of plugin: "BigQueryTable"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "dataset" with value: "dataset"
+    Then Replace input plugin property: "table" with value: "bqSourceTable"
+    Then Click on the Get Schema button
+    Then Click on the Validate button
+    Then Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "BigQuery2"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "table" with value: "bqTargetTable"
+    Then Replace input plugin property: "dataset" with value: "dataset"
+    Then Click on the Validate button
+    Then Close the Plugin Properties page
+    Then Rename the pipeline
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Wait till pipeline is in running state
+    Then Open and capture logs
+    Then Verify the pipeline status is "Succeeded"
+    Then Close the pipeline logs
+    Then Validate The Data From BQ To BQ With Actual And Expected File for: "ExpectedDirective_parse_Datetime"

wrangler-transform/src/e2e-test/features/Wrangler/ParseAsFixedLength.feature

@@ -0,0 +1,43 @@
+# Copyright © 2023 Cask Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+@Wrangler
+Feature:  Wrangler - Run time scenarios
+
+  @BQ_SOURCE_FXDLEN_TEST @BQ_SINK_TEST
+  Scenario: To verify User is able to run a pipeline using parse fixedlength directive
+    Given Open Datafusion Project to configure pipeline
+    Then Click on the Plus Green Button to import the pipelines
+    Then Select the file for importing the pipeline for the plugin "Directive_parse_Fixed_Length"
+    Then Navigate to the properties page of plugin: "BigQueryTable"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "dataset" with value: "dataset"
+    Then Replace input plugin property: "table" with value: "bqSourceTable"
+    Then Click on the Get Schema button
+    Then Click on the Validate button
+    Then Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "BigQuery2"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "table" with value: "bqTargetTable"
+    Then Replace input plugin property: "dataset" with value: "dataset"
+    Then Click on the Validate button
+    Then Close the Plugin Properties page
+    Then Rename the pipeline
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Wait till pipeline is in running state
+    Then Open and capture logs
+    Then Verify the pipeline status is "Succeeded"
+    Then Close the pipeline logs
+    Then Validate The Data From BQ To BQ With Actual And Expected File for: "ExpectedDirective_parse_FixedLength"

wrangler-transform/src/e2e-test/features/Wrangler/ParseAsHl7.feature

@@ -0,0 +1,43 @@
+# Copyright © 2023 Cask Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+@Wrangler
+Feature:  Wrangler - Run time scenarios
+
+  @BQ_SOURCE_HL7_TEST @BQ_SINK_TEST
+  Scenario: To verify User is able to run a pipeline using parse hl7 directive
+    Given Open Datafusion Project to configure pipeline
+    Then Click on the Plus Green Button to import the pipelines
+    Then Select the file for importing the pipeline for the plugin "Directive_parse_hl7"
+    Then Navigate to the properties page of plugin: "BigQueryTable"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "dataset" with value: "dataset"
+    Then Replace input plugin property: "table" with value: "bqSourceTable"
+    Then Click on the Get Schema button
+    Then Click on the Validate button
+    Then Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "BigQuery2"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "table" with value: "bqTargetTable"
+    Then Replace input plugin property: "dataset" with value: "dataset"
+    Then Click on the Validate button
+    Then Close the Plugin Properties page
+    Then Rename the pipeline
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Wait till pipeline is in running state
+    Then Open and capture logs
+    Then Verify the pipeline status is "Succeeded"
+    Then Close the pipeline logs
+    Then Validate The Data From BQ To BQ With Actual And Expected File for: "ExpectedDirective_parse_hl7"

wrangler-transform/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java

@@ -17,29 +17,21 @@
 package io.cdap.plugin.common.stepsdesign;
 
 import com.google.cloud.bigquery.BigQueryException;
-import com.google.cloud.storage.Blob;
-import com.google.cloud.storage.StorageException;
 import io.cdap.e2e.utils.BigQueryClient;
 import io.cdap.e2e.utils.PluginPropertyUtils;
-import io.cdap.e2e.utils.StorageClient;
 import io.cucumber.java.After;
 import io.cucumber.java.Before;
-import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.junit.Assert;
 import stepsdesign.BeforeActions;
 
 import java.io.IOException;
-import java.net.URISyntaxException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Paths;
-import java.sql.SQLException;
 import java.util.NoSuchElementException;
 import java.util.UUID;
 
-import static io.cdap.e2e.pages.locators.CdfGCSLocators.filePath;
-
 /**
  * Setup BQ for Wrangler tests.
  */
@@ -71,10 +63,33 @@ public static void deleteTempTargetBQTable() throws IOException, InterruptedExce
   /**
    * Create BigQuery table.
    */
-  @Before(order = 1, value = "@BQ_SOURCE_CSV_TEST")
-  public static void createTempSourceBQTable() throws IOException, InterruptedException {
-    createSourceBQTableWithQueries(PluginPropertyUtils.pluginProp("CreateBQTableQueryFileCsv"),
-                                   PluginPropertyUtils.pluginProp("InsertBQDataQueryFileCsv"));
+  @Before(order = 1, value = "@BQ_SOURCE_FXDLEN_TEST")
+  public static void createTempSourceBQTableFxdLen() throws IOException, InterruptedException {
+    createSourceBQTableWithQueries(PluginPropertyUtils.pluginProp("CreateBQDataQueryFileFxdLen"),
+            PluginPropertyUtils.pluginProp("InsertBQDataQueryFileFxdLen"));
+  }
+  @Before(order = 1, value = "@BQ_SOURCE_HL7_TEST")
+  public static void createTempSourceBQTableHl7() throws IOException, InterruptedException {
+    createSourceBQTableWithQueries(PluginPropertyUtils.pluginProp("CreateBQDataQueryFileHl7"),
+            PluginPropertyUtils.pluginProp("InsertBQDataQueryFileHl7"));
+  }
+  @Before(order = 1, value = "@BQ_SOURCE_TS_TEST")
+  public static void createTempSourceBQTableTimestamp() throws IOException, InterruptedException {
+    createSourceBQTableWithQueries(PluginPropertyUtils.pluginProp("CreateBQDataQueryFileTimestamp"),
+            PluginPropertyUtils.pluginProp("InsertBQDataQueryFileTimestamp"));
+  }
+  @Before(order = 1, value = "@BQ_SOURCE_DATETIME_TEST")
+  public static void createTempSourceBQTableDateTime() throws IOException, InterruptedException {
+    createSourceBQTableWithQueries(PluginPropertyUtils.pluginProp("CreateBQDataQueryFileDatetime"),
+            PluginPropertyUtils.pluginProp("InsertBQDataQueryFileDatetime"));
+  }
+
+  @After(order = 1, value = "@BQ_SOURCE_TEST")
+  public static void deleteTempSourceBQTable() throws IOException, InterruptedException {
+    String bqSourceTable = PluginPropertyUtils.pluginProp("bqSourceTable");
+    BigQueryClient.dropBqQuery(bqSourceTable);
+    BeforeActions.scenario.write("BQ source Table " + bqSourceTable + " deleted successfully");
+    PluginPropertyUtils.removePluginProp("bqSourceTable");
   }
 
   private static void createSourceBQTableWithQueries(String bqCreateTableQueryFile, String bqInsertDataQueryFile)

wrangler-transform/src/e2e-test/resources/BQValidationExpectedFiles/Directive_parse_DateTime

@@ -0,0 +1,3 @@
+{"create_date":"2023","id":1,"timecolumn":"2006-03-18"}
+{"create_date":"2023","id":2,"timecolumn":"2007-03-18"}
+{"create_date":"2023","id":3,"timecolumn":"2008-04-19"}

wrangler-transform/src/e2e-test/resources/BQValidationExpectedFiles/Directive_parse_Timestamp

@@ -0,0 +1,3 @@
+{"create_date":"2021-01-21T00:00:00Z","diff_date":594345600000,"format_price":"$1.00","id":1.0,"price":"$1","time":"2018-09-07T14:57:51.892Z","update_date":"2002-03-23T00:00:00"}
+{"create_date":"2022-01-22T00:00:00Z","diff_date":562723200000,"format_price":"$2.00","id":2.0,"price":"$2","time":"2018-09-07T14:57:51.896Z","update_date":"2004-03-24T00:00:00"}
+{"create_date":"2023-01-23T00:00:00Z","diff_date":652060800000,"format_price":"$3.00","id":3.0,"price":"$3","time":"2018-09-07T14:57:51.898Z","update_date":"2002-05-26T00:00:00"}

wrangler-transform/src/e2e-test/resources/BQValidationExpectedFiles/Directive_parse_fixedlength

@@ -0,0 +1,2 @@
+{"Url":"http://example.com:80/docs/books/tutorial/index.html?name=networking#DOWNLOADING","fixedlength":"21  10  ABCXYZ","fixedlength_1":"21","fixedlength_3":"  ABC","fixedlength_4":"XYZ","fixedlength_encode_base32":"GIYSAIBRGAQCAQKCINMFSWQ=","fixedlength_encode_base32_decode_base32":"21  10  ABCXYZ","id":"  10","url_authority":"example.com:80","url_filename":"/docs/books/tutorial/index.html?name=networking","url_host":"example.com","url_path":"/docs/books/tutorial/index.html","url_port":80,"url_protocol":"http","url_query":"name=networking","url_query_1":"name","url_query_2":"networking"}
+{"Url":"http://geeks.com:80/docs/chair/tutorial/index.html?name=networking#DOWNLOADING","fixedlength":"19  13  ABCXYZ","fixedlength_1":"19","fixedlength_3":"  ABC","fixedlength_4":"XYZ","fixedlength_encode_base32":"GE4SAIBRGMQCAQKCINMFSWQ=","fixedlength_encode_base32_decode_base32":"19  13  ABCXYZ","id":"  13","url_authority":"geeks.com:80","url_filename":"/docs/chair/tutorial/index.html?name=networking","url_host":"geeks.com","url_path":"/docs/chair/tutorial/index.html","url_port":80,"url_protocol":"http","url_query":"name=networking","url_query_1":"name","url_query_2":"networking"}

wrangler-transform/src/e2e-test/resources/BQValidationExpectedFiles/Directive_parse_hl7

@@ -0,0 +1,2 @@
+{"Body":"s��\u0011y�X��\u0006�H���","Body_hl7_MSH_9_1":"ALM","address":"test","id":"3"}
+{"Body":"F<��\u001c����#J��^�:","Body_hl7_MSH_9_1":"BLM","address":"address2","id":"4"}

wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryCreateTableQueryDatetime.txt

@@ -0,0 +1 @@
+create table `DATASET.TABLE_NAME` (id STRING, create_date STRING, timestamp STRING)

wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryCreateTableQueryFxdlen.txt

@@ -0,0 +1 @@
+create table `DATASET.TABLE_NAME` (url STRING, fixedlength STRING)

wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryCreateTableQueryTimestamp.txt

@@ -0,0 +1 @@
+create table `DATASET.TABLE_NAME` (create_date STRING, update_date STRING, time BIGINT, price STRING)

wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryCreateTableQueryhl7.txt

@@ -0,0 +1 @@
+create table `DATASET.TABLE_NAME` (address STRING, Body STRING)

wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryInsertDataQueryDatetime.txt

@@ -0,0 +1,5 @@
+INSERT INTO DATASET.TABLE_NAME (id,create_date,timestamp)
+VALUES
+('1','2021-01-21','2006-02-18T05:03:42Z[UTC]'),
+('2','2022-02-22','2007-01-18T04:03:22Z[UTC]'),
+('3','2023-03-23','2008-07-19T08:04:22Z[UTC]');

wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryInsertDataQueryFxdlen.txt

@@ -0,0 +1,5 @@
+INSERT INTO DATASET.TABLE_NAME (url,fixedlength)
+VALUES
+('http://example.com:80/docs/books/tutorial/index.html?name=networking#DOWNLOADING','21  10  ABCXYZ'),
+('http://geeks.com:80/docs/chair/tutorial/index.html?name=networking#DOWNLOADING','19  13  ABCXYZ'),
+('http://amazing.com:80/docs/tables/tutorial/index.html?name=networking#DOWNLOADING','18  14  CDEFGH');

wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryInsertDataQueryHl7.txt

@@ -0,0 +1,5 @@
+INSERT INTO DATASET.TABLE_NAME (address,Body)
+VALUES
+('address1','MSH|^~?2||.|||199908180016||ADT^A04|ADT.1.1698593|P|3'),
+('address2','MSH|^~?2||.|||199908180016||BSC^A04|ADT.1.1698593|P|4'),
+('','MSH|^~?2||.|||199908180016||JKL^A04|ADT.1.1698593|P|5');

wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryInsertDataQueryTimestamp.txt

@@ -0,0 +1,5 @@
+INSERT INTO DATASET.TABLE_NAME (create_date,update_date,time,price)
+VALUES
+('2021-01-21','2002-03-23',1536332271892,'$1'),
+('2022-01-22','2004-03-24',1536332271896,'$2'),
+('2023-01-23','2002-05-26',1536332271898,'$3');

wrangler-transform/src/e2e-test/resources/pluginParameters.properties

@@ -1,16 +1,29 @@
 #json file path
-Directive_parse_csv=testData/Wrangler\
-  /parse_csv_wrangle-cdap-data-pipeline.json
+Directive_parse_Fixed_Length=testData/Wrangler/parse_fixedlength_wrangle-cdap-data-pipeline.json
+Directive_parse_hl7=testData/Wrangler/parse_HL7_Wrangler-cdap-data-pipeline (1).json
+Directive_parse_Timestamp=testData/Wrangler/parse_timestamp_wrangle-cdap-data-pipeline.json
+Directive_parse_Datetime=testData/Wrangler/parse_datetime_wrangle-cdap-data-pipeline.json
 bqSourceTable=dummy
 sourcePath=example/hello.csv
 gcsSourceBucket=dummy
 #bq queries file path
-CreateBQTableQueryFileCsv=BQtesdata/BigQuery/BigQueryCreateTableQueryCsv.txt
-InsertBQDataQueryFileCsv=BQtesdata/BigQuery/BigQueryInsertDataQueryCsv.txt
+
+CreateBQDataQueryFileFxdLen=BQtesdata/BigQuery/BigQueryCreateTableQueryFxdlen.txt
+InsertBQDataQueryFileFxdLen=BQtesdata/BigQuery/BigQueryInsertDataQueryFxdlen.txt
+CreateBQDataQueryFileHl7=BQtesdata/BigQuery/BigQueryCreateTableQueryhl7.txt
+InsertBQDataQueryFileHl7=BQtesdata/BigQuery/BigQueryInsertDataQueryHl7.txt
+CreateBQDataQueryFileTimestamp=BQtesdata/BigQuery/BigQueryCreateTableQueryTimestamp.txt
+InsertBQDataQueryFileTimestamp=BQtesdata/BigQuery/BigQueryInsertDataQueryTimestamp.txt
+CreateBQDataQueryFileDatetime=BQtesdata/BigQuery/BigQueryCreateTableQueryDatetime.txt
+InsertBQDataQueryFileDatetime=BQtesdata/BigQuery/BigQueryInsertDataQueryDatetime.txt
 
 #bq properties
 projectId=cdf-athena
 dataset=test_automation
 dataset2=Wrangler
 #expectedBQFiles
-ExpectedDirective_parse_csv=BQValidationExpectedFiles/Directive_parse_csv
+
+ExpectedDirective_parse_FixedLength=BQValidationExpectedFiles/Directive_parse_fixedlength
+ExpectedDirective_parse_hl7=BQValidationExpectedFiles/Directive_parse_hl7
+ExpectedDirective_parse_Datetime=BQValidationExpectedFiles/Directive_parse_DateTime
+ExpectedDirective_parse_Timestamp=BQValidationExpectedFiles/Directive_parse_Timestamp