From c94601ba52761297163d1aeea5ecdcb6a0894f55 Mon Sep 17 00:00:00 2001 From: vikasrathee-cs Date: Tue, 15 Oct 2024 23:22:21 +0530 Subject: [PATCH 1/2] Added fix for date datatype Added fix for date datatype (cherry picked from commit ca328a2fea43e0be57ca183546d51010e11e6214) --- .../features/sink/OracleRunTime.feature | 51 +++++++++++++++++++ .../java/io.cdap.plugin/BQValidation.java | 48 ++++++++++++++--- .../java/io.cdap.plugin/OracleClient.java | 10 ++++ .../common.stepsdesign/TestSetupHooks.java | 30 +++++++++++ .../resources/pluginParameters.properties | 6 +++ .../BigQuery/CreateBQTableQueryFileDate.txt | 1 + .../BigQuery/InsertBQDataQueryFileDate.txt | 1 + .../plugin/oracle/OracleSourceDBRecord.java | 4 +- 8 files changed, 142 insertions(+), 9 deletions(-) create mode 100644 oracle-plugin/src/e2e-test/resources/testdata/BigQuery/CreateBQTableQueryFileDate.txt create mode 100644 oracle-plugin/src/e2e-test/resources/testdata/BigQuery/InsertBQDataQueryFileDate.txt diff --git a/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature b/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature index 67293700b..70b1bdba6 100644 --- a/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature +++ b/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature @@ -167,3 +167,54 @@ Feature: Oracle - Verify data transfer from BigQuery source to Oracle sink Then Verify the pipeline status is "Succeeded" Then Validate records transferred to target table with record counts of BigQuery table Then Validate the values of records transferred to target Oracle table is equal to the values from source BigQuery table with case + + + @BQ_SOURCE_TEST_DATE @ORACLE_DATE_TABLE + Scenario: To verify data is getting transferred from BigQuery source to Oracle sink successfully when schema is having date and timestamp fields + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "BigQuery" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "BigQuery" and "Oracle" to establish connection + Then Navigate to the properties page of plugin: "BigQuery" + Then Replace input plugin property: "project" with value: "projectId" + Then Enter input plugin property: "datasetProject" with value: "projectId" + Then Enter input plugin property: "referenceName" with value: "BQReferenceName" + Then Enter input plugin property: "dataset" with value: "dataset" + Then Enter input plugin property: "table" with value: "bqSourceTable" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputDatatypesDateTimeSchema" + Then Validate "BigQuery" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on preview data for Oracle sink + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate records transferred to target table with record counts of BigQuery table + Then Validate the values of records transferred to target Oracle table is equal to the values from source BigQuery table diff --git a/oracle-plugin/src/e2e-test/java/io.cdap.plugin/BQValidation.java b/oracle-plugin/src/e2e-test/java/io.cdap.plugin/BQValidation.java index b7d93c80a..b5a82e420 100644 --- a/oracle-plugin/src/e2e-test/java/io.cdap.plugin/BQValidation.java +++ b/oracle-plugin/src/e2e-test/java/io.cdap.plugin/BQValidation.java @@ -33,7 +33,12 @@ import java.sql.Types; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.time.LocalDateTime; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Base64; import java.util.Date; import java.util.List; @@ -44,6 +49,13 @@ public class BQValidation { + private static final List TIMESTAMP_DATE_FORMATS = Arrays.asList( + new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss"), + new SimpleDateFormat("yyyy-MM-dd")); + private static final List TIMESTAMP_TZ_DATE_FORMATS = Arrays.asList( + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssXXX"), + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXX")); + /** * Extracts entire data from source and target tables. * @@ -173,21 +185,43 @@ public static boolean compareResultSetAndJsonData(ResultSet rsSource, List Date: Wed, 23 Oct 2024 12:17:33 +0530 Subject: [PATCH 2/2] Bump up to 1.11.4 --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index db99429ff..46fbbdf28 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index f37390c37..52bd863c4 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index a3fe5bed9..0ea3d211a 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index cd3b7e1e1..6daba90c1 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 5d97d547b..185235a04 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index c51034cb5..85de33814 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 406a2614a..f74a73313 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 58d7600f9..f60c6e1b4 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index b2db66d2e..11176aad4 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index ed6d20f19..3d45a86f2 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 757ca5ee2..40f541c1d 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 4e8fb3048..bc189004a 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 34ec642c6..ee353ab20 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index a11b1d01d..d44e0615b 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 4c3902c3a..ee01680b1 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 Oracle plugin diff --git a/pom.xml b/pom.xml index ff9f58a58..7fe284df2 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.4-SNAPSHOT + 1.11.4 pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 2a9a28cfa..73f16b2f2 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 6805f04af..0bcca7c45 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 166564699..b6b7809d6 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.4-SNAPSHOT + 1.11.4 teradata-plugin