Skip to content

Commit

Permalink
addressed comments for version range
Browse files Browse the repository at this point in the history
  • Loading branch information
@AnkitCLI
AnkitCLI committed Jan 11, 2024
1 parent 2535080 commit 43a4ed7
Show file tree
Hide file tree
Showing 8 changed files with 29 additions and 73 deletions.
4 changes: 1 addition & 3 deletions wrangler-transform/src/e2e-test/features/Wrangler/DataTypeParsers.feature
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
@Wranglers
Feature: datatype parsers

@BQ_SOURCE_TS_TEST @BQ_SINK_TEST
@BQ_SOURCE_TS_TEST @BQ_SOURCE_TEST @BQ_SINK_TEST
Scenario: To verify User is able to run a pipeline using parse timestamp directive
Given Open Datafusion Project to configure pipeline
Then Click on the Plus Green Button to import the pipelines
Expand All @@ -24,8 +24,6 @@ Feature: datatype parsers
Then Replace input plugin property: "project" with value: "projectId"
Then Replace input plugin property: "dataset" with value: "dataset"
Then Replace input plugin property: "table" with value: "bqSourceTable"
Then Click on the Get Schema button
Then Click on the Validate button
Then Close the Plugin Properties page
Then Navigate to the properties page of plugin: "BigQuery2"
Then Replace input plugin property: "project" with value: "projectId"
Expand Down
14 changes: 4 additions & 10 deletions ...ransform/src/e2e-test/resources/testData/Wrangler/parseAsAvro-cdap-data-pipeline (1).json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"description": "Data Pipeline Application",
"artifact": {
"name": "cdap-data-pipeline",
"version": "6.10.0-SNAPSHOT",
"version": "[6.0.0, 7.0.0]",
"scope": "SYSTEM"
},
"config": {
Expand Down Expand Up @@ -37,9 +37,7 @@
"type": "batchsource",
"label": "BigQueryTable",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down Expand Up @@ -258,9 +256,7 @@
"type": "transform",
"label": "Wrangler",
"artifact": {
"name": "wrangler-transform",
"version": "4.11.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "wrangler-transform"
},
"properties": {
"directives": "parse-as-avro-file :body\nfilter-rows-on condition-true name == null || name =~ \"^\\W*$\"\ncut-character name nameNew 2-3\nset-type :value integer ",
Expand Down Expand Up @@ -377,9 +373,7 @@
"type": "batchsink",
"label": "BigQuery2",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down
14 changes: 4 additions & 10 deletions ...m/src/e2e-test/resources/testData/Wrangler/parse_HL7_Wrangler-cdap-data-pipeline (1).json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"description": "Data Pipeline Application",
"artifact": {
"name": "cdap-data-pipeline",
"version": "6.10.0-SNAPSHOT",
"version": "[6.0.0, 7.0.0]",
"scope": "SYSTEM"
},
"config": {
Expand Down Expand Up @@ -37,9 +37,7 @@
"type": "batchsource",
"label": "BigQueryTable",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down Expand Up @@ -76,9 +74,7 @@
"type": "transform",
"label": "Wrangler",
"artifact": {
"name": "wrangler-transform",
"version": "4.11.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "wrangler-transform"
},
"properties": {
"directives": "parse-as-hl7 :Body\nhash :Body MD5\nset-type :Body string \nkeep address,Body,Body_hl7_MSH_12,Body_hl7_MSH_9_1\nfind-and-replace address s/address1/test/g\nmask-shuffle :Body_hl7_MSH_9_1\nsend-to-error empty(address)\nrename :Body_hl7_MSH_12 :id ",
Expand Down Expand Up @@ -119,9 +115,7 @@
"type": "batchsink",
"label": "BigQuery2",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down
14 changes: 4 additions & 10 deletions ...nsform/src/e2e-test/resources/testData/Wrangler/parse_csv_wrangle-cdap-data-pipeline.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"description": "Data Pipeline Application",
"artifact": {
"name": "cdap-data-pipeline",
"version": "6.10.0-SNAPSHOT",
"version": "[6.0.0, 7.0.0]",
"scope": "SYSTEM"
},
"config": {
Expand Down Expand Up @@ -37,9 +37,7 @@
"type": "batchsource",
"label": "BigQueryTable",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down Expand Up @@ -76,9 +74,7 @@
"type": "transform",
"label": "Wrangler",
"artifact": {
"name": "wrangler-transform",
"version": "4.11.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "wrangler-transform"
},
"properties": {
"directives": "parse-as-csv :body ',' false\nrename body_1 new_id\nquantize body_4 body_q 1:2=20,3:4=40\nset-type :body_4 integer \ncolumns-replace s/^new_//g\nfill-null-or-empty :body_3 'shubh'\nset-headers :abc\nchange-column-case uppercase\ncleanse-column-names\nsplit-to-rows :id '#'",
Expand Down Expand Up @@ -119,9 +115,7 @@
"type": "batchsink",
"label": "BigQuery2",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down
14 changes: 4 additions & 10 deletions ...m/src/e2e-test/resources/testData/Wrangler/parse_datetime_wrangle-cdap-data-pipeline.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"description": "Data Pipeline Application",
"artifact": {
"name": "cdap-data-pipeline",
"version": "6.10.0-SNAPSHOT",
"version": "[6.0.0, 7.0.0]",
"scope": "SYSTEM"
},
"config": {
Expand Down Expand Up @@ -37,9 +37,7 @@
"type": "batchsource",
"label": "BigQueryTable",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down Expand Up @@ -257,9 +255,7 @@
"type": "transform",
"label": "Wrangler",
"artifact": {
"name": "wrangler-transform",
"version": "4.11.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "wrangler-transform"
},
"properties": {
"directives": "parse-as-datetime :timestamp \"yyyy-MM-dd'T'HH:mm:ssX'['z']'\"\ncurrent-datetime :create_date\ndatetime-to-timestamp :timestamp\nformat-datetime :create_date 'y'\nformat-date :timestamp yyyy-mm-dd\nrename timestamp timecolumn",
Expand Down Expand Up @@ -301,9 +297,7 @@
"type": "batchsink",
"label": "BigQuery2",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down
14 changes: 4 additions & 10 deletions ...rc/e2e-test/resources/testData/Wrangler/parse_fixedlength_wrangle-cdap-data-pipeline.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"description": "Data Pipeline Application",
"artifact": {
"name": "cdap-data-pipeline",
"version": "6.10.0-SNAPSHOT",
"version": "[6.0.0, 7.0.0]",
"scope": "SYSTEM"
},
"config": {
Expand Down Expand Up @@ -37,9 +37,7 @@
"type": "batchsource",
"label": "BigQueryTable",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down Expand Up @@ -76,9 +74,7 @@
"type": "transform",
"label": "Wrangler",
"artifact": {
"name": "wrangler-transform",
"version": "4.11.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "wrangler-transform"
},
"properties": {
"directives": "parse-as-fixed-length :fixedlength 2,4,5,3\nsplit-url :url\nwrite-as-csv :url_protocol\nurl-encode :url\nurl-decode :url\nencode base32 :fixedlength\ndecode base32 :fixedlength_encode_base32\nsplit-to-columns :url_query '='\nrename :fixedlength_2 :id\nfilter-rows-on condition-true fixedlength_4 !~ 'XYZ'",
Expand Down Expand Up @@ -119,9 +115,7 @@
"type": "batchsink",
"label": "BigQuery2",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down
14 changes: 4 additions & 10 deletions .../src/e2e-test/resources/testData/Wrangler/parse_log_wrangler_copy-cdap-data-pipeline.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"description": "Data Pipeline Application",
"artifact": {
"name": "cdap-data-pipeline",
"version": "6.11.0-SNAPSHOT",
"version": "[6.0.0, 7.0.0]",
"scope": "SYSTEM"
},
"config": {
Expand Down Expand Up @@ -37,9 +37,7 @@
"type": "batchsource",
"label": "BigQueryTable",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down Expand Up @@ -76,9 +74,7 @@
"type": "transform",
"label": "Wrangler",
"artifact": {
"name": "wrangler-transform",
"version": "4.11.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "wrangler-transform"
},
"properties": {
"directives": "parse-as-log :Body 'common'\ndrop :number_connection_client_logname_last\ndrop :number_connection_client_logname\ndrop :http_querystring_request_firstline_uri_query\nfill-null-or-empty :http_ref_request_firstline_uri_ref 'no value'\nfilter-rows-on condition-false ip_connection_client_host =$ \"1\"\nfilter-rows-on regex-match string_connection_client_user_last ^Tryck$\nextract-regex-groups :http_firstline_request_firstline GET*\nsplit-to-columns :ip_connection_client_host ,\ncatalog-lookup ICD-9 :Body\nfill-null-or-empty :Body_icd_9_description 'Body Post Catalog lookup'\nset-variable Pass string_request_status_last == \"200\" ? string_request_status_last : Pass\nset-type :time_day_request_receive_time_last_day integer ",
Expand Down Expand Up @@ -112,9 +108,7 @@
"type": "batchsink",
"label": "BigQuery2",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down
14 changes: 4 additions & 10 deletions .../src/e2e-test/resources/testData/Wrangler/parse_timestamp_wrangle-cdap-data-pipeline.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"description": "Data Pipeline Application",
"artifact": {
"name": "cdap-data-pipeline",
"version": "6.10.0-SNAPSHOT",
"version": "[6.0.0, 7.0.0]",
"scope": "SYSTEM"
},
"config": {
Expand Down Expand Up @@ -37,9 +37,7 @@
"type": "batchsource",
"label": "BigQueryTable",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down Expand Up @@ -76,9 +74,7 @@
"type": "transform",
"label": "Wrangler",
"artifact": {
"name": "wrangler-transform",
"version": "4.11.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "wrangler-transform"
},
"properties": {
"directives": "parse-timestamp :time\nparse-as-currency :price :newprice\nformat-as-currency :newprice :format_price\nparse-as-simple-date :create_date yyyy-MM-dd\nparse-as-simple-date :update_date yyyy-MM-dd\ndiff-date :create_date :update_date :diff_date\ntimestamp-to-datetime :update_date\nrename :newprice :id",
Expand Down Expand Up @@ -119,9 +115,7 @@
"type": "batchsink",
"label": "BigQuery2",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down

0 comments on commit 43a4ed7

Please sign in to comment.