diff --git a/docs/docs/en/guide/upgrade/upgrade.md b/docs/docs/en/guide/upgrade/upgrade.md index dfb3016a07d6..261e2380c8fd 100644 --- a/docs/docs/en/guide/upgrade/upgrade.md +++ b/docs/docs/en/guide/upgrade/upgrade.md @@ -63,6 +63,14 @@ Execution result: - Migrate lineage data to new table `t_ds_workflow_task_lineage`. - This script only performs upsert operations, not deletes. You can delete it manually if you need to. +### Drop Data-Quality Related Table + +Execute script: `sh ./tools/bin/drop-table.sh`. + +Execution result: + +- This script only performs `drop table` operations. + ### Upgrade Service - If you deploy with Pseudo-Cluster deployment, change it according to [Pseudo-Cluster](../installation/pseudo-cluster.md) section "Modify Configuration". diff --git a/docs/docs/zh/guide/upgrade/upgrade.md b/docs/docs/zh/guide/upgrade/upgrade.md index 19ccad73cab2..5801b1ea55c7 100644 --- a/docs/docs/zh/guide/upgrade/upgrade.md +++ b/docs/docs/zh/guide/upgrade/upgrade.md @@ -62,6 +62,14 @@ jar 包 并添加到 `./tools/libs` 目录下，设置以下环境变量 - 原血缘数据迁移至新血缘表 `t_ds_workflow_task_lineage`。 - 此脚本仅执行 upsert 操作，不执行删除操作，如果需要删除，您可以手动删除。 +### 删除数据质量相关表 + +执行脚本：`sh ./tools/bin/drop-table.sh`。 + +执行结果： + +- 此脚本仅执行 `drop table` 操作。 + ### 服务升级 #### 修改配置内容 diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql index 0259e67c3c54..fe24c473b195 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql @@ -1103,322 +1103,6 @@ CREATE TABLE t_ds_alert_plugin_instance PRIMARY KEY (id) ); --- --- Table structure for table `t_ds_dq_comparison_type` --- -DROP TABLE IF EXISTS `t_ds_dq_comparison_type`; -CREATE TABLE `t_ds_dq_comparison_type` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `type` varchar(255) NOT NULL, - `execute_sql` text DEFAULT NULL, - `output_table` varchar(100) DEFAULT NULL, - `name` varchar(255) DEFAULT NULL, - `create_time` datetime DEFAULT NULL, - `update_time` datetime DEFAULT NULL, - `is_inner_source` tinyint(1) DEFAULT '0', - PRIMARY KEY (`id`) -)ENGINE=InnoDB DEFAULT CHARSET=utf8; - -INSERT INTO `t_ds_dq_comparison_type` -(`id`, `type`, `execute_sql`, `output_table`, `name`, `create_time`, `update_time`, `is_inner_source`) -VALUES(1, 'FixValue', NULL, NULL, NULL, '2021-06-30 00:00:00.000', '2021-06-30 00:00:00.000', false); -INSERT INTO `t_ds_dq_comparison_type` -(`id`, `type`, `execute_sql`, `output_table`, `name`, `create_time`, `update_time`, `is_inner_source`) -VALUES(2, 'DailyAvg', 'select round(avg(statistics_value),2) as day_avg from t_ds_dq_task_statistics_value where data_time >=date_trunc(''DAY'', ${data_time}) and data_time < date_add(date_trunc(''day'', ${data_time}),1) and unique_code = ${unique_code} and statistics_name = ''${statistics_name}''', 'day_range', 'day_range.day_avg', '2021-06-30 00:00:00.000', '2021-06-30 00:00:00.000', true); -INSERT INTO `t_ds_dq_comparison_type` -(`id`, `type`, `execute_sql`, `output_table`, `name`, `create_time`, `update_time`, `is_inner_source`) -VALUES(3, 'WeeklyAvg', 'select round(avg(statistics_value),2) as week_avg from t_ds_dq_task_statistics_value where data_time >= date_trunc(''WEEK'', ${data_time}) and data_time = date_trunc(''MONTH'', ${data_time}) and data_time = date_add(date_trunc(''day'', ${data_time}),-7) and data_time = date_add(date_trunc(''day'', ${data_time}),-30) and data_time < date_trunc(''day'', ${data_time}) and unique_code = ${unique_code} and statistics_name = ''${statistics_name}''', 'last_thirty_days', 'last_thirty_days.last_30_avg', '2021-06-30 00:00:00.000', '2021-06-30 00:00:00.000', true); -INSERT INTO `t_ds_dq_comparison_type` -(`id`, `type`, `execute_sql`, `output_table`, `name`, `create_time`, `update_time`, `is_inner_source`) -VALUES(7, 'SrcTableTotalRows', 'SELECT COUNT(*) AS total FROM ${src_table} WHERE (${src_filter})', 'total_count', 'total_count.total', '2021-06-30 00:00:00.000', '2021-06-30 00:00:00.000', false); -INSERT INTO `t_ds_dq_comparison_type` -(`id`, `type`, `execute_sql`, `output_table`, `name`, `create_time`, `update_time`, `is_inner_source`) -VALUES(8, 'TargetTableTotalRows', 'SELECT COUNT(*) AS total FROM ${target_table} WHERE (${target_filter})', 'total_count', 'total_count.total', '2021-06-30 00:00:00.000', '2021-06-30 00:00:00.000', false); - --- --- Table structure for table `t_ds_dq_execute_result` --- -DROP TABLE IF EXISTS `t_ds_dq_execute_result`; -CREATE TABLE `t_ds_dq_execute_result` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `workflow_definition_id` int(11) DEFAULT NULL, - `workflow_instance_id` int(11) DEFAULT NULL, - `task_instance_id` int(11) DEFAULT NULL, - `rule_type` int(11) DEFAULT NULL, - `rule_name` varchar(255) DEFAULT NULL, - `statistics_value` double DEFAULT NULL, - `comparison_value` double DEFAULT NULL, - `check_type` int(11) DEFAULT NULL, - `threshold` double DEFAULT NULL, - `operator` int(11) DEFAULT NULL, - `failure_strategy` int(11) DEFAULT NULL, - `state` int(11) DEFAULT NULL, - `user_id` int(11) DEFAULT NULL, - `comparison_type` int(11) DEFAULT NULL, - `error_output_path` text DEFAULT NULL, - `create_time` datetime DEFAULT NULL, - `update_time` datetime DEFAULT NULL, - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- --- Table structure for table t_ds_dq_rule --- -DROP TABLE IF EXISTS `t_ds_dq_rule`; -CREATE TABLE `t_ds_dq_rule` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `name` varchar(255) DEFAULT NULL, - `type` int(11) DEFAULT NULL, - `user_id` int(11) DEFAULT NULL, - `create_time` datetime DEFAULT NULL, - `update_time` datetime DEFAULT NULL, - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -INSERT INTO `t_ds_dq_rule` -(`id`, `name`, `type`, `user_id`, `create_time`, `update_time`) -VALUES(1, '$t(null_check)', 0, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000'); -INSERT INTO `t_ds_dq_rule` -(`id`, `name`, `type`, `user_id`, `create_time`, `update_time`) -VALUES(2, '$t(custom_sql)', 1, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000'); -INSERT INTO `t_ds_dq_rule` -(`id`, `name`, `type`, `user_id`, `create_time`, `update_time`) -VALUES(3, '$t(multi_table_accuracy)', 2, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000'); -INSERT INTO `t_ds_dq_rule` -(`id`, `name`, `type`, `user_id`, `create_time`, `update_time`) -VALUES(4, '$t(multi_table_value_comparison)', 3, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000'); -INSERT INTO `t_ds_dq_rule` -(`id`, `name`, `type`, `user_id`, `create_time`, `update_time`) -VALUES(5, '$t(field_length_check)', 0, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000'); -INSERT INTO `t_ds_dq_rule` -(`id`, `name`, `type`, `user_id`, `create_time`, `update_time`) -VALUES(6, '$t(uniqueness_check)', 0, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000'); -INSERT INTO `t_ds_dq_rule` -(`id`, `name`, `type`, `user_id`, `create_time`, `update_time`) -VALUES(7, '$t(regexp_check)', 0, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000'); -INSERT INTO `t_ds_dq_rule` -(`id`, `name`, `type`, `user_id`, `create_time`, `update_time`) -VALUES(8, '$t(timeliness_check)', 0, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000'); -INSERT INTO `t_ds_dq_rule` -(`id`, `name`, `type`, `user_id`, `create_time`, `update_time`) -VALUES(9, '$t(enumeration_check)', 0, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000'); -INSERT INTO `t_ds_dq_rule` -(`id`, `name`, `type`, `user_id`, `create_time`, `update_time`) -VALUES(10, '$t(table_count_check)', 0, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000'); - --- --- Table structure for table `t_ds_dq_rule_execute_sql` --- -DROP TABLE IF EXISTS `t_ds_dq_rule_execute_sql`; -CREATE TABLE `t_ds_dq_rule_execute_sql` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `index` int(11) DEFAULT NULL, - `sql` text DEFAULT NULL, - `table_alias` varchar(255) DEFAULT NULL, - `type` int(11) DEFAULT NULL, - `is_error_output_sql` tinyint(1) DEFAULT '0', - `create_time` datetime DEFAULT NULL, - `update_time` datetime DEFAULT NULL, - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -INSERT INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(1, 1, 'SELECT COUNT(*) AS nulls FROM null_items', 'null_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(2, 1, 'SELECT COUNT(*) AS total FROM ${src_table} WHERE (${src_filter})', 'total_count', 2, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(3, 1, 'SELECT COUNT(*) AS miss from miss_items', 'miss_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(4, 1, 'SELECT COUNT(*) AS valids FROM invalid_length_items', 'invalid_length_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(5, 1, 'SELECT COUNT(*) AS total FROM ${target_table} WHERE (${target_filter})', 'total_count', 2, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(6, 1, 'SELECT ${src_field} FROM ${src_table} group by ${src_field} having count(*) > 1', 'duplicate_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(7, 1, 'SELECT COUNT(*) AS duplicates FROM duplicate_items', 'duplicate_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(8, 1, 'SELECT ${src_table}.* FROM (SELECT * FROM ${src_table} WHERE (${src_filter})) ${src_table} LEFT JOIN (SELECT * FROM ${target_table} WHERE (${target_filter})) ${target_table} ON ${on_clause} WHERE ${where_clause}', 'miss_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(9, 1, 'SELECT * FROM ${src_table} WHERE (${src_field} not regexp ''${regexp_pattern}'') AND (${src_filter}) ', 'regexp_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(10, 1, 'SELECT COUNT(*) AS regexps FROM regexp_items', 'regexp_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(11, 1, 'SELECT * FROM ${src_table} WHERE (to_unix_timestamp(${src_field}, ''${datetime_format}'')-to_unix_timestamp(''${deadline}'', ''${datetime_format}'') <= 0) AND (to_unix_timestamp(${src_field}, ''${datetime_format}'')-to_unix_timestamp(''${begin_time}'', ''${datetime_format}'') >= 0) AND (${src_filter}) ', 'timeliness_items', 0, 1, '2021-03-03 11:31:24.0', '2021-03-03 11:31:24.0'); -INSERT INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(12, 1, 'SELECT COUNT(*) AS timeliness FROM timeliness_items', 'timeliness_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(13, 1, 'SELECT * FROM ${src_table} where (${src_field} not in ( ${enum_list} ) or ${src_field} is null) AND (${src_filter}) ', 'enum_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(14, 1, 'SELECT COUNT(*) AS enums FROM enum_items', 'enum_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(15, 1, 'SELECT COUNT(*) AS total FROM ${src_table} WHERE (${src_filter})', 'table_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(16, 1, 'SELECT * FROM ${src_table} WHERE (${src_field} is null or ${src_field} = '''') AND (${src_filter})', 'null_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(17, 1, 'SELECT * FROM ${src_table} WHERE (length(${src_field}) ${logic_operator} ${field_length}) AND (${src_filter})', 'invalid_length_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); - --- --- Table structure for table `t_ds_dq_rule_input_entry` --- -DROP TABLE IF EXISTS `t_ds_dq_rule_input_entry`; -CREATE TABLE `t_ds_dq_rule_input_entry` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `field` varchar(255) DEFAULT NULL, - `type` varchar(255) DEFAULT NULL, - `title` varchar(255) DEFAULT NULL, - `data` varchar(255) DEFAULT NULL, - `options` text DEFAULT NULL, - `placeholder` varchar(255) DEFAULT NULL, - `option_source_type` int(11) DEFAULT NULL, - `data_type` int(11) DEFAULT NULL, - `input_type` int(11) DEFAULT NULL, - `is_show` tinyint(1) DEFAULT '1', - `can_edit` tinyint(1) DEFAULT '1', - `is_emit` tinyint(1) DEFAULT '0', - `is_validate` tinyint(1) DEFAULT '1', - `create_time` datetime DEFAULT NULL, - `update_time` datetime DEFAULT NULL, - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(1, 'src_connector_type', 'select', '$t(src_connector_type)', '', '[{"label":"HIVE","value":"HIVE"},{"label":"JDBC","value":"JDBC"}]', 'please select source connector type', 2, 2, 0, 1, 1, 1, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(2, 'src_datasource_id', 'select', '$t(src_datasource_id)', '', NULL, 'please select source datasource id', 1, 2, 0, 1, 1, 1, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(3, 'src_table', 'select', '$t(src_table)', NULL, NULL, 'Please enter source table name', 0, 0, 0, 1, 1, 1, 1, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(4, 'src_filter', 'input', '$t(src_filter)', NULL, NULL, 'Please enter filter expression', 0, 3, 0, 1, 1, 0, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(5, 'src_field', 'select', '$t(src_field)', NULL, NULL, 'Please enter column, only single column is supported', 0, 0, 0, 1, 1, 0, 1, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(6, 'statistics_name', 'input', '$t(statistics_name)', NULL, NULL, 'Please enter statistics name, the alias in statistics execute sql', 0, 0, 1, 0, 0, 0, 1, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(7, 'check_type', 'select', '$t(check_type)', '0', '[{"label":"Expected - Actual","value":"0"},{"label":"Actual - Expected","value":"1"},{"label":"Actual / Expected","value":"2"},{"label":"(Expected - Actual) / Expected","value":"3"}]', 'please select check type', 0, 0, 3, 1, 1, 1, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(8, 'operator', 'select', '$t(operator)', '0', '[{"label":"=","value":"0"},{"label":"<","value":"1"},{"label":"<=","value":"2"},{"label":">","value":"3"},{"label":">=","value":"4"},{"label":"!=","value":"5"}]', 'please select operator', 0, 0, 3, 1, 1, 0, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(9, 'threshold', 'input', '$t(threshold)', NULL, NULL, 'Please enter threshold, number is needed', 0, 2, 3, 1, 1, 0, 1, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(10, 'failure_strategy', 'select', '$t(failure_strategy)', '0', '[{"label":"Alert","value":"0"},{"label":"Block","value":"1"}]', 'please select failure strategy', 0, 0, 3, 1, 1, 0, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(11, 'target_connector_type', 'select', '$t(target_connector_type)', '', '[{"label":"HIVE","value":"HIVE"},{"label":"JDBC","value":"JDBC"}]', 'Please select target connector type', 2, 0, 0, 1, 1, 1, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(12, 'target_datasource_id', 'select', '$t(target_datasource_id)', '', NULL, 'Please select target datasource', 1, 2, 0, 1, 1, 1, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(13, 'target_table', 'select', '$t(target_table)', NULL, NULL, 'Please enter target table', 0, 0, 0, 1, 1, 1, 1, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(14, 'target_filter', 'input', '$t(target_filter)', NULL, NULL, 'Please enter target filter expression', 0, 3, 0, 1, 1, 0, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(15, 'mapping_columns', 'group', '$t(mapping_columns)', NULL, '[{"field":"src_field","props":{"placeholder":"Please input src field","rows":0,"disabled":false,"size":"small"},"type":"input","title":"src_field"},{"field":"operator","props":{"placeholder":"Please input operator","rows":0,"disabled":false,"size":"small"},"type":"input","title":"operator"},{"field":"target_field","props":{"placeholder":"Please input target field","rows":0,"disabled":false,"size":"small"},"type":"input","title":"target_field"}]', 'please enter mapping columns', 0, 0, 0, 1, 1, 0, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(16, 'statistics_execute_sql', 'textarea', '$t(statistics_execute_sql)', NULL, NULL, 'Please enter statistics execute sql', 0, 3, 0, 1, 1, 0, 1, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(17, 'comparison_name', 'input', '$t(comparison_name)', NULL, NULL, 'Please enter comparison name, the alias in comparison execute sql', 0, 0, 0, 0, 0, 0, 1, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(18, 'comparison_execute_sql', 'textarea', '$t(comparison_execute_sql)', NULL, NULL, 'Please enter comparison execute sql', 0, 3, 0, 1, 1, 0, 1, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(19, 'comparison_type', 'select', '$t(comparison_type)', '', NULL, 'Please enter comparison title', 3, 0, 2, 1, 0, 1, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(20, 'writer_connector_type', 'select', '$t(writer_connector_type)', '', '[{"label":"MYSQL","value":"0"},{"label":"POSTGRESQL","value":"1"}]', 'please select writer connector type', 0, 2, 0, 1, 1, 1, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(21, 'writer_datasource_id', 'select', '$t(writer_datasource_id)', '', NULL, 'please select writer datasource id', 1, 2, 0, 1, 1, 0, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(22, 'target_field', 'select', '$t(target_field)', NULL, NULL, 'Please enter column, only single column is supported', 0, 0, 0, 1, 1, 0, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(23, 'field_length', 'input', '$t(field_length)', NULL, NULL, 'Please enter length limit', 0, 3, 0, 1, 1, 0, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(24, 'logic_operator', 'select', '$t(logic_operator)', '=', '[{"label":"=","value":"="},{"label":"<","value":"<"},{"label":"<=","value":"<="},{"label":">","value":">"},{"label":">=","value":">="},{"label":"<>","value":"<>"}]', 'please select logic operator', 0, 0, 3, 1, 1, 0, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(25, 'regexp_pattern', 'input', '$t(regexp_pattern)', NULL, NULL, 'Please enter regexp pattern', 0, 0, 0, 1, 1, 0, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(26, 'deadline', 'input', '$t(deadline)', NULL, NULL, 'Please enter deadline', 0, 0, 0, 1, 1, 0, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(27, 'datetime_format', 'input', '$t(datetime_format)', NULL, NULL, 'Please enter datetime format', 0, 0, 0, 1, 1, 0, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(28, 'enum_list', 'input', '$t(enum_list)', NULL, NULL, 'Please enter enumeration', 0, 0, 0, 1, 1, 0, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(29, 'begin_time', 'input', '$t(begin_time)', NULL, NULL, 'Please enter begin time', 0, 0, 0, 1, 1, 0, 0, '2021-03-03 11:31:24.0', '2021-03-03 11:31:24.0'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(30, 'src_database', 'select', '$t(src_database)', NULL, NULL, 'Please select source database', 0, 0, 0, 1, 1, 1, 1, '2021-03-03 11:31:24.0', '2021-03-03 11:31:24.0'); -INSERT INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(31, 'target_database', 'select', '$t(target_database)', NULL, NULL, 'Please select target database', 0, 0, 0, 1, 1, 1, 1, '2021-03-03 11:31:24.0', '2021-03-03 11:31:24.0'); - --- --- Table structure for table `t_ds_dq_task_statistics_value` --- -DROP TABLE IF EXISTS `t_ds_dq_task_statistics_value`; -CREATE TABLE `t_ds_dq_task_statistics_value` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `workflow_definition_id` int(11) DEFAULT NULL, - `task_instance_id` int(11) DEFAULT NULL, - `rule_id` int(11) NOT NULL, - `unique_code` varchar(255) NULL, - `statistics_name` varchar(255) NULL, - `statistics_value` double NULL, - `data_time` datetime DEFAULT NULL, - `create_time` datetime DEFAULT NULL, - `update_time` datetime DEFAULT NULL, - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -- -- Table structure for table `t_ds_relation_rule_execute_sql` -- diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql index cfb4997fbeab..192508e4890f 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql @@ -1093,323 +1093,6 @@ CREATE TABLE `t_ds_relation_project_worker_group` ( ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE = utf8_bin; --- --- Table structure for table `t_ds_dq_comparison_type` --- -DROP TABLE IF EXISTS `t_ds_dq_comparison_type`; -CREATE TABLE `t_ds_dq_comparison_type` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `type` varchar(255) NOT NULL, - `execute_sql` text DEFAULT NULL, - `output_table` varchar(100) DEFAULT NULL, - `name` varchar(255) DEFAULT NULL, - `create_time` datetime DEFAULT NULL, - `update_time` datetime DEFAULT NULL, - `is_inner_source` tinyint(1) DEFAULT '0', - PRIMARY KEY (`id`) -)ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE = utf8_bin; - - -INSERT IGNORE INTO `t_ds_dq_comparison_type` -(`id`, `type`, `execute_sql`, `output_table`, `name`, `create_time`, `update_time`, `is_inner_source`) -VALUES(1, 'FixValue', NULL, NULL, NULL, current_timestamp, current_timestamp, false); -INSERT IGNORE INTO `t_ds_dq_comparison_type` -(`id`, `type`, `execute_sql`, `output_table`, `name`, `create_time`, `update_time`, `is_inner_source`) -VALUES(2, 'DailyAvg', 'select round(avg(statistics_value),2) as day_avg from t_ds_dq_task_statistics_value where data_time >=date_trunc(''DAY'', ${data_time}) and data_time < date_add(date_trunc(''day'', ${data_time}),1) and unique_code = ${unique_code} and statistics_name = ''${statistics_name}''', 'day_range', 'day_range.day_avg', current_timestamp, current_timestamp, true); -INSERT IGNORE INTO `t_ds_dq_comparison_type` -(`id`, `type`, `execute_sql`, `output_table`, `name`, `create_time`, `update_time`, `is_inner_source`) -VALUES(3, 'WeeklyAvg', 'select round(avg(statistics_value),2) as week_avg from t_ds_dq_task_statistics_value where data_time >= date_trunc(''WEEK'', ${data_time}) and data_time = date_trunc(''MONTH'', ${data_time}) and data_time = date_add(date_trunc(''day'', ${data_time}),-7) and data_time = date_add(date_trunc(''day'', ${data_time}),-30) and data_time < date_trunc(''day'', ${data_time}) and unique_code = ${unique_code} and statistics_name = ''${statistics_name}''', 'last_thirty_days', 'last_thirty_days.last_30_avg', current_timestamp, current_timestamp, true); -INSERT IGNORE INTO `t_ds_dq_comparison_type` -(`id`, `type`, `execute_sql`, `output_table`, `name`, `create_time`, `update_time`, `is_inner_source`) -VALUES(7, 'SrcTableTotalRows', 'SELECT COUNT(*) AS total FROM ${src_table} WHERE (${src_filter})', 'total_count', 'total_count.total', current_timestamp, current_timestamp, false); -INSERT IGNORE INTO `t_ds_dq_comparison_type` -(`id`, `type`, `execute_sql`, `output_table`, `name`, `create_time`, `update_time`, `is_inner_source`) -VALUES(8, 'TargetTableTotalRows', 'SELECT COUNT(*) AS total FROM ${target_table} WHERE (${target_filter})', 'total_count', 'total_count.total', current_timestamp, current_timestamp, false); - --- --- Table structure for table `t_ds_dq_execute_result` --- -DROP TABLE IF EXISTS `t_ds_dq_execute_result`; -CREATE TABLE `t_ds_dq_execute_result` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `workflow_definition_id` int(11) DEFAULT NULL, - `workflow_instance_id` int(11) DEFAULT NULL, - `task_instance_id` int(11) DEFAULT NULL, - `rule_type` int(11) DEFAULT NULL, - `rule_name` varchar(255) DEFAULT NULL, - `statistics_value` double DEFAULT NULL, - `comparison_value` double DEFAULT NULL, - `check_type` int(11) DEFAULT NULL, - `threshold` double DEFAULT NULL, - `operator` int(11) DEFAULT NULL, - `failure_strategy` int(11) DEFAULT NULL, - `state` int(11) DEFAULT NULL, - `user_id` int(11) DEFAULT NULL, - `comparison_type` int(11) DEFAULT NULL, - `error_output_path` text DEFAULT NULL, - `create_time` datetime DEFAULT NULL, - `update_time` datetime DEFAULT NULL, - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE = utf8_bin; - --- --- Table structure for table t_ds_dq_rule --- -DROP TABLE IF EXISTS `t_ds_dq_rule`; -CREATE TABLE `t_ds_dq_rule` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `name` varchar(255) DEFAULT NULL, - `type` int(11) DEFAULT NULL, - `user_id` int(11) DEFAULT NULL, - `create_time` datetime DEFAULT NULL, - `update_time` datetime DEFAULT NULL, - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE = utf8_bin; - -INSERT IGNORE INTO `t_ds_dq_rule` -(`id`, `name`, `type`, `user_id`, `create_time`, `update_time`) -VALUES(1, '$t(null_check)', 0, 1, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule` -(`id`, `name`, `type`, `user_id`, `create_time`, `update_time`) -VALUES(2, '$t(custom_sql)', 1, 1, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule` -(`id`, `name`, `type`, `user_id`, `create_time`, `update_time`) -VALUES(3, '$t(multi_table_accuracy)', 2, 1, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule` -(`id`, `name`, `type`, `user_id`, `create_time`, `update_time`) -VALUES(4, '$t(multi_table_value_comparison)', 3, 1, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule` -(`id`, `name`, `type`, `user_id`, `create_time`, `update_time`) -VALUES(5, '$t(field_length_check)', 0, 1, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule` -(`id`, `name`, `type`, `user_id`, `create_time`, `update_time`) -VALUES(6, '$t(uniqueness_check)', 0, 1, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule` -(`id`, `name`, `type`, `user_id`, `create_time`, `update_time`) -VALUES(7, '$t(regexp_check)', 0, 1, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule` -(`id`, `name`, `type`, `user_id`, `create_time`, `update_time`) -VALUES(8, '$t(timeliness_check)', 0, 1, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule` -(`id`, `name`, `type`, `user_id`, `create_time`, `update_time`) -VALUES(9, '$t(enumeration_check)', 0, 1, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule` -(`id`, `name`, `type`, `user_id`, `create_time`, `update_time`) -VALUES(10, '$t(table_count_check)', 0, 1, current_timestamp, current_timestamp); - --- --- Table structure for table `t_ds_dq_rule_execute_sql` --- -DROP TABLE IF EXISTS `t_ds_dq_rule_execute_sql`; -CREATE TABLE `t_ds_dq_rule_execute_sql` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `index` int(11) DEFAULT NULL, - `sql` text DEFAULT NULL, - `table_alias` varchar(255) DEFAULT NULL, - `type` int(11) DEFAULT NULL, - `is_error_output_sql` tinyint(1) DEFAULT '0', - `create_time` datetime DEFAULT NULL, - `update_time` datetime DEFAULT NULL, - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE = utf8_bin; - -INSERT IGNORE INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(1, 1, 'SELECT COUNT(*) AS nulls FROM null_items', 'null_count', 1, false, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(2, 1, 'SELECT COUNT(*) AS total FROM ${src_table} WHERE (${src_filter})', 'total_count', 2, false, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(3, 1, 'SELECT COUNT(*) AS miss from miss_items', 'miss_count', 1, false, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(4, 1, 'SELECT COUNT(*) AS valids FROM invalid_length_items', 'invalid_length_count', 1, false, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(5, 1, 'SELECT COUNT(*) AS total FROM ${target_table} WHERE (${target_filter})', 'total_count', 2, false, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(6, 1, 'SELECT ${src_field} FROM ${src_table} group by ${src_field} having count(*) > 1', 'duplicate_items', 0, true, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(7, 1, 'SELECT COUNT(*) AS duplicates FROM duplicate_items', 'duplicate_count', 1, false, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(8, 1, 'SELECT ${src_table}.* FROM (SELECT * FROM ${src_table} WHERE (${src_filter})) ${src_table} LEFT JOIN (SELECT * FROM ${target_table} WHERE (${target_filter})) ${target_table} ON ${on_clause} WHERE ${where_clause}', 'miss_items', 0, true, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(9, 1, 'SELECT * FROM ${src_table} WHERE (${src_field} not regexp ''${regexp_pattern}'') AND (${src_filter}) ', 'regexp_items', 0, true, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(10, 1, 'SELECT COUNT(*) AS regexps FROM regexp_items', 'regexp_count', 1, false, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(11, 1, 'SELECT * FROM ${src_table} WHERE (to_unix_timestamp(${src_field}, ''${datetime_format}'')-to_unix_timestamp(''${deadline}'', ''${datetime_format}'') <= 0) AND (to_unix_timestamp(${src_field}, ''${datetime_format}'')-to_unix_timestamp(''${begin_time}'', ''${datetime_format}'') >= 0) AND (${src_filter}) ', 'timeliness_items', 0, 1, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(12, 1, 'SELECT COUNT(*) AS timeliness FROM timeliness_items', 'timeliness_count', 1, false, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(13, 1, 'SELECT * FROM ${src_table} where (${src_field} not in ( ${enum_list} ) or ${src_field} is null) AND (${src_filter}) ', 'enum_items', 0, true, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(14, 1, 'SELECT COUNT(*) AS enums FROM enum_items', 'enum_count', 1, false, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(15, 1, 'SELECT COUNT(*) AS total FROM ${src_table} WHERE (${src_filter})', 'table_count', 1, false, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(16, 1, 'SELECT * FROM ${src_table} WHERE (${src_field} is null or ${src_field} = '''') AND (${src_filter})', 'null_items', 0, true, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_execute_sql` -(`id`, `index`, `sql`, `table_alias`, `type`, `is_error_output_sql`, `create_time`, `update_time`) -VALUES(17, 1, 'SELECT * FROM ${src_table} WHERE (length(${src_field}) ${logic_operator} ${field_length}) AND (${src_filter})', 'invalid_length_items', 0, true, current_timestamp, current_timestamp); - --- --- Table structure for table `t_ds_dq_rule_input_entry` --- -DROP TABLE IF EXISTS `t_ds_dq_rule_input_entry`; -CREATE TABLE `t_ds_dq_rule_input_entry` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `field` varchar(255) DEFAULT NULL, - `type` varchar(255) DEFAULT NULL, - `title` varchar(255) DEFAULT NULL, - `data` varchar(255) DEFAULT NULL, - `options` text DEFAULT NULL, - `placeholder` varchar(255) DEFAULT NULL, - `option_source_type` int(11) DEFAULT NULL, - `data_type` int(11) DEFAULT NULL, - `input_type` int(11) DEFAULT NULL, - `is_show` tinyint(1) DEFAULT '1', - `can_edit` tinyint(1) DEFAULT '1', - `is_emit` tinyint(1) DEFAULT '0', - `is_validate` tinyint(1) DEFAULT '1', - `create_time` datetime DEFAULT NULL, - `update_time` datetime DEFAULT NULL, - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE = utf8_bin; - -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(1, 'src_connector_type', 'select', '$t(src_connector_type)', '', '[{"label":"HIVE","value":"HIVE"},{"label":"JDBC","value":"JDBC"}]', 'please select source connector type', 2, 2, 0, 1, 1, 1, 0, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(2, 'src_datasource_id', 'select', '$t(src_datasource_id)', '', NULL, 'please select source datasource id', 1, 2, 0, 1, 1, 1, 0, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(3, 'src_table', 'select', '$t(src_table)', NULL, NULL, 'Please enter source table name', 0, 0, 0, 1, 1, 1, 1, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(4, 'src_filter', 'input', '$t(src_filter)', NULL, NULL, 'Please enter filter expression', 0, 3, 0, 1, 1, 0, 0, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(5, 'src_field', 'select', '$t(src_field)', NULL, NULL, 'Please enter column, only single column is supported', 0, 0, 0, 1, 1, 0, 1, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(6, 'statistics_name', 'input', '$t(statistics_name)', NULL, NULL, 'Please enter statistics name, the alias in statistics execute sql', 0, 0, 1, 0, 0, 0, 1, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(7, 'check_type', 'select', '$t(check_type)', '0', '[{"label":"Expected - Actual","value":"0"},{"label":"Actual - Expected","value":"1"},{"label":"Actual / Expected","value":"2"},{"label":"(Expected - Actual) / Expected","value":"3"}]', 'please select check type', 0, 0, 3, 1, 1, 1, 0, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(8, 'operator', 'select', '$t(operator)', '0', '[{"label":"=","value":"0"},{"label":"<","value":"1"},{"label":"<=","value":"2"},{"label":">","value":"3"},{"label":">=","value":"4"},{"label":"!=","value":"5"}]', 'please select operator', 0, 0, 3, 1, 1, 0, 0, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(9, 'threshold', 'input', '$t(threshold)', NULL, NULL, 'Please enter threshold, number is needed', 0, 2, 3, 1, 1, 0, 1, current_timestamp, current_timestamp); -INSERT Ignore INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(10, 'failure_strategy', 'select', '$t(failure_strategy)', '0', '[{"label":"Alert","value":"0"},{"label":"Block","value":"1"}]', 'please select failure strategy', 0, 0, 3, 1, 1, 0, 0, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(11, 'target_connector_type', 'select', '$t(target_connector_type)', '', '[{"label":"HIVE","value":"HIVE"},{"label":"JDBC","value":"JDBC"}]', 'Please select target connector type', 2, 0, 0, 1, 1, 1, 0, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(12, 'target_datasource_id', 'select', '$t(target_datasource_id)', '', NULL, 'Please select target datasource', 1, 2, 0, 1, 1, 1, 0, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(13, 'target_table', 'select', '$t(target_table)', NULL, NULL, 'Please enter target table', 0, 0, 0, 1, 1, 1, 1, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(14, 'target_filter', 'input', '$t(target_filter)', NULL, NULL, 'Please enter target filter expression', 0, 3, 0, 1, 1, 0, 0, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(15, 'mapping_columns', 'group', '$t(mapping_columns)', NULL, '[{"field":"src_field","props":{"placeholder":"Please input src field","rows":0,"disabled":false,"size":"small"},"type":"input","title":"src_field"},{"field":"operator","props":{"placeholder":"Please input operator","rows":0,"disabled":false,"size":"small"},"type":"input","title":"operator"},{"field":"target_field","props":{"placeholder":"Please input target field","rows":0,"disabled":false,"size":"small"},"type":"input","title":"target_field"}]', 'please enter mapping columns', 0, 0, 0, 1, 1, 0, 0, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(16, 'statistics_execute_sql', 'textarea', '$t(statistics_execute_sql)', NULL, NULL, 'Please enter statistics execute sql', 0, 3, 0, 1, 1, 0, 1, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(17, 'comparison_name', 'input', '$t(comparison_name)', NULL, NULL, 'Please enter comparison name, the alias in comparison execute sql', 0, 0, 0, 0, 0, 0, 1, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(18, 'comparison_execute_sql', 'textarea', '$t(comparison_execute_sql)', NULL, NULL, 'Please enter comparison execute sql', 0, 3, 0, 1, 1, 0, 1, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(19, 'comparison_type', 'select', '$t(comparison_type)', '', NULL, 'Please enter comparison title', 3, 0, 2, 1, 0, 1, 0, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(20, 'writer_connector_type', 'select', '$t(writer_connector_type)', '', '[{"label":"MYSQL","value":"0"},{"label":"POSTGRESQL","value":"1"}]', 'please select writer connector type', 0, 2, 0, 1, 1, 1, 0, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(21, 'writer_datasource_id', 'select', '$t(writer_datasource_id)', '', NULL, 'please select writer datasource id', 1, 2, 0, 1, 1, 0, 0, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(22, 'target_field', 'select', '$t(target_field)', NULL, NULL, 'Please enter column, only single column is supported', 0, 0, 0, 1, 1, 0, 0, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(23, 'field_length', 'input', '$t(field_length)', NULL, NULL, 'Please enter length limit', 0, 3, 0, 1, 1, 0, 0, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(24, 'logic_operator', 'select', '$t(logic_operator)', '=', '[{"label":"=","value":"="},{"label":"<","value":"<"},{"label":"<=","value":"<="},{"label":">","value":">"},{"label":">=","value":">="},{"label":"<>","value":"<>"}]', 'please select logic operator', 0, 0, 3, 1, 1, 0, 0, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(25, 'regexp_pattern', 'input', '$t(regexp_pattern)', NULL, NULL, 'Please enter regexp pattern', 0, 0, 0, 1, 1, 0, 0, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(26, 'deadline', 'input', '$t(deadline)', NULL, NULL, 'Please enter deadline', 0, 0, 0, 1, 1, 0, 0, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(27, 'datetime_format', 'input', '$t(datetime_format)', NULL, NULL, 'Please enter datetime format', 0, 0, 0, 1, 1, 0, 0, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(28, 'enum_list', 'input', '$t(enum_list)', NULL, NULL, 'Please enter enumeration', 0, 0, 0, 1, 1, 0, 0, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(29, 'begin_time', 'input', '$t(begin_time)', NULL, NULL, 'Please enter begin time', 0, 0, 0, 1, 1, 0, 0, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(30, 'src_database', 'select', '$t(src_database)', NULL, NULL, 'Please select source database', 0, 0, 0, 1, 1, 1, 1, current_timestamp, current_timestamp); -INSERT IGNORE INTO `t_ds_dq_rule_input_entry` -(`id`, `field`, `type`, `title`, `data`, `options`, `placeholder`, `option_source_type`, `data_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`) -VALUES(31, 'target_database', 'select', '$t(target_database)', NULL, NULL, 'Please select target database', 0, 0, 0, 1, 1, 1, 1, current_timestamp, current_timestamp); - --- --- Table structure for table `t_ds_dq_task_statistics_value` --- -DROP TABLE IF EXISTS `t_ds_dq_task_statistics_value`; -CREATE TABLE `t_ds_dq_task_statistics_value` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `workflow_definition_id` int(11) DEFAULT NULL, - `task_instance_id` int(11) DEFAULT NULL, - `rule_id` int(11) NOT NULL, - `unique_code` varchar(255) NULL, - `statistics_name` varchar(255) NULL, - `statistics_value` double NULL, - `data_time` datetime DEFAULT NULL, - `create_time` datetime DEFAULT NULL, - `update_time` datetime DEFAULT NULL, - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE = utf8_bin; - -- -- Table structure for table `t_ds_relation_rule_execute_sql` -- diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql index 6243772bf2d9..44ec38afa439 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql @@ -1095,317 +1095,6 @@ CREATE TABLE t_ds_alert_plugin_instance ( CONSTRAINT t_ds_alert_plugin_instance_pk PRIMARY KEY (id) ); --- --- Table structure for table t_ds_dq_comparison_type --- -DROP TABLE IF EXISTS t_ds_dq_comparison_type; -CREATE TABLE t_ds_dq_comparison_type ( - id serial NOT NULL, - "type" varchar NOT NULL, - execute_sql varchar NULL, - output_table varchar NULL, - "name" varchar NULL, - create_time timestamp NULL, - update_time timestamp NULL, - is_inner_source bool NULL, - CONSTRAINT t_ds_dq_comparison_type_pk PRIMARY KEY (id) -); -INSERT INTO t_ds_dq_comparison_type -(id, "type", execute_sql, output_table, "name", create_time, update_time, is_inner_source) -VALUES(1, 'FixValue', NULL, NULL, NULL, '2021-06-30 00:00:00.000', '2021-06-30 00:00:00.000', false); -INSERT INTO t_ds_dq_comparison_type -(id, "type", execute_sql, output_table, "name", create_time, update_time, is_inner_source) -VALUES(2, 'DailyAvg', 'select round(avg(statistics_value),2) as day_avg from t_ds_dq_task_statistics_value where data_time >=date_trunc(''DAY'', ${data_time}) and data_time < date_add(date_trunc(''day'', ${data_time}),1) and unique_code = ${unique_code} and statistics_name = ''${statistics_name}''', 'day_range', 'day_range.day_avg', '2021-06-30 00:00:00.000', '2021-06-30 00:00:00.000', true); -INSERT INTO t_ds_dq_comparison_type -(id, "type", execute_sql, output_table, "name", create_time, update_time, is_inner_source) -VALUES(3, 'WeeklyAvg', 'select round(avg(statistics_value),2) as week_avg from t_ds_dq_task_statistics_value where data_time >= date_trunc(''WEEK'', ${data_time}) and data_time = date_trunc(''MONTH'', ${data_time}) and data_time = date_add(date_trunc(''day'', ${data_time}),-7) and data_time = date_add(date_trunc(''day'', ${data_time}),-30) and data_time < date_trunc(''day'', ${data_time}) and unique_code = ${unique_code} and statistics_name = ''${statistics_name}''', 'last_thirty_days', 'last_thirty_days.last_30_avg', '2021-06-30 00:00:00.000', '2021-06-30 00:00:00.000', true); -INSERT INTO t_ds_dq_comparison_type -(id, "type", execute_sql, output_table, "name", create_time, update_time, is_inner_source) -VALUES(7, 'SrcTableTotalRows', 'SELECT COUNT(*) AS total FROM ${src_table} WHERE (${src_filter})', 'total_count', 'total_count.total', '2021-06-30 00:00:00.000', '2021-06-30 00:00:00.000', false); -INSERT INTO t_ds_dq_comparison_type -(id, "type", execute_sql, output_table, "name", create_time, update_time, is_inner_source) -VALUES(8, 'TargetTableTotalRows', 'SELECT COUNT(*) AS total FROM ${target_table} WHERE (${target_filter})', 'total_count', 'total_count.total', '2021-06-30 00:00:00.000', '2021-06-30 00:00:00.000', false); - --- --- Table structure for table t_ds_dq_execute_result --- -DROP TABLE IF EXISTS t_ds_dq_execute_result; -CREATE TABLE t_ds_dq_execute_result ( - id serial NOT NULL, - workflow_definition_id int4 NULL, - workflow_instance_id int4 NULL, - task_instance_id int4 NULL, - rule_type int4 NULL, - rule_name varchar(255) DEFAULT NULL, - statistics_value float8 NULL, - comparison_value float8 NULL, - check_type int4 NULL, - threshold float8 NULL, - "operator" int4 NULL, - failure_strategy int4 NULL, - state int4 NULL, - user_id int4 NULL, - create_time timestamp NULL, - update_time timestamp NULL, - comparison_type int4 NULL, - error_output_path text NULL, - CONSTRAINT t_ds_dq_execute_result_pk PRIMARY KEY (id) -); - --- --- Table structure for table t_ds_dq_rule --- -DROP TABLE IF EXISTS t_ds_dq_rule; -CREATE TABLE t_ds_dq_rule ( - id serial NOT NULL, - "name" varchar(255) DEFAULT NULL, - "type" int4 NULL, - user_id int4 NULL, - create_time timestamp NULL, - update_time timestamp NULL, - CONSTRAINT t_ds_dq_rule_pk PRIMARY KEY (id) -); -INSERT INTO t_ds_dq_rule -(id, "name", "type", user_id, create_time, update_time) -VALUES(1, '$t(null_check)', 0, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000'); -INSERT INTO t_ds_dq_rule -(id, "name", "type", user_id, create_time, update_time) -VALUES(2, '$t(custom_sql)', 1, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000'); -INSERT INTO t_ds_dq_rule -(id, "name", "type", user_id, create_time, update_time) -VALUES(3, '$t(multi_table_accuracy)', 2, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000'); -INSERT INTO t_ds_dq_rule -(id, "name", "type", user_id, create_time, update_time) -VALUES(4, '$t(multi_table_value_comparison)', 3, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000'); -INSERT INTO t_ds_dq_rule -(id, "name", "type", user_id, create_time, update_time) -VALUES(5, '$t(field_length_check)', 0, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000'); -INSERT INTO t_ds_dq_rule -(id, "name", "type", user_id, create_time, update_time) -VALUES(6, '$t(uniqueness_check)', 0, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000'); -INSERT INTO t_ds_dq_rule -(id, "name", "type", user_id, create_time, update_time) -VALUES(7, '$t(regexp_check)', 0, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000'); -INSERT INTO t_ds_dq_rule -(id, "name", "type", user_id, create_time, update_time) -VALUES(8, '$t(timeliness_check)', 0, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000'); -INSERT INTO t_ds_dq_rule -(id, "name", "type", user_id, create_time, update_time) -VALUES(9, '$t(enumeration_check)', 0, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000'); -INSERT INTO t_ds_dq_rule -(id, "name", "type", user_id, create_time, update_time) -VALUES(10, '$t(table_count_check)', 0, 1, '2020-01-12 00:00:00.000', '2020-01-12 00:00:00.000'); - --- --- Table structure for table t_ds_dq_rule_execute_sql --- -DROP TABLE IF EXISTS t_ds_dq_rule_execute_sql; -CREATE TABLE t_ds_dq_rule_execute_sql ( - id serial NOT NULL, - "index" int4 NULL, - "sql" text NULL, - table_alias varchar(255) DEFAULT NULL, - "type" int4 NULL, - create_time timestamp NULL, - update_time timestamp NULL, - is_error_output_sql bool NULL, - CONSTRAINT t_ds_dq_rule_execute_sql_pk PRIMARY KEY (id) -); -INSERT INTO t_ds_dq_rule_execute_sql -(id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) -VALUES(1, 1, 'SELECT COUNT(*) AS nulls FROM null_items', 'null_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_execute_sql -(id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) -VALUES(2, 1, 'SELECT COUNT(*) AS total FROM ${src_table} WHERE (${src_filter})', 'total_count', 2, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_execute_sql -(id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) -VALUES(3, 1, 'SELECT COUNT(*) AS miss from miss_items', 'miss_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_execute_sql -(id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) -VALUES(4, 1, 'SELECT COUNT(*) AS valids FROM invalid_length_items', 'invalid_length_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_execute_sql -(id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) -VALUES(5, 1, 'SELECT COUNT(*) AS total FROM ${target_table} WHERE (${target_filter})', 'total_count', 2, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_execute_sql -(id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) -VALUES(6, 1, 'SELECT ${src_field} FROM ${src_table} group by ${src_field} having count(*) > 1', 'duplicate_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_execute_sql -(id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) -VALUES(7, 1, 'SELECT COUNT(*) AS duplicates FROM duplicate_items', 'duplicate_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_execute_sql -(id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) -VALUES(8, 1, 'SELECT ${src_table}.* FROM (SELECT * FROM ${src_table} WHERE (${src_filter})) ${src_table} LEFT JOIN (SELECT * FROM ${target_table} WHERE (${target_filter})) ${target_table} ON ${on_clause} WHERE ${where_clause}', 'miss_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_execute_sql -(id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) -VALUES(9, 1, 'SELECT * FROM ${src_table} WHERE (${src_field} not regexp ''${regexp_pattern}'') AND (${src_filter}) ', 'regexp_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_execute_sql -(id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) -VALUES(10, 1, 'SELECT COUNT(*) AS regexps FROM regexp_items', 'regexp_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_execute_sql -(id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) -VALUES(11, 1, 'SELECT * FROM ${src_table} WHERE (to_unix_timestamp(${src_field}, ''${datetime_format}'')-to_unix_timestamp(''${deadline}'', ''${datetime_format}'') <= 0) AND (to_unix_timestamp(${src_field}, ''${datetime_format}'')-to_unix_timestamp(''${begin_time}'', ''${datetime_format}'') >= 0) AND (${src_filter}) ', 'timeliness_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_execute_sql -(id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) -VALUES(12, 1, 'SELECT COUNT(*) AS timeliness FROM timeliness_items', 'timeliness_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_execute_sql -(id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) -VALUES(13, 1, 'SELECT * FROM ${src_table} where (${src_field} not in ( ${enum_list} ) or ${src_field} is null) AND (${src_filter}) ', 'enum_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_execute_sql -(id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) -VALUES(14, 1, 'SELECT COUNT(*) AS enums FROM enum_items', 'enum_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_execute_sql -(id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) -VALUES(15, 1, 'SELECT COUNT(*) AS total FROM ${src_table} WHERE (${src_filter})', 'table_count', 1, false, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_execute_sql -(id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) -VALUES(16, 1, 'SELECT * FROM ${src_table} WHERE (${src_field} is null or ${src_field} = '''') AND (${src_filter})', 'null_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_execute_sql -(id, "index", "sql", table_alias, "type", is_error_output_sql, create_time, update_time) -VALUES(17, 1, 'SELECT * FROM ${src_table} WHERE (length(${src_field}) ${logic_operator} ${field_length}) AND (${src_filter})', 'invalid_length_items', 0, true, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); - --- --- Table structure for table t_ds_dq_rule_input_entry --- -DROP TABLE IF EXISTS t_ds_dq_rule_input_entry; -CREATE TABLE t_ds_dq_rule_input_entry ( - id serial NOT NULL, - field varchar(255) DEFAULT NULL, - "type" varchar(255) DEFAULT NULL, - title varchar(255) DEFAULT NULL, - data varchar(255) DEFAULT NULL, - "options" text DEFAULT NULL, - placeholder varchar(255) DEFAULT NULL, - option_source_type int4 NULL, - data_type int4 NULL, - input_type int4 NULL, - is_show int2 NULL DEFAULT '1'::smallint, - can_edit int2 NULL DEFAULT '1'::smallint, - is_emit int2 NULL DEFAULT '0'::smallint, - is_validate int2 NULL DEFAULT '0'::smallint, - create_time timestamp NULL, - update_time timestamp NULL, - CONSTRAINT t_ds_dq_rule_input_entry_pk PRIMARY KEY (id) -); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(1, 'src_connector_type', 'select', '$t(src_connector_type)', '', '[{"label":"HIVE","value":"HIVE"},{"label":"JDBC","value":"JDBC"}]', 'please select source connector type', 2, 2, 0, 1, 1, 1, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(2, 'src_datasource_id', 'select', '$t(src_datasource_id)', '', NULL, 'please select source datasource id', 1, 2, 0, 1, 1, 1, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(3, 'src_table', 'select', '$t(src_table)', NULL, NULL, 'Please enter source table name', 0, 0, 0, 1, 1, 1, 1, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(4, 'src_filter', 'input', '$t(src_filter)', NULL, NULL, 'Please enter filter expression', 0, 3, 0, 1, 1, 0, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(5, 'src_field', 'select', '$t(src_field)', NULL, NULL, 'Please enter column, only single column is supported', 0, 0, 0, 1, 1, 0, 1, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(6, 'statistics_name', 'input', '$t(statistics_name)', NULL, NULL, 'Please enter statistics name, the alias in statistics execute sql', 0, 0, 1, 0, 0, 0, 1, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(7, 'check_type', 'select', '$t(check_type)', '0', '[{"label":"Expected - Actual","value":"0"},{"label":"Actual - Expected","value":"1"},{"label":"Actual / Expected","value":"2"},{"label":"(Expected - Actual) / Expected","value":"3"}]', 'please select check type', 0, 0, 3, 1, 1, 1, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(8, 'operator', 'select', '$t(operator)', '0', '[{"label":"=","value":"0"},{"label":"<","value":"1"},{"label":"<=","value":"2"},{"label":">","value":"3"},{"label":">=","value":"4"},{"label":"!=","value":"5"}]', 'please select operator', 0, 0, 3, 1, 1, 0, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(9, 'threshold', 'input', '$t(threshold)', NULL, NULL, 'Please enter threshold, number is needed', 0, 2, 3, 1, 1, 0, 1, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(10, 'failure_strategy', 'select', '$t(failure_strategy)', '0', '[{"label":"Alert","value":"0"},{"label":"Block","value":"1"}]', 'please select failure strategy', 0, 0, 3, 1, 1, 0, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(11, 'target_connector_type', 'select', '$t(target_connector_type)', '', '[{"label":"HIVE","value":"HIVE"},{"label":"JDBC","value":"JDBC"}]', 'Please select target connector type', 2, 0, 0, 1, 1, 1, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(12, 'target_datasource_id', 'select', '$t(target_datasource_id)', '', NULL, 'Please select target datasource', 1, 2, 0, 1, 1, 1, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(13, 'target_table', 'select', '$t(target_table)', NULL, NULL, 'Please enter target table', 0, 0, 0, 1, 1, 1, 1, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(14, 'target_filter', 'input', '$t(target_filter)', NULL, NULL, 'Please enter target filter expression', 0, 3, 0, 1, 1, 0, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(15, 'mapping_columns', 'group', '$t(mapping_columns)', NULL, '[{"field":"src_field","props":{"placeholder":"Please input src field","rows":0,"disabled":false,"size":"small"},"type":"input","title":"src_field"},{"field":"operator","props":{"placeholder":"Please input operator","rows":0,"disabled":false,"size":"small"},"type":"input","title":"operator"},{"field":"target_field","props":{"placeholder":"Please input target field","rows":0,"disabled":false,"size":"small"},"type":"input","title":"target_field"}]', 'please enter mapping columns', 0, 0, 0, 1, 1, 0, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(16, 'statistics_execute_sql', 'textarea', '$t(statistics_execute_sql)', NULL, NULL, 'Please enter statistics execute sql', 0, 3, 0, 1, 1, 0, 1, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(17, 'comparison_name', 'input', '$t(comparison_name)', NULL, NULL, 'Please enter comparison name, the alias in comparison execute sql', 0, 0, 0, 0, 0, 0, 1, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(18, 'comparison_execute_sql', 'textarea', '$t(comparison_execute_sql)', NULL, NULL, 'Please enter comparison execute sql', 0, 3, 0, 1, 1, 0, 1, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(19, 'comparison_type', 'select', '$t(comparison_type)', '', NULL, 'Please enter comparison title', 3, 0, 2, 1, 0, 1, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(20, 'writer_connector_type', 'select', '$t(writer_connector_type)', '', '[{"label":"MYSQL","value":"0"},{"label":"POSTGRESQL","value":"1"}]', 'please select writer connector type', 0, 2, 0, 1, 1, 1, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(21, 'writer_datasource_id', 'select', '$t(writer_datasource_id)', '', NULL, 'please select writer datasource id', 1, 2, 0, 1, 1, 0, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(22, 'target_field', 'select', '$t(target_field)', NULL, NULL, 'Please enter column, only single column is supported', 0, 0, 0, 1, 1, 0, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(23, 'field_length', 'input', '$t(field_length)', NULL, NULL, 'Please enter length limit', 0, 3, 0, 1, 1, 0, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(24, 'logic_operator', 'select', '$t(logic_operator)', '=', '[{"label":"=","value":"="},{"label":"<","value":"<"},{"label":"<=","value":"<="},{"label":">","value":">"},{"label":">=","value":">="},{"label":"<>","value":"<>"}]', 'please select logic operator', 0, 0, 3, 1, 1, 0, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(25, 'regexp_pattern', 'input', '$t(regexp_pattern)', NULL, NULL, 'Please enter regexp pattern', 0, 0, 0, 1, 1, 0, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(26, 'deadline', 'input', '$t(deadline)', NULL, NULL, 'Please enter deadline', 0, 0, 0, 1, 1, 0, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(27, 'datetime_format', 'input', '$t(datetime_format)', NULL, NULL, 'Please enter datetime format', 0, 0, 0, 1, 1, 0, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(28, 'enum_list', 'input', '$t(enum_list)', NULL, NULL, 'Please enter enumeration', 0, 0, 0, 1, 1, 0, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(29, 'begin_time', 'input', '$t(begin_time)', NULL, NULL, 'Please enter begin time', 0, 0, 0, 1, 1, 0, 0, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(30, 'src_database', 'select', '$t(src_database)', NULL, NULL, 'Please select source database', 0, 0, 0, 1, 1, 1, 1, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); -INSERT INTO t_ds_dq_rule_input_entry -(id, field, "type", title, data, "options", placeholder, option_source_type, data_type, input_type, is_show, can_edit, is_emit, is_validate, create_time, update_time) -VALUES(31, 'target_database', 'select', '$t(target_database)', NULL, NULL, 'Please select target database', 0, 0, 0, 1, 1, 1, 1, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000'); - --- --- Table structure for table t_ds_dq_task_statistics_value --- -DROP TABLE IF EXISTS t_ds_dq_task_statistics_value; -CREATE TABLE t_ds_dq_task_statistics_value ( - id serial NOT NULL, - workflow_definition_id int4 NOT NULL, - task_instance_id int4 NULL, - rule_id int4 NOT NULL, - unique_code varchar NOT NULL, - statistics_name varchar NULL, - statistics_value float8 NULL, - data_time timestamp(0) NULL, - create_time timestamp(0) NULL, - update_time timestamp(0) NULL, - CONSTRAINT t_ds_dq_task_statistics_value_pk PRIMARY KEY (id) -); -- -- Table structure for table t_ds_relation_rule_execute_sql diff --git a/dolphinscheduler-tools/src/main/bin/drop-table.sh b/dolphinscheduler-tools/src/main/bin/drop-table.sh new file mode 100755 index 000000000000..7b1a1684ffe4 --- /dev/null +++ b/dolphinscheduler-tools/src/main/bin/drop-table.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +BIN_DIR=$(dirname $0) +DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd ${BIN_DIR}/../..;pwd)} +TOOLS_HOME=$(cd ${BIN_DIR}/..;pwd) + +if [ "$DOCKER" != "true" ]; then + source "$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh" +fi + +JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms4g -Xmx4g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} + +$JAVA_HOME/bin/java $JAVA_OPTS \ + -cp "$TOOLS_HOME/conf":"$TOOLS_HOME/sql":"$TOOLS_HOME/libs/*" \ + -Dspring.profiles.active=dropTable,${DATABASE} \ + org.apache.dolphinscheduler.tools.table.DropTable diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/table/DropTable.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/table/DropTable.java new file mode 100644 index 000000000000..f53db5c6995d --- /dev/null +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/table/DropTable.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.tools.table; + +import java.sql.SQLException; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.CommandLineRunner; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; + +@SpringBootApplication +@ComponentScan("org.apache.dolphinscheduler") +@Slf4j +public class DropTable { + + public static void main(String[] args) { + SpringApplication.run(DropTable.class, args); + } + + @Component + @Profile("dropTable") + static class DropTableRunner implements CommandLineRunner { + + @Autowired + private DropTableService dropTableService; + + @Override + public void run(String... args) throws SQLException { + log.info("Starting Drop Table..."); + dropTableService.dropTableOnce(); + } + } +} diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/table/DropTableService.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/table/DropTableService.java new file mode 100644 index 000000000000..7a253257c65e --- /dev/null +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/table/DropTableService.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.tools.table; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.util.ArrayList; +import java.util.List; + +import javax.sql.DataSource; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +@Slf4j +public class DropTableService { + + @Autowired + private DataSource dataSource; + + public void dropTableOnce() { + List dropTableDDLList = getDropTableDDL(); + + for (String dropTableDDL : dropTableDDLList) { + try { + try (Connection connection = dataSource.getConnection()) { + try (PreparedStatement pstmt = connection.prepareStatement(dropTableDDL)) { + pstmt.executeUpdate(); + } + } + } catch (Exception e) { + log.error("Failed to execute sql: {},", dropTableDDL, e); + } + } + } + + private List getDropTableDDL() { + List dropTableDDLList = new ArrayList<>(); + dropTableDDLList.add("DROP TABLE IF EXISTS t_ds_dq_comparison_type;"); + dropTableDDLList.add("DROP TABLE IF EXISTS t_ds_dq_rule_execute_sql;"); + dropTableDDLList.add("DROP TABLE IF EXISTS t_ds_dq_rule_input_entry;"); + dropTableDDLList.add("DROP TABLE IF EXISTS t_ds_dq_task_statistics_value;"); + return dropTableDDLList; + } + +}