Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[fix](jdbc catalog) fix and add mysql and doris extremum test #41679

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
+966 −25
Open

[fix](jdbc catalog) fix and add mysql and doris extremum test #41679

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6851f0c
[test](jdbc catalog) add mysql and doris extremum test
zy-kkk Oct 10, 2024
ad8d679
1
zy-kkk Oct 10, 2024
d6f7408
2
zy-kkk Oct 10, 2024
af6fa39
3
zy-kkk Oct 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
153 changes: 153 additions & 0 deletions docker/thirdparties/docker-compose/mysql/init/03-create-table.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -352,3 +352,156 @@ col_int_undef_signed2 int

create table doris_test.text_push (pk varchar(10));


create table doris_test.all_types_nullable (
`tinyint_u` tinyint unsigned,
`smallint_u` smallint unsigned,
`mediumint_u` mediumint unsigned,
`int_u` int unsigned,
`bigint_u` bigint unsigned,
`decimal1_u` decimal unsigned,
`decimal2_u` decimal(9, 2) unsigned,
`decimal3_u` decimal(18, 5) unsigned,
`decimal4_u` decimal(38, 10) unsigned,
`decimal5_u` decimal(65, 30) unsigned,
`double_u` double unsigned,
`float_u` float unsigned,
`boolean` boolean,
`tinyint` tinyint,
`smallint` smallint,
`mediumint` mediumint,
`int` int,
`bigint` bigint,
`double` double,
`float` float,
`decimal1` decimal,
`decimal2` decimal(9, 2),
`decimal3` decimal(18, 5) ,
`decimal4` decimal(38, 10),
`decimal5` decimal(65, 30),
`year` year,
`time1` time,
`time2` time(3),
`time3` time(6),
`date` date,
`datetime` datetime,
`timestamp1` timestamp null,
`timestamp2` timestamp(3) null,
`timestamp3` timestamp(6) null,
`char` char(5),
`varchar` varchar(10),
`text` text,
`blob` blob,
`json` json,
`set` set('Option1', 'Option2', 'Option3'),
`bit` bit(6),
`binary` binary(12),
`varbinary` varbinary(12),
`enum` enum('Value1', 'Value2', 'Value3')
) engine=innodb charset=utf8;


create table doris_test.all_types_non_nullable (
`tinyint_u` tinyint unsigned NOT NULL,
`smallint_u` smallint unsigned NOT NULL,
`mediumint_u` mediumint unsigned NOT NULL,
`int_u` int unsigned NOT NULL,
`bigint_u` bigint unsigned NOT NULL,
`decimal1_u` decimal unsigned NOT NULL,
`decimal2_u` decimal(9, 2) unsigned NOT NULL,
`decimal3_u` decimal(18, 5) unsigned NOT NULL,
`decimal4_u` decimal(38, 10) unsigned NOT NULL,
`decimal5_u` decimal(65, 30) unsigned NOT NULL,
`double_u` double unsigned NOT NULL,
`float_u` float unsigned NOT NULL,
`boolean` boolean NOT NULL,
`tinyint` tinyint NOT NULL,
`smallint` smallint NOT NULL,
`mediumint` mediumint NOT NULL,
`int` int NOT NULL,
`bigint` bigint NOT NULL,
`double` double NOT NULL,
`float` float NOT NULL,
`decimal1` decimal NOT NULL,
`decimal2` decimal(9, 2) NOT NULL,
`decimal3` decimal(18, 5) NOT NULL,
`decimal4` decimal(38, 10) NOT NULL,
`decimal5` decimal(65, 30) NOT NULL,
`year` year NOT NULL,
`time1` time NOT NULL,
`time2` time(3) NOT NULL,
`time3` time(6) NOT NULL,
`date` date NOT NULL,
`datetime` datetime NOT NULL,
`timestamp1` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
`timestamp2` timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
`timestamp3` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
`char` char(5) NOT NULL,
`varchar` varchar(10) NOT NULL,
`text` text NOT NULL,
`blob` blob NOT NULL,
`json` json NOT NULL,
`set` set('Option1', 'Option2', 'Option3') NOT NULL,
`bit` bit(6) NOT NULL,
`binary` binary(12) NOT NULL,
`varbinary` varbinary(12) NOT NULL,
`enum` enum('Value1', 'Value2', 'Value3') NOT NULL
) engine=innodb charset=utf8;


create table doris_test.all_types_multi_block (
`tinyint_u` tinyint unsigned,
`smallint_u` smallint unsigned,
`mediumint_u` mediumint unsigned,
`int_u` int unsigned,
`bigint_u` bigint unsigned,
`decimal1_u` decimal unsigned,
`decimal2_u` decimal(9, 2) unsigned,
`decimal3_u` decimal(18, 5) unsigned,
`decimal4_u` decimal(38, 10) unsigned,
`decimal5_u` decimal(65, 30) unsigned,
`double_u` double unsigned,
`float_u` float unsigned,
`boolean` boolean,
`tinyint` tinyint,
`smallint` smallint,
`mediumint` mediumint,
`int` int,
`bigint` bigint,
`double` double,
`float` float,
`decimal1` decimal,
`decimal2` decimal(9, 2),
`decimal3` decimal(18, 5) ,
`decimal4` decimal(38, 10),
`decimal5` decimal(65, 30),
`year` year,
`time1` time,
`time2` time(3),
`time3` time(6),
`date` date,
`datetime` datetime,
`timestamp1` timestamp null,
`timestamp2` timestamp(3) null,
`timestamp3` timestamp(6) null,
`char` char(5),
`varchar` varchar(10),
`text` text,
`blob` blob,
`json` json,
`set` set('Option1', 'Option2', 'Option3'),
`bit` bit(6),
`binary` binary(12),
`varbinary` varbinary(12),
`enum` enum('Value1', 'Value2', 'Value3')
) engine=innodb charset=utf8;


CREATE TABLE doris_test.`t_varchar` (
`varchar_col` varchar(21844)
);

CREATE TABLE doris_test.`t_char` (
`char_col` char(255) COLLATE utf8_bin DEFAULT NULL
);

37 changes: 37 additions & 0 deletions docker/thirdparties/docker-compose/mysql/init/04-insert.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1168,3 +1168,40 @@ insert into Doris.doris values ('doris');
insert into doris_test.compoundpredicate_test(pk,col_int_undef_signed,col_int_undef_signed2) values (0,null,23868),(1,68,-18),(2,19030,-125),(3,16539,null),(4,null,null),(5,null,-127),(6,14680,-26424),(7,-22270,12722),(8,null,null),(9,null,null),(10,null,7744),(11,null,-94),(12,16970,95),(13,null,7023),(14,null,1),(15,3679,-11),(16,null,-1079),(17,-22,null),(18,30995,null),(19,null,-79);

insert into doris_test.text_push values('a'),('aa'),('aaa');

insert into doris_test.all_types_nullable
values(0,0,0,0,0, 0, 0.00, 0.00000, 0.0000000000, 0.000000000000000000000000000000,0,0,false,-128,-32768,-8388608,-2147483648,-9223372036854775808,-1.7976931348623157E+308,-3.4028234E+38,-9999999999,-9999999.99,-9999999999999.99999,-9999999999999999999999999999.9999999999,-99999999999999999999999999999999999.999999999999999999999999999999,1901,'-838:59:59','-838:59:59.000','-838:59:59.000000','1000-01-01','1000-01-01 00:00:00','1970-01-01 00:00:01','1970-01-01 00:00:01.000','1970-01-01 00:00:01.000000','','','', '', '{}', '', b'000000', '', '', 'Value1'),
(255,65535,16777215,4294967295,18446744073709551615,9999999999, 9999999.99, 9999999999999.99999, 9999999999999999999999999999.9999999999, 99999999999999999999999999999999999.999999999999999999999999999999,1.7976931348623157E+308,3.4028234E+38,true,127,32767,8388607,2147483647,9223372036854775807,1.7976931348623157E+308,3.4028234E+38,9999999999,9999999.99,9999999999999.99999,9999999999999999999999999999.9999999999,99999999999999999999999999999999999.999999999999999999999999999999,2155,'838:59:59.000','838:59:59.000','838:59:59.000000','9999-12-31','9999-12-31 23:59:59','2038-01-19 03:14:07','2038-01-19 03:14:07.499','2038-01-19 03:14:07.499999','zzzz','abcdefgh','max', 'maxBLOB', '{"key": "maxJSON"}', 'Option1,Option2,Option3', b'111111', 'maxBIN', 'maxVARBIN', 'Value3'),
(0,0,0,0,0, 0, 0.00, 0.00000, 0.0000000000, 0.000000000000000000000000000000, 0, 0, false, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1901, '00:00:00', '00:00:00.000', '00:00:00.000000','1000-01-01','1000-01-01 00:00:00','1970-01-01 00:00:01','1970-01-01 00:00:01.000','1970-01-01 00:00:01.000000', '', '', '', '', '{}', '', b'', '', '', 'Value1'),
(NULL,NULL,NULL,NULL,NULL, NULL, NULL, NULL, NULL, NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);

insert into doris_test.all_types_non_nullable
values(0,0,0,0,0, 0, 0.00, 0.00000, 0.0000000000, 0.000000000000000000000000000000,0,0,false,-128,-32768,-8388608,-2147483648,-9223372036854775808,-1.7976931348623157E+308,-3.4028234E+38,-9999999999,-9999999.99,-9999999999999.99999,-9999999999999999999999999999.9999999999,-99999999999999999999999999999999999.999999999999999999999999999999,1901,'-838:59:59','-838:59:59.000','-838:59:59.000000','1000-01-01','1000-01-01 00:00:00','1970-01-01 00:00:01','1970-01-01 00:00:01.000','1970-01-01 00:00:01.000000','','','', '', '{}', '', b'000000', '', '', 'Value1'),
(255,65535,16777215,4294967295,18446744073709551615,9999999999, 9999999.99, 9999999999999.99999, 9999999999999999999999999999.9999999999, 99999999999999999999999999999999999.999999999999999999999999999999,1.7976931348623157E+308,3.4028234E+38,true,127,32767,8388607,2147483647,9223372036854775807,1.7976931348623157E+308,3.4028234E+38,9999999999,9999999.99,9999999999999.99999,9999999999999999999999999999.9999999999,99999999999999999999999999999999999.999999999999999999999999999999,2155,'838:59:59.000','838:59:59.000','838:59:59.000000','9999-12-31','9999-12-31 23:59:59','2038-01-19 03:14:07','2038-01-19 03:14:07.499','2038-01-19 03:14:07.499999','zzzz','abcdefgh','max', 'maxBLOB', '{"key": "maxJSON"}', 'Option1,Option2,Option3', b'111111', 'maxBIN', 'maxVARBIN', 'Value3'),
(0,0,0,0,0, 0, 0.00, 0.00000, 0.0000000000, 0.000000000000000000000000000000, 0, 0, false, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1901, '00:00:00', '00:00:00.000', '00:00:00.000000','1000-01-01','1000-01-01 00:00:00','1970-01-01 00:00:01','1970-01-01 00:00:01.000','1970-01-01 00:00:01.000000', '', '', '', '', '{}', '', b'', '', '', 'Value1');

insert into doris_test.all_types_multi_block select * from doris_test.all_types_nullable;
insert into doris_test.all_types_multi_block select * from doris_test.all_types_multi_block;
insert into doris_test.all_types_multi_block select * from doris_test.all_types_multi_block;
insert into doris_test.all_types_multi_block select * from doris_test.all_types_multi_block;
insert into doris_test.all_types_multi_block select * from doris_test.all_types_multi_block;
insert into doris_test.all_types_multi_block select * from doris_test.all_types_multi_block;
insert into doris_test.all_types_multi_block select * from doris_test.all_types_multi_block;
insert into doris_test.all_types_multi_block select * from doris_test.all_types_multi_block;
insert into doris_test.all_types_multi_block select * from doris_test.all_types_multi_block;
insert into doris_test.all_types_multi_block select * from doris_test.all_types_multi_block;
insert into doris_test.all_types_multi_block select * from doris_test.all_types_multi_block;
insert into doris_test.all_types_multi_block select * from doris_test.all_types_nullable;

INSERT INTO doris_test.t_varchar (varchar_col) VALUES ('a');

INSERT INTO doris_test.t_varchar (varchar_col) VALUES ('中');

INSERT INTO doris_test.t_varchar (varchar_col) VALUES (REPEAT('a', 21844));

INSERT INTO doris_test.t_varchar (varchar_col) VALUES (REPEAT('中', 21844));

INSERT INTO doris_test.t_char (char_col) VALUES (REPEAT('a', 255));

INSERT INTO doris_test.t_char (char_col) VALUES (REPEAT('中', 255));

41 changes: 33 additions & 8 deletions ...e-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/MySQLJdbcExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Types;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatterBuilder;
Expand Down Expand Up @@ -133,8 +134,14 @@ protected Object getColumnValue(int columnIndex, ColumnType type, String[] repla
case VARCHAR:
case ARRAY:
return resultSet.getObject(columnIndex + 1, String.class);
case STRING:
return resultSet.getObject(columnIndex + 1);
case STRING: {
int jdbcType = resultSetMetaData.getColumnType(columnIndex + 1);
if (jdbcType == Types.TIME) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add some comment to explain this if else

return resultSet.getString(columnIndex + 1);
} else {
return resultSet.getObject(columnIndex + 1);
}
}
default:
throw new IllegalArgumentException("Unsupported column type: " + type.getType());
}
Expand Down Expand Up @@ -192,8 +199,20 @@ protected ColumnValueConverter getOutputConverter(ColumnType columnType, String
}

private Object convertArray(Object input, ColumnType columnType) {
if (input == null) {
return null;
}
java.lang.reflect.Type listType = getListTypeForArray(columnType);
if (columnType.getType() == Type.BOOLEAN) {
if (columnType.getType() == Type.ARRAY) {
ColumnType childType = columnType.getChildTypes().get(0);
List<?> rawList = gson.fromJson((String) input, List.class);
return rawList.stream()
.map(element -> {
String elementJson = gson.toJson(element);
return convertArray(elementJson, childType);
})
.collect(Collectors.toList());
} else if (columnType.getType() == Type.BOOLEAN) {
List<?> list = gson.fromJson((String) input, List.class);
return list.stream().map(item -> {
if (item instanceof Boolean) {
Expand Down Expand Up @@ -228,11 +247,17 @@ private Object convertArray(Object input, ColumnType columnType) {
throw new IllegalArgumentException("Cannot convert " + item + " to LocalDateTime.");
}
}).collect(Collectors.toList());
} else if (columnType.getType() == Type.ARRAY) {
List<?> list = gson.fromJson((String) input, listType);
return list.stream()
.map(item -> convertArray(gson.toJson(item), columnType.getChildTypes().get(0)))
.collect(Collectors.toList());
} else if (columnType.getType() == Type.LARGEINT) {
List<?> list = gson.fromJson((String) input, List.class);
return list.stream().map(item -> {
if (item instanceof Number) {
return new BigDecimal(item.toString()).toBigInteger();
} else if (item instanceof String) {
return new BigDecimal((String) item).toBigInteger();
} else {
throw new IllegalArgumentException("Cannot convert " + item + " to BigInteger.");
}
}).collect(Collectors.toList());
} else {
return gson.fromJson((String) input, listType);
}
Expand Down
23 changes: 12 additions & 11 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,8 +195,8 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
case "BIGINT":
return Type.LARGEINT;
case "DECIMAL": {
int precision = fieldSchema.getColumnSize().orElse(0) + 1;
int scale = fieldSchema.getDecimalDigits().orElse(0);
int precision = fieldSchema.requiredColumnSize() + 1;
int scale = fieldSchema.requiredDecimalDigits();
return createDecimalOrStringType(precision, scale);
}
case "DOUBLE":
Expand Down Expand Up @@ -233,7 +233,7 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
case "DATETIME": {
// mysql can support microsecond
// use columnSize to calculate the precision of timestamp/datetime
int columnSize = fieldSchema.getColumnSize().orElse(0);
int columnSize = fieldSchema.requiredColumnSize();
int scale = columnSize > 19 ? columnSize - 20 : 0;
if (scale > 6) {
scale = 6;
Expand All @@ -248,18 +248,18 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
case "DOUBLE":
return Type.DOUBLE;
case "DECIMAL": {
int precision = fieldSchema.getColumnSize().orElse(0);
int scale = fieldSchema.getDecimalDigits().orElse(0);
int precision = fieldSchema.requiredColumnSize();
int scale = fieldSchema.requiredDecimalDigits();
return createDecimalOrStringType(precision, scale);
}
case "CHAR":
ScalarType charType = ScalarType.createType(PrimitiveType.CHAR);
charType.setLength(fieldSchema.getColumnSize().orElse(0));
charType.setLength(fieldSchema.requiredColumnSize());
return charType;
case "VARCHAR":
return ScalarType.createVarcharType(fieldSchema.getColumnSize().orElse(0));
case "BIT":
if (fieldSchema.getColumnSize().orElse(0) == 1) {
if (fieldSchema.requiredColumnSize() == 1) {
return Type.BOOLEAN;
} else {
return ScalarType.createStringType();
Expand Down Expand Up @@ -360,8 +360,8 @@ private Type dorisTypeToDoris(JdbcFieldSchema fieldSchema) {
return Type.DOUBLE;
case "DECIMAL":
case "DECIMALV3": {
int precision = fieldSchema.getColumnSize().orElse(0);
int scale = fieldSchema.getDecimalDigits().orElse(0);
int precision = fieldSchema.requiredColumnSize();
int scale = fieldSchema.requiredDecimalDigits();
return createDecimalOrStringType(precision, scale);
}
case "DATE":
Expand All @@ -377,11 +377,12 @@ private Type dorisTypeToDoris(JdbcFieldSchema fieldSchema) {
return ScalarType.createDatetimeV2Type(scale);
}
case "CHAR":
case "CHARACTER":
ScalarType charType = ScalarType.createType(PrimitiveType.CHAR);
charType.setLength(fieldSchema.getColumnSize().orElse(0));
charType.setLength(fieldSchema.requiredColumnSize());
return charType;
case "VARCHAR":
return ScalarType.createVarcharType(fieldSchema.getColumnSize().orElse(0));
return ScalarType.createVarcharType(fieldSchema.requiredColumnSize());
case "STRING":
case "TEXT":
case "JSON":
Expand Down
14 changes: 10 additions & 4 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/util/JdbcFieldSchema.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@ public class JdbcFieldSchema {
protected int charOctetLength;
protected boolean isAllowNull;


public JdbcFieldSchema(JdbcFieldSchema other) {
this.columnName = other.columnName;
this.dataType = other.dataType;
Expand All @@ -65,7 +64,7 @@ public JdbcFieldSchema(ResultSet rs) throws SQLException {
this.dataType = getInteger(rs, "DATA_TYPE").orElseThrow(() -> new IllegalStateException("DATA_TYPE is null"));
this.dataTypeName = Optional.ofNullable(rs.getString("TYPE_NAME"));
this.columnSize = getInteger(rs, "COLUMN_SIZE");
this.decimalDigits = getInteger(rs, "DECIMAL_DIGITS");
this.decimalDigits = getInteger(rs, "DECIMAL_DIGITS");
this.numPrecRadix = rs.getInt("NUM_PREC_RADIX");
this.isAllowNull = rs.getInt("NULLABLE") != DatabaseMetaData.columnNoNulls;
this.remarks = rs.getString("REMARKS");
Expand All @@ -77,7 +76,7 @@ public JdbcFieldSchema(ResultSet rs, Map<String, String> dataTypeOverrides) thro
this.dataType = getInteger(rs, "DATA_TYPE").orElseThrow(() -> new IllegalStateException("DATA_TYPE is null"));
this.dataTypeName = Optional.ofNullable(dataTypeOverrides.getOrDefault(columnName, rs.getString("TYPE_NAME")));
this.columnSize = getInteger(rs, "COLUMN_SIZE");
this.decimalDigits = getInteger(rs, "DECIMAL_DIGITS");
this.decimalDigits = getInteger(rs, "DECIMAL_DIGITS");
this.numPrecRadix = rs.getInt("NUM_PREC_RADIX");
this.isAllowNull = rs.getInt("NULLABLE") != 0;
this.remarks = rs.getString("REMARKS");
Expand All @@ -92,6 +91,14 @@ public JdbcFieldSchema(ResultSetMetaData metaData, int columnIndex) throws SQLEx
this.decimalDigits = Optional.of(metaData.getScale(columnIndex));
}

public int requiredColumnSize() {
return columnSize.orElseThrow(() -> new IllegalStateException("column size not present"));
}

public int requiredDecimalDigits() {
return decimalDigits.orElseThrow(() -> new IllegalStateException("decimal digits not present"));
}

protected static Optional<Integer> getInteger(ResultSet resultSet, String columnLabel)
throws SQLException {
int value = resultSet.getInt(columnLabel);
Expand All @@ -101,4 +108,3 @@ protected static Optional<Integer> getInteger(ResultSet resultSet, String column
return Optional.of(value);
}
}

Loading
Loading