From 5f456bfa69b4955d6c4deaa188ac4ab79934f04c Mon Sep 17 00:00:00 2001 From: Yuhui Date: Thu, 19 Oct 2023 22:19:28 +0800 Subject: [PATCH] [#523] test (trino-connector): Add SQL interface tests for trino-connector. (#524) ### What changes were proposed in this pull request? Add SQL interface end to end UT for trino-connector. It uses the Graviton connector to load the memory catalog and execute basic SQL tests. ### Why are the changes needed? Fix: #523 Add SQL interface end to end UT for trino-connector --- build.gradle.kts | 10 +- gradle/libs.versions.toml | 3 + trino-connector/build.gradle.kts | 31 +- .../connector/GravitinoConnectorFactory.java | 3 + .../trino/connector/GravitinoMetadata.java | 11 - .../trino/connector/GravitinoPlugin.java | 5 + .../catalog/CatalogConnectorAdapter.java | 8 +- .../catalog/CatalogConnectorFactory.java | 3 + .../CatalogConnectorMetadataAdapter.java | 96 +++++- .../catalog/hive/HiveConnectorAdapter.java | 2 +- .../catalog/hive/HiveMetadataAdapter.java | 99 +----- .../memory/MemoryConnectorAdapter.java | 36 ++ .../catalog/memory/MemoryMetadataAdapter.java | 21 ++ .../connector/metadata/GravitinoColumn.java | 2 +- .../connector/util/DataTypeTransformer.java | 7 +- .../trino/connector/TestGravitinoConfig.java | 18 +- .../connector/TestGravitinoConnector.java | 156 ++++++++- .../connector/TestGravitinoTableHandle.java | 9 +- .../metadata/TestGravitinoCatalog.java | 11 +- .../metadata/TestGravitinoColumn.java | 17 +- .../metadata/TestGravitinoSchema.java | 13 +- .../metadata/TestGravitinoTable.java | 21 +- .../connector/util/GravitinoRestApi.java | 308 ++++++++++++++++++ .../connector/util/MockGravitinoServer.java | 54 +++ .../util/TestDataTypeTransformer.java | 85 ++--- .../src/test/resources/log4j2.properties | 4 +- 26 files changed, 797 insertions(+), 236 deletions(-) create mode 100644 trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/memory/MemoryConnectorAdapter.java create mode 100644 trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/memory/MemoryMetadataAdapter.java create mode 100644 trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/util/GravitinoRestApi.java create mode 100644 trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/util/MockGravitinoServer.java diff --git a/build.gradle.kts b/build.gradle.kts index 82f3214712..5b994d5358 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -41,6 +41,10 @@ java { } } +dependencies { + testImplementation(libs.testng) +} + subprojects { apply(plugin = "jacoco") @@ -52,7 +56,11 @@ subprojects { tasks.configureEach { val skipTests = project.hasProperty("skipTests") if (!skipTests) { - useJUnitPlatform() + if (project.name == "trino-connector") { + useTestNG() + } else { + useJUnitPlatform() + } finalizedBy(tasks.getByName("jacocoTestReport")) } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index c8d8b6dbe0..56b7a9419f 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -30,6 +30,7 @@ trino = '426' spark = "3.4.1" scala-collection-compat = "2.7.0" sqlite-jdbc = "3.42.0.0" +testng = "7.7.1" protobuf-plugin = "0.9.2" @@ -91,10 +92,12 @@ iceberg-hive-metastore = { group = "org.apache.iceberg", name = "iceberg-hive-me trino-spi= { group = "io.trino", name = "trino-spi", version.ref = "trino" } trino-toolkit= { group = "io.trino", name = "trino-plugin-toolkit", version.ref = "trino" } trino-testing= { group = "io.trino", name = "trino-testing", version.ref = "trino" } +trino-memory= { group = "io.trino", name = "trino-memory", version.ref = "trino" } iceberg-spark-runtime = { group = "org.apache.iceberg", name = "iceberg-spark-runtime-3.4_2.13", version.ref = "iceberg" } spark-sql = { group = "org.apache.spark", name = "spark-sql_2.13", version.ref = "spark" } scala-collection-compat = { group = "org.scala-lang.modules", name = "scala-collection-compat_2.13", version.ref = "scala-collection-compat" } sqlite-jdbc = { group = "org.xerial", name = "sqlite-jdbc", version.ref = "sqlite-jdbc" } +testng = { group = "org.testng", name = "testng", version.ref = "testng" } spark-hive = { group = "org.apache.spark", name = "spark-hive_2.13", version.ref = "spark" } diff --git a/trino-connector/build.gradle.kts b/trino-connector/build.gradle.kts index acb7929b69..7a79d0032c 100644 --- a/trino-connector/build.gradle.kts +++ b/trino-connector/build.gradle.kts @@ -14,29 +14,26 @@ repositories { } dependencies { - implementation(project(":api")) - implementation(project(":common")) { - exclude("org.apache.logging.log4j") - } - implementation(project(":clients:client-java")) { - exclude("org.apache.logging.log4j") - } + implementation(project(":clients:client-java-runtime", configuration="shadow")) implementation(libs.jackson.databind) implementation(libs.jackson.annotations) implementation(libs.guava) implementation(libs.httpclient5) implementation(libs.commons.lang3) - implementation(libs.trino.spi) - implementation(libs.trino.toolkit) - implementation(libs.substrait.java.core) + implementation(libs.trino.spi) { + exclude("org.apache.logging.log4j") + } + implementation(libs.trino.toolkit) { + exclude("org.apache.logging.log4j") + } - testImplementation(libs.junit.jupiter.api) - testImplementation(libs.junit.jupiter.params) - testRuntimeOnly(libs.junit.jupiter.engine) - testImplementation(libs.trino.testing) - testImplementation(libs.mockito.core) - testImplementation(libs.mockserver.netty) - testImplementation(libs.mockserver.client.java) + testImplementation(libs.trino.testing) { + exclude("org.apache.logging.log4j") + } + testImplementation(libs.trino.memory) { + exclude("org.antlr") + exclude("org.apache.logging.log4j") + } } java { diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConnectorFactory.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConnectorFactory.java index 1b6013c89a..d06278a912 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConnectorFactory.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConnectorFactory.java @@ -72,6 +72,9 @@ public Connector create( catalogConnectorManager.getCatalogConnector(catalogName); Preconditions.checkNotNull(catalogConnectorContext, "catalogConnector is not null"); + // For testing + GravitinoPlugin.internalTestingConnector = catalogConnectorContext.getInternalConnector(); + return catalogConnectorContext.getConnector(); } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoMetadata.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoMetadata.java index b017d6fb49..cef7c52968 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoMetadata.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoMetadata.java @@ -21,7 +21,6 @@ import io.trino.spi.connector.ConnectorSession; import io.trino.spi.connector.ConnectorTableHandle; import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.ConnectorTableProperties; import io.trino.spi.connector.ConnectorTableVersion; import io.trino.spi.connector.RetryMode; import io.trino.spi.connector.SchemaTableName; @@ -68,16 +67,6 @@ public Map getSchemaProperties(ConnectorSession session, String return metadataAdapter.getSchemaProperties(schema); } - @Override - public ConnectorTableProperties getTableProperties( - ConnectorSession session, ConnectorTableHandle tableHandle) { - GravitinoTableHandle gravitinoTableHandle = (GravitinoTableHandle) tableHandle; - GravitinoTable table = - catalogConnectorMetadata.getTable( - gravitinoTableHandle.getSchemaName(), gravitinoTableHandle.getTableName()); - return metadataAdapter.getTableProperties(table); - } - @Override public GravitinoTableHandle getTableHandle( ConnectorSession session, diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoPlugin.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoPlugin.java index 942d0c8e10..5eac723a55 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoPlugin.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoPlugin.java @@ -6,10 +6,15 @@ import com.google.common.collect.ImmutableList; import io.trino.spi.Plugin; +import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorFactory; /** Trino plugin endpoint, using java spi mechanism */ public class GravitinoPlugin implements Plugin { + + // For testing. + public static Connector internalTestingConnector; + @Override public Iterable getConnectorFactories() { return ImmutableList.of(new GravitinoConnectorFactory()); diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorAdapter.java index dc2c595516..742c768289 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorAdapter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorAdapter.java @@ -25,11 +25,15 @@ default List> getTableProperties() { Map buildInternalConnectorConfig(GravitinoCatalog catalog); /** @return SchemaProperties list that used to validate schema properties. */ - List> getSchemaProperties(); + default List> getSchemaProperties() { + return emptyList(); + }; /** @return Return MetadataAdapter for special catalog connector. */ CatalogConnectorMetadataAdapter getMetadataAdapter(); /** @return ColumnProperties list that used to validate column properties. */ - List> getColumnProperties(); + default List> getColumnProperties() { + return emptyList(); + } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorFactory.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorFactory.java index 2dfea6a723..0a513ec973 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorFactory.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorFactory.java @@ -10,6 +10,7 @@ import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.client.GravitinoMetaLake; import com.datastrato.gravitino.trino.connector.catalog.hive.HiveConnectorAdapter; +import com.datastrato.gravitino.trino.connector.catalog.memory.MemoryConnectorAdapter; import com.datastrato.gravitino.trino.connector.metadata.GravitinoCatalog; import io.trino.spi.TrinoException; import io.trino.spi.connector.Connector; @@ -28,6 +29,8 @@ public CatalogConnectorFactory(CatalogInjector catalogInjector) { this.catalogInjector = catalogInjector; catalogBuilders.put("hive", new CatalogConnectorContext.Builder(new HiveConnectorAdapter())); + catalogBuilders.put( + "memory", new CatalogConnectorContext.Builder(new MemoryConnectorAdapter())); } public CatalogConnectorContext loadCatalogConnector( diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorMetadataAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorMetadataAdapter.java index 7d939bb29a..3c02e028f8 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorMetadataAdapter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorMetadataAdapter.java @@ -7,30 +7,112 @@ import com.datastrato.gravitino.trino.connector.metadata.GravitinoColumn; import com.datastrato.gravitino.trino.connector.metadata.GravitinoSchema; import com.datastrato.gravitino.trino.connector.metadata.GravitinoTable; +import com.datastrato.gravitino.trino.connector.util.DataTypeTransformer; import io.trino.spi.connector.ColumnMetadata; import io.trino.spi.connector.ConnectorTableMetadata; import io.trino.spi.connector.ConnectorTableProperties; +import io.trino.spi.connector.SchemaTableName; +import io.trino.spi.session.PropertyMetadata; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.commons.lang3.NotImplementedException; /** * This interface is used to handle different parts of catalog metadata from different catalog * connectors. */ -public interface CatalogConnectorMetadataAdapter { - Map getSchemaProperties(GravitinoSchema schema); +public class CatalogConnectorMetadataAdapter { + + protected final List> schemaProperties; + protected final List> tableProperties; + protected final List> columnProperties; + + protected CatalogConnectorMetadataAdapter( + List> schemaProperties, + List> tableProperties, + List> columnProperties) { + this.schemaProperties = schemaProperties; + this.tableProperties = tableProperties; + this.columnProperties = columnProperties; + } + + public Map getSchemaProperties(GravitinoSchema schema) { + return normalizeProperties(schema.getProperties(), schemaProperties); + } /** Transform gravitino table metadata to trino ConnectorTableMetadata */ - ConnectorTableMetadata getTableMetadata(GravitinoTable gravitinoTable); + public ConnectorTableMetadata getTableMetadata(GravitinoTable gravitinoTable) { + SchemaTableName schemaTableName = + new SchemaTableName(gravitinoTable.getSchemaName(), gravitinoTable.getName()); + ArrayList columnMetadataList = new ArrayList<>(); + for (GravitinoColumn column : gravitinoTable.getColumns()) { + columnMetadataList.add(getColumnMetadata(column)); + } + + Map properties = + normalizeProperties(gravitinoTable.getProperties(), tableProperties); + return new ConnectorTableMetadata( + schemaTableName, columnMetadataList, properties, Optional.of(gravitinoTable.getComment())); + } /** Transform trino ConnectorTableMetadata to gravitino table metadata */ - GravitinoTable createTable(ConnectorTableMetadata tableMetadata); + public GravitinoTable createTable(ConnectorTableMetadata tableMetadata) { + String tableName = tableMetadata.getTableSchema().getTable().getTableName(); + String schemaName = tableMetadata.getTableSchema().getTable().getSchemaName(); + String comment = tableMetadata.getComment().orElse(""); + Map properties = removeUnsetProperties(tableMetadata.getProperties()); + + List columns = new ArrayList<>(); + for (int i = 0; i < tableMetadata.getColumns().size(); i++) { + ColumnMetadata column = tableMetadata.getColumns().get(i); + columns.add( + new GravitinoColumn( + column.getName(), + DataTypeTransformer.getGravitinoType(column.getType(), column.isNullable()), + i, + column.getComment())); + } + return new GravitinoTable(schemaName, tableName, columns, comment, properties); + } /** Transform trino schema metadata to gravitino schema metadata */ - GravitinoSchema createSchema(String schemaName, Map properties); + public GravitinoSchema createSchema(String schemaName, Map properties) { + return new GravitinoSchema(schemaName, removeUnsetProperties(properties), ""); + } /** Transform gravitino column metadata to trino ColumnMetadata */ - ColumnMetadata getColumnMetadata(GravitinoColumn column); + public ColumnMetadata getColumnMetadata(GravitinoColumn column) { + return new ColumnMetadata(column.getName(), DataTypeTransformer.getTrinoType(column.getType())); + } /** Transform gravitino table properties to trino ConnectorTableProperties */ - ConnectorTableProperties getTableProperties(GravitinoTable table); + public ConnectorTableProperties getTableProperties(GravitinoTable table) { + throw new NotImplementedException(); + } + + /** Normalize gravitino attributes for trino */ + protected Map normalizeProperties( + Map properties, List> propertyTemplate) { + // TODO yuhui redo this function once gravitino table properties are supported.. + // Trino only supports properties defined in the propertyTemplate. + Map validProperties = new HashMap<>(); + for (PropertyMetadata propertyMetadata : propertyTemplate) { + String name = propertyMetadata.getName(); + if (properties.containsKey(name)) { + validProperties.put(name, properties.get(name)); + } + } + return validProperties; + } + + /** Remove trino unset attributes fro gravitino */ + protected Map removeUnsetProperties(Map properties) { + return properties.entrySet().stream() + .filter(e -> e.getValue() != null) + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toString())); + } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java index 7c2de49e76..c8bc68da71 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java @@ -27,7 +27,7 @@ public Map buildInternalConnectorConfig(GravitinoCatalog catalog config.put("connectorName", "hive"); Map properties = new HashMap<>(); - properties.put("hive.metastore.uri", catalog.getProperties("hive.metastore.uris", "")); + properties.put("hive.metastore.uri", catalog.getProperties("metastore.uris", "")); config.put("properties", properties); return config; } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveMetadataAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveMetadataAdapter.java index 3a1ef2c21f..eda93c2f80 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveMetadataAdapter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveMetadataAdapter.java @@ -5,111 +5,16 @@ package com.datastrato.gravitino.trino.connector.catalog.hive; import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorMetadataAdapter; -import com.datastrato.gravitino.trino.connector.metadata.GravitinoColumn; -import com.datastrato.gravitino.trino.connector.metadata.GravitinoSchema; -import com.datastrato.gravitino.trino.connector.metadata.GravitinoTable; -import com.datastrato.gravitino.trino.connector.util.DataTypeTransformer; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.ConnectorTableProperties; -import io.trino.spi.connector.SchemaTableName; import io.trino.spi.session.PropertyMetadata; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; -import org.apache.commons.lang3.NotImplementedException; /** Transforming gravitino hive metadata to trino. */ -public class HiveMetadataAdapter implements CatalogConnectorMetadataAdapter { - private final List> schemaProperties; - private final List> tableProperties; - private final List> columnProperties; +public class HiveMetadataAdapter extends CatalogConnectorMetadataAdapter { public HiveMetadataAdapter( List> schemaProperties, List> tableProperties, List> columnProperties) { - this.schemaProperties = schemaProperties; - this.tableProperties = tableProperties; - this.columnProperties = columnProperties; - } - - public ConnectorTableMetadata getTableMetadata(GravitinoTable gravitinoTable) { - SchemaTableName schemaTableName = - new SchemaTableName(gravitinoTable.getSchemaName(), gravitinoTable.getName()); - ArrayList columnMetadataList = new ArrayList<>(); - for (GravitinoColumn column : gravitinoTable.getColumns()) { - columnMetadataList.add(getColumnMetadata(column)); - } - - Map properties = - normalizeProperties(gravitinoTable.getProperties(), tableProperties); - return new ConnectorTableMetadata( - schemaTableName, columnMetadataList, properties, Optional.of(gravitinoTable.getComment())); - } - - @Override - public ColumnMetadata getColumnMetadata(GravitinoColumn column) { - return new ColumnMetadata(column.getName(), DataTypeTransformer.getTrinoType(column.getType())); - } - - @Override - public ConnectorTableProperties getTableProperties(GravitinoTable table) { - throw new NotImplementedException(); - } - - @Override - public Map getSchemaProperties(GravitinoSchema schema) { - return normalizeProperties(schema.getProperties(), schemaProperties); - } - - @Override - public GravitinoTable createTable(ConnectorTableMetadata tableMetadata) { - String tableName = tableMetadata.getTableSchema().getTable().getTableName(); - String schemaName = tableMetadata.getTableSchema().getTable().getSchemaName(); - String comment = tableMetadata.getComment().orElse(""); - Map properties = removeUnsetProperties(tableMetadata.getProperties()); - - List columns = new ArrayList<>(); - int index = 0; - for (int i = 0; i < tableMetadata.getColumns().size(); i++) { - ColumnMetadata column = tableMetadata.getColumns().get(i); - columns.add( - new GravitinoColumn( - column.getName(), - DataTypeTransformer.getGravitinoType(column.getType(), column.isNullable()), - index, - column.getComment())); - index++; - } - return new GravitinoTable(schemaName, tableName, columns, comment, properties); - } - - @Override - public GravitinoSchema createSchema(String schemaName, Map properties) { - return new GravitinoSchema(schemaName, removeUnsetProperties(properties), ""); - } - - private Map normalizeProperties( - Map properties, List> propertyTemplate) { - // TODO yuhui redo this function on gravitino table properties supported. - // Trino only supports properties defined in the propertyTemplate. - Map validProperties = new HashMap<>(); - for (PropertyMetadata propertyMetadata : propertyTemplate) { - String name = propertyMetadata.getName(); - if (properties.containsKey(name)) { - validProperties.put(name, properties.get(name)); - } - } - return validProperties; - } - - private Map removeUnsetProperties(Map properties) { - return properties.entrySet().stream() - .filter(e -> e.getValue() != null) - .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toString())); + super(schemaProperties, tableProperties, columnProperties); } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/memory/MemoryConnectorAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/memory/MemoryConnectorAdapter.java new file mode 100644 index 0000000000..427e3a4ada --- /dev/null +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/memory/MemoryConnectorAdapter.java @@ -0,0 +1,36 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.trino.connector.catalog.memory; + +import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorAdapter; +import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorMetadataAdapter; +import com.datastrato.gravitino.trino.connector.metadata.GravitinoCatalog; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * Support trino Memory connector for testing. Transforming Memory connector configuration and + * components into Gravitino connector. + */ +public class MemoryConnectorAdapter implements CatalogConnectorAdapter { + + @Override + public Map buildInternalConnectorConfig(GravitinoCatalog catalog) { + Map config = new HashMap<>(); + config.put("catalogHandle", catalog.getName() + ":normal:default"); + config.put("connectorName", "memory"); + + Map properties = new HashMap<>(); + config.put("properties", properties); + return config; + } + + @Override + public CatalogConnectorMetadataAdapter getMetadataAdapter() { + return new MemoryMetadataAdapter( + Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); + } +} diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/memory/MemoryMetadataAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/memory/MemoryMetadataAdapter.java new file mode 100644 index 0000000000..7752074dc1 --- /dev/null +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/memory/MemoryMetadataAdapter.java @@ -0,0 +1,21 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.trino.connector.catalog.memory; + +import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorMetadataAdapter; +import io.trino.spi.session.PropertyMetadata; +import java.util.List; + +/** Support trino memory connector for testing. */ +public class MemoryMetadataAdapter extends CatalogConnectorMetadataAdapter { + + MemoryMetadataAdapter( + List> schemaProperties, + List> tableProperties, + List> columnProperties) { + + super(schemaProperties, tableProperties, columnProperties); + } +} diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoColumn.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoColumn.java index 50548c1e1d..35f3b0aca2 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoColumn.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoColumn.java @@ -7,7 +7,7 @@ import static java.util.Objects.requireNonNull; import com.datastrato.gravitino.rel.Column; -import io.substrait.type.Type; +import com.datastrato.gravitino.shaded.io.substrait.type.Type; import java.util.Map; /** Help Gravitino connector access ColumnMetadata from gravitino client. */ diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/DataTypeTransformer.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/DataTypeTransformer.java index 282b97381a..5eb9975fad 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/DataTypeTransformer.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/DataTypeTransformer.java @@ -17,14 +17,14 @@ import static io.trino.spi.type.VarcharType.VARCHAR; import static io.trino.spi.type.VarcharType.createUnboundedVarcharType; -import io.substrait.type.TypeCreator; +import com.datastrato.gravitino.shaded.io.substrait.type.TypeCreator; import io.trino.spi.TrinoException; import io.trino.spi.type.Type; /** This class is used to transform datatype between gravitino and trino */ public class DataTypeTransformer { - public static Type getTrinoType(io.substrait.type.Type type) { + public static Type getTrinoType(com.datastrato.gravitino.shaded.io.substrait.type.Type type) { if (type.equals(TypeCreator.REQUIRED.STRING) || type.equals(TypeCreator.NULLABLE.STRING)) { return createUnboundedVarcharType(); } else if (type.equals(TypeCreator.REQUIRED.BOOLEAN) @@ -49,7 +49,8 @@ public static Type getTrinoType(io.substrait.type.Type type) { GRAVITINO_UNSUPPORTED_GRAVITINO_DATATYPE, "Unsupported gravitino datatype: " + type); } - public static io.substrait.type.Type getGravitinoType(Type type, boolean nullable) { + public static com.datastrato.gravitino.shaded.io.substrait.type.Type getGravitinoType( + Type type, boolean nullable) { if (type.equals(VARCHAR)) { return nullable ? TypeCreator.NULLABLE.STRING : TypeCreator.REQUIRED.STRING; } else if (type.equals(BOOLEAN)) { diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConfig.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConfig.java index 575a8a3997..1ad5da78d0 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConfig.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConfig.java @@ -5,20 +5,20 @@ package com.datastrato.gravitino.trino.connector; import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_MISSING_CONFIG; +import static org.testng.Assert.assertEquals; import com.google.common.collect.ImmutableMap; import io.trino.spi.TrinoException; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; +import org.testng.annotations.AfterTest; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; public class TestGravitinoConfig { - @BeforeAll + @BeforeTest public static void startup() throws Exception {} - @AfterAll + @AfterTest public static void shutdown() throws Exception {} @Test @@ -30,8 +30,8 @@ public void testGravitinoConfig() { GravitinoConfig config = new GravitinoConfig(configMap); - Assertions.assertEquals(gravitinoUrl, config.getURI()); - Assertions.assertEquals(metalake, config.getMetalake()); + assertEquals(gravitinoUrl, config.getURI()); + assertEquals(metalake, config.getMetalake()); } @Test @@ -40,7 +40,7 @@ public void testMissingConfig() { ImmutableMap configMap = ImmutableMap.of("gravitino.uri", gravitinoUrl); try { GravitinoConfig config = new GravitinoConfig(configMap); - Assertions.assertEquals(gravitinoUrl, config.getURI()); + assertEquals(gravitinoUrl, config.getURI()); } catch (TrinoException e) { if (e.getErrorCode() != GRAVITINO_MISSING_CONFIG.toErrorCode()) { throw e; diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java index 5a831587d2..5dce518dd2 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java @@ -4,43 +4,189 @@ */ package com.datastrato.gravitino.trino.connector; +import static io.trino.spi.type.IntegerType.INTEGER; +import static io.trino.spi.type.VarcharType.VARCHAR; import static io.trino.testing.TestingSession.testSessionBuilder; +import static java.lang.String.format; +import static java.util.Collections.emptyMap; import static org.assertj.core.api.Assertions.assertThat; +import static org.testng.Assert.assertEquals; +import com.datastrato.gravitino.trino.connector.util.MockGravitinoServer; import io.trino.Session; -import io.trino.testing.BaseConnectorTest; +import io.trino.plugin.memory.MemoryConnector; +import io.trino.plugin.memory.MemoryPlugin; +import io.trino.spi.connector.ColumnMetadata; +import io.trino.spi.connector.ConnectorMetadata; +import io.trino.spi.connector.ConnectorTableMetadata; +import io.trino.spi.connector.SchemaTableName; +import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.DistributedQueryRunner; +import io.trino.testing.MaterializedResult; +import io.trino.testing.MaterializedRow; import io.trino.testing.QueryRunner; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; +import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testng.annotations.Parameters; +import org.testng.annotations.Test; -public class TestGravitinoConnector extends BaseConnectorTest { +@Parameters({"-Xmx1G"}) +public class TestGravitinoConnector extends AbstractTestQueryFramework { private static final Logger LOG = LoggerFactory.getLogger(TestGravitinoConnector.class); + MockGravitinoServer server; + MemoryConnector memoryConnector; + + private int initGravitinoServer() throws Exception { + server = closeAfterClass(new MockGravitinoServer()); + server.start(0); + return server.getLocalPort(); + } + @Override protected QueryRunner createQueryRunner() throws Exception { - Session session = testSessionBuilder().setCatalog("gravitino").build(); + int port = initGravitinoServer(); + Session session = testSessionBuilder().setCatalog("gravitino").build(); QueryRunner queryRunner = null; try { // queryRunner = LocalQueryRunner.builder(session).build(); queryRunner = DistributedQueryRunner.builder(session).setNodeCount(1).build(); queryRunner.installPlugin(new GravitinoPlugin()); + queryRunner.installPlugin(new MemoryPlugin()); HashMap properties = new HashMap<>(); properties.put("gravitino.metalake", "test"); + properties.put("gravitino.uri", "http://127.0.0.1:" + port); + queryRunner.createCatalog("gravitino", "gravitino", properties); + } catch (Exception e) { throw new RuntimeException(e); } return queryRunner; } - @Override + @Test public void testCreateSchema() { - assertThat(computeActual("SHOW catalogs").getOnlyColumnAsSet()).contains("gravitino"); + String catalogName = "test.memory"; + // testing the catalogs + assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("gravitino"); + assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains(catalogName); + + String schemaName = "db_01"; + String fullSchemaName = String.format("\"%s\".%s", catalogName, schemaName); + assertThat(computeActual("show schemas from \"test.memory\"").getOnlyColumnAsSet()) + .doesNotContain(schemaName); + + assertUpdate("create schema " + fullSchemaName); + assertThat(computeActual("show schemas from \"test.memory\"").getOnlyColumnAsSet()) + .contains(schemaName); + + assertThat((String) computeScalar("show create schema " + fullSchemaName)) + .startsWith(format("CREATE SCHEMA %s", fullSchemaName)); + + // try to create duplicate schema + assertQueryFails( + "create schema " + fullSchemaName, format("line 1:1: Schema .* already exists")); + + // cleanup + assertUpdate("drop schema " + fullSchemaName); + + // verify DROP SCHEMA for non-existing schema + assertQueryFails("drop schema " + fullSchemaName, format("line 1:1: Schema .* does not exist")); + } + + @Test + public void testCreateTable() { + String schemaName = "db_01"; + String fullSchemaName = "\"test.memory\".db_01"; + String tableName = "tb_01"; + String fullTableName = fullSchemaName + "." + tableName; + + // preparing internal connector metadata. + preparingTestingData(schemaName, tableName); + + assertUpdate("create schema " + fullSchemaName); + + // try to get table + assertThat(computeActual("show tables from " + fullSchemaName).getOnlyColumnAsSet()) + .doesNotContain(tableName); + + // try to create table + assertUpdate("create table " + fullTableName + " (a varchar, b int)"); + assertThat(computeActual("show tables from " + fullSchemaName).getOnlyColumnAsSet()) + .contains(tableName); + + assertThat((String) computeScalar("show create table " + fullTableName)) + .startsWith(format("CREATE TABLE %s", fullTableName)); + + // cleanup + assertUpdate("drop table" + fullTableName); + assertUpdate("drop schema " + fullSchemaName); + + cleanTestingData(schemaName); + } + + private void preparingTestingData(String schemaName, String tableName) { + memoryConnector = (MemoryConnector) GravitinoPlugin.internalTestingConnector; + + // create schema + ConnectorMetadata metadata = memoryConnector.getMetadata(null, null); + metadata.createSchema(null, schemaName, emptyMap(), null); + + // create table + SchemaTableName schemaTableName = new SchemaTableName(schemaName, tableName); + ArrayList columnMetadataList = new ArrayList<>(); + columnMetadataList.add(new ColumnMetadata("a", VARCHAR)); + columnMetadataList.add(new ColumnMetadata("b", INTEGER)); + + ConnectorTableMetadata connectorTableMetadata = + new ConnectorTableMetadata( + schemaTableName, columnMetadataList, emptyMap(), Optional.of("")); + metadata.createTable(null, connectorTableMetadata, false); + } + + private void cleanTestingData(String schemaName) { + memoryConnector = (MemoryConnector) GravitinoPlugin.internalTestingConnector; + ConnectorMetadata metadata = memoryConnector.getMetadata(null, null); + // drop schema and tables + metadata.dropSchema(null, schemaName, true); + } + + @Test + public void testInsert() { + String schemaName = "db_01"; + String fullSchemaName = "\"test.memory\".db_01"; + String tableName = "tb_01"; + String fullTableName = fullSchemaName + "." + tableName; + + // preparing internal connector metadata. + preparingTestingData(schemaName, tableName); + + // create schema and table + assertUpdate("create schema \"test.memory\".db_01"); + assertUpdate("create table " + fullTableName + " (a varchar, b int)"); + + // insert some data. + assertUpdate(String.format("insert into %s (a, b) values ('ice', 12)", fullTableName), 1); + + // select data from the table. + MaterializedResult expectedResult = computeActual("select * from " + fullTableName); + assertEquals(expectedResult.getRowCount(), 1); + List expectedRows = expectedResult.getMaterializedRows(); + MaterializedRow row = expectedRows.get(0); + assertEquals(row.getField(0), "ice"); + assertEquals(row.getField(1), 12); + + // cleanup + assertUpdate("drop schema " + fullSchemaName + " cascade"); + cleanTestingData(schemaName); } } diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoTableHandle.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoTableHandle.java index ab64187773..652205b365 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoTableHandle.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoTableHandle.java @@ -4,11 +4,13 @@ */ package com.datastrato.gravitino.trino.connector; +import static org.testng.Assert.assertTrue; + import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import io.airlift.json.JsonCodec; import io.trino.spi.connector.ConnectorTableHandle; -import org.junit.jupiter.api.Test; +import org.testng.annotations.Test; public class TestGravitinoTableHandle { private final JsonCodec codec = @@ -19,7 +21,10 @@ public void testCreateFromJson() { GravitinoTableHandle expected = new GravitinoTableHandle("db1", "t1", new MockConnectorTableHandle("mock")); - codec.toJson(expected); + String jsonStr = codec.toJson(expected); + assertTrue(jsonStr.contains("db1")); + assertTrue(jsonStr.contains("t1")); + assertTrue(jsonStr.contains("mock")); } public static class MockConnectorTableHandle implements ConnectorTableHandle { diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoCatalog.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoCatalog.java index 1fdbd55df6..b860e8ff39 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoCatalog.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoCatalog.java @@ -4,17 +4,18 @@ */ package com.datastrato.gravitino.trino.connector.metadata; +import static org.testng.Assert.assertEquals; + import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.dto.AuditDTO; import com.datastrato.gravitino.dto.CatalogDTO; import java.time.Instant; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; +import org.testng.annotations.Test; public class TestGravitinoCatalog { @Test - void testGravitinoCatalog() { + public void testGravitinoCatalog() { String catalogName = "mock"; String provider = "hive"; CatalogDTO mockCatalog = @@ -27,7 +28,7 @@ void testGravitinoCatalog() { new AuditDTO.Builder().withCreator("creator").withCreateTime(Instant.now()).build()) .build(); GravitinoCatalog catalog = new GravitinoCatalog(mockCatalog); - Assertions.assertEquals(catalogName, catalog.getName()); - Assertions.assertEquals(provider, catalog.getProvider()); + assertEquals(catalogName, catalog.getName()); + assertEquals(provider, catalog.getProvider()); } } diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoColumn.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoColumn.java index f600009bcf..40d968bc93 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoColumn.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoColumn.java @@ -4,15 +4,16 @@ */ package com.datastrato.gravitino.trino.connector.metadata; +import static org.testng.Assert.assertEquals; + import com.datastrato.gravitino.dto.rel.ColumnDTO; -import io.substrait.type.TypeCreator; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; +import com.datastrato.gravitino.shaded.io.substrait.type.TypeCreator; +import org.testng.annotations.Test; public class TestGravitinoColumn { @Test - void testGravitinoColumn() { + public void testGravitinoColumn() { ColumnDTO columnDTO = new ColumnDTO.Builder() .withName("f1") @@ -22,9 +23,9 @@ void testGravitinoColumn() { GravitinoColumn column = new GravitinoColumn(columnDTO, 0); - Assertions.assertEquals(column.getName(), columnDTO.name()); - Assertions.assertEquals(column.getIndex(), 0); - Assertions.assertEquals(column.getComment(), columnDTO.comment()); - Assertions.assertEquals(column.getType(), columnDTO.dataType()); + assertEquals(column.getName(), columnDTO.name()); + assertEquals(column.getIndex(), 0); + assertEquals(column.getComment(), columnDTO.comment()); + assertEquals(column.getType(), columnDTO.dataType()); } } diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoSchema.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoSchema.java index 1296789b91..0eab2d32f9 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoSchema.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoSchema.java @@ -4,18 +4,19 @@ */ package com.datastrato.gravitino.trino.connector.metadata; +import static org.testng.Assert.assertEquals; + import com.datastrato.gravitino.dto.AuditDTO; import com.datastrato.gravitino.dto.rel.SchemaDTO; import java.time.Instant; import java.util.HashMap; import java.util.Map; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; +import org.testng.annotations.Test; public class TestGravitinoSchema { @Test - void testGravitinoSchema() { + public void testGravitinoSchema() { Map properties = new HashMap<>(); properties.put("prop1", "test prop1"); @@ -30,8 +31,8 @@ void testGravitinoSchema() { GravitinoSchema schema = new GravitinoSchema(schemaDTO); - Assertions.assertEquals(schema.getName(), schemaDTO.name()); - Assertions.assertEquals(schema.getComment(), schemaDTO.comment()); - Assertions.assertEquals(schema.getProperties(), schemaDTO.properties()); + assertEquals(schema.getName(), schemaDTO.name()); + assertEquals(schema.getComment(), schemaDTO.comment()); + assertEquals(schema.getProperties(), schemaDTO.properties()); } } diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoTable.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoTable.java index f8aefbbfd0..1d056406d4 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoTable.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoTable.java @@ -4,20 +4,21 @@ */ package com.datastrato.gravitino.trino.connector.metadata; +import static org.testng.Assert.assertEquals; + import com.datastrato.gravitino.dto.AuditDTO; import com.datastrato.gravitino.dto.rel.ColumnDTO; import com.datastrato.gravitino.dto.rel.TableDTO; -import io.substrait.type.TypeCreator; +import com.datastrato.gravitino.shaded.io.substrait.type.TypeCreator; import java.time.Instant; import java.util.HashMap; import java.util.Map; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; +import org.testng.annotations.Test; public class TestGravitinoTable { @Test - void testGravitinoTable() { + public void testGravitinoTable() { ColumnDTO[] columns = new ColumnDTO[] { new ColumnDTO.Builder() @@ -45,13 +46,13 @@ void testGravitinoTable() { GravitinoTable table = new GravitinoTable("db1", "table1", tableDTO); - Assertions.assertEquals(table.getName(), tableDTO.name()); - Assertions.assertEquals(table.getSchemaName(), "db1"); - Assertions.assertEquals(table.getColumns().size(), tableDTO.columns().length); + assertEquals(table.getName(), tableDTO.name()); + assertEquals(table.getSchemaName(), "db1"); + assertEquals(table.getColumns().size(), tableDTO.columns().length); for (int i = 0; i < table.getColumns().size(); i++) { - Assertions.assertEquals(table.getColumns().get(i).getName(), tableDTO.columns()[i].name()); + assertEquals(table.getColumns().get(i).getName(), tableDTO.columns()[i].name()); } - Assertions.assertEquals(table.getComment(), tableDTO.comment()); - Assertions.assertEquals(table.getProperties(), tableDTO.properties()); + assertEquals(table.getComment(), tableDTO.comment()); + assertEquals(table.getProperties(), tableDTO.properties()); } } diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/util/GravitinoRestApi.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/util/GravitinoRestApi.java new file mode 100644 index 0000000000..cfd3696889 --- /dev/null +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/util/GravitinoRestApi.java @@ -0,0 +1,308 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.trino.connector.util; + +import com.datastrato.gravitino.Catalog; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.dto.AuditDTO; +import com.datastrato.gravitino.dto.CatalogDTO; +import com.datastrato.gravitino.dto.MetalakeDTO; +import com.datastrato.gravitino.dto.rel.ColumnDTO; +import com.datastrato.gravitino.dto.rel.PartitionUtils; +import com.datastrato.gravitino.dto.rel.SchemaDTO; +import com.datastrato.gravitino.dto.rel.TableDTO; +import com.datastrato.gravitino.dto.responses.CatalogResponse; +import com.datastrato.gravitino.dto.responses.DropResponse; +import com.datastrato.gravitino.dto.responses.EntityListResponse; +import com.datastrato.gravitino.dto.responses.ErrorResponse; +import com.datastrato.gravitino.dto.responses.MetalakeResponse; +import com.datastrato.gravitino.dto.responses.SchemaResponse; +import com.datastrato.gravitino.dto.responses.TableResponse; +import com.datastrato.gravitino.dto.util.DTOConverters; +import com.datastrato.gravitino.exceptions.NoSuchCatalogException; +import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; +import com.datastrato.gravitino.exceptions.NoSuchSchemaException; +import com.datastrato.gravitino.exceptions.NoSuchTableException; +import com.datastrato.gravitino.exceptions.NonEmptySchemaException; +import com.datastrato.gravitino.exceptions.SchemaAlreadyExistsException; +import com.datastrato.gravitino.exceptions.TableAlreadyExistsException; +import com.datastrato.gravitino.json.JsonUtils; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.DefaultValue; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.QueryParam; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +@Path("/") +public class GravitinoRestApi { + + private static String testMetalakeName = "test"; + private static String testCatalogName = "memory"; + private static Map testSchemas = new HashMap<>(); + private static Map testTables = new HashMap<>(); + + @GET + @Path("/metalakes/{name}") + public Response loadMetalake(@PathParam("name") String metalakeName) throws Exception { + if (!testMetalakeName.equals(metalakeName)) { + throw new NoSuchMetalakeException("Metalake does not exist"); + } + MetalakeDTO metalake = + new MetalakeDTO.Builder() + .withName(metalakeName) + .withComment("comment") + .withAudit(new AuditDTO.Builder().build()) + .build(); + return ok(new MetalakeResponse(metalake)); + } + + @GET + @Path("/metalakes/{metalake}/catalogs") + public Response listCatalogs(@PathParam("metalake") String metalake) throws Exception { + NameIdentifier[] idents = {NameIdentifier.ofCatalog(metalake, testCatalogName)}; + return ok(new EntityListResponse(idents)); + } + + @GET + @Path("/metalakes/{metalake}/catalogs/{catalog}") + public Response loadCatalog( + @PathParam("metalake") String metalakeName, @PathParam("catalog") String catalogName) + throws Exception { + if (!metalakeName.equals(testMetalakeName) || !catalogName.equals(testCatalogName)) { + throw new NoSuchCatalogException("Catalog does not exist"); + } + CatalogDTO catalog = + new CatalogDTO.Builder() + .withName(testCatalogName) + .withProvider("memory") + .withType(Catalog.Type.RELATIONAL) + .withProperties(Collections.emptyMap()) + .withAudit(new AuditDTO.Builder().build()) + .build(); + + return ok(new CatalogResponse(catalog)); + } + + @GET + @Path("/metalakes/{metalake}/catalogs/{catalog}/schemas") + public Response listSchemas( + @PathParam("metalake") String metalake, @PathParam("catalog") String catalog) + throws Exception { + if (!metalake.equals(testMetalakeName) || !catalog.equals(testCatalogName)) { + throw new NoSuchCatalogException("Catalog does not exist"); + } + + NameIdentifier[] schemas = + testSchemas.entrySet().stream() + .map((e) -> NameIdentifier.ofSchema(testMetalakeName, testCatalogName, e.getKey())) + .toArray(NameIdentifier[]::new); + return ok(new EntityListResponse(schemas)); + } + + @POST + @Path("/metalakes/{metalake}/catalogs/{catalog}/schemas") + public Response createSchema( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + String requestStr) + throws Exception { + if (!metalake.equals(testMetalakeName) || !catalog.equals(testCatalogName)) { + throw new NoSuchCatalogException("Catalog does not exist"); + } + + SchemaDTO originSchema = JsonUtils.objectMapper().readValue(requestStr, SchemaDTO.class); + + if (testSchemas.containsKey(originSchema.name())) { + throw new SchemaAlreadyExistsException("Schema already exists"); + } + + SchemaDTO schemaDTO = + new SchemaDTO.Builder() + .withName(originSchema.name()) + .withComment(originSchema.comment()) + .withProperties(originSchema.properties()) + .withAudit(new AuditDTO.Builder().build()) + .build(); + testSchemas.put(schemaDTO.name(), schemaDTO); + + return ok(new SchemaResponse(schemaDTO)); + } + + @GET + @Path("/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}") + public Response loadSchema( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema) + throws Exception { + if (!metalake.equals(testMetalakeName) || !catalog.equals(testCatalogName)) { + throw new NoSuchCatalogException("Catalog does not exist"); + } + + SchemaDTO schemaDTO = testSchemas.get(schema); + if (schemaDTO == null) { + throw new NoSuchSchemaException("Schema does not exist"); + } + + return ok(new SchemaResponse(schemaDTO)); + } + + @DELETE + @Path("/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}") + public Response dropSchema( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @DefaultValue("false") @QueryParam("cascade") boolean cascade) + throws Exception { + if (!metalake.equals(testMetalakeName) || !catalog.equals(testCatalogName)) { + throw new NoSuchCatalogException("Catalog does not exist"); + } + if (!testTables.isEmpty() && cascade == false) { + throw new NonEmptySchemaException("No"); + } + testSchemas.remove(schema); + return ok(new DropResponse(true)); + } + + @GET + @Path("/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/tables") + public Response listTables( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema) + throws Exception { + + if (!metalake.equals(testMetalakeName) || !catalog.equals(testCatalogName)) { + throw new NoSuchCatalogException("Catalog does not exist"); + } + + if (!testSchemas.containsKey(schema)) { + throw new NoSuchSchemaException("Schema does not exist"); + } + + NameIdentifier[] tables = + testTables.entrySet().stream() + .map( + (e) -> + NameIdentifier.ofTable(testMetalakeName, testCatalogName, schema, e.getKey())) + .toArray(NameIdentifier[]::new); + return ok(new EntityListResponse(tables)); + } + + @POST + @Path("/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/tables") + public Response createTable( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + String requestStr) + throws Exception { + + if (!metalake.equals(testMetalakeName) || !catalog.equals(testCatalogName)) { + throw new NoSuchCatalogException("Catalog does not exist"); + } + + if (!testSchemas.containsKey(schema)) { + throw new NoSuchSchemaException("Schema does not exist"); + } + + TableDTO originTable = JsonUtils.objectMapper().readValue(requestStr, TableDTO.class); + + if (testTables.containsKey(originTable.name())) { + throw new TableAlreadyExistsException("Table already exists"); + } + + TableDTO table = + TableDTO.builder() + .withName(originTable.name()) + .withComment(originTable.comment()) + .withColumns((ColumnDTO[]) originTable.columns()) + .withProperties(originTable.properties()) + .withPartitions(PartitionUtils.toPartitions(originTable.partitioning())) + .withDistribution(DTOConverters.toDTO(originTable.distribution())) + .withSortOrders(DTOConverters.toDTOs(originTable.sortOrder())) + .withAudit(new AuditDTO.Builder().build()) + .build(); + + testTables.put(table.name(), table); + return ok(new TableResponse(table)); + } + + @GET + @Path("/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/tables/{table}") + public Response loadTable( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("table") String table) + throws Exception { + if (!metalake.equals(testMetalakeName) || !catalog.equals(testCatalogName)) { + throw new NoSuchCatalogException("Catalog does not exist"); + } + + if (!testSchemas.containsKey(schema)) { + throw new NoSuchSchemaException("Schema does not exist"); + } + + TableDTO tableDTO = testTables.get(table); + if (tableDTO == null) { + NoSuchTableException tableDoesNotExist = new NoSuchTableException("Table does not exist"); + return notFound( + NoSuchTableException.class.getSimpleName(), + tableDoesNotExist.getMessage(), + tableDoesNotExist); + } + return ok(new TableResponse(tableDTO)); + } + + @DELETE + @Path("/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/tables/{table}") + public Response dropTable( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("table") String table, + @QueryParam("purge") @DefaultValue("false") boolean purge) + throws Exception { + if (!metalake.equals(testMetalakeName) || !catalog.equals(testCatalogName)) { + throw new NoSuchCatalogException("Catalog does not exist"); + } + + if (!testTables.containsKey(schema)) { + throw new NoSuchSchemaException("Schema does not exist"); + } + + TableDTO tableDTO = testTables.get(table); + boolean dropped = false; + if (testTables.containsKey(table)) { + dropped = true; + testTables.remove(table); + } + + return ok(new DropResponse(dropped)); + } + + public static Response ok(T t) throws Exception { + String str = JsonUtils.objectMapper().writeValueAsString(t); + return Response.status(Response.Status.OK).entity(str).type(MediaType.APPLICATION_JSON).build(); + } + + public static Response notFound(String type, String msg, Throwable t) throws Exception { + ErrorResponse errorResponse = ErrorResponse.notFound(type, msg, t); + String str = JsonUtils.objectMapper().writeValueAsString(errorResponse); + return Response.status(Response.Status.NOT_FOUND) + .entity(str) + .type(MediaType.APPLICATION_JSON) + .build(); + } +} diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/util/MockGravitinoServer.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/util/MockGravitinoServer.java new file mode 100644 index 0000000000..9990d34979 --- /dev/null +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/util/MockGravitinoServer.java @@ -0,0 +1,54 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.trino.connector.util; + +import io.trino.testing.ResourcePresence; +import org.eclipse.jetty.server.NetworkConnector; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.glassfish.jersey.server.ResourceConfig; +import org.glassfish.jersey.servlet.ServletContainer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MockGravitinoServer implements AutoCloseable { + private static final Logger LOG = LoggerFactory.getLogger(MockGravitinoServer.class); + + private Server server; + + public void start(int port) throws Exception { + server = new Server(port); + ResourceConfig config = new ResourceConfig(); + config.packages("com.datastrato.gravitino.trino.connector.util"); + ServletHolder servlet = new ServletHolder(new ServletContainer(config)); + + ServletContextHandler context = new ServletContextHandler(server, "/"); + context.addServlet(servlet, "/api/*"); + server.setHandler(context); + server.start(); + } + + public void stop() throws Exception { + server.stop(); + } + + public int getLocalPort() throws Exception { + if (server.getConnectors().length == 0) + throw new Exception("MockGravitinoServer port is not initialized"); + NetworkConnector connector = (NetworkConnector) server.getConnectors()[0]; + return connector.getLocalPort(); + } + + @ResourcePresence + public boolean isRunning() { + return server.isRunning(); + } + + @Override + public void close() throws Exception { + stop(); + } +} diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/util/TestDataTypeTransformer.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/util/TestDataTypeTransformer.java index 1c7e72964c..80f526f3d5 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/util/TestDataTypeTransformer.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/util/TestDataTypeTransformer.java @@ -15,55 +15,44 @@ import static io.trino.spi.type.TimestampType.TIMESTAMP_SECONDS; import static io.trino.spi.type.VarbinaryType.VARBINARY; import static io.trino.spi.type.VarcharType.VARCHAR; +import static org.testng.Assert.assertEquals; -import io.substrait.type.TypeCreator; +import com.datastrato.gravitino.shaded.io.substrait.type.TypeCreator; import io.trino.spi.TrinoException; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; +import org.testng.annotations.Test; public class TestDataTypeTransformer { @Test public void testGetGravitinoType() { - Assertions.assertEquals( - DataTypeTransformer.getGravitinoType(VARCHAR, true), TypeCreator.NULLABLE.STRING); - Assertions.assertEquals( - DataTypeTransformer.getGravitinoType(VARCHAR, false), TypeCreator.REQUIRED.STRING); - - Assertions.assertEquals( - DataTypeTransformer.getGravitinoType(BOOLEAN, true), TypeCreator.NULLABLE.BOOLEAN); - Assertions.assertEquals( + assertEquals(DataTypeTransformer.getGravitinoType(VARCHAR, true), TypeCreator.NULLABLE.STRING); + assertEquals(DataTypeTransformer.getGravitinoType(VARCHAR, false), TypeCreator.REQUIRED.STRING); + + assertEquals(DataTypeTransformer.getGravitinoType(BOOLEAN, true), TypeCreator.NULLABLE.BOOLEAN); + assertEquals( DataTypeTransformer.getGravitinoType(BOOLEAN, false), TypeCreator.REQUIRED.BOOLEAN); - Assertions.assertEquals( - DataTypeTransformer.getGravitinoType(INTEGER, true), TypeCreator.NULLABLE.I32); - Assertions.assertEquals( - DataTypeTransformer.getGravitinoType(INTEGER, false), TypeCreator.REQUIRED.I32); + assertEquals(DataTypeTransformer.getGravitinoType(INTEGER, true), TypeCreator.NULLABLE.I32); + assertEquals(DataTypeTransformer.getGravitinoType(INTEGER, false), TypeCreator.REQUIRED.I32); - Assertions.assertEquals( - DataTypeTransformer.getGravitinoType(BIGINT, true), TypeCreator.NULLABLE.I64); - Assertions.assertEquals( - DataTypeTransformer.getGravitinoType(BIGINT, false), TypeCreator.REQUIRED.I64); + assertEquals(DataTypeTransformer.getGravitinoType(BIGINT, true), TypeCreator.NULLABLE.I64); + assertEquals(DataTypeTransformer.getGravitinoType(BIGINT, false), TypeCreator.REQUIRED.I64); - Assertions.assertEquals( - DataTypeTransformer.getGravitinoType(DOUBLE, true), TypeCreator.NULLABLE.FP64); - Assertions.assertEquals( - DataTypeTransformer.getGravitinoType(DOUBLE, false), TypeCreator.REQUIRED.FP64); + assertEquals(DataTypeTransformer.getGravitinoType(DOUBLE, true), TypeCreator.NULLABLE.FP64); + assertEquals(DataTypeTransformer.getGravitinoType(DOUBLE, false), TypeCreator.REQUIRED.FP64); - Assertions.assertEquals( + assertEquals( DataTypeTransformer.getGravitinoType(VARBINARY, true), TypeCreator.NULLABLE.BINARY); - Assertions.assertEquals( + assertEquals( DataTypeTransformer.getGravitinoType(VARBINARY, false), TypeCreator.REQUIRED.BINARY); - Assertions.assertEquals( - DataTypeTransformer.getGravitinoType(DATE, true), TypeCreator.NULLABLE.DATE); - Assertions.assertEquals( - DataTypeTransformer.getGravitinoType(DATE, false), TypeCreator.REQUIRED.DATE); + assertEquals(DataTypeTransformer.getGravitinoType(DATE, true), TypeCreator.NULLABLE.DATE); + assertEquals(DataTypeTransformer.getGravitinoType(DATE, false), TypeCreator.REQUIRED.DATE); - Assertions.assertEquals( + assertEquals( DataTypeTransformer.getGravitinoType(TIMESTAMP_SECONDS, true), TypeCreator.NULLABLE.TIMESTAMP); - Assertions.assertEquals( + assertEquals( DataTypeTransformer.getGravitinoType(TIMESTAMP_SECONDS, false), TypeCreator.REQUIRED.TIMESTAMP); @@ -78,32 +67,30 @@ public void testGetGravitinoType() { @Test public void testGetTrinoType() { - Assertions.assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.NULLABLE.STRING), VARCHAR); - Assertions.assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.REQUIRED.STRING), VARCHAR); + assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.NULLABLE.STRING), VARCHAR); + assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.REQUIRED.STRING), VARCHAR); - Assertions.assertEquals( - DataTypeTransformer.getTrinoType(TypeCreator.NULLABLE.BOOLEAN), BOOLEAN); - Assertions.assertEquals( - DataTypeTransformer.getTrinoType(TypeCreator.REQUIRED.BOOLEAN), BOOLEAN); + assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.NULLABLE.BOOLEAN), BOOLEAN); + assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.REQUIRED.BOOLEAN), BOOLEAN); - Assertions.assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.NULLABLE.I32), INTEGER); - Assertions.assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.REQUIRED.I32), INTEGER); + assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.NULLABLE.I32), INTEGER); + assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.REQUIRED.I32), INTEGER); - Assertions.assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.NULLABLE.I64), BIGINT); - Assertions.assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.REQUIRED.I64), BIGINT); + assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.NULLABLE.I64), BIGINT); + assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.REQUIRED.I64), BIGINT); - Assertions.assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.NULLABLE.I64), BIGINT); - Assertions.assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.REQUIRED.I64), BIGINT); + assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.NULLABLE.I64), BIGINT); + assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.REQUIRED.I64), BIGINT); - Assertions.assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.NULLABLE.FP64), DOUBLE); - Assertions.assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.REQUIRED.FP64), DOUBLE); + assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.NULLABLE.FP64), DOUBLE); + assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.REQUIRED.FP64), DOUBLE); - Assertions.assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.NULLABLE.DATE), DATE); - Assertions.assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.REQUIRED.DATE), DATE); + assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.NULLABLE.DATE), DATE); + assertEquals(DataTypeTransformer.getTrinoType(TypeCreator.REQUIRED.DATE), DATE); - Assertions.assertEquals( + assertEquals( DataTypeTransformer.getTrinoType(TypeCreator.NULLABLE.TIMESTAMP), TIMESTAMP_SECONDS); - Assertions.assertEquals( + assertEquals( DataTypeTransformer.getTrinoType(TypeCreator.REQUIRED.TIMESTAMP), TIMESTAMP_SECONDS); try { diff --git a/trino-connector/src/test/resources/log4j2.properties b/trino-connector/src/test/resources/log4j2.properties index 200adccfbf..a92b06a8e5 100644 --- a/trino-connector/src/test/resources/log4j2.properties +++ b/trino-connector/src/test/resources/log4j2.properties @@ -16,7 +16,7 @@ appender.console.layout.type = PatternLayout appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p [%t] %c{1}:%L - %m%n # Root logger level -rootLogger.level = debug +rootLogger.level = info # Root logger referring to console appender -rootLogger.appenderRef.stdout.ref = consoleLogger \ No newline at end of file +rootLogger.appenderRef.stdout.ref = consoleLogger