diff --git a/spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerBatch.java b/spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerBatch.java deleted file mode 100644 index d9fb8f77..00000000 --- a/spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerBatch.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.cloud.spark.spanner; - -import org.apache.spark.sql.connector.read.Batch; -import org.apache.spark.sql.connector.read.PartitionReaderFactory; -import org.apache.spark.sql.connector.read.Scan; -import org.apache.spark.sql.types.StructType; -import org.apache.spark.sql.util.CaseInsensitiveStringMap; - -public class SpannerBatch implements Batch { - @Override - public PartititionReaderFactory createReaderFactory() { - // TODO: Fill me in. - return null; - } - - @Override - public InputPartition[] planInputPartitions() { - // TODO: Fill me in. - return null; - } -} diff --git a/spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerDataReaderFactory.java b/spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerDataReaderFactory.java deleted file mode 100644 index 9ad773dc..00000000 --- a/spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerDataReaderFactory.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.cloud.spark.spanner; - -import org.apache.spark.sql.sources.v2.reader.DataReader; -import org.apache.spark.sql.sources.v2.reader.DataReaderFactory; - -public class SpannerDataReaderFactory implements DataReaderFactory { - @Override - public DataReader createDataReader() { - return new SpannerDataReader(); - } -} diff --git a/spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerDataReader.java b/spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerDataReader_java similarity index 98% rename from spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerDataReader.java rename to spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerDataReader_java index 83aeda3f..ae93441a 100644 --- a/spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerDataReader.java +++ b/spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerDataReader_java @@ -28,6 +28,6 @@ public boolean next() { @Override public T get() { // TODO: Fill me in. - return true; + return null; } } diff --git a/spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerDataSourceReader.java b/spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerDataSourceReader_java similarity index 100% rename from spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerDataSourceReader.java rename to spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerDataSourceReader_java diff --git a/spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerPartitionReader.java b/spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerPartitionReader.java index a967846d..39e524b9 100644 --- a/spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerPartitionReader.java +++ b/spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerPartitionReader.java @@ -17,9 +17,7 @@ import java.io.IOException; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.connector.read.Batch; -import org.apache.spark.sql.connector.read.ColumnarBatch; import org.apache.spark.sql.connector.read.InputPartition; -import org.apache.spark.sql.connector.read.Partition; import org.apache.spark.sql.connector.read.PartitionReader; import org.apache.spark.sql.connector.read.Scan; import org.apache.spark.sql.types.StructType; @@ -43,7 +41,6 @@ public void close() throws IOException { // TODO: Fill me in. } - @Override public boolean supportsColumnarReads(InputPartition partition) { // TODO: Fill me in. return false; diff --git a/spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerPartitionReaderFactory.java b/spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerPartitionReaderFactory_java similarity index 100% rename from spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerPartitionReaderFactory.java rename to spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerPartitionReaderFactory_java diff --git a/spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerSpark.java b/spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerSpark.java index 7320f4c4..91210b13 100644 --- a/spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerSpark.java +++ b/spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerSpark.java @@ -29,6 +29,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -42,17 +43,15 @@ import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.connector.catalog.SupportsRead; import org.apache.spark.sql.connector.catalog.Table; +import org.apache.spark.sql.connector.catalog.TableCapability; import org.apache.spark.sql.connector.catalog.TableProvider; import org.apache.spark.sql.connector.expressions.Transform; import org.apache.spark.sql.connector.read.PartitionReader; import org.apache.spark.sql.connector.read.ScanBuilder; -import org.apache.spark.sql.sources.v2.DataSource; -import org.apache.spark.sql.sources.v2.reader.DataSourceOptions; -import org.apache.spark.sql.sources.v2.reader.DataSourceReader; import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.util.CaseInsensitiveStringMap; -public class SpannerSpark implements TableProvider, SupportsRead, ReadSupport { +public class SpannerSpark implements TableProvider, SupportsRead { private BatchClient batchClient; private Map properties; @@ -195,16 +194,23 @@ public ScanBuilder newScanBuilder(CaseInsensitiveStringMap options) { return new SpannerScanBuilder(options); } + @Override + public Set capabilities() { + return null; + } + @Override public Map properties() { return this.properties; } - /* - * The entry point to create a reader. - */ @Override - public DataSourceReader createReader(DataSourceOptions options) { - return new SpannerDataSourceReader(options); + public String name() { + return "spanner"; + } + + @Override + public StructType schema() { + return null; } } diff --git a/spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerTable.java b/spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerTable.java index fba0f098..3b05a346 100644 --- a/spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerTable.java +++ b/spark-3.1-spanner-lib/src/main/java/com/google/cloud/spark/spanner/SpannerTable.java @@ -83,6 +83,7 @@ public StructType createSchema(String tableName, ResultSet rs) { } } this.tableSchema = schema; + return schema; } public static DataType ofSpannerStrType(String spannerStrType) {