From 41b11bfacad92a5c7117ddf62dd94f8aa8426e6a Mon Sep 17 00:00:00 2001 From: siddharth Date: Thu, 7 Dec 2017 13:57:30 -0800 Subject: [PATCH] Add Decimal Vector APIs to write Big Endian data --- .../templates/AbstractFieldWriter.java | 4 ++ .../AbstractPromotableFieldWriter.java | 8 ++++ .../codegen/templates/ComplexWriters.java | 7 +++ .../arrow/vector/NullableDecimalVector.java | 48 +++++++++++++++++++ 4 files changed, 67 insertions(+) diff --git a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java index 853f67fd0dd56..fce6876025a91 100644 --- a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java +++ b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java @@ -67,6 +67,10 @@ public void write(${name}Holder holder) { public void write${minor.class}(${friendlyType} value) { fail("${name}"); } + + public void writeBigEndianBytesToDecimal(byte[] value) { + fail("${name}"); + } diff --git a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java index 228c2c531f98f..7f4a13d4f06e8 100644 --- a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java +++ b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java @@ -16,6 +16,8 @@ * limitations under the License. */ +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.vector.types.Types; import org.apache.drill.common.types.TypeProtos.MinorType; <@pp.dropOutputFile /> @@ -82,6 +84,12 @@ public void write(${name}Holder holder) { getWriter(MinorType.${name?upper_case}).write${minor.class}(<#list fields as field>${field.name}<#if field_has_next>, ); } + <#if minor.class == "Decimal"> + public void writeBigEndianBytesToDecimal(byte[] value) { + getWriter(Types.MinorType.DECIMAL).writeBigEndianBytesToDecimal(value); + } + + public void writeNull() { } diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java index 406bbb39c7f4a..8cad12ac318a7 100644 --- a/java/vector/src/main/codegen/templates/ComplexWriters.java +++ b/java/vector/src/main/codegen/templates/ComplexWriters.java @@ -120,6 +120,11 @@ public void write(Nullable${minor.class}Holder h) { vector.setSafe(idx(), value); vector.setValueCount(idx()+1); } + + public void writeBigEndianBytesToDecimal(byte[] value) { + vector.setBigEndianSafe(idx(), value); + vector.setValueCount(idx()+1); + } <#if mode == "Nullable"> @@ -148,6 +153,8 @@ public interface ${eName}Writer extends BaseWriter { <#if minor.class == "Decimal"> public void write${minor.class}(${friendlyType} value); + + public void writeBigEndianBytesToDecimal(byte[] value); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java index dcc551094ae28..0d0a7c0ec1332 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java @@ -18,6 +18,7 @@ package org.apache.arrow.vector; +import com.google.common.base.Preconditions; import io.netty.buffer.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.complex.impl.DecimalReaderImpl; @@ -199,6 +200,43 @@ public void set(int index, ArrowBuf buffer) { valueBuffer.setBytes(index * TYPE_WIDTH, buffer, 0, TYPE_WIDTH); } + /** + * Set the decimal element at given index to the provided array of bytes. + * Decimal is now implemented as Little Endian. This API allows the user + * to pass a decimal value in the form of byte array in BE byte order. + * + * Consumers of Arrow code can use this API instead of first swapping + * the source bytes (doing a write and read) and then finally writing to + * ArrowBuf of decimal vector. + * + * This method takes care of adding the necessary padding if the length + * of byte array is less then 16 (length of decimal type). + * + * @param index position of element + * @param value array of bytes containing decimal in big endian byte order. + */ + public void setBigEndian(int index, byte[] value) { + assert value.length <= TYPE_WIDTH; + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int length = value.length; + int startIndex = index * TYPE_WIDTH; + if (length == TYPE_WIDTH) { + for (int i = TYPE_WIDTH - 1; i >= 3; i-=4) { + valueBuffer.setByte(startIndex, value[i]); + valueBuffer.setByte(startIndex + 1, value[i-1]); + valueBuffer.setByte(startIndex + 2, value[i-2]); + valueBuffer.setByte(startIndex + 3, value[i-3]); + startIndex += 4; + } + } else { + for (int i = length - 1; i >= 0; i--) { + valueBuffer.setByte(startIndex, value[i]); + startIndex++; + } + valueBuffer.setZero(startIndex, TYPE_WIDTH - length); + } + } + /** * Set the element at the given index to the given value. * @@ -266,6 +304,16 @@ public void setSafe(int index, ArrowBuf buffer) { set(index, buffer); } + /** + * Same as {@link #setBigEndian(int, byte[])} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + */ + public void setBigEndianSafe(int index, byte[] value) { + handleSafe(index); + setBigEndian(index, value); + } + /** * Same as {@link #set(int, int, ArrowBuf)} except that it handles the * case when index is greater than or equal to existing