Skip to content

Commit

Permalink
Vector Expansion Kernel O(n^2) -> O(n) (#4722)
Browse files Browse the repository at this point in the history
  • Loading branch information
@nbauernfeind
nbauernfeind authored Oct 25, 2023
1 parent 321bc3d commit f6dc16c
Show file tree
Hide file tree
Showing 10 changed files with 236 additions and 168 deletions.
45 changes: 26 additions & 19 deletions ...src/main/java/io/deephaven/extensions/barrage/chunk/vector/ByteVectorExpansionKernel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,9 @@
import io.deephaven.chunk.WritableObjectChunk;
import io.deephaven.chunk.attributes.Any;
import io.deephaven.chunk.attributes.ChunkPositions;
import io.deephaven.chunk.sized.SizedByteChunk;
import io.deephaven.engine.primitive.function.ByteConsumer;
import io.deephaven.engine.primitive.iterator.CloseablePrimitiveIteratorOfByte;
import io.deephaven.util.datastructures.LongSizedDataStructure;
import io.deephaven.vector.ByteVector;
import io.deephaven.vector.ByteVectorDirect;
import io.deephaven.vector.Vector;
Expand All @@ -37,24 +39,31 @@ public <A extends Any> WritableChunk<A> expand(
}

final ObjectChunk<ByteVector, A> typedSource = source.asObjectChunk();
final SizedByteChunk<A> resultWrapper = new SizedByteChunk<>();

int lenWritten = 0;
long totalSize = 0;
for (int i = 0; i < typedSource.size(); ++i) {
final ByteVector row = typedSource.get(i);
totalSize += row == null ? 0 : row.size();
}
final WritableByteChunk<A> result = WritableByteChunk.makeWritableChunk(
LongSizedDataStructure.intSize("ExpansionKernel", totalSize));
result.setSize(0);

perElementLengthDest.setSize(source.size() + 1);
for (int i = 0; i < typedSource.size(); ++i) {
final ByteVector row = typedSource.get(i);
final int len = row == null ? 0 : row.intSize("ByteVectorExpansionKernel");
perElementLengthDest.set(i, lenWritten);
final WritableByteChunk<A> result = resultWrapper.ensureCapacityPreserve(lenWritten + len);
for (int j = 0; j < len; ++j) {
result.set(lenWritten + j, row.get(j));
perElementLengthDest.set(i, result.size());
if (row == null) {
continue;
}
final ByteConsumer consumer = result::add;
try (final CloseablePrimitiveIteratorOfByte iter = row.iterator()) {
iter.forEachRemaining(consumer);
}
lenWritten += len;
result.setSize(lenWritten);
}
perElementLengthDest.set(typedSource.size(), lenWritten);
perElementLengthDest.set(typedSource.size(), result.size());

return resultWrapper.get();
return result;
}

@Override
Expand All @@ -81,15 +90,13 @@ public <A extends Any> WritableObjectChunk<Vector<?>, A> contract(

int lenRead = 0;
for (int i = 0; i < itemsInBatch; ++i) {
final int ROW_LEN = perElementLengthDest.get(i + 1) - perElementLengthDest.get(i);
if (ROW_LEN == 0) {
final int rowLen = perElementLengthDest.get(i + 1) - perElementLengthDest.get(i);
if (rowLen == 0) {
result.set(outOffset + i, ZERO_LENGTH_VECTOR);
} else {
final byte[] row = new byte[ROW_LEN];
for (int j = 0; j < ROW_LEN; ++j) {
row[j] = typedSource.get(lenRead + j);
}
lenRead += ROW_LEN;
final byte[] row = new byte[rowLen];
typedSource.copyToArray(lenRead, row, 0, rowLen);
lenRead += rowLen;
result.set(outOffset + i, new ByteVectorDirect(row));
}
}
Expand Down
45 changes: 26 additions & 19 deletions ...src/main/java/io/deephaven/extensions/barrage/chunk/vector/CharVectorExpansionKernel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@
import io.deephaven.chunk.WritableObjectChunk;
import io.deephaven.chunk.attributes.Any;
import io.deephaven.chunk.attributes.ChunkPositions;
import io.deephaven.chunk.sized.SizedCharChunk;
import io.deephaven.engine.primitive.function.CharConsumer;
import io.deephaven.engine.primitive.iterator.CloseablePrimitiveIteratorOfChar;
import io.deephaven.util.datastructures.LongSizedDataStructure;
import io.deephaven.vector.CharVector;
import io.deephaven.vector.CharVectorDirect;
import io.deephaven.vector.Vector;
Expand All @@ -32,24 +34,31 @@ public <A extends Any> WritableChunk<A> expand(
}

final ObjectChunk<CharVector, A> typedSource = source.asObjectChunk();
final SizedCharChunk<A> resultWrapper = new SizedCharChunk<>();

int lenWritten = 0;
long totalSize = 0;
for (int i = 0; i < typedSource.size(); ++i) {
final CharVector row = typedSource.get(i);
totalSize += row == null ? 0 : row.size();
}
final WritableCharChunk<A> result = WritableCharChunk.makeWritableChunk(
LongSizedDataStructure.intSize("ExpansionKernel", totalSize));
result.setSize(0);

perElementLengthDest.setSize(source.size() + 1);
for (int i = 0; i < typedSource.size(); ++i) {
final CharVector row = typedSource.get(i);
final int len = row == null ? 0 : row.intSize("CharVectorExpansionKernel");
perElementLengthDest.set(i, lenWritten);
final WritableCharChunk<A> result = resultWrapper.ensureCapacityPreserve(lenWritten + len);
for (int j = 0; j < len; ++j) {
result.set(lenWritten + j, row.get(j));
perElementLengthDest.set(i, result.size());
if (row == null) {
continue;
}
final CharConsumer consumer = result::add;
try (final CloseablePrimitiveIteratorOfChar iter = row.iterator()) {
iter.forEachRemaining(consumer);
}
lenWritten += len;
result.setSize(lenWritten);
}
perElementLengthDest.set(typedSource.size(), lenWritten);
perElementLengthDest.set(typedSource.size(), result.size());

return resultWrapper.get();
return result;
}

@Override
Expand All @@ -76,15 +85,13 @@ public <A extends Any> WritableObjectChunk<Vector<?>, A> contract(

int lenRead = 0;
for (int i = 0; i < itemsInBatch; ++i) {
final int ROW_LEN = perElementLengthDest.get(i + 1) - perElementLengthDest.get(i);
if (ROW_LEN == 0) {
final int rowLen = perElementLengthDest.get(i + 1) - perElementLengthDest.get(i);
if (rowLen == 0) {
result.set(outOffset + i, ZERO_LENGTH_VECTOR);
} else {
final char[] row = new char[ROW_LEN];
for (int j = 0; j < ROW_LEN; ++j) {
row[j] = typedSource.get(lenRead + j);
}
lenRead += ROW_LEN;
final char[] row = new char[rowLen];
typedSource.copyToArray(lenRead, row, 0, rowLen);
lenRead += rowLen;
result.set(outOffset + i, new CharVectorDirect(row));
}
}
Expand Down
46 changes: 27 additions & 19 deletions ...c/main/java/io/deephaven/extensions/barrage/chunk/vector/DoubleVectorExpansionKernel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
*/
package io.deephaven.extensions.barrage.chunk.vector;

import java.util.function.DoubleConsumer;

import io.deephaven.chunk.DoubleChunk;
import io.deephaven.chunk.Chunk;
import io.deephaven.chunk.IntChunk;
Expand All @@ -18,7 +20,8 @@
import io.deephaven.chunk.WritableObjectChunk;
import io.deephaven.chunk.attributes.Any;
import io.deephaven.chunk.attributes.ChunkPositions;
import io.deephaven.chunk.sized.SizedDoubleChunk;
import io.deephaven.engine.primitive.iterator.CloseablePrimitiveIteratorOfDouble;
import io.deephaven.util.datastructures.LongSizedDataStructure;
import io.deephaven.vector.DoubleVector;
import io.deephaven.vector.DoubleVectorDirect;
import io.deephaven.vector.Vector;
Expand All @@ -37,24 +40,31 @@ public <A extends Any> WritableChunk<A> expand(
}

final ObjectChunk<DoubleVector, A> typedSource = source.asObjectChunk();
final SizedDoubleChunk<A> resultWrapper = new SizedDoubleChunk<>();

int lenWritten = 0;
long totalSize = 0;
for (int i = 0; i < typedSource.size(); ++i) {
final DoubleVector row = typedSource.get(i);
totalSize += row == null ? 0 : row.size();
}
final WritableDoubleChunk<A> result = WritableDoubleChunk.makeWritableChunk(
LongSizedDataStructure.intSize("ExpansionKernel", totalSize));
result.setSize(0);

perElementLengthDest.setSize(source.size() + 1);
for (int i = 0; i < typedSource.size(); ++i) {
final DoubleVector row = typedSource.get(i);
final int len = row == null ? 0 : row.intSize("DoubleVectorExpansionKernel");
perElementLengthDest.set(i, lenWritten);
final WritableDoubleChunk<A> result = resultWrapper.ensureCapacityPreserve(lenWritten + len);
for (int j = 0; j < len; ++j) {
result.set(lenWritten + j, row.get(j));
perElementLengthDest.set(i, result.size());
if (row == null) {
continue;
}
final DoubleConsumer consumer = result::add;
try (final CloseablePrimitiveIteratorOfDouble iter = row.iterator()) {
iter.forEachRemaining(consumer);
}
lenWritten += len;
result.setSize(lenWritten);
}
perElementLengthDest.set(typedSource.size(), lenWritten);
perElementLengthDest.set(typedSource.size(), result.size());

return resultWrapper.get();
return result;
}

@Override
Expand All @@ -81,15 +91,13 @@ public <A extends Any> WritableObjectChunk<Vector<?>, A> contract(

int lenRead = 0;
for (int i = 0; i < itemsInBatch; ++i) {
final int ROW_LEN = perElementLengthDest.get(i + 1) - perElementLengthDest.get(i);
if (ROW_LEN == 0) {
final int rowLen = perElementLengthDest.get(i + 1) - perElementLengthDest.get(i);
if (rowLen == 0) {
result.set(outOffset + i, ZERO_LENGTH_VECTOR);
} else {
final double[] row = new double[ROW_LEN];
for (int j = 0; j < ROW_LEN; ++j) {
row[j] = typedSource.get(lenRead + j);
}
lenRead += ROW_LEN;
final double[] row = new double[rowLen];
typedSource.copyToArray(lenRead, row, 0, rowLen);
lenRead += rowLen;
result.set(outOffset + i, new DoubleVectorDirect(row));
}
}
Expand Down
45 changes: 26 additions & 19 deletions ...rc/main/java/io/deephaven/extensions/barrage/chunk/vector/FloatVectorExpansionKernel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,9 @@
import io.deephaven.chunk.WritableObjectChunk;
import io.deephaven.chunk.attributes.Any;
import io.deephaven.chunk.attributes.ChunkPositions;
import io.deephaven.chunk.sized.SizedFloatChunk;
import io.deephaven.engine.primitive.function.FloatConsumer;
import io.deephaven.engine.primitive.iterator.CloseablePrimitiveIteratorOfFloat;
import io.deephaven.util.datastructures.LongSizedDataStructure;
import io.deephaven.vector.FloatVector;
import io.deephaven.vector.FloatVectorDirect;
import io.deephaven.vector.Vector;
Expand All @@ -37,24 +39,31 @@ public <A extends Any> WritableChunk<A> expand(
}

final ObjectChunk<FloatVector, A> typedSource = source.asObjectChunk();
final SizedFloatChunk<A> resultWrapper = new SizedFloatChunk<>();

int lenWritten = 0;
long totalSize = 0;
for (int i = 0; i < typedSource.size(); ++i) {
final FloatVector row = typedSource.get(i);
totalSize += row == null ? 0 : row.size();
}
final WritableFloatChunk<A> result = WritableFloatChunk.makeWritableChunk(
LongSizedDataStructure.intSize("ExpansionKernel", totalSize));
result.setSize(0);

perElementLengthDest.setSize(source.size() + 1);
for (int i = 0; i < typedSource.size(); ++i) {
final FloatVector row = typedSource.get(i);
final int len = row == null ? 0 : row.intSize("FloatVectorExpansionKernel");
perElementLengthDest.set(i, lenWritten);
final WritableFloatChunk<A> result = resultWrapper.ensureCapacityPreserve(lenWritten + len);
for (int j = 0; j < len; ++j) {
result.set(lenWritten + j, row.get(j));
perElementLengthDest.set(i, result.size());
if (row == null) {
continue;
}
final FloatConsumer consumer = result::add;
try (final CloseablePrimitiveIteratorOfFloat iter = row.iterator()) {
iter.forEachRemaining(consumer);
}
lenWritten += len;
result.setSize(lenWritten);
}
perElementLengthDest.set(typedSource.size(), lenWritten);
perElementLengthDest.set(typedSource.size(), result.size());

return resultWrapper.get();
return result;
}

@Override
Expand All @@ -81,15 +90,13 @@ public <A extends Any> WritableObjectChunk<Vector<?>, A> contract(

int lenRead = 0;
for (int i = 0; i < itemsInBatch; ++i) {
final int ROW_LEN = perElementLengthDest.get(i + 1) - perElementLengthDest.get(i);
if (ROW_LEN == 0) {
final int rowLen = perElementLengthDest.get(i + 1) - perElementLengthDest.get(i);
if (rowLen == 0) {
result.set(outOffset + i, ZERO_LENGTH_VECTOR);
} else {
final float[] row = new float[ROW_LEN];
for (int j = 0; j < ROW_LEN; ++j) {
row[j] = typedSource.get(lenRead + j);
}
lenRead += ROW_LEN;
final float[] row = new float[rowLen];
typedSource.copyToArray(lenRead, row, 0, rowLen);
lenRead += rowLen;
result.set(outOffset + i, new FloatVectorDirect(row));
}
}
Expand Down
46 changes: 27 additions & 19 deletions .../src/main/java/io/deephaven/extensions/barrage/chunk/vector/IntVectorExpansionKernel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
*/
package io.deephaven.extensions.barrage.chunk.vector;

import java.util.function.IntConsumer;

import io.deephaven.chunk.IntChunk;
import io.deephaven.chunk.Chunk;
import io.deephaven.chunk.IntChunk;
Expand All @@ -18,7 +20,8 @@
import io.deephaven.chunk.WritableObjectChunk;
import io.deephaven.chunk.attributes.Any;
import io.deephaven.chunk.attributes.ChunkPositions;
import io.deephaven.chunk.sized.SizedIntChunk;
import io.deephaven.engine.primitive.iterator.CloseablePrimitiveIteratorOfInt;
import io.deephaven.util.datastructures.LongSizedDataStructure;
import io.deephaven.vector.IntVector;
import io.deephaven.vector.IntVectorDirect;
import io.deephaven.vector.Vector;
Expand All @@ -37,24 +40,31 @@ public <A extends Any> WritableChunk<A> expand(
}

final ObjectChunk<IntVector, A> typedSource = source.asObjectChunk();
final SizedIntChunk<A> resultWrapper = new SizedIntChunk<>();

int lenWritten = 0;
long totalSize = 0;
for (int i = 0; i < typedSource.size(); ++i) {
final IntVector row = typedSource.get(i);
totalSize += row == null ? 0 : row.size();
}
final WritableIntChunk<A> result = WritableIntChunk.makeWritableChunk(
LongSizedDataStructure.intSize("ExpansionKernel", totalSize));
result.setSize(0);

perElementLengthDest.setSize(source.size() + 1);
for (int i = 0; i < typedSource.size(); ++i) {
final IntVector row = typedSource.get(i);
final int len = row == null ? 0 : row.intSize("IntVectorExpansionKernel");
perElementLengthDest.set(i, lenWritten);
final WritableIntChunk<A> result = resultWrapper.ensureCapacityPreserve(lenWritten + len);
for (int j = 0; j < len; ++j) {
result.set(lenWritten + j, row.get(j));
perElementLengthDest.set(i, result.size());
if (row == null) {
continue;
}
final IntConsumer consumer = result::add;
try (final CloseablePrimitiveIteratorOfInt iter = row.iterator()) {
iter.forEachRemaining(consumer);
}
lenWritten += len;
result.setSize(lenWritten);
}
perElementLengthDest.set(typedSource.size(), lenWritten);
perElementLengthDest.set(typedSource.size(), result.size());

return resultWrapper.get();
return result;
}

@Override
Expand All @@ -81,15 +91,13 @@ public <A extends Any> WritableObjectChunk<Vector<?>, A> contract(

int lenRead = 0;
for (int i = 0; i < itemsInBatch; ++i) {
final int ROW_LEN = perElementLengthDest.get(i + 1) - perElementLengthDest.get(i);
if (ROW_LEN == 0) {
final int rowLen = perElementLengthDest.get(i + 1) - perElementLengthDest.get(i);
if (rowLen == 0) {
result.set(outOffset + i, ZERO_LENGTH_VECTOR);
} else {
final int[] row = new int[ROW_LEN];
for (int j = 0; j < ROW_LEN; ++j) {
row[j] = typedSource.get(lenRead + j);
}
lenRead += ROW_LEN;
final int[] row = new int[rowLen];
typedSource.copyToArray(lenRead, row, 0, rowLen);
lenRead += rowLen;
result.set(outOffset + i, new IntVectorDirect(row));
}
}
Expand Down
Loading

0 comments on commit f6dc16c

Please sign in to comment.