Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Provide query library functions for computing differences and sort order #5151

Merged
merged 12 commits into from
Mar 19, 2024
Merged

Provide query library functions for computing differences and sort order #5151

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
9351d55
New query lang function to compute the difference in array values.
chipkent Feb 13, 2024
be37e0b
Updated docs.
chipkent Feb 13, 2024
0d68bc4
Added query library functions to return the sort order of arrays.
chipkent Feb 13, 2024
86b2078
Addressing review comments
chipkent Feb 26, 2024
86afa9e
Added comparator functions.
chipkent Feb 28, 2024
9df1dee
Fixed the new comparators.
chipkent Feb 28, 2024
2e35901
Porting the new comparators to sort.
chipkent Feb 28, 2024
7b6edd6
Merge branch '5101_3413_diff_rank' of https://github.com/chipkent/dee…
chipkent Feb 28, 2024
d186a80
Sorts use the new comparators.
chipkent Feb 28, 2024
04f1b5b
Addressing review comments.
chipkent Mar 18, 2024
e5ab43c
Addressing review comments.
chipkent Mar 18, 2024
14f7fdb
Fix compile bug.
chipkent Mar 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 67 additions & 0 deletions engine/function/src/templates/Numeric.ftl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1588,6 +1588,73 @@ public class Numeric {
return product(new ${pt.vectorDirect}(values));
}

/**
* Returns the difference between elements in the input vector.
chipkent marked this conversation as resolved.
Show resolved Hide resolved
* A stride of 2 returns v(i)=e(i+2)-e(i), where v(i) is the ith computed value e(i) is the ith input value.
chipkent marked this conversation as resolved.
Show resolved Hide resolved
* A stride of -2 returns v(i)=e(i-2)-e(i), where v(i) is the ith computed value e(i) is the ith input value.
* The result is the same length as the input vector.
chipkent marked this conversation as resolved.
Show resolved Hide resolved
* Differences off the end of the input vector are null.
chipkent marked this conversation as resolved.
Show resolved Hide resolved
*
* @param stride number of elements to skip between consecutive elements.
chipkent marked this conversation as resolved.
Show resolved Hide resolved
* @param values input vector.
* @return difference between elements.
chipkent marked this conversation as resolved.
Show resolved Hide resolved
*/
public static ${pt.primitive}[] diff(int stride, ${pt.boxed}[] values) {
return diff(stride, unbox(values));
}

/**
* Returns the difference between elements in the input vector.
chipkent marked this conversation as resolved.
Show resolved Hide resolved
* A stride of 2 returns v(i)=e(i+2)-e(i), where v(i) is the ith computed value e(i) is the ith input value.
* A stride of -2 returns v(i)=e(i-2)-e(i), where v(i) is the ith computed value e(i) is the ith input value.
* The result is the same length as the input vector.
* Differences off the end of the input vector are null.
*
* @param stride number of elements to skip between consecutive elements.
* @param values input vector.
* @return difference between elements.
*/
public static ${pt.primitive}[] diff(int stride, ${pt.primitive}... values) {
return diff(stride, new ${pt.vectorDirect}(values));
}

/**
* Returns the difference between elements in the input vector.
chipkent marked this conversation as resolved.
Show resolved Hide resolved
* A stride of 2 returns v(i)=e(i+2)-e(i), where v(i) is the ith computed value e(i) is the ith input value.
* A stride of -2 returns v(i)=e(i-2)-e(i), where v(i) is the ith computed value e(i) is the ith input value.
* The result is the same length as the input vector.
* Differences off the end of the input vector are null.
*
* @param stride number of elements to skip between consecutive elements.
* @param values input vector.
* @return difference between elements.
*/
public static ${pt.primitive}[] diff(int stride, ${pt.vector} values) {
if (values == null) {
return null;
}

if (values.isEmpty()) {
return new ${pt.primitive}[0];
}

final int n = values.intSize("diff");
${pt.primitive}[] result = new ${pt.primitive}[n];

for(int i = 0; i < n; i++) {
chipkent marked this conversation as resolved.
Show resolved Hide resolved
${pt.primitive} v1 = values.get(i);
${pt.primitive} v2 = values.get(i+stride);
chipkent marked this conversation as resolved.
Show resolved Hide resolved

if (isNull(v1) || isNull(v2)) {
result[i] = ${pt.null};
} else {
result[i] = (${pt.primitive})(v2 - v1);
}
}

return result;
}

/**
* Returns the cumulative minimum. Null values are excluded.
*
Expand Down
225 changes: 225 additions & 0 deletions engine/function/src/templates/Sort.ftl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import org.apache.commons.lang3.ArrayUtils;

import java.util.Arrays;
import java.util.Comparator;
import java.util.stream.IntStream;

import static io.deephaven.util.QueryConstants.*;
import static io.deephaven.function.Basic.isNull;
Expand Down Expand Up @@ -81,6 +82,64 @@ public class Sort {
return sortObj(values, new NullNaNAwareComparator<>());
}

/**
* Returns the indices of values sorted from smallest to largest.
*
* @param values values.
* @param comparator value comparator.
* @return sorted indices.
*/
static public <T extends Comparable<? super T>> int[] sortIndexObj(final ObjectVector<T> values, final Comparator<T> comparator) {
if (values == null) {
return null;
}
if (values.isEmpty()) {
return new int[]{};
chipkent marked this conversation as resolved.
Show resolved Hide resolved
}

return IntStream.range(0, values.intSize("sortIndex"))
.boxed().sorted((i, j) -> comparator.compare(values.get(i),values.get(j)) )
chipkent marked this conversation as resolved.
Show resolved Hide resolved
.mapToInt(ele -> ele).toArray();
}

/**
* Returns the indices of values sorted from smallest to largest.
*
* @param values values.
* @return sorted indices.
*/
static public <T extends Comparable<? super T>> int[] sortIndexObj(final ObjectVector<T> values) {
return sortIndexObj(values, new NullNaNAwareComparator<>());
}

/**
* Returns the indices of values sorted from smallest to largest.
*
* @param values values.
* @param comparator value comparator.
* @return sorted indices.
*/
static public <T extends Comparable<? super T>> int[] sortIndexObj(final T[] values, final Comparator<T> comparator) {
if (values == null) {
return null;
}

return IntStream.range(0, values.length)
.boxed().sorted((i, j) -> comparator.compare(values[i],values[j]) )
chipkent marked this conversation as resolved.
Show resolved Hide resolved
.mapToInt(ele -> ele).toArray();
}

/**
* Returns the indices of values sorted from smallest to largest.
*
* @param values values.
* @return sorted indices.
*/
@SafeVarargs
static public <T extends Comparable<? super T>> int[] sortIndexObj(final T... values) {
return sortIndexObj(values, new NullNaNAwareComparator<>());
}

/**
* Returns sorted values from largest to smallest.
*
Expand Down Expand Up @@ -138,6 +197,63 @@ public class Sort {
return sortDescendingObj(values, new NullNaNAwareComparator<>());
}

/**
* Returns the indices of values sorted from largest to smallest.
*
* @param values values.
* @param comparator value comparator.
* @return sorted indices.
*/
static public <T extends Comparable<? super T>> int[] sortDescendingIndexObj(final ObjectVector<T> values, final Comparator<T> comparator) {
if (values == null) {
return null;
}

if (values.isEmpty()) {
return new int[]{};
chipkent marked this conversation as resolved.
Show resolved Hide resolved
}

return IntStream.range(0, values.intSize("sortIndex"))
.boxed().sorted((i, j) -> -comparator.compare(values.get(i),values.get(j)) )
chipkent marked this conversation as resolved.
Show resolved Hide resolved
.mapToInt(ele -> ele).toArray();
}

/**
* Returns the indices of values sorted from largest to smallest.
*
* @param values values.
* @return sorted indices.
*/
static public <T extends Comparable<? super T>> int[] sortDescendingIndexObj(final ObjectVector<T> values) {
return sortDescendingIndexObj(values, new NullNaNAwareComparator<>());
}

/**
* Returns the indices of values sorted from largest to smallest.
*
* @param values values.
* @param comparator value comparator.
* @return sorted indices.
*/
static public <T extends Comparable<? super T>> int[] sortDescendingIndexObj(final T[] values, final Comparator<T> comparator) {
if (values == null) {
return null;
}

return sortDescendingIndexObj(new ObjectVectorDirect<>(values), comparator);
}

/**
* Returns the indices of values sorted from largest to smallest.
*
* @param values values.
* @return sorted indices.
*/
@SafeVarargs
static public <T extends Comparable<? super T>> int[] sortDescendingIndexObj(final T... values) {
return sortDescendingIndexObj(values, new NullNaNAwareComparator<>());
}

<#list primitiveTypes as pt>
<#if pt.valueType.isNumber >

Expand Down Expand Up @@ -202,6 +318,63 @@ public class Sort {
return vs;
}

/**
* Returns the indices of values sorted from smallest to largest.
*
* @param values values.
* @return sorted indices.
*/
public static int[] sortIndex(final ${pt.vector} values) {
chipkent marked this conversation as resolved.
Show resolved Hide resolved
if (values == null) {
return null;
}

if (values.isEmpty()) {
return new int[]{};
chipkent marked this conversation as resolved.
Show resolved Hide resolved
}

return IntStream.range(0, values.intSize("sortIndex"))
.boxed().sorted((i, j) -> ${pt.boxed}.compare(values.get(i),values.get(j)) )
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Two issues:

  1. add one space, delete one: compare(values.get(i), values.get(j)))
  2. Here and everywhere in this file I have a concern.

the concern is that when applied to Float (or Double) this template will use Float.compare (or Double.compare) and in particular will not use a NaN-aware comparison.

this would make the sort inconsistent with the one used for object arrays, which does use the NullNanAwareComparator

The same NaN issue comes up when we use Arrays.sort() in this file on float[] and double[]

so I guess the first question is: do we care about NaN

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Resolved, but it came with boxing and copies.

.mapToInt(ele -> ele).toArray();
}

/**
* Returns the indices of values sorted from smallest to largest.
*
* @param values values.
* @return sorted indices.
*/
public static int[] sortIndex(final ${pt.primitive}... values) {
if (values == null) {
return null;
}

return sortIndex(new ${pt.vectorDirect}(values));
}

/**
* Returns the indices of values sorted from smallest to largest.
*
* @param values values.
* @return sorted indices.
*/
public static int[] sortIndex(final ${pt.boxed}[] values) {
if (values == null) {
return null;
}

if (values.length == 0) {
return new int[]{};
chipkent marked this conversation as resolved.
Show resolved Hide resolved
}

final ${pt.primitive}[] vs = new ${pt.primitive}[values.length];
for (int i = 0; i < values.length; i++) {
vs[i] = isNull(values[i]) ? ${pt.null} : values[i];
}

return sortIndex(new ${pt.vectorDirect}(vs));
}

/**
* Returns sorted values from largest to smallest.
*
Expand Down Expand Up @@ -254,6 +427,58 @@ public class Sort {
return result;
}

/**
* Returns the indices of values sorted from largest to smallest.
*
* @param values values.
* @return sorted indices.
*/
public static int[] sortDescendingIndex(final ${pt.vector} values) {
if (values == null) {
return null;
}

if (values.isEmpty()) {
return new int[]{};
chipkent marked this conversation as resolved.
Show resolved Hide resolved
}

return IntStream.range(0, values.intSize("sortIndex"))
.boxed().sorted((i, j) -> -${pt.boxed}.compare(values.get(i),values.get(j)) )
chipkent marked this conversation as resolved.
Show resolved Hide resolved
.mapToInt(ele -> ele).toArray();
}

/**
* Returns the indices of values sorted from largest to smallest.
*
* @param values values.
* @return sorted indices.
*/
public static int[] sortDescendingIndex(final ${pt.primitive}... values) {
if (values == null) {
return null;
}

return sortDescendingIndex(new ${pt.vectorDirect}(values));
}

/**
* Returns the indices of values sorted from largest to smallest.
*
* @param values values.
* @return sorted indices.
*/
public static int[] sortDescendingIndex(final ${pt.boxed}[] values) {
if (values == null) {
return null;
}

final ${pt.primitive}[] vs = new ${pt.primitive}[values.length];
for (int i = 0; i < values.length; i++) {
vs[i] = isNull(values[i]) ? ${pt.null} : values[i];
}

return sortDescendingIndex(new ${pt.vectorDirect}(vs));
}

</#if>
</#list>
Expand Down
17 changes: 17 additions & 0 deletions engine/function/src/templates/TestNumeric.ftl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -680,6 +680,23 @@ public class TestNumeric extends BaseArrayTestCase {
// }
// }

public void test${pt.boxed}Diff() {
assertEquals(new ${pt.primitive}[]{1, 2, 4, 8, ${pt.null}}, diff(1, new ${pt.primitive}[]{1, 2, 4, 8, 16}));
assertEquals(new ${pt.primitive}[]{3, 6, 12, ${pt.null}, ${pt.null}}, diff(2, new ${pt.primitive}[]{1, 2, 4, 8, 16}));
assertEquals(new ${pt.primitive}[]{${pt.null}, -1, -2, -4, -8}, diff(-1, new ${pt.primitive}[]{1, 2, 4, 8, 16}));
assertEquals(new ${pt.primitive}[]{${pt.null}, ${pt.null}, -3, -6, -12}, diff(-2, new ${pt.primitive}[]{1, 2, 4, 8, 16}));

assertEquals(new ${pt.primitive}[]{1, 2, 4, 8, ${pt.null}}, diff(1, new ${pt.boxed}[]{(${pt.primitive})1, (${pt.primitive})2, (${pt.primitive})4, (${pt.primitive})8, (${pt.primitive})16}));
assertEquals(new ${pt.primitive}[]{3, 6, 12, ${pt.null}, ${pt.null}}, diff(2, new ${pt.boxed}[]{(${pt.primitive})1, (${pt.primitive})2, (${pt.primitive})4, (${pt.primitive})8, (${pt.primitive})16}));
assertEquals(new ${pt.primitive}[]{${pt.null}, -1, -2, -4, -8}, diff(-1, new ${pt.boxed}[]{(${pt.primitive})1, (${pt.primitive})2, (${pt.primitive})4, (${pt.primitive})8, (${pt.primitive})16}));
assertEquals(new ${pt.primitive}[]{${pt.null}, ${pt.null}, -3, -6, -12}, diff(-2, new ${pt.boxed}[]{(${pt.primitive})1, (${pt.primitive})2, (${pt.primitive})4, (${pt.primitive})8, (${pt.primitive})16}));

assertEquals(new ${pt.primitive}[]{1, 2, 4, 8, ${pt.null}}, diff(1, new ${pt.vectorDirect}(new ${pt.primitive}[]{1, 2, 4, 8, 16})));
assertEquals(new ${pt.primitive}[]{3, 6, 12, ${pt.null}, ${pt.null}}, diff(2, new ${pt.vectorDirect}(new ${pt.primitive}[]{1, 2, 4, 8, 16})));
assertEquals(new ${pt.primitive}[]{${pt.null}, -1, -2, -4, -8}, diff(-1, new ${pt.vectorDirect}(new ${pt.primitive}[]{1, 2, 4, 8, 16})));
assertEquals(new ${pt.primitive}[]{${pt.null}, ${pt.null}, -3, -6, -12}, diff(-2, new ${pt.vectorDirect}(new ${pt.primitive}[]{1, 2, 4, 8, 16})));
}

public void test${pt.boxed}CumMinArray() {
assertEquals(new ${pt.primitive}[]{1, 1, 1, 1, 1}, cummin(new ${pt.primitive}[]{1, 2, 3, 4, 5}));
assertEquals(new ${pt.primitive}[]{5, 4, 3, 2, 1}, cummin(new ${pt.primitive}[]{5, 4, 3, 2, 1}));
Expand Down
Loading
Loading