Skip to content

Commit

Permalink
adding half hour and quarter hour calendar intervals
Browse files Browse the repository at this point in the history 
Signed-off-by: Bharathwaj G <[email protected]>
  • Loading branch information
@bharath-techie
bharath-techie committed Aug 19, 2024
1 parent 3f82b5a commit c2269e3
Show file tree
Hide file tree
Showing 12 changed files with 605 additions and 208 deletions.
125 changes: 117 additions & 8 deletions server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@
import org.opensearch.index.compositeindex.datacube.MetricStat;
import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration;
import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings;
import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitAdapter;
import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding;
import org.opensearch.index.compositeindex.datacube.startree.utils.date.ExtendedDateTimeUnit;
import org.opensearch.indices.IndicesService;
import org.opensearch.test.OpenSearchIntegTestCase;
import org.junit.After;
Expand Down Expand Up @@ -90,6 +93,61 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool
}
}

private static XContentBuilder createDateTestMapping() {
try {
return jsonBuilder().startObject()
.startObject("composite")
.startObject("startree-1")
.field("type", "star_tree")
.startObject("config")
.startObject("date_dimension")
.field("name", "timestamp")
.startArray("calendar_intervals")
.value("day")
.value("quarter-hour")
.value("half-hour")
.endArray()
.endObject()
.startArray("ordered_dimensions")
.startObject()
.field("name", "numeric_dv")
.endObject()
.endArray()
.startArray("metrics")
.startObject()
.field("name", "numeric_dv")
.endObject()
.endArray()
.endObject()
.endObject()
.endObject()
.startObject("properties")
.startObject("timestamp")
.field("type", "date")
.endObject()
.startObject("numeric_dv")
.field("type", "integer")
.field("doc_values", true)
.endObject()
.startObject("numeric")
.field("type", "integer")
.field("doc_values", false)
.endObject()
.startObject("keyword_dv")
.field("type", "keyword")
.field("doc_values", true)
.endObject()
.startObject("keyword")
.field("type", "keyword")
.field("doc_values", false)
.endObject()
.endObject()
.endObject();
} catch (IOException e) {
throw new IllegalStateException(e);
}
}

private static XContentBuilder createMaxDimTestMapping() {
try {
return jsonBuilder().startObject()
Expand All @@ -102,6 +160,7 @@ private static XContentBuilder createMaxDimTestMapping() {
.startArray("calendar_intervals")
.value("day")
.value("month")
.value("half-hour")
.endArray()
.endObject()
.startArray("ordered_dimensions")
Expand Down Expand Up @@ -258,11 +317,57 @@ public void testValidCompositeIndex() {
assertEquals("timestamp", starTreeFieldType.getDimensions().get(0).getField());
assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension);
DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0);
List<Rounding.DateTimeUnit> expectedTimeUnits = Arrays.asList(
Rounding.DateTimeUnit.MINUTES_OF_HOUR,
Rounding.DateTimeUnit.HOUR_OF_DAY
List<DateTimeUnitRounding> expectedTimeUnits = Arrays.asList(
new DateTimeUnitAdapter(Rounding.DateTimeUnit.MINUTES_OF_HOUR),
ExtendedDateTimeUnit.HALF_HOUR_OF_DAY
);
assertEquals(expectedTimeUnits, dateDim.getIntervals());
for (int i = 0; i < dateDim.getSortedCalendarIntervals().size(); i++) {
assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getSortedCalendarIntervals().get(i).shortName());
}
assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField());
assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField());
List<MetricStat> expectedMetrics = Arrays.asList(
MetricStat.AVG,
MetricStat.COUNT,
MetricStat.SUM,
MetricStat.MAX,
MetricStat.MIN
);
assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs());
assertEquals(
StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP,
starTreeFieldType.getStarTreeConfig().getBuildMode()
);
assertEquals(Collections.emptySet(), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims());
}
}
}
}

public void testValidCompositeIndexWithDates() {
prepareCreate(TEST_INDEX).setMapping(createDateTestMapping()).get();
Iterable<IndicesService> dataNodeInstances = internalCluster().getDataNodeInstances(IndicesService.class);
for (IndicesService service : dataNodeInstances) {
final Index index = resolveIndex("test");
if (service.hasIndex(index)) {
IndexService indexService = service.indexService(index);
Set<CompositeMappedFieldType> fts = indexService.mapperService().getCompositeFieldTypes();

for (CompositeMappedFieldType ft : fts) {
assertTrue(ft instanceof StarTreeMapper.StarTreeFieldType);
StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) ft;
assertEquals("timestamp", starTreeFieldType.getDimensions().get(0).getField());
assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension);
DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0);
List<DateTimeUnitRounding> expectedTimeUnits = Arrays.asList(
ExtendedDateTimeUnit.QUARTER_HOUR_OF_DAY,
ExtendedDateTimeUnit.HALF_HOUR_OF_DAY,
new DateTimeUnitAdapter(Rounding.DateTimeUnit.DAY_OF_MONTH)
);
for (int i = 0; i < dateDim.getIntervals().size(); i++) {
assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getSortedCalendarIntervals().get(i).shortName());
}
assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField());
assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField());
List<MetricStat> expectedMetrics = Arrays.asList(
Expand Down Expand Up @@ -342,11 +447,14 @@ public void testUpdateIndexWhenMappingIsSame() {
assertEquals("timestamp", starTreeFieldType.getDimensions().get(0).getField());
assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension);
DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0);
List<Rounding.DateTimeUnit> expectedTimeUnits = Arrays.asList(
Rounding.DateTimeUnit.MINUTES_OF_HOUR,
Rounding.DateTimeUnit.HOUR_OF_DAY
List<DateTimeUnitRounding> expectedTimeUnits = Arrays.asList(
new DateTimeUnitAdapter(Rounding.DateTimeUnit.MINUTES_OF_HOUR),
ExtendedDateTimeUnit.HALF_HOUR_OF_DAY
);
assertEquals(expectedTimeUnits, dateDim.getIntervals());
for (int i = 0; i < expectedTimeUnits.size(); i++) {
assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getIntervals().get(i).shortName());
}

assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField());
assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField());
List<MetricStat> expectedMetrics = Arrays.asList(
Expand Down Expand Up @@ -383,6 +491,7 @@ public void testMaxDimsCompositeIndex() {
MapperParsingException ex = expectThrows(
MapperParsingException.class,
() -> prepareCreate(TEST_INDEX).setMapping(createMaxDimTestMapping())
// Date dimension is considered as one dimension regardless of number of actual calendar intervals
.setSettings(Settings.builder().put(StarTreeIndexSettings.STAR_TREE_MAX_DIMENSIONS_SETTING.getKey(), 2))
.get()
);
Expand Down
33 changes: 0 additions & 33 deletions server/src/main/java/org/opensearch/common/Rounding.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,15 +62,11 @@
import java.time.temporal.TemporalQueries;
import java.time.zone.ZoneOffsetTransition;
import java.time.zone.ZoneRules;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.OptionalLong;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

/**
* A strategy for rounding milliseconds since epoch.
Expand Down Expand Up @@ -264,35 +260,6 @@ public static DateTimeUnit resolve(byte id) {
}
}

/**
* DateTimeUnit Comparator which tracks dateTimeUnits from second unit to year unit
*/
public static class DateTimeUnitComparator implements Comparator<DateTimeUnit> {
public static final Map<DateTimeUnit, Integer> ORDERED_DATE_TIME_UNIT = new HashMap<>();

static {
ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.SECOND_OF_MINUTE, 1);
ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.MINUTES_OF_HOUR, 2);
ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.HOUR_OF_DAY, 3);
ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.DAY_OF_MONTH, 4);
ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.WEEK_OF_WEEKYEAR, 5);
ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.MONTH_OF_YEAR, 6);
ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.QUARTER_OF_YEAR, 7);
ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.YEAR_OF_CENTURY, 8);
}

@Override
public int compare(DateTimeUnit unit1, DateTimeUnit unit2) {
return Integer.compare(ORDERED_DATE_TIME_UNIT.get(unit1), ORDERED_DATE_TIME_UNIT.get(unit2));
}
}

public static List<DateTimeUnit> getSortedDateTimeUnits(List<DateTimeUnit> dateTimeUnits) {
return dateTimeUnits.stream()
.sorted(Comparator.comparingInt(DateTimeUnitComparator.ORDERED_DATE_TIME_UNIT::get))
.collect(Collectors.toList());
}

public abstract void innerWriteTo(StreamOutput out) throws IOException;

@Override
Expand Down
59 changes: 50 additions & 9 deletions server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,19 @@
import org.opensearch.common.annotation.ExperimentalApi;
import org.opensearch.common.time.DateUtils;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding;
import org.opensearch.index.compositeindex.datacube.startree.utils.date.ExtendedDateTimeUnit;
import org.opensearch.index.mapper.CompositeDataCubeFieldType;
import org.opensearch.index.mapper.DateFieldMapper;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;

/**
* Date dimension class
Expand All @@ -27,30 +33,30 @@
*/
@ExperimentalApi
public class DateDimension implements Dimension {
private final List<Rounding.DateTimeUnit> calendarIntervals;
private final List<DateTimeUnitRounding> calendarIntervals;
public static final String CALENDAR_INTERVALS = "calendar_intervals";
public static final String DATE = "date";
private final String field;
private final List<Rounding.DateTimeUnit> sortedCalendarIntervals;
private final List<DateTimeUnitRounding> sortedCalendarIntervals;
private final DateFieldMapper.Resolution resolution;

public DateDimension(String field, List<Rounding.DateTimeUnit> calendarIntervals, DateFieldMapper.Resolution resolution) {
public DateDimension(String field, List<DateTimeUnitRounding> calendarIntervals, DateFieldMapper.Resolution resolution) {
this.field = field;
this.calendarIntervals = calendarIntervals;
// Sort from the lowest unit to the highest unit
this.sortedCalendarIntervals = Rounding.getSortedDateTimeUnits(calendarIntervals);
this.sortedCalendarIntervals = getSortedDateTimeUnits(calendarIntervals);
if (resolution == null) {
this.resolution = DateFieldMapper.Resolution.MILLISECONDS;
} else {
this.resolution = resolution;
}
}

public List<Rounding.DateTimeUnit> getIntervals() {
public List<DateTimeUnitRounding> getIntervals() {
return calendarIntervals;
}

public List<Rounding.DateTimeUnit> getSortedCalendarIntervals() {
public List<DateTimeUnitRounding> getSortedCalendarIntervals() {
return sortedCalendarIntervals;
}

Expand All @@ -64,7 +70,7 @@ public List<Rounding.DateTimeUnit> getSortedCalendarIntervals() {
*/
@Override
public int setDimensionValues(final Long val, final Long[] dims, int index) {
for (Rounding.DateTimeUnit dateTimeUnit : sortedCalendarIntervals) {
for (DateTimeUnitRounding dateTimeUnit : sortedCalendarIntervals) {
if (val == null) {
dims[index++] = null;
continue;
Expand All @@ -88,7 +94,7 @@ private long maybeConvertNanosToMillis(long nanoSecondsSinceEpoch) {
@Override
public List<String> getDimensionFieldsNames() {
List<String> fields = new ArrayList<>(calendarIntervals.size());
for (Rounding.DateTimeUnit interval : sortedCalendarIntervals) {
for (DateTimeUnitRounding interval : sortedCalendarIntervals) {
// TODO : revisit this post file format changes
fields.add(field + "_" + interval.shortName());
}
Expand All @@ -101,7 +107,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.field(CompositeDataCubeFieldType.NAME, this.getField());
builder.field(CompositeDataCubeFieldType.TYPE, DATE);
builder.startArray(CALENDAR_INTERVALS);
for (Rounding.DateTimeUnit interval : calendarIntervals) {
for (DateTimeUnitRounding interval : calendarIntervals) {
builder.value(interval.shortName());
}
builder.endArray();
Expand Down Expand Up @@ -131,4 +137,39 @@ public String getField() {
public int getNumSubDimensions() {
return calendarIntervals.size();
}

/**
* DateTimeUnit Comparator which tracks dateTimeUnits in sorted order from second unit to year unit
*/
public static class DateTimeUnitComparator implements Comparator<DateTimeUnitRounding> {
public static final Map<String, Integer> ORDERED_DATE_TIME_UNIT = new HashMap<>();

static {
ORDERED_DATE_TIME_UNIT.put(Rounding.DateTimeUnit.SECOND_OF_MINUTE.shortName(), 1);
ORDERED_DATE_TIME_UNIT.put(Rounding.DateTimeUnit.MINUTES_OF_HOUR.shortName(), 2);
ORDERED_DATE_TIME_UNIT.put(ExtendedDateTimeUnit.QUARTER_HOUR_OF_DAY.shortName(), 3);
ORDERED_DATE_TIME_UNIT.put(ExtendedDateTimeUnit.HALF_HOUR_OF_DAY.shortName(), 4);
ORDERED_DATE_TIME_UNIT.put(Rounding.DateTimeUnit.HOUR_OF_DAY.shortName(), 5);
ORDERED_DATE_TIME_UNIT.put(Rounding.DateTimeUnit.DAY_OF_MONTH.shortName(), 6);
ORDERED_DATE_TIME_UNIT.put(Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR.shortName(), 7);
ORDERED_DATE_TIME_UNIT.put(Rounding.DateTimeUnit.MONTH_OF_YEAR.shortName(), 8);
ORDERED_DATE_TIME_UNIT.put(Rounding.DateTimeUnit.QUARTER_OF_YEAR.shortName(), 9);
ORDERED_DATE_TIME_UNIT.put(Rounding.DateTimeUnit.YEAR_OF_CENTURY.shortName(), 10);
}

@Override
public int compare(DateTimeUnitRounding unit1, DateTimeUnitRounding unit2) {
return Integer.compare(
ORDERED_DATE_TIME_UNIT.getOrDefault(unit1.shortName(), Integer.MAX_VALUE),
ORDERED_DATE_TIME_UNIT.getOrDefault(unit2.shortName(), Integer.MAX_VALUE)
);
}
}

/**
* Returns a sorted list of dateTimeUnits based on the DateTimeUnitComparator
*/
public static List<DateTimeUnitRounding> getSortedDateTimeUnits(List<DateTimeUnitRounding> dateTimeUnits) {
return dateTimeUnits.stream().sorted(new DateTimeUnitComparator()).collect(Collectors.toList());
}
}
4 changes: 2 additions & 2 deletions server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@

package org.opensearch.index.compositeindex.datacube;

import org.opensearch.common.Rounding;
import org.opensearch.common.annotation.ExperimentalApi;
import org.opensearch.common.xcontent.support.XContentMapValues;
import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings;
import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding;
import org.opensearch.index.mapper.DateFieldMapper;
import org.opensearch.index.mapper.Mapper;
import org.opensearch.index.mapper.NumberFieldMapper;
Expand Down Expand Up @@ -70,7 +70,7 @@ private static DateDimension parseAndCreateDateDimension(
Map<String, Object> dimensionMap,
Mapper.TypeParser.ParserContext c
) {
List<Rounding.DateTimeUnit> calendarIntervals = new ArrayList<>();
List<DateTimeUnitRounding> calendarIntervals = new ArrayList<>();
List<String> intervalStrings = XContentMapValues.extractRawValues(CALENDAR_INTERVALS, dimensionMap)
.stream()
.map(Object::toString)
Expand Down
Loading

0 comments on commit c2269e3

Please sign in to comment.