Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Implemented ability for sorting and counting on solr side via 'direct index query" using indexQueryBuilder() #1182

Open
wants to merge 2 commits into
base: titan05
Choose a base branch
from
Open

Implemented ability for sorting and counting on solr side via 'direct index query" using indexQueryBuilder() #1182

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
188317b
Implemented ability for sorting on solr side via 'direct index query"…
dmytro-babichev Nov 12, 2015
9d6d20c
Implemented ability for counting vertices on solr side via 'direct in…
dmytro-babichev Nov 12, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 33 additions & 1 deletion titan-core/src/main/java/com/thinkaurelius/titan/graphdb/database/IndexSerializer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@

public class IndexSerializer {

public static final String ORDER_BY = "orderBy";

private static final Logger log = LoggerFactory.getLogger(IndexSerializer.class);

private static final int DEFAULT_OBJECT_BYTELEN = 30;
Expand Down Expand Up @@ -646,7 +648,11 @@ public Iterable<RawQuery.Result> executeQuery(IndexQueryBuilder query, final Ele
String queryStr = qB.toString();
if (replacements<=0) log.warn("Could not convert given {} index query: [{}]",resultType, query.getQuery());
log.info("Converted query string with {} replacements: [{}] => [{}]",replacements,query.getQuery(),queryStr);
RawQuery rawQuery=new RawQuery(index.getStoreName(),queryStr,query.getParameters());

// I need to map titan properties to solr properties in ORDER_BY parameters for sorting on the solr side
Parameter[] mappedParameters = mapQueryParameters(transaction, index, query.getParameters());

RawQuery rawQuery = new RawQuery(index.getStoreName(),queryStr,mappedParameters);
if (query.hasLimit()) rawQuery.setLimit(query.getLimit());
rawQuery.setOffset(query.getOffset());
return Iterables.transform(backendTx.rawQuery(index.getBackingIndexName(), rawQuery), new Function<RawQuery.Result<String>, RawQuery.Result>() {
Expand All @@ -658,6 +664,32 @@ public RawQuery.Result apply(@Nullable RawQuery.Result<String> result) {
});
}

/**
* Method maps 'orderBy' parameters like Parameter{"orderBy", Parameter{"name", "asc"}} to Parameter{"orderBy", Parameter{"ch5_t", "asc"}}
* @param transaction
* @param index
* @param parameters array of parameters passed via indexQueryBuilder()
* @return array of mapped 'orderBy' parameters and other parameters w/o changes
*/
private Parameter[] mapQueryParameters(StandardTitanTx transaction, MixedIndexType index, Parameter[] parameters) {
Parameter[] mappedParameters = new Parameter[parameters.length];
for (int i = 0; i < parameters.length; i++) {
Parameter parameter = parameters[i];
String parameterKey = parameter.getKey();
if (parameterKey.equals(ORDER_BY)) {
Parameter<Order> orderClause = (Parameter<Order>) parameter.getValue();
String titanProperty = orderClause.getKey();
PropertyKey propertyKey = transaction.getPropertyKey(titanProperty);
String mappedTitanProperty = key2Field(index, propertyKey);
mappedParameters[i] = Parameter.of(parameterKey, Parameter.of(mappedTitanProperty, orderClause.getValue()));
} else {
mappedParameters[i] = parameter; // just copy other parameters w/o changes
}
}
log.info("Mapped parameters {} to {}", Arrays.toString(parameters), Arrays.toString(mappedParameters));
return mappedParameters;
}


/* ################################################
Utility Functions
Expand Down
23 changes: 23 additions & 0 deletions titan-core/src/main/java/com/thinkaurelius/titan/graphdb/query/graph/IndexQueryBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import com.thinkaurelius.titan.core.TitanElement;
import com.thinkaurelius.titan.core.TitanIndexQuery;
import com.thinkaurelius.titan.core.TitanProperty;
import com.thinkaurelius.titan.core.Order;
import com.thinkaurelius.titan.diskstorage.indexing.RawQuery;
import com.thinkaurelius.titan.graphdb.database.IndexSerializer;
import com.thinkaurelius.titan.graphdb.internal.ElementCategory;
Expand Down Expand Up @@ -175,6 +176,12 @@ public IndexQueryBuilder addParameters(Parameter... paras) {
return this;
}

public IndexQueryBuilder orderBy(String field, String order) {
Parameter orderByClause = Parameter.of("orderBy", Parameter.of(field, order.equals("asc") ? Order.ASC : Order.DESC));
addParameter(orderByClause);
return this;
}

private Iterable<Result<TitanElement>> execute(ElementCategory resultType) {
Preconditions.checkNotNull(indexName);
Preconditions.checkNotNull(query);
Expand All @@ -196,6 +203,22 @@ public boolean apply(@Nullable Result<TitanElement> r) {
});
}

// We use this method to calculate total number of vertices, which satisfy the query.
public long executeCount() {
addParameter(Parameter.of("count", true)); // it is required to set parameter to influence on SolrIndex behavior
setPrefixInternal(VERTEX_PREFIX);
Preconditions.checkNotNull(indexName);
Preconditions.checkNotNull(query);
if (tx.hasModifications())
log.warn("Modifications in this transaction might not be accurately reflected in this index query: {}",
query);
Iterable<RawQuery.Result> results = serializer.executeQuery(this, ElementCategory.VERTEX, tx.getTxHandle(), tx);
for (RawQuery.Result result : results) {
return (long) result.getScore(); // count is set as score of result
}
return -1;
}

@Override
public Iterable<Result<Vertex>> vertices() {
setPrefixInternal(VERTEX_PREFIX);
Expand Down
45 changes: 44 additions & 1 deletion titan-solr/src/main/java/com/thinkaurelius/titan/diskstorage/solr/SolrIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import com.thinkaurelius.titan.core.TitanElement;
import com.thinkaurelius.titan.core.attribute.*;
import com.thinkaurelius.titan.core.schema.Mapping;
import com.thinkaurelius.titan.core.schema.Parameter;
import com.thinkaurelius.titan.diskstorage.*;
import com.thinkaurelius.titan.diskstorage.configuration.ConfigNamespace;
import com.thinkaurelius.titan.diskstorage.configuration.ConfigOption;
Expand Down Expand Up @@ -58,6 +59,9 @@
@PreInitializeConfigOptions
public class SolrIndex implements IndexProvider {

public static final String ORDER_BY = "orderBy";
public static final String COUNT = "count";

private static final Logger logger = LoggerFactory.getLogger(SolrIndex.class);

private static final String COLLECTION_PARAM = "collection";
Expand Down Expand Up @@ -452,17 +456,30 @@ public Iterable<RawQuery.Result<String>> query(RawQuery query, KeyInformation.In
List<RawQuery.Result<String>> result;
String collection = query.getStore();
String keyIdField = getKeyFieldId(collection);
List<SolrQuery.SortClause> solrSortClauses = getSolrSortClauses(query);
boolean isCountQuery = isCountQuery(query);

SolrQuery solrQuery = newQuery(collection, query.getQuery())
.addField(keyIdField)
.setIncludeScore(true)
.setStart(query.getOffset())
.setRows(query.hasLimit() ? query.getLimit() : maxResults);
.setRows(isCountQuery ? 0 : (query.hasLimit() ? query.getLimit() : maxResults)); // we set limit 0 for counting query

for (SolrQuery.SortClause solrSortClause : solrSortClauses) {
solrQuery.addSort(solrSortClause);
}

try {
QueryResponse response = solrServer.query(solrQuery);
if (logger.isDebugEnabled())
logger.debug("Executed query [{}] in {} ms", query.getQuery(), response.getElapsedTime());

if (isCountQuery) {
result = new ArrayList<RawQuery.Result<String>>(1);
result.add(new RawQuery.Result<String>(COUNT, response.getResults().getNumFound())); // return counts as score of a RawQuery.Result to not break an existing logic
return result;
}

int totalHits = response.getResults().size();
if (!query.hasLimit() && totalHits >= maxResults) {
logger.warn("Query result set truncated to first [{}] elements for query: {}", maxResults, query);
Expand All @@ -487,6 +504,32 @@ private static String escapeValue(Object value) {
return ClientUtils.escapeQueryChars(value.toString());
}

private boolean isCountQuery(RawQuery query) {
if (query.getParameters() != null) {
for (Parameter parameter : query.getParameters()) {
if (parameter.getKey().equals(COUNT)) {
return true;
}
}
}
return false;
}

private static List<SolrQuery.SortClause> getSolrSortClauses(RawQuery query) {
List<SolrQuery.SortClause> solrSortClauses = new ArrayList<SolrQuery.SortClause>();
if (query.getParameters() != null) {
for (Parameter parameter : query.getParameters()) {
if (parameter.getKey().equals(ORDER_BY)) {
Parameter<Order> sortClause = (Parameter<Order>) parameter.getValue();
solrSortClauses.add(SolrQuery.SortClause.create(sortClause.getKey(),
sortClause.getValue() == Order.ASC
? SolrQuery.ORDER.asc : SolrQuery.ORDER.desc));
}
}
}
return solrSortClauses;
}

public String buildQueryFilter(Condition<TitanElement> condition, KeyInformation.StoreRetriever informations) {
if (condition instanceof PredicateCondition) {
PredicateCondition<String, TitanElement> atom = (PredicateCondition<String, TitanElement>) condition;
Expand Down