use of org.apache.lucene.search.grouping.SearchGroup in project lucene-solr by apache.
the class GroupConverter method fromMutable.
static Collection<SearchGroup<BytesRef>> fromMutable(SchemaField field, Collection<SearchGroup<MutableValue>> values) {
if (values == null) {
return null;
}
FieldType fieldType = field.getType();
List<SearchGroup<BytesRef>> result = new ArrayList<>(values.size());
for (SearchGroup<MutableValue> original : values) {
SearchGroup<BytesRef> converted = new SearchGroup<BytesRef>();
converted.sortValues = original.sortValues;
if (original.groupValue.exists) {
BytesRefBuilder binary = new BytesRefBuilder();
fieldType.readableToIndexed(original.groupValue.toString(), binary);
converted.groupValue = binary.get();
} else {
converted.groupValue = null;
}
result.add(converted);
}
return result;
}
use of org.apache.lucene.search.grouping.SearchGroup in project lucene-solr by apache.
the class GroupConverter method toMutable.
static Collection<SearchGroup<MutableValue>> toMutable(SchemaField field, Collection<SearchGroup<BytesRef>> values) {
FieldType fieldType = field.getType();
List<SearchGroup<MutableValue>> result = new ArrayList<>(values.size());
for (SearchGroup<BytesRef> original : values) {
SearchGroup<MutableValue> converted = new SearchGroup<MutableValue>();
// ?
converted.sortValues = original.sortValues;
NumberType type = fieldType.getNumberType();
final MutableValue v;
switch(type) {
case INTEGER:
MutableValueInt mutableInt = new MutableValueInt();
if (original.groupValue == null) {
mutableInt.value = 0;
mutableInt.exists = false;
} else {
mutableInt.value = (Integer) fieldType.toObject(field, original.groupValue);
}
v = mutableInt;
break;
case FLOAT:
MutableValueFloat mutableFloat = new MutableValueFloat();
if (original.groupValue == null) {
mutableFloat.value = 0;
mutableFloat.exists = false;
} else {
mutableFloat.value = (Float) fieldType.toObject(field, original.groupValue);
}
v = mutableFloat;
break;
case DOUBLE:
MutableValueDouble mutableDouble = new MutableValueDouble();
if (original.groupValue == null) {
mutableDouble.value = 0;
mutableDouble.exists = false;
} else {
mutableDouble.value = (Double) fieldType.toObject(field, original.groupValue);
}
v = mutableDouble;
break;
case LONG:
MutableValueLong mutableLong = new MutableValueLong();
if (original.groupValue == null) {
mutableLong.value = 0;
mutableLong.exists = false;
} else {
mutableLong.value = (Long) fieldType.toObject(field, original.groupValue);
}
v = mutableLong;
break;
case DATE:
MutableValueDate mutableDate = new MutableValueDate();
if (original.groupValue == null) {
mutableDate.value = 0;
mutableDate.exists = false;
} else {
mutableDate.value = ((Date) fieldType.toObject(field, original.groupValue)).getTime();
}
v = mutableDate;
break;
default:
throw new AssertionError();
}
converted.groupValue = v;
result.add(converted);
}
return result;
}
use of org.apache.lucene.search.grouping.SearchGroup in project lucene-solr by apache.
the class SearchGroupsResultTransformer method transform.
/**
* {@inheritDoc}
*/
@Override
public NamedList transform(List<Command> data) throws IOException {
final NamedList<NamedList> result = new NamedList<>(data.size());
for (Command command : data) {
final NamedList<Object> commandResult = new NamedList<>(2);
if (SearchGroupsFieldCommand.class.isInstance(command)) {
SearchGroupsFieldCommand fieldCommand = (SearchGroupsFieldCommand) command;
final SearchGroupsFieldCommandResult fieldCommandResult = fieldCommand.result();
final Collection<SearchGroup<BytesRef>> searchGroups = fieldCommandResult.getSearchGroups();
if (searchGroups != null) {
commandResult.add(TOP_GROUPS, serializeSearchGroup(searchGroups, fieldCommand));
}
final Integer groupedCount = fieldCommandResult.getGroupCount();
if (groupedCount != null) {
commandResult.add(GROUP_COUNT, groupedCount);
}
} else {
continue;
}
result.add(command.getKey(), commandResult);
}
return result;
}
use of org.apache.lucene.search.grouping.SearchGroup in project lucene-solr by apache.
the class QueryComponent method process.
/**
* Actually run the query
*/
@Override
public void process(ResponseBuilder rb) throws IOException {
LOG.debug("process: {}", rb.req.getParams());
SolrQueryRequest req = rb.req;
SolrParams params = req.getParams();
if (!params.getBool(COMPONENT_NAME, true)) {
return;
}
SolrIndexSearcher searcher = req.getSearcher();
StatsCache statsCache = req.getCore().getStatsCache();
int purpose = params.getInt(ShardParams.SHARDS_PURPOSE, ShardRequest.PURPOSE_GET_TOP_IDS);
if ((purpose & ShardRequest.PURPOSE_GET_TERM_STATS) != 0) {
statsCache.returnLocalStats(rb, searcher);
return;
}
// check if we need to update the local copy of global dfs
if ((purpose & ShardRequest.PURPOSE_SET_TERM_STATS) != 0) {
// retrieve from request and update local cache
statsCache.receiveGlobalStats(req);
}
SolrQueryResponse rsp = rb.rsp;
IndexSchema schema = searcher.getSchema();
// Optional: This could also be implemented by the top-level searcher sending
// a filter that lists the ids... that would be transparent to
// the request handler, but would be more expensive (and would preserve score
// too if desired).
String ids = params.get(ShardParams.IDS);
if (ids != null) {
SchemaField idField = schema.getUniqueKeyField();
List<String> idArr = StrUtils.splitSmart(ids, ",", true);
int[] luceneIds = new int[idArr.size()];
int docs = 0;
if (idField.getType().isPointField()) {
for (int i = 0; i < idArr.size(); i++) {
int id = searcher.search(idField.getType().getFieldQuery(null, idField, idArr.get(i)), 1).scoreDocs[0].doc;
if (id >= 0) {
luceneIds[docs++] = id;
}
}
} else {
for (int i = 0; i < idArr.size(); i++) {
int id = searcher.getFirstMatch(new Term(idField.getName(), idField.getType().toInternal(idArr.get(i))));
if (id >= 0)
luceneIds[docs++] = id;
}
}
DocListAndSet res = new DocListAndSet();
res.docList = new DocSlice(0, docs, luceneIds, null, docs, 0);
if (rb.isNeedDocSet()) {
// TODO: create a cache for this!
List<Query> queries = new ArrayList<>();
queries.add(rb.getQuery());
List<Query> filters = rb.getFilters();
if (filters != null)
queries.addAll(filters);
res.docSet = searcher.getDocSet(queries);
}
rb.setResults(res);
ResultContext ctx = new BasicResultContext(rb);
rsp.addResponse(ctx);
return;
}
// -1 as flag if not set.
long timeAllowed = params.getLong(CommonParams.TIME_ALLOWED, -1L);
if (null != rb.getCursorMark() && 0 < timeAllowed) {
// fundamentally incompatible
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Can not search using both " + CursorMarkParams.CURSOR_MARK_PARAM + " and " + CommonParams.TIME_ALLOWED);
}
QueryCommand cmd = rb.getQueryCommand();
cmd.setTimeAllowed(timeAllowed);
req.getContext().put(SolrIndexSearcher.STATS_SOURCE, statsCache.get(req));
QueryResult result = new QueryResult();
cmd.setSegmentTerminateEarly(params.getBool(CommonParams.SEGMENT_TERMINATE_EARLY, CommonParams.SEGMENT_TERMINATE_EARLY_DEFAULT));
if (cmd.getSegmentTerminateEarly()) {
result.setSegmentTerminatedEarly(Boolean.FALSE);
}
//
// grouping / field collapsing
//
GroupingSpecification groupingSpec = rb.getGroupingSpec();
if (groupingSpec != null) {
// not supported, silently ignore any segmentTerminateEarly flag
cmd.setSegmentTerminateEarly(false);
try {
boolean needScores = (cmd.getFlags() & SolrIndexSearcher.GET_SCORES) != 0;
if (params.getBool(GroupParams.GROUP_DISTRIBUTED_FIRST, false)) {
CommandHandler.Builder topsGroupsActionBuilder = new CommandHandler.Builder().setQueryCommand(cmd).setNeedDocSet(// Order matters here
false).setIncludeHitCount(true).setSearcher(searcher);
for (String field : groupingSpec.getFields()) {
topsGroupsActionBuilder.addCommandField(new SearchGroupsFieldCommand.Builder().setField(schema.getField(field)).setGroupSort(groupingSpec.getGroupSort()).setTopNGroups(cmd.getOffset() + cmd.getLen()).setIncludeGroupCount(groupingSpec.isIncludeGroupCount()).build());
}
CommandHandler commandHandler = topsGroupsActionBuilder.build();
commandHandler.execute();
SearchGroupsResultTransformer serializer = new SearchGroupsResultTransformer(searcher);
rsp.add("firstPhase", commandHandler.processResult(result, serializer));
rsp.add("totalHitCount", commandHandler.getTotalHitCount());
rb.setResult(result);
return;
} else if (params.getBool(GroupParams.GROUP_DISTRIBUTED_SECOND, false)) {
CommandHandler.Builder secondPhaseBuilder = new CommandHandler.Builder().setQueryCommand(cmd).setTruncateGroups(groupingSpec.isTruncateGroups() && groupingSpec.getFields().length > 0).setSearcher(searcher);
int docsToCollect = Grouping.getMax(groupingSpec.getWithinGroupOffset(), groupingSpec.getWithinGroupLimit(), searcher.maxDoc());
docsToCollect = Math.max(docsToCollect, 1);
for (String field : groupingSpec.getFields()) {
SchemaField schemaField = schema.getField(field);
String[] topGroupsParam = params.getParams(GroupParams.GROUP_DISTRIBUTED_TOPGROUPS_PREFIX + field);
if (topGroupsParam == null) {
topGroupsParam = new String[0];
}
List<SearchGroup<BytesRef>> topGroups = new ArrayList<>(topGroupsParam.length);
for (String topGroup : topGroupsParam) {
SearchGroup<BytesRef> searchGroup = new SearchGroup<>();
if (!topGroup.equals(TopGroupsShardRequestFactory.GROUP_NULL_VALUE)) {
BytesRefBuilder builder = new BytesRefBuilder();
schemaField.getType().readableToIndexed(topGroup, builder);
searchGroup.groupValue = builder.get();
}
topGroups.add(searchGroup);
}
secondPhaseBuilder.addCommandField(new TopGroupsFieldCommand.Builder().setField(schemaField).setGroupSort(groupingSpec.getGroupSort()).setSortWithinGroup(groupingSpec.getSortWithinGroup()).setFirstPhaseGroups(topGroups).setMaxDocPerGroup(docsToCollect).setNeedScores(needScores).setNeedMaxScore(needScores).build());
}
for (String query : groupingSpec.getQueries()) {
secondPhaseBuilder.addCommandField(new Builder().setDocsToCollect(docsToCollect).setSort(groupingSpec.getGroupSort()).setQuery(query, rb.req).setDocSet(searcher).build());
}
CommandHandler commandHandler = secondPhaseBuilder.build();
commandHandler.execute();
TopGroupsResultTransformer serializer = new TopGroupsResultTransformer(rb);
rsp.add("secondPhase", commandHandler.processResult(result, serializer));
rb.setResult(result);
return;
}
int maxDocsPercentageToCache = params.getInt(GroupParams.GROUP_CACHE_PERCENTAGE, 0);
boolean cacheSecondPassSearch = maxDocsPercentageToCache >= 1 && maxDocsPercentageToCache <= 100;
Grouping.TotalCount defaultTotalCount = groupingSpec.isIncludeGroupCount() ? Grouping.TotalCount.grouped : Grouping.TotalCount.ungrouped;
// this is normally from "rows"
int limitDefault = cmd.getLen();
Grouping grouping = new Grouping(searcher, result, cmd, cacheSecondPassSearch, maxDocsPercentageToCache, groupingSpec.isMain());
grouping.setGroupSort(groupingSpec.getGroupSort()).setWithinGroupSort(groupingSpec.getSortWithinGroup()).setDefaultFormat(groupingSpec.getResponseFormat()).setLimitDefault(limitDefault).setDefaultTotalCount(defaultTotalCount).setDocsPerGroupDefault(groupingSpec.getWithinGroupLimit()).setGroupOffsetDefault(groupingSpec.getWithinGroupOffset()).setGetGroupedDocSet(groupingSpec.isTruncateGroups());
if (groupingSpec.getFields() != null) {
for (String field : groupingSpec.getFields()) {
grouping.addFieldCommand(field, rb.req);
}
}
if (groupingSpec.getFunctions() != null) {
for (String groupByStr : groupingSpec.getFunctions()) {
grouping.addFunctionCommand(groupByStr, rb.req);
}
}
if (groupingSpec.getQueries() != null) {
for (String groupByStr : groupingSpec.getQueries()) {
grouping.addQueryCommand(groupByStr, rb.req);
}
}
if (rb.isNeedDocList() || rb.isDebug()) {
// we need a single list of the returned docs
cmd.setFlags(SolrIndexSearcher.GET_DOCLIST);
}
grouping.execute();
if (grouping.isSignalCacheWarning()) {
rsp.add("cacheWarning", String.format(Locale.ROOT, "Cache limit of %d percent relative to maxdoc has exceeded. Please increase cache size or disable caching.", maxDocsPercentageToCache));
}
rb.setResult(result);
if (grouping.mainResult != null) {
ResultContext ctx = new BasicResultContext(rb, grouping.mainResult);
rsp.addResponse(ctx);
rsp.getToLog().add("hits", grouping.mainResult.matches());
} else if (!grouping.getCommands().isEmpty()) {
// Can never be empty since grouping.execute() checks for this.
rsp.add("grouped", result.groupedResults);
rsp.getToLog().add("hits", grouping.getCommands().get(0).getMatches());
}
return;
} catch (SyntaxError e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
}
// normal search result
searcher.search(result, cmd);
rb.setResult(result);
ResultContext ctx = new BasicResultContext(rb);
rsp.addResponse(ctx);
rsp.getToLog().add("hits", rb.getResults().docList.matches());
if (!rb.req.getParams().getBool(ShardParams.IS_SHARD, false)) {
if (null != rb.getNextCursorMark()) {
rb.rsp.add(CursorMarkParams.CURSOR_MARK_NEXT, rb.getNextCursorMark().getSerializedTotem());
}
}
if (rb.mergeFieldHandler != null) {
rb.mergeFieldHandler.handleMergeFields(rb, searcher);
} else {
doFieldSortValues(rb, searcher);
}
doPrefetch(rb);
}
use of org.apache.lucene.search.grouping.SearchGroup in project lucene-solr by apache.
the class TopGroupsFieldCommand method create.
@Override
public List<Collector> create() throws IOException {
if (firstPhaseGroups.isEmpty()) {
return Collections.emptyList();
}
final List<Collector> collectors = new ArrayList<>(1);
final FieldType fieldType = field.getType();
if (fieldType.getNumberType() != null) {
ValueSource vs = fieldType.getValueSource(field, null);
Collection<SearchGroup<MutableValue>> v = GroupConverter.toMutable(field, firstPhaseGroups);
secondPassCollector = new TopGroupsCollector<>(new ValueSourceGroupSelector(vs, new HashMap<>()), v, groupSort, withinGroupSort, maxDocPerGroup, needScores, needMaxScore, true);
} else {
secondPassCollector = new TopGroupsCollector<>(new TermGroupSelector(field.getName()), firstPhaseGroups, groupSort, withinGroupSort, maxDocPerGroup, needScores, needMaxScore, true);
}
collectors.add(secondPassCollector);
return collectors;
}
Aggregations