use of org.apache.solr.client.solrj.io.comp.MultipleFieldComparator in project lucene-solr by apache.
the class SolrTable method handleSelectDistinctMapReduce.
private TupleStream handleSelectDistinctMapReduce(final String zkHost, final String collection, final Properties properties, final List<Map.Entry<String, Class>> fields, final String query, final List<Pair<String, String>> orders, final Bucket[] buckets, final String limit) throws IOException {
int numWorkers = Integer.parseInt(properties.getProperty("numWorkers", "1"));
String fl = getFields(fields);
String sort = null;
StreamEqualitor ecomp = null;
StreamComparator comp = null;
if (orders != null && orders.size() > 0) {
StreamComparator[] adjustedSorts = adjustSorts(orders, buckets);
// Because of the way adjustSorts works we know that each FieldComparator has a single
// field name. For this reason we can just look at the leftFieldName
FieldEqualitor[] fieldEqualitors = new FieldEqualitor[adjustedSorts.length];
StringBuilder buf = new StringBuilder();
for (int i = 0; i < adjustedSorts.length; i++) {
FieldComparator fieldComparator = (FieldComparator) adjustedSorts[i];
fieldEqualitors[i] = new FieldEqualitor(fieldComparator.getLeftFieldName());
if (i > 0) {
buf.append(",");
}
buf.append(fieldComparator.getLeftFieldName()).append(" ").append(fieldComparator.getOrder().toString());
}
sort = buf.toString();
if (adjustedSorts.length == 1) {
ecomp = fieldEqualitors[0];
comp = adjustedSorts[0];
} else {
ecomp = new MultipleFieldEqualitor(fieldEqualitors);
comp = new MultipleFieldComparator(adjustedSorts);
}
} else {
StringBuilder sortBuf = new StringBuilder();
FieldEqualitor[] equalitors = new FieldEqualitor[buckets.length];
StreamComparator[] streamComparators = new StreamComparator[buckets.length];
for (int i = 0; i < buckets.length; i++) {
equalitors[i] = new FieldEqualitor(buckets[i].toString());
streamComparators[i] = new FieldComparator(buckets[i].toString(), ComparatorOrder.ASCENDING);
if (i > 0) {
sortBuf.append(',');
}
sortBuf.append(buckets[i].toString()).append(" asc");
}
sort = sortBuf.toString();
if (equalitors.length == 1) {
ecomp = equalitors[0];
comp = streamComparators[0];
} else {
ecomp = new MultipleFieldEqualitor(equalitors);
comp = new MultipleFieldComparator(streamComparators);
}
}
ModifiableSolrParams params = new ModifiableSolrParams();
params.set(CommonParams.FL, fl);
params.set(CommonParams.Q, query);
params.set(CommonParams.WT, CommonParams.JAVABIN);
//Always use the /export handler for Distinct Queries because it requires exporting full result sets.
params.set(CommonParams.QT, "/export");
if (numWorkers > 1) {
params.set("partitionKeys", getPartitionKeys(buckets));
}
params.set(SORT, sort);
TupleStream tupleStream = null;
CloudSolrStream cstream = new CloudSolrStream(zkHost, collection, params);
tupleStream = new UniqueStream(cstream, ecomp);
if (numWorkers > 1) {
// Do the unique in parallel
// Maintain the sort of the Tuples coming from the workers.
ParallelStream parallelStream = new ParallelStream(zkHost, collection, tupleStream, numWorkers, comp);
StreamFactory factory = new StreamFactory().withFunctionName("search", CloudSolrStream.class).withFunctionName("parallel", ParallelStream.class).withFunctionName("unique", UniqueStream.class);
parallelStream.setStreamFactory(factory);
tupleStream = parallelStream;
}
if (limit != null) {
tupleStream = new LimitStream(tupleStream, Integer.parseInt(limit));
}
return tupleStream;
}
use of org.apache.solr.client.solrj.io.comp.MultipleFieldComparator in project lucene-solr by apache.
the class BiJoinStream method createIterationComparator.
private StreamComparator createIterationComparator(StreamEqualitor eq, StreamComparator comp) throws IOException {
if (eq instanceof MultipleFieldEqualitor && comp instanceof MultipleFieldComparator) {
// we know the comp is at least as long as the eq because we've already validated the tuple order
StreamComparator[] compoundComps = new StreamComparator[((MultipleFieldEqualitor) eq).getEqs().length];
for (int idx = 0; idx < compoundComps.length; ++idx) {
StreamEqualitor sourceEqualitor = ((MultipleFieldEqualitor) eq).getEqs()[idx];
StreamComparator sourceComparator = ((MultipleFieldComparator) comp).getComps()[idx];
if (sourceEqualitor instanceof FieldEqualitor && sourceComparator instanceof FieldComparator) {
FieldEqualitor fieldEqualitor = (FieldEqualitor) sourceEqualitor;
FieldComparator fieldComparator = (FieldComparator) sourceComparator;
compoundComps[idx] = new FieldComparator(fieldEqualitor.getLeftFieldName(), fieldEqualitor.getRightFieldName(), fieldComparator.getOrder());
} else {
throw new IOException("Failed to create an iteration comparator");
}
}
return new MultipleFieldComparator(compoundComps);
} else if (comp instanceof MultipleFieldComparator) {
StreamEqualitor sourceEqualitor = eq;
StreamComparator sourceComparator = ((MultipleFieldComparator) comp).getComps()[0];
if (sourceEqualitor instanceof FieldEqualitor && sourceComparator instanceof FieldComparator) {
FieldEqualitor fieldEqualitor = (FieldEqualitor) sourceEqualitor;
FieldComparator fieldComparator = (FieldComparator) sourceComparator;
return new FieldComparator(fieldEqualitor.getLeftFieldName(), fieldEqualitor.getRightFieldName(), fieldComparator.getOrder());
} else {
throw new IOException("Failed to create an iteration comparator");
}
} else {
StreamEqualitor sourceEqualitor = eq;
StreamComparator sourceComparator = comp;
if (sourceEqualitor instanceof FieldEqualitor && sourceComparator instanceof FieldComparator) {
FieldEqualitor fieldEqualitor = (FieldEqualitor) sourceEqualitor;
FieldComparator fieldComparator = (FieldComparator) sourceComparator;
return new FieldComparator(fieldEqualitor.getLeftFieldName(), fieldEqualitor.getRightFieldName(), fieldComparator.getOrder());
} else {
throw new IOException("Failed to create an iteration comparator");
}
}
}
use of org.apache.solr.client.solrj.io.comp.MultipleFieldComparator in project lucene-solr by apache.
the class CloudSolrStream method parseComp.
private StreamComparator parseComp(String sort, String fl) throws IOException {
String[] fls = fl.split(",");
HashSet fieldSet = new HashSet();
for (String f : fls) {
//Handle spaces in the field list.
fieldSet.add(f.trim());
}
String[] sorts = sort.split(",");
StreamComparator[] comps = new StreamComparator[sorts.length];
for (int i = 0; i < sorts.length; i++) {
String s = sorts[i];
//This should take into account spaces in the sort spec.
String[] spec = s.trim().split("\\s+");
if (spec.length != 2) {
throw new IOException("Invalid sort spec:" + s);
}
String fieldName = spec[0].trim();
String order = spec[1].trim();
if (!fieldSet.contains(spec[0])) {
throw new IOException("Fields in the sort spec must be included in the field list:" + spec[0]);
}
// if there's an alias for the field then use the alias
if (null != fieldMappings && fieldMappings.containsKey(fieldName)) {
fieldName = fieldMappings.get(fieldName);
}
comps[i] = new FieldComparator(fieldName, order.equalsIgnoreCase("asc") ? ComparatorOrder.ASCENDING : ComparatorOrder.DESCENDING);
}
if (comps.length > 1) {
return new MultipleFieldComparator(comps);
} else {
return comps[0];
}
}
use of org.apache.solr.client.solrj.io.comp.MultipleFieldComparator in project lucene-solr by apache.
the class StreamFactory method constructComparator.
public StreamComparator constructComparator(String comparatorString, Class comparatorType) throws IOException {
if (comparatorString.contains(",")) {
String[] parts = comparatorString.split(",");
StreamComparator[] comps = new StreamComparator[parts.length];
for (int idx = 0; idx < parts.length; ++idx) {
comps[idx] = constructComparator(parts[idx].trim(), comparatorType);
}
return new MultipleFieldComparator(comps);
} else if (comparatorString.contains("=")) {
// expected format is "left=right order"
String[] parts = comparatorString.split("[ =]");
if (parts.length < 3) {
throw new IOException(String.format(Locale.ROOT, "Invalid comparator expression %s - expecting 'left=right order'", comparatorString));
}
String leftFieldName = null;
String rightFieldName = null;
String order = null;
for (String part : parts) {
// skip empty
if (null == part || 0 == part.trim().length()) {
continue;
}
// assign each in order
if (null == leftFieldName) {
leftFieldName = part.trim();
} else if (null == rightFieldName) {
rightFieldName = part.trim();
} else {
order = part.trim();
// we're done, stop looping
break;
}
}
if (null == leftFieldName || null == rightFieldName || null == order) {
throw new IOException(String.format(Locale.ROOT, "Invalid comparator expression %s - expecting 'left=right order'", comparatorString));
}
return (StreamComparator) createInstance(comparatorType, new Class[] { String.class, String.class, ComparatorOrder.class }, new Object[] { leftFieldName, rightFieldName, ComparatorOrder.fromString(order) });
} else {
// expected format is "field order"
String[] parts = comparatorString.split(" ");
if (2 != parts.length) {
throw new IOException(String.format(Locale.ROOT, "Invalid comparator expression %s - expecting 'field order'", comparatorString));
}
String fieldName = parts[0].trim();
String order = parts[1].trim();
return (StreamComparator) createInstance(comparatorType, new Class[] { String.class, ComparatorOrder.class }, new Object[] { fieldName, ComparatorOrder.fromString(order) });
}
}
use of org.apache.solr.client.solrj.io.comp.MultipleFieldComparator in project lucene-solr by apache.
the class StreamingTest method testMergeStream.
@Test
public void testMergeStream() throws Exception {
new UpdateRequest().add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0").add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0").add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3").add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4").add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
StreamContext streamContext = new StreamContext();
SolrClientCache solrClientCache = new SolrClientCache();
streamContext.setSolrClientCache(solrClientCache);
try {
//Test ascending
SolrParams sParamsA = mapParams("q", "id:(4 1)", "fl", "id,a_s,a_i", "sort", "a_i asc");
CloudSolrStream streamA = new CloudSolrStream(zkHost, COLLECTIONORALIAS, sParamsA);
SolrParams sParamsB = mapParams("q", "id:(0 2 3)", "fl", "id,a_s,a_i", "sort", "a_i asc");
CloudSolrStream streamB = new CloudSolrStream(zkHost, COLLECTIONORALIAS, sParamsB);
MergeStream mstream = new MergeStream(streamA, streamB, new FieldComparator("a_i", ComparatorOrder.ASCENDING));
mstream.setStreamContext(streamContext);
List<Tuple> tuples = getTuples(mstream);
assertEquals(5, tuples.size());
assertOrder(tuples, 0, 1, 2, 3, 4);
//Test descending
sParamsA = mapParams("q", "id:(4 1)", "fl", "id,a_s,a_i", "sort", "a_i desc");
streamA = new CloudSolrStream(zkHost, COLLECTIONORALIAS, sParamsA);
sParamsB = mapParams("q", "id:(0 2 3)", "fl", "id,a_s,a_i", "sort", "a_i desc");
streamB = new CloudSolrStream(zkHost, COLLECTIONORALIAS, sParamsB);
mstream = new MergeStream(streamA, streamB, new FieldComparator("a_i", ComparatorOrder.DESCENDING));
mstream.setStreamContext(streamContext);
tuples = getTuples(mstream);
assertEquals(5, tuples.size());
assertOrder(tuples, 4, 3, 2, 1, 0);
//Test compound sort
sParamsA = mapParams("q", "id:(2 4 1)", "fl", "id,a_s,a_i,a_f", "sort", "a_f asc,a_i asc");
streamA = new CloudSolrStream(zkHost, COLLECTIONORALIAS, sParamsA);
sParamsB = mapParams("q", "id:(0 3)", "fl", "id,a_s,a_i,a_f", "sort", "a_f asc,a_i asc");
streamB = new CloudSolrStream(zkHost, COLLECTIONORALIAS, sParamsB);
mstream = new MergeStream(streamA, streamB, new MultipleFieldComparator(new FieldComparator("a_f", ComparatorOrder.ASCENDING), new FieldComparator("a_i", ComparatorOrder.ASCENDING)));
mstream.setStreamContext(streamContext);
tuples = getTuples(mstream);
assertEquals(5, tuples.size());
assertOrder(tuples, 0, 2, 1, 3, 4);
sParamsA = mapParams("q", "id:(2 4 1)", "fl", "id,a_s,a_i,a_f", "sort", "a_f asc,a_i desc");
streamA = new CloudSolrStream(zkHost, COLLECTIONORALIAS, sParamsA);
sParamsB = mapParams("q", "id:(0 3)", "fl", "id,a_s,a_i,a_f", "sort", "a_f asc,a_i desc");
streamB = new CloudSolrStream(zkHost, COLLECTIONORALIAS, sParamsB);
mstream = new MergeStream(streamA, streamB, new MultipleFieldComparator(new FieldComparator("a_f", ComparatorOrder.ASCENDING), new FieldComparator("a_i", ComparatorOrder.DESCENDING)));
mstream.setStreamContext(streamContext);
tuples = getTuples(mstream);
assertEquals(5, tuples.size());
assertOrder(tuples, 2, 0, 1, 3, 4);
} finally {
solrClientCache.close();
}
}
Aggregations