Examples with StreamComparator org.apache.solr.client.solrj.io.comp.StreamComparator used on opensource projects

Example 1 with StreamComparator

use of org.apache.solr.client.solrj.io.comp.StreamComparator in project lucene-solr by apache.

the class BiJoinStream method createIterationComparator.

private StreamComparator createIterationComparator(StreamEqualitor eq, StreamComparator comp) throws IOException {
    if (eq instanceof MultipleFieldEqualitor && comp instanceof MultipleFieldComparator) {
        // we know the comp is at least as long as the eq because we've already validated the tuple order
        StreamComparator[] compoundComps = new StreamComparator[((MultipleFieldEqualitor) eq).getEqs().length];
        for (int idx = 0; idx < compoundComps.length; ++idx) {
            StreamEqualitor sourceEqualitor = ((MultipleFieldEqualitor) eq).getEqs()[idx];
            StreamComparator sourceComparator = ((MultipleFieldComparator) comp).getComps()[idx];
            if (sourceEqualitor instanceof FieldEqualitor && sourceComparator instanceof FieldComparator) {
                FieldEqualitor fieldEqualitor = (FieldEqualitor) sourceEqualitor;
                FieldComparator fieldComparator = (FieldComparator) sourceComparator;
                compoundComps[idx] = new FieldComparator(fieldEqualitor.getLeftFieldName(), fieldEqualitor.getRightFieldName(), fieldComparator.getOrder());
            } else {
                throw new IOException("Failed to create an iteration comparator");
            }
        }
        return new MultipleFieldComparator(compoundComps);
    } else if (comp instanceof MultipleFieldComparator) {
        StreamEqualitor sourceEqualitor = eq;
        StreamComparator sourceComparator = ((MultipleFieldComparator) comp).getComps()[0];
        if (sourceEqualitor instanceof FieldEqualitor && sourceComparator instanceof FieldComparator) {
            FieldEqualitor fieldEqualitor = (FieldEqualitor) sourceEqualitor;
            FieldComparator fieldComparator = (FieldComparator) sourceComparator;
            return new FieldComparator(fieldEqualitor.getLeftFieldName(), fieldEqualitor.getRightFieldName(), fieldComparator.getOrder());
        } else {
            throw new IOException("Failed to create an iteration comparator");
        }
    } else {
        StreamEqualitor sourceEqualitor = eq;
        StreamComparator sourceComparator = comp;
        if (sourceEqualitor instanceof FieldEqualitor && sourceComparator instanceof FieldComparator) {
            FieldEqualitor fieldEqualitor = (FieldEqualitor) sourceEqualitor;
            FieldComparator fieldComparator = (FieldComparator) sourceComparator;
            return new FieldComparator(fieldEqualitor.getLeftFieldName(), fieldEqualitor.getRightFieldName(), fieldComparator.getOrder());
        } else {
            throw new IOException("Failed to create an iteration comparator");
        }
    }
}

Example 2 with StreamComparator

use of org.apache.solr.client.solrj.io.comp.StreamComparator in project lucene-solr by apache.

the class CloudSolrStream method parseComp.

private StreamComparator parseComp(String sort, String fl) throws IOException {
    String[] fls = fl.split(",");
    HashSet fieldSet = new HashSet();
    for (String f : fls) {
        //Handle spaces in the field list.
        fieldSet.add(f.trim());
    }
    String[] sorts = sort.split(",");
    StreamComparator[] comps = new StreamComparator[sorts.length];
    for (int i = 0; i < sorts.length; i++) {
        String s = sorts[i];
        //This should take into account spaces in the sort spec.
        String[] spec = s.trim().split("\\s+");
        if (spec.length != 2) {
            throw new IOException("Invalid sort spec:" + s);
        }
        String fieldName = spec[0].trim();
        String order = spec[1].trim();
        if (!fieldSet.contains(spec[0])) {
            throw new IOException("Fields in the sort spec must be included in the field list:" + spec[0]);
        }
        // if there's an alias for the field then use the alias
        if (null != fieldMappings && fieldMappings.containsKey(fieldName)) {
            fieldName = fieldMappings.get(fieldName);
        }
        comps[i] = new FieldComparator(fieldName, order.equalsIgnoreCase("asc") ? ComparatorOrder.ASCENDING : ComparatorOrder.DESCENDING);
    }
    if (comps.length > 1) {
        return new MultipleFieldComparator(comps);
    } else {
        return comps[0];
    }
}

Example 3 with StreamComparator

use of org.apache.solr.client.solrj.io.comp.StreamComparator in project lucene-solr by apache.

the class StreamFactory method constructComparator.

public StreamComparator constructComparator(String comparatorString, Class comparatorType) throws IOException {
    if (comparatorString.contains(",")) {
        String[] parts = comparatorString.split(",");
        StreamComparator[] comps = new StreamComparator[parts.length];
        for (int idx = 0; idx < parts.length; ++idx) {
            comps[idx] = constructComparator(parts[idx].trim(), comparatorType);
        }
        return new MultipleFieldComparator(comps);
    } else if (comparatorString.contains("=")) {
        // expected format is "left=right order"
        String[] parts = comparatorString.split("[ =]");
        if (parts.length < 3) {
            throw new IOException(String.format(Locale.ROOT, "Invalid comparator expression %s - expecting 'left=right order'", comparatorString));
        }
        String leftFieldName = null;
        String rightFieldName = null;
        String order = null;
        for (String part : parts) {
            // skip empty
            if (null == part || 0 == part.trim().length()) {
                continue;
            }
            // assign each in order
            if (null == leftFieldName) {
                leftFieldName = part.trim();
            } else if (null == rightFieldName) {
                rightFieldName = part.trim();
            } else {
                order = part.trim();
                // we're done, stop looping
                break;
            }
        }
        if (null == leftFieldName || null == rightFieldName || null == order) {
            throw new IOException(String.format(Locale.ROOT, "Invalid comparator expression %s - expecting 'left=right order'", comparatorString));
        }
        return (StreamComparator) createInstance(comparatorType, new Class[] { String.class, String.class, ComparatorOrder.class }, new Object[] { leftFieldName, rightFieldName, ComparatorOrder.fromString(order) });
    } else {
        // expected format is "field order"
        String[] parts = comparatorString.split(" ");
        if (2 != parts.length) {
            throw new IOException(String.format(Locale.ROOT, "Invalid comparator expression %s - expecting 'field order'", comparatorString));
        }
        String fieldName = parts[0].trim();
        String order = parts[1].trim();
        return (StreamComparator) createInstance(comparatorType, new Class[] { String.class, ComparatorOrder.class }, new Object[] { fieldName, ComparatorOrder.fromString(order) });
    }
}

Example 4 with StreamComparator

use of org.apache.solr.client.solrj.io.comp.StreamComparator in project lucene-solr by apache.

the class SolrTable method handleSelectDistinctMapReduce.

private TupleStream handleSelectDistinctMapReduce(final String zkHost, final String collection, final Properties properties, final List<Map.Entry<String, Class>> fields, final String query, final List<Pair<String, String>> orders, final Bucket[] buckets, final String limit) throws IOException {
    int numWorkers = Integer.parseInt(properties.getProperty("numWorkers", "1"));
    String fl = getFields(fields);
    String sort = null;
    StreamEqualitor ecomp = null;
    StreamComparator comp = null;
    if (orders != null && orders.size() > 0) {
        StreamComparator[] adjustedSorts = adjustSorts(orders, buckets);
        // Because of the way adjustSorts works we know that each FieldComparator has a single
        // field name. For this reason we can just look at the leftFieldName
        FieldEqualitor[] fieldEqualitors = new FieldEqualitor[adjustedSorts.length];
        StringBuilder buf = new StringBuilder();
        for (int i = 0; i < adjustedSorts.length; i++) {
            FieldComparator fieldComparator = (FieldComparator) adjustedSorts[i];
            fieldEqualitors[i] = new FieldEqualitor(fieldComparator.getLeftFieldName());
            if (i > 0) {
                buf.append(",");
            }
            buf.append(fieldComparator.getLeftFieldName()).append(" ").append(fieldComparator.getOrder().toString());
        }
        sort = buf.toString();
        if (adjustedSorts.length == 1) {
            ecomp = fieldEqualitors[0];
            comp = adjustedSorts[0];
        } else {
            ecomp = new MultipleFieldEqualitor(fieldEqualitors);
            comp = new MultipleFieldComparator(adjustedSorts);
        }
    } else {
        StringBuilder sortBuf = new StringBuilder();
        FieldEqualitor[] equalitors = new FieldEqualitor[buckets.length];
        StreamComparator[] streamComparators = new StreamComparator[buckets.length];
        for (int i = 0; i < buckets.length; i++) {
            equalitors[i] = new FieldEqualitor(buckets[i].toString());
            streamComparators[i] = new FieldComparator(buckets[i].toString(), ComparatorOrder.ASCENDING);
            if (i > 0) {
                sortBuf.append(',');
            }
            sortBuf.append(buckets[i].toString()).append(" asc");
        }
        sort = sortBuf.toString();
        if (equalitors.length == 1) {
            ecomp = equalitors[0];
            comp = streamComparators[0];
        } else {
            ecomp = new MultipleFieldEqualitor(equalitors);
            comp = new MultipleFieldComparator(streamComparators);
        }
    }
    ModifiableSolrParams params = new ModifiableSolrParams();
    params.set(CommonParams.FL, fl);
    params.set(CommonParams.Q, query);
    params.set(CommonParams.WT, CommonParams.JAVABIN);
    //Always use the /export handler for Distinct Queries because it requires exporting full result sets.
    params.set(CommonParams.QT, "/export");
    if (numWorkers > 1) {
        params.set("partitionKeys", getPartitionKeys(buckets));
    }
    params.set(SORT, sort);
    TupleStream tupleStream = null;
    CloudSolrStream cstream = new CloudSolrStream(zkHost, collection, params);
    tupleStream = new UniqueStream(cstream, ecomp);
    if (numWorkers > 1) {
        // Do the unique in parallel
        // Maintain the sort of the Tuples coming from the workers.
        ParallelStream parallelStream = new ParallelStream(zkHost, collection, tupleStream, numWorkers, comp);
        StreamFactory factory = new StreamFactory().withFunctionName("search", CloudSolrStream.class).withFunctionName("parallel", ParallelStream.class).withFunctionName("unique", UniqueStream.class);
        parallelStream.setStreamFactory(factory);
        tupleStream = parallelStream;
    }
    if (limit != null) {
        tupleStream = new LimitStream(tupleStream, Integer.parseInt(limit));
    }
    return tupleStream;
}

Example 5 with StreamComparator

use of org.apache.solr.client.solrj.io.comp.StreamComparator in project lucene-solr by apache.

the class SolrTable method handleGroupByMapReduce.

private TupleStream handleGroupByMapReduce(String zk, String collection, Properties properties, final List<Map.Entry<String, Class>> fields, final String query, final List<Pair<String, String>> orders, final List<String> _buckets, final List<Pair<String, String>> metricPairs, final String limit, final String havingPredicate) throws IOException {
    Map<String, Class> fmap = new HashMap();
    for (Map.Entry<String, Class> entry : fields) {
        fmap.put(entry.getKey(), entry.getValue());
    }
    int numWorkers = Integer.parseInt(properties.getProperty("numWorkers", "1"));
    Bucket[] buckets = buildBuckets(_buckets, fields);
    Metric[] metrics = buildMetrics(metricPairs, false).toArray(new Metric[0]);
    if (metrics.length == 0) {
        return handleSelectDistinctMapReduce(zk, collection, properties, fields, query, orders, buckets, limit);
    } else {
        for (Metric metric : metrics) {
            Class c = fmap.get(metric.getIdentifier());
            if (Long.class.equals(c)) {
                metric.outputLong = true;
            }
        }
    }
    Set<String> fieldSet = getFieldSet(metrics, fields);
    if (metrics.length == 0) {
        throw new IOException("Group by queries must include atleast one aggregate function.");
    }
    String fl = getFields(fieldSet);
    String sortDirection = getSortDirection(orders);
    String sort = bucketSort(buckets, sortDirection);
    ModifiableSolrParams params = new ModifiableSolrParams();
    params.set(CommonParams.FL, fl);
    params.set(CommonParams.Q, query);
    params.set(CommonParams.WT, CommonParams.JAVABIN);
    //Always use the /export handler for Group By Queries because it requires exporting full result sets.
    params.set(CommonParams.QT, "/export");
    if (numWorkers > 1) {
        params.set("partitionKeys", getPartitionKeys(buckets));
    }
    params.set(SORT, sort);
    TupleStream tupleStream = null;
    CloudSolrStream cstream = new CloudSolrStream(zk, collection, params);
    tupleStream = new RollupStream(cstream, buckets, metrics);
    StreamFactory factory = new StreamFactory().withFunctionName("search", CloudSolrStream.class).withFunctionName("parallel", ParallelStream.class).withFunctionName("rollup", RollupStream.class).withFunctionName("sum", SumMetric.class).withFunctionName("min", MinMetric.class).withFunctionName("max", MaxMetric.class).withFunctionName("avg", MeanMetric.class).withFunctionName("count", CountMetric.class).withFunctionName("and", AndEvaluator.class).withFunctionName("or", OrEvaluator.class).withFunctionName("not", NotEvaluator.class).withFunctionName("eq", EqualsEvaluator.class).withFunctionName("gt", GreaterThanEvaluator.class).withFunctionName("lt", LessThanEvaluator.class).withFunctionName("val", RawValueEvaluator.class).withFunctionName("lteq", LessThanEqualToEvaluator.class).withFunctionName("having", HavingStream.class).withFunctionName("gteq", GreaterThanEqualToEvaluator.class);
    if (havingPredicate != null) {
        BooleanEvaluator booleanOperation = (BooleanEvaluator) factory.constructEvaluator(StreamExpressionParser.parse(havingPredicate));
        tupleStream = new HavingStream(tupleStream, booleanOperation);
    }
    if (numWorkers > 1) {
        // Do the rollups in parallel
        // Maintain the sort of the Tuples coming from the workers.
        StreamComparator comp = bucketSortComp(buckets, sortDirection);
        ParallelStream parallelStream = new ParallelStream(zk, collection, tupleStream, numWorkers, comp);
        parallelStream.setStreamFactory(factory);
        tupleStream = parallelStream;
    }
    if (orders != null && orders.size() > 0) {
        if (!sortsEqual(buckets, sortDirection, orders)) {
            int lim = (limit == null) ? 100 : Integer.parseInt(limit);
            StreamComparator comp = getComp(orders);
            //Rank the Tuples
            //If parallel stream is used ALL the Rolled up tuples from the workers will be ranked
            //Providing a true Top or Bottom.
            tupleStream = new RankStream(tupleStream, lim, comp);
        } else {
            // Only need to limit the result, not Rank the result
            if (limit != null) {
                tupleStream = new LimitStream(tupleStream, Integer.parseInt(limit));
            }
        }
    } else {
        //No order by, check for limit
        if (limit != null) {
            tupleStream = new LimitStream(tupleStream, Integer.parseInt(limit));
        }
    }
    return tupleStream;
}