Examples with FieldComparator org.apache.solr.client.solrj.io.comp.FieldComparator used on opensource projects

Example 26 with FieldComparator

use of org.apache.solr.client.solrj.io.comp.FieldComparator in project lucene-solr by apache.

the class ReducerStream method convertToEqualitor.

private StreamEqualitor convertToEqualitor(StreamComparator comp) {
    if (comp instanceof MultipleFieldComparator) {
        MultipleFieldComparator mComp = (MultipleFieldComparator) comp;
        StreamEqualitor[] eqs = new StreamEqualitor[mComp.getComps().length];
        for (int idx = 0; idx < mComp.getComps().length; ++idx) {
            eqs[idx] = convertToEqualitor(mComp.getComps()[idx]);
        }
        return new MultipleFieldEqualitor(eqs);
    } else {
        FieldComparator fComp = (FieldComparator) comp;
        return new FieldEqualitor(fComp.getLeftFieldName(), fComp.getRightFieldName());
    }
}

Example 27 with FieldComparator

use of org.apache.solr.client.solrj.io.comp.FieldComparator in project lucene-solr by apache.

the class StreamExpressionTest method testParallelPriorityStream.

@Test
public void testParallelPriorityStream() throws Exception {
    Assume.assumeTrue(!useAlias);
    new UpdateRequest().add(id, "0", "a_s", "hello1", "a_i", "0", "a_f", "1").add(id, "2", "a_s", "hello1", "a_i", "2", "a_f", "2").add(id, "3", "a_s", "hello1", "a_i", "3", "a_f", "3").add(id, "4", "a_s", "hello1", "a_i", "4", "a_f", "4").add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "5").add(id, "5", "a_s", "hello", "a_i", "10", "a_f", "6").add(id, "6", "a_s", "hello", "a_i", "11", "a_f", "7").add(id, "7", "a_s", "hello", "a_i", "12", "a_f", "8").add(id, "8", "a_s", "hello", "a_i", "13", "a_f", "9").add(id, "9", "a_s", "hello1", "a_i", "14", "a_f", "10").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
    StreamFactory factory = new StreamFactory().withCollectionZkHost("collection1", cluster.getZkServer().getZkAddress()).withFunctionName("topic", TopicStream.class).withFunctionName("parallel", ParallelStream.class).withFunctionName("priority", PriorityStream.class);
    StreamExpression expression;
    TupleStream stream;
    List<Tuple> tuples;
    SolrClientCache cache = new SolrClientCache();
    try {
        FieldComparator comp = new FieldComparator("a_i", ComparatorOrder.ASCENDING);
        expression = StreamExpressionParser.parse("parallel(collection1, workers=2, sort=\"_version_ asc\", priority(topic(collection1, collection1, q=\"a_s:hello\", fl=\"id,a_i\", id=1000000, initialCheckpoint=0, partitionKeys=id)," + "topic(collection1, collection1, q=\"a_s:hello1\", fl=\"id,a_i\", id=2000000, initialCheckpoint=0, partitionKeys=id)))");
        stream = factory.constructStream(expression);
        StreamContext context = new StreamContext();
        context.setSolrClientCache(cache);
        stream.setStreamContext(context);
        tuples = getTuples(stream);
        Collections.sort(tuples, comp);
        //The tuples from the first topic (high priority) should be returned.
        assertEquals(tuples.size(), 4);
        assertOrder(tuples, 5, 6, 7, 8);
        expression = StreamExpressionParser.parse("parallel(collection1, workers=2, sort=\"_version_ asc\", priority(topic(collection1, collection1, q=\"a_s:hello\", fl=\"id,a_i\", id=1000000, initialCheckpoint=0, partitionKeys=id)," + "topic(collection1, collection1, q=\"a_s:hello1\", fl=\"id,a_i\", id=2000000, initialCheckpoint=0, partitionKeys=id)))");
        stream = factory.constructStream(expression);
        context = new StreamContext();
        context.setSolrClientCache(cache);
        stream.setStreamContext(context);
        tuples = getTuples(stream);
        Collections.sort(tuples, comp);
        //The Tuples from the second topic (Low priority) should be returned.
        assertEquals(tuples.size(), 6);
        assertOrder(tuples, 0, 1, 2, 3, 4, 9);
        expression = StreamExpressionParser.parse("parallel(collection1, workers=2, sort=\"_version_ asc\", priority(topic(collection1, collection1, q=\"a_s:hello\", fl=\"id,a_i\", id=1000000, initialCheckpoint=0, partitionKeys=id)," + "topic(collection1, collection1, q=\"a_s:hello1\", fl=\"id,a_i\", id=2000000, initialCheckpoint=0, partitionKeys=id)))");
        stream = factory.constructStream(expression);
        context = new StreamContext();
        context.setSolrClientCache(cache);
        stream.setStreamContext(context);
        tuples = getTuples(stream);
        //Both queus are empty.
        assertEquals(tuples.size(), 0);
    } finally {
        cache.close();
    }
}

Example 28 with FieldComparator

use of org.apache.solr.client.solrj.io.comp.FieldComparator in project lucene-solr by apache.

the class StreamExpressionTest method testRandomStream.

@Test
public void testRandomStream() throws Exception {
    UpdateRequest update = new UpdateRequest();
    for (int idx = 0; idx < 1000; ++idx) {
        String idxString = new Integer(idx).toString();
        update.add(id, idxString, "a_s", "hello" + idxString, "a_i", idxString, "a_f", idxString);
    }
    update.commit(cluster.getSolrClient(), COLLECTIONORALIAS);
    StreamExpression expression;
    TupleStream stream;
    StreamFactory factory = new StreamFactory().withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress()).withFunctionName("random", RandomStream.class);
    StreamContext context = new StreamContext();
    SolrClientCache cache = new SolrClientCache();
    try {
        context.setSolrClientCache(cache);
        expression = StreamExpressionParser.parse("random(" + COLLECTIONORALIAS + ", q=\"*:*\", rows=\"1000\", fl=\"id, a_i\")");
        stream = factory.constructStream(expression);
        stream.setStreamContext(context);
        List<Tuple> tuples1 = getTuples(stream);
        assert (tuples1.size() == 1000);
        expression = StreamExpressionParser.parse("random(" + COLLECTIONORALIAS + ", q=\"*:*\", rows=\"1000\", fl=\"id, a_i\")");
        stream = factory.constructStream(expression);
        stream.setStreamContext(context);
        List<Tuple> tuples2 = getTuples(stream);
        assert (tuples2.size() == 1000);
        boolean different = false;
        for (int i = 0; i < tuples1.size(); i++) {
            Tuple tuple1 = tuples1.get(i);
            Tuple tuple2 = tuples2.get(i);
            if (!tuple1.get("id").equals(tuple2.get(id))) {
                different = true;
                break;
            }
        }
        assertTrue(different);
        Collections.sort(tuples1, new FieldComparator("id", ComparatorOrder.ASCENDING));
        Collections.sort(tuples2, new FieldComparator("id", ComparatorOrder.ASCENDING));
        for (int i = 0; i < tuples1.size(); i++) {
            Tuple tuple1 = tuples1.get(i);
            Tuple tuple2 = tuples2.get(i);
            if (!tuple1.get("id").equals(tuple2.get(id))) {
                assert (tuple1.getLong("id").equals(tuple2.get("a_i")));
            }
        }
        expression = StreamExpressionParser.parse("random(" + COLLECTIONORALIAS + ", q=\"*:*\", rows=\"1\", fl=\"id, a_i\")");
        stream = factory.constructStream(expression);
        stream.setStreamContext(context);
        List<Tuple> tuples3 = getTuples(stream);
        assert (tuples3.size() == 1);
        //Exercise the /stream handler
        ModifiableSolrParams sParams = new ModifiableSolrParams(StreamingTest.mapParams(CommonParams.QT, "/stream"));
        sParams.add("expr", "random(" + COLLECTIONORALIAS + ", q=\"*:*\", rows=\"1\", fl=\"id, a_i\")");
        JettySolrRunner jetty = cluster.getJettySolrRunner(0);
        SolrStream solrStream = new SolrStream(jetty.getBaseUrl().toString() + "/collection1", sParams);
        List<Tuple> tuples4 = getTuples(solrStream);
        assert (tuples4.size() == 1);
    } finally {
        cache.close();
    }
}

Example 29 with FieldComparator

use of org.apache.solr.client.solrj.io.comp.FieldComparator in project lucene-solr by apache.

the class StreamingTest method testParallelRollupStream.

@Test
public void testParallelRollupStream() throws Exception {
    new UpdateRequest().add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1").add(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2").add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3").add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4").add(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5").add(id, "5", "a_s", "hello3", "a_i", "10", "a_f", "6").add(id, "6", "a_s", "hello4", "a_i", "11", "a_f", "7").add(id, "7", "a_s", "hello3", "a_i", "12", "a_f", "8").add(id, "8", "a_s", "hello3", "a_i", "13", "a_f", "9").add(id, "9", "a_s", "hello0", "a_i", "14", "a_f", "10").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
    StreamContext streamContext = new StreamContext();
    SolrClientCache solrClientCache = new SolrClientCache();
    streamContext.setSolrClientCache(solrClientCache);
    try {
        SolrParams sParamsA = mapParams("q", "*:*", "fl", "a_s,a_i,a_f", "sort", "a_s asc", "partitionKeys", "a_s");
        CloudSolrStream stream = new CloudSolrStream(zkHost, COLLECTIONORALIAS, sParamsA);
        Bucket[] buckets = { new Bucket("a_s") };
        Metric[] metrics = { new SumMetric("a_i"), new SumMetric("a_f"), new MinMetric("a_i"), new MinMetric("a_f"), new MaxMetric("a_i"), new MaxMetric("a_f"), new MeanMetric("a_i"), new MeanMetric("a_f"), new CountMetric() };
        RollupStream rollupStream = new RollupStream(stream, buckets, metrics);
        ParallelStream parallelStream = parallelStream(rollupStream, new FieldComparator("a_s", ComparatorOrder.ASCENDING));
        attachStreamFactory(parallelStream);
        parallelStream.setStreamContext(streamContext);
        List<Tuple> tuples = getTuples(parallelStream);
        assertEquals(3, tuples.size());
        //Test Long and Double Sums
        Tuple tuple = tuples.get(0);
        String bucket = tuple.getString("a_s");
        Double sumi = tuple.getDouble("sum(a_i)");
        Double sumf = tuple.getDouble("sum(a_f)");
        Double mini = tuple.getDouble("min(a_i)");
        Double minf = tuple.getDouble("min(a_f)");
        Double maxi = tuple.getDouble("max(a_i)");
        Double maxf = tuple.getDouble("max(a_f)");
        Double avgi = tuple.getDouble("avg(a_i)");
        Double avgf = tuple.getDouble("avg(a_f)");
        Double count = tuple.getDouble("count(*)");
        assertEquals("hello0", bucket);
        assertEquals(17, sumi.doubleValue(), 0.001);
        assertEquals(18, sumf.doubleValue(), 0.001);
        assertEquals(0, mini.doubleValue(), 0.001);
        assertEquals(1, minf.doubleValue(), 0.001);
        assertEquals(14, maxi.doubleValue(), 0.001);
        assertEquals(10, maxf.doubleValue(), 0.001);
        assertEquals(4.25, avgi.doubleValue(), 0.001);
        assertEquals(4.5, avgf.doubleValue(), 0.001);
        assertEquals(4, count.doubleValue(), 0.001);
        tuple = tuples.get(1);
        bucket = tuple.getString("a_s");
        sumi = tuple.getDouble("sum(a_i)");
        sumf = tuple.getDouble("sum(a_f)");
        mini = tuple.getDouble("min(a_i)");
        minf = tuple.getDouble("min(a_f)");
        maxi = tuple.getDouble("max(a_i)");
        maxf = tuple.getDouble("max(a_f)");
        avgi = tuple.getDouble("avg(a_i)");
        avgf = tuple.getDouble("avg(a_f)");
        count = tuple.getDouble("count(*)");
        assertEquals("hello3", bucket);
        assertEquals(38, sumi.doubleValue(), 0.001);
        assertEquals(26, sumf.doubleValue(), 0.001);
        assertEquals(3, mini.doubleValue(), 0.001);
        assertEquals(3, minf.doubleValue(), 0.001);
        assertEquals(13, maxi.doubleValue(), 0.001);
        assertEquals(9, maxf.doubleValue(), 0.001);
        assertEquals(9.5, avgi.doubleValue(), 0.001);
        assertEquals(6.5, avgf.doubleValue(), 0.001);
        assertEquals(4, count.doubleValue(), 0.001);
        tuple = tuples.get(2);
        bucket = tuple.getString("a_s");
        sumi = tuple.getDouble("sum(a_i)");
        sumf = tuple.getDouble("sum(a_f)");
        mini = tuple.getDouble("min(a_i)");
        minf = tuple.getDouble("min(a_f)");
        maxi = tuple.getDouble("max(a_i)");
        maxf = tuple.getDouble("max(a_f)");
        avgi = tuple.getDouble("avg(a_i)");
        avgf = tuple.getDouble("avg(a_f)");
        count = tuple.getDouble("count(*)");
        assertEquals("hello4", bucket);
        assertEquals(15, sumi.longValue());
        assertEquals(11, sumf.doubleValue(), 0.001);
        assertEquals(4, mini.doubleValue(), 0.001);
        assertEquals(4, minf.doubleValue(), 0.001);
        assertEquals(11, maxi.doubleValue(), 0.001);
        assertEquals(7, maxf.doubleValue(), 0.001);
        assertEquals(7.5, avgi.doubleValue(), 0.001);
        assertEquals(5.5, avgf.doubleValue(), 0.001);
        assertEquals(2, count.doubleValue(), 0.001);
    } finally {
        solrClientCache.close();
    }
}

Example 30 with FieldComparator

use of org.apache.solr.client.solrj.io.comp.FieldComparator in project lucene-solr by apache.

the class StreamingTest method testNonePartitionKeys.

@Test
public void testNonePartitionKeys() throws Exception {
    new UpdateRequest().add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1").add(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2").add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3").add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4").add(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5").add(id, "5", "a_s", "hello3", "a_i", "10", "a_f", "6").add(id, "6", "a_s", "hello4", "a_i", "11", "a_f", "7").add(id, "7", "a_s", "hello3", "a_i", "12", "a_f", "8").add(id, "8", "a_s", "hello3", "a_i", "13", "a_f", "9").add(id, "9", "a_s", "hello0", "a_i", "14", "a_f", "10").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
    StreamContext streamContext = new StreamContext();
    SolrClientCache solrClientCache = new SolrClientCache();
    streamContext.setSolrClientCache(solrClientCache);
    try {
        SolrParams sParamsA = StreamingTest.mapParams("q", "*:*", "fl", "id,a_s,a_i,a_f", "sort", "a_s asc,a_f asc", "partitionKeys", "none");
        CloudSolrStream stream = new CloudSolrStream(zkHost, COLLECTIONORALIAS, sParamsA);
        ParallelStream pstream = parallelStream(stream, new FieldComparator("a_s", ComparatorOrder.ASCENDING));
        attachStreamFactory(pstream);
        pstream.setStreamContext(streamContext);
        List<Tuple> tuples = getTuples(pstream);
        // Each tuple will be double counted.
        assert (tuples.size() == (10 * numWorkers));
    } finally {
        solrClientCache.close();
    }
}