Search in sources :

Example 71 with SolrClientCache

use of org.apache.solr.client.solrj.io.SolrClientCache in project lucene-solr by apache.

the class StreamExpressionTest method testParallelPriorityStream.

@Test
public void testParallelPriorityStream() throws Exception {
    Assume.assumeTrue(!useAlias);
    new UpdateRequest().add(id, "0", "a_s", "hello1", "a_i", "0", "a_f", "1").add(id, "2", "a_s", "hello1", "a_i", "2", "a_f", "2").add(id, "3", "a_s", "hello1", "a_i", "3", "a_f", "3").add(id, "4", "a_s", "hello1", "a_i", "4", "a_f", "4").add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "5").add(id, "5", "a_s", "hello", "a_i", "10", "a_f", "6").add(id, "6", "a_s", "hello", "a_i", "11", "a_f", "7").add(id, "7", "a_s", "hello", "a_i", "12", "a_f", "8").add(id, "8", "a_s", "hello", "a_i", "13", "a_f", "9").add(id, "9", "a_s", "hello1", "a_i", "14", "a_f", "10").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
    StreamFactory factory = new StreamFactory().withCollectionZkHost("collection1", cluster.getZkServer().getZkAddress()).withFunctionName("topic", TopicStream.class).withFunctionName("parallel", ParallelStream.class).withFunctionName("priority", PriorityStream.class);
    StreamExpression expression;
    TupleStream stream;
    List<Tuple> tuples;
    SolrClientCache cache = new SolrClientCache();
    try {
        FieldComparator comp = new FieldComparator("a_i", ComparatorOrder.ASCENDING);
        expression = StreamExpressionParser.parse("parallel(collection1, workers=2, sort=\"_version_ asc\", priority(topic(collection1, collection1, q=\"a_s:hello\", fl=\"id,a_i\", id=1000000, initialCheckpoint=0, partitionKeys=id)," + "topic(collection1, collection1, q=\"a_s:hello1\", fl=\"id,a_i\", id=2000000, initialCheckpoint=0, partitionKeys=id)))");
        stream = factory.constructStream(expression);
        StreamContext context = new StreamContext();
        context.setSolrClientCache(cache);
        stream.setStreamContext(context);
        tuples = getTuples(stream);
        Collections.sort(tuples, comp);
        //The tuples from the first topic (high priority) should be returned.
        assertEquals(tuples.size(), 4);
        assertOrder(tuples, 5, 6, 7, 8);
        expression = StreamExpressionParser.parse("parallel(collection1, workers=2, sort=\"_version_ asc\", priority(topic(collection1, collection1, q=\"a_s:hello\", fl=\"id,a_i\", id=1000000, initialCheckpoint=0, partitionKeys=id)," + "topic(collection1, collection1, q=\"a_s:hello1\", fl=\"id,a_i\", id=2000000, initialCheckpoint=0, partitionKeys=id)))");
        stream = factory.constructStream(expression);
        context = new StreamContext();
        context.setSolrClientCache(cache);
        stream.setStreamContext(context);
        tuples = getTuples(stream);
        Collections.sort(tuples, comp);
        //The Tuples from the second topic (Low priority) should be returned.
        assertEquals(tuples.size(), 6);
        assertOrder(tuples, 0, 1, 2, 3, 4, 9);
        expression = StreamExpressionParser.parse("parallel(collection1, workers=2, sort=\"_version_ asc\", priority(topic(collection1, collection1, q=\"a_s:hello\", fl=\"id,a_i\", id=1000000, initialCheckpoint=0, partitionKeys=id)," + "topic(collection1, collection1, q=\"a_s:hello1\", fl=\"id,a_i\", id=2000000, initialCheckpoint=0, partitionKeys=id)))");
        stream = factory.constructStream(expression);
        context = new StreamContext();
        context.setSolrClientCache(cache);
        stream.setStreamContext(context);
        tuples = getTuples(stream);
        //Both queus are empty.
        assertEquals(tuples.size(), 0);
    } finally {
        cache.close();
    }
}
Also used : StreamExpression(org.apache.solr.client.solrj.io.stream.expr.StreamExpression) UpdateRequest(org.apache.solr.client.solrj.request.UpdateRequest) StreamFactory(org.apache.solr.client.solrj.io.stream.expr.StreamFactory) SolrClientCache(org.apache.solr.client.solrj.io.SolrClientCache) FieldComparator(org.apache.solr.client.solrj.io.comp.FieldComparator) Tuple(org.apache.solr.client.solrj.io.Tuple) Test(org.junit.Test)

Example 72 with SolrClientCache

use of org.apache.solr.client.solrj.io.SolrClientCache in project lucene-solr by apache.

the class StreamExpressionTest method testMergeStream.

@Test
public void testMergeStream() throws Exception {
    new UpdateRequest().add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0").add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0").add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3").add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4").add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
    StreamExpression expression;
    TupleStream stream;
    List<Tuple> tuples;
    StreamFactory factory = new StreamFactory().withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress()).withFunctionName("search", CloudSolrStream.class).withFunctionName("unique", UniqueStream.class).withFunctionName("merge", MergeStream.class);
    // Basic test
    expression = StreamExpressionParser.parse("merge(" + "search(" + COLLECTIONORALIAS + ", q=\"id:(0 3 4)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\")," + "search(" + COLLECTIONORALIAS + ", q=\"id:(1)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\")," + "on=\"a_f asc\")");
    stream = new MergeStream(expression, factory);
    StreamContext streamContext = new StreamContext();
    SolrClientCache solrClientCache = new SolrClientCache();
    streamContext.setSolrClientCache(solrClientCache);
    try {
        stream.setStreamContext(streamContext);
        tuples = getTuples(stream);
        assert (tuples.size() == 4);
        assertOrder(tuples, 0, 1, 3, 4);
        // Basic test desc
        expression = StreamExpressionParser.parse("merge(" + "search(" + COLLECTIONORALIAS + ", q=\"id:(0 3 4)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f desc\")," + "search(" + COLLECTIONORALIAS + ", q=\"id:(1)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f desc\")," + "on=\"a_f desc\")");
        stream = new MergeStream(expression, factory);
        stream.setStreamContext(streamContext);
        tuples = getTuples(stream);
        assert (tuples.size() == 4);
        assertOrder(tuples, 4, 3, 1, 0);
        // Basic w/multi comp
        expression = StreamExpressionParser.parse("merge(" + "search(" + COLLECTIONORALIAS + ", q=\"id:(0 3 4)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_s asc\")," + "search(" + COLLECTIONORALIAS + ", q=\"id:(1 2)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_s asc\")," + "on=\"a_f asc, a_s asc\")");
        stream = new MergeStream(expression, factory);
        stream.setStreamContext(streamContext);
        tuples = getTuples(stream);
        assert (tuples.size() == 5);
        assertOrder(tuples, 0, 2, 1, 3, 4);
        // full factory w/multi comp
        stream = factory.constructStream("merge(" + "search(" + COLLECTIONORALIAS + ", q=\"id:(0 3 4)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_s asc\")," + "search(" + COLLECTIONORALIAS + ", q=\"id:(1 2)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_s asc\")," + "on=\"a_f asc, a_s asc\")");
        stream.setStreamContext(streamContext);
        tuples = getTuples(stream);
        assert (tuples.size() == 5);
        assertOrder(tuples, 0, 2, 1, 3, 4);
        // full factory w/multi streams
        stream = factory.constructStream("merge(" + "search(" + COLLECTIONORALIAS + ", q=\"id:(0 4)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_s asc\")," + "search(" + COLLECTIONORALIAS + ", q=\"id:(1)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_s asc\")," + "search(" + COLLECTIONORALIAS + ", q=\"id:(2)\", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_s asc\")," + "on=\"a_f asc\")");
        stream.setStreamContext(streamContext);
        tuples = getTuples(stream);
        assert (tuples.size() == 4);
        assertOrder(tuples, 0, 2, 1, 4);
    } finally {
        solrClientCache.close();
    }
}
Also used : StreamExpression(org.apache.solr.client.solrj.io.stream.expr.StreamExpression) UpdateRequest(org.apache.solr.client.solrj.request.UpdateRequest) StreamFactory(org.apache.solr.client.solrj.io.stream.expr.StreamFactory) SolrClientCache(org.apache.solr.client.solrj.io.SolrClientCache) Tuple(org.apache.solr.client.solrj.io.Tuple) Test(org.junit.Test)

Example 73 with SolrClientCache

use of org.apache.solr.client.solrj.io.SolrClientCache in project lucene-solr by apache.

the class StreamExpressionTest method testSignificantTermsStream.

@Test
public void testSignificantTermsStream() throws Exception {
    UpdateRequest updateRequest = new UpdateRequest();
    for (int i = 0; i < 5000; i++) {
        updateRequest.add(id, "a" + i, "test_t", "a b c d m l");
    }
    for (int i = 0; i < 5000; i++) {
        updateRequest.add(id, "b" + i, "test_t", "a b e f");
    }
    for (int i = 0; i < 900; i++) {
        updateRequest.add(id, "c" + i, "test_t", "c");
    }
    for (int i = 0; i < 600; i++) {
        updateRequest.add(id, "d" + i, "test_t", "d");
    }
    for (int i = 0; i < 500; i++) {
        updateRequest.add(id, "e" + i, "test_t", "m");
    }
    updateRequest.commit(cluster.getSolrClient(), COLLECTIONORALIAS);
    TupleStream stream;
    List<Tuple> tuples;
    StreamFactory factory = new StreamFactory().withCollectionZkHost("collection1", cluster.getZkServer().getZkAddress()).withDefaultZkHost(cluster.getZkServer().getZkAddress()).withFunctionName("significantTerms", SignificantTermsStream.class);
    StreamContext streamContext = new StreamContext();
    SolrClientCache cache = new SolrClientCache();
    streamContext.setSolrClientCache(cache);
    try {
        String significantTerms = "significantTerms(collection1, q=\"id:a*\",  field=\"test_t\", limit=3, minTermLength=1, maxDocFreq=\".5\")";
        stream = factory.constructStream(significantTerms);
        stream.setStreamContext(streamContext);
        tuples = getTuples(stream);
        assert (tuples.size() == 3);
        assertTrue(tuples.get(0).get("term").equals("l"));
        assertTrue(tuples.get(0).getLong("background") == 5000);
        assertTrue(tuples.get(0).getLong("foreground") == 5000);
        assertTrue(tuples.get(1).get("term").equals("m"));
        assertTrue(tuples.get(1).getLong("background") == 5500);
        assertTrue(tuples.get(1).getLong("foreground") == 5000);
        assertTrue(tuples.get(2).get("term").equals("d"));
        assertTrue(tuples.get(2).getLong("background") == 5600);
        assertTrue(tuples.get(2).getLong("foreground") == 5000);
        //Test maxDocFreq
        significantTerms = "significantTerms(collection1, q=\"id:a*\",  field=\"test_t\", limit=3, maxDocFreq=2650, minTermLength=1)";
        stream = factory.constructStream(significantTerms);
        stream.setStreamContext(streamContext);
        tuples = getTuples(stream);
        assert (tuples.size() == 1);
        assertTrue(tuples.get(0).get("term").equals("l"));
        assertTrue(tuples.get(0).getLong("background") == 5000);
        assertTrue(tuples.get(0).getLong("foreground") == 5000);
        //Test maxDocFreq percentage
        significantTerms = "significantTerms(collection1, q=\"id:a*\",  field=\"test_t\", limit=3, maxDocFreq=\".45\", minTermLength=1)";
        stream = factory.constructStream(significantTerms);
        stream.setStreamContext(streamContext);
        tuples = getTuples(stream);
        assert (tuples.size() == 1);
        assertTrue(tuples.get(0).get("term").equals("l"));
        assertTrue(tuples.get(0).getLong("background") == 5000);
        assertTrue(tuples.get(0).getLong("foreground") == 5000);
        //Test min doc freq
        significantTerms = "significantTerms(collection1, q=\"id:a*\",  field=\"test_t\", limit=3, minDocFreq=\"2700\", minTermLength=1, maxDocFreq=\".5\")";
        stream = factory.constructStream(significantTerms);
        stream.setStreamContext(streamContext);
        tuples = getTuples(stream);
        assert (tuples.size() == 3);
        assertTrue(tuples.get(0).get("term").equals("m"));
        assertTrue(tuples.get(0).getLong("background") == 5500);
        assertTrue(tuples.get(0).getLong("foreground") == 5000);
        assertTrue(tuples.get(1).get("term").equals("d"));
        assertTrue(tuples.get(1).getLong("background") == 5600);
        assertTrue(tuples.get(1).getLong("foreground") == 5000);
        assertTrue(tuples.get(2).get("term").equals("c"));
        assertTrue(tuples.get(2).getLong("background") == 5900);
        assertTrue(tuples.get(2).getLong("foreground") == 5000);
        //Test min doc freq percent
        significantTerms = "significantTerms(collection1, q=\"id:a*\",  field=\"test_t\", limit=3, minDocFreq=\".478\", minTermLength=1, maxDocFreq=\".5\")";
        stream = factory.constructStream(significantTerms);
        stream.setStreamContext(streamContext);
        tuples = getTuples(stream);
        assert (tuples.size() == 1);
        assertTrue(tuples.get(0).get("term").equals("c"));
        assertTrue(tuples.get(0).getLong("background") == 5900);
        assertTrue(tuples.get(0).getLong("foreground") == 5000);
        //Test limit
        significantTerms = "significantTerms(collection1, q=\"id:a*\",  field=\"test_t\", limit=2, minDocFreq=\"2700\", minTermLength=1, maxDocFreq=\".5\")";
        stream = factory.constructStream(significantTerms);
        stream.setStreamContext(streamContext);
        tuples = getTuples(stream);
        assert (tuples.size() == 2);
        assertTrue(tuples.get(0).get("term").equals("m"));
        assertTrue(tuples.get(0).getLong("background") == 5500);
        assertTrue(tuples.get(0).getLong("foreground") == 5000);
        assertTrue(tuples.get(1).get("term").equals("d"));
        assertTrue(tuples.get(1).getLong("background") == 5600);
        assertTrue(tuples.get(1).getLong("foreground") == 5000);
        //Test term length
        significantTerms = "significantTerms(collection1, q=\"id:a*\",  field=\"test_t\", limit=2, minDocFreq=\"2700\", minTermLength=2)";
        stream = factory.constructStream(significantTerms);
        stream.setStreamContext(streamContext);
        tuples = getTuples(stream);
        assert (tuples.size() == 0);
        //Test with shards parameter
        List<String> shardUrls = TupleStream.getShards(cluster.getZkServer().getZkAddress(), COLLECTIONORALIAS, streamContext);
        Map<String, List<String>> shardsMap = new HashMap();
        shardsMap.put("myCollection", shardUrls);
        StreamContext context = new StreamContext();
        context.put("shards", shardsMap);
        context.setSolrClientCache(cache);
        significantTerms = "significantTerms(myCollection, q=\"id:a*\",  field=\"test_t\", limit=2, minDocFreq=\"2700\", minTermLength=1, maxDocFreq=\".5\")";
        stream = factory.constructStream(significantTerms);
        stream.setStreamContext(context);
        tuples = getTuples(stream);
        assert (tuples.size() == 2);
        assertTrue(tuples.get(0).get("term").equals("m"));
        assertTrue(tuples.get(0).getLong("background") == 5500);
        assertTrue(tuples.get(0).getLong("foreground") == 5000);
        assertTrue(tuples.get(1).get("term").equals("d"));
        assertTrue(tuples.get(1).getLong("background") == 5600);
        assertTrue(tuples.get(1).getLong("foreground") == 5000);
        //Execersise the /stream hander
        //Add the shards http parameter for the myCollection
        StringBuilder buf = new StringBuilder();
        for (String shardUrl : shardUrls) {
            if (buf.length() > 0) {
                buf.append(",");
            }
            buf.append(shardUrl);
        }
        ModifiableSolrParams solrParams = new ModifiableSolrParams();
        solrParams.add("qt", "/stream");
        solrParams.add("expr", significantTerms);
        solrParams.add("myCollection.shards", buf.toString());
        SolrStream solrStream = new SolrStream(shardUrls.get(0), solrParams);
        tuples = getTuples(solrStream);
        assert (tuples.size() == 2);
        assertTrue(tuples.get(0).get("term").equals("m"));
        assertTrue(tuples.get(0).getLong("background") == 5500);
        assertTrue(tuples.get(0).getLong("foreground") == 5000);
        assertTrue(tuples.get(1).get("term").equals("d"));
        assertTrue(tuples.get(1).getLong("background") == 5600);
        assertTrue(tuples.get(1).getLong("foreground") == 5000);
        try {
            ModifiableSolrParams solrParamsBad = new ModifiableSolrParams();
            solrParamsBad.add("qt", "/stream");
            solrParamsBad.add("expr", significantTerms);
            solrStream = new SolrStream(shardUrls.get(0), solrParamsBad);
            tuples = getTuples(solrStream);
            throw new Exception("Exception should have been thrown above");
        } catch (IOException e) {
            assertTrue(e.getMessage().contains("Slices not found for myCollection"));
        }
    } finally {
        cache.close();
    }
}
Also used : UpdateRequest(org.apache.solr.client.solrj.request.UpdateRequest) HashMap(java.util.HashMap) IOException(java.io.IOException) ModifiableSolrParams(org.apache.solr.common.params.ModifiableSolrParams) IOException(java.io.IOException) StreamFactory(org.apache.solr.client.solrj.io.stream.expr.StreamFactory) SolrClientCache(org.apache.solr.client.solrj.io.SolrClientCache) ArrayList(java.util.ArrayList) List(java.util.List) Tuple(org.apache.solr.client.solrj.io.Tuple) Test(org.junit.Test)

Example 74 with SolrClientCache

use of org.apache.solr.client.solrj.io.SolrClientCache in project lucene-solr by apache.

the class StreamExpressionTest method testLeftOuterJoinStream.

@Test
public void testLeftOuterJoinStream() throws Exception {
    new UpdateRequest().add(id, "1", "side_s", "left", "join1_i", "0", "join2_s", "a", "ident_s", // 8, 9
    "left_1").add(id, "15", "side_s", "left", "join1_i", "0", "join2_s", "a", "ident_s", // 8, 9
    "left_1").add(id, "2", "side_s", "left", "join1_i", "0", "join2_s", "b", "ident_s", "left_2").add(id, "3", "side_s", "left", "join1_i", "1", "join2_s", "a", "ident_s", // 10
    "left_3").add(id, "4", "side_s", "left", "join1_i", "1", "join2_s", "b", "ident_s", // 11
    "left_4").add(id, "5", "side_s", "left", "join1_i", "1", "join2_s", "c", "ident_s", // 12
    "left_5").add(id, "6", "side_s", "left", "join1_i", "2", "join2_s", "d", "ident_s", "left_6").add(id, "7", "side_s", "left", "join1_i", "3", "join2_s", "e", "ident_s", // 14
    "left_7").add(id, "8", "side_s", "right", "join1_i", "0", "join2_s", "a", "ident_s", "right_1", "join3_i", // 1,15
    "0").add(id, "9", "side_s", "right", "join1_i", "0", "join2_s", "a", "ident_s", "right_2", "join3_i", // 1,15
    "0").add(id, "10", "side_s", "right", "join1_i", "1", "join2_s", "a", "ident_s", "right_3", "join3_i", // 3
    "1").add(id, "11", "side_s", "right", "join1_i", "1", "join2_s", "b", "ident_s", "right_4", "join3_i", // 4
    "1").add(id, "12", "side_s", "right", "join1_i", "1", "join2_s", "c", "ident_s", "right_5", "join3_i", // 5
    "1").add(id, "13", "side_s", "right", "join1_i", "2", "join2_s", "dad", "ident_s", "right_6", "join3_i", "2").add(id, "14", "side_s", "right", "join1_i", "3", "join2_s", "e", "ident_s", "right_7", "join3_i", // 7
    "3").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
    StreamExpression expression;
    TupleStream stream;
    List<Tuple> tuples;
    StreamContext streamContext = new StreamContext();
    SolrClientCache solrClientCache = new SolrClientCache();
    streamContext.setSolrClientCache(solrClientCache);
    StreamFactory factory = new StreamFactory().withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress()).withFunctionName("search", CloudSolrStream.class).withFunctionName("leftOuterJoin", LeftOuterJoinStream.class);
    // Basic test
    try {
        expression = StreamExpressionParser.parse("leftOuterJoin(" + "search(" + COLLECTIONORALIAS + ", q=\"side_s:left\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"join1_i asc, join2_s asc, id asc\")," + "search(" + COLLECTIONORALIAS + ", q=\"side_s:right\", fl=\"join1_i,join2_s,ident_s\", sort=\"join1_i asc, join2_s asc\")," + "on=\"join1_i=join1_i, join2_s=join2_s\")");
        stream = new LeftOuterJoinStream(expression, factory);
        stream.setStreamContext(streamContext);
        tuples = getTuples(stream);
        assert (tuples.size() == 10);
        assertOrder(tuples, 1, 1, 15, 15, 2, 3, 4, 5, 6, 7);
        // Basic desc
        expression = StreamExpressionParser.parse("leftOuterJoin(" + "search(" + COLLECTIONORALIAS + ", q=\"side_s:left\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"join1_i desc, join2_s asc\")," + "search(" + COLLECTIONORALIAS + ", q=\"side_s:right\", fl=\"join1_i,join2_s,ident_s\", sort=\"join1_i desc, join2_s asc\")," + "on=\"join1_i=join1_i, join2_s=join2_s\")");
        stream = new LeftOuterJoinStream(expression, factory);
        stream.setStreamContext(streamContext);
        tuples = getTuples(stream);
        assert (tuples.size() == 10);
        assertOrder(tuples, 7, 6, 3, 4, 5, 1, 1, 15, 15, 2);
        // Results in both searches, no join matches
        expression = StreamExpressionParser.parse("leftOuterJoin(" + "search(" + COLLECTIONORALIAS + ", q=\"side_s:left\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"ident_s asc\")," + "search(" + COLLECTIONORALIAS + ", q=\"side_s:right\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"ident_s asc\", aliases=\"id=right.id, join1_i=right.join1_i, join2_s=right.join2_s, ident_s=right.ident_s\")," + "on=\"ident_s=right.ident_s\")");
        stream = new LeftOuterJoinStream(expression, factory);
        stream.setStreamContext(streamContext);
        tuples = getTuples(stream);
        assert (tuples.size() == 8);
        assertOrder(tuples, 1, 15, 2, 3, 4, 5, 6, 7);
        // Differing field names
        expression = StreamExpressionParser.parse("leftOuterJoin(" + "search(" + COLLECTIONORALIAS + ", q=\"side_s:left\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"join1_i asc, join2_s asc, id asc\")," + "search(" + COLLECTIONORALIAS + ", q=\"side_s:right\", fl=\"join3_i,join2_s,ident_s\", sort=\"join3_i asc, join2_s asc\", aliases=\"join3_i=aliasesField\")," + "on=\"join1_i=aliasesField, join2_s=join2_s\")");
        stream = new LeftOuterJoinStream(expression, factory);
        stream.setStreamContext(streamContext);
        tuples = getTuples(stream);
        assert (tuples.size() == 10);
        assertOrder(tuples, 1, 1, 15, 15, 2, 3, 4, 5, 6, 7);
    } finally {
        solrClientCache.close();
    }
}
Also used : StreamExpression(org.apache.solr.client.solrj.io.stream.expr.StreamExpression) UpdateRequest(org.apache.solr.client.solrj.request.UpdateRequest) StreamFactory(org.apache.solr.client.solrj.io.stream.expr.StreamFactory) SolrClientCache(org.apache.solr.client.solrj.io.SolrClientCache) Tuple(org.apache.solr.client.solrj.io.Tuple) Test(org.junit.Test)

Example 75 with SolrClientCache

use of org.apache.solr.client.solrj.io.SolrClientCache in project lucene-solr by apache.

the class StreamExpressionTest method testRollupStream.

@Test
public void testRollupStream() throws Exception {
    new UpdateRequest().add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1").add(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2").add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3").add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4").add(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5").add(id, "5", "a_s", "hello3", "a_i", "10", "a_f", "6").add(id, "6", "a_s", "hello4", "a_i", "11", "a_f", "7").add(id, "7", "a_s", "hello3", "a_i", "12", "a_f", "8").add(id, "8", "a_s", "hello3", "a_i", "13", "a_f", "9").add(id, "9", "a_s", "hello0", "a_i", "14", "a_f", "10").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
    StreamFactory factory = new StreamFactory().withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress()).withFunctionName("search", CloudSolrStream.class).withFunctionName("rollup", RollupStream.class).withFunctionName("sum", SumMetric.class).withFunctionName("min", MinMetric.class).withFunctionName("max", MaxMetric.class).withFunctionName("avg", MeanMetric.class).withFunctionName("count", CountMetric.class);
    StreamExpression expression;
    TupleStream stream;
    List<Tuple> tuples;
    StreamContext streamContext = new StreamContext();
    SolrClientCache solrClientCache = new SolrClientCache();
    streamContext.setSolrClientCache(solrClientCache);
    try {
        expression = StreamExpressionParser.parse("rollup(" + "search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"a_s,a_i,a_f\", sort=\"a_s asc\")," + "over=\"a_s\"," + "sum(a_i)," + "sum(a_f)," + "min(a_i)," + "min(a_f)," + "max(a_i)," + "max(a_f)," + "avg(a_i)," + "avg(a_f)," + "count(*)," + ")");
        stream = factory.constructStream(expression);
        stream.setStreamContext(streamContext);
        tuples = getTuples(stream);
        assert (tuples.size() == 3);
        //Test Long and Double Sums
        Tuple tuple = tuples.get(0);
        String bucket = tuple.getString("a_s");
        Double sumi = tuple.getDouble("sum(a_i)");
        Double sumf = tuple.getDouble("sum(a_f)");
        Double mini = tuple.getDouble("min(a_i)");
        Double minf = tuple.getDouble("min(a_f)");
        Double maxi = tuple.getDouble("max(a_i)");
        Double maxf = tuple.getDouble("max(a_f)");
        Double avgi = tuple.getDouble("avg(a_i)");
        Double avgf = tuple.getDouble("avg(a_f)");
        Double count = tuple.getDouble("count(*)");
        assertTrue(bucket.equals("hello0"));
        assertTrue(sumi.doubleValue() == 17.0D);
        assertTrue(sumf.doubleValue() == 18.0D);
        assertTrue(mini.doubleValue() == 0.0D);
        assertTrue(minf.doubleValue() == 1.0D);
        assertTrue(maxi.doubleValue() == 14.0D);
        assertTrue(maxf.doubleValue() == 10.0D);
        assertTrue(avgi.doubleValue() == 4.25D);
        assertTrue(avgf.doubleValue() == 4.5D);
        assertTrue(count.doubleValue() == 4);
        tuple = tuples.get(1);
        bucket = tuple.getString("a_s");
        sumi = tuple.getDouble("sum(a_i)");
        sumf = tuple.getDouble("sum(a_f)");
        mini = tuple.getDouble("min(a_i)");
        minf = tuple.getDouble("min(a_f)");
        maxi = tuple.getDouble("max(a_i)");
        maxf = tuple.getDouble("max(a_f)");
        avgi = tuple.getDouble("avg(a_i)");
        avgf = tuple.getDouble("avg(a_f)");
        count = tuple.getDouble("count(*)");
        assertTrue(bucket.equals("hello3"));
        assertTrue(sumi.doubleValue() == 38.0D);
        assertTrue(sumf.doubleValue() == 26.0D);
        assertTrue(mini.doubleValue() == 3.0D);
        assertTrue(minf.doubleValue() == 3.0D);
        assertTrue(maxi.doubleValue() == 13.0D);
        assertTrue(maxf.doubleValue() == 9.0D);
        assertTrue(avgi.doubleValue() == 9.5D);
        assertTrue(avgf.doubleValue() == 6.5D);
        assertTrue(count.doubleValue() == 4);
        tuple = tuples.get(2);
        bucket = tuple.getString("a_s");
        sumi = tuple.getDouble("sum(a_i)");
        sumf = tuple.getDouble("sum(a_f)");
        mini = tuple.getDouble("min(a_i)");
        minf = tuple.getDouble("min(a_f)");
        maxi = tuple.getDouble("max(a_i)");
        maxf = tuple.getDouble("max(a_f)");
        avgi = tuple.getDouble("avg(a_i)");
        avgf = tuple.getDouble("avg(a_f)");
        count = tuple.getDouble("count(*)");
        assertTrue(bucket.equals("hello4"));
        assertTrue(sumi.longValue() == 15);
        assertTrue(sumf.doubleValue() == 11.0D);
        assertTrue(mini.doubleValue() == 4.0D);
        assertTrue(minf.doubleValue() == 4.0D);
        assertTrue(maxi.doubleValue() == 11.0D);
        assertTrue(maxf.doubleValue() == 7.0D);
        assertTrue(avgi.doubleValue() == 7.5D);
        assertTrue(avgf.doubleValue() == 5.5D);
        assertTrue(count.doubleValue() == 2);
    } finally {
        solrClientCache.close();
    }
}
Also used : UpdateRequest(org.apache.solr.client.solrj.request.UpdateRequest) MinMetric(org.apache.solr.client.solrj.io.stream.metrics.MinMetric) StreamExpression(org.apache.solr.client.solrj.io.stream.expr.StreamExpression) MeanMetric(org.apache.solr.client.solrj.io.stream.metrics.MeanMetric) StreamFactory(org.apache.solr.client.solrj.io.stream.expr.StreamFactory) SolrClientCache(org.apache.solr.client.solrj.io.SolrClientCache) Tuple(org.apache.solr.client.solrj.io.Tuple) Test(org.junit.Test)

Aggregations

SolrClientCache (org.apache.solr.client.solrj.io.SolrClientCache)96 Tuple (org.apache.solr.client.solrj.io.Tuple)92 UpdateRequest (org.apache.solr.client.solrj.request.UpdateRequest)88 Test (org.junit.Test)88 StreamFactory (org.apache.solr.client.solrj.io.stream.expr.StreamFactory)61 ModifiableSolrParams (org.apache.solr.common.params.ModifiableSolrParams)39 StreamExpression (org.apache.solr.client.solrj.io.stream.expr.StreamExpression)36 SolrParams (org.apache.solr.common.params.SolrParams)27 FieldComparator (org.apache.solr.client.solrj.io.comp.FieldComparator)20 MultipleFieldComparator (org.apache.solr.client.solrj.io.comp.MultipleFieldComparator)14 MeanMetric (org.apache.solr.client.solrj.io.stream.metrics.MeanMetric)13 MinMetric (org.apache.solr.client.solrj.io.stream.metrics.MinMetric)12 CountMetric (org.apache.solr.client.solrj.io.stream.metrics.CountMetric)7 MaxMetric (org.apache.solr.client.solrj.io.stream.metrics.MaxMetric)7 IOException (java.io.IOException)6 HashMap (java.util.HashMap)6 JettySolrRunner (org.apache.solr.client.solrj.embedded.JettySolrRunner)6 FieldEqualitor (org.apache.solr.client.solrj.io.eq.FieldEqualitor)6 StreamContext (org.apache.solr.client.solrj.io.stream.StreamContext)6 SumMetric (org.apache.solr.client.solrj.io.stream.metrics.SumMetric)6