Examples with StreamFactory org.apache.solr.client.solrj.io.stream.expr.StreamFactory used on opensource projects

Example 21 with StreamFactory

use of org.apache.solr.client.solrj.io.stream.expr.StreamFactory in project lucene-solr by apache.

the class StreamExpressionTest method testOuterHashJoinStream.

@Test
public void testOuterHashJoinStream() throws Exception {
    new UpdateRequest().add(id, "1", "side_s", "left", "join1_i", "0", "join2_s", "a", "ident_s", // 8, 9
    "left_1").add(id, "15", "side_s", "left", "join1_i", "0", "join2_s", "a", "ident_s", // 8, 9
    "left_1").add(id, "2", "side_s", "left", "join1_i", "0", "join2_s", "b", "ident_s", "left_2").add(id, "3", "side_s", "left", "join1_i", "1", "join2_s", "a", "ident_s", // 10
    "left_3").add(id, "4", "side_s", "left", "join1_i", "1", "join2_s", "b", "ident_s", // 11
    "left_4").add(id, "5", "side_s", "left", "join1_i", "1", "join2_s", "c", "ident_s", // 12
    "left_5").add(id, "6", "side_s", "left", "join1_i", "2", "join2_s", "d", "ident_s", "left_6").add(id, "7", "side_s", "left", "join1_i", "3", "join2_s", "e", "ident_s", // 14
    "left_7").add(id, "8", "side_s", "right", "join1_i", "0", "join2_s", "a", "ident_s", "right_1", "join3_i", // 1,15
    "0").add(id, "9", "side_s", "right", "join1_i", "0", "join2_s", "a", "ident_s", "right_2", "join3_i", // 1,15
    "0").add(id, "10", "side_s", "right", "join1_i", "1", "join2_s", "a", "ident_s", "right_3", "join3_i", // 3
    "1").add(id, "11", "side_s", "right", "join1_i", "1", "join2_s", "b", "ident_s", "right_4", "join3_i", // 4
    "1").add(id, "12", "side_s", "right", "join1_i", "1", "join2_s", "c", "ident_s", "right_5", "join3_i", // 5
    "1").add(id, "13", "side_s", "right", "join1_i", "2", "join2_s", "dad", "ident_s", "right_6", "join3_i", "2").add(id, "14", "side_s", "right", "join1_i", "3", "join2_s", "e", "ident_s", "right_7", "join3_i", // 7
    "3").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
    StreamExpression expression;
    TupleStream stream;
    List<Tuple> tuples;
    StreamContext streamContext = new StreamContext();
    SolrClientCache solrClientCache = new SolrClientCache();
    streamContext.setSolrClientCache(solrClientCache);
    StreamFactory factory = new StreamFactory().withCollectionZkHost("collection1", cluster.getZkServer().getZkAddress()).withFunctionName("search", CloudSolrStream.class).withFunctionName("outerHashJoin", OuterHashJoinStream.class);
    try {
        // Basic test
        expression = StreamExpressionParser.parse("outerHashJoin(" + "search(collection1, q=\"side_s:left\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"join1_i asc, join2_s asc, id asc\")," + "hashed=search(collection1, q=\"side_s:right\", fl=\"join1_i,join2_s,ident_s\", sort=\"join1_i asc, join2_s asc\")," + "on=\"join1_i, join2_s\")");
        stream = new OuterHashJoinStream(expression, factory);
        stream.setStreamContext(streamContext);
        tuples = getTuples(stream);
        assert (tuples.size() == 10);
        assertOrder(tuples, 1, 1, 15, 15, 2, 3, 4, 5, 6, 7);
        // Basic desc
        expression = StreamExpressionParser.parse("outerHashJoin(" + "search(collection1, q=\"side_s:left\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"join1_i desc, join2_s asc\")," + "hashed=search(collection1, q=\"side_s:right\", fl=\"join1_i,join2_s,ident_s\", sort=\"join1_i desc, join2_s asc\")," + "on=\"join1_i, join2_s\")");
        stream = new OuterHashJoinStream(expression, factory);
        stream.setStreamContext(streamContext);
        tuples = getTuples(stream);
        assert (tuples.size() == 10);
        assertOrder(tuples, 7, 6, 3, 4, 5, 1, 1, 15, 15, 2);
        // Results in both searches, no join matches
        expression = StreamExpressionParser.parse("outerHashJoin(" + "search(collection1, q=\"side_s:left\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"ident_s asc\")," + "hashed=search(collection1, q=\"side_s:right\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"ident_s asc\")," + "on=\"ident_s\")");
        stream = new OuterHashJoinStream(expression, factory);
        stream.setStreamContext(streamContext);
        tuples = getTuples(stream);
        assert (tuples.size() == 8);
        assertOrder(tuples, 1, 15, 2, 3, 4, 5, 6, 7);
        // Basic test
        expression = StreamExpressionParser.parse("outerHashJoin(" + "search(collection1, q=\"side_s:left\", fl=\"id,join1_i,join2_s,ident_s\", sort=\"join1_i asc, join2_s asc, id asc\")," + "hashed=search(collection1, q=\"side_s:right\", fl=\"join3_i,join2_s,ident_s\", sort=\"join2_s asc\")," + "on=\"join1_i=join3_i, join2_s\")");
        stream = new OuterHashJoinStream(expression, factory);
        stream.setStreamContext(streamContext);
        tuples = getTuples(stream);
        assert (tuples.size() == 10);
        assertOrder(tuples, 1, 1, 15, 15, 2, 3, 4, 5, 6, 7);
    } finally {
        solrClientCache.close();
    }
}

Example 22 with StreamFactory

use of org.apache.solr.client.solrj.io.stream.expr.StreamFactory in project lucene-solr by apache.

the class StreamExpressionTest method testCommitStream.

////////////////////////////////////////////
@Test
public void testCommitStream() throws Exception {
    CollectionAdminRequest.createCollection("destinationCollection", "conf", 2, 1).process(cluster.getSolrClient());
    AbstractDistribZkTestBase.waitForRecoveriesToFinish("destinationCollection", cluster.getSolrClient().getZkStateReader(), false, true, TIMEOUT);
    new UpdateRequest().add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0", "s_multi", "aaaa", "s_multi", "bbbb", "i_multi", "4", "i_multi", "7").add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0", "s_multi", "aaaa1", "s_multi", "bbbb1", "i_multi", "44", "i_multi", "77").add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3", "s_multi", "aaaa2", "s_multi", "bbbb2", "i_multi", "444", "i_multi", "777").add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4", "s_multi", "aaaa3", "s_multi", "bbbb3", "i_multi", "4444", "i_multi", "7777").add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1", "s_multi", "aaaa4", "s_multi", "bbbb4", "i_multi", "44444", "i_multi", "77777").commit(cluster.getSolrClient(), "collection1");
    StreamExpression expression;
    TupleStream stream;
    Tuple t;
    StreamContext streamContext = new StreamContext();
    SolrClientCache solrClientCache = new SolrClientCache();
    streamContext.setSolrClientCache(solrClientCache);
    StreamFactory factory = new StreamFactory().withCollectionZkHost("collection1", cluster.getZkServer().getZkAddress()).withCollectionZkHost("destinationCollection", cluster.getZkServer().getZkAddress()).withFunctionName("search", CloudSolrStream.class).withFunctionName("update", UpdateStream.class).withFunctionName("commit", CommitStream.class);
    try {
        //Copy all docs to destinationCollection
        expression = StreamExpressionParser.parse("commit(destinationCollection, batchSize=2, update(destinationCollection, batchSize=5, search(collection1, q=*:*, fl=\"id,a_s,a_i,a_f,s_multi,i_multi\", sort=\"a_f asc, a_i asc\")))");
        stream = factory.constructStream(expression);
        stream.setStreamContext(streamContext);
        List<Tuple> tuples = getTuples(stream);
        //Ensure that all CommitStream tuples indicate the correct number of copied/indexed docs
        assert (tuples.size() == 1);
        t = tuples.get(0);
        assert (t.EOF == false);
        assertEquals(5, t.get("batchIndexed"));
        //Ensure that destinationCollection actually has the new docs.
        expression = StreamExpressionParser.parse("search(destinationCollection, q=*:*, fl=\"id,a_s,a_i,a_f,s_multi,i_multi\", sort=\"a_i asc\")");
        stream = new CloudSolrStream(expression, factory);
        stream.setStreamContext(streamContext);
        tuples = getTuples(stream);
        assertEquals(5, tuples.size());
        Tuple tuple = tuples.get(0);
        assert (tuple.getLong("id") == 0);
        assert (tuple.get("a_s").equals("hello0"));
        assert (tuple.getLong("a_i") == 0);
        assert (tuple.getDouble("a_f") == 0.0);
        assertList(tuple.getStrings("s_multi"), "aaaa", "bbbb");
        assertList(tuple.getLongs("i_multi"), Long.parseLong("4"), Long.parseLong("7"));
        tuple = tuples.get(1);
        assert (tuple.getLong("id") == 1);
        assert (tuple.get("a_s").equals("hello1"));
        assert (tuple.getLong("a_i") == 1);
        assert (tuple.getDouble("a_f") == 1.0);
        assertList(tuple.getStrings("s_multi"), "aaaa4", "bbbb4");
        assertList(tuple.getLongs("i_multi"), Long.parseLong("44444"), Long.parseLong("77777"));
        tuple = tuples.get(2);
        assert (tuple.getLong("id") == 2);
        assert (tuple.get("a_s").equals("hello2"));
        assert (tuple.getLong("a_i") == 2);
        assert (tuple.getDouble("a_f") == 0.0);
        assertList(tuple.getStrings("s_multi"), "aaaa1", "bbbb1");
        assertList(tuple.getLongs("i_multi"), Long.parseLong("44"), Long.parseLong("77"));
        tuple = tuples.get(3);
        assert (tuple.getLong("id") == 3);
        assert (tuple.get("a_s").equals("hello3"));
        assert (tuple.getLong("a_i") == 3);
        assert (tuple.getDouble("a_f") == 3.0);
        assertList(tuple.getStrings("s_multi"), "aaaa2", "bbbb2");
        assertList(tuple.getLongs("i_multi"), Long.parseLong("444"), Long.parseLong("777"));
        tuple = tuples.get(4);
        assert (tuple.getLong("id") == 4);
        assert (tuple.get("a_s").equals("hello4"));
        assert (tuple.getLong("a_i") == 4);
        assert (tuple.getDouble("a_f") == 4.0);
        assertList(tuple.getStrings("s_multi"), "aaaa3", "bbbb3");
        assertList(tuple.getLongs("i_multi"), Long.parseLong("4444"), Long.parseLong("7777"));
    } finally {
        CollectionAdminRequest.deleteCollection("destinationCollection").process(cluster.getSolrClient());
        solrClientCache.close();
    }
}

Example 23 with StreamFactory

use of org.apache.solr.client.solrj.io.stream.expr.StreamFactory in project lucene-solr by apache.

the class StreamExpressionTest method testParallelComplementStream.

@Test
public void testParallelComplementStream() throws Exception {
    new UpdateRequest().add(id, "0", "a_s", "setA", "a_i", "0").add(id, "2", "a_s", "setA", "a_i", "1").add(id, "3", "a_s", "setA", "a_i", "2").add(id, "4", "a_s", "setA", "a_i", "3").add(id, "5", "a_s", "setB", "a_i", "2").add(id, "6", "a_s", "setB", "a_i", "3").add(id, "9", "a_s", "setB", "a_i", "5").add(id, "7", "a_s", "setAB", "a_i", "0").add(id, "8", "a_s", "setAB", "a_i", "6").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
    StreamFactory streamFactory = new StreamFactory().withCollectionZkHost("collection1", cluster.getZkServer().getZkAddress()).withFunctionName("search", CloudSolrStream.class).withFunctionName("complement", ComplementStream.class).withFunctionName("parallel", ParallelStream.class);
    StreamContext streamContext = new StreamContext();
    SolrClientCache solrClientCache = new SolrClientCache();
    streamContext.setSolrClientCache(solrClientCache);
    try {
        final String zkHost = cluster.getZkServer().getZkAddress();
        final TupleStream stream = streamFactory.constructStream("parallel(" + "collection1, " + "complement(" + "search(collection1, q=a_s:(setA || setAB), fl=\"id,a_s,a_i\", sort=\"a_i asc, a_s asc\", partitionKeys=\"a_i\")," + "search(collection1, q=a_s:(setB || setAB), fl=\"id,a_s,a_i\", sort=\"a_i asc\", partitionKeys=\"a_i\")," + "on=\"a_i\")," + "workers=\"2\", zkHost=\"" + zkHost + "\", sort=\"a_i asc\")");
        stream.setStreamContext(streamContext);
        final List<Tuple> tuples = getTuples(stream);
        assert (tuples.size() == 1);
        assertOrder(tuples, 2);
    } finally {
        solrClientCache.close();
    }
}

Example 24 with StreamFactory

use of org.apache.solr.client.solrj.io.stream.expr.StreamFactory in project lucene-solr by apache.

the class StreamExpressionTest method testParallelShuffleStream.

@Test
public void testParallelShuffleStream() throws Exception {
    new UpdateRequest().add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0").add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0").add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3").add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4").add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1").add(id, "5", "a_s", "hello1", "a_i", "10", "a_f", "1").add(id, "6", "a_s", "hello1", "a_i", "11", "a_f", "5").add(id, "7", "a_s", "hello1", "a_i", "12", "a_f", "5").add(id, "8", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "9", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "10", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "11", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "12", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "13", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "14", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "15", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "16", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "17", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "18", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "19", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "20", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "21", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "22", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "23", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "24", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "25", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "26", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "27", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "28", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "29", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "30", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "31", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "32", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "33", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "34", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "35", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "36", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "37", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "38", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "39", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "40", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "41", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "42", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "43", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "44", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "45", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "46", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "47", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "48", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "49", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "50", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "51", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "52", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "53", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "54", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "55", "a_s", "hello1", "a_i", "13", "a_f", "4").add(id, "56", "a_s", "hello1", "a_i", "13", "a_f", "1000").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
    StreamContext streamContext = new StreamContext();
    SolrClientCache solrClientCache = new SolrClientCache();
    streamContext.setSolrClientCache(solrClientCache);
    String zkHost = cluster.getZkServer().getZkAddress();
    StreamFactory streamFactory = new StreamFactory().withCollectionZkHost(COLLECTIONORALIAS, zkHost).withFunctionName("shuffle", ShuffleStream.class).withFunctionName("unique", UniqueStream.class).withFunctionName("parallel", ParallelStream.class);
    try {
        ParallelStream pstream = (ParallelStream) streamFactory.constructStream("parallel(" + COLLECTIONORALIAS + ", unique(shuffle(collection1, q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_i asc\", partitionKeys=\"a_f\"), over=\"a_f\"), workers=\"2\", zkHost=\"" + zkHost + "\", sort=\"a_f asc\")");
        pstream.setStreamFactory(streamFactory);
        pstream.setStreamContext(streamContext);
        List<Tuple> tuples = getTuples(pstream);
        assert (tuples.size() == 6);
        assertOrder(tuples, 0, 1, 3, 4, 6, 56);
        //Test the eofTuples
        Map<String, Tuple> eofTuples = pstream.getEofTuples();
        //There should be an EOF tuple for each worker.
        assert (eofTuples.size() == 2);
        assert (pstream.toExpression(streamFactory).toString().contains("shuffle"));
    } finally {
        solrClientCache.close();
    }
}

Example 25 with StreamFactory

use of org.apache.solr.client.solrj.io.stream.expr.StreamFactory in project lucene-solr by apache.

the class StreamExpressionTest method testParallelMergeStream.

@Test
public void testParallelMergeStream() throws Exception {
    new UpdateRequest().add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0").add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0").add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3").add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4").add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1").add(id, "5", "a_s", "hello0", "a_i", "10", "a_f", "0").add(id, "6", "a_s", "hello2", "a_i", "8", "a_f", "0").add(id, "7", "a_s", "hello3", "a_i", "7", "a_f", "3").add(id, "8", "a_s", "hello4", "a_i", "11", "a_f", "4").add(id, "9", "a_s", "hello1", "a_i", "100", "a_f", "1").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
    String zkHost = cluster.getZkServer().getZkAddress();
    StreamFactory streamFactory = new StreamFactory().withCollectionZkHost(COLLECTIONORALIAS, zkHost).withFunctionName("search", CloudSolrStream.class).withFunctionName("unique", UniqueStream.class).withFunctionName("top", RankStream.class).withFunctionName("group", ReducerStream.class).withFunctionName("merge", MergeStream.class).withFunctionName("parallel", ParallelStream.class);
    StreamContext streamContext = new StreamContext();
    SolrClientCache solrClientCache = new SolrClientCache();
    streamContext.setSolrClientCache(solrClientCache);
    try {
        //Test ascending
        ParallelStream pstream = (ParallelStream) streamFactory.constructStream("parallel(" + COLLECTIONORALIAS + ", merge(search(" + COLLECTIONORALIAS + ", q=\"id:(4 1 8 7 9)\", fl=\"id,a_s,a_i\", sort=\"a_i asc\", partitionKeys=\"a_i\"), search(" + COLLECTIONORALIAS + ", q=\"id:(0 2 3 6)\", fl=\"id,a_s,a_i\", sort=\"a_i asc\", partitionKeys=\"a_i\"), on=\"a_i asc\"), workers=\"2\", zkHost=\"" + zkHost + "\", sort=\"a_i asc\")");
        pstream.setStreamContext(streamContext);
        List<Tuple> tuples = getTuples(pstream);
        assert (tuples.size() == 9);
        assertOrder(tuples, 0, 1, 2, 3, 4, 7, 6, 8, 9);
        //Test descending
        pstream = (ParallelStream) streamFactory.constructStream("parallel(" + COLLECTIONORALIAS + ", merge(search(" + COLLECTIONORALIAS + ", q=\"id:(4 1 8 9)\", fl=\"id,a_s,a_i\", sort=\"a_i desc\", partitionKeys=\"a_i\"), search(" + COLLECTIONORALIAS + ", q=\"id:(0 2 3 6)\", fl=\"id,a_s,a_i\", sort=\"a_i desc\", partitionKeys=\"a_i\"), on=\"a_i desc\"), workers=\"2\", zkHost=\"" + zkHost + "\", sort=\"a_i desc\")");
        pstream.setStreamContext(streamContext);
        tuples = getTuples(pstream);
        assert (tuples.size() == 8);
        assertOrder(tuples, 9, 8, 6, 4, 3, 2, 1, 0);
    } finally {
        solrClientCache.close();
    }
}