Examples with SolrClientCache org.apache.solr.client.solrj.io.SolrClientCache used on opensource projects

Example 31 with SolrClientCache

use of org.apache.solr.client.solrj.io.SolrClientCache in project lucene-solr by apache.

the class StreamExpressionTest method testSortStream.

@Test
public void testSortStream() throws Exception {
    new UpdateRequest().add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0").add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0").add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3").add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4").add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1").add(id, "5", "a_s", "hello1", "a_i", "1", "a_f", "2").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
    StreamExpression expression;
    TupleStream stream;
    List<Tuple> tuples;
    StreamContext streamContext = new StreamContext();
    SolrClientCache solrClientCache = new SolrClientCache();
    streamContext.setSolrClientCache(solrClientCache);
    try {
        StreamFactory factory = new StreamFactory().withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress()).withFunctionName("search", CloudSolrStream.class).withFunctionName("sort", SortStream.class);
        // Basic test
        stream = factory.constructStream("sort(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"), by=\"a_i asc\")");
        stream.setStreamContext(streamContext);
        tuples = getTuples(stream);
        assert (tuples.size() == 6);
        assertOrder(tuples, 0, 1, 5, 2, 3, 4);
        // Basic test desc
        stream = factory.constructStream("sort(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"), by=\"a_i desc\")");
        stream.setStreamContext(streamContext);
        tuples = getTuples(stream);
        assert (tuples.size() == 6);
        assertOrder(tuples, 4, 3, 2, 1, 5, 0);
        // Basic w/multi comp
        stream = factory.constructStream("sort(search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc\"), by=\"a_i asc, a_f desc\")");
        stream.setStreamContext(streamContext);
        tuples = getTuples(stream);
        assert (tuples.size() == 6);
        assertOrder(tuples, 0, 5, 1, 2, 3, 4);
    } finally {
        solrClientCache.close();
    }
}

Example 32 with SolrClientCache

use of org.apache.solr.client.solrj.io.SolrClientCache in project lucene-solr by apache.

the class StreamExpressionTest method testParallelIntersectStream.

@Test
public void testParallelIntersectStream() throws Exception {
    new UpdateRequest().add(id, "0", "a_s", "setA", "a_i", "0").add(id, "2", "a_s", "setA", "a_i", "1").add(id, "3", "a_s", "setA", "a_i", "2").add(id, "4", "a_s", "setA", "a_i", "3").add(id, "5", "a_s", "setB", "a_i", "2").add(id, "6", "a_s", "setB", "a_i", "3").add(id, "7", "a_s", "setAB", "a_i", "0").add(id, "8", "a_s", "setAB", "a_i", "6").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
    StreamFactory streamFactory = new StreamFactory().withCollectionZkHost("collection1", cluster.getZkServer().getZkAddress()).withFunctionName("search", CloudSolrStream.class).withFunctionName("intersect", IntersectStream.class).withFunctionName("parallel", ParallelStream.class);
    // basic
    StreamContext streamContext = new StreamContext();
    SolrClientCache solrClientCache = new SolrClientCache();
    streamContext.setSolrClientCache(solrClientCache);
    try {
        String zkHost = cluster.getZkServer().getZkAddress();
        final TupleStream stream = streamFactory.constructStream("parallel(" + "collection1, " + "intersect(" + "search(collection1, q=a_s:(setA || setAB), fl=\"id,a_s,a_i\", sort=\"a_i asc, a_s asc\", partitionKeys=\"a_i\")," + "search(collection1, q=a_s:(setB || setAB), fl=\"id,a_s,a_i\", sort=\"a_i asc\", partitionKeys=\"a_i\")," + "on=\"a_i\")," + "workers=\"2\", zkHost=\"" + zkHost + "\", sort=\"a_i asc\")");
        stream.setStreamContext(streamContext);
        final List<Tuple> tuples = getTuples(stream);
        assert (tuples.size() == 5);
        assertOrder(tuples, 0, 7, 3, 4, 8);
    } finally {
        solrClientCache.close();
    }
}

Example 33 with SolrClientCache

use of org.apache.solr.client.solrj.io.SolrClientCache in project lucene-solr by apache.

the class StreamExpressionTest method testExecutorStream.

@Test
public void testExecutorStream() throws Exception {
    CollectionAdminRequest.createCollection("workQueue", "conf", 2, 1).process(cluster.getSolrClient());
    AbstractDistribZkTestBase.waitForRecoveriesToFinish("workQueue", cluster.getSolrClient().getZkStateReader(), false, true, TIMEOUT);
    CollectionAdminRequest.createCollection("mainCorpus", "conf", 2, 1).process(cluster.getSolrClient());
    AbstractDistribZkTestBase.waitForRecoveriesToFinish("mainCorpus", cluster.getSolrClient().getZkStateReader(), false, true, TIMEOUT);
    CollectionAdminRequest.createCollection("destination", "conf", 2, 1).process(cluster.getSolrClient());
    AbstractDistribZkTestBase.waitForRecoveriesToFinish("destination", cluster.getSolrClient().getZkStateReader(), false, true, TIMEOUT);
    UpdateRequest workRequest = new UpdateRequest();
    UpdateRequest dataRequest = new UpdateRequest();
    for (int i = 0; i < 500; i++) {
        workRequest.add(id, String.valueOf(i), "expr_s", "update(destination, batchSize=50, search(mainCorpus, q=id:" + i + ", rows=1, sort=\"id asc\", fl=\"id, body_t, field_i\"))");
        dataRequest.add(id, String.valueOf(i), "body_t", "hello world " + i, "field_i", Integer.toString(i));
    }
    workRequest.commit(cluster.getSolrClient(), "workQueue");
    dataRequest.commit(cluster.getSolrClient(), "mainCorpus");
    String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString() + "/destination";
    TupleStream executorStream;
    ModifiableSolrParams paramsLoc;
    StreamFactory factory = new StreamFactory().withCollectionZkHost("workQueue", cluster.getZkServer().getZkAddress()).withCollectionZkHost("mainCorpus", cluster.getZkServer().getZkAddress()).withCollectionZkHost("destination", cluster.getZkServer().getZkAddress()).withFunctionName("search", CloudSolrStream.class).withFunctionName("executor", ExecutorStream.class).withFunctionName("update", UpdateStream.class);
    String executorExpression = "executor(threads=3, search(workQueue, q=\"*:*\", fl=\"id, expr_s\", rows=1000, sort=\"id desc\"))";
    executorStream = factory.constructStream(executorExpression);
    StreamContext context = new StreamContext();
    SolrClientCache clientCache = new SolrClientCache();
    context.setSolrClientCache(clientCache);
    executorStream.setStreamContext(context);
    getTuples(executorStream);
    //Destination collection should now contain all the records in the main corpus.
    cluster.getSolrClient().commit("destination");
    paramsLoc = new ModifiableSolrParams();
    paramsLoc.set("expr", "search(destination, q=\"*:*\", fl=\"id, body_t, field_i\", rows=1000, sort=\"field_i asc\")");
    paramsLoc.set("qt", "/stream");
    SolrStream solrStream = new SolrStream(url, paramsLoc);
    List<Tuple> tuples = getTuples(solrStream);
    assertTrue(tuples.size() == 500);
    for (int i = 0; i < 500; i++) {
        Tuple tuple = tuples.get(i);
        long ivalue = tuple.getLong("field_i");
        String body = tuple.getString("body_t");
        assertTrue(ivalue == i);
        assertTrue(body.equals("hello world " + i));
    }
    solrStream.close();
    clientCache.close();
    CollectionAdminRequest.deleteCollection("workQueue").process(cluster.getSolrClient());
    CollectionAdminRequest.deleteCollection("mainCorpus").process(cluster.getSolrClient());
    CollectionAdminRequest.deleteCollection("destination").process(cluster.getSolrClient());
}

Example 34 with SolrClientCache

use of org.apache.solr.client.solrj.io.SolrClientCache in project lucene-solr by apache.

the class FeaturesSelectionStream method open.

/**
   * Opens the CloudSolrStream
   *
   ***/
public void open() throws IOException {
    if (cache == null) {
        isCloseCache = true;
        cache = new SolrClientCache();
    } else {
        isCloseCache = false;
    }
    this.cloudSolrClient = this.cache.getCloudSolrClient(zkHost);
    this.executorService = ExecutorUtil.newMDCAwareCachedThreadPool(new SolrjNamedThreadFactory("FeaturesSelectionStream"));
}

Example 35 with SolrClientCache

use of org.apache.solr.client.solrj.io.SolrClientCache in project lucene-solr by apache.

the class StreamingTest method testMergeStream.

@Test
public void testMergeStream() throws Exception {
    new UpdateRequest().add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0").add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0").add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3").add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4").add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
    StreamContext streamContext = new StreamContext();
    SolrClientCache solrClientCache = new SolrClientCache();
    streamContext.setSolrClientCache(solrClientCache);
    try {
        //Test ascending
        SolrParams sParamsA = mapParams("q", "id:(4 1)", "fl", "id,a_s,a_i", "sort", "a_i asc");
        CloudSolrStream streamA = new CloudSolrStream(zkHost, COLLECTIONORALIAS, sParamsA);
        SolrParams sParamsB = mapParams("q", "id:(0 2 3)", "fl", "id,a_s,a_i", "sort", "a_i asc");
        CloudSolrStream streamB = new CloudSolrStream(zkHost, COLLECTIONORALIAS, sParamsB);
        MergeStream mstream = new MergeStream(streamA, streamB, new FieldComparator("a_i", ComparatorOrder.ASCENDING));
        mstream.setStreamContext(streamContext);
        List<Tuple> tuples = getTuples(mstream);
        assertEquals(5, tuples.size());
        assertOrder(tuples, 0, 1, 2, 3, 4);
        //Test descending
        sParamsA = mapParams("q", "id:(4 1)", "fl", "id,a_s,a_i", "sort", "a_i desc");
        streamA = new CloudSolrStream(zkHost, COLLECTIONORALIAS, sParamsA);
        sParamsB = mapParams("q", "id:(0 2 3)", "fl", "id,a_s,a_i", "sort", "a_i desc");
        streamB = new CloudSolrStream(zkHost, COLLECTIONORALIAS, sParamsB);
        mstream = new MergeStream(streamA, streamB, new FieldComparator("a_i", ComparatorOrder.DESCENDING));
        mstream.setStreamContext(streamContext);
        tuples = getTuples(mstream);
        assertEquals(5, tuples.size());
        assertOrder(tuples, 4, 3, 2, 1, 0);
        //Test compound sort
        sParamsA = mapParams("q", "id:(2 4 1)", "fl", "id,a_s,a_i,a_f", "sort", "a_f asc,a_i asc");
        streamA = new CloudSolrStream(zkHost, COLLECTIONORALIAS, sParamsA);
        sParamsB = mapParams("q", "id:(0 3)", "fl", "id,a_s,a_i,a_f", "sort", "a_f asc,a_i asc");
        streamB = new CloudSolrStream(zkHost, COLLECTIONORALIAS, sParamsB);
        mstream = new MergeStream(streamA, streamB, new MultipleFieldComparator(new FieldComparator("a_f", ComparatorOrder.ASCENDING), new FieldComparator("a_i", ComparatorOrder.ASCENDING)));
        mstream.setStreamContext(streamContext);
        tuples = getTuples(mstream);
        assertEquals(5, tuples.size());
        assertOrder(tuples, 0, 2, 1, 3, 4);
        sParamsA = mapParams("q", "id:(2 4 1)", "fl", "id,a_s,a_i,a_f", "sort", "a_f asc,a_i desc");
        streamA = new CloudSolrStream(zkHost, COLLECTIONORALIAS, sParamsA);
        sParamsB = mapParams("q", "id:(0 3)", "fl", "id,a_s,a_i,a_f", "sort", "a_f asc,a_i desc");
        streamB = new CloudSolrStream(zkHost, COLLECTIONORALIAS, sParamsB);
        mstream = new MergeStream(streamA, streamB, new MultipleFieldComparator(new FieldComparator("a_f", ComparatorOrder.ASCENDING), new FieldComparator("a_i", ComparatorOrder.DESCENDING)));
        mstream.setStreamContext(streamContext);
        tuples = getTuples(mstream);
        assertEquals(5, tuples.size());
        assertOrder(tuples, 2, 0, 1, 3, 4);
    } finally {
        solrClientCache.close();
    }
}