Examples with TupleStream org.apache.solr.client.solrj.io.stream.TupleStream used on opensource projects

Example 1 with TupleStream

use of org.apache.solr.client.solrj.io.stream.TupleStream in project lucene-solr by apache.

the class TestJavabinTupleStreamParser method testSimple.

public void testSimple() throws IOException {
    List<Map<String, Object>> l = new ArrayList();
    l.add(Utils.makeMap("id", 1, "f", 1.0f, "s", "Some str 1"));
    l.add(Utils.makeMap("id", 2, "f", 2.0f, "s", "Some str 2"));
    l.add(Utils.makeMap("id", 3, "f", 1.0f, "s", "Some str 3"));
    l.add(Utils.makeMap("EOF", true, "RESPONSE_TIME", 206, "sleepMillis", 1000));
    Iterator<Map<String, Object>> iterator = l.iterator();
    TupleStream tupleStream = new TupleStream() {

        @Override
        public void setStreamContext(StreamContext context) {
        }

        @Override
        public List<TupleStream> children() {
            return null;
        }

        @Override
        public void open() throws IOException {
        }

        @Override
        public void close() throws IOException {
        }

        @Override
        public Tuple read() throws IOException {
            if (iterator.hasNext())
                return new Tuple(iterator.next());
            else
                return null;
        }

        @Override
        public StreamComparator getStreamSort() {
            return null;
        }

        @Override
        public Explanation toExplanation(StreamFactory factory) throws IOException {
            return new StreamExplanation(getStreamNodeId().toString()).withFunctionName("Dummy").withImplementingClass(this.getClass().getName()).withExpressionType(Explanation.ExpressionType.STREAM_SOURCE).withExpression("--non-expressible--");
        }
    };
    byte[] bytes = serialize(tupleStream);
    JavabinTupleStreamParser parser = new JavabinTupleStreamParser(new ByteArrayInputStream(bytes), true);
    Map m = parser.next();
    assertEquals(1L, m.get("id"));
    assertEquals(1.0, (Double) m.get("f"), 0.01);
    m = parser.next();
    assertEquals(2L, m.get("id"));
    assertEquals(2.0, (Double) m.get("f"), 0.01);
    m = parser.next();
    assertEquals(3L, m.get("id"));
    assertEquals(1.0, (Double) m.get("f"), 0.01);
    m = parser.next();
    assertEquals(Boolean.TRUE, m.get("EOF"));
    parser = new JavabinTupleStreamParser(new ByteArrayInputStream(bytes), false);
    m = parser.next();
    assertEquals(1, m.get("id"));
    assertEquals(1.0, (Float) m.get("f"), 0.01);
    m = parser.next();
    assertEquals(2, m.get("id"));
    assertEquals(2.0, (Float) m.get("f"), 0.01);
    m = parser.next();
    assertEquals(3, m.get("id"));
    assertEquals(1.0, (Float) m.get("f"), 0.01);
    m = parser.next();
    assertEquals(Boolean.TRUE, m.get("EOF"));
}

Example 2 with TupleStream

use of org.apache.solr.client.solrj.io.stream.TupleStream in project lucene-solr by apache.

the class SQLHandler method handleRequestBody.

public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
    ModifiableSolrParams params = new ModifiableSolrParams(req.getParams());
    params = adjustParams(params);
    req.setParams(params);
    String sql = params.get("stmt");
    // Set defaults for parameters
    params.set("numWorkers", params.getInt("numWorkers", 1));
    params.set("workerCollection", params.get("workerCollection", defaultWorkerCollection));
    params.set("workerZkhost", params.get("workerZkhost", defaultZkhost));
    params.set("aggregationMode", params.get("aggregationMode", "facet"));
    TupleStream tupleStream = null;
    try {
        if (!isCloud) {
            throw new IllegalStateException(sqlNonCloudErrorMsg);
        }
        if (sql == null) {
            throw new Exception("stmt parameter cannot be null");
        }
        String url = CalciteSolrDriver.CONNECT_STRING_PREFIX;
        Properties properties = new Properties();
        // Add all query parameters
        Iterator<String> parameterNamesIterator = params.getParameterNamesIterator();
        while (parameterNamesIterator.hasNext()) {
            String param = parameterNamesIterator.next();
            properties.setProperty(param, params.get(param));
        }
        // Set these last to ensure that they are set properly
        properties.setProperty("lex", Lex.MYSQL.toString());
        properties.setProperty("zk", defaultZkhost);
        String driverClass = CalciteSolrDriver.class.getCanonicalName();
        // JDBC driver requires metadata from the SQLHandler. Default to false since this adds a new Metadata stream.
        boolean includeMetadata = params.getBool("includeMetadata", false);
        tupleStream = new SqlHandlerStream(url, sql, null, properties, driverClass, includeMetadata);
        tupleStream = new StreamHandler.TimerStream(new ExceptionStream(tupleStream));
        rsp.add("result-set", tupleStream);
    } catch (Exception e) {
        //Catch the SQL parsing and query transformation exceptions.
        if (tupleStream != null) {
            tupleStream.close();
        }
        SolrException.log(logger, e);
        rsp.add("result-set", new StreamHandler.DummyErrorStream(e));
    }
}

Example 3 with TupleStream

use of org.apache.solr.client.solrj.io.stream.TupleStream in project lucene-solr by apache.

the class TestGraphMLResponseWriter method testGraphMLOutput.

@Test
public void testGraphMLOutput() throws Exception {
    // Just need a request to attach the stream and traversal to.
    SolrQueryRequest request = req("blah", "blah");
    SolrQueryResponse response = new SolrQueryResponse();
    Map context = request.getContext();
    //Simulates a GatherNodesStream
    TupleStream stream = new TestStream();
    Traversal traversal = new Traversal();
    context.put("traversal", traversal);
    context.put("stream", stream);
    StringWriter writer = new StringWriter();
    GraphMLResponseWriter graphMLResponseWriter = new GraphMLResponseWriter();
    graphMLResponseWriter.write(writer, request, response);
    String graphML = writer.toString();
    //Validate the nodes
    String error = h.validateXPath(graphML, "//graph/node[1][@id ='bill']", "//graph/node[2][@id ='jim']", "//graph/node[3][@id ='max']");
    if (error != null) {
        throw new Exception(error);
    }
    //Validate the edges
    error = h.validateXPath(graphML, "//graph/edge[1][@source ='jim']", "//graph/edge[1][@target ='bill']", "//graph/edge[2][@source ='max']", "//graph/edge[2][@target ='bill']", "//graph/edge[3][@source ='max']", "//graph/edge[3][@target ='jim']", "//graph/edge[4][@source ='jim']", "//graph/edge[4][@target ='max']");
    if (error != null) {
        throw new Exception(error);
    }
}

Example 4 with TupleStream

use of org.apache.solr.client.solrj.io.stream.TupleStream in project lucene-solr by apache.

the class TestSQLHandler method testSelectDistinct.

private void testSelectDistinct() throws Exception {
    try {
        CloudJettyRunner jetty = this.cloudJettys.get(0);
        del("*:*");
        commit();
        indexr("id", "1", "text", "XXXX XXXX", "str_s", "a", "field_i", "1");
        indexr("id", "2", "text", "XXXX XXXX", "str_s", "b", "field_i", "2");
        indexr("id", "3", "text", "XXXX XXXX", "str_s", "a", "field_i", "20");
        indexr("id", "4", "text", "XXXX XXXX", "str_s", "b", "field_i", "2");
        indexr("id", "5", "text", "XXXX XXXX", "str_s", "c", "field_i", "30");
        indexr("id", "6", "text", "XXXX XXXX", "str_s", "c", "field_i", "30");
        indexr("id", "7", "text", "XXXX XXXX", "str_s", "c", "field_i", "50");
        indexr("id", "8", "text", "XXXX XXXX", "str_s", "c", "field_i", "60");
        commit();
        SolrParams sParams = mapParams(CommonParams.QT, "/sql", "aggregationMode", "map_reduce", "stmt", "select distinct str_s, field_i from collection1 order by str_s asc, field_i asc");
        System.out.println("##################### testSelectDistinct()");
        TupleStream solrStream = new SolrStream(jetty.url, sParams);
        List<Tuple> tuples = getTuples(solrStream);
        assert (tuples.size() == 6);
        Tuple tuple = tuples.get(0);
        assert (tuple.get("str_s").equals("a"));
        assert (tuple.getLong("field_i") == 1);
        tuple = tuples.get(1);
        assert (tuple.get("str_s").equals("a"));
        assert (tuple.getLong("field_i") == 20);
        tuple = tuples.get(2);
        assert (tuple.get("str_s").equals("b"));
        assert (tuple.getLong("field_i") == 2);
        tuple = tuples.get(3);
        assert (tuple.get("str_s").equals("c"));
        assert (tuple.getLong("field_i") == 30);
        tuple = tuples.get(4);
        assert (tuple.get("str_s").equals("c"));
        assert (tuple.getLong("field_i") == 50);
        tuple = tuples.get(5);
        assert (tuple.get("str_s").equals("c"));
        assert (tuple.getLong("field_i") == 60);
        //reverse the sort
        sParams = mapParams(CommonParams.QT, "/sql", "aggregationMode", "map_reduce", "stmt", "select distinct str_s, field_i from collection1 order by str_s desc, field_i desc");
        solrStream = new SolrStream(jetty.url, sParams);
        tuples = getTuples(solrStream);
        assert (tuples.size() == 6);
        tuple = tuples.get(0);
        assert (tuple.get("str_s").equals("c"));
        assert (tuple.getLong("field_i") == 60);
        tuple = tuples.get(1);
        assert (tuple.get("str_s").equals("c"));
        assert (tuple.getLong("field_i") == 50);
        tuple = tuples.get(2);
        assert (tuple.get("str_s").equals("c"));
        assert (tuple.getLong("field_i") == 30);
        tuple = tuples.get(3);
        assert (tuple.get("str_s").equals("b"));
        assert (tuple.getLong("field_i") == 2);
        tuple = tuples.get(4);
        assert (tuple.get("str_s").equals("a"));
        assert (tuple.getLong("field_i") == 20);
        tuple = tuples.get(5);
        assert (tuple.get("str_s").equals("a"));
        assert (tuple.getLong("field_i") == 1);
        sParams = mapParams(CommonParams.QT, "/sql", "aggregationMode", "map_reduce", "stmt", "select distinct str_s as myString, field_i from collection1 order by myString desc, field_i desc");
        solrStream = new SolrStream(jetty.url, sParams);
        tuples = getTuples(solrStream);
        assert (tuples.size() == 6);
        tuple = tuples.get(0);
        assert (tuple.get("myString").equals("c"));
        assert (tuple.getLong("field_i") == 60);
        tuple = tuples.get(1);
        assert (tuple.get("myString").equals("c"));
        assert (tuple.getLong("field_i") == 50);
        tuple = tuples.get(2);
        assert (tuple.get("myString").equals("c"));
        assert (tuple.getLong("field_i") == 30);
        tuple = tuples.get(3);
        assert (tuple.get("myString").equals("b"));
        assert (tuple.getLong("field_i") == 2);
        tuple = tuples.get(4);
        assert (tuple.get("myString").equals("a"));
        assert (tuple.getLong("field_i") == 20);
        tuple = tuples.get(5);
        assert (tuple.get("myString").equals("a"));
        assert (tuple.getLong("field_i") == 1);
        //test with limit
        sParams = mapParams(CommonParams.QT, "/sql", "aggregationMode", "map_reduce", "stmt", "select distinct str_s, field_i from collection1 order by str_s desc, field_i desc limit 2");
        solrStream = new SolrStream(jetty.url, sParams);
        tuples = getTuples(solrStream);
        assert (tuples.size() == 2);
        tuple = tuples.get(0);
        assert (tuple.get("str_s").equals("c"));
        assert (tuple.getLong("field_i") == 60);
        tuple = tuples.get(1);
        assert (tuple.get("str_s").equals("c"));
        assert (tuple.getLong("field_i") == 50);
        // Test without a sort. Sort should be asc by default.
        sParams = mapParams(CommonParams.QT, "/sql", "aggregationMode", "map_reduce", "stmt", "select distinct str_s, field_i from collection1");
        solrStream = new SolrStream(jetty.url, sParams);
        tuples = getTuples(solrStream);
        assert (tuples.size() == 6);
        tuple = tuples.get(0);
        assert (tuple.get("str_s").equals("a"));
        assert (tuple.getLong("field_i") == 1);
        tuple = tuples.get(1);
        assert (tuple.get("str_s").equals("a"));
        assert (tuple.getLong("field_i") == 20);
        tuple = tuples.get(2);
        assert (tuple.get("str_s").equals("b"));
        assert (tuple.getLong("field_i") == 2);
        tuple = tuples.get(3);
        assert (tuple.get("str_s").equals("c"));
        assert (tuple.getLong("field_i") == 30);
        tuple = tuples.get(4);
        assert (tuple.get("str_s").equals("c"));
        assert (tuple.getLong("field_i") == 50);
        tuple = tuples.get(5);
        assert (tuple.get("str_s").equals("c"));
        assert (tuple.getLong("field_i") == 60);
        // Test with a predicate.
        sParams = mapParams(CommonParams.QT, "/sql", "aggregationMode", "map_reduce", "stmt", "select distinct str_s, field_i from collection1 where str_s = 'a'");
        solrStream = new SolrStream(jetty.url, sParams);
        tuples = getTuples(solrStream);
        assert (tuples.size() == 2);
        tuple = tuples.get(0);
        assert (tuple.get("str_s").equals("a"));
        assert (tuple.getLong("field_i") == 1);
        tuple = tuples.get(1);
        assert (tuple.get("str_s").equals("a"));
        assert (tuple.getLong("field_i") == 20);
    } finally {
        delete();
    }
}

Example 5 with TupleStream

use of org.apache.solr.client.solrj.io.stream.TupleStream in project lucene-solr by apache.

the class GraphExpressionTest method testScoreNodesFacetStream.

@Test
public void testScoreNodesFacetStream() throws Exception {
    new UpdateRequest().add(id, "0", "basket_s", "basket1", "product_ss", "product1", "product_ss", "product3", "product_ss", "product5", "price_f", "1").add(id, "3", "basket_s", "basket2", "product_ss", "product1", "product_ss", "product6", "product_ss", "product7", "price_f", "1").add(id, "6", "basket_s", "basket3", "product_ss", "product4", "product_ss", "product3", "product_ss", "product1", "price_f", "1").add(id, "9", "basket_s", "basket4", "product_ss", "product4", "product_ss", "product3", "product_ss", "product1", "price_f", "1").commit(cluster.getSolrClient(), COLLECTION);
    List<Tuple> tuples = null;
    TupleStream stream = null;
    StreamContext context = new StreamContext();
    SolrClientCache cache = new SolrClientCache();
    context.setSolrClientCache(cache);
    StreamFactory factory = new StreamFactory().withCollectionZkHost("collection1", cluster.getZkServer().getZkAddress()).withDefaultZkHost(cluster.getZkServer().getZkAddress()).withFunctionName("gatherNodes", GatherNodesStream.class).withFunctionName("scoreNodes", ScoreNodesStream.class).withFunctionName("search", CloudSolrStream.class).withFunctionName("facet", FacetStream.class).withFunctionName("sort", SortStream.class).withFunctionName("count", CountMetric.class).withFunctionName("avg", MeanMetric.class).withFunctionName("sum", SumMetric.class).withFunctionName("min", MinMetric.class).withFunctionName("max", MaxMetric.class);
    String expr = "sort(by=\"nodeScore desc\",scoreNodes(facet(collection1, q=\"product_ss:product3\", buckets=\"product_ss\", bucketSorts=\"count(*) desc\", bucketSizeLimit=100, count(*))))";
    stream = factory.constructStream(expr);
    context = new StreamContext();
    context.setSolrClientCache(cache);
    stream.setStreamContext(context);
    tuples = getTuples(stream);
    Tuple tuple = tuples.get(0);
    assert (tuple.getString("node").equals("product3"));
    assert (tuple.getLong("docFreq") == 3);
    assert (tuple.getLong("count(*)") == 3);
    Tuple tuple0 = tuples.get(1);
    assert (tuple0.getString("node").equals("product4"));
    assert (tuple0.getLong("docFreq") == 2);
    assert (tuple0.getLong("count(*)") == 2);
    Tuple tuple1 = tuples.get(2);
    assert (tuple1.getString("node").equals("product1"));
    assert (tuple1.getLong("docFreq") == 4);
    assert (tuple1.getLong("count(*)") == 3);
    Tuple tuple2 = tuples.get(3);
    assert (tuple2.getString("node").equals("product5"));
    assert (tuple2.getLong("docFreq") == 1);
    assert (tuple2.getLong("count(*)") == 1);
    cache.close();
}