Examples with StreamExpression org.apache.solr.client.solrj.io.stream.expr.StreamExpression used on opensource projects

Example 56 with StreamExpression

use of org.apache.solr.client.solrj.io.stream.expr.StreamExpression in project lucene-solr by apache.

the class StreamExpressionTest method testParallelRollupStream.

@Test
public void testParallelRollupStream() throws Exception {
    new UpdateRequest().add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1").add(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2").add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3").add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4").add(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5").add(id, "5", "a_s", "hello3", "a_i", "10", "a_f", "6").add(id, "6", "a_s", "hello4", "a_i", "11", "a_f", "7").add(id, "7", "a_s", "hello3", "a_i", "12", "a_f", "8").add(id, "8", "a_s", "hello3", "a_i", "13", "a_f", "9").add(id, "9", "a_s", "hello0", "a_i", "14", "a_f", "10").commit(cluster.getSolrClient(), COLLECTIONORALIAS);
    StreamFactory factory = new StreamFactory().withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress()).withFunctionName("search", CloudSolrStream.class).withFunctionName("parallel", ParallelStream.class).withFunctionName("rollup", RollupStream.class).withFunctionName("sum", SumMetric.class).withFunctionName("min", MinMetric.class).withFunctionName("max", MaxMetric.class).withFunctionName("avg", MeanMetric.class).withFunctionName("count", CountMetric.class);
    StreamContext streamContext = new StreamContext();
    SolrClientCache solrClientCache = new SolrClientCache();
    streamContext.setSolrClientCache(solrClientCache);
    StreamExpression expression;
    TupleStream stream;
    List<Tuple> tuples;
    try {
        expression = StreamExpressionParser.parse("parallel(" + COLLECTIONORALIAS + "," + "rollup(" + "search(" + COLLECTIONORALIAS + ", q=*:*, fl=\"a_s,a_i,a_f\", sort=\"a_s asc\", partitionKeys=\"a_s\")," + "over=\"a_s\"," + "sum(a_i)," + "sum(a_f)," + "min(a_i)," + "min(a_f)," + "max(a_i)," + "max(a_f)," + "avg(a_i)," + "avg(a_f)," + "count(*)" + ")," + "workers=\"2\", zkHost=\"" + cluster.getZkServer().getZkAddress() + "\", sort=\"a_s asc\")");
        stream = factory.constructStream(expression);
        stream.setStreamContext(streamContext);
        tuples = getTuples(stream);
        assert (tuples.size() == 3);
        //Test Long and Double Sums
        Tuple tuple = tuples.get(0);
        String bucket = tuple.getString("a_s");
        Double sumi = tuple.getDouble("sum(a_i)");
        Double sumf = tuple.getDouble("sum(a_f)");
        Double mini = tuple.getDouble("min(a_i)");
        Double minf = tuple.getDouble("min(a_f)");
        Double maxi = tuple.getDouble("max(a_i)");
        Double maxf = tuple.getDouble("max(a_f)");
        Double avgi = tuple.getDouble("avg(a_i)");
        Double avgf = tuple.getDouble("avg(a_f)");
        Double count = tuple.getDouble("count(*)");
        assertTrue(bucket.equals("hello0"));
        assertTrue(sumi.doubleValue() == 17.0D);
        assertTrue(sumf.doubleValue() == 18.0D);
        assertTrue(mini.doubleValue() == 0.0D);
        assertTrue(minf.doubleValue() == 1.0D);
        assertTrue(maxi.doubleValue() == 14.0D);
        assertTrue(maxf.doubleValue() == 10.0D);
        assertTrue(avgi.doubleValue() == 4.25D);
        assertTrue(avgf.doubleValue() == 4.5D);
        assertTrue(count.doubleValue() == 4);
        tuple = tuples.get(1);
        bucket = tuple.getString("a_s");
        sumi = tuple.getDouble("sum(a_i)");
        sumf = tuple.getDouble("sum(a_f)");
        mini = tuple.getDouble("min(a_i)");
        minf = tuple.getDouble("min(a_f)");
        maxi = tuple.getDouble("max(a_i)");
        maxf = tuple.getDouble("max(a_f)");
        avgi = tuple.getDouble("avg(a_i)");
        avgf = tuple.getDouble("avg(a_f)");
        count = tuple.getDouble("count(*)");
        assertTrue(bucket.equals("hello3"));
        assertTrue(sumi.doubleValue() == 38.0D);
        assertTrue(sumf.doubleValue() == 26.0D);
        assertTrue(mini.doubleValue() == 3.0D);
        assertTrue(minf.doubleValue() == 3.0D);
        assertTrue(maxi.doubleValue() == 13.0D);
        assertTrue(maxf.doubleValue() == 9.0D);
        assertTrue(avgi.doubleValue() == 9.5D);
        assertTrue(avgf.doubleValue() == 6.5D);
        assertTrue(count.doubleValue() == 4);
        tuple = tuples.get(2);
        bucket = tuple.getString("a_s");
        sumi = tuple.getDouble("sum(a_i)");
        sumf = tuple.getDouble("sum(a_f)");
        mini = tuple.getDouble("min(a_i)");
        minf = tuple.getDouble("min(a_f)");
        maxi = tuple.getDouble("max(a_i)");
        maxf = tuple.getDouble("max(a_f)");
        avgi = tuple.getDouble("avg(a_i)");
        avgf = tuple.getDouble("avg(a_f)");
        count = tuple.getDouble("count(*)");
        assertTrue(bucket.equals("hello4"));
        assertTrue(sumi.longValue() == 15);
        assertTrue(sumf.doubleValue() == 11.0D);
        assertTrue(mini.doubleValue() == 4.0D);
        assertTrue(minf.doubleValue() == 4.0D);
        assertTrue(maxi.doubleValue() == 11.0D);
        assertTrue(maxf.doubleValue() == 7.0D);
        assertTrue(avgi.doubleValue() == 7.5D);
        assertTrue(avgf.doubleValue() == 5.5D);
        assertTrue(count.doubleValue() == 2);
    } finally {
        solrClientCache.close();
    }
}

Example 57 with StreamExpression

use of org.apache.solr.client.solrj.io.stream.expr.StreamExpression in project lucene-solr by apache.

the class StreamHandler method handleRequestBody.

public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
    SolrParams params = req.getParams();
    params = adjustParams(params);
    req.setParams(params);
    if (params.get("action") != null) {
        handleAdmin(req, rsp, params);
        return;
    }
    TupleStream tupleStream;
    try {
        StreamExpression streamExpression = StreamExpressionParser.parse(params.get("expr"));
        if (this.streamFactory.isEvaluator(streamExpression)) {
            StreamExpression tupleExpression = new StreamExpression("tuple");
            tupleExpression.addParameter(new StreamExpressionNamedParameter("return-value", streamExpression));
            tupleStream = this.streamFactory.constructStream(tupleExpression);
        } else {
            tupleStream = this.streamFactory.constructStream(streamExpression);
        }
    } catch (Exception e) {
        //Catch exceptions that occur while the stream is being created. This will include streaming expression parse rules.
        SolrException.log(logger, e);
        rsp.add("result-set", new DummyErrorStream(e));
        return;
    }
    int worker = params.getInt("workerID", 0);
    int numWorkers = params.getInt("numWorkers", 1);
    StreamContext context = new StreamContext();
    context.put("shards", getCollectionShards(params));
    context.workerID = worker;
    context.numWorkers = numWorkers;
    context.setSolrClientCache(clientCache);
    context.setModelCache(modelCache);
    context.put("core", this.coreName);
    context.put("solr-core", req.getCore());
    tupleStream.setStreamContext(context);
    // if asking for explanation then go get it
    if (params.getBool("explain", false)) {
        rsp.add("explanation", tupleStream.toExplanation(this.streamFactory));
    }
    if (tupleStream instanceof DaemonStream) {
        DaemonStream daemonStream = (DaemonStream) tupleStream;
        if (daemons.containsKey(daemonStream.getId())) {
            daemons.remove(daemonStream.getId()).close();
        }
        daemonStream.setDaemons(daemons);
        //This will start the deamonStream
        daemonStream.open();
        daemons.put(daemonStream.getId(), daemonStream);
        rsp.add("result-set", new DaemonResponseStream("Deamon:" + daemonStream.getId() + " started on " + coreName));
    } else {
        rsp.add("result-set", new TimerStream(new ExceptionStream(tupleStream)));
    }
}

Example 58 with StreamExpression

use of org.apache.solr.client.solrj.io.stream.expr.StreamExpression in project lucene-solr by apache.

the class StreamExpressionParserTest method testParsing.

@Test
public void testParsing() throws Exception {
    StreamExpression actual, expected;
    actual = StreamExpressionParser.parse("aliases(a_i=alias.a_i)");
    expected = new StreamExpression("aliases").withParameter(new StreamExpressionNamedParameter("a_i", "alias.a_i"));
    assertEquals(expected, actual);
    actual = StreamExpressionParser.parse("search(a,b)");
    expected = new StreamExpression("search").withParameter("a").withParameter("b");
    assertEquals(expected, actual);
    actual = StreamExpressionParser.parse("search(collection1, q=*:*, sort=\"fieldA desc, fieldB asc, fieldC asc\")");
    expected = new StreamExpression("search").withParameter(new StreamExpressionValue("collection1")).withParameter(new StreamExpressionNamedParameter("q").withParameter("*:*")).withParameter(new StreamExpressionNamedParameter("sort").withParameter("fieldA desc, fieldB asc, fieldC asc"));
    assertEquals(expected, actual);
    actual = StreamExpressionParser.parse("unique(search(collection1, q=*:*, sort=\"fieldA desc, fieldB asc, fieldC asc\"))");
    expected = new StreamExpression("unique").withParameter(new StreamExpression("search").withParameter(new StreamExpressionValue("collection1")).withParameter(new StreamExpressionNamedParameter("q").withParameter("*:*")).withParameter(new StreamExpressionNamedParameter("sort").withParameter("fieldA desc, fieldB asc, fieldC asc")));
    assertEquals(expected, actual);
    actual = StreamExpressionParser.parse("unique(search(collection1, q=*:*, sort=\"fieldA desc, fieldB asc, fieldC asc\"), alt=search(collection1, foo=bar))");
    expected = new StreamExpression("unique").withParameter(new StreamExpression("search").withParameter(new StreamExpressionValue("collection1")).withParameter(new StreamExpressionNamedParameter("q").withParameter("*:*")).withParameter(new StreamExpressionNamedParameter("sort").withParameter("fieldA desc, fieldB asc, fieldC asc"))).withParameter(new StreamExpressionNamedParameter("alt").withParameter(new StreamExpression("search").withParameter("collection1").withParameter(new StreamExpressionNamedParameter("foo").withParameter("bar"))));
    assertEquals(expected, actual);
    actual = StreamExpressionParser.parse("innerJoin(" + "left=search(collection1, q=*:*, fl=\"fieldA,fieldB,fieldC\", sort=\"fieldA asc, fieldB asc\")," + "right=search(collection2, q=*:*, fl=\"fieldA,fieldD\", sort=fieldA asc)," + "on(equals(fieldA), notEquals(fieldC,fieldD))" + ")");
    expected = new StreamExpression("innerJoin").withParameter(new StreamExpressionNamedParameter("left").withParameter(new StreamExpression("search").withParameter("collection1").withParameter(new StreamExpressionNamedParameter("q", "*:*")).withParameter(new StreamExpressionNamedParameter("fl", "fieldA,fieldB,fieldC")).withParameter(new StreamExpressionNamedParameter("sort", "fieldA asc, fieldB asc")))).withParameter(new StreamExpressionNamedParameter("right").withParameter(new StreamExpression("search").withParameter("collection2").withParameter(new StreamExpressionNamedParameter("q", "*:*")).withParameter(new StreamExpressionNamedParameter("fl", "fieldA,fieldD")).withParameter(new StreamExpressionNamedParameter("sort", "fieldA asc")))).withParameter(new StreamExpression("on").withParameter(new StreamExpression("equals").withParameter("fieldA")).withParameter(new StreamExpression("notEquals").withParameter("fieldC").withParameter("fieldD")));
    assertEquals(expected, actual);
}

Example 59 with StreamExpression

use of org.apache.solr.client.solrj.io.stream.expr.StreamExpression in project lucene-solr by apache.

the class TopicStream method toExpression.

@Override
public StreamExpression toExpression(StreamFactory factory) throws IOException {
    // function name
    StreamExpression expression = new StreamExpression(factory.getFunctionName(this.getClass()));
    expression.addParameter(checkpointCollection);
    // collection
    expression.addParameter(collection);
    ModifiableSolrParams mParams = new ModifiableSolrParams(params);
    for (Entry<String, String[]> param : mParams.getMap().entrySet()) {
        String value = String.join(",", param.getValue());
        // SOLR-8409: This is a special case where the params contain a " character
        // Do note that in any other BASE streams with parameters where a " might come into play
        // that this same replacement needs to take place.
        value = value.replace("\"", "\\\"");
        expression.addParameter(new StreamExpressionNamedParameter(param.getKey(), value));
    }
    // zkHost
    expression.addParameter(new StreamExpressionNamedParameter("zkHost", zkHost));
    expression.addParameter(new StreamExpressionNamedParameter(ID, id));
    if (initialCheckpoint > -1) {
        expression.addParameter(new StreamExpressionNamedParameter("initialCheckpoint", Long.toString(initialCheckpoint)));
    }
    expression.addParameter(new StreamExpressionNamedParameter("checkpointEvery", Long.toString(checkpointEvery)));
    return expression;
}

Example 60 with StreamExpression

use of org.apache.solr.client.solrj.io.stream.expr.StreamExpression in project lucene-solr by apache.

the class SqlStream method toExpression.

@Override
public StreamExpression toExpression(StreamFactory factory) throws IOException {
    // functionName(collectionName, param1, param2, ..., paramN, sort="comp", [aliases="field=alias,..."])
    // function name
    StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
    // collection
    expression.addParameter(collection);
    // parameters
    ModifiableSolrParams mParams = new ModifiableSolrParams(SolrParams.toMultiMap(params.toNamedList()));
    for (Entry<String, String[]> param : mParams.getMap().entrySet()) {
        String value = String.join(",", param.getValue());
        // SOLR-8409: This is a special case where the params contain a " character
        // Do note that in any other BASE streams with parameters where a " might come into play
        // that this same replacement needs to take place.
        value = value.replace("\"", "\\\"");
        expression.addParameter(new StreamExpressionNamedParameter(param.getKey(), value));
    }
    // zkHost
    expression.addParameter(new StreamExpressionNamedParameter("zkHost", zkHost));
    return expression;
}