Examples with Query net.opentsdb.query.pojo.Query used on opensource projects

Example 1 with Query

use of net.opentsdb.query.pojo.Query in project opentsdb by OpenTSDB.

the class QueryExecutor method execute.

/**
   * Execute the RPC and serialize the response
   * @param query The HTTP query to parse and and return results to
   */
public void execute(final HttpQuery query) {
    http_query = query;
    final QueryStats query_stats = new QueryStats(query.getRemoteAddress(), ts_query, query.getHeaders());
    ts_query.setQueryStats(query_stats);
    /**
     * Sends the serialized results to the caller. This should be the very
     * last callback executed.
     */
    class CompleteCB implements Callback<Object, ChannelBuffer> {

        @Override
        public Object call(final ChannelBuffer cb) throws Exception {
            query.sendReply(cb);
            return null;
        }
    }
    /**
     * After all of the queries have run and we have data (or not) then we
     * need to compile the iterators.
     * This class could probably be improved:
     * First we iterate over the results AND for each result, iterate over
     * the expressions, giving a time synced iterator to each expression that 
     * needs the result set.
     * THEN we iterate over the expressions again and build a DAG to determine
     * if any of the expressions require the output of an expression. If so
     * then we add the expressions to the proper parent and compile them in
     * order.
     * After all of that we're ready to start serializing and iterating
     * over the results.
     */
    class QueriesCB implements Callback<Object, ArrayList<DataPoints[]>> {

        public Object call(final ArrayList<DataPoints[]> query_results) throws Exception {
            for (int i = 0; i < query_results.size(); i++) {
                final TSSubQuery sub = ts_query.getQueries().get(i);
                Iterator<Entry<String, TSSubQuery>> it = sub_queries.entrySet().iterator();
                while (it.hasNext()) {
                    final Entry<String, TSSubQuery> entry = it.next();
                    if (entry.getValue().equals(sub)) {
                        sub_query_results.put(entry.getKey(), query_results.get(i));
                        for (final ExpressionIterator ei : expressions.values()) {
                            if (ei.getVariableNames().contains(entry.getKey())) {
                                final TimeSyncedIterator tsi = new TimeSyncedIterator(entry.getKey(), sub.getFilterTagKs(), query_results.get(i));
                                final NumericFillPolicy fill = fills.get(entry.getKey());
                                if (fill != null) {
                                    tsi.setFillPolicy(fill);
                                }
                                ei.addResults(entry.getKey(), tsi);
                                if (LOG.isDebugEnabled()) {
                                    LOG.debug("Added results for " + entry.getKey() + " to " + ei.getId());
                                }
                            }
                        }
                    }
                }
            }
            // handle nested expressions
            final DirectedAcyclicGraph<String, DefaultEdge> graph = new DirectedAcyclicGraph<String, DefaultEdge>(DefaultEdge.class);
            for (final Entry<String, ExpressionIterator> eii : expressions.entrySet()) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug(String.format("Expression entry key is %s, value is %s", eii.getKey(), eii.getValue().toString()));
                    LOG.debug(String.format("Time to loop through the variable names " + "for %s", eii.getKey()));
                }
                if (!graph.containsVertex(eii.getKey())) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Adding vertex " + eii.getKey());
                    }
                    graph.addVertex(eii.getKey());
                }
                for (final String var : eii.getValue().getVariableNames()) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug(String.format("var is %s", var));
                    }
                    final ExpressionIterator ei = expressions.get(var);
                    if (ei != null) {
                        if (LOG.isDebugEnabled()) {
                            LOG.debug(String.format("The expression iterator for %s is %s", var, ei.toString()));
                        }
                        // TODO - really ought to calculate this earlier
                        if (eii.getKey().equals(var)) {
                            throw new IllegalArgumentException("Self referencing expression found: " + eii.getKey());
                        }
                        if (LOG.isDebugEnabled()) {
                            LOG.debug("Nested expression detected. " + eii.getKey() + " depends on " + var);
                        }
                        if (!graph.containsVertex(eii.getKey())) {
                            if (LOG.isDebugEnabled()) {
                                LOG.debug("Added vertex " + eii.getKey());
                            }
                            graph.addVertex(eii.getKey());
                        } else if (LOG.isDebugEnabled()) {
                            LOG.debug("Already contains vertex " + eii.getKey());
                        }
                        if (!graph.containsVertex(var)) {
                            if (LOG.isDebugEnabled()) {
                                LOG.debug("Added vertex " + var);
                            }
                            graph.addVertex(var);
                        } else if (LOG.isDebugEnabled()) {
                            LOG.debug("Already contains vertex " + var);
                        }
                        try {
                            if (LOG.isDebugEnabled()) {
                                LOG.debug("Added Edge " + eii.getKey() + " - " + var);
                            }
                            graph.addDagEdge(eii.getKey(), var);
                        } catch (CycleFoundException cfe) {
                            throw new IllegalArgumentException("Circular reference found: " + eii.getKey(), cfe);
                        }
                    } else if (LOG.isDebugEnabled()) {
                        LOG.debug(String.format("The expression iterator for %s is null", var));
                    }
                }
            }
            // compile all of the expressions
            final long intersect_start = DateTime.currentTimeMillis();
            final Integer expressionLength = expressions.size();
            final ExpressionIterator[] compile_stack = new ExpressionIterator[expressionLength];
            final TopologicalOrderIterator<String, DefaultEdge> it = new TopologicalOrderIterator<String, DefaultEdge>(graph);
            if (LOG.isDebugEnabled()) {
                LOG.debug(String.format("Expressions Size is %d", expressionLength));
                LOG.debug(String.format("Topology Iterator %s", it.toString()));
            }
            int i = 0;
            while (it.hasNext()) {
                String next = it.next();
                if (LOG.isDebugEnabled()) {
                    LOG.debug(String.format("Expression: %s", next));
                }
                ExpressionIterator ei = expressions.get(next);
                if (LOG.isDebugEnabled()) {
                    LOG.debug(String.format("Expression Iterator: %s", ei.toString()));
                }
                if (ei == null) {
                    LOG.error(String.format("The expression iterator for %s is null", next));
                }
                compile_stack[i] = ei;
                if (LOG.isDebugEnabled()) {
                    LOG.debug(String.format("Added expression %s to compile_stack[%d]", next, i));
                }
                i++;
            }
            if (i != expressionLength) {
                throw new IOException(String.format(" Internal Error: Fewer " + "expressions where added to the compile stack than " + "expressions.size (%d instead of %d)", i, expressionLength));
            }
            if (LOG.isDebugEnabled()) {
                LOG.debug(String.format("compile stack length: %d", compile_stack.length));
            }
            for (int x = compile_stack.length - 1; x >= 0; x--) {
                if (compile_stack[x] == null) {
                    throw new NullPointerException(String.format("Item %d in " + "compile_stack[] is null", x));
                }
                // look for and add expressions
                for (final String var : compile_stack[x].getVariableNames()) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug(String.format("Looking for variable %s for %s", var, compile_stack[x].getId()));
                    }
                    ExpressionIterator source = expressions.get(var);
                    if (source != null) {
                        compile_stack[x].addResults(var, source.getCopy());
                        if (LOG.isDebugEnabled()) {
                            LOG.debug(String.format("Adding expression %s to %s", source.getId(), compile_stack[x].getId()));
                        }
                    }
                }
                compile_stack[x].compile();
                if (LOG.isDebugEnabled()) {
                    LOG.debug(String.format("Successfully compiled %s", compile_stack[x].getId()));
                }
            }
            if (LOG.isDebugEnabled()) {
                LOG.debug("Finished compilations in " + (DateTime.currentTimeMillis() - intersect_start) + " ms");
            }
            return serialize().addCallback(new CompleteCB()).addErrback(new ErrorCB());
        }
    }
    /**
     * Callback executed after we have resolved the metric, tag names and tag
     * values to their respective UIDs. This callback then runs the actual 
     * queries and fetches their results.
     */
    class BuildCB implements Callback<Deferred<Object>, net.opentsdb.core.Query[]> {

        @Override
        public Deferred<Object> call(final net.opentsdb.core.Query[] queries) {
            final ArrayList<Deferred<DataPoints[]>> deferreds = new ArrayList<Deferred<DataPoints[]>>(queries.length);
            for (final net.opentsdb.core.Query query : queries) {
                deferreds.add(query.runAsync());
            }
            return Deferred.groupInOrder(deferreds).addCallback(new QueriesCB()).addErrback(new ErrorCB());
        }
    }
    // TODO - only run the ones that will be involved in an output. Folks WILL
    // ask for stuff they don't need.... *sigh*
    ts_query.buildQueriesAsync(tsdb).addCallback(new BuildCB()).addErrback(new ErrorCB());
}

Example 2 with Query

use of net.opentsdb.query.pojo.Query in project opentsdb by OpenTSDB.

the class TestQueryExecutor method twoExpressionsDefaultOutput.

@Test
public void twoExpressionsDefaultOutput() throws Exception {
    oneExtraSameE();
    expressions = Arrays.asList(Expression.Builder().setId("e").setExpression("a + b").setJoin(intersection).build(), Expression.Builder().setId("e2").setExpression("a * b").setJoin(intersection).build());
    final Query q = Query.Builder().setExpressions(expressions).setFilters(filters).setMetrics(metrics).setName("q1").setTime(time).build();
    final String json = JSON.serializeToString(q);
    final QueryRpc rpc = new QueryRpc();
    final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/exp", json);
    NettyMocks.mockChannelFuture(query);
    rpc.execute(tsdb, query);
    final String response = query.response().getContent().toString(Charset.forName("UTF-8"));
    assertTrue(response.contains("\"id\":\"e\""));
    assertTrue(response.contains("\"dps\":[[1431561600000,12.0,18.0]"));
    assertTrue(response.contains("[1431561660000,14.0,20.0]"));
    assertTrue(response.contains("[1431561720000,16.0,22.0]"));
    assertTrue(response.contains("\"firstTimestamp\":1431561600000"));
    assertTrue(response.contains("\"index\":1"));
    assertTrue(response.contains("\"metrics\":[\"A\",\"B\"]"));
    assertTrue(response.contains("\"index\":2"));
    assertTrue(response.contains("\"id\":\"e2\""));
    assertTrue(response.contains("\"dps\":[[1431561600000,11.0,56.0]"));
    assertTrue(response.contains("[1431561660000,24.0,75.0]"));
    assertTrue(response.contains("[1431561720000,39.0,96.0]"));
}

Example 3 with Query

use of net.opentsdb.query.pojo.Query in project opentsdb by OpenTSDB.

the class TestQueryExecutor method nestedExpressionsOneLevelDefaultOutput.

@Test
public void nestedExpressionsOneLevelDefaultOutput() throws Exception {
    oneExtraSameE();
    expressions = Arrays.asList(Expression.Builder().setId("e").setExpression("a + b").setJoin(intersection).build(), Expression.Builder().setId("e2").setExpression("e * 2").setJoin(intersection).build());
    final Query q = Query.Builder().setExpressions(expressions).setFilters(filters).setMetrics(metrics).setName("q1").setTime(time).build();
    final String json = JSON.serializeToString(q);
    final QueryRpc rpc = new QueryRpc();
    final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/exp", json);
    NettyMocks.mockChannelFuture(query);
    rpc.execute(tsdb, query);
    final String response = query.response().getContent().toString(Charset.forName("UTF-8"));
    assertTrue(response.contains("\"id\":\"e\""));
    assertTrue(response.contains("\"dps\":[[1431561600000,12.0,18.0]"));
    assertTrue(response.contains("[1431561660000,14.0,20.0]"));
    assertTrue(response.contains("[1431561720000,16.0,22.0]"));
    assertTrue(response.contains("\"firstTimestamp\":1431561600000"));
    assertTrue(response.contains("\"index\":1"));
    assertTrue(response.contains("\"metrics\":[\"A\",\"B\"]"));
    assertTrue(response.contains("\"index\":2"));
    assertTrue(response.contains("\"id\":\"e2\""));
    assertTrue(response.contains("\"dps\":[[1431561600000,24.0,36.0]"));
    assertTrue(response.contains("[1431561660000,28.0,40.0]"));
    assertTrue(response.contains("[1431561720000,32.0,44.0]"));
}

Example 4 with Query

use of net.opentsdb.query.pojo.Query in project opentsdb by OpenTSDB.

the class TestQueryExecutor method oneExpressionDefaultOutput.

@Test
public void oneExpressionDefaultOutput() throws Exception {
    oneExtraSameE();
    final Query q = Query.Builder().setExpressions(expressions).setFilters(filters).setMetrics(metrics).setName("q1").setTime(time).build();
    final String json = JSON.serializeToString(q);
    final QueryRpc rpc = new QueryRpc();
    final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/exp", json);
    NettyMocks.mockChannelFuture(query);
    rpc.execute(tsdb, query);
    final String response = query.response().getContent().toString(Charset.forName("UTF-8"));
    assertTrue(response.contains("\"id\":\"e\""));
    assertTrue(response.contains("\"dps\":[[1431561600000,12.0,18.0]"));
    assertTrue(response.contains("[1431561660000,14.0,20.0]"));
    assertTrue(response.contains("[1431561720000,16.0,22.0]"));
    assertTrue(response.contains("\"firstTimestamp\":1431561600000"));
    assertTrue(response.contains("\"index\":1"));
    assertTrue(response.contains("\"metrics\":[\"A\",\"B\"]"));
    assertTrue(response.contains("\"index\":2"));
// TODO - more asserts once we settle on names
}

Example 5 with Query

use of net.opentsdb.query.pojo.Query in project opentsdb by OpenTSDB.

the class TestQueryExecutor method nestedExpressionsTwoLevelsDefaultOutputOrdering.

@Test
public void nestedExpressionsTwoLevelsDefaultOutputOrdering() throws Exception {
    oneExtraSameE();
    expressions = Arrays.asList(Expression.Builder().setId("e2").setExpression("e * 2").setJoin(intersection).build(), Expression.Builder().setId("e4").setExpression("e2 + e3").setJoin(intersection).build(), Expression.Builder().setId("e3").setExpression("e * 2").setJoin(intersection).build(), Expression.Builder().setId("e").setExpression("a + b").setJoin(intersection).build());
    final Query q = Query.Builder().setExpressions(expressions).setFilters(filters).setMetrics(metrics).setName("q1").setTime(time).build();
    final String json = JSON.serializeToString(q);
    final QueryRpc rpc = new QueryRpc();
    final HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query/exp", json);
    NettyMocks.mockChannelFuture(query);
    rpc.execute(tsdb, query);
    final String response = query.response().getContent().toString(Charset.forName("UTF-8"));
    assertTrue(response.contains("\"id\":\"e\""));
    assertTrue(response.contains("\"dps\":[[1431561600000,12.0,18.0]"));
    assertTrue(response.contains("[1431561660000,14.0,20.0]"));
    assertTrue(response.contains("[1431561720000,16.0,22.0]"));
    assertTrue(response.contains("\"firstTimestamp\":1431561600000"));
    assertTrue(response.contains("\"index\":1"));
    assertTrue(response.contains("\"metrics\":[\"A\",\"B\"]"));
    assertTrue(response.contains("\"index\":2"));
    assertTrue(response.contains("\"id\":\"e2\""));
    assertTrue(response.contains("\"dps\":[[1431561600000,24.0,36.0]"));
    assertTrue(response.contains("[1431561660000,28.0,40.0]"));
    assertTrue(response.contains("[1431561720000,32.0,44.0]"));
    assertTrue(response.contains("\"id\":\"e3\""));
    assertTrue(response.contains("\"id\":\"e4\""));
    assertTrue(response.contains("\"dps\":[[1431561600000,48.0,72.0]"));
    assertTrue(response.contains("[1431561660000,56.0,80.0]"));
    assertTrue(response.contains("[1431561720000,64.0,88.0]"));
}