Search in sources :

Example 1 with TSQuery

use of net.opentsdb.core.TSQuery in project opentsdb by OpenTSDB.

the class QueryRpc method handleQuery.

/**
   * Processing for a data point query
   * @param tsdb The TSDB to which we belong
   * @param query The HTTP query to parse/respond
   * @param allow_expressions Whether or not expressions should be parsed
   * (based on the endpoint)
   */
private void handleQuery(final TSDB tsdb, final HttpQuery query, final boolean allow_expressions) {
    final long start = DateTime.currentTimeMillis();
    final TSQuery data_query;
    final List<ExpressionTree> expressions;
    if (query.method() == HttpMethod.POST) {
        switch(query.apiVersion()) {
            case 0:
            case 1:
                data_query = query.serializer().parseQueryV1();
                break;
            default:
                query_invalid.incrementAndGet();
                throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, "Requested API version not implemented", "Version " + query.apiVersion() + " is not implemented");
        }
        expressions = null;
    } else {
        expressions = new ArrayList<ExpressionTree>();
        data_query = parseQuery(tsdb, query, expressions);
    }
    if (query.getAPIMethod() == HttpMethod.DELETE && tsdb.getConfig().getBoolean("tsd.http.query.allow_delete")) {
        data_query.setDelete(true);
    }
    // validate and then compile the queries
    try {
        LOG.debug(data_query.toString());
        data_query.validateAndSetQuery();
    } catch (Exception e) {
        throw new BadRequestException(HttpResponseStatus.BAD_REQUEST, e.getMessage(), data_query.toString(), e);
    }
    // if the user tried this query multiple times from the same IP and src port
    // they'll be rejected on subsequent calls
    final QueryStats query_stats = new QueryStats(query.getRemoteAddress(), data_query, query.getPrintableHeaders());
    data_query.setQueryStats(query_stats);
    query.setStats(query_stats);
    final int nqueries = data_query.getQueries().size();
    final ArrayList<DataPoints[]> results = new ArrayList<DataPoints[]>(nqueries);
    final List<Annotation> globals = new ArrayList<Annotation>();
    /** This has to be attached to callbacks or we may never respond to clients */
    class ErrorCB implements Callback<Object, Exception> {

        public Object call(final Exception e) throws Exception {
            Throwable ex = e;
            try {
                LOG.error("Query exception: ", e);
                if (ex instanceof DeferredGroupException) {
                    ex = e.getCause();
                    while (ex != null && ex instanceof DeferredGroupException) {
                        ex = ex.getCause();
                    }
                    if (ex == null) {
                        LOG.error("The deferred group exception didn't have a cause???");
                    }
                }
                if (ex instanceof RpcTimedOutException) {
                    query_stats.markSerialized(HttpResponseStatus.REQUEST_TIMEOUT, ex);
                    query.badRequest(new BadRequestException(HttpResponseStatus.REQUEST_TIMEOUT, ex.getMessage()));
                    query_exceptions.incrementAndGet();
                } else if (ex instanceof HBaseException) {
                    query_stats.markSerialized(HttpResponseStatus.FAILED_DEPENDENCY, ex);
                    query.badRequest(new BadRequestException(HttpResponseStatus.FAILED_DEPENDENCY, ex.getMessage()));
                    query_exceptions.incrementAndGet();
                } else if (ex instanceof QueryException) {
                    query_stats.markSerialized(((QueryException) ex).getStatus(), ex);
                    query.badRequest(new BadRequestException(((QueryException) ex).getStatus(), ex.getMessage()));
                    query_exceptions.incrementAndGet();
                } else if (ex instanceof BadRequestException) {
                    query_stats.markSerialized(((BadRequestException) ex).getStatus(), ex);
                    query.badRequest((BadRequestException) ex);
                    query_invalid.incrementAndGet();
                } else if (ex instanceof NoSuchUniqueName) {
                    query_stats.markSerialized(HttpResponseStatus.BAD_REQUEST, ex);
                    query.badRequest(new BadRequestException(ex));
                    query_invalid.incrementAndGet();
                } else {
                    query_stats.markSerialized(HttpResponseStatus.INTERNAL_SERVER_ERROR, ex);
                    query.badRequest(new BadRequestException(ex));
                    query_exceptions.incrementAndGet();
                }
            } catch (RuntimeException ex2) {
                LOG.error("Exception thrown during exception handling", ex2);
                query_stats.markSerialized(HttpResponseStatus.INTERNAL_SERVER_ERROR, ex2);
                query.sendReply(HttpResponseStatus.INTERNAL_SERVER_ERROR, ex2.getMessage().getBytes());
                query_exceptions.incrementAndGet();
            }
            return null;
        }
    }
    /**
     * After all of the queries have run, we get the results in the order given
     * and add dump the results in an array
     */
    class QueriesCB implements Callback<Object, ArrayList<DataPoints[]>> {

        public Object call(final ArrayList<DataPoints[]> query_results) throws Exception {
            if (allow_expressions) {
                // process each of the expressions into a new list, then merge it
                // with the original. This avoids possible recursion loops.
                final List<DataPoints[]> expression_results = new ArrayList<DataPoints[]>(expressions.size());
                // let exceptions bubble up
                for (final ExpressionTree expression : expressions) {
                    expression_results.add(expression.evaluate(query_results));
                }
                results.addAll(expression_results);
            } else {
                results.addAll(query_results);
            }
            /** Simply returns the buffer once serialization is complete and logs it */
            class SendIt implements Callback<Object, ChannelBuffer> {

                public Object call(final ChannelBuffer buffer) throws Exception {
                    query.sendReply(buffer);
                    query_success.incrementAndGet();
                    return null;
                }
            }
            switch(query.apiVersion()) {
                case 0:
                case 1:
                    query.serializer().formatQueryAsyncV1(data_query, results, globals).addCallback(new SendIt()).addErrback(new ErrorCB());
                    break;
                default:
                    query_invalid.incrementAndGet();
                    throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, "Requested API version not implemented", "Version " + query.apiVersion() + " is not implemented");
            }
            return null;
        }
    }
    /**
     * Callback executed after we have resolved the metric, tag names and tag
     * values to their respective UIDs. This callback then runs the actual 
     * queries and fetches their results.
     */
    class BuildCB implements Callback<Deferred<Object>, Query[]> {

        @Override
        public Deferred<Object> call(final Query[] queries) {
            final ArrayList<Deferred<DataPoints[]>> deferreds = new ArrayList<Deferred<DataPoints[]>>(queries.length);
            for (final Query query : queries) {
                deferreds.add(query.runAsync());
            }
            return Deferred.groupInOrder(deferreds).addCallback(new QueriesCB());
        }
    }
    /** Handles storing the global annotations after fetching them */
    class GlobalCB implements Callback<Object, List<Annotation>> {

        public Object call(final List<Annotation> annotations) throws Exception {
            globals.addAll(annotations);
            return data_query.buildQueriesAsync(tsdb).addCallback(new BuildCB());
        }
    }
    // when complete
    if (!data_query.getNoAnnotations() && data_query.getGlobalAnnotations()) {
        Annotation.getGlobalAnnotations(tsdb, data_query.startTime() / 1000, data_query.endTime() / 1000).addCallback(new GlobalCB()).addErrback(new ErrorCB());
    } else {
        data_query.buildQueriesAsync(tsdb).addCallback(new BuildCB()).addErrback(new ErrorCB());
    }
}
Also used : Query(net.opentsdb.core.Query) TSUIDQuery(net.opentsdb.meta.TSUIDQuery) TSQuery(net.opentsdb.core.TSQuery) TSSubQuery(net.opentsdb.core.TSSubQuery) Deferred(com.stumbleupon.async.Deferred) ArrayList(java.util.ArrayList) DataPoints(net.opentsdb.core.DataPoints) ChannelBuffer(org.jboss.netty.buffer.ChannelBuffer) TSQuery(net.opentsdb.core.TSQuery) DeferredGroupException(com.stumbleupon.async.DeferredGroupException) ExpressionTree(net.opentsdb.query.expression.ExpressionTree) ArrayList(java.util.ArrayList) List(java.util.List) RpcTimedOutException(org.hbase.async.RpcTimedOutException) QueryException(net.opentsdb.core.QueryException) DeferredGroupException(com.stumbleupon.async.DeferredGroupException) IOException(java.io.IOException) RpcTimedOutException(org.hbase.async.RpcTimedOutException) HBaseException(org.hbase.async.HBaseException) IncomingDataPoint(net.opentsdb.core.IncomingDataPoint) Annotation(net.opentsdb.meta.Annotation) QueryException(net.opentsdb.core.QueryException) Callback(com.stumbleupon.async.Callback) QueryStats(net.opentsdb.stats.QueryStats) HBaseException(org.hbase.async.HBaseException) NoSuchUniqueName(net.opentsdb.uid.NoSuchUniqueName)

Example 2 with TSQuery

use of net.opentsdb.core.TSQuery in project opentsdb by OpenTSDB.

the class BaseTimeSyncedIteratorTest method runQueries.

/**
   * Executes the queries against MockBase through the regular pipeline and stores
   * the results in {@linke #results}
   * @param subs The queries to execute
   */
protected void runQueries(final ArrayList<TSSubQuery> subs) throws Exception {
    query = new TSQuery();
    query.setStart(Long.toString(START_TS));
    query.setQueries(subs);
    query.validateAndSetQuery();
    final Query[] compiled = query.buildQueries(tsdb);
    results = new HashMap<String, Pair<TSSubQuery, DataPoints[]>>(compiled.length);
    iterators = new HashMap<String, ITimeSyncedIterator>(compiled.length);
    int index = 0;
    for (final Query q : compiled) {
        final DataPoints[] dps = q.runAsync().join();
        results.put(Integer.toString(index), new Pair<TSSubQuery, DataPoints[]>(query.getQueries().get(index), dps));
        final ITimeSyncedIterator it = new TimeSyncedIterator(Integer.toString(index), query.getQueries().get(index).getFilterTagKs(), dps);
        it.setFillPolicy(new NumericFillPolicy(FillPolicy.NOT_A_NUMBER));
        iterators.put(Integer.toString(index), it);
        index++;
    }
}
Also used : Query(net.opentsdb.core.Query) TSQuery(net.opentsdb.core.TSQuery) TSSubQuery(net.opentsdb.core.TSSubQuery) DataPoints(net.opentsdb.core.DataPoints) TSSubQuery(net.opentsdb.core.TSSubQuery) TSQuery(net.opentsdb.core.TSQuery) Pair(net.opentsdb.utils.Pair)

Example 3 with TSQuery

use of net.opentsdb.core.TSQuery in project opentsdb by OpenTSDB.

the class TestQueryStats method markCompleteTimeout.

@Test
public void markCompleteTimeout() throws Exception {
    final TSQuery query = new TSQuery();
    query.setStart("1h-ago");
    final QueryStats stats = new QueryStats(remote, query, headers);
    final RuntimeException timeout = new RuntimeException("Timeout!");
    stats.markSerialized(HttpResponseStatus.REQUEST_TIMEOUT, timeout);
    final Map<String, Object> map = QueryStats.getRunningAndCompleteStats();
    assertNotNull(map);
    assertEquals(0, ((List<Object>) map.get("running")).size());
    assertEquals(1, ((Collection<QueryStats>) map.get("completed")).size());
    final QueryStats completed = ((Collection<QueryStats>) map.get("completed")).iterator().next();
    assertEquals(408, completed.getHttpResponse().getCode());
    assertTrue(completed.getException().startsWith("Timeout!\n"));
}
Also used : TSQuery(net.opentsdb.core.TSQuery) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 4 with TSQuery

use of net.opentsdb.core.TSQuery in project opentsdb by OpenTSDB.

the class TestQueryStats method ctorNullHeaders.

@Test
public void ctorNullHeaders() throws Exception {
    final TSQuery query = new TSQuery();
    query.setStart("1h-ago");
    final QueryStats stats = new QueryStats(remote, query, null);
    assertNotNull(stats);
    final Map<String, Object> map = QueryStats.getRunningAndCompleteStats();
    assertNotNull(map);
    assertEquals(1, ((List<Object>) map.get("running")).size());
    assertEquals(0, ((Collection<QueryStats>) map.get("completed")).size());
}
Also used : TSQuery(net.opentsdb.core.TSQuery) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 5 with TSQuery

use of net.opentsdb.core.TSQuery in project opentsdb by OpenTSDB.

the class TestQueryStats method testEqualsNull.

@Test
public void testEqualsNull() throws Exception {
    final TSQuery query = new TSQuery();
    query.setStart("1h-ago");
    final QueryStats stats = new QueryStats(remote, query, headers);
    assertFalse(stats.equals(null));
}
Also used : TSQuery(net.opentsdb.core.TSQuery) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Aggregations

TSQuery (net.opentsdb.core.TSQuery)63 Test (org.junit.Test)57 PrepareForTest (org.powermock.core.classloader.annotations.PrepareForTest)57 TSSubQuery (net.opentsdb.core.TSSubQuery)30 ArrayList (java.util.ArrayList)18 DataPoints (net.opentsdb.core.DataPoints)17 Annotation (net.opentsdb.meta.Annotation)15 MockDataPoints (net.opentsdb.storage.MockDataPoints)13 Matchers.anyString (org.mockito.Matchers.anyString)10 ChannelBuffer (org.jboss.netty.buffer.ChannelBuffer)9 TagVWildcardFilter (net.opentsdb.query.filter.TagVWildcardFilter)7 Query (net.opentsdb.core.Query)4 Callback (com.stumbleupon.async.Callback)3 IOException (java.io.IOException)3 NoSuchUniqueId (net.opentsdb.uid.NoSuchUniqueId)3 Deferred (com.stumbleupon.async.Deferred)2 List (java.util.List)2 DataPoint (net.opentsdb.core.DataPoint)2 QueryException (net.opentsdb.core.QueryException)2 TagVLiteralOrFilter (net.opentsdb.query.filter.TagVLiteralOrFilter)2