Examples with Query net.opentsdb.core.Query used on opensource projects

Example 1 with Query

use of net.opentsdb.core.Query in project opentsdb by OpenTSDB.

the class CliQuery method doQuery.

private static Plot doQuery(final TSDB tsdb, final String[] args, final boolean want_plot) {
    final ArrayList<String> plotparams = new ArrayList<String>();
    final ArrayList<Query> queries = new ArrayList<Query>();
    final ArrayList<String> plotoptions = new ArrayList<String>();
    parseCommandLineQuery(args, tsdb, queries, plotparams, plotoptions);
    if (queries.isEmpty()) {
        usage(null, "Not enough arguments, need at least one query.", 2);
    }
    final Plot plot = (want_plot ? new Plot(queries.get(0).getStartTime(), queries.get(0).getEndTime()) : null);
    if (want_plot) {
        plot.setParams(parsePlotParams(plotparams));
    }
    final int nqueries = queries.size();
    for (int i = 0; i < nqueries; i++) {
        // TODO(tsuna): Optimization: run each query in parallel.
        final StringBuilder buf = want_plot ? null : new StringBuilder();
        for (final DataPoints datapoints : queries.get(i).run()) {
            if (want_plot) {
                plot.add(datapoints, plotoptions.get(i));
            } else {
                final String metric = datapoints.metricName();
                final String tagz = datapoints.getTags().toString();
                for (final DataPoint datapoint : datapoints) {
                    buf.append(metric).append(' ').append(datapoint.timestamp()).append(' ');
                    if (datapoint.isInteger()) {
                        buf.append(datapoint.longValue());
                    } else {
                        buf.append(String.format("%f", datapoint.doubleValue()));
                    }
                    buf.append(' ').append(tagz).append('\n');
                    System.out.print(buf);
                    buf.delete(0, buf.length());
                }
            }
        }
    }
    return plot;
}

Example 2 with Query

use of net.opentsdb.core.Query in project opentsdb by OpenTSDB.

the class QueryRpc method handleQuery.

/**
   * Processing for a data point query
   * @param tsdb The TSDB to which we belong
   * @param query The HTTP query to parse/respond
   * @param allow_expressions Whether or not expressions should be parsed
   * (based on the endpoint)
   */
private void handleQuery(final TSDB tsdb, final HttpQuery query, final boolean allow_expressions) {
    final long start = DateTime.currentTimeMillis();
    final TSQuery data_query;
    final List<ExpressionTree> expressions;
    if (query.method() == HttpMethod.POST) {
        switch(query.apiVersion()) {
            case 0:
            case 1:
                data_query = query.serializer().parseQueryV1();
                break;
            default:
                query_invalid.incrementAndGet();
                throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, "Requested API version not implemented", "Version " + query.apiVersion() + " is not implemented");
        }
        expressions = null;
    } else {
        expressions = new ArrayList<ExpressionTree>();
        data_query = parseQuery(tsdb, query, expressions);
    }
    if (query.getAPIMethod() == HttpMethod.DELETE && tsdb.getConfig().getBoolean("tsd.http.query.allow_delete")) {
        data_query.setDelete(true);
    }
    // validate and then compile the queries
    try {
        LOG.debug(data_query.toString());
        data_query.validateAndSetQuery();
    } catch (Exception e) {
        throw new BadRequestException(HttpResponseStatus.BAD_REQUEST, e.getMessage(), data_query.toString(), e);
    }
    // if the user tried this query multiple times from the same IP and src port
    // they'll be rejected on subsequent calls
    final QueryStats query_stats = new QueryStats(query.getRemoteAddress(), data_query, query.getPrintableHeaders());
    data_query.setQueryStats(query_stats);
    query.setStats(query_stats);
    final int nqueries = data_query.getQueries().size();
    final ArrayList<DataPoints[]> results = new ArrayList<DataPoints[]>(nqueries);
    final List<Annotation> globals = new ArrayList<Annotation>();
    /** This has to be attached to callbacks or we may never respond to clients */
    class ErrorCB implements Callback<Object, Exception> {

        public Object call(final Exception e) throws Exception {
            Throwable ex = e;
            try {
                LOG.error("Query exception: ", e);
                if (ex instanceof DeferredGroupException) {
                    ex = e.getCause();
                    while (ex != null && ex instanceof DeferredGroupException) {
                        ex = ex.getCause();
                    }
                    if (ex == null) {
                        LOG.error("The deferred group exception didn't have a cause???");
                    }
                }
                if (ex instanceof RpcTimedOutException) {
                    query_stats.markSerialized(HttpResponseStatus.REQUEST_TIMEOUT, ex);
                    query.badRequest(new BadRequestException(HttpResponseStatus.REQUEST_TIMEOUT, ex.getMessage()));
                    query_exceptions.incrementAndGet();
                } else if (ex instanceof HBaseException) {
                    query_stats.markSerialized(HttpResponseStatus.FAILED_DEPENDENCY, ex);
                    query.badRequest(new BadRequestException(HttpResponseStatus.FAILED_DEPENDENCY, ex.getMessage()));
                    query_exceptions.incrementAndGet();
                } else if (ex instanceof QueryException) {
                    query_stats.markSerialized(((QueryException) ex).getStatus(), ex);
                    query.badRequest(new BadRequestException(((QueryException) ex).getStatus(), ex.getMessage()));
                    query_exceptions.incrementAndGet();
                } else if (ex instanceof BadRequestException) {
                    query_stats.markSerialized(((BadRequestException) ex).getStatus(), ex);
                    query.badRequest((BadRequestException) ex);
                    query_invalid.incrementAndGet();
                } else if (ex instanceof NoSuchUniqueName) {
                    query_stats.markSerialized(HttpResponseStatus.BAD_REQUEST, ex);
                    query.badRequest(new BadRequestException(ex));
                    query_invalid.incrementAndGet();
                } else {
                    query_stats.markSerialized(HttpResponseStatus.INTERNAL_SERVER_ERROR, ex);
                    query.badRequest(new BadRequestException(ex));
                    query_exceptions.incrementAndGet();
                }
            } catch (RuntimeException ex2) {
                LOG.error("Exception thrown during exception handling", ex2);
                query_stats.markSerialized(HttpResponseStatus.INTERNAL_SERVER_ERROR, ex2);
                query.sendReply(HttpResponseStatus.INTERNAL_SERVER_ERROR, ex2.getMessage().getBytes());
                query_exceptions.incrementAndGet();
            }
            return null;
        }
    }
    /**
     * After all of the queries have run, we get the results in the order given
     * and add dump the results in an array
     */
    class QueriesCB implements Callback<Object, ArrayList<DataPoints[]>> {

        public Object call(final ArrayList<DataPoints[]> query_results) throws Exception {
            if (allow_expressions) {
                // process each of the expressions into a new list, then merge it
                // with the original. This avoids possible recursion loops.
                final List<DataPoints[]> expression_results = new ArrayList<DataPoints[]>(expressions.size());
                // let exceptions bubble up
                for (final ExpressionTree expression : expressions) {
                    expression_results.add(expression.evaluate(query_results));
                }
                results.addAll(expression_results);
            } else {
                results.addAll(query_results);
            }
            /** Simply returns the buffer once serialization is complete and logs it */
            class SendIt implements Callback<Object, ChannelBuffer> {

                public Object call(final ChannelBuffer buffer) throws Exception {
                    query.sendReply(buffer);
                    query_success.incrementAndGet();
                    return null;
                }
            }
            switch(query.apiVersion()) {
                case 0:
                case 1:
                    query.serializer().formatQueryAsyncV1(data_query, results, globals).addCallback(new SendIt()).addErrback(new ErrorCB());
                    break;
                default:
                    query_invalid.incrementAndGet();
                    throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, "Requested API version not implemented", "Version " + query.apiVersion() + " is not implemented");
            }
            return null;
        }
    }
    /**
     * Callback executed after we have resolved the metric, tag names and tag
     * values to their respective UIDs. This callback then runs the actual 
     * queries and fetches their results.
     */
    class BuildCB implements Callback<Deferred<Object>, Query[]> {

        @Override
        public Deferred<Object> call(final Query[] queries) {
            final ArrayList<Deferred<DataPoints[]>> deferreds = new ArrayList<Deferred<DataPoints[]>>(queries.length);
            for (final Query query : queries) {
                deferreds.add(query.runAsync());
            }
            return Deferred.groupInOrder(deferreds).addCallback(new QueriesCB());
        }
    }
    /** Handles storing the global annotations after fetching them */
    class GlobalCB implements Callback<Object, List<Annotation>> {

        public Object call(final List<Annotation> annotations) throws Exception {
            globals.addAll(annotations);
            return data_query.buildQueriesAsync(tsdb).addCallback(new BuildCB());
        }
    }
    // when complete
    if (!data_query.getNoAnnotations() && data_query.getGlobalAnnotations()) {
        Annotation.getGlobalAnnotations(tsdb, data_query.startTime() / 1000, data_query.endTime() / 1000).addCallback(new GlobalCB()).addErrback(new ErrorCB());
    } else {
        data_query.buildQueriesAsync(tsdb).addCallback(new BuildCB()).addErrback(new ErrorCB());
    }
}

Example 3 with Query

use of net.opentsdb.core.Query in project opentsdb by OpenTSDB.

the class BaseTimeSyncedIteratorTest method runQueries.

/**
   * Executes the queries against MockBase through the regular pipeline and stores
   * the results in {@linke #results}
   * @param subs The queries to execute
   */
protected void runQueries(final ArrayList<TSSubQuery> subs) throws Exception {
    query = new TSQuery();
    query.setStart(Long.toString(START_TS));
    query.setQueries(subs);
    query.validateAndSetQuery();
    final Query[] compiled = query.buildQueries(tsdb);
    results = new HashMap<String, Pair<TSSubQuery, DataPoints[]>>(compiled.length);
    iterators = new HashMap<String, ITimeSyncedIterator>(compiled.length);
    int index = 0;
    for (final Query q : compiled) {
        final DataPoints[] dps = q.runAsync().join();
        results.put(Integer.toString(index), new Pair<TSSubQuery, DataPoints[]>(query.getQueries().get(index), dps));
        final ITimeSyncedIterator it = new TimeSyncedIterator(Integer.toString(index), query.getQueries().get(index).getFilterTagKs(), dps);
        it.setFillPolicy(new NumericFillPolicy(FillPolicy.NOT_A_NUMBER));
        iterators.put(Integer.toString(index), it);
        index++;
    }
}

Example 4 with Query

use of net.opentsdb.core.Query in project opentsdb by OpenTSDB.

the class QueryExample method main.

public static void main(final String[] args) throws IOException {
    // Set these as arguments so you don't have to keep path information in
    // source files 
    String pathToConfigFile = (args != null && args.length > 0 ? args[0] : null);
    // Create a config object with a path to the file for parsing. Or manually
    // override settings.
    // e.g. config.overrideConfig("tsd.storage.hbase.zk_quorum", "localhost");
    final Config config;
    if (pathToConfigFile != null && !pathToConfigFile.isEmpty()) {
        config = new Config(pathToConfigFile);
    } else {
        // Search for a default config from /etc/opentsdb/opentsdb.conf, etc.
        config = new Config(true);
    }
    final TSDB tsdb = new TSDB(config);
    // main query
    final TSQuery query = new TSQuery();
    // use any string format from
    // http://opentsdb.net/docs/build/html/user_guide/query/dates.html
    query.setStart("1h-ago");
    // Optional: set other global query params
    // at least one sub query required. This is where you specify the metric and
    // tags
    final TSSubQuery subQuery = new TSSubQuery();
    subQuery.setMetric("my.tsdb.test.metric");
    // filters are optional but useful.
    final List<TagVFilter> filters = new ArrayList<TagVFilter>(1);
    filters.add(new TagVFilter.Builder().setType("literal_or").setFilter("example1").setTagk("script").setGroupBy(true).build());
    subQuery.setFilters(filters);
    // you do have to set an aggregator. Just provide the name as a string
    subQuery.setAggregator("sum");
    // IMPORTANT: don't forget to add the subQuery
    final ArrayList<TSSubQuery> subQueries = new ArrayList<TSSubQuery>(1);
    subQueries.add(subQuery);
    query.setQueries(subQueries);
    // otherwise we aggregate on the second. 
    query.setMsResolution(true);
    // make sure the query is valid. This will throw exceptions if something
    // is missing
    query.validateAndSetQuery();
    // compile the queries into TsdbQuery objects behind the scenes
    Query[] tsdbqueries = query.buildQueries(tsdb);
    // create some arrays for storing the results and the async calls
    final int nqueries = tsdbqueries.length;
    final ArrayList<DataPoints[]> results = new ArrayList<DataPoints[]>(nqueries);
    final ArrayList<Deferred<DataPoints[]>> deferreds = new ArrayList<Deferred<DataPoints[]>>(nqueries);
    // deferred in an array so we can wait for them to complete.
    for (int i = 0; i < nqueries; i++) {
        deferreds.add(tsdbqueries[i].runAsync());
    }
    // Start timer
    long startTime = DateTime.nanoTime();
    // query has finished
    class QueriesCB implements Callback<Object, ArrayList<DataPoints[]>> {

        public Object call(final ArrayList<DataPoints[]> queryResults) throws Exception {
            results.addAll(queryResults);
            return null;
        }
    }
    // Make sure to handle any errors that might crop up
    class QueriesEB implements Callback<Object, Exception> {

        @Override
        public Object call(final Exception e) throws Exception {
            System.err.println("Queries failed");
            e.printStackTrace();
            return null;
        }
    }
    // have completed.
    try {
        Deferred.groupInOrder(deferreds).addCallback(new QueriesCB()).addErrback(new QueriesEB()).join();
    } catch (Exception e) {
        e.printStackTrace();
    }
    // End timer.
    double elapsedTime = DateTime.msFromNanoDiff(DateTime.nanoTime(), startTime);
    System.out.println("Query returned in: " + elapsedTime + " milliseconds.");
    // results and do any processing necessary.
    for (final DataPoints[] dataSets : results) {
        for (final DataPoints data : dataSets) {
            System.out.print(data.metricName());
            Map<String, String> resolvedTags = data.getTags();
            for (final Map.Entry<String, String> pair : resolvedTags.entrySet()) {
                System.out.print(" " + pair.getKey() + "=" + pair.getValue());
            }
            System.out.print("\n");
            final SeekableView it = data.iterator();
            /*
         * An important point about SeekableView:
         * Because no data is copied during iteration and no new object gets
         * created, the DataPoint returned must not be stored and gets
         * invalidated as soon as next is called on the iterator (actually it
         * doesn't get invalidated but rather its contents changes). If you want
         * to store individual data points, you need to copy the timestamp and
         * value out of each DataPoint into your own data structures.
         * 
         * In the vast majority of cases, the iterator will be used to go once
         * through all the data points, which is why it's not a problem if the
         * iterator acts just as a transient "view". Iterating will be very
         * cheap since no memory allocation is required (except to instantiate
         * the actual iterator at the beginning).
         */
            while (it.hasNext()) {
                final DataPoint dp = it.next();
                System.out.println("  " + dp.timestamp() + " " + (dp.isInteger() ? dp.longValue() : dp.doubleValue()));
            }
            System.out.println("");
        }
    }
    // Gracefully shutdown connection to TSDB
    try {
        tsdb.shutdown().join();
    } catch (InterruptedException e) {
        e.printStackTrace();
    } catch (Exception e) {
        e.printStackTrace();
    }
}

Example 5 with Query

use of net.opentsdb.core.Query in project opentsdb by OpenTSDB.

the class GraphHandler method doGraph.

// TODO(HugoMFernandes): Most of this (query-related) logic is implemented in
// net.opentsdb.tsd.QueryRpc.java (which actually does this asynchronously),
// so we should refactor both classes to split the actual logic used to
// generate the data from the actual visualization (removing all duped code).
private void doGraph(final TSDB tsdb, final HttpQuery query) throws IOException {
    final String basepath = getGnuplotBasePath(tsdb, query);
    long start_time = DateTime.parseDateTimeString(query.getRequiredQueryStringParam("start"), query.getQueryStringParam("tz"));
    final boolean nocache = query.hasQueryStringParam("nocache");
    if (start_time == -1) {
        throw BadRequestException.missingParameter("start");
    } else {
        // temp fixup to seconds from ms until the rest of TSDB supports ms
        // Note you can't append this to the DateTime.parseDateTimeString() call as
        // it clobbers -1 results
        start_time /= 1000;
    }
    long end_time = DateTime.parseDateTimeString(query.getQueryStringParam("end"), query.getQueryStringParam("tz"));
    final long now = System.currentTimeMillis() / 1000;
    if (end_time == -1) {
        end_time = now;
    } else {
        // temp fixup to seconds from ms until the rest of TSDB supports ms
        // Note you can't append this to the DateTime.parseDateTimeString() call as
        // it clobbers -1 results
        end_time /= 1000;
    }
    final int max_age = computeMaxAge(query, start_time, end_time, now);
    if (!nocache && isDiskCacheHit(query, end_time, max_age, basepath)) {
        return;
    }
    // Parse TSQuery from HTTP query
    final TSQuery tsquery = QueryRpc.parseQuery(tsdb, query);
    tsquery.validateAndSetQuery();
    // Build the queries for the parsed TSQuery
    Query[] tsdbqueries = tsquery.buildQueries(tsdb);
    List<String> options = query.getQueryStringParams("o");
    if (options == null) {
        options = new ArrayList<String>(tsdbqueries.length);
        for (int i = 0; i < tsdbqueries.length; i++) {
            options.add("");
        }
    } else if (options.size() != tsdbqueries.length) {
        throw new BadRequestException(options.size() + " `o' parameters, but " + tsdbqueries.length + " `m' parameters.");
    }
    for (final Query tsdbquery : tsdbqueries) {
        try {
            tsdbquery.setStartTime(start_time);
        } catch (IllegalArgumentException e) {
            throw new BadRequestException("start time: " + e.getMessage());
        }
        try {
            tsdbquery.setEndTime(end_time);
        } catch (IllegalArgumentException e) {
            throw new BadRequestException("end time: " + e.getMessage());
        }
    }
    final Plot plot = new Plot(start_time, end_time, DateTime.timezones.get(query.getQueryStringParam("tz")));
    setPlotDimensions(query, plot);
    setPlotParams(query, plot);
    final int nqueries = tsdbqueries.length;
    @SuppressWarnings("unchecked") final HashSet<String>[] aggregated_tags = new HashSet[nqueries];
    int npoints = 0;
    for (int i = 0; i < nqueries; i++) {
        try {
            // execute the TSDB query!
            // XXX This is slow and will block Netty.  TODO(tsuna): Don't block.
            // TODO(tsuna): Optimization: run each query in parallel.
            final DataPoints[] series = tsdbqueries[i].run();
            for (final DataPoints datapoints : series) {
                plot.add(datapoints, options.get(i));
                aggregated_tags[i] = new HashSet<String>();
                aggregated_tags[i].addAll(datapoints.getAggregatedTags());
                npoints += datapoints.aggregatedSize();
            }
        } catch (RuntimeException e) {
            logInfo(query, "Query failed (stack trace coming): " + tsdbqueries[i]);
            throw e;
        }
        // free()
        tsdbqueries[i] = null;
    }
    // free()
    tsdbqueries = null;
    if (query.hasQueryStringParam("ascii")) {
        respondAsciiQuery(query, max_age, basepath, plot);
        return;
    }
    final RunGnuplot rungnuplot = new RunGnuplot(query, max_age, plot, basepath, aggregated_tags, npoints);
    class ErrorCB implements Callback<Object, Exception> {

        public Object call(final Exception e) throws Exception {
            LOG.warn("Failed to retrieve global annotations: ", e);
            throw e;
        }
    }
    class GlobalCB implements Callback<Object, List<Annotation>> {

        public Object call(final List<Annotation> global_annotations) throws Exception {
            rungnuplot.plot.setGlobals(global_annotations);
            execGnuplot(rungnuplot, query);
            return null;
        }
    }
    // Fetch global annotations, if needed
    if (!tsquery.getNoAnnotations() && tsquery.getGlobalAnnotations()) {
        Annotation.getGlobalAnnotations(tsdb, start_time, end_time).addCallback(new GlobalCB()).addErrback(new ErrorCB());
    } else {
        execGnuplot(rungnuplot, query);
    }
}