Examples with TSSubQuery net.opentsdb.core.TSSubQuery used on opensource projects

Example 16 with TSSubQuery

use of net.opentsdb.core.TSSubQuery in project opentsdb by OpenTSDB.

the class TestQueryRpc method parseQueryMTypeWGroupByAndWildcardFilterExplicit.

@Test
public void parseQueryMTypeWGroupByAndWildcardFilterExplicit() throws Exception {
    HttpQuery query = NettyMocks.getQuery(tsdb, "/api/query?start=1h-ago&m=sum:sys.cpu.0{colo=lga}{host=wildcard(*quirm)}");
    TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions);
    TSSubQuery sub = tsq.getQueries().get(0);
    sub.validateAndSetQuery();
    assertTrue(sub.getFilters().get(0) instanceof TagVWildcardFilter);
    assertTrue(sub.getFilters().get(1) instanceof TagVLiteralOrFilter);
}

Example 17 with TSSubQuery

use of net.opentsdb.core.TSSubQuery in project opentsdb by OpenTSDB.

the class TestQueryRpc method parseQueryMTypeWDS.

@Test
public void parseQueryMTypeWDS() throws Exception {
    HttpQuery query = NettyMocks.getQuery(tsdb, "/api/query?start=1h-ago&m=sum:1h-avg:sys.cpu.0");
    TSQuery tsq = (TSQuery) parseQuery.invoke(rpc, tsdb, query, expressions);
    TSSubQuery sub = tsq.getQueries().get(0);
    assertEquals("1h-avg", sub.getDownsample());
}

Example 18 with TSSubQuery

use of net.opentsdb.core.TSSubQuery in project opentsdb by OpenTSDB.

the class QueryExample method main.

public static void main(final String[] args) throws IOException {
    // Set these as arguments so you don't have to keep path information in
    // source files 
    String pathToConfigFile = (args != null && args.length > 0 ? args[0] : null);
    // Create a config object with a path to the file for parsing. Or manually
    // override settings.
    // e.g. config.overrideConfig("tsd.storage.hbase.zk_quorum", "localhost");
    final Config config;
    if (pathToConfigFile != null && !pathToConfigFile.isEmpty()) {
        config = new Config(pathToConfigFile);
    } else {
        // Search for a default config from /etc/opentsdb/opentsdb.conf, etc.
        config = new Config(true);
    }
    final TSDB tsdb = new TSDB(config);
    // main query
    final TSQuery query = new TSQuery();
    // use any string format from
    // http://opentsdb.net/docs/build/html/user_guide/query/dates.html
    query.setStart("1h-ago");
    // Optional: set other global query params
    // at least one sub query required. This is where you specify the metric and
    // tags
    final TSSubQuery subQuery = new TSSubQuery();
    subQuery.setMetric("my.tsdb.test.metric");
    // filters are optional but useful.
    final List<TagVFilter> filters = new ArrayList<TagVFilter>(1);
    filters.add(new TagVFilter.Builder().setType("literal_or").setFilter("example1").setTagk("script").setGroupBy(true).build());
    subQuery.setFilters(filters);
    // you do have to set an aggregator. Just provide the name as a string
    subQuery.setAggregator("sum");
    // IMPORTANT: don't forget to add the subQuery
    final ArrayList<TSSubQuery> subQueries = new ArrayList<TSSubQuery>(1);
    subQueries.add(subQuery);
    query.setQueries(subQueries);
    // otherwise we aggregate on the second. 
    query.setMsResolution(true);
    // make sure the query is valid. This will throw exceptions if something
    // is missing
    query.validateAndSetQuery();
    // compile the queries into TsdbQuery objects behind the scenes
    Query[] tsdbqueries = query.buildQueries(tsdb);
    // create some arrays for storing the results and the async calls
    final int nqueries = tsdbqueries.length;
    final ArrayList<DataPoints[]> results = new ArrayList<DataPoints[]>(nqueries);
    final ArrayList<Deferred<DataPoints[]>> deferreds = new ArrayList<Deferred<DataPoints[]>>(nqueries);
    // deferred in an array so we can wait for them to complete.
    for (int i = 0; i < nqueries; i++) {
        deferreds.add(tsdbqueries[i].runAsync());
    }
    // Start timer
    long startTime = DateTime.nanoTime();
    // query has finished
    class QueriesCB implements Callback<Object, ArrayList<DataPoints[]>> {

        public Object call(final ArrayList<DataPoints[]> queryResults) throws Exception {
            results.addAll(queryResults);
            return null;
        }
    }
    // Make sure to handle any errors that might crop up
    class QueriesEB implements Callback<Object, Exception> {

        @Override
        public Object call(final Exception e) throws Exception {
            System.err.println("Queries failed");
            e.printStackTrace();
            return null;
        }
    }
    // have completed.
    try {
        Deferred.groupInOrder(deferreds).addCallback(new QueriesCB()).addErrback(new QueriesEB()).join();
    } catch (Exception e) {
        e.printStackTrace();
    }
    // End timer.
    double elapsedTime = DateTime.msFromNanoDiff(DateTime.nanoTime(), startTime);
    System.out.println("Query returned in: " + elapsedTime + " milliseconds.");
    // results and do any processing necessary.
    for (final DataPoints[] dataSets : results) {
        for (final DataPoints data : dataSets) {
            System.out.print(data.metricName());
            Map<String, String> resolvedTags = data.getTags();
            for (final Map.Entry<String, String> pair : resolvedTags.entrySet()) {
                System.out.print(" " + pair.getKey() + "=" + pair.getValue());
            }
            System.out.print("\n");
            final SeekableView it = data.iterator();
            /*
         * An important point about SeekableView:
         * Because no data is copied during iteration and no new object gets
         * created, the DataPoint returned must not be stored and gets
         * invalidated as soon as next is called on the iterator (actually it
         * doesn't get invalidated but rather its contents changes). If you want
         * to store individual data points, you need to copy the timestamp and
         * value out of each DataPoint into your own data structures.
         * 
         * In the vast majority of cases, the iterator will be used to go once
         * through all the data points, which is why it's not a problem if the
         * iterator acts just as a transient "view". Iterating will be very
         * cheap since no memory allocation is required (except to instantiate
         * the actual iterator at the beginning).
         */
            while (it.hasNext()) {
                final DataPoint dp = it.next();
                System.out.println("  " + dp.timestamp() + " " + (dp.isInteger() ? dp.longValue() : dp.doubleValue()));
            }
            System.out.println("");
        }
    }
    // Gracefully shutdown connection to TSDB
    try {
        tsdb.shutdown().join();
    } catch (InterruptedException e) {
        e.printStackTrace();
    } catch (Exception e) {
        e.printStackTrace();
    }
}

Example 19 with TSSubQuery

use of net.opentsdb.core.TSSubQuery in project opentsdb by OpenTSDB.

the class HttpJsonSerializer method formatQueryAsyncV1.

/**
   * Format the results from a timeseries data query
   * @param data_query The TSQuery object used to fetch the results
   * @param results The data fetched from storage
   * @param globals An optional list of global annotation objects
   * @return A Deferred<ChannelBuffer> object to pass on to the caller
   * @throws IOException if serialization failed
   * @since 2.2
   */
public Deferred<ChannelBuffer> formatQueryAsyncV1(final TSQuery data_query, final List<DataPoints[]> results, final List<Annotation> globals) throws IOException {
    final long start = DateTime.currentTimeMillis();
    final boolean as_arrays = this.query.hasQueryStringParam("arrays");
    final String jsonp = this.query.getQueryStringParam("jsonp");
    // buffers and an array list to stored the deferreds
    final ChannelBuffer response = ChannelBuffers.dynamicBuffer();
    final OutputStream output = new ChannelBufferOutputStream(response);
    // too bad an inner class can't modify a primitive. This is a work around 
    final List<Boolean> timeout_flag = new ArrayList<Boolean>(1);
    timeout_flag.add(false);
    // start with JSONp if we're told to
    if (jsonp != null && !jsonp.isEmpty()) {
        output.write((jsonp + "(").getBytes(query.getCharset()));
    }
    // start the JSON generator and write the opening array
    final JsonGenerator json = JSON.getFactory().createGenerator(output);
    json.writeStartArray();
    /**
     * Every individual data point set (the result of a query and possibly a
     * group by) will initiate an asynchronous metric/tag UID to name resolution
     * and then print to the buffer.
     * NOTE that because this is asynchronous, the order of results is
     * indeterminate.
     */
    class DPsResolver implements Callback<Deferred<Object>, Object> {

        /** Has to be final to be shared with the nested classes */
        final StringBuilder metric = new StringBuilder(256);

        /** Resolved tags */
        final Map<String, String> tags = new HashMap<String, String>();

        /** Resolved aggregated tags */
        final List<String> agg_tags = new ArrayList<String>();

        /** A list storing the metric and tag resolve calls */
        final List<Deferred<Object>> resolve_deferreds = new ArrayList<Deferred<Object>>();

        /** The data points to serialize */
        final DataPoints dps;

        /** Starting time in nanos when we sent the UID resolution queries off */
        long uid_start;

        public DPsResolver(final DataPoints dps) {
            this.dps = dps;
        }

        /** Resolves the metric UID to a name*/
        class MetricResolver implements Callback<Object, String> {

            public Object call(final String metric) throws Exception {
                DPsResolver.this.metric.append(metric);
                return null;
            }
        }

        /** Resolves the tag UIDs to a key/value string set */
        class TagResolver implements Callback<Object, Map<String, String>> {

            public Object call(final Map<String, String> tags) throws Exception {
                DPsResolver.this.tags.putAll(tags);
                return null;
            }
        }

        /** Resolves aggregated tags */
        class AggTagResolver implements Callback<Object, List<String>> {

            public Object call(final List<String> tags) throws Exception {
                DPsResolver.this.agg_tags.addAll(tags);
                return null;
            }
        }

        /** After the metric and tags have been resolved, this will print the
       * results to the output buffer in the proper format.
       */
        class WriteToBuffer implements Callback<Object, ArrayList<Object>> {

            final DataPoints dps;

            /**
         * Default ctor that takes a data point set
         * @param dps Datapoints to print
         */
            public WriteToBuffer(final DataPoints dps) {
                this.dps = dps;
            }

            /**
         * Handles writing the data to the output buffer. The results of the
         * deferreds don't matter as they will be stored in the class final
         * variables.
         */
            public Object call(final ArrayList<Object> deferreds) throws Exception {
                data_query.getQueryStats().addStat(dps.getQueryIndex(), QueryStat.UID_TO_STRING_TIME, (DateTime.nanoTime() - uid_start));
                final long local_serialization_start = DateTime.nanoTime();
                final TSSubQuery orig_query = data_query.getQueries().get(dps.getQueryIndex());
                json.writeStartObject();
                json.writeStringField("metric", metric.toString());
                json.writeFieldName("tags");
                json.writeStartObject();
                if (dps.getTags() != null) {
                    for (Map.Entry<String, String> tag : tags.entrySet()) {
                        json.writeStringField(tag.getKey(), tag.getValue());
                    }
                }
                json.writeEndObject();
                json.writeFieldName("aggregateTags");
                json.writeStartArray();
                if (dps.getAggregatedTags() != null) {
                    for (String atag : agg_tags) {
                        json.writeString(atag);
                    }
                }
                json.writeEndArray();
                if (data_query.getShowQuery()) {
                    json.writeObjectField("query", orig_query);
                }
                if (data_query.getShowTSUIDs()) {
                    json.writeFieldName("tsuids");
                    json.writeStartArray();
                    final List<String> tsuids = dps.getTSUIDs();
                    Collections.sort(tsuids);
                    for (String tsuid : tsuids) {
                        json.writeString(tsuid);
                    }
                    json.writeEndArray();
                }
                if (!data_query.getNoAnnotations()) {
                    final List<Annotation> annotations = dps.getAnnotations();
                    if (annotations != null) {
                        Collections.sort(annotations);
                        json.writeArrayFieldStart("annotations");
                        for (Annotation note : annotations) {
                            json.writeObject(note);
                        }
                        json.writeEndArray();
                    }
                    if (globals != null && !globals.isEmpty()) {
                        Collections.sort(globals);
                        json.writeArrayFieldStart("globalAnnotations");
                        for (Annotation note : globals) {
                            json.writeObject(note);
                        }
                        json.writeEndArray();
                    }
                }
                // now the fun stuff, dump the data and time just the iteration over
                // the data points
                final long dps_start = DateTime.nanoTime();
                json.writeFieldName("dps");
                long counter = 0;
                // default is to write a map, otherwise write arrays
                if (!timeout_flag.get(0) && as_arrays) {
                    json.writeStartArray();
                    for (final DataPoint dp : dps) {
                        if (dp.timestamp() < data_query.startTime() || dp.timestamp() > data_query.endTime()) {
                            continue;
                        }
                        final long timestamp = data_query.getMsResolution() ? dp.timestamp() : dp.timestamp() / 1000;
                        json.writeStartArray();
                        json.writeNumber(timestamp);
                        if (dp.isInteger()) {
                            json.writeNumber(dp.longValue());
                        } else {
                            // Report missing intervals as null or NaN.
                            final double value = dp.doubleValue();
                            if (Double.isNaN(value) && orig_query.fillPolicy() == FillPolicy.NULL) {
                                json.writeNull();
                            } else {
                                json.writeNumber(dp.doubleValue());
                            }
                        }
                        json.writeEndArray();
                        ++counter;
                    }
                    json.writeEndArray();
                } else if (!timeout_flag.get(0)) {
                    json.writeStartObject();
                    for (final DataPoint dp : dps) {
                        if (dp.timestamp() < (data_query.startTime()) || dp.timestamp() > (data_query.endTime())) {
                            continue;
                        }
                        final long timestamp = data_query.getMsResolution() ? dp.timestamp() : dp.timestamp() / 1000;
                        if (dp.isInteger()) {
                            json.writeNumberField(Long.toString(timestamp), dp.longValue());
                        } else {
                            // Report missing intervals as null or NaN.
                            final double value = dp.doubleValue();
                            if (Double.isNaN(value) && orig_query.fillPolicy() == FillPolicy.NULL) {
                                json.writeNumberField(Long.toString(timestamp), null);
                            } else {
                                json.writeNumberField(Long.toString(timestamp), dp.doubleValue());
                            }
                        }
                        ++counter;
                    }
                    json.writeEndObject();
                } else {
                    // skipping data points all together due to timeout
                    json.writeStartObject();
                    json.writeEndObject();
                }
                final long agg_time = DateTime.nanoTime() - dps_start;
                data_query.getQueryStats().addStat(dps.getQueryIndex(), QueryStat.AGGREGATION_TIME, agg_time);
                data_query.getQueryStats().addStat(dps.getQueryIndex(), QueryStat.AGGREGATED_SIZE, counter);
                // yeah, it's a little early but we need to dump it out with the results.
                data_query.getQueryStats().addStat(dps.getQueryIndex(), QueryStat.SERIALIZATION_TIME, DateTime.nanoTime() - local_serialization_start);
                if (!timeout_flag.get(0) && data_query.getShowStats()) {
                    int query_index = (dps == null) ? -1 : dps.getQueryIndex();
                    QueryStats stats = data_query.getQueryStats();
                    if (query_index >= 0) {
                        json.writeFieldName("stats");
                        final Map<String, Object> s = stats.getQueryStats(query_index, false);
                        if (s != null) {
                            json.writeObject(s);
                        } else {
                            json.writeStringField("ERROR", "NO STATS FOUND");
                        }
                    }
                }
                // close the results for this particular query
                json.writeEndObject();
                return null;
            }
        }

        /**
       * When called, initiates a resolution of metric and tag UIDs to names, 
       * then prints to the output buffer once they are completed.
       */
        public Deferred<Object> call(final Object obj) throws Exception {
            this.uid_start = DateTime.nanoTime();
            resolve_deferreds.add(dps.metricNameAsync().addCallback(new MetricResolver()));
            resolve_deferreds.add(dps.getTagsAsync().addCallback(new TagResolver()));
            resolve_deferreds.add(dps.getAggregatedTagsAsync().addCallback(new AggTagResolver()));
            return Deferred.group(resolve_deferreds).addCallback(new WriteToBuffer(dps));
        }
    }
    // We want the serializer to execute serially so we need to create a callback
    // chain so that when one DPsResolver is finished, it triggers the next to
    // start serializing.
    final Deferred<Object> cb_chain = new Deferred<Object>();
    for (DataPoints[] separate_dps : results) {
        for (DataPoints dps : separate_dps) {
            try {
                cb_chain.addCallback(new DPsResolver(dps));
            } catch (Exception e) {
                throw new RuntimeException("Unexpected error durring resolution", e);
            }
        }
    }
    /** Final callback to close out the JSON array and return our results */
    class FinalCB implements Callback<ChannelBuffer, Object> {

        public ChannelBuffer call(final Object obj) throws Exception {
            // Call this here so we rollup sub metrics into a summary. It's not
            // completely accurate, of course, because we still have to write the
            // summary and close the writer. But it's close.
            data_query.getQueryStats().markSerializationSuccessful();
            // TODO - yeah, I've heard this sucks, we need to figure out a better way.
            if (data_query.getShowSummary()) {
                final QueryStats stats = data_query.getQueryStats();
                json.writeStartObject();
                json.writeFieldName("statsSummary");
                json.writeObject(stats.getStats(true, true));
                json.writeEndObject();
            }
            // IMPORTANT Make sure the close the JSON array and the generator
            json.writeEndArray();
            json.close();
            if (jsonp != null && !jsonp.isEmpty()) {
                output.write(")".getBytes());
            }
            return response;
        }
    }
    // trigger the callback chain here
    cb_chain.callback(null);
    return cb_chain.addCallback(new FinalCB());
}

Example 20 with TSSubQuery

use of net.opentsdb.core.TSSubQuery in project opentsdb by OpenTSDB.

the class BaseTimeSyncedIteratorTest method queryA_DD.

/**
   * Queries for A only with a filter of tag value "D" for tag key "D"
   */
protected void queryA_DD() throws Exception {
    final ArrayList<TSSubQuery> subs = new ArrayList<TSSubQuery>(2);
    TSSubQuery sub = new TSSubQuery();
    final HashMap<String, String> query_tags = new HashMap<String, String>(1);
    query_tags.put("D", "D");
    sub = new TSSubQuery();
    sub.setMetric("A");
    sub.setTags(query_tags);
    sub.setAggregator("sum");
    subs.add(sub);
    runQueries(subs);
}