Examples with DataPoint net.opentsdb.core.DataPoint used on opensource projects

Example 31 with DataPoint

use of net.opentsdb.core.DataPoint in project opentsdb by OpenTSDB.

the class QueryExample method main.

public static void main(final String[] args) throws IOException {
    // Set these as arguments so you don't have to keep path information in
    // source files 
    String pathToConfigFile = (args != null && args.length > 0 ? args[0] : null);
    // Create a config object with a path to the file for parsing. Or manually
    // override settings.
    // e.g. config.overrideConfig("tsd.storage.hbase.zk_quorum", "localhost");
    final Config config;
    if (pathToConfigFile != null && !pathToConfigFile.isEmpty()) {
        config = new Config(pathToConfigFile);
    } else {
        // Search for a default config from /etc/opentsdb/opentsdb.conf, etc.
        config = new Config(true);
    }
    final TSDB tsdb = new TSDB(config);
    // main query
    final TSQuery query = new TSQuery();
    // use any string format from
    // http://opentsdb.net/docs/build/html/user_guide/query/dates.html
    query.setStart("1h-ago");
    // Optional: set other global query params
    // at least one sub query required. This is where you specify the metric and
    // tags
    final TSSubQuery subQuery = new TSSubQuery();
    subQuery.setMetric("my.tsdb.test.metric");
    // filters are optional but useful.
    final List<TagVFilter> filters = new ArrayList<TagVFilter>(1);
    filters.add(new TagVFilter.Builder().setType("literal_or").setFilter("example1").setTagk("script").setGroupBy(true).build());
    subQuery.setFilters(filters);
    // you do have to set an aggregator. Just provide the name as a string
    subQuery.setAggregator("sum");
    // IMPORTANT: don't forget to add the subQuery
    final ArrayList<TSSubQuery> subQueries = new ArrayList<TSSubQuery>(1);
    subQueries.add(subQuery);
    query.setQueries(subQueries);
    // otherwise we aggregate on the second. 
    query.setMsResolution(true);
    // make sure the query is valid. This will throw exceptions if something
    // is missing
    query.validateAndSetQuery();
    // compile the queries into TsdbQuery objects behind the scenes
    Query[] tsdbqueries = query.buildQueries(tsdb);
    // create some arrays for storing the results and the async calls
    final int nqueries = tsdbqueries.length;
    final ArrayList<DataPoints[]> results = new ArrayList<DataPoints[]>(nqueries);
    final ArrayList<Deferred<DataPoints[]>> deferreds = new ArrayList<Deferred<DataPoints[]>>(nqueries);
    // deferred in an array so we can wait for them to complete.
    for (int i = 0; i < nqueries; i++) {
        deferreds.add(tsdbqueries[i].runAsync());
    }
    // Start timer
    long startTime = DateTime.nanoTime();
    // query has finished
    class QueriesCB implements Callback<Object, ArrayList<DataPoints[]>> {

        public Object call(final ArrayList<DataPoints[]> queryResults) throws Exception {
            results.addAll(queryResults);
            return null;
        }
    }
    // Make sure to handle any errors that might crop up
    class QueriesEB implements Callback<Object, Exception> {

        @Override
        public Object call(final Exception e) throws Exception {
            System.err.println("Queries failed");
            e.printStackTrace();
            return null;
        }
    }
    // have completed.
    try {
        Deferred.groupInOrder(deferreds).addCallback(new QueriesCB()).addErrback(new QueriesEB()).join();
    } catch (Exception e) {
        e.printStackTrace();
    }
    // End timer.
    double elapsedTime = DateTime.msFromNanoDiff(DateTime.nanoTime(), startTime);
    System.out.println("Query returned in: " + elapsedTime + " milliseconds.");
    // results and do any processing necessary.
    for (final DataPoints[] dataSets : results) {
        for (final DataPoints data : dataSets) {
            System.out.print(data.metricName());
            Map<String, String> resolvedTags = data.getTags();
            for (final Map.Entry<String, String> pair : resolvedTags.entrySet()) {
                System.out.print(" " + pair.getKey() + "=" + pair.getValue());
            }
            System.out.print("\n");
            final SeekableView it = data.iterator();
            /*
         * An important point about SeekableView:
         * Because no data is copied during iteration and no new object gets
         * created, the DataPoint returned must not be stored and gets
         * invalidated as soon as next is called on the iterator (actually it
         * doesn't get invalidated but rather its contents changes). If you want
         * to store individual data points, you need to copy the timestamp and
         * value out of each DataPoint into your own data structures.
         * 
         * In the vast majority of cases, the iterator will be used to go once
         * through all the data points, which is why it's not a problem if the
         * iterator acts just as a transient "view". Iterating will be very
         * cheap since no memory allocation is required (except to instantiate
         * the actual iterator at the beginning).
         */
            while (it.hasNext()) {
                final DataPoint dp = it.next();
                System.out.println("  " + dp.timestamp() + " " + (dp.isInteger() ? dp.longValue() : dp.doubleValue()));
            }
            System.out.println("");
        }
    }
    // Gracefully shutdown connection to TSDB
    try {
        tsdb.shutdown().join();
    } catch (InterruptedException e) {
        e.printStackTrace();
    } catch (Exception e) {
        e.printStackTrace();
    }
}

Example 32 with DataPoint

use of net.opentsdb.core.DataPoint in project opentsdb by OpenTSDB.

the class Plot method dumpToFiles.

/**
   * Generates the Gnuplot script and data files.
   * @param basepath The base path to use.  A number of new files will be
   * created and their names will all start with this string.
   * @return The number of data points sent to Gnuplot.  This can be less
   * than the number of data points involved in the query due to things like
   * aggregation or downsampling.
   * @throws IOException if there was an error while writing one of the files.
   */
public int dumpToFiles(final String basepath) throws IOException {
    int npoints = 0;
    final int nseries = datapoints.size();
    final String[] datafiles = nseries > 0 ? new String[nseries] : null;
    FileSystem.checkDirectory(new File(basepath).getParent(), Const.MUST_BE_WRITEABLE, Const.CREATE_IF_NEEDED);
    for (int i = 0; i < nseries; i++) {
        datafiles[i] = basepath + "_" + i + ".dat";
        final PrintWriter datafile = new PrintWriter(datafiles[i]);
        try {
            for (final DataPoint d : datapoints.get(i)) {
                final long ts = d.timestamp() / 1000;
                if (d.isInteger()) {
                    datafile.print(ts + utc_offset);
                    datafile.print(' ');
                    datafile.print(d.longValue());
                } else {
                    final double value = d.doubleValue();
                    if (Double.isInfinite(value)) {
                        // Infinity is invalid.
                        throw new IllegalStateException("Infinity found in" + " datapoints #" + i + ": " + value + " d=" + d);
                    } else if (Double.isNaN(value)) {
                        // NaNs should be skipped.
                        continue;
                    }
                    datafile.print(ts + utc_offset);
                    datafile.print(' ');
                    datafile.print(value);
                }
                datafile.print('\n');
                if (ts >= (start_time & UNSIGNED) && ts <= (end_time & UNSIGNED)) {
                    npoints++;
                }
            }
        } finally {
            datafile.close();
        }
    }
    if (npoints == 0) {
        // Gnuplot doesn't like empty graphs when xrange and yrange aren't
        // entirely defined, because it can't decide on good ranges with no
        // data.  We always set the xrange, but the yrange is supplied by the
        // user.  Let's make sure it defines a min and a max.
        // Doesn't matter what values we use.
        params.put("yrange", "[0:10]");
    }
    writeGnuplotScript(basepath, datafiles);
    return npoints;
}

Example 33 with DataPoint

use of net.opentsdb.core.DataPoint in project opentsdb by OpenTSDB.

the class HighestCurrent method evaluate.

@Override
public DataPoints[] evaluate(final TSQuery data_query, final List<DataPoints[]> query_results, final List<String> params) {
    if (data_query == null) {
        throw new IllegalArgumentException("Missing time series query");
    }
    if (query_results == null || query_results.isEmpty()) {
        return new DataPoints[] {};
    }
    // TODO(cl) - allow for empty top-n maybe? Just sort the results by max?
    if (params == null || params.isEmpty()) {
        throw new IllegalArgumentException("Need aggregation window for moving average");
    }
    String param = params.get(0);
    if (param == null || param.length() == 0) {
        throw new IllegalArgumentException("Missing top n value " + "(number of series to return)");
    }
    int topn = 0;
    if (param.matches("^[0-9]+$")) {
        try {
            topn = Integer.parseInt(param);
        } catch (NumberFormatException nfe) {
            throw new IllegalArgumentException("Invalid parameter, must be an integer", nfe);
        }
    } else {
        throw new IllegalArgumentException("Unparseable top n value: " + param);
    }
    if (topn < 1) {
        throw new IllegalArgumentException("Top n value must be greater " + "than zero: " + topn);
    }
    int num_results = 0;
    for (DataPoints[] results : query_results) {
        num_results += results.length;
    }
    final PostAggregatedDataPoints[] post_agg_results = new PostAggregatedDataPoints[num_results];
    int ix = 0;
    // one or more sub queries (m=...&m=...&m=...)
    for (final DataPoints[] sub_query_result : query_results) {
        // group bys (m=sum:foo{host=*})
        for (final DataPoints dps : sub_query_result) {
            // TODO(cl) - Avoid iterating and copying if we can help it. We should
            // be able to pass the original DataPoints object to the seekable view
            // and then iterate through it.
            final List<DataPoint> mutable_points = new ArrayList<DataPoint>();
            for (final DataPoint point : dps) {
                mutable_points.add(point.isInteger() ? MutableDataPoint.ofLongValue(point.timestamp(), point.longValue()) : MutableDataPoint.ofDoubleValue(point.timestamp(), point.doubleValue()));
            }
            post_agg_results[ix++] = new PostAggregatedDataPoints(dps, mutable_points.toArray(new DataPoint[mutable_points.size()]));
        }
    }
    final SeekableView[] views = new SeekableView[num_results];
    for (int i = 0; i < num_results; i++) {
        views[i] = post_agg_results[i].iterator();
    }
    final MaxLatestAggregator aggregator = new MaxLatestAggregator(Aggregators.Interpolation.LERP, "maxLatest", num_results, data_query.startTime(), data_query.endTime());
    final SeekableView view = (new AggregationIterator(views, data_query.startTime(), data_query.endTime(), aggregator, Aggregators.Interpolation.LERP, false));
    // slurp all the points even though we aren't using them at this stage
    while (view.hasNext()) {
        final DataPoint mdp = view.next();
        @SuppressWarnings("unused") final Object o = mdp.isInteger() ? mdp.longValue() : mdp.doubleValue();
    }
    final long[] max_longs = aggregator.getLongMaxes();
    final double[] max_doubles = aggregator.getDoubleMaxes();
    final TopNSortingEntry[] max_by_ts = new TopNSortingEntry[num_results];
    if (aggregator.hasDoubles() && aggregator.hasLongs()) {
        for (int i = 0; i < num_results; i++) {
            max_by_ts[i] = new TopNSortingEntry(Math.max((double) max_longs[i], max_doubles[i]), i);
        }
    } else if (aggregator.hasLongs() && !aggregator.hasDoubles()) {
        for (int i = 0; i < num_results; i++) {
            max_by_ts[i] = new TopNSortingEntry((double) max_longs[i], i);
        }
    } else if (aggregator.hasDoubles() && !aggregator.hasLongs()) {
        for (int i = 0; i < num_results; i++) {
            max_by_ts[i] = new TopNSortingEntry(max_doubles[i], i);
        }
    }
    Arrays.sort(max_by_ts);
    final int result_count = Math.min(topn, num_results);
    final DataPoints[] results = new DataPoints[result_count];
    for (int i = 0; i < result_count; i++) {
        results[i] = post_agg_results[max_by_ts[i].pos];
    }
    return results;
}

Example 34 with DataPoint

use of net.opentsdb.core.DataPoint in project opentsdb by OpenTSDB.

the class HighestMax method evaluate.

@Override
public DataPoints[] evaluate(final TSQuery data_query, final List<DataPoints[]> query_results, final List<String> params) {
    if (data_query == null) {
        throw new IllegalArgumentException("Missing time series query");
    }
    if (query_results == null || query_results.isEmpty()) {
        return new DataPoints[] {};
    }
    // TODO(cl) - allow for empty top-n maybe? Just sort the results by max?
    if (params == null || params.isEmpty()) {
        throw new IllegalArgumentException("Need aggregation window for moving average");
    }
    String param = params.get(0);
    if (param == null || param.length() == 0) {
        throw new IllegalArgumentException("Missing top n value " + "(number of series to return)");
    }
    int topn = 0;
    if (param.matches("^[0-9]+$")) {
        try {
            topn = Integer.parseInt(param);
        } catch (NumberFormatException nfe) {
            throw new IllegalArgumentException("Invalid parameter, must be an integer", nfe);
        }
    } else {
        throw new IllegalArgumentException("Unparseable top n value: " + param);
    }
    if (topn < 1) {
        throw new IllegalArgumentException("Top n value must be greater " + "than zero: " + topn);
    }
    int num_results = 0;
    for (DataPoints[] results : query_results) {
        num_results += results.length;
    }
    final PostAggregatedDataPoints[] post_agg_results = new PostAggregatedDataPoints[num_results];
    int ix = 0;
    // one or more sub queries (m=...&m=...&m=...)
    for (final DataPoints[] sub_query_result : query_results) {
        // group bys (m=sum:foo{host=*})
        for (final DataPoints dps : sub_query_result) {
            // TODO(cl) - Avoid iterating and copying if we can help it. We should
            // be able to pass the original DataPoints object to the seekable view
            // and then iterate through it.
            final List<DataPoint> mutable_points = new ArrayList<DataPoint>();
            for (final DataPoint point : dps) {
                mutable_points.add(point.isInteger() ? MutableDataPoint.ofLongValue(point.timestamp(), point.longValue()) : MutableDataPoint.ofDoubleValue(point.timestamp(), point.doubleValue()));
            }
            post_agg_results[ix++] = new PostAggregatedDataPoints(dps, mutable_points.toArray(new DataPoint[mutable_points.size()]));
        }
    }
    final SeekableView[] views = new SeekableView[num_results];
    for (int i = 0; i < num_results; i++) {
        views[i] = post_agg_results[i].iterator();
    }
    final MaxCacheAggregator aggregator = new MaxCacheAggregator(Aggregators.Interpolation.LERP, "maxCache", num_results, data_query.startTime(), data_query.endTime());
    final SeekableView view = (new AggregationIterator(views, data_query.startTime(), data_query.endTime(), aggregator, Aggregators.Interpolation.LERP, false));
    // slurp all the points even though we aren't using them at this stage
    while (view.hasNext()) {
        final DataPoint mdp = view.next();
        @SuppressWarnings("unused") final Object o = mdp.isInteger() ? mdp.longValue() : mdp.doubleValue();
    }
    final long[] max_longs = aggregator.getLongMaxes();
    final double[] max_doubles = aggregator.getDoubleMaxes();
    final TopNSortingEntry[] max_by_ts = new TopNSortingEntry[num_results];
    if (aggregator.hasDoubles() && aggregator.hasLongs()) {
        for (int i = 0; i < num_results; i++) {
            max_by_ts[i] = new TopNSortingEntry(Math.max((double) max_longs[i], max_doubles[i]), i);
        }
    } else if (aggregator.hasLongs() && !aggregator.hasDoubles()) {
        for (int i = 0; i < num_results; i++) {
            max_by_ts[i] = new TopNSortingEntry((double) max_longs[i], i);
        }
    } else if (aggregator.hasDoubles() && !aggregator.hasLongs()) {
        for (int i = 0; i < num_results; i++) {
            max_by_ts[i] = new TopNSortingEntry(max_doubles[i], i);
        }
    }
    Arrays.sort(max_by_ts);
    final int result_count = Math.min(topn, num_results);
    final DataPoints[] results = new DataPoints[result_count];
    for (int i = 0; i < result_count; i++) {
        results[i] = post_agg_results[max_by_ts[i].pos];
    }
    return results;
}

Example 35 with DataPoint

use of net.opentsdb.core.DataPoint in project opentsdb by OpenTSDB.

the class MovingAverage method evaluate.

@Override
public DataPoints[] evaluate(final TSQuery data_query, final List<DataPoints[]> query_results, final List<String> params) {
    if (data_query == null) {
        throw new IllegalArgumentException("Missing time series query");
    }
    if (query_results == null || query_results.isEmpty()) {
        return new DataPoints[] {};
    }
    if (params == null || params.isEmpty()) {
        throw new IllegalArgumentException("Missing moving average window size");
    }
    String param = params.get(0);
    if (param == null || param.isEmpty()) {
        throw new IllegalArgumentException("Missing moving average window size");
    }
    param = param.trim();
    long condition = -1;
    boolean is_time_unit = false;
    if (param.matches("^[0-9]+$")) {
        try {
            condition = Integer.parseInt(param);
        } catch (NumberFormatException nfe) {
            throw new IllegalArgumentException("Invalid parameter, must be an integer", nfe);
        }
    } else if (param.startsWith("'") && param.endsWith("'")) {
        condition = parseParam(param);
        is_time_unit = true;
    } else {
        throw new IllegalArgumentException("Unparseable window size: " + param);
    }
    if (condition <= 0) {
        throw new IllegalArgumentException("Moving average window must be an " + "integer greater than zero");
    }
    int num_results = 0;
    for (final DataPoints[] results : query_results) {
        num_results += results.length;
    }
    final PostAggregatedDataPoints[] post_agg_results = new PostAggregatedDataPoints[num_results];
    int ix = 0;
    // one or more queries (m=...&m=...&m=...)
    for (final DataPoints[] sub_query_result : query_results) {
        // group bys (m=sum:foo{host=*})
        for (final DataPoints dps : sub_query_result) {
            // TODO(cl) - Avoid iterating and copying if we can help it. We should
            // be able to pass the original DataPoints object to the seekable view
            // and then iterate through it.
            final List<DataPoint> mutable_points = new ArrayList<DataPoint>();
            for (final DataPoint point : dps) {
                // avoid flip-flopping between integers and floats, always use double
                // for average.
                mutable_points.add(MutableDataPoint.ofDoubleValue(point.timestamp(), point.toDouble()));
            }
            post_agg_results[ix++] = new PostAggregatedDataPoints(dps, mutable_points.toArray(new DataPoint[mutable_points.size()]));
        }
    }
    final DataPoints[] results = new DataPoints[num_results];
    for (int i = 0; i < num_results; i++) {
        final Aggregator moving_average = new MovingAverageAggregator(Aggregators.Interpolation.LERP, "movingAverage", condition, is_time_unit);
        final SeekableView[] metrics_groups = new SeekableView[] { post_agg_results[i].iterator() };
        final SeekableView view = new AggregationIterator(metrics_groups, data_query.startTime(), data_query.endTime(), moving_average, Aggregators.Interpolation.LERP, false);
        final List<DataPoint> points = new ArrayList<DataPoint>();
        while (view.hasNext()) {
            final DataPoint mdp = view.next();
            points.add(MutableDataPoint.ofDoubleValue(mdp.timestamp(), mdp.toDouble()));
        }
        results[i] = new PostAggregatedDataPoints(post_agg_results[i], points.toArray(new DataPoint[points.size()]));
    }
    return results;
}