Examples with TSDB net.opentsdb.core.TSDB used on opensource projects

Example 21 with TSDB

use of net.opentsdb.core.TSDB in project opentsdb by OpenTSDB.

the class QueryExample method main.

public static void main(final String[] args) throws IOException {
    // Set these as arguments so you don't have to keep path information in
    // source files 
    String pathToConfigFile = (args != null && args.length > 0 ? args[0] : null);
    // Create a config object with a path to the file for parsing. Or manually
    // override settings.
    // e.g. config.overrideConfig("tsd.storage.hbase.zk_quorum", "localhost");
    final Config config;
    if (pathToConfigFile != null && !pathToConfigFile.isEmpty()) {
        config = new Config(pathToConfigFile);
    } else {
        // Search for a default config from /etc/opentsdb/opentsdb.conf, etc.
        config = new Config(true);
    }
    final TSDB tsdb = new TSDB(config);
    // main query
    final TSQuery query = new TSQuery();
    // use any string format from
    // http://opentsdb.net/docs/build/html/user_guide/query/dates.html
    query.setStart("1h-ago");
    // Optional: set other global query params
    // at least one sub query required. This is where you specify the metric and
    // tags
    final TSSubQuery subQuery = new TSSubQuery();
    subQuery.setMetric("my.tsdb.test.metric");
    // filters are optional but useful.
    final List<TagVFilter> filters = new ArrayList<TagVFilter>(1);
    filters.add(new TagVFilter.Builder().setType("literal_or").setFilter("example1").setTagk("script").setGroupBy(true).build());
    subQuery.setFilters(filters);
    // you do have to set an aggregator. Just provide the name as a string
    subQuery.setAggregator("sum");
    // IMPORTANT: don't forget to add the subQuery
    final ArrayList<TSSubQuery> subQueries = new ArrayList<TSSubQuery>(1);
    subQueries.add(subQuery);
    query.setQueries(subQueries);
    // otherwise we aggregate on the second. 
    query.setMsResolution(true);
    // make sure the query is valid. This will throw exceptions if something
    // is missing
    query.validateAndSetQuery();
    // compile the queries into TsdbQuery objects behind the scenes
    Query[] tsdbqueries = query.buildQueries(tsdb);
    // create some arrays for storing the results and the async calls
    final int nqueries = tsdbqueries.length;
    final ArrayList<DataPoints[]> results = new ArrayList<DataPoints[]>(nqueries);
    final ArrayList<Deferred<DataPoints[]>> deferreds = new ArrayList<Deferred<DataPoints[]>>(nqueries);
    // deferred in an array so we can wait for them to complete.
    for (int i = 0; i < nqueries; i++) {
        deferreds.add(tsdbqueries[i].runAsync());
    }
    // Start timer
    long startTime = DateTime.nanoTime();
    // query has finished
    class QueriesCB implements Callback<Object, ArrayList<DataPoints[]>> {

        public Object call(final ArrayList<DataPoints[]> queryResults) throws Exception {
            results.addAll(queryResults);
            return null;
        }
    }
    // Make sure to handle any errors that might crop up
    class QueriesEB implements Callback<Object, Exception> {

        @Override
        public Object call(final Exception e) throws Exception {
            System.err.println("Queries failed");
            e.printStackTrace();
            return null;
        }
    }
    // have completed.
    try {
        Deferred.groupInOrder(deferreds).addCallback(new QueriesCB()).addErrback(new QueriesEB()).join();
    } catch (Exception e) {
        e.printStackTrace();
    }
    // End timer.
    double elapsedTime = DateTime.msFromNanoDiff(DateTime.nanoTime(), startTime);
    System.out.println("Query returned in: " + elapsedTime + " milliseconds.");
    // results and do any processing necessary.
    for (final DataPoints[] dataSets : results) {
        for (final DataPoints data : dataSets) {
            System.out.print(data.metricName());
            Map<String, String> resolvedTags = data.getTags();
            for (final Map.Entry<String, String> pair : resolvedTags.entrySet()) {
                System.out.print(" " + pair.getKey() + "=" + pair.getValue());
            }
            System.out.print("\n");
            final SeekableView it = data.iterator();
            /*
         * An important point about SeekableView:
         * Because no data is copied during iteration and no new object gets
         * created, the DataPoint returned must not be stored and gets
         * invalidated as soon as next is called on the iterator (actually it
         * doesn't get invalidated but rather its contents changes). If you want
         * to store individual data points, you need to copy the timestamp and
         * value out of each DataPoint into your own data structures.
         * 
         * In the vast majority of cases, the iterator will be used to go once
         * through all the data points, which is why it's not a problem if the
         * iterator acts just as a transient "view". Iterating will be very
         * cheap since no memory allocation is required (except to instantiate
         * the actual iterator at the beginning).
         */
            while (it.hasNext()) {
                final DataPoint dp = it.next();
                System.out.println("  " + dp.timestamp() + " " + (dp.isInteger() ? dp.longValue() : dp.doubleValue()));
            }
            System.out.println("");
        }
    }
    // Gracefully shutdown connection to TSDB
    try {
        tsdb.shutdown().join();
    } catch (InterruptedException e) {
        e.printStackTrace();
    } catch (Exception e) {
        e.printStackTrace();
    }
}

Example 22 with TSDB

use of net.opentsdb.core.TSDB in project opentsdb by OpenTSDB.

the class Fsck method main.

/**
   * The main class executed from the "tsdb" script
   * @param args Command line arguments to parse
   * @throws Exception If something goes pear shaped
   */
public static void main(String[] args) throws Exception {
    ArgP argp = new ArgP();
    argp.addOption("--help", "Print help information.");
    CliOptions.addCommon(argp);
    FsckOptions.addDataOptions(argp);
    args = CliOptions.parse(argp, args);
    if (argp.has("--help")) {
        usage(argp, "", 0);
    }
    Config config = CliOptions.getConfig(argp);
    final FsckOptions options = new FsckOptions(argp, config);
    final TSDB tsdb = new TSDB(config);
    final ArrayList<Query> queries = new ArrayList<Query>();
    if (args != null && args.length > 0) {
        CliQuery.parseCommandLineQuery(args, tsdb, queries, null, null);
    }
    if (queries.isEmpty() && !argp.has("--full-scan")) {
        usage(argp, "Must supply a query or use the '--full-scan' flag", 1);
    }
    tsdb.checkNecessaryTablesExist().joinUninterruptibly();
    argp = null;
    final Fsck fsck = new Fsck(tsdb, options);
    try {
        if (!queries.isEmpty()) {
            fsck.runQueries(queries);
        } else {
            fsck.runFullTable();
        }
    } finally {
        tsdb.shutdown().joinUninterruptibly();
    }
    System.exit(fsck.totalErrors() == 0 ? 0 : 1);
}

Example 23 with TSDB

use of net.opentsdb.core.TSDB in project opentsdb by OpenTSDB.

the class Search method main.

/**
   * Entry point to run the search utility
   * @param args Command line arguments
   * @throws Exception If something goes wrong
   */
public static void main(String[] args) throws Exception {
    ArgP argp = new ArgP();
    CliOptions.addCommon(argp);
    argp.addOption("--use-data-table", "Scan against the raw data table instead of the meta data table.");
    args = CliOptions.parse(argp, args);
    if (args == null) {
        usage(argp, "Invalid usage");
        System.exit(2);
    } else if (args.length < 1) {
        usage(argp, "Not enough arguments");
        System.exit(2);
    }
    final boolean use_data_table = argp.has("--use-data-table");
    Config config = CliOptions.getConfig(argp);
    final TSDB tsdb = new TSDB(config);
    tsdb.checkNecessaryTablesExist().joinUninterruptibly();
    int rc;
    try {
        rc = runCommand(tsdb, use_data_table, args);
    } finally {
        try {
            tsdb.getClient().shutdown().joinUninterruptibly();
            LOG.info("Gracefully shutdown the TSD");
        } catch (Exception e) {
            LOG.error("Unexpected exception while shutting down", e);
            rc = 42;
        }
    }
    System.exit(rc);
}

Example 24 with TSDB

use of net.opentsdb.core.TSDB in project opentsdb by OpenTSDB.

the class TSDMain method main.

public static void main(String[] args) throws IOException {
    Logger log = LoggerFactory.getLogger(TSDMain.class);
    log.info("Starting.");
    log.info(BuildData.revisionString());
    log.info(BuildData.buildString());
    try {
        // Release a FD we don't need.
        System.in.close();
    } catch (Exception e) {
        log.warn("Failed to close stdin", e);
    }
    final ArgP argp = new ArgP();
    CliOptions.addCommon(argp);
    argp.addOption("--port", "NUM", "TCP port to listen on.");
    argp.addOption("--bind", "ADDR", "Address to bind to (default: 0.0.0.0).");
    argp.addOption("--staticroot", "PATH", "Web root from which to serve static files (/s URLs).");
    argp.addOption("--cachedir", "PATH", "Directory under which to cache result of requests.");
    argp.addOption("--worker-threads", "NUM", "Number for async io workers (default: cpu * 2).");
    argp.addOption("--async-io", "true|false", "Use async NIO (default true) or traditional blocking io");
    argp.addOption("--read-only", "true|false", "Set tsd.mode to ro (default false)");
    argp.addOption("--disable-ui", "true|false", "Set tsd.core.enable_ui to false (default true)");
    argp.addOption("--disable-api", "true|false", "Set tsd.core.enable_api to false (default true)");
    argp.addOption("--backlog", "NUM", "Size of connection attempt queue (default: 3072 or kernel" + " somaxconn.");
    argp.addOption("--max-connections", "NUM", "Maximum number of connections to accept");
    argp.addOption("--flush-interval", "MSEC", "Maximum time for which a new data point can be buffered" + " (default: " + DEFAULT_FLUSH_INTERVAL + ").");
    argp.addOption("--statswport", "Force all stats to include the port");
    CliOptions.addAutoMetricFlag(argp);
    args = CliOptions.parse(argp, args);
    // free().
    args = null;
    // get a config object
    Config config = CliOptions.getConfig(argp);
    // check for the required parameters
    try {
        if (config.getString("tsd.http.staticroot").isEmpty())
            usage(argp, "Missing static root directory", 1);
    } catch (NullPointerException npe) {
        usage(argp, "Missing static root directory", 1);
    }
    try {
        if (config.getString("tsd.http.cachedir").isEmpty())
            usage(argp, "Missing cache directory", 1);
    } catch (NullPointerException npe) {
        usage(argp, "Missing cache directory", 1);
    }
    try {
        if (!config.hasProperty("tsd.network.port"))
            usage(argp, "Missing network port", 1);
        config.getInt("tsd.network.port");
    } catch (NumberFormatException nfe) {
        usage(argp, "Invalid network port setting", 1);
    }
    // validate the cache and staticroot directories
    try {
        FileSystem.checkDirectory(config.getString("tsd.http.staticroot"), !Const.MUST_BE_WRITEABLE, Const.DONT_CREATE);
        FileSystem.checkDirectory(config.getString("tsd.http.cachedir"), Const.MUST_BE_WRITEABLE, Const.CREATE_IF_NEEDED);
    } catch (IllegalArgumentException e) {
        usage(argp, e.getMessage(), 3);
    }
    final ServerSocketChannelFactory factory;
    int connections_limit = 0;
    try {
        connections_limit = config.getInt("tsd.core.connections.limit");
    } catch (NumberFormatException nfe) {
        usage(argp, "Invalid connections limit", 1);
    }
    if (config.getBoolean("tsd.network.async_io")) {
        int workers = Runtime.getRuntime().availableProcessors() * 2;
        if (config.hasProperty("tsd.network.worker_threads")) {
            try {
                workers = config.getInt("tsd.network.worker_threads");
            } catch (NumberFormatException nfe) {
                usage(argp, "Invalid worker thread count", 1);
            }
        }
        final Executor executor = Executors.newCachedThreadPool();
        final NioServerBossPool boss_pool = new NioServerBossPool(executor, 1, new Threads.BossThreadNamer());
        final NioWorkerPool worker_pool = new NioWorkerPool(executor, workers, new Threads.WorkerThreadNamer());
        factory = new NioServerSocketChannelFactory(boss_pool, worker_pool);
    } else {
        factory = new OioServerSocketChannelFactory(Executors.newCachedThreadPool(), Executors.newCachedThreadPool(), new Threads.PrependThreadNamer());
    }
    StartupPlugin startup = null;
    try {
        startup = loadStartupPlugins(config);
    } catch (IllegalArgumentException e) {
        usage(argp, e.getMessage(), 3);
    } catch (Exception e) {
        throw new RuntimeException("Initialization failed", e);
    }
    try {
        tsdb = new TSDB(config);
        if (startup != null) {
            tsdb.setStartupPlugin(startup);
        }
        tsdb.initializePlugins(true);
        if (config.getBoolean("tsd.storage.hbase.prefetch_meta")) {
            tsdb.preFetchHBaseMeta();
        }
        // Make sure we don't even start if we can't find our tables.
        tsdb.checkNecessaryTablesExist().joinUninterruptibly();
        registerShutdownHook();
        final ServerBootstrap server = new ServerBootstrap(factory);
        // This manager is capable of lazy init, but we force an init
        // here to fail fast.
        final RpcManager manager = RpcManager.instance(tsdb);
        server.setPipelineFactory(new PipelineFactory(tsdb, manager, connections_limit));
        if (config.hasProperty("tsd.network.backlog")) {
            server.setOption("backlog", config.getInt("tsd.network.backlog"));
        }
        server.setOption("child.tcpNoDelay", config.getBoolean("tsd.network.tcp_no_delay"));
        server.setOption("child.keepAlive", config.getBoolean("tsd.network.keep_alive"));
        server.setOption("reuseAddress", config.getBoolean("tsd.network.reuse_address"));
        // null is interpreted as the wildcard address.
        InetAddress bindAddress = null;
        if (config.hasProperty("tsd.network.bind")) {
            bindAddress = InetAddress.getByName(config.getString("tsd.network.bind"));
        }
        // we validated the network port config earlier
        final InetSocketAddress addr = new InetSocketAddress(bindAddress, config.getInt("tsd.network.port"));
        server.bind(addr);
        if (startup != null) {
            startup.setReady(tsdb);
        }
        log.info("Ready to serve on " + addr);
    } catch (Throwable e) {
        factory.releaseExternalResources();
        try {
            if (tsdb != null)
                tsdb.shutdown().joinUninterruptibly();
        } catch (Exception e2) {
            log.error("Failed to shutdown HBase client", e2);
        }
        throw new RuntimeException("Initialization failed", e);
    }
// The server is now running in separate threads, we can exit main.
}

Example 25 with TSDB

use of net.opentsdb.core.TSDB in project opentsdb by OpenTSDB.

the class DumpSeries method main.

public static void main(String[] args) throws Exception {
    ArgP argp = new ArgP();
    CliOptions.addCommon(argp);
    argp.addOption("--import", "Prints the rows in a format suitable for" + " the 'import' command.");
    argp.addOption("--delete", "Deletes rows as they are scanned.");
    args = CliOptions.parse(argp, args);
    if (args == null) {
        usage(argp, "Invalid usage.", 1);
    } else if (args.length < 3) {
        usage(argp, "Not enough arguments.", 2);
    }
    // get a config object
    Config config = CliOptions.getConfig(argp);
    final TSDB tsdb = new TSDB(config);
    tsdb.checkNecessaryTablesExist().joinUninterruptibly();
    final byte[] table = config.getString("tsd.storage.hbase.data_table").getBytes();
    final boolean delete = argp.has("--delete");
    final boolean importformat = delete || argp.has("--import");
    argp = null;
    try {
        doDump(tsdb, tsdb.getClient(), table, delete, importformat, args);
    } finally {
        tsdb.shutdown().joinUninterruptibly();
    }
}