Examples with TreeBuilder net.opentsdb.tree.TreeBuilder used on opensource projects

Example 1 with TreeBuilder

use of net.opentsdb.tree.TreeBuilder in project opentsdb by OpenTSDB.

the class TreeRpc method handleTest.

/**
   * Runs the specified TSMeta object through a tree's rule set to determine
   * what the results would be or debug a meta that wasn't added to a tree
   * successfully
   * @param tsdb The TSDB to which we belong
   * @param query The HTTP query to work with
   * @throws BadRequestException if the request was invalid.
   */
private void handleTest(TSDB tsdb, HttpQuery query) {
    final Map<String, Object> map;
    if (query.hasContent()) {
        map = query.serializer().parseTreeTSUIDsListV1();
    } else {
        map = parseTSUIDsList(query);
    }
    final Integer tree_id = (Integer) map.get("treeId");
    if (tree_id == null) {
        throw new BadRequestException("Missing or invalid Tree ID");
    }
    // make sure the tree exists
    Tree tree = null;
    try {
        tree = Tree.fetchTree(tsdb, tree_id).joinUninterruptibly();
        if (tree == null) {
            throw new BadRequestException(HttpResponseStatus.NOT_FOUND, "Unable to locate tree: " + tree_id);
        }
        // ugly, but keeps from having to create a dedicated class just to 
        // convert one field.
        @SuppressWarnings("unchecked") final List<String> tsuids = (List<String>) map.get("tsuids");
        if (tsuids == null || tsuids.isEmpty()) {
            throw new BadRequestException("Missing or empty TSUID list");
        }
        if (query.getAPIMethod() == HttpMethod.GET || query.getAPIMethod() == HttpMethod.POST || query.getAPIMethod() == HttpMethod.PUT) {
            final HashMap<String, HashMap<String, Object>> results = new HashMap<String, HashMap<String, Object>>(tsuids.size());
            final TreeBuilder builder = new TreeBuilder(tsdb, tree);
            for (String tsuid : tsuids) {
                final HashMap<String, Object> tsuid_results = new HashMap<String, Object>();
                try {
                    final TSMeta meta = TSMeta.getTSMeta(tsdb, tsuid).joinUninterruptibly();
                    // message to the results and move on to the next TSUID
                    if (meta == null) {
                        tsuid_results.put("branch", null);
                        tsuid_results.put("meta", null);
                        final ArrayList<String> messages = new ArrayList<String>(1);
                        messages.add("Unable to locate TSUID meta data");
                        tsuid_results.put("messages", messages);
                        results.put(tsuid, tsuid_results);
                        continue;
                    }
                    builder.processTimeseriesMeta(meta, true).joinUninterruptibly();
                    tsuid_results.put("branch", builder.getRootBranch());
                    tsuid_results.put("meta", meta);
                    tsuid_results.put("messages", builder.getTestMessage());
                    results.put(tsuid, tsuid_results);
                } catch (DeferredGroupException e) {
                    // we want to catch NSU errors and handle them gracefully for
                    // TSUIDs where they may have been deleted
                    Throwable ex = e;
                    while (ex.getClass().equals(DeferredGroupException.class)) {
                        ex = ex.getCause();
                    }
                    if (ex.getClass().equals(NoSuchUniqueId.class)) {
                        tsuid_results.put("branch", null);
                        tsuid_results.put("meta", null);
                        final ArrayList<String> messages = new ArrayList<String>(1);
                        messages.add("TSUID was missing a UID name: " + ex.getMessage());
                        tsuid_results.put("messages", messages);
                        results.put(tsuid, tsuid_results);
                    }
                }
            }
            query.sendReply(query.serializer().formatTreeTestV1(results));
        } else {
            throw new BadRequestException(HttpResponseStatus.BAD_REQUEST, "Unsupported HTTP request method");
        }
    } catch (BadRequestException e) {
        throw e;
    } catch (IllegalArgumentException e) {
        throw new BadRequestException(e);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 2 with TreeBuilder

use of net.opentsdb.tree.TreeBuilder in project opentsdb by OpenTSDB.

the class TreeSync method run.

/**
   * Performs a tree synchronization using a table scanner across the UID table
   * @return 0 if completed successfully, something else if an error occurred
   */
public void run() {
    final Scanner scanner = getScanner();
    // start the process by loading all of the trees in the system
    final List<Tree> trees;
    try {
        trees = Tree.fetchAllTrees(tsdb).joinUninterruptibly();
        LOG.info("[" + thread_id + "] Complete");
    } catch (Exception e) {
        LOG.error("[" + thread_id + "] Unexpected Exception", e);
        throw new RuntimeException("[" + thread_id + "] Unexpected exception", e);
    }
    if (trees == null) {
        LOG.warn("No tree definitions were found");
        return;
    } else {
        boolean has_enabled_tree = false;
        for (Tree tree : trees) {
            if (tree.getEnabled()) {
                has_enabled_tree = true;
                break;
            }
        }
        if (!has_enabled_tree) {
            LOG.warn("No enabled trees were found");
            return;
        }
        LOG.info("Found [" + trees.size() + "] trees");
    }
    // setup an array for storing the tree processing calls so we can block 
    // until each call has completed
    final ArrayList<Deferred<Boolean>> tree_calls = new ArrayList<Deferred<Boolean>>();
    final Deferred<Boolean> completed = new Deferred<Boolean>();
    /**
     * Scanner callback that loops through the UID table recursively until 
     * the scanner returns a null row set.
     */
    final class TsuidScanner implements Callback<Deferred<Boolean>, ArrayList<ArrayList<KeyValue>>> {

        /**
       * Fetches the next set of rows from the scanner, adding this class as a 
       * callback
       * @return A meaningless deferred used to wait on until processing has
       * completed
       */
        public Deferred<Boolean> scan() {
            return scanner.nextRows().addCallbackDeferring(this);
        }

        @Override
        public Deferred<Boolean> call(ArrayList<ArrayList<KeyValue>> rows) throws Exception {
            if (rows == null) {
                completed.callback(true);
                return null;
            }
            for (final ArrayList<KeyValue> row : rows) {
                // convert to a string one time
                final String tsuid = UniqueId.uidToString(row.get(0).key());
                /**
           * A throttling callback used to wait for the current TSMeta to 
           * complete processing through the trees before continuing on with 
           * the next set.
           */
                final class TreeBuilderBufferCB implements Callback<Boolean, ArrayList<ArrayList<Boolean>>> {

                    @Override
                    public Boolean call(ArrayList<ArrayList<Boolean>> builder_calls) throws Exception {
                        //LOG.debug("Processed [" + builder_calls.size() + "] tree_calls");
                        return true;
                    }
                }
                /**
           * Executed after parsing a TSMeta object and loading all of the
           * associated UIDMetas. Once the meta has been loaded, this callback
           * runs it through each of the configured TreeBuilder objects and
           * stores the resulting deferred in an array. Once processing of all
           * of the rules has completed, we group the deferreds and call
           * BufferCB() to wait for their completion.
           */
                final class ParseCB implements Callback<Deferred<Boolean>, TSMeta> {

                    final ArrayList<Deferred<ArrayList<Boolean>>> builder_calls = new ArrayList<Deferred<ArrayList<Boolean>>>();

                    @Override
                    public Deferred<Boolean> call(TSMeta meta) throws Exception {
                        if (meta != null) {
                            LOG.debug("Processing TSMeta: " + meta + " w value: " + JSON.serializeToString(meta));
                            // copy the trees into a tree builder object and iterate through
                            // each builder. We need to do this as a builder is not thread
                            // safe and cannot be used asynchronously.
                            final ArrayList<TreeBuilder> tree_builders = new ArrayList<TreeBuilder>(trees.size());
                            for (Tree tree : trees) {
                                if (!tree.getEnabled()) {
                                    continue;
                                }
                                final TreeBuilder builder = new TreeBuilder(tsdb, tree);
                                tree_builders.add(builder);
                            }
                            for (TreeBuilder builder : tree_builders) {
                                builder_calls.add(builder.processTimeseriesMeta(meta));
                            }
                            return Deferred.group(builder_calls).addCallback(new TreeBuilderBufferCB());
                        } else {
                            return Deferred.fromResult(false);
                        }
                    }
                }
                /**
           * An error handler used to catch issues when loading the TSMeta such
           * as a missing UID name. In these situations we want to log that the 
           * TSMeta had an issue and continue on.
           */
                final class ErrBack implements Callback<Deferred<Boolean>, Exception> {

                    @Override
                    public Deferred<Boolean> call(Exception e) throws Exception {
                        if (e.getClass().equals(IllegalStateException.class)) {
                            LOG.error("Invalid data when processing TSUID [" + tsuid + "]", e);
                        } else if (e.getClass().equals(IllegalArgumentException.class)) {
                            LOG.error("Invalid data when processing TSUID [" + tsuid + "]", e);
                        } else if (e.getClass().equals(NoSuchUniqueId.class)) {
                            LOG.warn("Timeseries [" + tsuid + "] includes a non-existant UID: " + e.getMessage());
                        } else {
                            LOG.error("[" + thread_id + "] Exception while processing TSUID [" + tsuid + "]", e);
                        }
                        return Deferred.fromResult(false);
                    }
                }
                // matched a TSMeta column, so request a parsing and loading of
                // associated UIDMeta objects, then pass it off to callbacks for 
                // parsing through the trees.
                final Deferred<Boolean> process_tsmeta = TSMeta.parseFromColumn(tsdb, row.get(0), true).addCallbackDeferring(new ParseCB());
                process_tsmeta.addErrback(new ErrBack());
                tree_calls.add(process_tsmeta);
            }
            /**
         * Another buffer callback that waits for the current set of TSMetas to
         * complete their tree calls before we fetch another set of rows from
         * the scanner. This necessary to avoid OOM issues.
         */
            final class ContinueCB implements Callback<Deferred<Boolean>, ArrayList<Boolean>> {

                @Override
                public Deferred<Boolean> call(ArrayList<Boolean> tsuids) throws Exception {
                    LOG.debug("Processed [" + tsuids.size() + "] tree_calls, continuing");
                    tree_calls.clear();
                    return scan();
                }
            }
            // request the next set of rows from the scanner, but wait until the
            // current set of TSMetas has been processed so we don't slaughter our
            // host
            Deferred.group(tree_calls).addCallback(new ContinueCB());
            return Deferred.fromResult(null);
        }
    }
    /**
     * Used to capture unhandled exceptions from the scanner callbacks and 
     * exit the thread properly
     */
    final class ErrBack implements Callback<Deferred<Boolean>, Exception> {

        @Override
        public Deferred<Boolean> call(Exception e) throws Exception {
            LOG.error("Unexpected exception", e);
            completed.callback(false);
            return Deferred.fromResult(false);
        }
    }
    final TsuidScanner tree_scanner = new TsuidScanner();
    tree_scanner.scan().addErrback(new ErrBack());
    try {
        completed.joinUninterruptibly();
        LOG.info("[" + thread_id + "] Complete");
    } catch (Exception e) {
        LOG.error("[" + thread_id + "] Scanner Exception", e);
        throw new RuntimeException("[" + thread_id + "] Scanner exception", e);
    }
    return;
}