Examples with Callback com.stumbleupon.async.Callback used on opensource projects

Example 46 with Callback

use of com.stumbleupon.async.Callback in project opentsdb by OpenTSDB.

the class Tree method fetchNotMatched.

/**
 * Returns the not-matched set from storage for the given tree, optionally for
 * only the list of TSUIDs provided.
 * <b>Note:</b> This can potentially be a large list if the rule set was
 * written poorly and there were many timeseries so only call this
 * without a list of TSUIDs if you feel confident the number is small.
 * @param tsdb TSDB to use for storage access
 * @param tree_id ID of the tree to fetch non matches for
 * @param tsuids An optional list of TSUIDs to fetch non-matches for. This may
 * be empty or null, in which case all non-matches for the tree will be
 * returned.
 * @return A list of not-matched mappings or null if nothing was found
 * @throws HBaseException if there was an issue
 * @throws IllegalArgumentException if the tree ID was invalid
 */
public static Deferred<Map<String, String>> fetchNotMatched(final TSDB tsdb, final int tree_id, final List<String> tsuids) {
    if (tree_id < 1 || tree_id > 65535) {
        throw new IllegalArgumentException("Invalid Tree ID");
    }
    final byte[] row_key = new byte[TREE_ID_WIDTH + 1];
    System.arraycopy(idToBytes(tree_id), 0, row_key, 0, TREE_ID_WIDTH);
    row_key[TREE_ID_WIDTH] = NOT_MATCHED_ROW_SUFFIX;
    final GetRequest get = new GetRequest(tsdb.treeTable(), row_key);
    get.family(TREE_FAMILY);
    // of qualifiers so we only fetch those columns.
    if (tsuids != null && !tsuids.isEmpty()) {
        final byte[][] qualifiers = new byte[tsuids.size()][];
        int index = 0;
        for (String tsuid : tsuids) {
            final byte[] qualifier = new byte[NOT_MATCHED_PREFIX.length + (tsuid.length() / 2)];
            System.arraycopy(NOT_MATCHED_PREFIX, 0, qualifier, 0, NOT_MATCHED_PREFIX.length);
            final byte[] tsuid_bytes = UniqueId.stringToUid(tsuid);
            System.arraycopy(tsuid_bytes, 0, qualifier, NOT_MATCHED_PREFIX.length, tsuid_bytes.length);
            qualifiers[index] = qualifier;
            index++;
        }
        get.qualifiers(qualifiers);
    }
    /**
     * Called after issuing the row get request to parse out the results and
     * compile the list of collisions.
     */
    final class GetCB implements Callback<Deferred<Map<String, String>>, ArrayList<KeyValue>> {

        @Override
        public Deferred<Map<String, String>> call(final ArrayList<KeyValue> row) throws Exception {
            if (row == null || row.isEmpty()) {
                final Map<String, String> empty = new HashMap<String, String>(0);
                return Deferred.fromResult(empty);
            }
            Map<String, String> not_matched = new HashMap<String, String>(row.size());
            for (KeyValue column : row) {
                final byte[] parsed_tsuid = Arrays.copyOfRange(column.qualifier(), NOT_MATCHED_PREFIX.length, column.qualifier().length);
                not_matched.put(UniqueId.uidToString(parsed_tsuid), new String(column.value(), CHARSET));
            }
            return Deferred.fromResult(not_matched);
        }
    }
    return tsdb.getClient().get(get).addCallbackDeferring(new GetCB());
}

Example 47 with Callback

use of com.stumbleupon.async.Callback in project opentsdb by OpenTSDB.

the class Tree method flushNotMatched.

/**
 * Attempts to flush the non-matches to storage. The storage call is a PUT so
 * it will overwrite any existing columns, but since each column is the TSUID
 * it should only exist once and the data shouldn't change.
 * <b>Note:</b> This will also clear the local {@link #not_matched} map
 * @param tsdb The TSDB to use for storage access
 * @return A meaningless deferred (will always be true since we need to group
 * it with tree store calls) for the caller to wait on
 * @throws HBaseException if there was an issue
 */
public Deferred<Boolean> flushNotMatched(final TSDB tsdb) {
    if (!store_failures) {
        not_matched.clear();
        return Deferred.fromResult(true);
    }
    final byte[] row_key = new byte[TREE_ID_WIDTH + 1];
    System.arraycopy(idToBytes(tree_id), 0, row_key, 0, TREE_ID_WIDTH);
    row_key[TREE_ID_WIDTH] = NOT_MATCHED_ROW_SUFFIX;
    final byte[][] qualifiers = new byte[not_matched.size()][];
    final byte[][] values = new byte[not_matched.size()][];
    int index = 0;
    for (Map.Entry<String, String> entry : not_matched.entrySet()) {
        qualifiers[index] = new byte[NOT_MATCHED_PREFIX.length + (entry.getKey().length() / 2)];
        System.arraycopy(NOT_MATCHED_PREFIX, 0, qualifiers[index], 0, NOT_MATCHED_PREFIX.length);
        final byte[] tsuid = UniqueId.stringToUid(entry.getKey());
        System.arraycopy(tsuid, 0, qualifiers[index], NOT_MATCHED_PREFIX.length, tsuid.length);
        values[index] = entry.getValue().getBytes(CHARSET);
        index++;
    }
    final PutRequest put = new PutRequest(tsdb.treeTable(), row_key, TREE_FAMILY, qualifiers, values);
    not_matched.clear();
    /**
     * Super simple callback used to convert the Deferred&lt;Object&gt; to a
     * Deferred&lt;Boolean&gt; so that it can be grouped with other storage
     * calls
     */
    final class PutCB implements Callback<Deferred<Boolean>, Object> {

        @Override
        public Deferred<Boolean> call(Object result) throws Exception {
            return Deferred.fromResult(true);
        }
    }
    return tsdb.getClient().put(put).addCallbackDeferring(new PutCB());
}

Example 48 with Callback

use of com.stumbleupon.async.Callback in project opentsdb by OpenTSDB.

the class Tree method deleteTree.

/**
 * Attempts to delete all branches, leaves, collisions and not-matched entries
 * for the given tree. Optionally can delete the tree definition and rules as
 * well.
 * <b>Warning:</b> This call can take a long time to complete so it should
 * only be done from a command line or issues once via RPC and allowed to
 * process. Multiple deletes running at the same time on the same tree
 * shouldn't be an issue but it's a waste of resources.
 * @param tsdb The TSDB to use for storage access
 * @param tree_id ID of the tree to delete
 * @param delete_definition Whether or not the tree definition and rule set
 * should be deleted as well
 * @return True if the deletion completed successfully, false if there was an
 * issue.
 * @throws HBaseException if there was an issue
 * @throws IllegalArgumentException if the tree ID was invalid
 */
public static Deferred<Boolean> deleteTree(final TSDB tsdb, final int tree_id, final boolean delete_definition) {
    if (tree_id < 1 || tree_id > 65535) {
        throw new IllegalArgumentException("Invalid Tree ID");
    }
    // scan all of the rows starting with the tree ID. We can't just delete the
    // rows as there may be other types of data. Thus we have to check the
    // qualifiers of every column to see if it's safe to delete
    final byte[] start = idToBytes(tree_id);
    final byte[] end = idToBytes(tree_id + 1);
    final Scanner scanner = tsdb.getClient().newScanner(tsdb.treeTable());
    scanner.setStartKey(start);
    scanner.setStopKey(end);
    scanner.setFamily(TREE_FAMILY);
    final Deferred<Boolean> completed = new Deferred<Boolean>();
    /**
     * Scanner callback that loops through all rows between tree id and
     * tree id++ searching for tree related columns to delete.
     */
    final class DeleteTreeScanner implements Callback<Deferred<Boolean>, ArrayList<ArrayList<KeyValue>>> {

        // list where we'll store delete requests for waiting on
        private final ArrayList<Deferred<Object>> delete_deferreds = new ArrayList<Deferred<Object>>();

        /**
         * Fetches the next set of rows from the scanner and adds this class as
         * a callback
         * @return The list of delete requests when the scanner returns a null set
         */
        public Deferred<Boolean> deleteTree() {
            return scanner.nextRows().addCallbackDeferring(this);
        }

        @Override
        public Deferred<Boolean> call(ArrayList<ArrayList<KeyValue>> rows) throws Exception {
            if (rows == null) {
                completed.callback(true);
                return null;
            }
            for (final ArrayList<KeyValue> row : rows) {
                // one delete request per row. We'll almost always delete the whole
                // row, so just preallocate the entire row.
                ArrayList<byte[]> qualifiers = new ArrayList<byte[]>(row.size());
                for (KeyValue column : row) {
                    // tree
                    if (delete_definition && Bytes.equals(TREE_QUALIFIER, column.qualifier())) {
                        LOG.trace("Deleting tree defnition in row: " + Branch.idToString(column.key()));
                        qualifiers.add(column.qualifier());
                    // branches
                    } else if (Bytes.equals(Branch.BRANCH_QUALIFIER(), column.qualifier())) {
                        LOG.trace("Deleting branch in row: " + Branch.idToString(column.key()));
                        qualifiers.add(column.qualifier());
                    // leaves
                    } else if (column.qualifier().length > Leaf.LEAF_PREFIX().length && Bytes.memcmp(Leaf.LEAF_PREFIX(), column.qualifier(), 0, Leaf.LEAF_PREFIX().length) == 0) {
                        LOG.trace("Deleting leaf in row: " + Branch.idToString(column.key()));
                        qualifiers.add(column.qualifier());
                    // collisions
                    } else if (column.qualifier().length > COLLISION_PREFIX.length && Bytes.memcmp(COLLISION_PREFIX, column.qualifier(), 0, COLLISION_PREFIX.length) == 0) {
                        LOG.trace("Deleting collision in row: " + Branch.idToString(column.key()));
                        qualifiers.add(column.qualifier());
                    // not matched
                    } else if (column.qualifier().length > NOT_MATCHED_PREFIX.length && Bytes.memcmp(NOT_MATCHED_PREFIX, column.qualifier(), 0, NOT_MATCHED_PREFIX.length) == 0) {
                        LOG.trace("Deleting not matched in row: " + Branch.idToString(column.key()));
                        qualifiers.add(column.qualifier());
                    // tree rule
                    } else if (delete_definition && column.qualifier().length > TreeRule.RULE_PREFIX().length && Bytes.memcmp(TreeRule.RULE_PREFIX(), column.qualifier(), 0, TreeRule.RULE_PREFIX().length) == 0) {
                        LOG.trace("Deleting tree rule in row: " + Branch.idToString(column.key()));
                        qualifiers.add(column.qualifier());
                    }
                }
                if (qualifiers.size() > 0) {
                    final DeleteRequest delete = new DeleteRequest(tsdb.treeTable(), row.get(0).key(), TREE_FAMILY, qualifiers.toArray(new byte[qualifiers.size()][]));
                    delete_deferreds.add(tsdb.getClient().delete(delete));
                }
            }
            /**
             * Callback used as a kind of buffer so that we don't wind up loading
             * thousands or millions of delete requests into memory and possibly run
             * into a StackOverflowError or general OOM. The scanner defaults are
             * our limit so each pass of the scanner will wait for the previous set
             * of deferreds to complete before continuing
             */
            final class ContinueCB implements Callback<Deferred<Boolean>, ArrayList<Object>> {

                public Deferred<Boolean> call(ArrayList<Object> objects) {
                    LOG.debug("Purged [" + objects.size() + "] columns, continuing");
                    delete_deferreds.clear();
                    // call ourself again to get the next set of rows from the scanner
                    return deleteTree();
                }
            }
            // call ourself again after waiting for the existing delete requests
            // to complete
            Deferred.group(delete_deferreds).addCallbackDeferring(new ContinueCB());
            return null;
        }
    }
    // start the scanner
    new DeleteTreeScanner().deleteTree();
    return completed;
}

Example 49 with Callback

use of com.stumbleupon.async.Callback in project opentsdb by OpenTSDB.

the class TreeSync method run.

/**
 * Performs a tree synchronization using a table scanner across the UID table
 * @return 0 if completed successfully, something else if an error occurred
 */
public void run() {
    final Scanner scanner = getScanner();
    // start the process by loading all of the trees in the system
    final List<Tree> trees;
    try {
        trees = Tree.fetchAllTrees(tsdb).joinUninterruptibly();
        LOG.info("[" + thread_id + "] Complete");
    } catch (Exception e) {
        LOG.error("[" + thread_id + "] Unexpected Exception", e);
        throw new RuntimeException("[" + thread_id + "] Unexpected exception", e);
    }
    if (trees == null) {
        LOG.warn("No tree definitions were found");
        return;
    } else {
        boolean has_enabled_tree = false;
        for (Tree tree : trees) {
            if (tree.getEnabled()) {
                has_enabled_tree = true;
                break;
            }
        }
        if (!has_enabled_tree) {
            LOG.warn("No enabled trees were found");
            return;
        }
        LOG.info("Found [" + trees.size() + "] trees");
    }
    // setup an array for storing the tree processing calls so we can block
    // until each call has completed
    final ArrayList<Deferred<Boolean>> tree_calls = new ArrayList<Deferred<Boolean>>();
    final Deferred<Boolean> completed = new Deferred<Boolean>();
    /**
     * Scanner callback that loops through the UID table recursively until
     * the scanner returns a null row set.
     */
    final class TsuidScanner implements Callback<Deferred<Boolean>, ArrayList<ArrayList<KeyValue>>> {

        /**
         * Fetches the next set of rows from the scanner, adding this class as a
         * callback
         * @return A meaningless deferred used to wait on until processing has
         * completed
         */
        public Deferred<Boolean> scan() {
            return scanner.nextRows().addCallbackDeferring(this);
        }

        @Override
        public Deferred<Boolean> call(ArrayList<ArrayList<KeyValue>> rows) throws Exception {
            if (rows == null) {
                completed.callback(true);
                return null;
            }
            for (final ArrayList<KeyValue> row : rows) {
                // convert to a string one time
                final String tsuid = UniqueId.uidToString(row.get(0).key());
                /**
                 * A throttling callback used to wait for the current TSMeta to
                 * complete processing through the trees before continuing on with
                 * the next set.
                 */
                final class TreeBuilderBufferCB implements Callback<Boolean, ArrayList<ArrayList<Boolean>>> {

                    @Override
                    public Boolean call(ArrayList<ArrayList<Boolean>> builder_calls) throws Exception {
                        // LOG.debug("Processed [" + builder_calls.size() + "] tree_calls");
                        return true;
                    }
                }
                /**
                 * Executed after parsing a TSMeta object and loading all of the
                 * associated UIDMetas. Once the meta has been loaded, this callback
                 * runs it through each of the configured TreeBuilder objects and
                 * stores the resulting deferred in an array. Once processing of all
                 * of the rules has completed, we group the deferreds and call
                 * BufferCB() to wait for their completion.
                 */
                final class ParseCB implements Callback<Deferred<Boolean>, TSMeta> {

                    final ArrayList<Deferred<ArrayList<Boolean>>> builder_calls = new ArrayList<Deferred<ArrayList<Boolean>>>();

                    @Override
                    public Deferred<Boolean> call(TSMeta meta) throws Exception {
                        if (meta != null) {
                            LOG.debug("Processing TSMeta: " + meta + " w value: " + JSON.serializeToString(meta));
                            // copy the trees into a tree builder object and iterate through
                            // each builder. We need to do this as a builder is not thread
                            // safe and cannot be used asynchronously.
                            final ArrayList<TreeBuilder> tree_builders = new ArrayList<TreeBuilder>(trees.size());
                            for (Tree tree : trees) {
                                if (!tree.getEnabled()) {
                                    continue;
                                }
                                final TreeBuilder builder = new TreeBuilder(tsdb, tree);
                                tree_builders.add(builder);
                            }
                            for (TreeBuilder builder : tree_builders) {
                                builder_calls.add(builder.processTimeseriesMeta(meta));
                            }
                            return Deferred.group(builder_calls).addCallback(new TreeBuilderBufferCB());
                        } else {
                            return Deferred.fromResult(false);
                        }
                    }
                }
                /**
                 * An error handler used to catch issues when loading the TSMeta such
                 * as a missing UID name. In these situations we want to log that the
                 * TSMeta had an issue and continue on.
                 */
                final class ErrBack implements Callback<Deferred<Boolean>, Exception> {

                    @Override
                    public Deferred<Boolean> call(Exception e) throws Exception {
                        if (e.getClass().equals(IllegalStateException.class)) {
                            LOG.error("Invalid data when processing TSUID [" + tsuid + "]", e);
                        } else if (e.getClass().equals(IllegalArgumentException.class)) {
                            LOG.error("Invalid data when processing TSUID [" + tsuid + "]", e);
                        } else if (e.getClass().equals(NoSuchUniqueId.class)) {
                            LOG.warn("Timeseries [" + tsuid + "] includes a non-existant UID: " + e.getMessage());
                        } else {
                            LOG.error("[" + thread_id + "] Exception while processing TSUID [" + tsuid + "]", e);
                        }
                        return Deferred.fromResult(false);
                    }
                }
                // matched a TSMeta column, so request a parsing and loading of
                // associated UIDMeta objects, then pass it off to callbacks for
                // parsing through the trees.
                final Deferred<Boolean> process_tsmeta = TSMeta.parseFromColumn(tsdb, row.get(0), true).addCallbackDeferring(new ParseCB());
                process_tsmeta.addErrback(new ErrBack());
                tree_calls.add(process_tsmeta);
            }
            /**
             * Another buffer callback that waits for the current set of TSMetas to
             * complete their tree calls before we fetch another set of rows from
             * the scanner. This necessary to avoid OOM issues.
             */
            final class ContinueCB implements Callback<Deferred<Boolean>, ArrayList<Boolean>> {

                @Override
                public Deferred<Boolean> call(ArrayList<Boolean> tsuids) throws Exception {
                    LOG.debug("Processed [" + tsuids.size() + "] tree_calls, continuing");
                    tree_calls.clear();
                    return scan();
                }
            }
            // request the next set of rows from the scanner, but wait until the
            // current set of TSMetas has been processed so we don't slaughter our
            // host
            Deferred.group(tree_calls).addCallback(new ContinueCB());
            return Deferred.fromResult(null);
        }
    }
    /**
     * Used to capture unhandled exceptions from the scanner callbacks and
     * exit the thread properly
     */
    final class ErrBack implements Callback<Deferred<Boolean>, Exception> {

        @Override
        public Deferred<Boolean> call(Exception e) throws Exception {
            LOG.error("Unexpected exception", e);
            completed.callback(false);
            return Deferred.fromResult(false);
        }
    }
    final TsuidScanner tree_scanner = new TsuidScanner();
    tree_scanner.scan().addErrback(new ErrBack());
    try {
        completed.joinUninterruptibly();
        LOG.info("[" + thread_id + "] Complete");
    } catch (Exception e) {
        LOG.error("[" + thread_id + "] Scanner Exception", e);
        throw new RuntimeException("[" + thread_id + "] Scanner exception", e);
    }
    return;
}

Example 50 with Callback

use of com.stumbleupon.async.Callback in project opentsdb by OpenTSDB.

the class Branch method fetchBranch.

/**
 * Attempts to fetch the branch, it's leaves and all child branches.
 * The UID names for each leaf may also be loaded if configured.
 * @param tsdb The TSDB to use for storage access
 * @param branch_id ID of the branch to retrieve
 * @param load_leaf_uids Whether or not to load UID names for each leaf
 * @return A branch if found, null if it did not exist
 * @throws JSONException if the object could not be deserialized
 */
public static Deferred<Branch> fetchBranch(final TSDB tsdb, final byte[] branch_id, final boolean load_leaf_uids) {
    final Deferred<Branch> result = new Deferred<Branch>();
    final Scanner scanner = setupBranchScanner(tsdb, branch_id);
    // This is the branch that will be loaded with data from the scanner and
    // returned at the end of the process.
    final Branch branch = new Branch();
    // A list of deferreds to wait on for child leaf processing
    final ArrayList<Deferred<Object>> leaf_group = new ArrayList<Deferred<Object>>();
    /**
     * Exception handler to catch leaves with an invalid UID name due to a
     * possible deletion. This will allow the scanner to keep loading valid
     * leaves and ignore problems. The fsck tool can be used to clean up
     * orphaned leaves. If we catch something other than an NSU, it will
     * re-throw the exception
     */
    final class LeafErrBack implements Callback<Object, Exception> {

        final byte[] qualifier;

        public LeafErrBack(final byte[] qualifier) {
            this.qualifier = qualifier;
        }

        @Override
        public Object call(final Exception e) throws Exception {
            Throwable ex = e;
            while (ex.getClass().equals(DeferredGroupException.class)) {
                ex = ex.getCause();
            }
            if (ex.getClass().equals(NoSuchUniqueId.class)) {
                LOG.debug("Invalid UID for leaf: " + idToString(qualifier) + " in branch: " + idToString(branch_id), ex);
            } else {
                throw (Exception) ex;
            }
            return null;
        }
    }
    /**
     * Called after a leaf has been loaded successfully and adds the leaf
     * to the branch's leaf set. Also lazily initializes the leaf set if it
     * hasn't been.
     */
    final class LeafCB implements Callback<Object, Leaf> {

        @Override
        public Object call(final Leaf leaf) throws Exception {
            if (leaf != null) {
                if (branch.leaves == null) {
                    branch.leaves = new HashMap<Integer, Leaf>();
                }
                branch.leaves.put(leaf.hashCode(), leaf);
            }
            return null;
        }
    }
    /**
     * Scanner callback executed recursively each time we get a set of data
     * from storage. This is responsible for determining what columns are
     * returned and issuing requests to load leaf objects.
     * When the scanner returns a null set of rows, the method initiates the
     * final callback.
     */
    final class FetchBranchCB implements Callback<Object, ArrayList<ArrayList<KeyValue>>> {

        /**
         * Starts the scanner and is called recursively to fetch the next set of
         * rows from the scanner.
         * @return The branch if loaded successfully, null if the branch was not
         * found.
         */
        public Object fetchBranch() {
            return scanner.nextRows().addCallback(this);
        }

        /**
         * Loops through each row of the scanner results and parses out branch
         * definitions and child leaves.
         * @return The final branch callback if the scanner returns a null set
         */
        @Override
        public Object call(final ArrayList<ArrayList<KeyValue>> rows) throws Exception {
            if (rows == null) {
                if (branch.tree_id < 1 || branch.path == null) {
                    result.callback(null);
                } else {
                    result.callback(branch);
                }
                return null;
            }
            for (final ArrayList<KeyValue> row : rows) {
                for (KeyValue column : row) {
                    // matched a branch column
                    if (Bytes.equals(BRANCH_QUALIFIER, column.qualifier())) {
                        if (Bytes.equals(branch_id, column.key())) {
                            // it's *this* branch. We deserialize to a new object and copy
                            // since the columns could be in any order and we may get a
                            // leaf before the branch
                            final Branch local_branch = JSON.parseToObject(column.value(), Branch.class);
                            branch.path = local_branch.path;
                            branch.display_name = local_branch.display_name;
                            branch.tree_id = Tree.bytesToId(column.key());
                        } else {
                            // it's a child branch
                            final Branch child = JSON.parseToObject(column.value(), Branch.class);
                            child.tree_id = Tree.bytesToId(column.key());
                            branch.addChild(child);
                        }
                    // parse out a leaf
                    } else if (Bytes.memcmp(Leaf.LEAF_PREFIX(), column.qualifier(), 0, Leaf.LEAF_PREFIX().length) == 0) {
                        if (Bytes.equals(branch_id, column.key())) {
                            // process a leaf and skip if the UIDs for the TSUID can't be
                            // found. Add an errback to catch NoSuchUniqueId exceptions
                            leaf_group.add(Leaf.parseFromStorage(tsdb, column, load_leaf_uids).addCallbacks(new LeafCB(), new LeafErrBack(column.qualifier())));
                        } else {
                        // TODO - figure out an efficient way to increment a counter in
                        // the child branch with the # of leaves it has
                        }
                    }
                }
            }
            // recursively call ourself to fetch more results from the scanner
            return fetchBranch();
        }
    }
    // start scanning
    new FetchBranchCB().fetchBranch();
    return result;
}