Examples with Callback com.stumbleupon.async.Callback used on opensource projects

Example 21 with Callback

use of com.stumbleupon.async.Callback in project opentsdb by OpenTSDB.

the class Tree method fetchAllTrees.

/**
   * Attempts to retrieve all trees from the UID table, including their rules.
   * If no trees were found, the result will be an empty list
   * @param tsdb The TSDB to use for storage
   * @return A list of tree objects. May be empty if none were found
   */
public static Deferred<List<Tree>> fetchAllTrees(final TSDB tsdb) {
    final Deferred<List<Tree>> result = new Deferred<List<Tree>>();
    /**
     * Scanner callback that recursively calls itself to load the next set of
     * rows from storage. When the scanner returns a null, the callback will
     * return with the list of trees discovered.
     */
    final class AllTreeScanner implements Callback<Object, ArrayList<ArrayList<KeyValue>>> {

        private final List<Tree> trees = new ArrayList<Tree>();

        private final Scanner scanner;

        public AllTreeScanner() {
            scanner = setupAllTreeScanner(tsdb);
        }

        /**
       * Fetches the next set of results from the scanner and adds this class
       * as a callback.
       * @return A list of trees if the scanner has reached the end
       */
        public Object fetchTrees() {
            return scanner.nextRows().addCallback(this);
        }

        @Override
        public Object call(ArrayList<ArrayList<KeyValue>> rows) throws Exception {
            if (rows == null) {
                result.callback(trees);
                return null;
            }
            for (ArrayList<KeyValue> row : rows) {
                final Tree tree = new Tree();
                for (KeyValue column : row) {
                    if (column.qualifier().length >= TREE_QUALIFIER.length && Bytes.memcmp(TREE_QUALIFIER, column.qualifier()) == 0) {
                        // it's *this* tree. We deserialize to a new object and copy
                        // since the columns could be in any order and we may get a rule 
                        // before the tree object
                        final Tree local_tree = JSON.parseToObject(column.value(), Tree.class);
                        tree.created = local_tree.created;
                        tree.description = local_tree.description;
                        tree.name = local_tree.name;
                        tree.notes = local_tree.notes;
                        tree.strict_match = local_tree.strict_match;
                        tree.enabled = local_tree.enabled;
                        tree.store_failures = local_tree.store_failures;
                        // WARNING: Since the JSON data in storage doesn't contain the tree
                        // ID, we need to parse it from the row key
                        tree.setTreeId(bytesToId(row.get(0).key()));
                    // tree rule
                    } else if (column.qualifier().length > TreeRule.RULE_PREFIX().length && Bytes.memcmp(TreeRule.RULE_PREFIX(), column.qualifier(), 0, TreeRule.RULE_PREFIX().length) == 0) {
                        final TreeRule rule = TreeRule.parseFromStorage(column);
                        tree.addRule(rule);
                    }
                }
                // only add the tree if we parsed a valid ID
                if (tree.tree_id > 0) {
                    trees.add(tree);
                }
            }
            // recurse to get the next set of rows from the scanner
            return fetchTrees();
        }
    }
    // start the scanning process
    new AllTreeScanner().fetchTrees();
    return result;
}

Example 22 with Callback

use of com.stumbleupon.async.Callback in project opentsdb by OpenTSDB.

the class Tree method fetchTree.

/**
   * Attempts to fetch the given tree from storage, loading the rule set at
   * the same time.
   * @param tsdb The TSDB to use for access
   * @param tree_id The Tree to fetch
   * @return A tree object if found, null if the tree did not exist
   * @throws IllegalArgumentException if the tree ID was invalid
   * @throws HBaseException if a storage exception occurred
   * @throws JSONException if the object could not be deserialized
   */
public static Deferred<Tree> fetchTree(final TSDB tsdb, final int tree_id) {
    if (tree_id < 1 || tree_id > 65535) {
        throw new IllegalArgumentException("Invalid Tree ID");
    }
    // fetch the whole row
    final GetRequest get = new GetRequest(tsdb.treeTable(), idToBytes(tree_id));
    get.family(TREE_FAMILY);
    /**
     * Called from the GetRequest with results from storage. Loops through the
     * columns and loads the tree definition and rules
     */
    final class FetchTreeCB implements Callback<Deferred<Tree>, ArrayList<KeyValue>> {

        @Override
        public Deferred<Tree> call(ArrayList<KeyValue> row) throws Exception {
            if (row == null || row.isEmpty()) {
                return Deferred.fromResult(null);
            }
            final Tree tree = new Tree();
            // WARNING: Since the JSON in storage doesn't store the tree ID, we need
            // to loadi t from the row key.
            tree.setTreeId(bytesToId(row.get(0).key()));
            for (KeyValue column : row) {
                if (Bytes.memcmp(TREE_QUALIFIER, column.qualifier()) == 0) {
                    // it's *this* tree. We deserialize to a new object and copy
                    // since the columns could be in any order and we may get a rule 
                    // before the tree object
                    final Tree local_tree = JSON.parseToObject(column.value(), Tree.class);
                    tree.created = local_tree.created;
                    tree.description = local_tree.description;
                    tree.name = local_tree.name;
                    tree.notes = local_tree.notes;
                    tree.strict_match = local_tree.strict_match;
                    tree.enabled = local_tree.enabled;
                    tree.store_failures = local_tree.store_failures;
                // Tree rule
                } else if (Bytes.memcmp(TreeRule.RULE_PREFIX(), column.qualifier(), 0, TreeRule.RULE_PREFIX().length) == 0) {
                    final TreeRule rule = TreeRule.parseFromStorage(column);
                    tree.addRule(rule);
                }
            }
            return Deferred.fromResult(tree);
        }
    }
    // issue the get request
    return tsdb.getClient().get(get).addCallbackDeferring(new FetchTreeCB());
}

Example 23 with Callback

use of com.stumbleupon.async.Callback in project opentsdb by OpenTSDB.

the class Tree method flushCollisions.

/**
   * Attempts to flush the collisions to storage. The storage call is a PUT so
   * it will overwrite any existing columns, but since each column is the TSUID
   * it should only exist once and the data shouldn't change.
   * <b>Note:</b> This will also clear the local {@link #collisions} map
   * @param tsdb The TSDB to use for storage access
   * @return A meaningless deferred (will always be true since we need to group
   * it with tree store calls) for the caller to wait on
   * @throws HBaseException if there was an issue
   */
public Deferred<Boolean> flushCollisions(final TSDB tsdb) {
    if (!store_failures) {
        collisions.clear();
        return Deferred.fromResult(true);
    }
    final byte[] row_key = new byte[TREE_ID_WIDTH + 1];
    System.arraycopy(idToBytes(tree_id), 0, row_key, 0, TREE_ID_WIDTH);
    row_key[TREE_ID_WIDTH] = COLLISION_ROW_SUFFIX;
    final byte[][] qualifiers = new byte[collisions.size()][];
    final byte[][] values = new byte[collisions.size()][];
    int index = 0;
    for (Map.Entry<String, String> entry : collisions.entrySet()) {
        qualifiers[index] = new byte[COLLISION_PREFIX.length + (entry.getKey().length() / 2)];
        System.arraycopy(COLLISION_PREFIX, 0, qualifiers[index], 0, COLLISION_PREFIX.length);
        final byte[] tsuid = UniqueId.stringToUid(entry.getKey());
        System.arraycopy(tsuid, 0, qualifiers[index], COLLISION_PREFIX.length, tsuid.length);
        values[index] = entry.getValue().getBytes(CHARSET);
        index++;
    }
    final PutRequest put = new PutRequest(tsdb.treeTable(), row_key, TREE_FAMILY, qualifiers, values);
    collisions.clear();
    /**
     * Super simple callback used to convert the Deferred&lt;Object&gt; to a 
     * Deferred&lt;Boolean&gt; so that it can be grouped with other storage
     * calls
     */
    final class PutCB implements Callback<Deferred<Boolean>, Object> {

        @Override
        public Deferred<Boolean> call(Object result) throws Exception {
            return Deferred.fromResult(true);
        }
    }
    return tsdb.getClient().put(put).addCallbackDeferring(new PutCB());
}

Example 24 with Callback

use of com.stumbleupon.async.Callback in project opentsdb by OpenTSDB.

the class TagVFilter method resolveTags.

/**
   * Resolves both the tagk to it's UID and a list of literal tag values to
   * their UIDs. A filter may match a literal set (e.g. the pipe filter) in which
   * case we can build the row key scanner with these values.
   * Note that if "tsd.query.skip_unresolved_tagvs" is set in the config then
   * any tag value UIDs that couldn't be found will be excluded.
   * @param tsdb The TSDB to use for the lookup
   * @param literals The list of unique strings to lookup
   * @return A deferred to let the caller know that the lookup was completed.
   * The value will be the tag UID (unless it's an exception of course)
   */
public Deferred<byte[]> resolveTags(final TSDB tsdb, final Set<String> literals) {
    final Config config = tsdb.getConfig();
    /**
     * Allows the filter to avoid killing the entire query when we can't resolve
     * a tag value to a UID.
     */
    class TagVErrback implements Callback<byte[], Exception> {

        @Override
        public byte[] call(final Exception e) throws Exception {
            if (config.getBoolean("tsd.query.skip_unresolved_tagvs")) {
                LOG.warn("Query tag value not found: " + e.getMessage());
                return null;
            } else {
                throw e;
            }
        }
    }
    /**
     * Stores the non-null UIDs in the local list and then sorts them in
     * prep for use in the regex filter
     */
    class ResolvedTagVCB implements Callback<byte[], ArrayList<byte[]>> {

        @Override
        public byte[] call(final ArrayList<byte[]> results) throws Exception {
            tagv_uids = new ArrayList<byte[]>(results.size() - 1);
            for (final byte[] tagv : results) {
                if (tagv != null) {
                    tagv_uids.add(tagv);
                }
            }
            Collections.sort(tagv_uids, Bytes.MEMCMP);
            return tagk_bytes;
        }
    }
    /**
     * Super simple callback to set the local tagk and returns null so it won't
     * be included in the tag value UID lookups.
     */
    class ResolvedTagKCB implements Callback<byte[], byte[]> {

        @Override
        public byte[] call(final byte[] uid) throws Exception {
            tagk_bytes = uid;
            return null;
        }
    }
    final List<Deferred<byte[]>> tagvs = new ArrayList<Deferred<byte[]>>(literals.size());
    for (final String tagv : literals) {
        tagvs.add(tsdb.getUIDAsync(UniqueIdType.TAGV, tagv).addErrback(new TagVErrback()));
    }
    // ugly hack to resolve the tagk UID. The callback will return null and we'll
    // remove it from the UID list.
    tagvs.add(tsdb.getUIDAsync(UniqueIdType.TAGK, tagk).addCallback(new ResolvedTagKCB()));
    return Deferred.group(tagvs).addCallback(new ResolvedTagVCB());
}

Example 25 with Callback

use of com.stumbleupon.async.Callback in project opentsdb by OpenTSDB.

the class QueryExecutor method serialize.

/**
   * Writes the results to a ChannelBuffer to return to the caller. This will
   * iterate over all of the outputs and drop in meta data where appropriate.
   * @throws Exception if something went pear shaped
   */
private Deferred<ChannelBuffer> serialize() throws Exception {
    final long start = System.currentTimeMillis();
    // buffers and an array list to stored the deferreds
    final ChannelBuffer response = ChannelBuffers.dynamicBuffer();
    final OutputStream output_stream = new ChannelBufferOutputStream(response);
    final JsonGenerator json = JSON.getFactory().createGenerator(output_stream);
    json.writeStartObject();
    json.writeFieldName("outputs");
    json.writeStartArray();
    // We want the serializer to execute serially so we need to create a callback
    // chain so that when one DPsResolver is finished, it triggers the next to
    // start serializing.
    final Deferred<Object> cb_chain = new Deferred<Object>();
    // default to the expressions if there, or fall back to the metrics
    final List<Output> outputs;
    if (query.getOutputs() == null || query.getOutputs().isEmpty()) {
        if (query.getExpressions() != null && !query.getExpressions().isEmpty()) {
            outputs = new ArrayList<Output>(query.getExpressions().size());
            for (final Expression exp : query.getExpressions()) {
                outputs.add(Output.Builder().setId(exp.getId()).build());
            }
        } else if (query.getMetrics() != null && !query.getMetrics().isEmpty()) {
            outputs = new ArrayList<Output>(query.getMetrics().size());
            for (final Metric metric : query.getMetrics()) {
                outputs.add(Output.Builder().setId(metric.getId()).build());
            }
        } else {
            throw new IllegalArgumentException("How did we get here?? No metrics or expressions??");
        }
    } else {
        outputs = query.getOutputs();
    }
    for (final Output output : outputs) {
        if (expressions != null) {
            final ExpressionIterator it = expressions.get(output.getId());
            if (it != null) {
                cb_chain.addCallback(new SerializeExpressionIterator(tsdb, json, output, it, ts_query));
                continue;
            }
        }
        if (query.getMetrics() != null && !query.getMetrics().isEmpty()) {
            final TSSubQuery sub = sub_queries.get(output.getId());
            if (sub != null) {
                final TimeSyncedIterator it = new TimeSyncedIterator(output.getId(), sub.getFilterTagKs(), sub_query_results.get(output.getId()));
                cb_chain.addCallback(new SerializeSubIterator(tsdb, json, output, it));
                continue;
            }
        } else {
            LOG.warn("Couldn't find a variable matching: " + output.getId() + " in query " + query);
        }
    }
    /** Final callback to close out the JSON array and return our results */
    class FinalCB implements Callback<ChannelBuffer, Object> {

        public ChannelBuffer call(final Object obj) throws Exception {
            json.writeEndArray();
            //        ts_query.getQueryStats().setTimeSerialization(
            //            DateTime.currentTimeMillis() - start);
            ts_query.getQueryStats().markSerializationSuccessful();
            // dump the original query
            if (true) {
                json.writeFieldName("query");
                json.writeObject(QueryExecutor.this.query);
            }
            // IMPORTANT Make sure the close the JSON array and the generator
            json.writeEndObject();
            json.close();
            return response;
        }
    }
    // trigger the callback chain here
    cb_chain.callback(null);
    return cb_chain.addCallback(new FinalCB());
}