Examples with Deferred com.stumbleupon.async.Deferred used on opensource projects

Example 6 with Deferred

use of com.stumbleupon.async.Deferred in project opentsdb by OpenTSDB.

the class AddDataExample method main.

public static void main(final String[] args) throws Exception {
    processArgs(args);
    // Create a config object with a path to the file for parsing. Or manually
    // override settings.
    // e.g. config.overrideConfig("tsd.storage.hbase.zk_quorum", "localhost");
    final Config config;
    if (pathToConfigFile != null && !pathToConfigFile.isEmpty()) {
        config = new Config(pathToConfigFile);
    } else {
        // Search for a default config from /etc/opentsdb/opentsdb.conf, etc.
        config = new Config(true);
    }
    final TSDB tsdb = new TSDB(config);
    // Declare new metric
    String metricName = "my.tsdb.test.metric";
    // First check to see it doesn't already exist
    // we don't actually need this for the first
    byte[] byteMetricUID;
    // addPoint().
    try {
        byteMetricUID = tsdb.getUID(UniqueIdType.METRIC, metricName);
    } catch (IllegalArgumentException iae) {
        System.out.println("Metric name not valid.");
        iae.printStackTrace();
        System.exit(1);
    } catch (NoSuchUniqueName nsune) {
        // If not, great. Create it.
        byteMetricUID = tsdb.assignUid("metric", metricName);
    }
    // Make a single datum
    long timestamp = System.currentTimeMillis() / 1000;
    long value = 314159;
    // Make key-val
    Map<String, String> tags = new HashMap<String, String>(1);
    tags.put("script", "example1");
    // Start timer
    long startTime1 = System.currentTimeMillis();
    // Write a number of data points at 30 second intervals. Each write will 
    // return a deferred (similar to a Java Future or JS Promise) that will 
    // be called on completion with either a "null" value on success or an
    // exception.
    int n = 100;
    ArrayList<Deferred<Object>> deferreds = new ArrayList<Deferred<Object>>(n);
    for (int i = 0; i < n; i++) {
        Deferred<Object> deferred = tsdb.addPoint(metricName, timestamp, value + i, tags);
        deferreds.add(deferred);
        timestamp += 30;
    }
    // Add the callbacks to the deferred object. (They might have already
    // returned, btw)
    // This will cause the calling thread to wait until the add has completed.
    System.out.println("Waiting for deferred result to return...");
    Deferred.groupInOrder(deferreds).addErrback(new AddDataExample().new errBack()).addCallback(new AddDataExample().new succBack()).join();
    // Alternatively you can add another callback here or use a join with a 
    // timeout argument.
    // End timer.
    long elapsedTime1 = System.currentTimeMillis() - startTime1;
    System.out.println("\nAdding " + n + " points took: " + elapsedTime1 + " milliseconds.\n");
    // Gracefully shutdown connection to TSDB. This is CRITICAL as it will 
    // flush any pending operations to HBase.
    tsdb.shutdown().join();
}

Example 7 with Deferred

use of com.stumbleupon.async.Deferred in project opentsdb by OpenTSDB.

the class Annotation method deleteRange.

/**
   * Deletes global or TSUID associated annotiations for the given time range.
   * @param tsdb The TSDB object to use for storage access
   * @param tsuid An optional TSUID. If set to null, then global annotations for
   * the given range will be deleted
   * @param start_time A start timestamp in milliseconds
   * @param end_time An end timestamp in millseconds
   * @return The number of annotations deleted
   * @throws IllegalArgumentException if the timestamps are invalid
   * @since 2.1
   */
public static Deferred<Integer> deleteRange(final TSDB tsdb, final byte[] tsuid, final long start_time, final long end_time) {
    if (end_time < 1) {
        throw new IllegalArgumentException("The end timestamp has not been set");
    }
    if (end_time < start_time) {
        throw new IllegalArgumentException("The end timestamp cannot be less than the start timestamp");
    }
    final List<Deferred<Object>> delete_requests = new ArrayList<Deferred<Object>>();
    int width = tsuid != null ? Const.SALT_WIDTH() + tsuid.length + Const.TIMESTAMP_BYTES : Const.SALT_WIDTH() + TSDB.metrics_width() + Const.TIMESTAMP_BYTES;
    final byte[] start_row = new byte[width];
    final byte[] end_row = new byte[width];
    // downsample to seconds for the row keys
    final long start = start_time / 1000;
    final long end = end_time / 1000;
    final long normalized_start = (start - (start % Const.MAX_TIMESPAN));
    final long normalized_end = (end - (end % Const.MAX_TIMESPAN) + Const.MAX_TIMESPAN);
    Bytes.setInt(start_row, (int) normalized_start, Const.SALT_WIDTH() + TSDB.metrics_width());
    Bytes.setInt(end_row, (int) normalized_end, Const.SALT_WIDTH() + TSDB.metrics_width());
    if (tsuid != null) {
        // first copy the metric UID then the tags
        System.arraycopy(tsuid, 0, start_row, Const.SALT_WIDTH(), TSDB.metrics_width());
        System.arraycopy(tsuid, 0, end_row, Const.SALT_WIDTH(), TSDB.metrics_width());
        width = Const.SALT_WIDTH() + TSDB.metrics_width() + Const.TIMESTAMP_BYTES;
        final int remainder = tsuid.length - TSDB.metrics_width();
        System.arraycopy(tsuid, TSDB.metrics_width(), start_row, width, remainder);
        System.arraycopy(tsuid, TSDB.metrics_width(), end_row, width, remainder);
    }
    /**
     * Iterates through the scanner results in an asynchronous manner, returning
     * once the scanner returns a null result set.
     */
    final class ScannerCB implements Callback<Deferred<List<Deferred<Object>>>, ArrayList<ArrayList<KeyValue>>> {

        final Scanner scanner;

        public ScannerCB() {
            scanner = tsdb.getClient().newScanner(tsdb.dataTable());
            scanner.setStartKey(start_row);
            scanner.setStopKey(end_row);
            scanner.setFamily(FAMILY);
            if (tsuid != null) {
                final List<String> tsuids = new ArrayList<String>(1);
                tsuids.add(UniqueId.uidToString(tsuid));
                Internal.createAndSetTSUIDFilter(scanner, tsuids);
            }
        }

        public Deferred<List<Deferred<Object>>> scan() {
            return scanner.nextRows().addCallbackDeferring(this);
        }

        @Override
        public Deferred<List<Deferred<Object>>> call(final ArrayList<ArrayList<KeyValue>> rows) throws Exception {
            if (rows == null || rows.isEmpty()) {
                return Deferred.fromResult(delete_requests);
            }
            for (final ArrayList<KeyValue> row : rows) {
                final long base_time = Internal.baseTime(tsdb, row.get(0).key());
                for (KeyValue column : row) {
                    if ((column.qualifier().length == 3 || column.qualifier().length == 5) && column.qualifier()[0] == PREFIX()) {
                        final long timestamp = timeFromQualifier(column.qualifier(), base_time);
                        if (timestamp < start_time || timestamp > end_time) {
                            continue;
                        }
                        final DeleteRequest delete = new DeleteRequest(tsdb.dataTable(), column.key(), FAMILY, column.qualifier());
                        delete_requests.add(tsdb.getClient().delete(delete));
                    }
                }
            }
            return scan();
        }
    }
    /** Called when the scanner is done. Delete requests may still be pending */
    final class ScannerDoneCB implements Callback<Deferred<ArrayList<Object>>, List<Deferred<Object>>> {

        @Override
        public Deferred<ArrayList<Object>> call(final List<Deferred<Object>> deletes) throws Exception {
            return Deferred.group(delete_requests);
        }
    }
    /** Waits on the group of deferreds to complete before returning the count */
    final class GroupCB implements Callback<Deferred<Integer>, ArrayList<Object>> {

        @Override
        public Deferred<Integer> call(final ArrayList<Object> deletes) throws Exception {
            return Deferred.fromResult(deletes.size());
        }
    }
    Deferred<ArrayList<Object>> scanner_done = new ScannerCB().scan().addCallbackDeferring(new ScannerDoneCB());
    return scanner_done.addCallbackDeferring(new GroupCB());
}

Example 8 with Deferred

use of com.stumbleupon.async.Deferred in project opentsdb by OpenTSDB.

the class Branch method storeBranch.

/**
   * Attempts to write the branch definition and optionally child leaves to
   * storage via CompareAndSets.
   * Each returned deferred will be a boolean regarding whether the CAS call 
   * was successful or not. This will be a mix of the branch call and leaves.
   * Some of these may be false, which is OK, because if the branch
   * definition already exists, we don't need to re-write it. Leaves will
   * return false if there was a collision.
   * @param tsdb The TSDB to use for access
   * @param tree The tree to record collisions to
   * @param store_leaves Whether or not child leaves should be written to
   * storage
   * @return A list of deferreds to wait on for completion.
   * @throws HBaseException if there was an issue
   * @throws IllegalArgumentException if the tree ID was missing or data was 
   * missing
   */
public Deferred<ArrayList<Boolean>> storeBranch(final TSDB tsdb, final Tree tree, final boolean store_leaves) {
    if (tree_id < 1 || tree_id > 65535) {
        throw new IllegalArgumentException("Missing or invalid tree ID");
    }
    final ArrayList<Deferred<Boolean>> storage_results = new ArrayList<Deferred<Boolean>>(leaves != null ? leaves.size() + 1 : 1);
    // compile the row key by making sure the display_name is in the path set
    // row ID = <treeID>[<parent.display_name.hashCode()>...]
    final byte[] row = this.compileBranchId();
    // compile the object for storage, this will toss exceptions if we are
    // missing anything important
    final byte[] storage_data = toStorageJson();
    final PutRequest put = new PutRequest(tsdb.treeTable(), row, Tree.TREE_FAMILY(), BRANCH_QUALIFIER, storage_data);
    put.setBufferable(true);
    storage_results.add(tsdb.getClient().compareAndSet(put, new byte[0]));
    // store leaves if told to and put the storage calls in our deferred group
    if (store_leaves && leaves != null && !leaves.isEmpty()) {
        for (final Leaf leaf : leaves.values()) {
            storage_results.add(leaf.storeLeaf(tsdb, row, tree));
        }
    }
    return Deferred.group(storage_results);
}

Example 9 with Deferred

use of com.stumbleupon.async.Deferred in project opentsdb by OpenTSDB.

the class TSUIDQuery method getTSMetas.

/**
   * Returns all TSMeta objects stored for timeseries defined by this query. The 
   * query is similar to TsdbQuery without any aggregations. Returns an empty 
   * list, when no TSMetas are found. Only returns stored TSMetas.
   * <p>
   * NOTE: If you called {@link #setQuery(String, Map)} successfully this will
   * immediately scan the meta table. But if you used the CTOR to set the
   * metric and tags it will attempt to resolve those and may return an exception.
   * @return A list of existing TSMetas for the timeseries covered by the query.
   * @throws IllegalArgumentException When either no metric was specified or the
   * tag map was null (Empty map is OK).
   */
public Deferred<List<TSMeta>> getTSMetas() {
    class ResolutionCB implements Callback<Deferred<List<TSMeta>>, Object> {

        @Override
        public Deferred<List<TSMeta>> call(final Object done) throws Exception {
            final Scanner scanner = getScanner();
            scanner.setQualifier(TSMeta.META_QUALIFIER());
            final Deferred<List<TSMeta>> results = new Deferred<List<TSMeta>>();
            final List<TSMeta> tsmetas = new ArrayList<TSMeta>();
            final List<Deferred<TSMeta>> tsmeta_group = new ArrayList<Deferred<TSMeta>>();
            final class TSMetaGroupCB implements Callback<Object, ArrayList<TSMeta>> {

                @Override
                public List<TSMeta> call(ArrayList<TSMeta> ts) throws Exception {
                    for (TSMeta tsm : ts) {
                        if (tsm != null) {
                            tsmetas.add(tsm);
                        }
                    }
                    results.callback(tsmetas);
                    return null;
                }

                @Override
                public String toString() {
                    return "TSMeta callback";
                }
            }
            final class ErrBack implements Callback<Object, Exception> {

                @Override
                public Object call(final Exception e) throws Exception {
                    results.callback(e);
                    return null;
                }

                @Override
                public String toString() {
                    return "Error callback";
                }
            }
            /**
         * Scanner callback that will call itself while iterating through the 
         * tsdb-meta table.
         * 
         * Keeps track of a Set of Deferred TSMeta calls. When all rows are scanned,
         * will wait for all TSMeta calls to be completed and then create the result 
         * list.
         */
            final class ScannerCB implements Callback<Object, ArrayList<ArrayList<KeyValue>>> {

                /**
           * Starts the scanner and is called recursively to fetch the next set of
           * rows from the scanner.
           * @return The map of spans if loaded successfully, null if no data was
           * found
           */
                public Object scan() {
                    return scanner.nextRows().addCallback(this).addErrback(new ErrBack());
                }

                /**
           * Loops through each row of the scanner results and parses out data
           * points and optional meta data
           * @return null if no rows were found, otherwise the TreeMap with spans
           */
                @Override
                public Object call(final ArrayList<ArrayList<KeyValue>> rows) throws Exception {
                    try {
                        if (rows == null) {
                            Deferred.group(tsmeta_group).addCallback(new TSMetaGroupCB()).addErrback(new ErrBack());
                            return null;
                        }
                        for (final ArrayList<KeyValue> row : rows) {
                            tsmeta_group.add(TSMeta.parseFromColumn(tsdb, row.get(0), true));
                        }
                        return scan();
                    } catch (Exception e) {
                        results.callback(e);
                        return null;
                    }
                }
            }
            new ScannerCB().scan();
            return results;
        }

        @Override
        public String toString() {
            return "TSMeta scan callback";
        }
    }
    if (metric_uid == null) {
        return resolveMetric().addCallbackDeferring(new ResolutionCB());
    }
    try {
        return new ResolutionCB().call(null);
    } catch (Exception e) {
        return Deferred.fromError(e);
    }
}

Example 10 with Deferred

use of com.stumbleupon.async.Deferred in project opentsdb by OpenTSDB.

the class UIDMeta method syncToStorage.

/**
   * Attempts a CompareAndSet storage call, loading the object from storage, 
   * synchronizing changes, and attempting a put.
   * <b>Note:</b> If the local object didn't have any fields set by the caller
   * then the data will not be written.
   * @param tsdb The TSDB to use for storage access
   * @param overwrite When the RPC method is PUT, will overwrite all user
   * accessible fields
   * @return True if the storage call was successful, false if the object was
   * modified in storage during the CAS call. If false, retry the call. Other 
   * failures will result in an exception being thrown.
   * @throws HBaseException if there was an issue fetching
   * @throws IllegalArgumentException if parsing failed
   * @throws NoSuchUniqueId If the UID does not exist
   * @throws IllegalStateException if the data hasn't changed. This is OK!
   * @throws JSONException if the object could not be serialized
   */
public Deferred<Boolean> syncToStorage(final TSDB tsdb, final boolean overwrite) {
    if (uid == null || uid.isEmpty()) {
        throw new IllegalArgumentException("Missing UID");
    }
    if (type == null) {
        throw new IllegalArgumentException("Missing type");
    }
    boolean has_changes = false;
    for (Map.Entry<String, Boolean> entry : changed.entrySet()) {
        if (entry.getValue()) {
            has_changes = true;
            break;
        }
    }
    if (!has_changes) {
        LOG.debug(this + " does not have changes, skipping sync to storage");
        throw new IllegalStateException("No changes detected in UID meta data");
    }
    /**
     * Callback used to verify that the UID to name mapping exists. Uses the TSD
     * for verification so the name may be cached. If the name does not exist
     * it will throw a NoSuchUniqueId and the meta data will not be saved to
     * storage
     */
    final class NameCB implements Callback<Deferred<Boolean>, String> {

        private final UIDMeta local_meta;

        public NameCB(final UIDMeta meta) {
            local_meta = meta;
        }

        /**
       *  Nested callback used to merge and store the meta data after verifying
       *  that the UID mapping exists. It has to access the {@code local_meta} 
       *  object so that's why it's nested within the NameCB class
       */
        final class StoreUIDMeta implements Callback<Deferred<Boolean>, ArrayList<KeyValue>> {

            /**
         * Executes the CompareAndSet after merging changes
         * @return True if the CAS was successful, false if the stored data
         * was modified during flight.
         */
            @Override
            public Deferred<Boolean> call(final ArrayList<KeyValue> row) throws Exception {
                final UIDMeta stored_meta;
                if (row == null || row.isEmpty()) {
                    stored_meta = null;
                } else {
                    stored_meta = JSON.parseToObject(row.get(0).value(), UIDMeta.class);
                    stored_meta.initializeChangedMap();
                }
                final byte[] original_meta = row == null || row.isEmpty() ? new byte[0] : row.get(0).value();
                if (stored_meta != null) {
                    local_meta.syncMeta(stored_meta, overwrite);
                }
                // verify the name is set locally just to be safe
                if (name == null || name.isEmpty()) {
                    local_meta.name = name;
                }
                final PutRequest put = new PutRequest(tsdb.uidTable(), UniqueId.stringToUid(uid), FAMILY, (type.toString().toLowerCase() + "_meta").getBytes(CHARSET), local_meta.getStorageJSON());
                return tsdb.getClient().compareAndSet(put, original_meta);
            }
        }

        /**
       * NameCB method that fetches the object from storage for merging and
       * use in the CAS call
       * @return The results of the {@link #StoreUIDMeta} callback
       */
        @Override
        public Deferred<Boolean> call(final String name) throws Exception {
            final GetRequest get = new GetRequest(tsdb.uidTable(), UniqueId.stringToUid(uid));
            get.family(FAMILY);
            get.qualifier((type.toString().toLowerCase() + "_meta").getBytes(CHARSET));
            // #2 deferred
            return tsdb.getClient().get(get).addCallbackDeferring(new StoreUIDMeta());
        }
    }
    // start the callback chain by veryfing that the UID name mapping exists
    return tsdb.getUidName(type, UniqueId.stringToUid(uid)).addCallbackDeferring(new NameCB(this));
}