Examples with Callback com.stumbleupon.async.Callback used on opensource projects

Example 11 with Callback

use of com.stumbleupon.async.Callback in project opentsdb by OpenTSDB.

the class TSUIDQuery method getTSMetas.

/**
   * Returns all TSMeta objects stored for timeseries defined by this query. The 
   * query is similar to TsdbQuery without any aggregations. Returns an empty 
   * list, when no TSMetas are found. Only returns stored TSMetas.
   * <p>
   * NOTE: If you called {@link #setQuery(String, Map)} successfully this will
   * immediately scan the meta table. But if you used the CTOR to set the
   * metric and tags it will attempt to resolve those and may return an exception.
   * @return A list of existing TSMetas for the timeseries covered by the query.
   * @throws IllegalArgumentException When either no metric was specified or the
   * tag map was null (Empty map is OK).
   */
public Deferred<List<TSMeta>> getTSMetas() {
    class ResolutionCB implements Callback<Deferred<List<TSMeta>>, Object> {

        @Override
        public Deferred<List<TSMeta>> call(final Object done) throws Exception {
            final Scanner scanner = getScanner();
            scanner.setQualifier(TSMeta.META_QUALIFIER());
            final Deferred<List<TSMeta>> results = new Deferred<List<TSMeta>>();
            final List<TSMeta> tsmetas = new ArrayList<TSMeta>();
            final List<Deferred<TSMeta>> tsmeta_group = new ArrayList<Deferred<TSMeta>>();
            final class TSMetaGroupCB implements Callback<Object, ArrayList<TSMeta>> {

                @Override
                public List<TSMeta> call(ArrayList<TSMeta> ts) throws Exception {
                    for (TSMeta tsm : ts) {
                        if (tsm != null) {
                            tsmetas.add(tsm);
                        }
                    }
                    results.callback(tsmetas);
                    return null;
                }

                @Override
                public String toString() {
                    return "TSMeta callback";
                }
            }
            final class ErrBack implements Callback<Object, Exception> {

                @Override
                public Object call(final Exception e) throws Exception {
                    results.callback(e);
                    return null;
                }

                @Override
                public String toString() {
                    return "Error callback";
                }
            }
            /**
         * Scanner callback that will call itself while iterating through the 
         * tsdb-meta table.
         * 
         * Keeps track of a Set of Deferred TSMeta calls. When all rows are scanned,
         * will wait for all TSMeta calls to be completed and then create the result 
         * list.
         */
            final class ScannerCB implements Callback<Object, ArrayList<ArrayList<KeyValue>>> {

                /**
           * Starts the scanner and is called recursively to fetch the next set of
           * rows from the scanner.
           * @return The map of spans if loaded successfully, null if no data was
           * found
           */
                public Object scan() {
                    return scanner.nextRows().addCallback(this).addErrback(new ErrBack());
                }

                /**
           * Loops through each row of the scanner results and parses out data
           * points and optional meta data
           * @return null if no rows were found, otherwise the TreeMap with spans
           */
                @Override
                public Object call(final ArrayList<ArrayList<KeyValue>> rows) throws Exception {
                    try {
                        if (rows == null) {
                            Deferred.group(tsmeta_group).addCallback(new TSMetaGroupCB()).addErrback(new ErrBack());
                            return null;
                        }
                        for (final ArrayList<KeyValue> row : rows) {
                            tsmeta_group.add(TSMeta.parseFromColumn(tsdb, row.get(0), true));
                        }
                        return scan();
                    } catch (Exception e) {
                        results.callback(e);
                        return null;
                    }
                }
            }
            new ScannerCB().scan();
            return results;
        }

        @Override
        public String toString() {
            return "TSMeta scan callback";
        }
    }
    if (metric_uid == null) {
        return resolveMetric().addCallbackDeferring(new ResolutionCB());
    }
    try {
        return new ResolutionCB().call(null);
    } catch (Exception e) {
        return Deferred.fromError(e);
    }
}

Example 12 with Callback

use of com.stumbleupon.async.Callback in project opentsdb by OpenTSDB.

the class UIDMeta method syncToStorage.

/**
   * Attempts a CompareAndSet storage call, loading the object from storage, 
   * synchronizing changes, and attempting a put.
   * <b>Note:</b> If the local object didn't have any fields set by the caller
   * then the data will not be written.
   * @param tsdb The TSDB to use for storage access
   * @param overwrite When the RPC method is PUT, will overwrite all user
   * accessible fields
   * @return True if the storage call was successful, false if the object was
   * modified in storage during the CAS call. If false, retry the call. Other 
   * failures will result in an exception being thrown.
   * @throws HBaseException if there was an issue fetching
   * @throws IllegalArgumentException if parsing failed
   * @throws NoSuchUniqueId If the UID does not exist
   * @throws IllegalStateException if the data hasn't changed. This is OK!
   * @throws JSONException if the object could not be serialized
   */
public Deferred<Boolean> syncToStorage(final TSDB tsdb, final boolean overwrite) {
    if (uid == null || uid.isEmpty()) {
        throw new IllegalArgumentException("Missing UID");
    }
    if (type == null) {
        throw new IllegalArgumentException("Missing type");
    }
    boolean has_changes = false;
    for (Map.Entry<String, Boolean> entry : changed.entrySet()) {
        if (entry.getValue()) {
            has_changes = true;
            break;
        }
    }
    if (!has_changes) {
        LOG.debug(this + " does not have changes, skipping sync to storage");
        throw new IllegalStateException("No changes detected in UID meta data");
    }
    /**
     * Callback used to verify that the UID to name mapping exists. Uses the TSD
     * for verification so the name may be cached. If the name does not exist
     * it will throw a NoSuchUniqueId and the meta data will not be saved to
     * storage
     */
    final class NameCB implements Callback<Deferred<Boolean>, String> {

        private final UIDMeta local_meta;

        public NameCB(final UIDMeta meta) {
            local_meta = meta;
        }

        /**
       *  Nested callback used to merge and store the meta data after verifying
       *  that the UID mapping exists. It has to access the {@code local_meta} 
       *  object so that's why it's nested within the NameCB class
       */
        final class StoreUIDMeta implements Callback<Deferred<Boolean>, ArrayList<KeyValue>> {

            /**
         * Executes the CompareAndSet after merging changes
         * @return True if the CAS was successful, false if the stored data
         * was modified during flight.
         */
            @Override
            public Deferred<Boolean> call(final ArrayList<KeyValue> row) throws Exception {
                final UIDMeta stored_meta;
                if (row == null || row.isEmpty()) {
                    stored_meta = null;
                } else {
                    stored_meta = JSON.parseToObject(row.get(0).value(), UIDMeta.class);
                    stored_meta.initializeChangedMap();
                }
                final byte[] original_meta = row == null || row.isEmpty() ? new byte[0] : row.get(0).value();
                if (stored_meta != null) {
                    local_meta.syncMeta(stored_meta, overwrite);
                }
                // verify the name is set locally just to be safe
                if (name == null || name.isEmpty()) {
                    local_meta.name = name;
                }
                final PutRequest put = new PutRequest(tsdb.uidTable(), UniqueId.stringToUid(uid), FAMILY, (type.toString().toLowerCase() + "_meta").getBytes(CHARSET), local_meta.getStorageJSON());
                return tsdb.getClient().compareAndSet(put, original_meta);
            }
        }

        /**
       * NameCB method that fetches the object from storage for merging and
       * use in the CAS call
       * @return The results of the {@link #StoreUIDMeta} callback
       */
        @Override
        public Deferred<Boolean> call(final String name) throws Exception {
            final GetRequest get = new GetRequest(tsdb.uidTable(), UniqueId.stringToUid(uid));
            get.family(FAMILY);
            get.qualifier((type.toString().toLowerCase() + "_meta").getBytes(CHARSET));
            // #2 deferred
            return tsdb.getClient().get(get).addCallbackDeferring(new StoreUIDMeta());
        }
    }
    // start the callback chain by veryfing that the UID name mapping exists
    return tsdb.getUidName(type, UniqueId.stringToUid(uid)).addCallbackDeferring(new NameCB(this));
}

Example 13 with Callback

use of com.stumbleupon.async.Callback in project opentsdb by OpenTSDB.

the class UIDMeta method getUIDMeta.

/**
   * Verifies the UID object exists, then attempts to fetch the meta from 
   * storage and if not found, returns a default object.
   * <p>
   * The reason for returning a default object (with the type, uid and name set)
   * is due to users who may have just enabled meta data or have upgraded; we 
   * want to return valid data. If they modify the entry, it will write to 
   * storage. You can tell it's a default if the {@code created} value is 0. If
   * the meta was generated at UID assignment or updated by the meta sync CLI
   * command, it will have a valid created timestamp.
   * @param tsdb The TSDB to use for storage access
   * @param type The type of UID to fetch
   * @param uid The ID of the meta to fetch
   * @return A UIDMeta from storage or a default
   * @throws HBaseException if there was an issue fetching
   * @throws NoSuchUniqueId If the UID does not exist
   */
public static Deferred<UIDMeta> getUIDMeta(final TSDB tsdb, final UniqueIdType type, final byte[] uid) {
    /**
     * Callback used to verify that the UID to name mapping exists. Uses the TSD
     * for verification so the name may be cached. If the name does not exist
     * it will throw a NoSuchUniqueId and the meta data will not be returned. 
     * This helps in case the user deletes a UID but the meta data is still 
     * stored. The fsck utility can be used later to cleanup orphaned objects.
     */
    class NameCB implements Callback<Deferred<UIDMeta>, String> {

        /**
       * Called after verifying that the name mapping exists
       * @return The results of {@link #FetchMetaCB}
       */
        @Override
        public Deferred<UIDMeta> call(final String name) throws Exception {
            /**
         * Inner class called to retrieve the meta data after verifying that the
         * name mapping exists. It requires the name to set the default, hence
         * the reason it's nested.
         */
            class FetchMetaCB implements Callback<Deferred<UIDMeta>, ArrayList<KeyValue>> {

                /**
           * Called to parse the response of our storage GET call after 
           * verification
           * @return The stored UIDMeta or a default object if the meta data
           * did not exist 
           */
                @Override
                public Deferred<UIDMeta> call(ArrayList<KeyValue> row) throws Exception {
                    if (row == null || row.isEmpty()) {
                        // return the default
                        final UIDMeta meta = new UIDMeta();
                        meta.uid = UniqueId.uidToString(uid);
                        meta.type = type;
                        meta.name = name;
                        return Deferred.fromResult(meta);
                    }
                    final UIDMeta meta = JSON.parseToObject(row.get(0).value(), UIDMeta.class);
                    // overwrite the name and UID
                    meta.name = name;
                    meta.uid = UniqueId.uidToString(uid);
                    // fix missing types
                    if (meta.type == null) {
                        final String qualifier = new String(row.get(0).qualifier(), CHARSET);
                        meta.type = UniqueId.stringToUniqueIdType(qualifier.substring(0, qualifier.indexOf("_meta")));
                    }
                    meta.initializeChangedMap();
                    return Deferred.fromResult(meta);
                }
            }
            final GetRequest get = new GetRequest(tsdb.uidTable(), uid);
            get.family(FAMILY);
            get.qualifier((type.toString().toLowerCase() + "_meta").getBytes(CHARSET));
            return tsdb.getClient().get(get).addCallbackDeferring(new FetchMetaCB());
        }
    }
    // verify that the UID is still in the map before fetching from storage
    return tsdb.getUidName(type, uid).addCallbackDeferring(new NameCB());
}

Example 14 with Callback

use of com.stumbleupon.async.Callback in project opentsdb by OpenTSDB.

the class MetaSync method run.

/**
   * Loops through the entire TSDB data set and exits when complete.
   */
public void run() {
    // list of deferred calls used to act as a buffer
    final ArrayList<Deferred<Boolean>> storage_calls = new ArrayList<Deferred<Boolean>>();
    final Deferred<Object> result = new Deferred<Object>();
    final class ErrBack implements Callback<Object, Exception> {

        @Override
        public Object call(Exception e) throws Exception {
            Throwable ex = e;
            while (ex.getClass().equals(DeferredGroupException.class)) {
                if (ex.getCause() == null) {
                    LOG.warn("Unable to get to the root cause of the DGE");
                    break;
                }
                ex = ex.getCause();
            }
            LOG.error("Sync thread failed with exception", ex);
            result.callback(null);
            return null;
        }
    }
    final ErrBack err_back = new ErrBack();
    /**
     * Called when we have encountered a previously un-processed UIDMeta object.
     * This callback will update the "created" timestamp of the UIDMeta and
     * store the update, replace corrupted metas and update search plugins.
     */
    final class UidCB implements Callback<Deferred<Boolean>, UIDMeta> {

        private final UniqueIdType type;

        private final byte[] uid;

        private final long timestamp;

        /**
       * Constructor that initializes the local callback
       * @param type The type of UIDMeta we're dealing with
       * @param uid The UID of the meta object as a byte array
       * @param timestamp The timestamp of the timeseries when this meta
       * was first detected
       */
        public UidCB(final UniqueIdType type, final byte[] uid, final long timestamp) {
            this.type = type;
            this.uid = uid;
            this.timestamp = timestamp;
        }

        /**
       * A nested class called after fetching a UID name to use when creating a
       * new UIDMeta object if the previous object was corrupted. Also pushes
       * the meta off to the search plugin.
       */
        final class UidNameCB implements Callback<Deferred<Boolean>, String> {

            @Override
            public Deferred<Boolean> call(final String name) throws Exception {
                UIDMeta new_meta = new UIDMeta(type, uid, name);
                new_meta.setCreated(timestamp);
                tsdb.indexUIDMeta(new_meta);
                LOG.info("Replacing corrupt UID [" + UniqueId.uidToString(uid) + "] of type [" + type + "]");
                return new_meta.syncToStorage(tsdb, true);
            }
        }

        @Override
        public Deferred<Boolean> call(final UIDMeta meta) throws Exception {
            // otherwise it's probably an accurate timestamp
            if (meta.getCreated() > (timestamp + 3600) || meta.getCreated() == 0) {
                LOG.info("Updating UID [" + UniqueId.uidToString(uid) + "] of type [" + type + "]");
                meta.setCreated(timestamp);
                // consider it corrupt and replace it with a new object
                if (meta.getUID() == null || meta.getUID().isEmpty() || meta.getType() == null) {
                    return tsdb.getUidName(type, uid).addCallbackDeferring(new UidNameCB());
                } else {
                    // the meta was good, just needed a timestamp update so sync to
                    // search and storage
                    tsdb.indexUIDMeta(meta);
                    LOG.info("Syncing valid UID [" + UniqueId.uidToString(uid) + "] of type [" + type + "]");
                    return meta.syncToStorage(tsdb, false);
                }
            } else {
                LOG.debug("UID [" + UniqueId.uidToString(uid) + "] of type [" + type + "] is up to date in storage");
                return Deferred.fromResult(true);
            }
        }
    }
    /**
     * Called to handle a previously unprocessed TSMeta object. This callback
     * will update the "created" timestamp, create a new TSMeta object if
     * missing, and update search plugins.
     */
    final class TSMetaCB implements Callback<Deferred<Boolean>, TSMeta> {

        private final String tsuid_string;

        private final byte[] tsuid;

        private final long timestamp;

        /**
       * Default constructor
       * @param tsuid ID of the timeseries
       * @param timestamp The timestamp when the first data point was recorded
       */
        public TSMetaCB(final byte[] tsuid, final long timestamp) {
            this.tsuid = tsuid;
            tsuid_string = UniqueId.uidToString(tsuid);
            this.timestamp = timestamp;
        }

        @Override
        public Deferred<Boolean> call(final TSMeta meta) throws Exception {
            /** Called to process the new meta through the search plugin and tree code */
            final class IndexCB implements Callback<Deferred<Boolean>, TSMeta> {

                @Override
                public Deferred<Boolean> call(final TSMeta new_meta) throws Exception {
                    tsdb.indexTSMeta(new_meta);
                    // pass through the trees
                    return tsdb.processTSMetaThroughTrees(new_meta);
                }
            }
            /** Called to load the newly created meta object for passage onto the
         * search plugin and tree builder if configured
         */
            final class GetCB implements Callback<Deferred<Boolean>, Boolean> {

                @Override
                public final Deferred<Boolean> call(final Boolean exists) throws Exception {
                    if (exists) {
                        return TSMeta.getTSMeta(tsdb, tsuid_string).addCallbackDeferring(new IndexCB());
                    } else {
                        return Deferred.fromResult(false);
                    }
                }
            }
            /** Errback on the store new call to catch issues */
            class ErrBack implements Callback<Object, Exception> {

                public Object call(final Exception e) throws Exception {
                    LOG.warn("Failed creating meta for: " + tsuid + " with exception: ", e);
                    return null;
                }
            }
            // new one
            if (meta == null) {
                /**
           * Called after successfully creating a TSMeta counter and object,
           * used to convert the deferred long to a boolean so it can be
           * combined with other calls for waiting.
           */
                final class CreatedCB implements Callback<Deferred<Boolean>, Long> {

                    @Override
                    public Deferred<Boolean> call(Long value) throws Exception {
                        LOG.info("Created counter and meta for timeseries [" + tsuid_string + "]");
                        return Deferred.fromResult(true);
                    }
                }
                /**
           * Called after checking to see if the counter exists and is used
           * to determine if we should create a new counter AND meta or just a
           * new meta
           */
                final class CounterCB implements Callback<Deferred<Boolean>, Boolean> {

                    @Override
                    public Deferred<Boolean> call(final Boolean exists) throws Exception {
                        if (!exists) {
                            // here or in the local callback
                            return TSMeta.incrementAndGetCounter(tsdb, tsuid).addCallbackDeferring(new CreatedCB());
                        } else {
                            TSMeta new_meta = new TSMeta(tsuid, timestamp);
                            tsdb.indexTSMeta(new_meta);
                            LOG.info("Counter exists but meta was null, creating meta data " + "for timeseries [" + tsuid_string + "]");
                            return new_meta.storeNew(tsdb).addCallbackDeferring(new GetCB()).addErrback(new ErrBack());
                        }
                    }
                }
                // improperly before the meta is flushed to storage.
                return TSMeta.counterExistsInStorage(tsdb, tsuid).addCallbackDeferring(new CounterCB());
            }
            // corrupted
            if (meta.getTSUID() == null || meta.getTSUID().isEmpty()) {
                LOG.warn("Replacing corrupt meta data for timeseries [" + tsuid_string + "]");
                TSMeta new_meta = new TSMeta(tsuid, timestamp);
                tsdb.indexTSMeta(new_meta);
                return new_meta.storeNew(tsdb).addCallbackDeferring(new GetCB()).addErrback(new ErrBack());
            } else {
                // hour otherwise it's probably an accurate timestamp
                if (meta.getCreated() > (timestamp + 3600) || meta.getCreated() == 0) {
                    meta.setCreated(timestamp);
                    tsdb.indexTSMeta(meta);
                    LOG.info("Updated created timestamp for timeseries [" + tsuid_string + "]");
                    return meta.syncToStorage(tsdb, false);
                }
                LOG.debug("TSUID [" + tsuid_string + "] is up to date in storage");
                return Deferred.fromResult(false);
            }
        }
    }
    /**
     * Scanner callback that recursively loops through all of the data point
     * rows. Note that we don't process the actual data points, just the row
     * keys.
     */
    final class MetaScanner implements Callback<Object, ArrayList<ArrayList<KeyValue>>> {

        private byte[] last_tsuid = null;

        private String tsuid_string = "";

        /**
       * Fetches the next set of rows from the scanner and adds this class as
       * a callback
       * @return A meaningless deferred to wait on until all data rows have
       * been processed.
       */
        public Object scan() {
            return scanner.nextRows().addCallback(this).addErrback(err_back);
        }

        @Override
        public Object call(ArrayList<ArrayList<KeyValue>> rows) throws Exception {
            if (rows == null) {
                result.callback(null);
                return null;
            }
            for (final ArrayList<KeyValue> row : rows) {
                try {
                    final byte[] tsuid = UniqueId.getTSUIDFromKey(row.get(0).key(), TSDB.metrics_width(), Const.TIMESTAMP_BYTES);
                    // so we save time
                    if (last_tsuid != null && Arrays.equals(last_tsuid, tsuid)) {
                        continue;
                    }
                    last_tsuid = tsuid;
                    // see if we've already processed this tsuid and if so, continue
                    if (processed_tsuids.contains(Arrays.hashCode(tsuid))) {
                        continue;
                    }
                    tsuid_string = UniqueId.uidToString(tsuid);
                    /**
             * An error callback used to catch issues with a particular timeseries
             * or UIDMeta such as a missing UID name. We want to continue
             * processing when this happens so we'll just log the error and
             * the user can issue a command later to clean up orphaned meta
             * entries.
             */
                    final class RowErrBack implements Callback<Object, Exception> {

                        @Override
                        public Object call(Exception e) throws Exception {
                            Throwable ex = e;
                            while (ex.getClass().equals(DeferredGroupException.class)) {
                                if (ex.getCause() == null) {
                                    LOG.warn("Unable to get to the root cause of the DGE");
                                    break;
                                }
                                ex = ex.getCause();
                            }
                            if (ex.getClass().equals(IllegalStateException.class)) {
                                LOG.error("Invalid data when processing TSUID [" + tsuid_string + "]: " + ex.getMessage());
                            } else if (ex.getClass().equals(IllegalArgumentException.class)) {
                                LOG.error("Invalid data when processing TSUID [" + tsuid_string + "]: " + ex.getMessage());
                            } else if (ex.getClass().equals(NoSuchUniqueId.class)) {
                                LOG.warn("Timeseries [" + tsuid_string + "] includes a non-existant UID: " + ex.getMessage());
                            } else {
                                LOG.error("Unknown exception processing row: " + row, ex);
                            }
                            return null;
                        }
                    }
                    // add tsuid to the processed list
                    processed_tsuids.add(Arrays.hashCode(tsuid));
                    // we may have a new TSUID or UIDs, so fetch the timestamp of the 
                    // row for use as the "created" time. Depending on speed we could 
                    // parse datapoints, but for now the hourly row time is enough
                    final long timestamp = Bytes.getUnsignedInt(row.get(0).key(), Const.SALT_WIDTH() + TSDB.metrics_width());
                    LOG.debug("[" + thread_id + "] Processing TSUID: " + tsuid_string + "  row timestamp: " + timestamp);
                    // now process the UID metric meta data
                    final byte[] metric_uid_bytes = Arrays.copyOfRange(tsuid, 0, TSDB.metrics_width());
                    final String metric_uid = UniqueId.uidToString(metric_uid_bytes);
                    Long last_get = metric_uids.get(metric_uid);
                    if (last_get == null || last_get == 0 || timestamp < last_get) {
                        // fetch and update. Returns default object if the meta doesn't
                        // exist, so we can just call sync on this to create a missing
                        // entry
                        final UidCB cb = new UidCB(UniqueIdType.METRIC, metric_uid_bytes, timestamp);
                        final Deferred<Boolean> process_uid = UIDMeta.getUIDMeta(tsdb, UniqueIdType.METRIC, metric_uid_bytes).addCallbackDeferring(cb).addErrback(new RowErrBack());
                        storage_calls.add(process_uid);
                        metric_uids.put(metric_uid, timestamp);
                    }
                    // loop through the tags and process their meta
                    final List<byte[]> tags = UniqueId.getTagsFromTSUID(tsuid_string);
                    int idx = 0;
                    for (byte[] tag : tags) {
                        final UniqueIdType type = (idx % 2 == 0) ? UniqueIdType.TAGK : UniqueIdType.TAGV;
                        idx++;
                        final String uid = UniqueId.uidToString(tag);
                        // check the maps to see if we need to bother updating
                        if (type == UniqueIdType.TAGK) {
                            last_get = tagk_uids.get(uid);
                        } else {
                            last_get = tagv_uids.get(uid);
                        }
                        if (last_get != null && last_get != 0 && last_get <= timestamp) {
                            continue;
                        }
                        // fetch and update. Returns default object if the meta doesn't
                        // exist, so we can just call sync on this to create a missing
                        // entry
                        final UidCB cb = new UidCB(type, tag, timestamp);
                        final Deferred<Boolean> process_uid = UIDMeta.getUIDMeta(tsdb, type, tag).addCallbackDeferring(cb).addErrback(new RowErrBack());
                        storage_calls.add(process_uid);
                        if (type == UniqueIdType.TAGK) {
                            tagk_uids.put(uid, timestamp);
                        } else {
                            tagv_uids.put(uid, timestamp);
                        }
                    }
                    // handle the timeseries meta last so we don't record it if one
                    // or more of the UIDs had an issue
                    final Deferred<Boolean> process_tsmeta = TSMeta.getTSMeta(tsdb, tsuid_string).addCallbackDeferring(new TSMetaCB(tsuid, timestamp)).addErrback(new RowErrBack());
                    storage_calls.add(process_tsmeta);
                } catch (RuntimeException e) {
                    LOG.error("Processing row " + row + " failed with exception: " + e.getMessage());
                    LOG.debug("Row: " + row + " stack trace: ", e);
                }
            }
            /**
         * A buffering callback used to avoid StackOverflowError exceptions
         * where the list of deferred calls can exceed the limit. Instead we'll
         * process the Scanner's limit in rows, wait for all of the storage
         * calls to complete, then continue on to the next set.
         */
            final class ContinueCB implements Callback<Object, ArrayList<Boolean>> {

                @Override
                public Object call(ArrayList<Boolean> puts) throws Exception {
                    storage_calls.clear();
                    return scan();
                }
            }
            /**
         * Catch exceptions in one of the grouped calls and continue scanning.
         * Without this the user may not see the exception and the thread will
         * just die silently.
         */
            final class ContinueEB implements Callback<Object, Exception> {

                @Override
                public Object call(Exception e) throws Exception {
                    Throwable ex = e;
                    while (ex.getClass().equals(DeferredGroupException.class)) {
                        if (ex.getCause() == null) {
                            LOG.warn("Unable to get to the root cause of the DGE");
                            break;
                        }
                        ex = ex.getCause();
                    }
                    LOG.error("[" + thread_id + "] Upstream Exception: ", ex);
                    return scan();
                }
            }
            // call ourself again but wait for the current set of storage calls to
            // complete so we don't OOM
            Deferred.group(storage_calls).addCallback(new ContinueCB()).addErrback(new ContinueEB());
            return null;
        }
    }
    final MetaScanner scanner = new MetaScanner();
    try {
        scanner.scan();
        result.joinUninterruptibly();
        LOG.info("[" + thread_id + "] Complete");
    } catch (Exception e) {
        LOG.error("[" + thread_id + "] Scanner Exception", e);
        throw new RuntimeException("[" + thread_id + "] Scanner exception", e);
    }
}

Example 15 with Callback

use of com.stumbleupon.async.Callback in project opentsdb by OpenTSDB.

the class TextImporter method importFile.

private static int importFile(final HBaseClient client, final TSDB tsdb, final String path, final boolean skip_errors) throws IOException {
    final long start_time = System.nanoTime();
    long ping_start_time = start_time;
    final BufferedReader in = open(path);
    String line = null;
    int points = 0;
    try {
        final class Errback implements Callback<Object, Exception> {

            public Object call(final Exception arg) {
                if (arg instanceof PleaseThrottleException) {
                    final PleaseThrottleException e = (PleaseThrottleException) arg;
                    LOG.warn("Need to throttle, HBase isn't keeping up.", e);
                    throttle = true;
                    final HBaseRpc rpc = e.getFailedRpc();
                    if (rpc instanceof PutRequest) {
                        // Don't lose edits.
                        client.put((PutRequest) rpc);
                    }
                    return null;
                }
                LOG.error("Exception caught while processing file " + path, arg);
                System.exit(2);
                return arg;
            }

            public String toString() {
                return "importFile errback";
            }
        }
        ;
        final Errback errback = new Errback();
        LOG.info("reading from file:" + path);
        while ((line = in.readLine()) != null) {
            final String[] words = Tags.splitString(line, ' ');
            final String metric = words[0];
            if (metric.length() <= 0) {
                if (skip_errors) {
                    LOG.error("invalid metric: " + metric);
                    LOG.error("error while processing file " + path + " line=" + line + "... Continuing");
                    continue;
                } else {
                    throw new RuntimeException("invalid metric: " + metric);
                }
            }
            final long timestamp;
            try {
                timestamp = Tags.parseLong(words[1]);
                if (timestamp <= 0) {
                    if (skip_errors) {
                        LOG.error("invalid timestamp: " + timestamp);
                        LOG.error("error while processing file " + path + " line=" + line + "... Continuing");
                        continue;
                    } else {
                        throw new RuntimeException("invalid timestamp: " + timestamp);
                    }
                }
            } catch (final RuntimeException e) {
                if (skip_errors) {
                    LOG.error("invalid timestamp: " + e.getMessage());
                    LOG.error("error while processing file " + path + " line=" + line + "... Continuing");
                    continue;
                } else {
                    throw e;
                }
            }
            final String value = words[2];
            if (value.length() <= 0) {
                if (skip_errors) {
                    LOG.error("invalid value: " + value);
                    LOG.error("error while processing file " + path + " line=" + line + "... Continuing");
                    continue;
                } else {
                    throw new RuntimeException("invalid value: " + value);
                }
            }
            try {
                final HashMap<String, String> tags = new HashMap<String, String>();
                for (int i = 3; i < words.length; i++) {
                    if (!words[i].isEmpty()) {
                        Tags.parse(tags, words[i]);
                    }
                }
                final WritableDataPoints dp = getDataPoints(tsdb, metric, tags);
                Deferred<Object> d;
                if (Tags.looksLikeInteger(value)) {
                    d = dp.addPoint(timestamp, Tags.parseLong(value));
                } else {
                    // floating point value
                    d = dp.addPoint(timestamp, Float.parseFloat(value));
                }
                d.addErrback(errback);
                points++;
                if (points % 1000000 == 0) {
                    final long now = System.nanoTime();
                    ping_start_time = (now - ping_start_time) / 1000000;
                    LOG.info(String.format("... %d data points in %dms (%.1f points/s)", points, ping_start_time, (1000000 * 1000.0 / ping_start_time)));
                    ping_start_time = now;
                }
                if (throttle) {
                    LOG.info("Throttling...");
                    long throttle_time = System.nanoTime();
                    try {
                        d.joinUninterruptibly();
                    } catch (final Exception e) {
                        throw new RuntimeException("Should never happen", e);
                    }
                    throttle_time = System.nanoTime() - throttle_time;
                    if (throttle_time < 1000000000L) {
                        LOG.info("Got throttled for only " + throttle_time + "ns, sleeping a bit now");
                        try {
                            Thread.sleep(1000);
                        } catch (InterruptedException e) {
                            throw new RuntimeException("interrupted", e);
                        }
                    }
                    LOG.info("Done throttling...");
                    throttle = false;
                }
            } catch (final RuntimeException e) {
                if (skip_errors) {
                    LOG.error("Exception: " + e.getMessage());
                    LOG.error("error while processing file " + path + " line=" + line + "... Continuing");
                    continue;
                } else {
                    throw e;
                }
            }
        }
    } catch (RuntimeException e) {
        LOG.error("Exception caught while processing file " + path + " line=[" + line + "]", e);
        throw e;
    } finally {
        in.close();
    }
    final long time_delta = (System.nanoTime() - start_time) / 1000000;
    LOG.info(String.format("Processed %s in %d ms, %d data points" + " (%.1f points/s)", path, time_delta, points, (points * 1000.0 / time_delta)));
    return points;
}