Examples with Callback com.stumbleupon.async.Callback used on opensource projects

Example 36 with Callback

use of com.stumbleupon.async.Callback in project opentsdb by OpenTSDB.

the class TSMeta method getFromStorage.

/**
   * Attempts to fetch the timeseries meta data from storage. 
   * This method will fetch the {@code counter} and {@code meta} columns.
   * <b>Note:</b> This method will not load the UIDMeta objects.
   * @param tsdb The TSDB to use for storage access
   * @param tsuid The UID of the meta to fetch
   * @return A TSMeta object if found, null if not
   * @throws HBaseException if there was an issue fetching
   * @throws IllegalArgumentException if parsing failed
   * @throws JSONException if the data was corrupted
   */
private static Deferred<TSMeta> getFromStorage(final TSDB tsdb, final byte[] tsuid) {
    /**
     * Called after executing the GetRequest to parse the meta data.
     */
    final class GetCB implements Callback<Deferred<TSMeta>, ArrayList<KeyValue>> {

        /**
       * @return Null if the meta did not exist or a valid TSMeta object if it
       * did.
       */
        @Override
        public Deferred<TSMeta> call(final ArrayList<KeyValue> row) throws Exception {
            if (row == null || row.isEmpty()) {
                return Deferred.fromResult(null);
            }
            long dps = 0;
            long last_received = 0;
            TSMeta meta = null;
            for (KeyValue column : row) {
                if (Arrays.equals(COUNTER_QUALIFIER, column.qualifier())) {
                    dps = Bytes.getLong(column.value());
                    last_received = column.timestamp() / 1000;
                } else if (Arrays.equals(META_QUALIFIER, column.qualifier())) {
                    meta = JSON.parseToObject(column.value(), TSMeta.class);
                }
            }
            if (meta == null) {
                LOG.warn("Found a counter TSMeta column without a meta for TSUID: " + UniqueId.uidToString(row.get(0).key()));
                return Deferred.fromResult(null);
            }
            meta.total_dps = dps;
            meta.last_received = last_received;
            return Deferred.fromResult(meta);
        }
    }
    final GetRequest get = new GetRequest(tsdb.metaTable(), tsuid);
    get.family(FAMILY);
    get.qualifiers(new byte[][] { COUNTER_QUALIFIER, META_QUALIFIER });
    return tsdb.getClient().get(get).addCallbackDeferring(new GetCB());
}

Example 37 with Callback

use of com.stumbleupon.async.Callback in project opentsdb by OpenTSDB.

the class TSMeta method metaExistsInStorage.

/**
   * Determines if an entry exists in storage or not. 
   * This is used by the UID Manager tool to determine if we need to write a 
   * new TSUID entry or not. It will not attempt to verify if the stored data is 
   * valid, just checks to see if something is stored in the proper column.
   * @param tsdb  The TSDB to use for storage access
   * @param tsuid The UID of the meta to verify
   * @return True if data was found, false if not
   * @throws HBaseException if there was an issue fetching
   */
public static Deferred<Boolean> metaExistsInStorage(final TSDB tsdb, final String tsuid) {
    final GetRequest get = new GetRequest(tsdb.metaTable(), UniqueId.stringToUid(tsuid));
    get.family(FAMILY);
    get.qualifier(META_QUALIFIER);
    /**
     * Callback from the GetRequest that simply determines if the row is empty
     * or not
     */
    final class ExistsCB implements Callback<Boolean, ArrayList<KeyValue>> {

        @Override
        public Boolean call(ArrayList<KeyValue> row) throws Exception {
            if (row == null || row.isEmpty() || row.get(0).value() == null) {
                return false;
            }
            return true;
        }
    }
    return tsdb.getClient().get(get).addCallback(new ExistsCB());
}

Example 38 with Callback

use of com.stumbleupon.async.Callback in project opentsdb by OpenTSDB.

the class TSUIDQuery method getLastWriteTimes.

/**
   * Fetches a list of TSUIDs given the metric and optional tag pairs. The query
   * format is similar to TsdbQuery but doesn't support grouping operators for 
   * tags. Only TSUIDs that had "ts_counter" qualifiers will be returned.
   * <p>
   * NOTE: If you called {@link #setQuery(String, Map)} successfully this will
   * immediately scan the meta table. But if you used the CTOR to set the
   * metric and tags it will attempt to resolve those and may return an exception.
   * @return A map of TSUIDs to the last timestamp (in milliseconds) when the
   * "ts_counter" was updated. Note that the timestamp will be the time stored
   * by HBase, not the actual timestamp of the data point. If nothing was
   * found, the map will be empty but not null.
   * @throws IllegalArgumentException if the metric was not set or the tag map
   * was null
   */
public Deferred<ByteMap<Long>> getLastWriteTimes() {
    class ResolutionCB implements Callback<Deferred<ByteMap<Long>>, Object> {

        @Override
        public Deferred<ByteMap<Long>> call(Object arg0) throws Exception {
            final Scanner scanner = getScanner();
            scanner.setQualifier(TSMeta.COUNTER_QUALIFIER());
            final Deferred<ByteMap<Long>> results = new Deferred<ByteMap<Long>>();
            final ByteMap<Long> tsuids = new ByteMap<Long>();
            final class ErrBack implements Callback<Object, Exception> {

                @Override
                public Object call(final Exception e) throws Exception {
                    results.callback(e);
                    return null;
                }

                @Override
                public String toString() {
                    return "Error callback";
                }
            }
            /**
         * Scanner callback that will call itself while iterating through the 
         * tsdb-meta table
         */
            final class ScannerCB implements Callback<Object, ArrayList<ArrayList<KeyValue>>> {

                /**
           * Starts the scanner and is called recursively to fetch the next set of
           * rows from the scanner.
           * @return The map of spans if loaded successfully, null if no data was
           * found
           */
                public Object scan() {
                    return scanner.nextRows().addCallback(this).addErrback(new ErrBack());
                }

                /**
           * Loops through each row of the scanner results and parses out data
           * points and optional meta data
           * @return null if no rows were found, otherwise the TreeMap with spans
           */
                @Override
                public Object call(final ArrayList<ArrayList<KeyValue>> rows) throws Exception {
                    try {
                        if (rows == null) {
                            results.callback(tsuids);
                            return null;
                        }
                        for (final ArrayList<KeyValue> row : rows) {
                            final byte[] tsuid = row.get(0).key();
                            tsuids.put(tsuid, row.get(0).timestamp());
                        }
                        return scan();
                    } catch (Exception e) {
                        results.callback(e);
                        return null;
                    }
                }
            }
            new ScannerCB().scan();
            return results;
        }

        @Override
        public String toString() {
            return "Last counter time callback";
        }
    }
    if (metric_uid == null) {
        return resolveMetric().addCallbackDeferring(new ResolutionCB());
    }
    try {
        return new ResolutionCB().call(null);
    } catch (Exception e) {
        return Deferred.fromError(e);
    }
}

Example 39 with Callback

use of com.stumbleupon.async.Callback in project opentsdb by OpenTSDB.

the class TSUIDQuery method getLastPoint.

/**
   * Attempts to fetch the last data point for the given metric or TSUID.
   * If back_scan == 0 and meta is enabled via 
   * "tsd.core.meta.enable_tsuid_tracking" or 
   * "tsd.core.meta.enable_tsuid_incrementing" then we will look up the metric
   * or TSUID in the meta table first and use the counter there to get the
   * last write time.
   * <p>
   * However if backscan is set, then we'll start with the current time and 
   * iterate back "back_scan" number of hours until we find a value.
   * <p>
   * @param resolve_names Whether or not to resolve the UIDs back to their
   * names when we find a value.
   * @param back_scan The number of hours back in time to scan
   * @return A data point if found, null if not. Or an exception if something
   * went pear shaped.
   */
public Deferred<IncomingDataPoint> getLastPoint(final boolean resolve_names, final int back_scan) {
    if (back_scan < 0) {
        throw new IllegalArgumentException("Backscan must be zero or a positive number");
    }
    this.resolve_names = resolve_names;
    this.back_scan = back_scan;
    final boolean meta_enabled = tsdb.getConfig().enable_tsuid_tracking() || tsdb.getConfig().enable_tsuid_incrementing();
    class TSUIDCB implements Callback<Deferred<IncomingDataPoint>, byte[]> {

        @Override
        public Deferred<IncomingDataPoint> call(final byte[] incoming_tsuid) throws Exception {
            if (tsuid == null && incoming_tsuid == null) {
                return Deferred.fromError(new RuntimeException("Both incoming and " + "supplied TSUIDs were null for " + TSUIDQuery.this));
            } else if (incoming_tsuid != null) {
                setTSUID(incoming_tsuid);
            }
            if (back_scan < 1 && meta_enabled) {
                final GetRequest get = new GetRequest(tsdb.metaTable(), tsuid);
                get.family(TSMeta.FAMILY());
                get.qualifier(TSMeta.COUNTER_QUALIFIER());
                return tsdb.getClient().get(get).addCallbackDeferring(new MetaCB());
            }
            if (last_timestamp > 0) {
                last_timestamp = Internal.baseTime(last_timestamp);
            } else {
                last_timestamp = Internal.baseTime(DateTime.currentTimeMillis());
            }
            final byte[] key = RowKey.rowKeyFromTSUID(tsdb, tsuid, last_timestamp);
            final GetRequest get = new GetRequest(tsdb.dataTable(), key);
            get.family(TSDB.FAMILY());
            return tsdb.getClient().get(get).addCallbackDeferring(new LastPointCB());
        }

        @Override
        public String toString() {
            return "TSUID callback";
        }
    }
    if (tsuid == null) {
        return tsuidFromMetric(tsdb, metric, tags).addCallbackDeferring(new TSUIDCB());
    }
    try {
        // damn typed exceptions....
        return new TSUIDCB().call(null);
    } catch (Exception e) {
        return Deferred.fromError(e);
    }
}

Example 40 with Callback

use of com.stumbleupon.async.Callback in project opentsdb by OpenTSDB.

the class TSDB method addPointInternal.

private Deferred<Object> addPointInternal(final String metric, final long timestamp, final byte[] value, final Map<String, String> tags, final short flags) {
    // we only accept positive unix epoch timestamps in seconds or milliseconds
    if (timestamp < 0 || ((timestamp & Const.SECOND_MASK) != 0 && timestamp > 9999999999999L)) {
        throw new IllegalArgumentException((timestamp < 0 ? "negative " : "bad") + " timestamp=" + timestamp + " when trying to add value=" + Arrays.toString(value) + '/' + flags + " to metric=" + metric + ", tags=" + tags);
    }
    IncomingDataPoints.checkMetricAndTags(metric, tags);
    final byte[] row = IncomingDataPoints.rowKeyTemplate(this, metric, tags);
    final long base_time;
    final byte[] qualifier = Internal.buildQualifier(timestamp, flags);
    if ((timestamp & Const.SECOND_MASK) != 0) {
        // drop the ms timestamp to seconds to calculate the base timestamp
        base_time = ((timestamp / 1000) - ((timestamp / 1000) % Const.MAX_TIMESPAN));
    } else {
        base_time = (timestamp - (timestamp % Const.MAX_TIMESPAN));
    }
    /** Callback executed for chaining filter calls to see if the value
     * should be written or not. */
    final class WriteCB implements Callback<Deferred<Object>, Boolean> {

        @Override
        public Deferred<Object> call(final Boolean allowed) throws Exception {
            if (!allowed) {
                rejected_dps.incrementAndGet();
                return Deferred.fromResult(null);
            }
            Bytes.setInt(row, (int) base_time, metrics.width() + Const.SALT_WIDTH());
            RowKey.prefixKeyWithSalt(row);
            Deferred<Object> result = null;
            if (config.enable_appends()) {
                final AppendDataPoints kv = new AppendDataPoints(qualifier, value);
                final AppendRequest point = new AppendRequest(table, row, FAMILY, AppendDataPoints.APPEND_COLUMN_QUALIFIER, kv.getBytes());
                result = client.append(point);
            } else {
                scheduleForCompaction(row, (int) base_time);
                final PutRequest point = new PutRequest(table, row, FAMILY, qualifier, value);
                result = client.put(point);
            }
            // Count all added datapoints, not just those that came in through PUT rpc
            // Will there be others? Well, something could call addPoint programatically right?
            datapoints_added.incrementAndGet();
            if (!config.enable_realtime_ts() && !config.enable_tsuid_incrementing() && !config.enable_tsuid_tracking() && rt_publisher == null) {
                return result;
            }
            final byte[] tsuid = UniqueId.getTSUIDFromKey(row, METRICS_WIDTH, Const.TIMESTAMP_BYTES);
            // if the meta cache plugin is instantiated then tracking goes through it
            if (meta_cache != null) {
                meta_cache.increment(tsuid);
            } else {
                if (config.enable_tsuid_tracking()) {
                    if (config.enable_realtime_ts()) {
                        if (config.enable_tsuid_incrementing()) {
                            TSMeta.incrementAndGetCounter(TSDB.this, tsuid);
                        } else {
                            TSMeta.storeIfNecessary(TSDB.this, tsuid);
                        }
                    } else {
                        final PutRequest tracking = new PutRequest(meta_table, tsuid, TSMeta.FAMILY(), TSMeta.COUNTER_QUALIFIER(), Bytes.fromLong(1));
                        client.put(tracking);
                    }
                }
            }
            if (rt_publisher != null) {
                rt_publisher.sinkDataPoint(metric, timestamp, value, tags, tsuid, flags);
            }
            return result;
        }

        @Override
        public String toString() {
            return "addPointInternal Write Callback";
        }
    }
    if (ts_filter != null && ts_filter.filterDataPoints()) {
        return ts_filter.allowDataPoint(metric, timestamp, value, tags, flags).addCallbackDeferring(new WriteCB());
    }
    return Deferred.fromResult(true).addCallbackDeferring(new WriteCB());
}