Examples with Callback com.stumbleupon.async.Callback used on opensource projects

Example 1 with Callback

use of com.stumbleupon.async.Callback in project opentsdb by OpenTSDB.

the class TsdbQuery method configureFromQuery.

@Override
public Deferred<Object> configureFromQuery(final TSQuery query, final int index) {
    if (query.getQueries() == null || query.getQueries().isEmpty()) {
        throw new IllegalArgumentException("Missing sub queries");
    }
    if (index < 0 || index > query.getQueries().size()) {
        throw new IllegalArgumentException("Query index was out of range");
    }
    final TSSubQuery sub_query = query.getQueries().get(index);
    setStartTime(query.startTime());
    setEndTime(query.endTime());
    setDelete(query.getDelete());
    query_index = index;
    query_stats = query.getQueryStats();
    // set common options
    aggregator = sub_query.aggregator();
    rate = sub_query.getRate();
    rate_options = sub_query.getRateOptions();
    if (rate_options == null) {
        rate_options = new RateOptions();
    }
    downsampler = sub_query.downsamplingSpecification();
    filters = sub_query.getFilters();
    explicit_tags = sub_query.getExplicitTags();
    // if we have tsuids set, that takes precedence
    if (sub_query.getTsuids() != null && !sub_query.getTsuids().isEmpty()) {
        tsuids = new ArrayList<String>(sub_query.getTsuids());
        String first_metric = "";
        for (final String tsuid : tsuids) {
            if (first_metric.isEmpty()) {
                first_metric = tsuid.substring(0, TSDB.metrics_width() * 2).toUpperCase();
                continue;
            }
            final String metric = tsuid.substring(0, TSDB.metrics_width() * 2).toUpperCase();
            if (!first_metric.equals(metric)) {
                throw new IllegalArgumentException("One or more TSUIDs did not share the same metric [" + first_metric + "] [" + metric + "]");
            }
        }
        return Deferred.fromResult(null);
    } else {
        /** Triggers the group by resolution if we had filters to resolve */
        class FilterCB implements Callback<Object, ArrayList<byte[]>> {

            @Override
            public Object call(final ArrayList<byte[]> results) throws Exception {
                findGroupBys();
                return null;
            }
        }
        /** Resolve and group by tags after resolving the metric */
        class MetricCB implements Callback<Deferred<Object>, byte[]> {

            @Override
            public Deferred<Object> call(final byte[] uid) throws Exception {
                metric = uid;
                if (filters != null) {
                    final List<Deferred<byte[]>> deferreds = new ArrayList<Deferred<byte[]>>(filters.size());
                    for (final TagVFilter filter : filters) {
                        deferreds.add(filter.resolveTagkName(tsdb));
                    }
                    return Deferred.group(deferreds).addCallback(new FilterCB());
                } else {
                    return Deferred.fromResult(null);
                }
            }
        }
        // fire off the callback chain by resolving the metric first
        return tsdb.metrics.getIdAsync(sub_query.getMetric()).addCallbackDeferring(new MetricCB());
    }
}

Example 2 with Callback

use of com.stumbleupon.async.Callback in project opentsdb by OpenTSDB.

the class IncomingDataPoints method addPointInternal.

/**
 * Implements {@link #addPoint} by storing a value with a specific flag.
 *
 * @param timestamp
 *          The timestamp to associate with the value.
 * @param value
 *          The value to store.
 * @param flags
 *          Flags to store in the qualifier (size and type of the data point).
 * @return A deferred object that indicates the completion of the request.
 */
private Deferred<Object> addPointInternal(final long timestamp, final byte[] value, final short flags) {
    if (row == null) {
        throw new IllegalStateException("setSeries() never called!");
    }
    final boolean ms_timestamp = (timestamp & Const.SECOND_MASK) != 0;
    // we only accept unix epoch timestamps in seconds or milliseconds
    if (timestamp < 0 || (ms_timestamp && timestamp > 9999999999999L)) {
        throw new IllegalArgumentException((timestamp < 0 ? "negative " : "bad") + " timestamp=" + timestamp + " when trying to add value=" + Arrays.toString(value) + " to " + this);
    }
    // always maintain last_ts in milliseconds
    if ((ms_timestamp ? timestamp : timestamp * 1000) <= last_ts) {
        if (allow_out_of_order_data) {
            // TSDB add function.
            return tsdb.addPointInternal(metric, timestamp, value, tags, flags);
        } else {
            throw new IllegalArgumentException("New timestamp=" + timestamp + " is less than or equal to previous=" + last_ts + " when trying to add value=" + Arrays.toString(value) + " to " + this);
        }
    }
    /**
     * Callback executed for chaining filter calls to see if the value
     * should be written or not.
     */
    final class WriteCB implements Callback<Deferred<Object>, Boolean> {

        @Override
        public Deferred<Object> call(final Boolean allowed) throws Exception {
            if (!allowed) {
                return Deferred.fromResult(null);
            }
            last_ts = (ms_timestamp ? timestamp : timestamp * 1000);
            long base_time = baseTime();
            long incoming_base_time;
            if (ms_timestamp) {
                // drop the ms timestamp to seconds to calculate the base timestamp
                incoming_base_time = ((timestamp / 1000) - ((timestamp / 1000) % Const.MAX_TIMESPAN));
            } else {
                incoming_base_time = (timestamp - (timestamp % Const.MAX_TIMESPAN));
            }
            if (incoming_base_time - base_time >= Const.MAX_TIMESPAN) {
                // Need to start a new row as we've exceeded Const.MAX_TIMESPAN.
                base_time = updateBaseTime((ms_timestamp ? timestamp / 1000 : timestamp));
            }
            // Java is so stupid with its auto-promotion of int to float.
            final byte[] qualifier = Internal.buildQualifier(timestamp, flags);
            // TODO(tsuna): Add an errback to handle some error cases here.
            if (tsdb.getConfig().enable_appends()) {
                final AppendDataPoints kv = new AppendDataPoints(qualifier, value);
                final AppendRequest point = new AppendRequest(tsdb.table, row, TSDB.FAMILY, AppendDataPoints.APPEND_COLUMN_QUALIFIER, kv.getBytes());
                point.setDurable(!batch_import);
                return tsdb.client.append(point);
            /* .addBoth(cb) */
            } else {
                final PutRequest point = RequestBuilder.buildPutRequest(tsdb.getConfig(), tsdb.table, row, TSDB.FAMILY, qualifier, value, timestamp);
                point.setDurable(!batch_import);
                return tsdb.client.put(point);
            }
        }

        @Override
        public String toString() {
            return "IncomingDataPoints.addPointInternal Write Callback";
        }
    }
    if (tsdb.getTSfilter() != null && tsdb.getTSfilter().filterDataPoints()) {
        return tsdb.getTSfilter().allowDataPoint(metric, timestamp, value, tags, flags).addCallbackDeferring(new WriteCB());
    }
    return Deferred.fromResult(true).addCallbackDeferring(new WriteCB());
}

Example 3 with Callback

use of com.stumbleupon.async.Callback in project opentsdb by OpenTSDB.

the class TsdbQuery method configureFromQuery.

public Deferred<Object> configureFromQuery(final TSQuery query, final int index, boolean force_raw) {
    if (query.getQueries() == null || query.getQueries().isEmpty()) {
        throw new IllegalArgumentException("Missing sub queries");
    }
    if (index < 0 || index > query.getQueries().size()) {
        throw new IllegalArgumentException("Query index was out of range");
    }
    final TSSubQuery sub_query = query.getQueries().get(index);
    setStartTime(query.startTime());
    setEndTime(query.endTime());
    setDelete(query.getDelete());
    query_index = index;
    query_stats = query.getQueryStats();
    // set common options
    aggregator = sub_query.aggregator();
    rate = sub_query.getRate();
    rate_options = sub_query.getRateOptions();
    if (rate_options == null) {
        rate_options = new RateOptions();
    }
    downsampler = sub_query.downsamplingSpecification();
    pre_aggregate = sub_query.isPreAggregate();
    rollup_usage = sub_query.getRollupUsage();
    filters = sub_query.getFilters();
    explicit_tags = sub_query.getExplicitTags();
    override_fuzzy_filter = sub_query.getUseFuzzyFilter();
    override_multi_get = sub_query.getUseMultiGets();
    max_bytes = tsdb.getQueryByteLimits().getByteLimit(sub_query.getMetric());
    if (tsdb.getConfig().getBoolean("tsd.query.limits.bytes.allow_override") && query.overrideByteLimit()) {
        max_bytes = 0;
    }
    max_data_points = tsdb.getQueryByteLimits().getDataPointLimit(sub_query.getMetric());
    if (tsdb.getConfig().getBoolean("tsd.query.limits.data_points.allow_override") && query.overrideDataPointLimit()) {
        max_data_points = 0;
    }
    // set percentile options
    percentiles = sub_query.getPercentiles();
    show_histogram_buckets = sub_query.getShowHistogramBuckets();
    if (!force_raw && rollup_usage != ROLLUP_USAGE.ROLLUP_RAW) {
        // Check whether the down sampler is set and rollup is enabled
        transformDownSamplerToRollupQuery(aggregator, sub_query.getDownsample());
    }
    sub_query.setTsdbQuery(this);
    if (use_multi_gets && override_multi_get && multiget_with_search) {
        row_key_literals_list = Lists.newArrayList();
    }
    // if we have tsuids set, that takes precedence
    if (sub_query.getTsuids() != null && !sub_query.getTsuids().isEmpty()) {
        tsuids = new ArrayList<String>(sub_query.getTsuids());
        String first_metric = "";
        for (final String tsuid : tsuids) {
            if (first_metric.isEmpty()) {
                first_metric = tsuid.substring(0, TSDB.metrics_width() * 2).toUpperCase();
                continue;
            }
            final String metric = tsuid.substring(0, TSDB.metrics_width() * 2).toUpperCase();
            if (!first_metric.equals(metric)) {
                throw new IllegalArgumentException("One or more TSUIDs did not share the same metric [" + first_metric + "] [" + metric + "]");
            }
        }
        return Deferred.fromResult(null);
    } else {
        /**
         * Triggers the group by resolution if we had filters to resolve
         */
        class FilterCB implements Callback<Object, ArrayList<byte[]>> {

            @Override
            public Object call(final ArrayList<byte[]> results) throws Exception {
                findGroupBys();
                return null;
            }
        }
        /**
         * Resolve and group by tags after resolving the metric
         */
        class MetricCB implements Callback<Deferred<Object>, byte[]> {

            @Override
            public Deferred<Object> call(final byte[] uid) throws Exception {
                metric = uid;
                if (filters != null) {
                    if (use_multi_gets && override_multi_get && multiget_with_search) {
                        class ErrorCB implements Callback<Deferred<Object>, Exception> {

                            @Override
                            public Deferred<Object> call(Exception arg) throws Exception {
                                LOG.info("Doing scans because meta query is failed", arg);
                                if (explicit_tags) {
                                    search_query_failure = true;
                                } else {
                                    override_multi_get = false;
                                    use_multi_gets = false;
                                }
                                return Deferred.group(resolveTagFilters()).addCallback(new FilterCB());
                            }
                        }
                        class SuccessCB implements Callback<Deferred<Object>, List<ByteMap<byte[][]>>> {

                            @Override
                            public Deferred<Object> call(final List<ByteMap<byte[][]>> results) throws Exception {
                                row_key_literals_list.addAll(results);
                                return Deferred.fromResult(null);
                            }
                        }
                        tsdb.getSearchPlugin().resolveTSQuery(query, index).addCallbackDeferring(new SuccessCB()).addErrback(new ErrorCB());
                    }
                    return Deferred.group(resolveTagFilters()).addCallback(new FilterCB());
                } else {
                    return Deferred.fromResult(null);
                }
            }

            private List<Deferred<byte[]>> resolveTagFilters() {
                final List<Deferred<byte[]>> deferreds = new ArrayList<Deferred<byte[]>>(filters.size());
                for (final TagVFilter filter : filters) {
                    // determine if the user is asking for pre-agg data
                    if (filter instanceof TagVLiteralOrFilter && tsdb.getAggTagKey() != null) {
                        if (filter.getTagk().equals(tsdb.getAggTagKey())) {
                            if (tsdb.getRawTagValue() != null && !filter.getFilter().equals(tsdb.getRawTagValue())) {
                                pre_aggregate = true;
                            }
                        }
                    }
                    deferreds.add(filter.resolveTagkName(tsdb));
                }
                return deferreds;
            }
        }
        // fire off the callback chain by resolving the metric first
        return tsdb.metrics.getIdAsync(sub_query.getMetric()).addCallbackDeferring(new MetricCB());
    }
}

Example 4 with Callback

use of com.stumbleupon.async.Callback in project opentsdb by OpenTSDB.

the class Annotation method syncToStorage.

/**
 * Attempts a CompareAndSet storage call, loading the object from storage,
 * synchronizing changes, and attempting a put.
 * <b>Note:</b> If the local object didn't have any fields set by the caller
 * or there weren't any changes, then the data will not be written and an
 * exception will be thrown.
 * @param tsdb The TSDB to use for storage access
 * @param overwrite When the RPC method is PUT, will overwrite all user
 * accessible fields
 * True if the storage call was successful, false if the object was
 * modified in storage during the CAS call. If false, retry the call. Other
 * failures will result in an exception being thrown.
 * @throws HBaseException if there was an issue
 * @throws IllegalArgumentException if required data was missing such as the
 * {@code #start_time}
 * @throws IllegalStateException if the data hasn't changed. This is OK!
 * @throws JSONException if the object could not be serialized
 */
public Deferred<Boolean> syncToStorage(final TSDB tsdb, final Boolean overwrite) {
    if (start_time < 1) {
        throw new IllegalArgumentException("The start timestamp has not been set");
    }
    boolean has_changes = false;
    for (Map.Entry<String, Boolean> entry : changed.entrySet()) {
        if (entry.getValue()) {
            has_changes = true;
            break;
        }
    }
    if (!has_changes) {
        LOG.debug(this + " does not have changes, skipping sync to storage");
        throw new IllegalStateException("No changes detected in Annotation data");
    }
    final class StoreCB implements Callback<Deferred<Boolean>, Annotation> {

        @Override
        public Deferred<Boolean> call(final Annotation stored_note) throws Exception {
            final byte[] original_note = stored_note == null ? new byte[0] : stored_note.getStorageJSON();
            if (stored_note != null) {
                Annotation.this.syncNote(stored_note, overwrite);
            }
            final byte[] tsuid_byte = tsuid != null && !tsuid.isEmpty() ? UniqueId.stringToUid(tsuid) : null;
            final PutRequest put = RequestBuilder.buildPutRequest(tsdb.getConfig(), tsdb.dataTable(), getRowKey(start_time, tsuid_byte), FAMILY, getQualifier(start_time), Annotation.this.getStorageJSON(), start_time);
            return tsdb.getClient().compareAndSet(put, original_note);
        }
    }
    if (tsuid != null && !tsuid.isEmpty()) {
        return getAnnotation(tsdb, UniqueId.stringToUid(tsuid), start_time).addCallbackDeferring(new StoreCB());
    }
    return getAnnotation(tsdb, start_time).addCallbackDeferring(new StoreCB());
}

Example 5 with Callback

use of com.stumbleupon.async.Callback in project opentsdb by OpenTSDB.

the class Annotation method deleteRange.

/**
 * Deletes global or TSUID associated annotiations for the given time range.
 * @param tsdb The TSDB object to use for storage access
 * @param tsuid An optional TSUID. If set to null, then global annotations for
 * the given range will be deleted
 * @param start_time A start timestamp in milliseconds
 * @param end_time An end timestamp in millseconds
 * @return The number of annotations deleted
 * @throws IllegalArgumentException if the timestamps are invalid
 * @since 2.1
 */
public static Deferred<Integer> deleteRange(final TSDB tsdb, final byte[] tsuid, final long start_time, final long end_time) {
    if (end_time < 1) {
        throw new IllegalArgumentException("The end timestamp has not been set");
    }
    if (end_time < start_time) {
        throw new IllegalArgumentException("The end timestamp cannot be less than the start timestamp");
    }
    final List<Deferred<Object>> delete_requests = new ArrayList<Deferred<Object>>();
    int width = tsuid != null ? Const.SALT_WIDTH() + tsuid.length + Const.TIMESTAMP_BYTES : Const.SALT_WIDTH() + TSDB.metrics_width() + Const.TIMESTAMP_BYTES;
    final byte[] start_row = new byte[width];
    final byte[] end_row = new byte[width];
    // downsample to seconds for the row keys
    final long start = start_time / 1000;
    final long end = end_time / 1000;
    final long normalized_start = (start - (start % Const.MAX_TIMESPAN));
    final long normalized_end = (end - (end % Const.MAX_TIMESPAN) + Const.MAX_TIMESPAN);
    Bytes.setInt(start_row, (int) normalized_start, Const.SALT_WIDTH() + TSDB.metrics_width());
    Bytes.setInt(end_row, (int) normalized_end, Const.SALT_WIDTH() + TSDB.metrics_width());
    if (tsuid != null) {
        // first copy the metric UID then the tags
        System.arraycopy(tsuid, 0, start_row, Const.SALT_WIDTH(), TSDB.metrics_width());
        System.arraycopy(tsuid, 0, end_row, Const.SALT_WIDTH(), TSDB.metrics_width());
        width = Const.SALT_WIDTH() + TSDB.metrics_width() + Const.TIMESTAMP_BYTES;
        final int remainder = tsuid.length - TSDB.metrics_width();
        System.arraycopy(tsuid, TSDB.metrics_width(), start_row, width, remainder);
        System.arraycopy(tsuid, TSDB.metrics_width(), end_row, width, remainder);
    }
    /**
     * Iterates through the scanner results in an asynchronous manner, returning
     * once the scanner returns a null result set.
     */
    final class ScannerCB implements Callback<Deferred<List<Deferred<Object>>>, ArrayList<ArrayList<KeyValue>>> {

        final Scanner scanner;

        public ScannerCB() {
            scanner = tsdb.getClient().newScanner(tsdb.dataTable());
            scanner.setStartKey(start_row);
            scanner.setStopKey(end_row);
            scanner.setFamily(FAMILY);
            if (tsuid != null) {
                final List<String> tsuids = new ArrayList<String>(1);
                tsuids.add(UniqueId.uidToString(tsuid));
                Internal.createAndSetTSUIDFilter(scanner, tsuids);
            }
        }

        public Deferred<List<Deferred<Object>>> scan() {
            return scanner.nextRows().addCallbackDeferring(this);
        }

        @Override
        public Deferred<List<Deferred<Object>>> call(final ArrayList<ArrayList<KeyValue>> rows) throws Exception {
            if (rows == null || rows.isEmpty()) {
                return Deferred.fromResult(delete_requests);
            }
            for (final ArrayList<KeyValue> row : rows) {
                final long base_time = Internal.baseTime(tsdb, row.get(0).key());
                for (KeyValue column : row) {
                    if ((column.qualifier().length == 3 || column.qualifier().length == 5) && column.qualifier()[0] == PREFIX()) {
                        final long timestamp = timeFromQualifier(column.qualifier(), base_time);
                        if (timestamp < start_time || timestamp > end_time) {
                            continue;
                        }
                        final DeleteRequest delete = new DeleteRequest(tsdb.dataTable(), column.key(), FAMILY, column.qualifier());
                        delete_requests.add(tsdb.getClient().delete(delete));
                    }
                }
            }
            return scan();
        }
    }
    /**
     * Called when the scanner is done. Delete requests may still be pending
     */
    final class ScannerDoneCB implements Callback<Deferred<ArrayList<Object>>, List<Deferred<Object>>> {

        @Override
        public Deferred<ArrayList<Object>> call(final List<Deferred<Object>> deletes) throws Exception {
            return Deferred.group(delete_requests);
        }
    }
    /**
     * Waits on the group of deferreds to complete before returning the count
     */
    final class GroupCB implements Callback<Deferred<Integer>, ArrayList<Object>> {

        @Override
        public Deferred<Integer> call(final ArrayList<Object> deletes) throws Exception {
            return Deferred.fromResult(deletes.size());
        }
    }
    Deferred<ArrayList<Object>> scanner_done = new ScannerCB().scan().addCallbackDeferring(new ScannerDoneCB());
    return scanner_done.addCallbackDeferring(new GroupCB());
}