Examples with AppendRequest org.hbase.async.AppendRequest used on opensource projects

Example 1 with AppendRequest

use of org.hbase.async.AppendRequest in project opentsdb by OpenTSDB.

the class IncomingDataPoints method addPointInternal.

/**
 * Implements {@link #addPoint} by storing a value with a specific flag.
 *
 * @param timestamp
 *          The timestamp to associate with the value.
 * @param value
 *          The value to store.
 * @param flags
 *          Flags to store in the qualifier (size and type of the data point).
 * @return A deferred object that indicates the completion of the request.
 */
private Deferred<Object> addPointInternal(final long timestamp, final byte[] value, final short flags) {
    if (row == null) {
        throw new IllegalStateException("setSeries() never called!");
    }
    final boolean ms_timestamp = (timestamp & Const.SECOND_MASK) != 0;
    // we only accept unix epoch timestamps in seconds or milliseconds
    if (timestamp < 0 || (ms_timestamp && timestamp > 9999999999999L)) {
        throw new IllegalArgumentException((timestamp < 0 ? "negative " : "bad") + " timestamp=" + timestamp + " when trying to add value=" + Arrays.toString(value) + " to " + this);
    }
    // always maintain last_ts in milliseconds
    if ((ms_timestamp ? timestamp : timestamp * 1000) <= last_ts) {
        if (allow_out_of_order_data) {
            // TSDB add function.
            return tsdb.addPointInternal(metric, timestamp, value, tags, flags);
        } else {
            throw new IllegalArgumentException("New timestamp=" + timestamp + " is less than or equal to previous=" + last_ts + " when trying to add value=" + Arrays.toString(value) + " to " + this);
        }
    }
    /**
     * Callback executed for chaining filter calls to see if the value
     * should be written or not.
     */
    final class WriteCB implements Callback<Deferred<Object>, Boolean> {

        @Override
        public Deferred<Object> call(final Boolean allowed) throws Exception {
            if (!allowed) {
                return Deferred.fromResult(null);
            }
            last_ts = (ms_timestamp ? timestamp : timestamp * 1000);
            long base_time = baseTime();
            long incoming_base_time;
            if (ms_timestamp) {
                // drop the ms timestamp to seconds to calculate the base timestamp
                incoming_base_time = ((timestamp / 1000) - ((timestamp / 1000) % Const.MAX_TIMESPAN));
            } else {
                incoming_base_time = (timestamp - (timestamp % Const.MAX_TIMESPAN));
            }
            if (incoming_base_time - base_time >= Const.MAX_TIMESPAN) {
                // Need to start a new row as we've exceeded Const.MAX_TIMESPAN.
                base_time = updateBaseTime((ms_timestamp ? timestamp / 1000 : timestamp));
            }
            // Java is so stupid with its auto-promotion of int to float.
            final byte[] qualifier = Internal.buildQualifier(timestamp, flags);
            // TODO(tsuna): Add an errback to handle some error cases here.
            if (tsdb.getConfig().enable_appends()) {
                final AppendDataPoints kv = new AppendDataPoints(qualifier, value);
                final AppendRequest point = new AppendRequest(tsdb.table, row, TSDB.FAMILY, AppendDataPoints.APPEND_COLUMN_QUALIFIER, kv.getBytes());
                point.setDurable(!batch_import);
                return tsdb.client.append(point);
            /* .addBoth(cb) */
            } else {
                final PutRequest point = RequestBuilder.buildPutRequest(tsdb.getConfig(), tsdb.table, row, TSDB.FAMILY, qualifier, value, timestamp);
                point.setDurable(!batch_import);
                return tsdb.client.put(point);
            }
        }

        @Override
        public String toString() {
            return "IncomingDataPoints.addPointInternal Write Callback";
        }
    }
    if (tsdb.getTSfilter() != null && tsdb.getTSfilter().filterDataPoints()) {
        return tsdb.getTSfilter().allowDataPoint(metric, timestamp, value, tags, flags).addCallbackDeferring(new WriteCB());
    }
    return Deferred.fromResult(true).addCallbackDeferring(new WriteCB());
}

Example 2 with AppendRequest

use of org.hbase.async.AppendRequest in project opentsdb by OpenTSDB.

the class TSDB method addPointInternal.

private Deferred<Object> addPointInternal(final String metric, final long timestamp, final byte[] value, final Map<String, String> tags, final short flags) {
    // we only accept positive unix epoch timestamps in seconds or milliseconds
    if (timestamp < 0 || ((timestamp & Const.SECOND_MASK) != 0 && timestamp > 9999999999999L)) {
        throw new IllegalArgumentException((timestamp < 0 ? "negative " : "bad") + " timestamp=" + timestamp + " when trying to add value=" + Arrays.toString(value) + '/' + flags + " to metric=" + metric + ", tags=" + tags);
    }
    IncomingDataPoints.checkMetricAndTags(metric, tags);
    final byte[] row = IncomingDataPoints.rowKeyTemplate(this, metric, tags);
    final long base_time;
    final byte[] qualifier = Internal.buildQualifier(timestamp, flags);
    if ((timestamp & Const.SECOND_MASK) != 0) {
        // drop the ms timestamp to seconds to calculate the base timestamp
        base_time = ((timestamp / 1000) - ((timestamp / 1000) % Const.MAX_TIMESPAN));
    } else {
        base_time = (timestamp - (timestamp % Const.MAX_TIMESPAN));
    }
    /** Callback executed for chaining filter calls to see if the value
     * should be written or not. */
    final class WriteCB implements Callback<Deferred<Object>, Boolean> {

        @Override
        public Deferred<Object> call(final Boolean allowed) throws Exception {
            if (!allowed) {
                rejected_dps.incrementAndGet();
                return Deferred.fromResult(null);
            }
            Bytes.setInt(row, (int) base_time, metrics.width() + Const.SALT_WIDTH());
            RowKey.prefixKeyWithSalt(row);
            Deferred<Object> result = null;
            if (config.enable_appends()) {
                final AppendDataPoints kv = new AppendDataPoints(qualifier, value);
                final AppendRequest point = new AppendRequest(table, row, FAMILY, AppendDataPoints.APPEND_COLUMN_QUALIFIER, kv.getBytes());
                result = client.append(point);
            } else {
                scheduleForCompaction(row, (int) base_time);
                final PutRequest point = new PutRequest(table, row, FAMILY, qualifier, value);
                result = client.put(point);
            }
            // Count all added datapoints, not just those that came in through PUT rpc
            // Will there be others? Well, something could call addPoint programatically right?
            datapoints_added.incrementAndGet();
            if (!config.enable_realtime_ts() && !config.enable_tsuid_incrementing() && !config.enable_tsuid_tracking() && rt_publisher == null) {
                return result;
            }
            final byte[] tsuid = UniqueId.getTSUIDFromKey(row, METRICS_WIDTH, Const.TIMESTAMP_BYTES);
            // if the meta cache plugin is instantiated then tracking goes through it
            if (meta_cache != null) {
                meta_cache.increment(tsuid);
            } else {
                if (config.enable_tsuid_tracking()) {
                    if (config.enable_realtime_ts()) {
                        if (config.enable_tsuid_incrementing()) {
                            TSMeta.incrementAndGetCounter(TSDB.this, tsuid);
                        } else {
                            TSMeta.storeIfNecessary(TSDB.this, tsuid);
                        }
                    } else {
                        final PutRequest tracking = new PutRequest(meta_table, tsuid, TSMeta.FAMILY(), TSMeta.COUNTER_QUALIFIER(), Bytes.fromLong(1));
                        client.put(tracking);
                    }
                }
            }
            if (rt_publisher != null) {
                rt_publisher.sinkDataPoint(metric, timestamp, value, tags, tsuid, flags);
            }
            return result;
        }

        @Override
        public String toString() {
            return "addPointInternal Write Callback";
        }
    }
    if (ts_filter != null && ts_filter.filterDataPoints()) {
        return ts_filter.allowDataPoint(metric, timestamp, value, tags, flags).addCallbackDeferring(new WriteCB());
    }
    return Deferred.fromResult(true).addCallbackDeferring(new WriteCB());
}

Example 3 with AppendRequest

use of org.hbase.async.AppendRequest in project opentsdb by OpenTSDB.

the class TSDB method storeIntoDB.

private final Deferred<Object> storeIntoDB(final String metric, final long timestamp, final byte[] value, final Map<String, String> tags, final short flags, final byte[] row, final byte[] qualifier) {
    final long base_time;
    if ((timestamp & Const.SECOND_MASK) != 0) {
        // drop the ms timestamp to seconds to calculate the base timestamp
        base_time = ((timestamp / 1000) - ((timestamp / 1000) % Const.MAX_TIMESPAN));
    } else {
        base_time = (timestamp - (timestamp % Const.MAX_TIMESPAN));
    }
    /**
     * Callback executed for chaining filter calls to see if the value
     * should be written or not.
     */
    final class WriteCB implements Callback<Deferred<Object>, Boolean> {

        @Override
        public Deferred<Object> call(final Boolean allowed) throws Exception {
            if (!allowed) {
                rejected_dps.incrementAndGet();
                return Deferred.fromResult(null);
            }
            Bytes.setInt(row, (int) base_time, metrics.width() + Const.SALT_WIDTH());
            RowKey.prefixKeyWithSalt(row);
            Deferred<Object> result = null;
            if (!isHistogram(qualifier) && config.enable_appends()) {
                if (config.use_otsdb_timestamp()) {
                    LOG.error("Cannot use Date Tiered Compaction with AppendPoints. Please turn off either of them.");
                }
                final AppendDataPoints kv = new AppendDataPoints(qualifier, value);
                final AppendRequest point = new AppendRequest(table, row, FAMILY, AppendDataPoints.APPEND_COLUMN_QUALIFIER, kv.getBytes());
                result = client.append(point);
            } else if (!isHistogram(qualifier)) {
                scheduleForCompaction(row, (int) base_time);
                final PutRequest point = RequestBuilder.buildPutRequest(config, table, row, FAMILY, qualifier, value, timestamp);
                result = client.put(point);
            } else {
                scheduleForCompaction(row, (int) base_time);
                final PutRequest histo_point = new PutRequest(table, row, FAMILY, qualifier, value);
                result = client.put(histo_point);
            }
            // Count all added datapoints, not just those that came in through PUT rpc
            // Will there be others? Well, something could call addPoint programatically right?
            datapoints_added.incrementAndGet();
            if (!config.enable_realtime_ts() && !config.enable_tsuid_incrementing() && !config.enable_tsuid_tracking() && rt_publisher == null) {
                return result;
            }
            final byte[] tsuid = UniqueId.getTSUIDFromKey(row, METRICS_WIDTH, Const.TIMESTAMP_BYTES);
            // if the meta cache plugin is instantiated then tracking goes through it
            if (meta_cache != null) {
                meta_cache.increment(tsuid);
            } else {
                if (config.enable_tsuid_tracking()) {
                    if (config.enable_realtime_ts()) {
                        if (config.enable_tsuid_incrementing()) {
                            TSMeta.incrementAndGetCounter(TSDB.this, tsuid);
                        } else {
                            TSMeta.storeIfNecessary(TSDB.this, tsuid);
                        }
                    } else {
                        final PutRequest tracking = new PutRequest(meta_table, tsuid, TSMeta.FAMILY(), TSMeta.COUNTER_QUALIFIER(), Bytes.fromLong(1));
                        client.put(tracking);
                    }
                }
            }
            if (rt_publisher != null) {
                if (isHistogram(qualifier)) {
                    rt_publisher.publishHistogramPoint(metric, timestamp, value, tags, tsuid);
                } else {
                    rt_publisher.sinkDataPoint(metric, timestamp, value, tags, tsuid, flags);
                }
            }
            return result;
        }

        @Override
        public String toString() {
            return "addPointInternal Write Callback";
        }
    }
    if (ts_filter != null && ts_filter.filterDataPoints()) {
        if (isHistogram(qualifier)) {
            return ts_filter.allowHistogramPoint(metric, timestamp, value, tags).addCallbackDeferring(new WriteCB());
        } else {
            return ts_filter.allowDataPoint(metric, timestamp, value, tags, flags).addCallbackDeferring(new WriteCB());
        }
    }
    return Deferred.fromResult(true).addCallbackDeferring(new WriteCB());
}