Examples with PutRequest org.hbase.async.PutRequest used on opensource projects

Example 6 with PutRequest

use of org.hbase.async.PutRequest in project opentsdb by OpenTSDB.

the class TextImporter method importFile.

private static int importFile(final HBaseClient client, final TSDB tsdb, final String path, final boolean skip_errors) throws IOException {
    final long start_time = System.nanoTime();
    long ping_start_time = start_time;
    final BufferedReader in = open(path);
    String line = null;
    int points = 0;
    try {
        final class Errback implements Callback<Object, Exception> {

            public Object call(final Exception arg) {
                if (arg instanceof PleaseThrottleException) {
                    final PleaseThrottleException e = (PleaseThrottleException) arg;
                    LOG.warn("Need to throttle, HBase isn't keeping up.", e);
                    throttle = true;
                    final HBaseRpc rpc = e.getFailedRpc();
                    if (rpc instanceof PutRequest) {
                        // Don't lose edits.
                        client.put((PutRequest) rpc);
                    }
                    return null;
                }
                LOG.error("Exception caught while processing file " + path, arg);
                System.exit(2);
                return arg;
            }

            public String toString() {
                return "importFile errback";
            }
        }
        ;
        final Errback errback = new Errback();
        LOG.info("reading from file:" + path);
        while ((line = in.readLine()) != null) {
            final String[] words = Tags.splitString(line, ' ');
            final String metric = words[0];
            if (metric.length() <= 0) {
                if (skip_errors) {
                    LOG.error("invalid metric: " + metric);
                    LOG.error("error while processing file " + path + " line=" + line + "... Continuing");
                    continue;
                } else {
                    throw new RuntimeException("invalid metric: " + metric);
                }
            }
            final long timestamp;
            try {
                timestamp = Tags.parseLong(words[1]);
                if (timestamp <= 0) {
                    if (skip_errors) {
                        LOG.error("invalid timestamp: " + timestamp);
                        LOG.error("error while processing file " + path + " line=" + line + "... Continuing");
                        continue;
                    } else {
                        throw new RuntimeException("invalid timestamp: " + timestamp);
                    }
                }
            } catch (final RuntimeException e) {
                if (skip_errors) {
                    LOG.error("invalid timestamp: " + e.getMessage());
                    LOG.error("error while processing file " + path + " line=" + line + "... Continuing");
                    continue;
                } else {
                    throw e;
                }
            }
            final String value = words[2];
            if (value.length() <= 0) {
                if (skip_errors) {
                    LOG.error("invalid value: " + value);
                    LOG.error("error while processing file " + path + " line=" + line + "... Continuing");
                    continue;
                } else {
                    throw new RuntimeException("invalid value: " + value);
                }
            }
            try {
                final HashMap<String, String> tags = new HashMap<String, String>();
                for (int i = 3; i < words.length; i++) {
                    if (!words[i].isEmpty()) {
                        Tags.parse(tags, words[i]);
                    }
                }
                final WritableDataPoints dp = getDataPoints(tsdb, metric, tags);
                Deferred<Object> d;
                if (Tags.looksLikeInteger(value)) {
                    d = dp.addPoint(timestamp, Tags.parseLong(value));
                } else {
                    // floating point value
                    d = dp.addPoint(timestamp, Float.parseFloat(value));
                }
                d.addErrback(errback);
                points++;
                if (points % 1000000 == 0) {
                    final long now = System.nanoTime();
                    ping_start_time = (now - ping_start_time) / 1000000;
                    LOG.info(String.format("... %d data points in %dms (%.1f points/s)", points, ping_start_time, (1000000 * 1000.0 / ping_start_time)));
                    ping_start_time = now;
                }
                if (throttle) {
                    LOG.info("Throttling...");
                    long throttle_time = System.nanoTime();
                    try {
                        d.joinUninterruptibly();
                    } catch (final Exception e) {
                        throw new RuntimeException("Should never happen", e);
                    }
                    throttle_time = System.nanoTime() - throttle_time;
                    if (throttle_time < 1000000000L) {
                        LOG.info("Got throttled for only " + throttle_time + "ns, sleeping a bit now");
                        try {
                            Thread.sleep(1000);
                        } catch (InterruptedException e) {
                            throw new RuntimeException("interrupted", e);
                        }
                    }
                    LOG.info("Done throttling...");
                    throttle = false;
                }
            } catch (final RuntimeException e) {
                if (skip_errors) {
                    LOG.error("Exception: " + e.getMessage());
                    LOG.error("error while processing file " + path + " line=" + line + "... Continuing");
                    continue;
                } else {
                    throw e;
                }
            }
        }
    } catch (RuntimeException e) {
        LOG.error("Exception caught while processing file " + path + " line=[" + line + "]", e);
        throw e;
    } finally {
        in.close();
    }
    final long time_delta = (System.nanoTime() - start_time) / 1000000;
    LOG.info(String.format("Processed %s in %d ms, %d data points" + " (%.1f points/s)", path, time_delta, points, (points * 1000.0 / time_delta)));
    return points;
}

Example 7 with PutRequest

use of org.hbase.async.PutRequest in project opentsdb by OpenTSDB.

the class TreeRule method syncToStorage.

/**
   * Attempts to write the rule to storage via CompareAndSet, merging changes
   * with an existing rule.
   * <b>Note:</b> If the local object didn't have any fields set by the caller
   * or there weren't any changes, then the data will not be written and an 
   * exception will be thrown.
   * <b>Note:</b> This method also validates the rule, making sure that proper
   * combinations of data exist before writing to storage.
   * @param tsdb The TSDB to use for storage access
   * @param overwrite When the RPC method is PUT, will overwrite all user
   * accessible fields
   * @return True if the CAS call succeeded, false if the stored data was
   * modified in flight. This should be retried if that happens.
   * @throws HBaseException if there was an issue
   * @throws IllegalArgumentException if parsing failed or the tree ID was 
   * invalid or validation failed
   * @throws IllegalStateException if the data hasn't changed. This is OK!
   * @throws JSONException if the object could not be serialized
   */
public Deferred<Boolean> syncToStorage(final TSDB tsdb, final boolean overwrite) {
    if (tree_id < 1 || tree_id > 65535) {
        throw new IllegalArgumentException("Invalid Tree ID");
    }
    // if there aren't any changes, save time and bandwidth by not writing to
    // storage
    boolean has_changes = false;
    for (Map.Entry<String, Boolean> entry : changed.entrySet()) {
        if (entry.getValue()) {
            has_changes = true;
            break;
        }
    }
    if (!has_changes) {
        LOG.trace(this + " does not have changes, skipping sync to storage");
        throw new IllegalStateException("No changes detected in the rule");
    }
    /**
     * Executes the CAS after retrieving existing rule from storage, if it
     * exists.
     */
    final class StoreCB implements Callback<Deferred<Boolean>, TreeRule> {

        final TreeRule local_rule;

        public StoreCB(final TreeRule local_rule) {
            this.local_rule = local_rule;
        }

        /**
       * @return True if the CAS was successful, false if not
       */
        @Override
        public Deferred<Boolean> call(final TreeRule fetched_rule) {
            TreeRule stored_rule = fetched_rule;
            final byte[] original_rule = stored_rule == null ? new byte[0] : JSON.serializeToBytes(stored_rule);
            if (stored_rule == null) {
                stored_rule = local_rule;
            } else {
                if (!stored_rule.copyChanges(local_rule, overwrite)) {
                    LOG.debug(this + " does not have changes, skipping sync to storage");
                    throw new IllegalStateException("No changes detected in the rule");
                }
            }
            // reset the local change map so we don't keep writing on subsequent
            // requests
            initializeChangedMap();
            // validate before storing
            stored_rule.validateRule();
            final PutRequest put = new PutRequest(tsdb.treeTable(), Tree.idToBytes(tree_id), Tree.TREE_FAMILY(), getQualifier(level, order), JSON.serializeToBytes(stored_rule));
            return tsdb.getClient().compareAndSet(put, original_rule);
        }
    }
    // start the callback chain by fetching from storage
    return fetchRule(tsdb, tree_id, level, order).addCallbackDeferring(new StoreCB(this));
}

Example 8 with PutRequest

use of org.hbase.async.PutRequest in project opentsdb by OpenTSDB.

the class Tree method flushCollisions.

/**
   * Attempts to flush the collisions to storage. The storage call is a PUT so
   * it will overwrite any existing columns, but since each column is the TSUID
   * it should only exist once and the data shouldn't change.
   * <b>Note:</b> This will also clear the local {@link #collisions} map
   * @param tsdb The TSDB to use for storage access
   * @return A meaningless deferred (will always be true since we need to group
   * it with tree store calls) for the caller to wait on
   * @throws HBaseException if there was an issue
   */
public Deferred<Boolean> flushCollisions(final TSDB tsdb) {
    if (!store_failures) {
        collisions.clear();
        return Deferred.fromResult(true);
    }
    final byte[] row_key = new byte[TREE_ID_WIDTH + 1];
    System.arraycopy(idToBytes(tree_id), 0, row_key, 0, TREE_ID_WIDTH);
    row_key[TREE_ID_WIDTH] = COLLISION_ROW_SUFFIX;
    final byte[][] qualifiers = new byte[collisions.size()][];
    final byte[][] values = new byte[collisions.size()][];
    int index = 0;
    for (Map.Entry<String, String> entry : collisions.entrySet()) {
        qualifiers[index] = new byte[COLLISION_PREFIX.length + (entry.getKey().length() / 2)];
        System.arraycopy(COLLISION_PREFIX, 0, qualifiers[index], 0, COLLISION_PREFIX.length);
        final byte[] tsuid = UniqueId.stringToUid(entry.getKey());
        System.arraycopy(tsuid, 0, qualifiers[index], COLLISION_PREFIX.length, tsuid.length);
        values[index] = entry.getValue().getBytes(CHARSET);
        index++;
    }
    final PutRequest put = new PutRequest(tsdb.treeTable(), row_key, TREE_FAMILY, qualifiers, values);
    collisions.clear();
    /**
     * Super simple callback used to convert the Deferred&lt;Object&gt; to a 
     * Deferred&lt;Boolean&gt; so that it can be grouped with other storage
     * calls
     */
    final class PutCB implements Callback<Deferred<Boolean>, Object> {

        @Override
        public Deferred<Boolean> call(Object result) throws Exception {
            return Deferred.fromResult(true);
        }
    }
    return tsdb.getClient().put(put).addCallbackDeferring(new PutCB());
}

Example 9 with PutRequest

use of org.hbase.async.PutRequest in project YCSB by brianfrankcooper.

the class AsyncHBaseClient method update.

@Override
public Status update(String table, String key, HashMap<String, ByteIterator> values) {
    setTable(table);
    if (debug) {
        System.out.println("Setting up put for key: " + key);
    }
    final byte[][] qualifiers = new byte[values.size()][];
    final byte[][] byteValues = new byte[values.size()][];
    int idx = 0;
    for (final Entry<String, ByteIterator> entry : values.entrySet()) {
        qualifiers[idx] = entry.getKey().getBytes();
        byteValues[idx++] = entry.getValue().toArray();
        if (debug) {
            System.out.println("Adding field/value " + entry.getKey() + "/" + Bytes.pretty(entry.getValue().toArray()) + " to put request");
        }
    }
    final PutRequest put = new PutRequest(lastTableBytes, key.getBytes(), columnFamilyBytes, qualifiers, byteValues);
    if (!durability) {
        put.setDurable(false);
    }
    if (!clientSideBuffering) {
        put.setBufferable(false);
        try {
            client.put(put).join(joinTimeout);
        } catch (InterruptedException e) {
            System.err.println("Thread interrupted");
            Thread.currentThread().interrupt();
        } catch (Exception e) {
            System.err.println("Failure reading from row with key " + key + ": " + e.getMessage());
            return Status.ERROR;
        }
    } else {
        // hooray! Asynchronous write. But without a callback and an async
        // YCSB call we don't know whether it succeeded or not
        client.put(put);
    }
    return Status.OK;
}

Example 10 with PutRequest

use of org.hbase.async.PutRequest in project opentsdb by OpenTSDB.

the class TSMeta method syncToStorage.

/**
   * Attempts a CompareAndSet storage call, loading the object from storage, 
   * synchronizing changes, and attempting a put. Also verifies that associated 
   * UID name mappings exist before merging.
   * <b>Note:</b> If the local object didn't have any fields set by the caller
   * or there weren't any changes, then the data will not be written and an 
   * exception will be thrown.
   * <b>Note:</b> We do not store the UIDMeta information with TSMeta's since
   * users may change a single UIDMeta object and we don't want to update every
   * TSUID that includes that object with the new data. Instead, UIDMetas are
   * merged into the TSMeta on retrieval so we always have canonical data. This
   * also saves space in storage. 
   * @param tsdb The TSDB to use for storage access
   * @param overwrite When the RPC method is PUT, will overwrite all user
   * accessible fields
   * @return True if the storage call was successful, false if the object was
   * modified in storage during the CAS call. If false, retry the call. Other 
   * failures will result in an exception being thrown.
   * @throws HBaseException if there was an issue
   * @throws IllegalArgumentException if parsing failed
   * @throws NoSuchUniqueId If any of the UID name mappings do not exist
   * @throws IllegalStateException if the data hasn't changed. This is OK!
   * @throws JSONException if the object could not be serialized
   */
public Deferred<Boolean> syncToStorage(final TSDB tsdb, final boolean overwrite) {
    if (tsuid == null || tsuid.isEmpty()) {
        throw new IllegalArgumentException("Missing TSUID");
    }
    boolean has_changes = false;
    for (Map.Entry<String, Boolean> entry : changed.entrySet()) {
        if (entry.getValue()) {
            has_changes = true;
            break;
        }
    }
    if (!has_changes) {
        LOG.debug(this + " does not have changes, skipping sync to storage");
        throw new IllegalStateException("No changes detected in TSUID meta data");
    }
    /**
     * Callback used to verify that the UID name mappings exist. We don't need
     * to process the actual name, we just want it to throw an error if any
     * of the UIDs don't exist.
     */
    class UidCB implements Callback<Object, String> {

        @Override
        public Object call(String name) throws Exception {
            // nothing to do as missing mappings will throw a NoSuchUniqueId
            return null;
        }
    }
    // parse out the tags from the tsuid
    final List<byte[]> parsed_tags = UniqueId.getTagsFromTSUID(tsuid);
    // Deferred group used to accumulate UidCB callbacks so the next call
    // can wait until all of the UIDs have been verified
    ArrayList<Deferred<Object>> uid_group = new ArrayList<Deferred<Object>>(parsed_tags.size() + 1);
    // calculate the metric UID and fetch it's name mapping
    final byte[] metric_uid = UniqueId.stringToUid(tsuid.substring(0, TSDB.metrics_width() * 2));
    uid_group.add(tsdb.getUidName(UniqueIdType.METRIC, metric_uid).addCallback(new UidCB()));
    int idx = 0;
    for (byte[] tag : parsed_tags) {
        if (idx % 2 == 0) {
            uid_group.add(tsdb.getUidName(UniqueIdType.TAGK, tag).addCallback(new UidCB()));
        } else {
            uid_group.add(tsdb.getUidName(UniqueIdType.TAGV, tag).addCallback(new UidCB()));
        }
        idx++;
    }
    /**
     * Callback executed after all of the UID mappings have been verified. This
     * will then proceed with the CAS call.
     */
    final class ValidateCB implements Callback<Deferred<Boolean>, ArrayList<Object>> {

        private final TSMeta local_meta;

        public ValidateCB(final TSMeta local_meta) {
            this.local_meta = local_meta;
        }

        /**
       * Nested class that executes the CAS after retrieving existing TSMeta
       * from storage.
       */
        final class StoreCB implements Callback<Deferred<Boolean>, TSMeta> {

            /**
         * Executes the CAS if the TSMeta was successfully retrieved
         * @return True if the CAS was successful, false if the stored data
         * was modified in flight
         * @throws IllegalArgumentException if the TSMeta did not exist in
         * storage. Only the TSD should be able to create TSMeta objects.
         */
            @Override
            public Deferred<Boolean> call(TSMeta stored_meta) throws Exception {
                if (stored_meta == null) {
                    throw new IllegalArgumentException("Requested TSMeta did not exist");
                }
                final byte[] original_meta = stored_meta.getStorageJSON();
                local_meta.syncMeta(stored_meta, overwrite);
                final PutRequest put = new PutRequest(tsdb.metaTable(), UniqueId.stringToUid(local_meta.tsuid), FAMILY, META_QUALIFIER, local_meta.getStorageJSON());
                return tsdb.getClient().compareAndSet(put, original_meta);
            }
        }

        /**
       * Called on UID mapping verification and continues executing the CAS 
       * procedure.
       * @return Results from the {@link #StoreCB} callback
       */
        @Override
        public Deferred<Boolean> call(ArrayList<Object> validated) throws Exception {
            return getFromStorage(tsdb, UniqueId.stringToUid(tsuid)).addCallbackDeferring(new StoreCB());
        }
    }
    // Begins the callback chain by validating that the UID mappings exist
    return Deferred.group(uid_group).addCallbackDeferring(new ValidateCB(this));
}