Examples with DeleteRequest org.hbase.async.DeleteRequest used on opensource projects

Example 1 with DeleteRequest

use of org.hbase.async.DeleteRequest in project YCSB by brianfrankcooper.

the class AsyncHBaseClient method delete.

@Override
public Status delete(String table, String key) {
    setTable(table);
    if (debug) {
        System.out.println("Doing delete for key: " + key);
    }
    final DeleteRequest delete = new DeleteRequest(lastTableBytes, key.getBytes(), columnFamilyBytes);
    if (!durability) {
        delete.setDurable(false);
    }
    if (!clientSideBuffering) {
        delete.setBufferable(false);
        try {
            client.delete(delete).join(joinTimeout);
        } catch (InterruptedException e) {
            System.err.println("Thread interrupted");
            Thread.currentThread().interrupt();
        } catch (Exception e) {
            System.err.println("Failure reading from row with key " + key + ": " + e.getMessage());
            return Status.ERROR;
        }
    } else {
        // hooray! Asynchronous write. But without a callback and an async
        // YCSB call we don't know whether it succeeded or not
        client.delete(delete);
    }
    return Status.OK;
}

Example 2 with DeleteRequest

use of org.hbase.async.DeleteRequest in project opentsdb by OpenTSDB.

the class Annotation method deleteRange.

/**
 * Deletes global or TSUID associated annotiations for the given time range.
 * @param tsdb The TSDB object to use for storage access
 * @param tsuid An optional TSUID. If set to null, then global annotations for
 * the given range will be deleted
 * @param start_time A start timestamp in milliseconds
 * @param end_time An end timestamp in millseconds
 * @return The number of annotations deleted
 * @throws IllegalArgumentException if the timestamps are invalid
 * @since 2.1
 */
public static Deferred<Integer> deleteRange(final TSDB tsdb, final byte[] tsuid, final long start_time, final long end_time) {
    if (end_time < 1) {
        throw new IllegalArgumentException("The end timestamp has not been set");
    }
    if (end_time < start_time) {
        throw new IllegalArgumentException("The end timestamp cannot be less than the start timestamp");
    }
    final List<Deferred<Object>> delete_requests = new ArrayList<Deferred<Object>>();
    int width = tsuid != null ? Const.SALT_WIDTH() + tsuid.length + Const.TIMESTAMP_BYTES : Const.SALT_WIDTH() + TSDB.metrics_width() + Const.TIMESTAMP_BYTES;
    final byte[] start_row = new byte[width];
    final byte[] end_row = new byte[width];
    // downsample to seconds for the row keys
    final long start = start_time / 1000;
    final long end = end_time / 1000;
    final long normalized_start = (start - (start % Const.MAX_TIMESPAN));
    final long normalized_end = (end - (end % Const.MAX_TIMESPAN) + Const.MAX_TIMESPAN);
    Bytes.setInt(start_row, (int) normalized_start, Const.SALT_WIDTH() + TSDB.metrics_width());
    Bytes.setInt(end_row, (int) normalized_end, Const.SALT_WIDTH() + TSDB.metrics_width());
    if (tsuid != null) {
        // first copy the metric UID then the tags
        System.arraycopy(tsuid, 0, start_row, Const.SALT_WIDTH(), TSDB.metrics_width());
        System.arraycopy(tsuid, 0, end_row, Const.SALT_WIDTH(), TSDB.metrics_width());
        width = Const.SALT_WIDTH() + TSDB.metrics_width() + Const.TIMESTAMP_BYTES;
        final int remainder = tsuid.length - TSDB.metrics_width();
        System.arraycopy(tsuid, TSDB.metrics_width(), start_row, width, remainder);
        System.arraycopy(tsuid, TSDB.metrics_width(), end_row, width, remainder);
    }
    /**
     * Iterates through the scanner results in an asynchronous manner, returning
     * once the scanner returns a null result set.
     */
    final class ScannerCB implements Callback<Deferred<List<Deferred<Object>>>, ArrayList<ArrayList<KeyValue>>> {

        final Scanner scanner;

        public ScannerCB() {
            scanner = tsdb.getClient().newScanner(tsdb.dataTable());
            scanner.setStartKey(start_row);
            scanner.setStopKey(end_row);
            scanner.setFamily(FAMILY);
            if (tsuid != null) {
                final List<String> tsuids = new ArrayList<String>(1);
                tsuids.add(UniqueId.uidToString(tsuid));
                Internal.createAndSetTSUIDFilter(scanner, tsuids);
            }
        }

        public Deferred<List<Deferred<Object>>> scan() {
            return scanner.nextRows().addCallbackDeferring(this);
        }

        @Override
        public Deferred<List<Deferred<Object>>> call(final ArrayList<ArrayList<KeyValue>> rows) throws Exception {
            if (rows == null || rows.isEmpty()) {
                return Deferred.fromResult(delete_requests);
            }
            for (final ArrayList<KeyValue> row : rows) {
                final long base_time = Internal.baseTime(tsdb, row.get(0).key());
                for (KeyValue column : row) {
                    if ((column.qualifier().length == 3 || column.qualifier().length == 5) && column.qualifier()[0] == PREFIX()) {
                        final long timestamp = timeFromQualifier(column.qualifier(), base_time);
                        if (timestamp < start_time || timestamp > end_time) {
                            continue;
                        }
                        final DeleteRequest delete = new DeleteRequest(tsdb.dataTable(), column.key(), FAMILY, column.qualifier());
                        delete_requests.add(tsdb.getClient().delete(delete));
                    }
                }
            }
            return scan();
        }
    }
    /**
     * Called when the scanner is done. Delete requests may still be pending
     */
    final class ScannerDoneCB implements Callback<Deferred<ArrayList<Object>>, List<Deferred<Object>>> {

        @Override
        public Deferred<ArrayList<Object>> call(final List<Deferred<Object>> deletes) throws Exception {
            return Deferred.group(delete_requests);
        }
    }
    /**
     * Waits on the group of deferreds to complete before returning the count
     */
    final class GroupCB implements Callback<Deferred<Integer>, ArrayList<Object>> {

        @Override
        public Deferred<Integer> call(final ArrayList<Object> deletes) throws Exception {
            return Deferred.fromResult(deletes.size());
        }
    }
    Deferred<ArrayList<Object>> scanner_done = new ScannerCB().scan().addCallbackDeferring(new ScannerDoneCB());
    return scanner_done.addCallbackDeferring(new GroupCB());
}

Example 3 with DeleteRequest

use of org.hbase.async.DeleteRequest in project opentsdb by OpenTSDB.

the class Annotation method delete.

/**
 * Attempts to mark an Annotation object for deletion. Note that if the
 * annoation does not exist in storage, this delete call will not throw an
 * error.
 * @param tsdb The TSDB to use for storage access
 * @return A meaningless Deferred for the caller to wait on until the call is
 * complete. The value may be null.
 */
public Deferred<Object> delete(final TSDB tsdb) {
    if (start_time < 1) {
        throw new IllegalArgumentException("The start timestamp has not been set");
    }
    final byte[] tsuid_byte = tsuid != null && !tsuid.isEmpty() ? UniqueId.stringToUid(tsuid) : null;
    final DeleteRequest delete = new DeleteRequest(tsdb.dataTable(), getRowKey(start_time, tsuid_byte), FAMILY, getQualifier(start_time));
    return tsdb.getClient().delete(delete);
}

Example 4 with DeleteRequest

use of org.hbase.async.DeleteRequest in project opentsdb by OpenTSDB.

the class UidManager method fsck.

/**
 * Implements the {@code fsck} subcommand.
 * @param client The HBase client to use.
 * @param table The name of the HBase table to use.
 * @return The exit status of the command (0 means success).
 */
private static int fsck(final HBaseClient client, final byte[] table, final boolean fix, final boolean fix_unknowns) {
    if (fix) {
        LOG.info("----------------------------------");
        LOG.info("-    Running fsck in FIX mode    -");
        LOG.info("-      Remove Unknowns: " + fix_unknowns + "     -");
        LOG.info("----------------------------------");
    } else {
        LOG.info("Running in log only mode");
    }
    final class Uids {

        int errors;

        long maxid;

        long max_found_id;

        short width;

        final HashMap<String, String> id2name = new HashMap<String, String>();

        final HashMap<String, String> name2id = new HashMap<String, String>();

        void error(final KeyValue kv, final String msg) {
            error(msg + ".  kv=" + kv);
        }

        void error(final String msg) {
            LOG.error(msg);
            errors++;
        }

        /*
       * Replaces or creates the reverse map in storage and in the local map
       */
        void restoreReverseMap(final String kind, final String name, final String uid) {
            final PutRequest put = new PutRequest(table, UniqueId.stringToUid(uid), CliUtils.NAME_FAMILY, CliUtils.toBytes(kind), CliUtils.toBytes(name));
            client.put(put);
            id2name.put(uid, name);
            LOG.info("FIX: Restoring " + kind + " reverse mapping: " + uid + " -> " + name);
        }

        /*
       * Removes the reverse map from storage only
       */
        void removeReverseMap(final String kind, final String name, final String uid) {
            // clean up meta data too
            final byte[][] qualifiers = new byte[2][];
            qualifiers[0] = CliUtils.toBytes(kind);
            if (Bytes.equals(CliUtils.METRICS, qualifiers[0])) {
                qualifiers[1] = CliUtils.METRICS_META;
            } else if (Bytes.equals(CliUtils.TAGK, qualifiers[0])) {
                qualifiers[1] = CliUtils.TAGK_META;
            } else if (Bytes.equals(CliUtils.TAGV, qualifiers[0])) {
                qualifiers[1] = CliUtils.TAGV_META;
            }
            final DeleteRequest delete = new DeleteRequest(table, UniqueId.stringToUid(uid), CliUtils.NAME_FAMILY, qualifiers);
            client.delete(delete);
            // can't remove from the id2name map as this will be called while looping
            LOG.info("FIX: Removed " + kind + " reverse mapping: " + uid + " -> " + name);
        }
    }
    final long start_time = System.nanoTime();
    final HashMap<String, Uids> name2uids = new HashMap<String, Uids>();
    final Scanner scanner = client.newScanner(table);
    scanner.setMaxNumRows(1024);
    int kvcount = 0;
    try {
        ArrayList<ArrayList<KeyValue>> rows;
        while ((rows = scanner.nextRows().joinUninterruptibly()) != null) {
            for (final ArrayList<KeyValue> row : rows) {
                for (final KeyValue kv : row) {
                    kvcount++;
                    final byte[] qualifier = kv.qualifier();
                    // TODO - validate meta data in the future, for now skip it
                    if (Bytes.equals(qualifier, TSMeta.META_QUALIFIER()) || Bytes.equals(qualifier, TSMeta.COUNTER_QUALIFIER()) || Bytes.equals(qualifier, CliUtils.METRICS_META) || Bytes.equals(qualifier, CliUtils.TAGK_META) || Bytes.equals(qualifier, CliUtils.TAGV_META)) {
                        continue;
                    }
                    if (!Bytes.equals(qualifier, CliUtils.METRICS) && !Bytes.equals(qualifier, CliUtils.TAGK) && !Bytes.equals(qualifier, CliUtils.TAGV)) {
                        LOG.warn("Unknown qualifier " + UniqueId.uidToString(qualifier) + " in row " + UniqueId.uidToString(kv.key()));
                        if (fix && fix_unknowns) {
                            final DeleteRequest delete = new DeleteRequest(table, kv.key(), kv.family(), qualifier);
                            client.delete(delete);
                            LOG.info("FIX: Removed unknown qualifier " + UniqueId.uidToString(qualifier) + " in row " + UniqueId.uidToString(kv.key()));
                        }
                        continue;
                    }
                    final String kind = CliUtils.fromBytes(kv.qualifier());
                    Uids uids = name2uids.get(kind);
                    if (uids == null) {
                        uids = new Uids();
                        name2uids.put(kind, uids);
                    }
                    final byte[] key = kv.key();
                    final byte[] family = kv.family();
                    final byte[] value = kv.value();
                    if (Bytes.equals(key, CliUtils.MAXID_ROW)) {
                        if (value.length != 8) {
                            uids.error(kv, "Invalid maximum ID for " + kind + ": should be on 8 bytes: ");
                        // TODO - a fix would be to find the max used ID for the type
                        // and store that in the max row.
                        } else {
                            uids.maxid = Bytes.getLong(value);
                            LOG.info("Maximum ID for " + kind + ": " + uids.maxid);
                        }
                    } else {
                        short idwidth = 0;
                        if (Bytes.equals(family, CliUtils.ID_FAMILY)) {
                            idwidth = (short) value.length;
                            final String skey = CliUtils.fromBytes(key);
                            final String svalue = UniqueId.uidToString(value);
                            final long max_found_id;
                            if (Bytes.equals(qualifier, CliUtils.METRICS)) {
                                max_found_id = UniqueId.uidToLong(value, TSDB.metrics_width());
                            } else if (Bytes.equals(qualifier, CliUtils.TAGK)) {
                                max_found_id = UniqueId.uidToLong(value, TSDB.tagk_width());
                            } else {
                                max_found_id = UniqueId.uidToLong(value, TSDB.tagv_width());
                            }
                            if (uids.max_found_id < max_found_id) {
                                uids.max_found_id = max_found_id;
                            }
                            final String id = uids.name2id.put(skey, svalue);
                            if (id != null) {
                                uids.error(kv, "Duplicate forward " + kind + " mapping: " + skey + " -> " + id + " and " + skey + " -> " + svalue);
                            }
                        } else if (Bytes.equals(family, CliUtils.NAME_FAMILY)) {
                            final String skey = UniqueId.uidToString(key);
                            final String svalue = CliUtils.fromBytes(value);
                            idwidth = (short) key.length;
                            final String name = uids.id2name.put(skey, svalue);
                            if (name != null) {
                                uids.error(kv, "Duplicate reverse " + kind + "  mapping: " + svalue + " -> " + name + " and " + svalue + " -> " + skey);
                            }
                        }
                        if (uids.width == 0) {
                            uids.width = idwidth;
                        } else if (uids.width != idwidth) {
                            uids.error(kv, "Invalid " + kind + " ID of length " + idwidth + " (expected: " + uids.width + ')');
                        }
                    }
                }
            }
        }
    } catch (HBaseException e) {
        LOG.error("Error while scanning HBase, scanner=" + scanner, e);
        throw e;
    } catch (Exception e) {
        LOG.error("WTF?  Unexpected exception type, scanner=" + scanner, e);
        throw new AssertionError("Should never happen");
    }
    // Match up all forward mappings with their reverse mappings and vice
    // versa and make sure they agree.
    int errors = 0;
    for (final Map.Entry<String, Uids> entry : name2uids.entrySet()) {
        final String kind = entry.getKey();
        final Uids uids = entry.getValue();
        // This will be used in the event that we run into an inconsistent forward
        // mapping that could mean a single UID was assigned to different names.
        // It SHOULD NEVER HAPPEN, but it could.
        HashMap<String, TreeSet<String>> uid_collisions = null;
        // These are harmful and shouldn't exist.
        for (final Map.Entry<String, String> nameid : uids.name2id.entrySet()) {
            final String name = nameid.getKey();
            final String id = nameid.getValue();
            final String found = uids.id2name.get(id);
            if (found == null) {
                uids.error("Forward " + kind + " mapping is missing reverse" + " mapping: " + name + " -> " + id);
                if (fix) {
                    uids.restoreReverseMap(kind, name, id);
                }
            } else if (!found.equals(name)) {
                uids.error("Forward " + kind + " mapping " + name + " -> " + id + " is different than reverse mapping: " + id + " -> " + found);
                final String id2 = uids.name2id.get(found);
                if (id2 != null) {
                    uids.error("Inconsistent forward " + kind + " mapping " + name + " -> " + id + " vs " + name + " -> " + found + " / " + found + " -> " + id2);
                    // series.
                    if (fix) {
                        // once, as needed, since it's expensive.
                        if (uid_collisions == null) {
                            uid_collisions = new HashMap<String, TreeSet<String>>(uids.name2id.size());
                            for (final Map.Entry<String, String> row : uids.name2id.entrySet()) {
                                TreeSet<String> names = uid_collisions.get(row.getValue());
                                if (names == null) {
                                    names = new TreeSet<String>();
                                    uid_collisions.put(row.getValue(), names);
                                }
                                names.add(row.getKey());
                            }
                        }
                        // series *should* be OK and we can just fix the reverse map.
                        if (uid_collisions.containsKey(id) && uid_collisions.get(id).size() <= 1) {
                            uids.restoreReverseMap(kind, name, id);
                        }
                    }
                } else {
                    uids.error("Duplicate forward " + kind + " mapping " + name + " -> " + id + " and " + id2 + " -> " + found);
                    if (fix) {
                        uids.restoreReverseMap(kind, name, id);
                    }
                }
            }
        }
        // Scan through the UID collisions map and fix the screw ups
        if (uid_collisions != null) {
            for (Map.Entry<String, TreeSet<String>> collision : uid_collisions.entrySet()) {
                if (collision.getValue().size() <= 1) {
                    continue;
                }
                // The data in any time series with the errant UID is
                // a mashup of with all of the names. The best thing to do is
                // start over. We'll rename the old time series so the user can
                // still see it if they want to, but delete the forward mappings
                // so that UIDs can be reassigned and clean series started.
                // - concatenate all of the names into
                // "fsck.<name1>.<name2>[...<nameN>]"
                // - delete the forward mappings for all of the names
                // - create a mapping with the fsck'd name pointing to the id
                final StringBuilder fsck_builder = new StringBuilder("fsck");
                final String id = collision.getKey();
                // compile the new fsck'd name and remove each of the duplicate keys
                for (String name : collision.getValue()) {
                    fsck_builder.append(".").append(name);
                    final DeleteRequest delete = new DeleteRequest(table, CliUtils.toBytes(name), CliUtils.ID_FAMILY, CliUtils.toBytes(kind));
                    client.delete(delete);
                    uids.name2id.remove(name);
                    LOG.info("FIX: Removed forward " + kind + " mapping for " + name + " -> " + id);
                }
                // write the new forward map
                final String fsck_name = fsck_builder.toString();
                final PutRequest put = new PutRequest(table, CliUtils.toBytes(fsck_name), CliUtils.ID_FAMILY, CliUtils.toBytes(kind), UniqueId.stringToUid(id));
                client.put(put);
                LOG.info("FIX: Created forward " + kind + " mapping for fsck'd UID " + fsck_name + " -> " + collision.getKey());
                // we still need to fix the uids map for the reverse run through below
                uids.name2id.put(fsck_name, collision.getKey());
                uids.restoreReverseMap(kind, fsck_name, id);
                LOG.error("----------------------------------");
                LOG.error("-     UID COLLISION DETECTED     -");
                LOG.error("Corrupted UID [" + collision.getKey() + "] renamed to [" + fsck_name + "]");
                LOG.error("----------------------------------");
            }
        }
        // These are harmless but shouldn't frequently occur.
        for (final Map.Entry<String, String> idname : uids.id2name.entrySet()) {
            final String name = idname.getValue();
            final String id = idname.getKey();
            final String found = uids.name2id.get(name);
            if (found == null) {
                LOG.warn("Reverse " + kind + " mapping is missing forward" + " mapping: " + name + " -> " + id);
                if (fix) {
                    uids.removeReverseMap(kind, name, id);
                }
            } else if (!found.equals(id)) {
                final String name2 = uids.id2name.get(found);
                if (name2 != null) {
                    uids.error("Inconsistent reverse " + kind + " mapping " + id + " -> " + name + " vs " + found + " -> " + name + " / " + name2 + " -> " + found);
                    if (fix) {
                        uids.removeReverseMap(kind, name, id);
                    }
                } else {
                    uids.error("Duplicate reverse " + kind + " mapping " + id + " -> " + name + " and " + found + " -> " + name2);
                    if (fix) {
                        uids.removeReverseMap(kind, name, id);
                    }
                }
            }
        }
        final int maxsize = Math.max(uids.id2name.size(), uids.name2id.size());
        if (uids.maxid > maxsize) {
            LOG.warn("Max ID for " + kind + " is " + uids.maxid + " but only " + maxsize + " entries were found.  Maybe " + (uids.maxid - maxsize) + " IDs were deleted?");
        } else if (uids.maxid < uids.max_found_id) {
            uids.error("We found an ID of " + uids.max_found_id + " for " + kind + " but the max ID is only " + uids.maxid + "!  Future IDs may be double-assigned!");
            if (fix) {
                // IDs than to under-run.
                if (uids.max_found_id == Long.MAX_VALUE) {
                    LOG.error("Ran out of UIDs for " + kind + ". Unable to fix max ID");
                } else {
                    final long diff = uids.max_found_id - uids.maxid;
                    final AtomicIncrementRequest air = new AtomicIncrementRequest(table, CliUtils.MAXID_ROW, CliUtils.ID_FAMILY, CliUtils.toBytes(kind), diff);
                    client.atomicIncrement(air);
                    LOG.info("FIX: Updated max ID for " + kind + " to " + uids.max_found_id);
                }
            }
        }
        if (uids.errors > 0) {
            LOG.error(kind + ": Found " + uids.errors + " errors.");
            errors += uids.errors;
        }
    }
    final long timing = (System.nanoTime() - start_time) / 1000000;
    LOG.info(kvcount + " KVs analyzed in " + timing + "ms (~" + (kvcount * 1000 / timing) + " KV/s)");
    if (errors == 0) {
        LOG.info("No errors found.");
        return 0;
    }
    LOG.warn(errors + " errors found.");
    return errors;
}

Example 5 with DeleteRequest

use of org.hbase.async.DeleteRequest in project opentsdb by OpenTSDB.

the class TreeRule method deleteAllRules.

/**
 * Attempts to delete all rules belonging to the given tree.
 * @param tsdb The TSDB to use for storage access
 * @param tree_id ID of the tree the rules belongs to
 * @return A deferred to wait on for completion. The value has no meaning and
 * may be null.
 * @throws HBaseException if there was an issue
 * @throws IllegalArgumentException if the one of the required parameters was
 * missing
 */
public static Deferred<Object> deleteAllRules(final TSDB tsdb, final int tree_id) {
    if (tree_id < 1 || tree_id > 65535) {
        throw new IllegalArgumentException("Invalid Tree ID");
    }
    // fetch the whole row
    final GetRequest get = new GetRequest(tsdb.treeTable(), Tree.idToBytes(tree_id));
    get.family(Tree.TREE_FAMILY());
    /**
     * Called after fetching the requested row. If the row is empty, we just
     * return, otherwise we compile a list of qualifiers to delete and submit
     * a single delete request to storage.
     */
    final class GetCB implements Callback<Deferred<Object>, ArrayList<KeyValue>> {

        @Override
        public Deferred<Object> call(final ArrayList<KeyValue> row) throws Exception {
            if (row == null || row.isEmpty()) {
                return Deferred.fromResult(null);
            }
            final ArrayList<byte[]> qualifiers = new ArrayList<byte[]>(row.size());
            for (KeyValue column : row) {
                if (column.qualifier().length > RULE_PREFIX.length && Bytes.memcmp(RULE_PREFIX, column.qualifier(), 0, RULE_PREFIX.length) == 0) {
                    qualifiers.add(column.qualifier());
                }
            }
            final DeleteRequest delete = new DeleteRequest(tsdb.treeTable(), Tree.idToBytes(tree_id), Tree.TREE_FAMILY(), qualifiers.toArray(new byte[qualifiers.size()][]));
            return tsdb.getClient().delete(delete);
        }
    }
    return tsdb.getClient().get(get).addCallbackDeferring(new GetCB());
}