use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.
the class Annotation method deleteRange.
/**
* Deletes global or TSUID associated annotiations for the given time range.
* @param tsdb The TSDB object to use for storage access
* @param tsuid An optional TSUID. If set to null, then global annotations for
* the given range will be deleted
* @param start_time A start timestamp in milliseconds
* @param end_time An end timestamp in millseconds
* @return The number of annotations deleted
* @throws IllegalArgumentException if the timestamps are invalid
* @since 2.1
*/
public static Deferred<Integer> deleteRange(final TSDB tsdb, final byte[] tsuid, final long start_time, final long end_time) {
if (end_time < 1) {
throw new IllegalArgumentException("The end timestamp has not been set");
}
if (end_time < start_time) {
throw new IllegalArgumentException("The end timestamp cannot be less than the start timestamp");
}
final List<Deferred<Object>> delete_requests = new ArrayList<Deferred<Object>>();
int width = tsuid != null ? Const.SALT_WIDTH() + tsuid.length + Const.TIMESTAMP_BYTES : Const.SALT_WIDTH() + TSDB.metrics_width() + Const.TIMESTAMP_BYTES;
final byte[] start_row = new byte[width];
final byte[] end_row = new byte[width];
// downsample to seconds for the row keys
final long start = start_time / 1000;
final long end = end_time / 1000;
final long normalized_start = (start - (start % Const.MAX_TIMESPAN));
final long normalized_end = (end - (end % Const.MAX_TIMESPAN) + Const.MAX_TIMESPAN);
Bytes.setInt(start_row, (int) normalized_start, Const.SALT_WIDTH() + TSDB.metrics_width());
Bytes.setInt(end_row, (int) normalized_end, Const.SALT_WIDTH() + TSDB.metrics_width());
if (tsuid != null) {
// first copy the metric UID then the tags
System.arraycopy(tsuid, 0, start_row, Const.SALT_WIDTH(), TSDB.metrics_width());
System.arraycopy(tsuid, 0, end_row, Const.SALT_WIDTH(), TSDB.metrics_width());
width = Const.SALT_WIDTH() + TSDB.metrics_width() + Const.TIMESTAMP_BYTES;
final int remainder = tsuid.length - TSDB.metrics_width();
System.arraycopy(tsuid, TSDB.metrics_width(), start_row, width, remainder);
System.arraycopy(tsuid, TSDB.metrics_width(), end_row, width, remainder);
}
/**
* Iterates through the scanner results in an asynchronous manner, returning
* once the scanner returns a null result set.
*/
final class ScannerCB implements Callback<Deferred<List<Deferred<Object>>>, ArrayList<ArrayList<KeyValue>>> {
final Scanner scanner;
public ScannerCB() {
scanner = tsdb.getClient().newScanner(tsdb.dataTable());
scanner.setStartKey(start_row);
scanner.setStopKey(end_row);
scanner.setFamily(FAMILY);
if (tsuid != null) {
final List<String> tsuids = new ArrayList<String>(1);
tsuids.add(UniqueId.uidToString(tsuid));
Internal.createAndSetTSUIDFilter(scanner, tsuids);
}
}
public Deferred<List<Deferred<Object>>> scan() {
return scanner.nextRows().addCallbackDeferring(this);
}
@Override
public Deferred<List<Deferred<Object>>> call(final ArrayList<ArrayList<KeyValue>> rows) throws Exception {
if (rows == null || rows.isEmpty()) {
return Deferred.fromResult(delete_requests);
}
for (final ArrayList<KeyValue> row : rows) {
final long base_time = Internal.baseTime(tsdb, row.get(0).key());
for (KeyValue column : row) {
if ((column.qualifier().length == 3 || column.qualifier().length == 5) && column.qualifier()[0] == PREFIX()) {
final long timestamp = timeFromQualifier(column.qualifier(), base_time);
if (timestamp < start_time || timestamp > end_time) {
continue;
}
final DeleteRequest delete = new DeleteRequest(tsdb.dataTable(), column.key(), FAMILY, column.qualifier());
delete_requests.add(tsdb.getClient().delete(delete));
}
}
}
return scan();
}
}
/** Called when the scanner is done. Delete requests may still be pending */
final class ScannerDoneCB implements Callback<Deferred<ArrayList<Object>>, List<Deferred<Object>>> {
@Override
public Deferred<ArrayList<Object>> call(final List<Deferred<Object>> deletes) throws Exception {
return Deferred.group(delete_requests);
}
}
/** Waits on the group of deferreds to complete before returning the count */
final class GroupCB implements Callback<Deferred<Integer>, ArrayList<Object>> {
@Override
public Deferred<Integer> call(final ArrayList<Object> deletes) throws Exception {
return Deferred.fromResult(deletes.size());
}
}
Deferred<ArrayList<Object>> scanner_done = new ScannerCB().scan().addCallbackDeferring(new ScannerDoneCB());
return scanner_done.addCallbackDeferring(new GroupCB());
}
use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.
the class TreeSync method getScanner.
/**
* Returns a scanner set to scan the range configured for this thread
* @return A scanner on the "name" CF configured for the specified range
* @throws HBaseException if something goes boom
*/
private Scanner getScanner() throws HBaseException {
final short metric_width = TSDB.metrics_width();
final byte[] start_row = Arrays.copyOfRange(Bytes.fromLong(start_id), 8 - metric_width, 8);
final byte[] end_row = Arrays.copyOfRange(Bytes.fromLong(end_id), 8 - metric_width, 8);
LOG.debug("[" + thread_id + "] Start row: " + UniqueId.uidToString(start_row));
LOG.debug("[" + thread_id + "] End row: " + UniqueId.uidToString(end_row));
final Scanner scanner = tsdb.getClient().newScanner(tsdb.metaTable());
scanner.setStartKey(start_row);
scanner.setStopKey(end_row);
scanner.setFamily("name".getBytes(CHARSET));
scanner.setQualifier("ts_meta".getBytes(CHARSET));
return scanner;
}
use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.
the class UidManager method fsck.
/**
* Implements the {@code fsck} subcommand.
* @param client The HBase client to use.
* @param table The name of the HBase table to use.
* @return The exit status of the command (0 means success).
*/
private static int fsck(final HBaseClient client, final byte[] table, final boolean fix, final boolean fix_unknowns) {
if (fix) {
LOG.info("----------------------------------");
LOG.info("- Running fsck in FIX mode -");
LOG.info("- Remove Unknowns: " + fix_unknowns + " -");
LOG.info("----------------------------------");
} else {
LOG.info("Running in log only mode");
}
final class Uids {
int errors;
long maxid;
long max_found_id;
short width;
final HashMap<String, String> id2name = new HashMap<String, String>();
final HashMap<String, String> name2id = new HashMap<String, String>();
void error(final KeyValue kv, final String msg) {
error(msg + ". kv=" + kv);
}
void error(final String msg) {
LOG.error(msg);
errors++;
}
/*
* Replaces or creates the reverse map in storage and in the local map
*/
void restoreReverseMap(final String kind, final String name, final String uid) {
final PutRequest put = new PutRequest(table, UniqueId.stringToUid(uid), CliUtils.NAME_FAMILY, CliUtils.toBytes(kind), CliUtils.toBytes(name));
client.put(put);
id2name.put(uid, name);
LOG.info("FIX: Restoring " + kind + " reverse mapping: " + uid + " -> " + name);
}
/*
* Removes the reverse map from storage only
*/
void removeReverseMap(final String kind, final String name, final String uid) {
// clean up meta data too
final byte[][] qualifiers = new byte[2][];
qualifiers[0] = CliUtils.toBytes(kind);
if (Bytes.equals(CliUtils.METRICS, qualifiers[0])) {
qualifiers[1] = CliUtils.METRICS_META;
} else if (Bytes.equals(CliUtils.TAGK, qualifiers[0])) {
qualifiers[1] = CliUtils.TAGK_META;
} else if (Bytes.equals(CliUtils.TAGV, qualifiers[0])) {
qualifiers[1] = CliUtils.TAGV_META;
}
final DeleteRequest delete = new DeleteRequest(table, UniqueId.stringToUid(uid), CliUtils.NAME_FAMILY, qualifiers);
client.delete(delete);
// can't remove from the id2name map as this will be called while looping
LOG.info("FIX: Removed " + kind + " reverse mapping: " + uid + " -> " + name);
}
}
final long start_time = System.nanoTime();
final HashMap<String, Uids> name2uids = new HashMap<String, Uids>();
final Scanner scanner = client.newScanner(table);
scanner.setMaxNumRows(1024);
int kvcount = 0;
try {
ArrayList<ArrayList<KeyValue>> rows;
while ((rows = scanner.nextRows().joinUninterruptibly()) != null) {
for (final ArrayList<KeyValue> row : rows) {
for (final KeyValue kv : row) {
kvcount++;
final byte[] qualifier = kv.qualifier();
// TODO - validate meta data in the future, for now skip it
if (Bytes.equals(qualifier, TSMeta.META_QUALIFIER()) || Bytes.equals(qualifier, TSMeta.COUNTER_QUALIFIER()) || Bytes.equals(qualifier, CliUtils.METRICS_META) || Bytes.equals(qualifier, CliUtils.TAGK_META) || Bytes.equals(qualifier, CliUtils.TAGV_META)) {
continue;
}
if (!Bytes.equals(qualifier, CliUtils.METRICS) && !Bytes.equals(qualifier, CliUtils.TAGK) && !Bytes.equals(qualifier, CliUtils.TAGV)) {
LOG.warn("Unknown qualifier " + UniqueId.uidToString(qualifier) + " in row " + UniqueId.uidToString(kv.key()));
if (fix && fix_unknowns) {
final DeleteRequest delete = new DeleteRequest(table, kv.key(), kv.family(), qualifier);
client.delete(delete);
LOG.info("FIX: Removed unknown qualifier " + UniqueId.uidToString(qualifier) + " in row " + UniqueId.uidToString(kv.key()));
}
continue;
}
final String kind = CliUtils.fromBytes(kv.qualifier());
Uids uids = name2uids.get(kind);
if (uids == null) {
uids = new Uids();
name2uids.put(kind, uids);
}
final byte[] key = kv.key();
final byte[] family = kv.family();
final byte[] value = kv.value();
if (Bytes.equals(key, CliUtils.MAXID_ROW)) {
if (value.length != 8) {
uids.error(kv, "Invalid maximum ID for " + kind + ": should be on 8 bytes: ");
// TODO - a fix would be to find the max used ID for the type
// and store that in the max row.
} else {
uids.maxid = Bytes.getLong(value);
LOG.info("Maximum ID for " + kind + ": " + uids.maxid);
}
} else {
short idwidth = 0;
if (Bytes.equals(family, CliUtils.ID_FAMILY)) {
idwidth = (short) value.length;
final String skey = CliUtils.fromBytes(key);
final String svalue = UniqueId.uidToString(value);
final long max_found_id;
if (Bytes.equals(qualifier, CliUtils.METRICS)) {
max_found_id = UniqueId.uidToLong(value, TSDB.metrics_width());
} else if (Bytes.equals(qualifier, CliUtils.TAGK)) {
max_found_id = UniqueId.uidToLong(value, TSDB.tagk_width());
} else {
max_found_id = UniqueId.uidToLong(value, TSDB.tagv_width());
}
if (uids.max_found_id < max_found_id) {
uids.max_found_id = max_found_id;
}
final String id = uids.name2id.put(skey, svalue);
if (id != null) {
uids.error(kv, "Duplicate forward " + kind + " mapping: " + skey + " -> " + id + " and " + skey + " -> " + svalue);
}
} else if (Bytes.equals(family, CliUtils.NAME_FAMILY)) {
final String skey = UniqueId.uidToString(key);
final String svalue = CliUtils.fromBytes(value);
idwidth = (short) key.length;
final String name = uids.id2name.put(skey, svalue);
if (name != null) {
uids.error(kv, "Duplicate reverse " + kind + " mapping: " + svalue + " -> " + name + " and " + svalue + " -> " + skey);
}
}
if (uids.width == 0) {
uids.width = idwidth;
} else if (uids.width != idwidth) {
uids.error(kv, "Invalid " + kind + " ID of length " + idwidth + " (expected: " + uids.width + ')');
}
}
}
}
}
} catch (HBaseException e) {
LOG.error("Error while scanning HBase, scanner=" + scanner, e);
throw e;
} catch (Exception e) {
LOG.error("WTF? Unexpected exception type, scanner=" + scanner, e);
throw new AssertionError("Should never happen");
}
// Match up all forward mappings with their reverse mappings and vice
// versa and make sure they agree.
int errors = 0;
for (final Map.Entry<String, Uids> entry : name2uids.entrySet()) {
final String kind = entry.getKey();
final Uids uids = entry.getValue();
// This will be used in the event that we run into an inconsistent forward
// mapping that could mean a single UID was assigned to different names.
// It SHOULD NEVER HAPPEN, but it could.
HashMap<String, TreeSet<String>> uid_collisions = null;
// These are harmful and shouldn't exist.
for (final Map.Entry<String, String> nameid : uids.name2id.entrySet()) {
final String name = nameid.getKey();
final String id = nameid.getValue();
final String found = uids.id2name.get(id);
if (found == null) {
uids.error("Forward " + kind + " mapping is missing reverse" + " mapping: " + name + " -> " + id);
if (fix) {
uids.restoreReverseMap(kind, name, id);
}
} else if (!found.equals(name)) {
uids.error("Forward " + kind + " mapping " + name + " -> " + id + " is different than reverse mapping: " + id + " -> " + found);
final String id2 = uids.name2id.get(found);
if (id2 != null) {
uids.error("Inconsistent forward " + kind + " mapping " + name + " -> " + id + " vs " + name + " -> " + found + " / " + found + " -> " + id2);
// series.
if (fix) {
// once, as needed, since it's expensive.
if (uid_collisions == null) {
uid_collisions = new HashMap<String, TreeSet<String>>(uids.name2id.size());
for (final Map.Entry<String, String> row : uids.name2id.entrySet()) {
TreeSet<String> names = uid_collisions.get(row.getValue());
if (names == null) {
names = new TreeSet<String>();
uid_collisions.put(row.getValue(), names);
}
names.add(row.getKey());
}
}
// series *should* be OK and we can just fix the reverse map.
if (uid_collisions.containsKey(id) && uid_collisions.get(id).size() <= 1) {
uids.restoreReverseMap(kind, name, id);
}
}
} else {
uids.error("Duplicate forward " + kind + " mapping " + name + " -> " + id + " and " + id2 + " -> " + found);
if (fix) {
uids.restoreReverseMap(kind, name, id);
}
}
}
}
// Scan through the UID collisions map and fix the screw ups
if (uid_collisions != null) {
for (Map.Entry<String, TreeSet<String>> collision : uid_collisions.entrySet()) {
if (collision.getValue().size() <= 1) {
continue;
}
// The data in any time series with the errant UID is
// a mashup of with all of the names. The best thing to do is
// start over. We'll rename the old time series so the user can
// still see it if they want to, but delete the forward mappings
// so that UIDs can be reassigned and clean series started.
// - concatenate all of the names into
// "fsck.<name1>.<name2>[...<nameN>]"
// - delete the forward mappings for all of the names
// - create a mapping with the fsck'd name pointing to the id
final StringBuilder fsck_builder = new StringBuilder("fsck");
final String id = collision.getKey();
// compile the new fsck'd name and remove each of the duplicate keys
for (String name : collision.getValue()) {
fsck_builder.append(".").append(name);
final DeleteRequest delete = new DeleteRequest(table, CliUtils.toBytes(name), CliUtils.ID_FAMILY, CliUtils.toBytes(kind));
client.delete(delete);
uids.name2id.remove(name);
LOG.info("FIX: Removed forward " + kind + " mapping for " + name + " -> " + id);
}
// write the new forward map
final String fsck_name = fsck_builder.toString();
final PutRequest put = new PutRequest(table, CliUtils.toBytes(fsck_name), CliUtils.ID_FAMILY, CliUtils.toBytes(kind), UniqueId.stringToUid(id));
client.put(put);
LOG.info("FIX: Created forward " + kind + " mapping for fsck'd UID " + fsck_name + " -> " + collision.getKey());
// we still need to fix the uids map for the reverse run through below
uids.name2id.put(fsck_name, collision.getKey());
uids.restoreReverseMap(kind, fsck_name, id);
LOG.error("----------------------------------");
LOG.error("- UID COLLISION DETECTED -");
LOG.error("Corrupted UID [" + collision.getKey() + "] renamed to [" + fsck_name + "]");
LOG.error("----------------------------------");
}
}
// These are harmless but shouldn't frequently occur.
for (final Map.Entry<String, String> idname : uids.id2name.entrySet()) {
final String name = idname.getValue();
final String id = idname.getKey();
final String found = uids.name2id.get(name);
if (found == null) {
LOG.warn("Reverse " + kind + " mapping is missing forward" + " mapping: " + name + " -> " + id);
if (fix) {
uids.removeReverseMap(kind, name, id);
}
} else if (!found.equals(id)) {
final String name2 = uids.id2name.get(found);
if (name2 != null) {
uids.error("Inconsistent reverse " + kind + " mapping " + id + " -> " + name + " vs " + found + " -> " + name + " / " + name2 + " -> " + found);
if (fix) {
uids.removeReverseMap(kind, name, id);
}
} else {
uids.error("Duplicate reverse " + kind + " mapping " + id + " -> " + name + " and " + found + " -> " + name2);
if (fix) {
uids.removeReverseMap(kind, name, id);
}
}
}
}
final int maxsize = Math.max(uids.id2name.size(), uids.name2id.size());
if (uids.maxid > maxsize) {
LOG.warn("Max ID for " + kind + " is " + uids.maxid + " but only " + maxsize + " entries were found. Maybe " + (uids.maxid - maxsize) + " IDs were deleted?");
} else if (uids.maxid < uids.max_found_id) {
uids.error("We found an ID of " + uids.max_found_id + " for " + kind + " but the max ID is only " + uids.maxid + "! Future IDs may be double-assigned!");
if (fix) {
// IDs than to under-run.
if (uids.max_found_id == Long.MAX_VALUE) {
LOG.error("Ran out of UIDs for " + kind + ". Unable to fix max ID");
} else {
final long diff = uids.max_found_id - uids.maxid;
final AtomicIncrementRequest air = new AtomicIncrementRequest(table, CliUtils.MAXID_ROW, CliUtils.ID_FAMILY, CliUtils.toBytes(kind), diff);
client.atomicIncrement(air);
LOG.info("FIX: Updated max ID for " + kind + " to " + uids.max_found_id);
}
}
}
if (uids.errors > 0) {
LOG.error(kind + ": Found " + uids.errors + " errors.");
errors += uids.errors;
}
}
final long timing = (System.nanoTime() - start_time) / 1000000;
LOG.info(kvcount + " KVs analyzed in " + timing + "ms (~" + (kvcount * 1000 / timing) + " KV/s)");
if (errors == 0) {
LOG.info("No errors found.");
return 0;
}
LOG.warn(errors + " errors found.");
return errors;
}
use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.
the class UidManager method grep.
/**
* Implements the {@code grep} subcommand.
* @param client The HBase client to use.
* @param table The name of the HBase table to use.
* @param ignorecase Whether or not to ignore the case while grepping.
* @param args Command line arguments ({@code [kind] RE}).
* @return The exit status of the command (0 means at least 1 match).
*/
private static int grep(final HBaseClient client, final byte[] table, final boolean ignorecase, final String[] args) {
final Scanner scanner = client.newScanner(table);
scanner.setMaxNumRows(1024);
String regexp;
scanner.setFamily(CliUtils.ID_FAMILY);
if (args.length == 3) {
scanner.setQualifier(CliUtils.toBytes(args[1]));
regexp = args[2];
} else {
regexp = args[1];
}
if (ignorecase) {
regexp = "(?i)" + regexp;
}
scanner.setKeyRegexp(regexp, CliUtils.CHARSET);
boolean found = false;
try {
ArrayList<ArrayList<KeyValue>> rows;
while ((rows = scanner.nextRows().joinUninterruptibly()) != null) {
for (final ArrayList<KeyValue> row : rows) {
found |= printResult(row, CliUtils.ID_FAMILY, true);
}
}
} catch (HBaseException e) {
LOG.error("Error while scanning HBase, scanner=" + scanner, e);
throw e;
} catch (Exception e) {
LOG.error("WTF? Unexpected exception type, scanner=" + scanner, e);
throw new AssertionError("Should never happen");
}
return found ? 0 : 1;
}
use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.
the class Branch method setupBranchScanner.
/**
* Configures an HBase scanner to fetch the requested branch and all child
* branches. It uses a row key regex filter to match any rows starting with
* the given branch and another INT_WIDTH bytes deep. Deeper branches are
* ignored.
* @param tsdb The TSDB to use for storage access
* @param branch_id ID of the branch to fetch
* @return An HBase scanner ready for scanning
*/
private static Scanner setupBranchScanner(final TSDB tsdb, final byte[] branch_id) {
final byte[] start = branch_id;
final byte[] end = Arrays.copyOf(branch_id, branch_id.length);
final Scanner scanner = tsdb.getClient().newScanner(tsdb.treeTable());
scanner.setStartKey(start);
// increment the tree ID so we scan the whole tree
byte[] tree_id = new byte[INT_WIDTH];
for (int i = 0; i < Tree.TREE_ID_WIDTH(); i++) {
tree_id[i + (INT_WIDTH - Tree.TREE_ID_WIDTH())] = end[i];
}
int id = Bytes.getInt(tree_id) + 1;
tree_id = Bytes.fromInt(id);
for (int i = 0; i < Tree.TREE_ID_WIDTH(); i++) {
end[i] = tree_id[i + (INT_WIDTH - Tree.TREE_ID_WIDTH())];
}
scanner.setStopKey(end);
scanner.setFamily(Tree.TREE_FAMILY());
// TODO - use the column filter to fetch only branches and leaves, ignore
// collisions, no matches and other meta
// set the regex filter
// we want one branch below the current ID so we want something like:
// {0, 1, 1, 2, 3, 4 } where { 0, 1 } is the tree ID, { 1, 2, 3, 4 } is the
// branch
// "^\\Q\000\001\001\002\003\004\\E(?:.{4})$"
final StringBuilder buf = new StringBuilder((start.length * 6) + 20);
buf.append(// Ensure we use the DOTALL flag.
"(?s)" + "^\\Q");
for (final byte b : start) {
buf.append((char) (b & 0xFF));
}
buf.append("\\E(?:.{").append(INT_WIDTH).append("})?$");
scanner.setKeyRegexp(buf.toString(), CHARSET);
return scanner;
}
Aggregations