Examples with KeyValue org.hbase.async.KeyValue used on opensource projects

Example 96 with KeyValue

use of org.hbase.async.KeyValue in project opentsdb by OpenTSDB.

the class TSMeta method storeIfNecessary.

/**
   * Attempts to fetch the meta column and if null, attempts to write a new 
   * column using {@link #storeNew}.
   * @param tsdb The TSDB instance to use for access.
   * @param tsuid The TSUID of the time series.
   * @return A deferred with a true if the meta exists or was created, false
   * if the meta did not exist and writing failed.
   */
public static Deferred<Boolean> storeIfNecessary(final TSDB tsdb, final byte[] tsuid) {
    final GetRequest get = new GetRequest(tsdb.metaTable(), tsuid);
    get.family(FAMILY);
    get.qualifier(META_QUALIFIER);
    final class CreateNewCB implements Callback<Deferred<Boolean>, Object> {

        @Override
        public Deferred<Boolean> call(Object arg0) throws Exception {
            final TSMeta meta = new TSMeta(tsuid, System.currentTimeMillis() / 1000);
            final class FetchNewCB implements Callback<Deferred<Boolean>, TSMeta> {

                @Override
                public Deferred<Boolean> call(TSMeta stored_meta) throws Exception {
                    // pass to the search plugin
                    tsdb.indexTSMeta(stored_meta);
                    // pass through the trees
                    tsdb.processTSMetaThroughTrees(stored_meta);
                    return Deferred.fromResult(true);
                }
            }
            final class StoreNewCB implements Callback<Deferred<Boolean>, Boolean> {

                @Override
                public Deferred<Boolean> call(Boolean success) throws Exception {
                    if (!success) {
                        LOG.warn("Unable to save metadata: " + meta);
                        return Deferred.fromResult(false);
                    }
                    LOG.info("Successfullly created new TSUID entry for: " + meta);
                    return new LoadUIDs(tsdb, UniqueId.uidToString(tsuid)).call(meta).addCallbackDeferring(new FetchNewCB());
                }
            }
            return meta.storeNew(tsdb).addCallbackDeferring(new StoreNewCB());
        }
    }
    final class ExistsCB implements Callback<Deferred<Boolean>, ArrayList<KeyValue>> {

        @Override
        public Deferred<Boolean> call(ArrayList<KeyValue> row) throws Exception {
            if (row == null || row.isEmpty() || row.get(0).value() == null) {
                return new CreateNewCB().call(null);
            }
            return Deferred.fromResult(true);
        }
    }
    return tsdb.getClient().get(get).addCallbackDeferring(new ExistsCB());
}

Example 97 with KeyValue

use of org.hbase.async.KeyValue in project opentsdb by OpenTSDB.

the class TSMeta method getFromStorage.

/**
   * Attempts to fetch the timeseries meta data from storage. 
   * This method will fetch the {@code counter} and {@code meta} columns.
   * <b>Note:</b> This method will not load the UIDMeta objects.
   * @param tsdb The TSDB to use for storage access
   * @param tsuid The UID of the meta to fetch
   * @return A TSMeta object if found, null if not
   * @throws HBaseException if there was an issue fetching
   * @throws IllegalArgumentException if parsing failed
   * @throws JSONException if the data was corrupted
   */
private static Deferred<TSMeta> getFromStorage(final TSDB tsdb, final byte[] tsuid) {
    /**
     * Called after executing the GetRequest to parse the meta data.
     */
    final class GetCB implements Callback<Deferred<TSMeta>, ArrayList<KeyValue>> {

        /**
       * @return Null if the meta did not exist or a valid TSMeta object if it
       * did.
       */
        @Override
        public Deferred<TSMeta> call(final ArrayList<KeyValue> row) throws Exception {
            if (row == null || row.isEmpty()) {
                return Deferred.fromResult(null);
            }
            long dps = 0;
            long last_received = 0;
            TSMeta meta = null;
            for (KeyValue column : row) {
                if (Arrays.equals(COUNTER_QUALIFIER, column.qualifier())) {
                    dps = Bytes.getLong(column.value());
                    last_received = column.timestamp() / 1000;
                } else if (Arrays.equals(META_QUALIFIER, column.qualifier())) {
                    meta = JSON.parseToObject(column.value(), TSMeta.class);
                }
            }
            if (meta == null) {
                LOG.warn("Found a counter TSMeta column without a meta for TSUID: " + UniqueId.uidToString(row.get(0).key()));
                return Deferred.fromResult(null);
            }
            meta.total_dps = dps;
            meta.last_received = last_received;
            return Deferred.fromResult(meta);
        }
    }
    final GetRequest get = new GetRequest(tsdb.metaTable(), tsuid);
    get.family(FAMILY);
    get.qualifiers(new byte[][] { COUNTER_QUALIFIER, META_QUALIFIER });
    return tsdb.getClient().get(get).addCallbackDeferring(new GetCB());
}

Example 98 with KeyValue

use of org.hbase.async.KeyValue in project opentsdb by OpenTSDB.

the class TSMeta method metaExistsInStorage.

/**
   * Determines if an entry exists in storage or not. 
   * This is used by the UID Manager tool to determine if we need to write a 
   * new TSUID entry or not. It will not attempt to verify if the stored data is 
   * valid, just checks to see if something is stored in the proper column.
   * @param tsdb  The TSDB to use for storage access
   * @param tsuid The UID of the meta to verify
   * @return True if data was found, false if not
   * @throws HBaseException if there was an issue fetching
   */
public static Deferred<Boolean> metaExistsInStorage(final TSDB tsdb, final String tsuid) {
    final GetRequest get = new GetRequest(tsdb.metaTable(), UniqueId.stringToUid(tsuid));
    get.family(FAMILY);
    get.qualifier(META_QUALIFIER);
    /**
     * Callback from the GetRequest that simply determines if the row is empty
     * or not
     */
    final class ExistsCB implements Callback<Boolean, ArrayList<KeyValue>> {

        @Override
        public Boolean call(ArrayList<KeyValue> row) throws Exception {
            if (row == null || row.isEmpty() || row.get(0).value() == null) {
                return false;
            }
            return true;
        }
    }
    return tsdb.getClient().get(get).addCallback(new ExistsCB());
}

Example 99 with KeyValue

use of org.hbase.async.KeyValue in project opentsdb by OpenTSDB.

the class TSUIDQuery method getLastWriteTimes.

/**
   * Fetches a list of TSUIDs given the metric and optional tag pairs. The query
   * format is similar to TsdbQuery but doesn't support grouping operators for 
   * tags. Only TSUIDs that had "ts_counter" qualifiers will be returned.
   * <p>
   * NOTE: If you called {@link #setQuery(String, Map)} successfully this will
   * immediately scan the meta table. But if you used the CTOR to set the
   * metric and tags it will attempt to resolve those and may return an exception.
   * @return A map of TSUIDs to the last timestamp (in milliseconds) when the
   * "ts_counter" was updated. Note that the timestamp will be the time stored
   * by HBase, not the actual timestamp of the data point. If nothing was
   * found, the map will be empty but not null.
   * @throws IllegalArgumentException if the metric was not set or the tag map
   * was null
   */
public Deferred<ByteMap<Long>> getLastWriteTimes() {
    class ResolutionCB implements Callback<Deferred<ByteMap<Long>>, Object> {

        @Override
        public Deferred<ByteMap<Long>> call(Object arg0) throws Exception {
            final Scanner scanner = getScanner();
            scanner.setQualifier(TSMeta.COUNTER_QUALIFIER());
            final Deferred<ByteMap<Long>> results = new Deferred<ByteMap<Long>>();
            final ByteMap<Long> tsuids = new ByteMap<Long>();
            final class ErrBack implements Callback<Object, Exception> {

                @Override
                public Object call(final Exception e) throws Exception {
                    results.callback(e);
                    return null;
                }

                @Override
                public String toString() {
                    return "Error callback";
                }
            }
            /**
         * Scanner callback that will call itself while iterating through the 
         * tsdb-meta table
         */
            final class ScannerCB implements Callback<Object, ArrayList<ArrayList<KeyValue>>> {

                /**
           * Starts the scanner and is called recursively to fetch the next set of
           * rows from the scanner.
           * @return The map of spans if loaded successfully, null if no data was
           * found
           */
                public Object scan() {
                    return scanner.nextRows().addCallback(this).addErrback(new ErrBack());
                }

                /**
           * Loops through each row of the scanner results and parses out data
           * points and optional meta data
           * @return null if no rows were found, otherwise the TreeMap with spans
           */
                @Override
                public Object call(final ArrayList<ArrayList<KeyValue>> rows) throws Exception {
                    try {
                        if (rows == null) {
                            results.callback(tsuids);
                            return null;
                        }
                        for (final ArrayList<KeyValue> row : rows) {
                            final byte[] tsuid = row.get(0).key();
                            tsuids.put(tsuid, row.get(0).timestamp());
                        }
                        return scan();
                    } catch (Exception e) {
                        results.callback(e);
                        return null;
                    }
                }
            }
            new ScannerCB().scan();
            return results;
        }

        @Override
        public String toString() {
            return "Last counter time callback";
        }
    }
    if (metric_uid == null) {
        return resolveMetric().addCallbackDeferring(new ResolutionCB());
    }
    try {
        return new ResolutionCB().call(null);
    } catch (Exception e) {
        return Deferred.fromError(e);
    }
}

Example 100 with KeyValue

use of org.hbase.async.KeyValue in project opentsdb by OpenTSDB.

the class SaltScanner method mergeAndReturnResults.

/**
   * Called once all of the scanners have reported back in to record our
   * latency and merge the results into the spans map. If there was an exception
   * stored then we'll return that instead.
   */
private void mergeAndReturnResults() {
    final long hbase_time = System.currentTimeMillis();
    TsdbQuery.scanlatency.add((int) (hbase_time - start_time));
    long rows = 0;
    if (exception != null) {
        LOG.error("After all of the scanners finished, at " + "least one threw an exception", exception);
        results.callback(exception);
        return;
    }
    // Merge sorted spans together
    final long merge_start = DateTime.nanoTime();
    for (final List<KeyValue> kvs : kv_map.values()) {
        if (kvs == null || kvs.isEmpty()) {
            LOG.warn("Found a key value list that was null or empty");
            continue;
        }
        for (final KeyValue kv : kvs) {
            if (kv == null) {
                LOG.warn("Found a key value item that was null");
                continue;
            }
            if (kv.key() == null) {
                LOG.warn("A key for a kv was null");
                continue;
            }
            Span datapoints = spans.get(kv.key());
            if (datapoints == null) {
                datapoints = new Span(tsdb);
                spans.put(kv.key(), datapoints);
            }
            if (annotation_map.containsKey(kv.key())) {
                for (final Annotation note : annotation_map.get(kv.key())) {
                    datapoints.getAnnotations().add(note);
                }
                annotation_map.remove(kv.key());
            }
            try {
                datapoints.addRow(kv);
                rows++;
            } catch (RuntimeException e) {
                LOG.error("Exception adding row to span", e);
                throw e;
            }
        }
    }
    kv_map.clear();
    for (final byte[] key : annotation_map.keySet()) {
        Span datapoints = spans.get(key);
        if (datapoints == null) {
            datapoints = new Span(tsdb);
            spans.put(key, datapoints);
        }
        for (final Annotation note : annotation_map.get(key)) {
            datapoints.getAnnotations().add(note);
        }
    }
    if (query_stats != null) {
        query_stats.addStat(query_index, QueryStat.SCANNER_MERGE_TIME, (DateTime.nanoTime() - merge_start));
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Scanning completed in " + (hbase_time - start_time) + " ms, " + rows + " rows, and stored in " + spans.size() + " spans");
        LOG.debug("It took " + (System.currentTimeMillis() - hbase_time) + " ms, " + " to merge and sort the rows into a tree map");
    }
    results.callback(spans);
}