Examples with ByteMap org.hbase.async.Bytes.ByteMap used on opensource projects

Example 1 with ByteMap

use of org.hbase.async.Bytes.ByteMap in project opentsdb by OpenTSDB.

the class IntersectionIterator method flattenTags.

/**
   * Flattens the appropriate tags into a single byte array
   * @param use_query_tags Whether or not to include tags returned with the
   * results or just use those group by'd in the query
   * @param include_agg_tags Whether or not to include the aggregated tags in
   * the identifier
   * @param tags The map of tags from the result set
   * @param agg_tags The list of aggregated tags
   * @param sub The sub query iterator
   * @return A byte array with the flattened tag keys and values. Note that
   * if the tags set is empty, this may return an empty array (but not a null
   * array)
   */
static byte[] flattenTags(final boolean use_query_tags, final boolean include_agg_tags, final ByteMap<byte[]> tags, final ByteSet agg_tags, final ITimeSyncedIterator sub) {
    if (tags.isEmpty()) {
        return HBaseClient.EMPTY_ARRAY;
    }
    final ByteSet query_tagks;
    // NOTE: We MAY need the agg tags but I'm not sure yet
    final int tag_size;
    if (use_query_tags) {
        int i = 0;
        if (sub.getQueryTagKs() != null && !sub.getQueryTagKs().isEmpty()) {
            query_tagks = sub.getQueryTagKs();
            for (final Map.Entry<byte[], byte[]> pair : tags.entrySet()) {
                if (query_tagks.contains(pair.getKey())) {
                    i++;
                }
            }
        } else {
            query_tagks = new ByteSet();
        }
        tag_size = i;
    } else {
        query_tagks = new ByteSet();
        tag_size = tags.size();
    }
    int len = (tag_size * (TSDB.tagk_width() + TSDB.tagv_width())) + (include_agg_tags ? (agg_tags.size() * TSDB.tagk_width()) : 0);
    final byte[] tagks = new byte[len];
    int i = 0;
    for (final Map.Entry<byte[], byte[]> pair : tags.entrySet()) {
        if (use_query_tags && !query_tagks.contains(pair.getKey())) {
            continue;
        }
        System.arraycopy(pair.getKey(), 0, tagks, i, TSDB.tagk_width());
        i += TSDB.tagk_width();
        System.arraycopy(pair.getValue(), 0, tagks, i, TSDB.tagv_width());
        i += TSDB.tagv_width();
    }
    if (include_agg_tags) {
        for (final byte[] tagk : agg_tags) {
            System.arraycopy(tagk, 0, tagks, i, TSDB.tagk_width());
            i += TSDB.tagk_width();
        }
    }
    return tagks;
}

Example 2 with ByteMap

use of org.hbase.async.Bytes.ByteMap in project opentsdb by OpenTSDB.

the class QueryRpc method handleLastDataPointQuery.

/**
   * Processes a last data point query
   * @param tsdb The TSDB to which we belong
   * @param query The HTTP query to parse/respond
   */
private void handleLastDataPointQuery(final TSDB tsdb, final HttpQuery query) {
    final LastPointQuery data_query;
    if (query.method() == HttpMethod.POST) {
        switch(query.apiVersion()) {
            case 0:
            case 1:
                data_query = query.serializer().parseLastPointQueryV1();
                break;
            default:
                throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, "Requested API version not implemented", "Version " + query.apiVersion() + " is not implemented");
        }
    } else {
        data_query = this.parseLastPointQuery(tsdb, query);
    }
    if (data_query.sub_queries == null || data_query.sub_queries.isEmpty()) {
        throw new BadRequestException(HttpResponseStatus.BAD_REQUEST, "Missing sub queries");
    }
    // a list of deferreds to wait on
    final ArrayList<Deferred<Object>> calls = new ArrayList<Deferred<Object>>();
    // final results for serialization
    final List<IncomingDataPoint> results = new ArrayList<IncomingDataPoint>();
    /**
     * Used to catch exceptions 
     */
    final class ErrBack implements Callback<Object, Exception> {

        public Object call(final Exception e) throws Exception {
            Throwable ex = e;
            while (ex.getClass().equals(DeferredGroupException.class)) {
                if (ex.getCause() == null) {
                    LOG.warn("Unable to get to the root cause of the DGE");
                    break;
                }
                ex = ex.getCause();
            }
            if (ex instanceof RuntimeException) {
                throw new BadRequestException(ex);
            } else {
                throw e;
            }
        }

        @Override
        public String toString() {
            return "Error back";
        }
    }
    final class FetchCB implements Callback<Deferred<Object>, ArrayList<IncomingDataPoint>> {

        @Override
        public Deferred<Object> call(final ArrayList<IncomingDataPoint> dps) throws Exception {
            synchronized (results) {
                for (final IncomingDataPoint dp : dps) {
                    if (dp != null) {
                        results.add(dp);
                    }
                }
            }
            return Deferred.fromResult(null);
        }

        @Override
        public String toString() {
            return "Fetched data points CB";
        }
    }
    /**
     * Called after scanning the tsdb-meta table for TSUIDs that match the given
     * metric and/or tags. If matches were found, it fires off a number of
     * getLastPoint requests, adding the deferreds to the calls list
     */
    final class TSUIDQueryCB implements Callback<Deferred<Object>, ByteMap<Long>> {

        public Deferred<Object> call(final ByteMap<Long> tsuids) throws Exception {
            if (tsuids == null || tsuids.isEmpty()) {
                return null;
            }
            final ArrayList<Deferred<IncomingDataPoint>> deferreds = new ArrayList<Deferred<IncomingDataPoint>>(tsuids.size());
            for (Map.Entry<byte[], Long> entry : tsuids.entrySet()) {
                deferreds.add(TSUIDQuery.getLastPoint(tsdb, entry.getKey(), data_query.getResolveNames(), data_query.getBackScan(), entry.getValue()));
            }
            return Deferred.group(deferreds).addCallbackDeferring(new FetchCB());
        }

        @Override
        public String toString() {
            return "TSMeta scan CB";
        }
    }
    /**
     * Used to wait on the list of data point deferreds. Once they're all done
     * this will return the results to the call via the serializer
     */
    final class FinalCB implements Callback<Object, ArrayList<Object>> {

        public Object call(final ArrayList<Object> done) throws Exception {
            query.sendReply(query.serializer().formatLastPointQueryV1(results));
            return null;
        }

        @Override
        public String toString() {
            return "Final CB";
        }
    }
    try {
        // start executing the queries
        for (final LastPointSubQuery sub_query : data_query.getQueries()) {
            final ArrayList<Deferred<IncomingDataPoint>> deferreds = new ArrayList<Deferred<IncomingDataPoint>>();
            // process the TSUIDs and ignore the metric/tags
            if (sub_query.getTSUIDs() != null && !sub_query.getTSUIDs().isEmpty()) {
                for (final String tsuid : sub_query.getTSUIDs()) {
                    final TSUIDQuery tsuid_query = new TSUIDQuery(tsdb, UniqueId.stringToUid(tsuid));
                    deferreds.add(tsuid_query.getLastPoint(data_query.getResolveNames(), data_query.getBackScan()));
                }
            } else {
                @SuppressWarnings("unchecked") final TSUIDQuery tsuid_query = new TSUIDQuery(tsdb, sub_query.getMetric(), sub_query.getTags() != null ? sub_query.getTags() : Collections.EMPTY_MAP);
                if (data_query.getBackScan() > 0) {
                    deferreds.add(tsuid_query.getLastPoint(data_query.getResolveNames(), data_query.getBackScan()));
                } else {
                    calls.add(tsuid_query.getLastWriteTimes().addCallbackDeferring(new TSUIDQueryCB()));
                }
            }
            if (deferreds.size() > 0) {
                calls.add(Deferred.group(deferreds).addCallbackDeferring(new FetchCB()));
            }
        }
        Deferred.group(calls).addCallback(new FinalCB()).addErrback(new ErrBack()).joinUninterruptibly();
    } catch (Exception e) {
        Throwable ex = e;
        while (ex.getClass().equals(DeferredGroupException.class)) {
            if (ex.getCause() == null) {
                LOG.warn("Unable to get to the root cause of the DGE");
                break;
            }
            ex = ex.getCause();
        }
        if (ex instanceof RuntimeException) {
            throw new BadRequestException(ex);
        } else {
            throw new RuntimeException("Shouldn't be here", e);
        }
    }
}

Example 3 with ByteMap

use of org.hbase.async.Bytes.ByteMap in project opentsdb by OpenTSDB.

the class UniqueIdRpc method getTSUIDForMetric.

/**
   * Parses a query string "m=metric{tagk1=tagv1,...}" type query and returns
   * a tsuid.
   * @param data_query The query we're building
   * @throws BadRequestException if we are unable to parse the query or it is
   * missing components
   * @todo - make this asynchronous
   */
private String getTSUIDForMetric(final String query_string, TSDB tsdb) {
    if (query_string == null || query_string.isEmpty()) {
        throw new BadRequestException("The query string was empty");
    }
    // m is of the following forms:
    // metric[{tag=value,...}]
    // where the parts in square brackets `[' .. `]' are optional.
    final HashMap<String, String> tags = new HashMap<String, String>();
    String metric = null;
    try {
        metric = Tags.parseWithMetric(query_string, tags);
    } catch (IllegalArgumentException e) {
        throw new BadRequestException(e);
    }
    // sort the UIDs on tagk values
    final ByteMap<byte[]> tag_uids = new ByteMap<byte[]>();
    for (final Entry<String, String> pair : tags.entrySet()) {
        tag_uids.put(tsdb.getUID(UniqueIdType.TAGK, pair.getKey()), tsdb.getUID(UniqueIdType.TAGV, pair.getValue()));
    }
    // Byte Buffer to generate TSUID, pre allocated to the size of the TSUID
    final ByteArrayOutputStream buf = new ByteArrayOutputStream(TSDB.metrics_width() + tag_uids.size() * (TSDB.tagk_width() + TSDB.tagv_width()));
    try {
        buf.write(tsdb.getUID(UniqueIdType.METRIC, metric));
        for (final Entry<byte[], byte[]> uids : tag_uids.entrySet()) {
            buf.write(uids.getKey());
            buf.write(uids.getValue());
        }
    } catch (IOException e) {
        throw new BadRequestException(e);
    }
    final String tsuid = UniqueId.uidToString(buf.toByteArray());
    return tsuid;
}

Example 4 with ByteMap

use of org.hbase.async.Bytes.ByteMap in project opentsdb by OpenTSDB.

the class TsdbQuery method findGroupBys.

/**
   * Populates the {@link #group_bys} and {@link #row_key_literals}'s with 
   * values pulled from the filters. 
   */
private void findGroupBys() {
    if (filters == null || filters.isEmpty()) {
        return;
    }
    row_key_literals = new ByteMap<byte[][]>();
    Collections.sort(filters);
    final Iterator<TagVFilter> current_iterator = filters.iterator();
    final Iterator<TagVFilter> look_ahead = filters.iterator();
    byte[] tagk = null;
    TagVFilter next = look_ahead.hasNext() ? look_ahead.next() : null;
    int row_key_literals_count = 0;
    while (current_iterator.hasNext()) {
        next = look_ahead.hasNext() ? look_ahead.next() : null;
        int gbs = 0;
        // sorted!
        final ByteMap<Void> literals = new ByteMap<Void>();
        final List<TagVFilter> literal_filters = new ArrayList<TagVFilter>();
        TagVFilter current = null;
        do {
            // yeah, I'm breakin out the do!!!
            current = current_iterator.next();
            if (tagk == null) {
                tagk = new byte[TSDB.tagk_width()];
                System.arraycopy(current.getTagkBytes(), 0, tagk, 0, TSDB.tagk_width());
            }
            if (current.isGroupBy()) {
                gbs++;
            }
            if (!current.getTagVUids().isEmpty()) {
                for (final byte[] uid : current.getTagVUids()) {
                    literals.put(uid, null);
                }
                literal_filters.add(current);
            }
            if (next != null && Bytes.memcmp(tagk, next.getTagkBytes()) != 0) {
                break;
            }
            next = look_ahead.hasNext() ? look_ahead.next() : null;
        } while (current_iterator.hasNext() && Bytes.memcmp(tagk, current.getTagkBytes()) == 0);
        if (gbs > 0) {
            if (group_bys == null) {
                group_bys = new ArrayList<byte[]>();
            }
            group_bys.add(current.getTagkBytes());
        }
        if (literals.size() > 0) {
            if (literals.size() + row_key_literals_count > tsdb.getConfig().getInt("tsd.query.filter.expansion_limit")) {
                LOG.debug("Skipping literals for " + current.getTagk() + " as it exceedes the limit");
            } else {
                final byte[][] values = new byte[literals.size()][];
                literals.keySet().toArray(values);
                row_key_literals.put(current.getTagkBytes(), values);
                row_key_literals_count += values.length;
                for (final TagVFilter filter : literal_filters) {
                    filter.setPostScan(false);
                }
            }
        } else {
            row_key_literals.put(current.getTagkBytes(), null);
        }
    }
}

Example 5 with ByteMap

use of org.hbase.async.Bytes.ByteMap in project opentsdb by OpenTSDB.

the class TSUIDQuery method getLastWriteTimes.

/**
   * Fetches a list of TSUIDs given the metric and optional tag pairs. The query
   * format is similar to TsdbQuery but doesn't support grouping operators for 
   * tags. Only TSUIDs that had "ts_counter" qualifiers will be returned.
   * <p>
   * NOTE: If you called {@link #setQuery(String, Map)} successfully this will
   * immediately scan the meta table. But if you used the CTOR to set the
   * metric and tags it will attempt to resolve those and may return an exception.
   * @return A map of TSUIDs to the last timestamp (in milliseconds) when the
   * "ts_counter" was updated. Note that the timestamp will be the time stored
   * by HBase, not the actual timestamp of the data point. If nothing was
   * found, the map will be empty but not null.
   * @throws IllegalArgumentException if the metric was not set or the tag map
   * was null
   */
public Deferred<ByteMap<Long>> getLastWriteTimes() {
    class ResolutionCB implements Callback<Deferred<ByteMap<Long>>, Object> {

        @Override
        public Deferred<ByteMap<Long>> call(Object arg0) throws Exception {
            final Scanner scanner = getScanner();
            scanner.setQualifier(TSMeta.COUNTER_QUALIFIER());
            final Deferred<ByteMap<Long>> results = new Deferred<ByteMap<Long>>();
            final ByteMap<Long> tsuids = new ByteMap<Long>();
            final class ErrBack implements Callback<Object, Exception> {

                @Override
                public Object call(final Exception e) throws Exception {
                    results.callback(e);
                    return null;
                }

                @Override
                public String toString() {
                    return "Error callback";
                }
            }
            /**
         * Scanner callback that will call itself while iterating through the 
         * tsdb-meta table
         */
            final class ScannerCB implements Callback<Object, ArrayList<ArrayList<KeyValue>>> {

                /**
           * Starts the scanner and is called recursively to fetch the next set of
           * rows from the scanner.
           * @return The map of spans if loaded successfully, null if no data was
           * found
           */
                public Object scan() {
                    return scanner.nextRows().addCallback(this).addErrback(new ErrBack());
                }

                /**
           * Loops through each row of the scanner results and parses out data
           * points and optional meta data
           * @return null if no rows were found, otherwise the TreeMap with spans
           */
                @Override
                public Object call(final ArrayList<ArrayList<KeyValue>> rows) throws Exception {
                    try {
                        if (rows == null) {
                            results.callback(tsuids);
                            return null;
                        }
                        for (final ArrayList<KeyValue> row : rows) {
                            final byte[] tsuid = row.get(0).key();
                            tsuids.put(tsuid, row.get(0).timestamp());
                        }
                        return scan();
                    } catch (Exception e) {
                        results.callback(e);
                        return null;
                    }
                }
            }
            new ScannerCB().scan();
            return results;
        }

        @Override
        public String toString() {
            return "Last counter time callback";
        }
    }
    if (metric_uid == null) {
        return resolveMetric().addCallbackDeferring(new ResolutionCB());
    }
    try {
        return new ResolutionCB().call(null);
    } catch (Exception e) {
        return Deferred.fromError(e);
    }
}