Examples with DeferredGroupException com.stumbleupon.async.DeferredGroupException used on opensource projects

Example 1 with DeferredGroupException

use of com.stumbleupon.async.DeferredGroupException in project opentsdb by OpenTSDB.

the class TSDB method shutdown.

/**
   * Gracefully shuts down this TSD instance.
   * <p>
   * The method must call {@code shutdown()} on all plugins as well as flush the
   * compaction queue.
   * @return A {@link Deferred} that will be called once all the un-committed
   * data has been successfully and durably stored, and all resources used by
   * this instance have been released.  The value of the deferred object
   * return is meaningless and unspecified, and can be {@code null}.
   * @throws HBaseException (deferred) if there was a problem sending
   * un-committed data to HBase.  Please refer to the {@link HBaseException}
   * hierarchy to handle the possible failures.  Some of them are easily
   * recoverable by retrying, some are not.
   */
public Deferred<Object> shutdown() {
    final ArrayList<Deferred<Object>> deferreds = new ArrayList<Deferred<Object>>();
    final class FinalShutdown implements Callback<Object, Object> {

        @Override
        public Object call(Object result) throws Exception {
            if (result instanceof Exception) {
                LOG.error("A previous shutdown failed", (Exception) result);
            }
            final Set<Timeout> timeouts = timer.stop();
            // TODO - at some point we should clean these up.
            if (timeouts.size() > 0) {
                LOG.warn("There were " + timeouts.size() + " timer tasks queued");
            }
            LOG.info("Completed shutting down the TSDB");
            return Deferred.fromResult(null);
        }
    }
    final class SEHShutdown implements Callback<Object, Object> {

        @Override
        public Object call(Object result) throws Exception {
            if (result instanceof Exception) {
                LOG.error("Shutdown of the HBase client failed", (Exception) result);
            }
            LOG.info("Shutting down storage exception handler plugin: " + storage_exception_handler.getClass().getCanonicalName());
            return storage_exception_handler.shutdown().addBoth(new FinalShutdown());
        }

        @Override
        public String toString() {
            return "SEHShutdown";
        }
    }
    final class HClientShutdown implements Callback<Deferred<Object>, ArrayList<Object>> {

        public Deferred<Object> call(final ArrayList<Object> args) {
            if (storage_exception_handler != null) {
                return client.shutdown().addBoth(new SEHShutdown());
            }
            return client.shutdown().addBoth(new FinalShutdown());
        }

        public String toString() {
            return "shutdown HBase client";
        }
    }
    final class ShutdownErrback implements Callback<Object, Exception> {

        public Object call(final Exception e) {
            final Logger LOG = LoggerFactory.getLogger(ShutdownErrback.class);
            if (e instanceof DeferredGroupException) {
                final DeferredGroupException ge = (DeferredGroupException) e;
                for (final Object r : ge.results()) {
                    if (r instanceof Exception) {
                        LOG.error("Failed to shutdown the TSD", (Exception) r);
                    }
                }
            } else {
                LOG.error("Failed to shutdown the TSD", e);
            }
            return new HClientShutdown().call(null);
        }

        public String toString() {
            return "shutdown HBase client after error";
        }
    }
    final class CompactCB implements Callback<Object, ArrayList<Object>> {

        public Object call(ArrayList<Object> compactions) throws Exception {
            return null;
        }
    }
    if (config.enable_compactions()) {
        LOG.info("Flushing compaction queue");
        deferreds.add(compactionq.flush().addCallback(new CompactCB()));
    }
    if (startup != null) {
        LOG.info("Shutting down startup plugin: " + startup.getClass().getCanonicalName());
        deferreds.add(startup.shutdown());
    }
    if (search != null) {
        LOG.info("Shutting down search plugin: " + search.getClass().getCanonicalName());
        deferreds.add(search.shutdown());
    }
    if (rt_publisher != null) {
        LOG.info("Shutting down RT plugin: " + rt_publisher.getClass().getCanonicalName());
        deferreds.add(rt_publisher.shutdown());
    }
    if (meta_cache != null) {
        LOG.info("Shutting down meta cache plugin: " + meta_cache.getClass().getCanonicalName());
        deferreds.add(meta_cache.shutdown());
    }
    if (storage_exception_handler != null) {
        LOG.info("Shutting down storage exception handler plugin: " + storage_exception_handler.getClass().getCanonicalName());
        deferreds.add(storage_exception_handler.shutdown());
    }
    if (ts_filter != null) {
        LOG.info("Shutting down time series filter plugin: " + ts_filter.getClass().getCanonicalName());
        deferreds.add(ts_filter.shutdown());
    }
    if (uid_filter != null) {
        LOG.info("Shutting down UID filter plugin: " + uid_filter.getClass().getCanonicalName());
        deferreds.add(uid_filter.shutdown());
    }
    // wait for plugins to shutdown before we close the client
    return deferreds.size() > 0 ? Deferred.group(deferreds).addCallbackDeferring(new HClientShutdown()).addErrback(new ShutdownErrback()) : new HClientShutdown().call(null);
}

Example 2 with DeferredGroupException

use of com.stumbleupon.async.DeferredGroupException in project opentsdb by OpenTSDB.

the class QueryRpc method handleQuery.

/**
   * Processing for a data point query
   * @param tsdb The TSDB to which we belong
   * @param query The HTTP query to parse/respond
   * @param allow_expressions Whether or not expressions should be parsed
   * (based on the endpoint)
   */
private void handleQuery(final TSDB tsdb, final HttpQuery query, final boolean allow_expressions) {
    final long start = DateTime.currentTimeMillis();
    final TSQuery data_query;
    final List<ExpressionTree> expressions;
    if (query.method() == HttpMethod.POST) {
        switch(query.apiVersion()) {
            case 0:
            case 1:
                data_query = query.serializer().parseQueryV1();
                break;
            default:
                query_invalid.incrementAndGet();
                throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, "Requested API version not implemented", "Version " + query.apiVersion() + " is not implemented");
        }
        expressions = null;
    } else {
        expressions = new ArrayList<ExpressionTree>();
        data_query = parseQuery(tsdb, query, expressions);
    }
    if (query.getAPIMethod() == HttpMethod.DELETE && tsdb.getConfig().getBoolean("tsd.http.query.allow_delete")) {
        data_query.setDelete(true);
    }
    // validate and then compile the queries
    try {
        LOG.debug(data_query.toString());
        data_query.validateAndSetQuery();
    } catch (Exception e) {
        throw new BadRequestException(HttpResponseStatus.BAD_REQUEST, e.getMessage(), data_query.toString(), e);
    }
    // if the user tried this query multiple times from the same IP and src port
    // they'll be rejected on subsequent calls
    final QueryStats query_stats = new QueryStats(query.getRemoteAddress(), data_query, query.getPrintableHeaders());
    data_query.setQueryStats(query_stats);
    query.setStats(query_stats);
    final int nqueries = data_query.getQueries().size();
    final ArrayList<DataPoints[]> results = new ArrayList<DataPoints[]>(nqueries);
    final List<Annotation> globals = new ArrayList<Annotation>();
    /** This has to be attached to callbacks or we may never respond to clients */
    class ErrorCB implements Callback<Object, Exception> {

        public Object call(final Exception e) throws Exception {
            Throwable ex = e;
            try {
                LOG.error("Query exception: ", e);
                if (ex instanceof DeferredGroupException) {
                    ex = e.getCause();
                    while (ex != null && ex instanceof DeferredGroupException) {
                        ex = ex.getCause();
                    }
                    if (ex == null) {
                        LOG.error("The deferred group exception didn't have a cause???");
                    }
                }
                if (ex instanceof RpcTimedOutException) {
                    query_stats.markSerialized(HttpResponseStatus.REQUEST_TIMEOUT, ex);
                    query.badRequest(new BadRequestException(HttpResponseStatus.REQUEST_TIMEOUT, ex.getMessage()));
                    query_exceptions.incrementAndGet();
                } else if (ex instanceof HBaseException) {
                    query_stats.markSerialized(HttpResponseStatus.FAILED_DEPENDENCY, ex);
                    query.badRequest(new BadRequestException(HttpResponseStatus.FAILED_DEPENDENCY, ex.getMessage()));
                    query_exceptions.incrementAndGet();
                } else if (ex instanceof QueryException) {
                    query_stats.markSerialized(((QueryException) ex).getStatus(), ex);
                    query.badRequest(new BadRequestException(((QueryException) ex).getStatus(), ex.getMessage()));
                    query_exceptions.incrementAndGet();
                } else if (ex instanceof BadRequestException) {
                    query_stats.markSerialized(((BadRequestException) ex).getStatus(), ex);
                    query.badRequest((BadRequestException) ex);
                    query_invalid.incrementAndGet();
                } else if (ex instanceof NoSuchUniqueName) {
                    query_stats.markSerialized(HttpResponseStatus.BAD_REQUEST, ex);
                    query.badRequest(new BadRequestException(ex));
                    query_invalid.incrementAndGet();
                } else {
                    query_stats.markSerialized(HttpResponseStatus.INTERNAL_SERVER_ERROR, ex);
                    query.badRequest(new BadRequestException(ex));
                    query_exceptions.incrementAndGet();
                }
            } catch (RuntimeException ex2) {
                LOG.error("Exception thrown during exception handling", ex2);
                query_stats.markSerialized(HttpResponseStatus.INTERNAL_SERVER_ERROR, ex2);
                query.sendReply(HttpResponseStatus.INTERNAL_SERVER_ERROR, ex2.getMessage().getBytes());
                query_exceptions.incrementAndGet();
            }
            return null;
        }
    }
    /**
     * After all of the queries have run, we get the results in the order given
     * and add dump the results in an array
     */
    class QueriesCB implements Callback<Object, ArrayList<DataPoints[]>> {

        public Object call(final ArrayList<DataPoints[]> query_results) throws Exception {
            if (allow_expressions) {
                // process each of the expressions into a new list, then merge it
                // with the original. This avoids possible recursion loops.
                final List<DataPoints[]> expression_results = new ArrayList<DataPoints[]>(expressions.size());
                // let exceptions bubble up
                for (final ExpressionTree expression : expressions) {
                    expression_results.add(expression.evaluate(query_results));
                }
                results.addAll(expression_results);
            } else {
                results.addAll(query_results);
            }
            /** Simply returns the buffer once serialization is complete and logs it */
            class SendIt implements Callback<Object, ChannelBuffer> {

                public Object call(final ChannelBuffer buffer) throws Exception {
                    query.sendReply(buffer);
                    query_success.incrementAndGet();
                    return null;
                }
            }
            switch(query.apiVersion()) {
                case 0:
                case 1:
                    query.serializer().formatQueryAsyncV1(data_query, results, globals).addCallback(new SendIt()).addErrback(new ErrorCB());
                    break;
                default:
                    query_invalid.incrementAndGet();
                    throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, "Requested API version not implemented", "Version " + query.apiVersion() + " is not implemented");
            }
            return null;
        }
    }
    /**
     * Callback executed after we have resolved the metric, tag names and tag
     * values to their respective UIDs. This callback then runs the actual 
     * queries and fetches their results.
     */
    class BuildCB implements Callback<Deferred<Object>, Query[]> {

        @Override
        public Deferred<Object> call(final Query[] queries) {
            final ArrayList<Deferred<DataPoints[]>> deferreds = new ArrayList<Deferred<DataPoints[]>>(queries.length);
            for (final Query query : queries) {
                deferreds.add(query.runAsync());
            }
            return Deferred.groupInOrder(deferreds).addCallback(new QueriesCB());
        }
    }
    /** Handles storing the global annotations after fetching them */
    class GlobalCB implements Callback<Object, List<Annotation>> {

        public Object call(final List<Annotation> annotations) throws Exception {
            globals.addAll(annotations);
            return data_query.buildQueriesAsync(tsdb).addCallback(new BuildCB());
        }
    }
    // when complete
    if (!data_query.getNoAnnotations() && data_query.getGlobalAnnotations()) {
        Annotation.getGlobalAnnotations(tsdb, data_query.startTime() / 1000, data_query.endTime() / 1000).addCallback(new GlobalCB()).addErrback(new ErrorCB());
    } else {
        data_query.buildQueriesAsync(tsdb).addCallback(new BuildCB()).addErrback(new ErrorCB());
    }
}

Example 3 with DeferredGroupException

use of com.stumbleupon.async.DeferredGroupException in project opentsdb by OpenTSDB.

the class SearchRpc method processLookup.

/**
   * Processes a lookup query against the tsdb-meta table, returning (and 
   * resolving) the TSUIDs of any series that matched the query.
   * @param tsdb The TSDB to which we belong
   * @param query The HTTP query to work with
   * @param search_query A search query configured with at least a metric
   * or a list of tag pairs. If neither are set, the method will throw an error.
   * @throws BadRequestException if the metric and tags are null or empty or
   * a UID fails to resolve.
   * @since 2.1
   */
private void processLookup(final TSDB tsdb, final HttpQuery query, final SearchQuery search_query) {
    if (search_query.getMetric() == null && (search_query.getTags() == null || search_query.getTags().size() < 1)) {
        throw new BadRequestException("Missing metric and tags. Please supply at least one value.");
    }
    final long start = System.currentTimeMillis();
    class MetricCB implements Callback<Object, String> {

        final Map<String, Object> series;

        MetricCB(final Map<String, Object> series) {
            this.series = series;
        }

        @Override
        public Object call(final String name) throws Exception {
            series.put("metric", name);
            return null;
        }
    }
    class TagsCB implements Callback<Object, HashMap<String, String>> {

        final Map<String, Object> series;

        TagsCB(final Map<String, Object> series) {
            this.series = series;
        }

        @Override
        public Object call(final HashMap<String, String> names) throws Exception {
            series.put("tags", names);
            return null;
        }
    }
    class Serialize implements Callback<Object, ArrayList<Object>> {

        final List<Object> results;

        Serialize(final List<Object> results) {
            this.results = results;
        }

        @Override
        public Object call(final ArrayList<Object> ignored) throws Exception {
            search_query.setResults(results);
            search_query.setTime(System.currentTimeMillis() - start);
            query.sendReply(query.serializer().formatSearchResultsV1(search_query));
            return null;
        }
    }
    class LookupCB implements Callback<Deferred<Object>, List<byte[]>> {

        @Override
        public Deferred<Object> call(final List<byte[]> tsuids) throws Exception {
            final List<Object> results = new ArrayList<Object>(tsuids.size());
            search_query.setTotalResults(tsuids.size());
            final ArrayList<Deferred<Object>> deferreds = new ArrayList<Deferred<Object>>(tsuids.size());
            for (final byte[] tsuid : tsuids) {
                // has to be concurrent if the uid table is split across servers
                final Map<String, Object> series = new ConcurrentHashMap<String, Object>(3);
                results.add(series);
                series.put("tsuid", UniqueId.uidToString(tsuid));
                byte[] metric_uid = Arrays.copyOfRange(tsuid, 0, TSDB.metrics_width());
                deferreds.add(tsdb.getUidName(UniqueIdType.METRIC, metric_uid).addCallback(new MetricCB(series)));
                final List<byte[]> tag_ids = UniqueId.getTagPairsFromTSUID(tsuid);
                deferreds.add(Tags.resolveIdsAsync(tsdb, tag_ids).addCallback(new TagsCB(series)));
            }
            return Deferred.group(deferreds).addCallback(new Serialize(results));
        }
    }
    class ErrCB implements Callback<Object, Exception> {

        @Override
        public Object call(final Exception e) throws Exception {
            if (e instanceof NoSuchUniqueId) {
                query.sendReply(HttpResponseStatus.NOT_FOUND, query.serializer().formatErrorV1(new BadRequestException(HttpResponseStatus.NOT_FOUND, "Unable to resolve one or more TSUIDs", (NoSuchUniqueId) e)));
            } else if (e instanceof NoSuchUniqueName) {
                query.sendReply(HttpResponseStatus.NOT_FOUND, query.serializer().formatErrorV1(new BadRequestException(HttpResponseStatus.NOT_FOUND, "Unable to resolve one or more UIDs", (NoSuchUniqueName) e)));
            } else if (e instanceof DeferredGroupException) {
                final Throwable ex = Exceptions.getCause((DeferredGroupException) e);
                if (ex instanceof NoSuchUniqueId) {
                    query.sendReply(HttpResponseStatus.NOT_FOUND, query.serializer().formatErrorV1(new BadRequestException(HttpResponseStatus.NOT_FOUND, "Unable to resolve one or more TSUIDs", (NoSuchUniqueId) ex)));
                } else if (ex instanceof NoSuchUniqueName) {
                    query.sendReply(HttpResponseStatus.NOT_FOUND, query.serializer().formatErrorV1(new BadRequestException(HttpResponseStatus.NOT_FOUND, "Unable to resolve one or more UIDs", (NoSuchUniqueName) ex)));
                } else {
                    query.sendReply(HttpResponseStatus.INTERNAL_SERVER_ERROR, query.serializer().formatErrorV1(new BadRequestException(HttpResponseStatus.INTERNAL_SERVER_ERROR, "Unexpected exception", ex)));
                }
            } else {
                query.sendReply(HttpResponseStatus.INTERNAL_SERVER_ERROR, query.serializer().formatErrorV1(new BadRequestException(HttpResponseStatus.INTERNAL_SERVER_ERROR, "Unexpected exception", e)));
            }
            return null;
        }
    }
    new TimeSeriesLookup(tsdb, search_query).lookupAsync().addCallback(new LookupCB()).addErrback(new ErrCB());
}

Example 4 with DeferredGroupException

use of com.stumbleupon.async.DeferredGroupException in project opentsdb by OpenTSDB.

the class TreeRpc method handleTest.

/**
   * Runs the specified TSMeta object through a tree's rule set to determine
   * what the results would be or debug a meta that wasn't added to a tree
   * successfully
   * @param tsdb The TSDB to which we belong
   * @param query The HTTP query to work with
   * @throws BadRequestException if the request was invalid.
   */
private void handleTest(TSDB tsdb, HttpQuery query) {
    final Map<String, Object> map;
    if (query.hasContent()) {
        map = query.serializer().parseTreeTSUIDsListV1();
    } else {
        map = parseTSUIDsList(query);
    }
    final Integer tree_id = (Integer) map.get("treeId");
    if (tree_id == null) {
        throw new BadRequestException("Missing or invalid Tree ID");
    }
    // make sure the tree exists
    Tree tree = null;
    try {
        tree = Tree.fetchTree(tsdb, tree_id).joinUninterruptibly();
        if (tree == null) {
            throw new BadRequestException(HttpResponseStatus.NOT_FOUND, "Unable to locate tree: " + tree_id);
        }
        // ugly, but keeps from having to create a dedicated class just to 
        // convert one field.
        @SuppressWarnings("unchecked") final List<String> tsuids = (List<String>) map.get("tsuids");
        if (tsuids == null || tsuids.isEmpty()) {
            throw new BadRequestException("Missing or empty TSUID list");
        }
        if (query.getAPIMethod() == HttpMethod.GET || query.getAPIMethod() == HttpMethod.POST || query.getAPIMethod() == HttpMethod.PUT) {
            final HashMap<String, HashMap<String, Object>> results = new HashMap<String, HashMap<String, Object>>(tsuids.size());
            final TreeBuilder builder = new TreeBuilder(tsdb, tree);
            for (String tsuid : tsuids) {
                final HashMap<String, Object> tsuid_results = new HashMap<String, Object>();
                try {
                    final TSMeta meta = TSMeta.getTSMeta(tsdb, tsuid).joinUninterruptibly();
                    // message to the results and move on to the next TSUID
                    if (meta == null) {
                        tsuid_results.put("branch", null);
                        tsuid_results.put("meta", null);
                        final ArrayList<String> messages = new ArrayList<String>(1);
                        messages.add("Unable to locate TSUID meta data");
                        tsuid_results.put("messages", messages);
                        results.put(tsuid, tsuid_results);
                        continue;
                    }
                    builder.processTimeseriesMeta(meta, true).joinUninterruptibly();
                    tsuid_results.put("branch", builder.getRootBranch());
                    tsuid_results.put("meta", meta);
                    tsuid_results.put("messages", builder.getTestMessage());
                    results.put(tsuid, tsuid_results);
                } catch (DeferredGroupException e) {
                    // we want to catch NSU errors and handle them gracefully for
                    // TSUIDs where they may have been deleted
                    Throwable ex = e;
                    while (ex.getClass().equals(DeferredGroupException.class)) {
                        ex = ex.getCause();
                    }
                    if (ex.getClass().equals(NoSuchUniqueId.class)) {
                        tsuid_results.put("branch", null);
                        tsuid_results.put("meta", null);
                        final ArrayList<String> messages = new ArrayList<String>(1);
                        messages.add("TSUID was missing a UID name: " + ex.getMessage());
                        tsuid_results.put("messages", messages);
                        results.put(tsuid, tsuid_results);
                    }
                }
            }
            query.sendReply(query.serializer().formatTreeTestV1(results));
        } else {
            throw new BadRequestException(HttpResponseStatus.BAD_REQUEST, "Unsupported HTTP request method");
        }
    } catch (BadRequestException e) {
        throw e;
    } catch (IllegalArgumentException e) {
        throw new BadRequestException(e);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 5 with DeferredGroupException

use of com.stumbleupon.async.DeferredGroupException in project opentsdb by OpenTSDB.

the class TestQueryRpc method postQueryNoMetricBadRequest.

@Test
public void postQueryNoMetricBadRequest() throws Exception {
    final DeferredGroupException dge = mock(DeferredGroupException.class);
    when(dge.getCause()).thenReturn(new NoSuchUniqueName("foo", "metrics"));
    when(query_result.configureFromQuery((TSQuery) any(), anyInt())).thenReturn(Deferred.fromError(dge));
    HttpQuery query = NettyMocks.postQuery(tsdb, "/api/query", "{\"start\":1425440315306,\"queries\":" + "[{\"metric\":\"nonexistent\",\"aggregator\":\"sum\",\"rate\":true," + "\"rateOptions\":{\"counter\":false}}]}");
    rpc.execute(tsdb, query);
    assertEquals(HttpResponseStatus.BAD_REQUEST, query.response().getStatus());
    final String json = query.response().getContent().toString(Charset.forName("UTF-8"));
    assertTrue(json.contains("No such name for 'foo': 'metrics'"));
}