use of com.stumbleupon.async.Deferred in project opentsdb by OpenTSDB.
the class QueryRpc method handleQuery.
/**
* Processing for a data point query
* @param tsdb The TSDB to which we belong
* @param query The HTTP query to parse/respond
* @param allow_expressions Whether or not expressions should be parsed
* (based on the endpoint)
*/
private void handleQuery(final TSDB tsdb, final HttpQuery query, final boolean allow_expressions) {
final long start = DateTime.currentTimeMillis();
final TSQuery data_query;
final List<ExpressionTree> expressions;
if (query.method() == HttpMethod.POST) {
switch(query.apiVersion()) {
case 0:
case 1:
data_query = query.serializer().parseQueryV1();
break;
default:
query_invalid.incrementAndGet();
throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, "Requested API version not implemented", "Version " + query.apiVersion() + " is not implemented");
}
expressions = null;
} else {
expressions = new ArrayList<ExpressionTree>();
data_query = parseQuery(tsdb, query, expressions);
}
if (query.getAPIMethod() == HttpMethod.DELETE && tsdb.getConfig().getBoolean("tsd.http.query.allow_delete")) {
data_query.setDelete(true);
}
// validate and then compile the queries
try {
LOG.debug(data_query.toString());
data_query.validateAndSetQuery();
} catch (Exception e) {
throw new BadRequestException(HttpResponseStatus.BAD_REQUEST, e.getMessage(), data_query.toString(), e);
}
// if the user tried this query multiple times from the same IP and src port
// they'll be rejected on subsequent calls
final QueryStats query_stats = new QueryStats(query.getRemoteAddress(), data_query, query.getPrintableHeaders());
data_query.setQueryStats(query_stats);
query.setStats(query_stats);
final int nqueries = data_query.getQueries().size();
final ArrayList<DataPoints[]> results = new ArrayList<DataPoints[]>(nqueries);
final List<Annotation> globals = new ArrayList<Annotation>();
/** This has to be attached to callbacks or we may never respond to clients */
class ErrorCB implements Callback<Object, Exception> {
public Object call(final Exception e) throws Exception {
Throwable ex = e;
try {
LOG.error("Query exception: ", e);
if (ex instanceof DeferredGroupException) {
ex = e.getCause();
while (ex != null && ex instanceof DeferredGroupException) {
ex = ex.getCause();
}
if (ex == null) {
LOG.error("The deferred group exception didn't have a cause???");
}
}
if (ex instanceof RpcTimedOutException) {
query_stats.markSerialized(HttpResponseStatus.REQUEST_TIMEOUT, ex);
query.badRequest(new BadRequestException(HttpResponseStatus.REQUEST_TIMEOUT, ex.getMessage()));
query_exceptions.incrementAndGet();
} else if (ex instanceof HBaseException) {
query_stats.markSerialized(HttpResponseStatus.FAILED_DEPENDENCY, ex);
query.badRequest(new BadRequestException(HttpResponseStatus.FAILED_DEPENDENCY, ex.getMessage()));
query_exceptions.incrementAndGet();
} else if (ex instanceof QueryException) {
query_stats.markSerialized(((QueryException) ex).getStatus(), ex);
query.badRequest(new BadRequestException(((QueryException) ex).getStatus(), ex.getMessage()));
query_exceptions.incrementAndGet();
} else if (ex instanceof BadRequestException) {
query_stats.markSerialized(((BadRequestException) ex).getStatus(), ex);
query.badRequest((BadRequestException) ex);
query_invalid.incrementAndGet();
} else if (ex instanceof NoSuchUniqueName) {
query_stats.markSerialized(HttpResponseStatus.BAD_REQUEST, ex);
query.badRequest(new BadRequestException(ex));
query_invalid.incrementAndGet();
} else {
query_stats.markSerialized(HttpResponseStatus.INTERNAL_SERVER_ERROR, ex);
query.badRequest(new BadRequestException(ex));
query_exceptions.incrementAndGet();
}
} catch (RuntimeException ex2) {
LOG.error("Exception thrown during exception handling", ex2);
query_stats.markSerialized(HttpResponseStatus.INTERNAL_SERVER_ERROR, ex2);
query.sendReply(HttpResponseStatus.INTERNAL_SERVER_ERROR, ex2.getMessage().getBytes());
query_exceptions.incrementAndGet();
}
return null;
}
}
/**
* After all of the queries have run, we get the results in the order given
* and add dump the results in an array
*/
class QueriesCB implements Callback<Object, ArrayList<DataPoints[]>> {
public Object call(final ArrayList<DataPoints[]> query_results) throws Exception {
if (allow_expressions) {
// process each of the expressions into a new list, then merge it
// with the original. This avoids possible recursion loops.
final List<DataPoints[]> expression_results = new ArrayList<DataPoints[]>(expressions.size());
// let exceptions bubble up
for (final ExpressionTree expression : expressions) {
expression_results.add(expression.evaluate(query_results));
}
results.addAll(expression_results);
} else {
results.addAll(query_results);
}
/** Simply returns the buffer once serialization is complete and logs it */
class SendIt implements Callback<Object, ChannelBuffer> {
public Object call(final ChannelBuffer buffer) throws Exception {
query.sendReply(buffer);
query_success.incrementAndGet();
return null;
}
}
switch(query.apiVersion()) {
case 0:
case 1:
query.serializer().formatQueryAsyncV1(data_query, results, globals).addCallback(new SendIt()).addErrback(new ErrorCB());
break;
default:
query_invalid.incrementAndGet();
throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, "Requested API version not implemented", "Version " + query.apiVersion() + " is not implemented");
}
return null;
}
}
/**
* Callback executed after we have resolved the metric, tag names and tag
* values to their respective UIDs. This callback then runs the actual
* queries and fetches their results.
*/
class BuildCB implements Callback<Deferred<Object>, Query[]> {
@Override
public Deferred<Object> call(final Query[] queries) {
final ArrayList<Deferred<DataPoints[]>> deferreds = new ArrayList<Deferred<DataPoints[]>>(queries.length);
for (final Query query : queries) {
deferreds.add(query.runAsync());
}
return Deferred.groupInOrder(deferreds).addCallback(new QueriesCB());
}
}
/** Handles storing the global annotations after fetching them */
class GlobalCB implements Callback<Object, List<Annotation>> {
public Object call(final List<Annotation> annotations) throws Exception {
globals.addAll(annotations);
return data_query.buildQueriesAsync(tsdb).addCallback(new BuildCB());
}
}
// when complete
if (!data_query.getNoAnnotations() && data_query.getGlobalAnnotations()) {
Annotation.getGlobalAnnotations(tsdb, data_query.startTime() / 1000, data_query.endTime() / 1000).addCallback(new GlobalCB()).addErrback(new ErrorCB());
} else {
data_query.buildQueriesAsync(tsdb).addCallback(new BuildCB()).addErrback(new ErrorCB());
}
}
use of com.stumbleupon.async.Deferred in project opentsdb by OpenTSDB.
the class QueryRpc method handleLastDataPointQuery.
/**
* Processes a last data point query
* @param tsdb The TSDB to which we belong
* @param query The HTTP query to parse/respond
*/
private void handleLastDataPointQuery(final TSDB tsdb, final HttpQuery query) {
final LastPointQuery data_query;
if (query.method() == HttpMethod.POST) {
switch(query.apiVersion()) {
case 0:
case 1:
data_query = query.serializer().parseLastPointQueryV1();
break;
default:
throw new BadRequestException(HttpResponseStatus.NOT_IMPLEMENTED, "Requested API version not implemented", "Version " + query.apiVersion() + " is not implemented");
}
} else {
data_query = this.parseLastPointQuery(tsdb, query);
}
if (data_query.sub_queries == null || data_query.sub_queries.isEmpty()) {
throw new BadRequestException(HttpResponseStatus.BAD_REQUEST, "Missing sub queries");
}
// a list of deferreds to wait on
final ArrayList<Deferred<Object>> calls = new ArrayList<Deferred<Object>>();
// final results for serialization
final List<IncomingDataPoint> results = new ArrayList<IncomingDataPoint>();
/**
* Used to catch exceptions
*/
final class ErrBack implements Callback<Object, Exception> {
public Object call(final Exception e) throws Exception {
Throwable ex = e;
while (ex.getClass().equals(DeferredGroupException.class)) {
if (ex.getCause() == null) {
LOG.warn("Unable to get to the root cause of the DGE");
break;
}
ex = ex.getCause();
}
if (ex instanceof RuntimeException) {
throw new BadRequestException(ex);
} else {
throw e;
}
}
@Override
public String toString() {
return "Error back";
}
}
final class FetchCB implements Callback<Deferred<Object>, ArrayList<IncomingDataPoint>> {
@Override
public Deferred<Object> call(final ArrayList<IncomingDataPoint> dps) throws Exception {
synchronized (results) {
for (final IncomingDataPoint dp : dps) {
if (dp != null) {
results.add(dp);
}
}
}
return Deferred.fromResult(null);
}
@Override
public String toString() {
return "Fetched data points CB";
}
}
/**
* Called after scanning the tsdb-meta table for TSUIDs that match the given
* metric and/or tags. If matches were found, it fires off a number of
* getLastPoint requests, adding the deferreds to the calls list
*/
final class TSUIDQueryCB implements Callback<Deferred<Object>, ByteMap<Long>> {
public Deferred<Object> call(final ByteMap<Long> tsuids) throws Exception {
if (tsuids == null || tsuids.isEmpty()) {
return null;
}
final ArrayList<Deferred<IncomingDataPoint>> deferreds = new ArrayList<Deferred<IncomingDataPoint>>(tsuids.size());
for (Map.Entry<byte[], Long> entry : tsuids.entrySet()) {
deferreds.add(TSUIDQuery.getLastPoint(tsdb, entry.getKey(), data_query.getResolveNames(), data_query.getBackScan(), entry.getValue()));
}
return Deferred.group(deferreds).addCallbackDeferring(new FetchCB());
}
@Override
public String toString() {
return "TSMeta scan CB";
}
}
/**
* Used to wait on the list of data point deferreds. Once they're all done
* this will return the results to the call via the serializer
*/
final class FinalCB implements Callback<Object, ArrayList<Object>> {
public Object call(final ArrayList<Object> done) throws Exception {
query.sendReply(query.serializer().formatLastPointQueryV1(results));
return null;
}
@Override
public String toString() {
return "Final CB";
}
}
try {
// start executing the queries
for (final LastPointSubQuery sub_query : data_query.getQueries()) {
final ArrayList<Deferred<IncomingDataPoint>> deferreds = new ArrayList<Deferred<IncomingDataPoint>>();
// process the TSUIDs and ignore the metric/tags
if (sub_query.getTSUIDs() != null && !sub_query.getTSUIDs().isEmpty()) {
for (final String tsuid : sub_query.getTSUIDs()) {
final TSUIDQuery tsuid_query = new TSUIDQuery(tsdb, UniqueId.stringToUid(tsuid));
deferreds.add(tsuid_query.getLastPoint(data_query.getResolveNames(), data_query.getBackScan()));
}
} else {
@SuppressWarnings("unchecked") final TSUIDQuery tsuid_query = new TSUIDQuery(tsdb, sub_query.getMetric(), sub_query.getTags() != null ? sub_query.getTags() : Collections.EMPTY_MAP);
if (data_query.getBackScan() > 0) {
deferreds.add(tsuid_query.getLastPoint(data_query.getResolveNames(), data_query.getBackScan()));
} else {
calls.add(tsuid_query.getLastWriteTimes().addCallbackDeferring(new TSUIDQueryCB()));
}
}
if (deferreds.size() > 0) {
calls.add(Deferred.group(deferreds).addCallbackDeferring(new FetchCB()));
}
}
Deferred.group(calls).addCallback(new FinalCB()).addErrback(new ErrBack()).joinUninterruptibly();
} catch (Exception e) {
Throwable ex = e;
while (ex.getClass().equals(DeferredGroupException.class)) {
if (ex.getCause() == null) {
LOG.warn("Unable to get to the root cause of the DGE");
break;
}
ex = ex.getCause();
}
if (ex instanceof RuntimeException) {
throw new BadRequestException(ex);
} else {
throw new RuntimeException("Shouldn't be here", e);
}
}
}
use of com.stumbleupon.async.Deferred in project opentsdb by OpenTSDB.
the class SearchRpc method processLookup.
/**
* Processes a lookup query against the tsdb-meta table, returning (and
* resolving) the TSUIDs of any series that matched the query.
* @param tsdb The TSDB to which we belong
* @param query The HTTP query to work with
* @param search_query A search query configured with at least a metric
* or a list of tag pairs. If neither are set, the method will throw an error.
* @throws BadRequestException if the metric and tags are null or empty or
* a UID fails to resolve.
* @since 2.1
*/
private void processLookup(final TSDB tsdb, final HttpQuery query, final SearchQuery search_query) {
if (search_query.getMetric() == null && (search_query.getTags() == null || search_query.getTags().size() < 1)) {
throw new BadRequestException("Missing metric and tags. Please supply at least one value.");
}
final long start = System.currentTimeMillis();
class MetricCB implements Callback<Object, String> {
final Map<String, Object> series;
MetricCB(final Map<String, Object> series) {
this.series = series;
}
@Override
public Object call(final String name) throws Exception {
series.put("metric", name);
return null;
}
}
class TagsCB implements Callback<Object, HashMap<String, String>> {
final Map<String, Object> series;
TagsCB(final Map<String, Object> series) {
this.series = series;
}
@Override
public Object call(final HashMap<String, String> names) throws Exception {
series.put("tags", names);
return null;
}
}
class Serialize implements Callback<Object, ArrayList<Object>> {
final List<Object> results;
Serialize(final List<Object> results) {
this.results = results;
}
@Override
public Object call(final ArrayList<Object> ignored) throws Exception {
search_query.setResults(results);
search_query.setTime(System.currentTimeMillis() - start);
query.sendReply(query.serializer().formatSearchResultsV1(search_query));
return null;
}
}
class LookupCB implements Callback<Deferred<Object>, List<byte[]>> {
@Override
public Deferred<Object> call(final List<byte[]> tsuids) throws Exception {
final List<Object> results = new ArrayList<Object>(tsuids.size());
search_query.setTotalResults(tsuids.size());
final ArrayList<Deferred<Object>> deferreds = new ArrayList<Deferred<Object>>(tsuids.size());
for (final byte[] tsuid : tsuids) {
// has to be concurrent if the uid table is split across servers
final Map<String, Object> series = new ConcurrentHashMap<String, Object>(3);
results.add(series);
series.put("tsuid", UniqueId.uidToString(tsuid));
byte[] metric_uid = Arrays.copyOfRange(tsuid, 0, TSDB.metrics_width());
deferreds.add(tsdb.getUidName(UniqueIdType.METRIC, metric_uid).addCallback(new MetricCB(series)));
final List<byte[]> tag_ids = UniqueId.getTagPairsFromTSUID(tsuid);
deferreds.add(Tags.resolveIdsAsync(tsdb, tag_ids).addCallback(new TagsCB(series)));
}
return Deferred.group(deferreds).addCallback(new Serialize(results));
}
}
class ErrCB implements Callback<Object, Exception> {
@Override
public Object call(final Exception e) throws Exception {
if (e instanceof NoSuchUniqueId) {
query.sendReply(HttpResponseStatus.NOT_FOUND, query.serializer().formatErrorV1(new BadRequestException(HttpResponseStatus.NOT_FOUND, "Unable to resolve one or more TSUIDs", (NoSuchUniqueId) e)));
} else if (e instanceof NoSuchUniqueName) {
query.sendReply(HttpResponseStatus.NOT_FOUND, query.serializer().formatErrorV1(new BadRequestException(HttpResponseStatus.NOT_FOUND, "Unable to resolve one or more UIDs", (NoSuchUniqueName) e)));
} else if (e instanceof DeferredGroupException) {
final Throwable ex = Exceptions.getCause((DeferredGroupException) e);
if (ex instanceof NoSuchUniqueId) {
query.sendReply(HttpResponseStatus.NOT_FOUND, query.serializer().formatErrorV1(new BadRequestException(HttpResponseStatus.NOT_FOUND, "Unable to resolve one or more TSUIDs", (NoSuchUniqueId) ex)));
} else if (ex instanceof NoSuchUniqueName) {
query.sendReply(HttpResponseStatus.NOT_FOUND, query.serializer().formatErrorV1(new BadRequestException(HttpResponseStatus.NOT_FOUND, "Unable to resolve one or more UIDs", (NoSuchUniqueName) ex)));
} else {
query.sendReply(HttpResponseStatus.INTERNAL_SERVER_ERROR, query.serializer().formatErrorV1(new BadRequestException(HttpResponseStatus.INTERNAL_SERVER_ERROR, "Unexpected exception", ex)));
}
} else {
query.sendReply(HttpResponseStatus.INTERNAL_SERVER_ERROR, query.serializer().formatErrorV1(new BadRequestException(HttpResponseStatus.INTERNAL_SERVER_ERROR, "Unexpected exception", e)));
}
return null;
}
}
new TimeSeriesLookup(tsdb, search_query).lookupAsync().addCallback(new LookupCB()).addErrback(new ErrCB());
}
use of com.stumbleupon.async.Deferred in project opentsdb by OpenTSDB.
the class BaseTsdbTest method setupTagvMaps.
/** Adds the static UIDs to the tag values UID mock object */
void setupTagvMaps() {
when(tag_values.getId(TAGV_STRING)).thenReturn(TAGV_BYTES);
when(tag_values.getOrCreateId(TAGV_STRING)).thenReturn(TAGV_BYTES);
when(tag_values.getIdAsync(TAGV_STRING)).thenAnswer(new Answer<Deferred<byte[]>>() {
@Override
public Deferred<byte[]> answer(InvocationOnMock invocation) throws Throwable {
return Deferred.fromResult(TAGV_BYTES);
}
});
when(tag_values.getOrCreateIdAsync(TAGV_STRING)).thenReturn(Deferred.fromResult(TAGV_BYTES));
when(tag_values.getId(TAGV_B_STRING)).thenReturn(TAGV_B_BYTES);
when(tag_values.getOrCreateId(TAGV_B_STRING)).thenReturn(TAGV_B_BYTES);
when(tag_values.getIdAsync(TAGV_B_STRING)).thenAnswer(new Answer<Deferred<byte[]>>() {
@Override
public Deferred<byte[]> answer(InvocationOnMock invocation) throws Throwable {
return Deferred.fromResult(TAGV_B_BYTES);
}
});
when(tag_values.getOrCreateIdAsync(TAGV_B_STRING)).thenReturn(Deferred.fromResult(TAGV_B_BYTES));
when(tag_values.getNameAsync(TAGV_BYTES)).thenReturn(Deferred.fromResult(TAGV_STRING));
when(tag_values.getNameAsync(TAGV_B_BYTES)).thenReturn(Deferred.fromResult(TAGV_B_STRING));
when(tag_values.getNameAsync(NSUI_TAGV)).thenThrow(new NoSuchUniqueId("tagv", NSUI_TAGV));
final NoSuchUniqueName nsun = new NoSuchUniqueName(NSUN_TAGV, "tagv");
when(tag_values.getId(NSUN_TAGV)).thenThrow(nsun);
when(tag_values.getIdAsync(NSUN_TAGV)).thenReturn(Deferred.<byte[]>fromError(nsun));
// Iterate over the tagv UIDs and handle both forward and reverse
for (final Map.Entry<String, byte[]> uid : TAGV_UIDS.entrySet()) {
when(tag_values.getId(uid.getKey())).thenReturn(uid.getValue());
when(tag_values.getIdAsync(uid.getKey())).thenAnswer(new Answer<Deferred<byte[]>>() {
@Override
public Deferred<byte[]> answer(InvocationOnMock invocation) throws Throwable {
return Deferred.fromResult(uid.getValue());
}
});
when(tag_values.getOrCreateId(uid.getKey())).thenReturn(uid.getValue());
when(tag_values.getNameAsync(uid.getValue())).thenAnswer(new Answer<Deferred<String>>() {
@Override
public Deferred<String> answer(InvocationOnMock invocation) throws Throwable {
return Deferred.fromResult(uid.getKey());
}
});
}
}
use of com.stumbleupon.async.Deferred in project opentsdb by OpenTSDB.
the class BaseTsdbTest method setupMetricMaps.
/** Adds the static UIDs to the metrics UID mock object */
void setupMetricMaps() {
when(metrics.getId(METRIC_STRING)).thenReturn(METRIC_BYTES);
when(metrics.getIdAsync(METRIC_STRING)).thenAnswer(new Answer<Deferred<byte[]>>() {
@Override
public Deferred<byte[]> answer(InvocationOnMock invocation) throws Throwable {
return Deferred.fromResult(METRIC_BYTES);
}
});
when(metrics.getOrCreateId(METRIC_STRING)).thenReturn(METRIC_BYTES);
when(metrics.getId(METRIC_B_STRING)).thenReturn(METRIC_B_BYTES);
when(metrics.getIdAsync(METRIC_B_STRING)).thenAnswer(new Answer<Deferred<byte[]>>() {
@Override
public Deferred<byte[]> answer(InvocationOnMock invocation) throws Throwable {
return Deferred.fromResult(METRIC_B_BYTES);
}
});
when(metrics.getOrCreateId(METRIC_B_STRING)).thenReturn(METRIC_B_BYTES);
when(metrics.getNameAsync(METRIC_BYTES)).thenAnswer(new Answer<Deferred<String>>() {
@Override
public Deferred<String> answer(InvocationOnMock invocation) throws Throwable {
return Deferred.fromResult(METRIC_STRING);
}
});
when(metrics.getNameAsync(METRIC_B_BYTES)).thenAnswer(new Answer<Deferred<String>>() {
@Override
public Deferred<String> answer(InvocationOnMock invocation) throws Throwable {
return Deferred.fromResult(METRIC_B_STRING);
}
});
when(metrics.getNameAsync(NSUI_METRIC)).thenThrow(new NoSuchUniqueId("metrics", NSUI_METRIC));
final NoSuchUniqueName nsun = new NoSuchUniqueName(NSUN_METRIC, "metrics");
when(metrics.getId(NSUN_METRIC)).thenThrow(nsun);
when(metrics.getIdAsync(NSUN_METRIC)).thenReturn(Deferred.<byte[]>fromError(nsun));
when(metrics.getOrCreateId(NSUN_METRIC)).thenThrow(nsun);
// Iterate over the metric UIDs and handle both forward and reverse
for (final Map.Entry<String, byte[]> uid : METRIC_UIDS.entrySet()) {
when(metrics.getId(uid.getKey())).thenReturn(uid.getValue());
when(metrics.getIdAsync(uid.getKey())).thenAnswer(new Answer<Deferred<byte[]>>() {
@Override
public Deferred<byte[]> answer(InvocationOnMock invocation) throws Throwable {
return Deferred.fromResult(uid.getValue());
}
});
when(metrics.getOrCreateId(uid.getKey())).thenReturn(uid.getValue());
when(metrics.getNameAsync(uid.getValue())).thenAnswer(new Answer<Deferred<String>>() {
@Override
public Deferred<String> answer(InvocationOnMock invocation) throws Throwable {
return Deferred.fromResult(uid.getKey());
}
});
}
}
Aggregations