use of org.jgrapht.experimental.dag.DirectedAcyclicGraph in project opentsdb by OpenTSDB.
the class QueryExecutor method execute.
/**
* Execute the RPC and serialize the response
* @param query The HTTP query to parse and and return results to
*/
public void execute(final HttpQuery query) {
http_query = query;
final QueryStats query_stats = new QueryStats(query.getRemoteAddress(), ts_query, query.getHeaders());
ts_query.setQueryStats(query_stats);
/**
* Sends the serialized results to the caller. This should be the very
* last callback executed.
*/
class CompleteCB implements Callback<Object, ChannelBuffer> {
@Override
public Object call(final ChannelBuffer cb) throws Exception {
query.sendReply(cb);
return null;
}
}
/**
* After all of the queries have run and we have data (or not) then we
* need to compile the iterators.
* This class could probably be improved:
* First we iterate over the results AND for each result, iterate over
* the expressions, giving a time synced iterator to each expression that
* needs the result set.
* THEN we iterate over the expressions again and build a DAG to determine
* if any of the expressions require the output of an expression. If so
* then we add the expressions to the proper parent and compile them in
* order.
* After all of that we're ready to start serializing and iterating
* over the results.
*/
class QueriesCB implements Callback<Object, ArrayList<DataPoints[]>> {
public Object call(final ArrayList<DataPoints[]> query_results) throws Exception {
for (int i = 0; i < query_results.size(); i++) {
final TSSubQuery sub = ts_query.getQueries().get(i);
Iterator<Entry<String, TSSubQuery>> it = sub_queries.entrySet().iterator();
while (it.hasNext()) {
final Entry<String, TSSubQuery> entry = it.next();
if (entry.getValue().equals(sub)) {
sub_query_results.put(entry.getKey(), query_results.get(i));
for (final ExpressionIterator ei : expressions.values()) {
if (ei.getVariableNames().contains(entry.getKey())) {
final TimeSyncedIterator tsi = new TimeSyncedIterator(entry.getKey(), sub.getFilterTagKs(), query_results.get(i));
final NumericFillPolicy fill = fills.get(entry.getKey());
if (fill != null) {
tsi.setFillPolicy(fill);
}
ei.addResults(entry.getKey(), tsi);
if (LOG.isDebugEnabled()) {
LOG.debug("Added results for " + entry.getKey() + " to " + ei.getId());
}
}
}
}
}
}
// handle nested expressions
final DirectedAcyclicGraph<String, DefaultEdge> graph = new DirectedAcyclicGraph<String, DefaultEdge>(DefaultEdge.class);
for (final Entry<String, ExpressionIterator> eii : expressions.entrySet()) {
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("Expression entry key is %s, value is %s", eii.getKey(), eii.getValue().toString()));
LOG.debug(String.format("Time to loop through the variable names " + "for %s", eii.getKey()));
}
if (!graph.containsVertex(eii.getKey())) {
if (LOG.isDebugEnabled()) {
LOG.debug("Adding vertex " + eii.getKey());
}
graph.addVertex(eii.getKey());
}
for (final String var : eii.getValue().getVariableNames()) {
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("var is %s", var));
}
final ExpressionIterator ei = expressions.get(var);
if (ei != null) {
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("The expression iterator for %s is %s", var, ei.toString()));
}
// TODO - really ought to calculate this earlier
if (eii.getKey().equals(var)) {
throw new IllegalArgumentException("Self referencing expression found: " + eii.getKey());
}
if (LOG.isDebugEnabled()) {
LOG.debug("Nested expression detected. " + eii.getKey() + " depends on " + var);
}
if (!graph.containsVertex(eii.getKey())) {
if (LOG.isDebugEnabled()) {
LOG.debug("Added vertex " + eii.getKey());
}
graph.addVertex(eii.getKey());
} else if (LOG.isDebugEnabled()) {
LOG.debug("Already contains vertex " + eii.getKey());
}
if (!graph.containsVertex(var)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Added vertex " + var);
}
graph.addVertex(var);
} else if (LOG.isDebugEnabled()) {
LOG.debug("Already contains vertex " + var);
}
try {
if (LOG.isDebugEnabled()) {
LOG.debug("Added Edge " + eii.getKey() + " - " + var);
}
graph.addDagEdge(eii.getKey(), var);
} catch (CycleFoundException cfe) {
throw new IllegalArgumentException("Circular reference found: " + eii.getKey(), cfe);
}
} else if (LOG.isDebugEnabled()) {
LOG.debug(String.format("The expression iterator for %s is null", var));
}
}
}
// compile all of the expressions
final long intersect_start = DateTime.currentTimeMillis();
final Integer expressionLength = expressions.size();
final ExpressionIterator[] compile_stack = new ExpressionIterator[expressionLength];
final TopologicalOrderIterator<String, DefaultEdge> it = new TopologicalOrderIterator<String, DefaultEdge>(graph);
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("Expressions Size is %d", expressionLength));
LOG.debug(String.format("Topology Iterator %s", it.toString()));
}
int i = 0;
while (it.hasNext()) {
String next = it.next();
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("Expression: %s", next));
}
ExpressionIterator ei = expressions.get(next);
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("Expression Iterator: %s", ei.toString()));
}
if (ei == null) {
LOG.error(String.format("The expression iterator for %s is null", next));
}
compile_stack[i] = ei;
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("Added expression %s to compile_stack[%d]", next, i));
}
i++;
}
if (i != expressionLength) {
throw new IOException(String.format(" Internal Error: Fewer " + "expressions where added to the compile stack than " + "expressions.size (%d instead of %d)", i, expressionLength));
}
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("compile stack length: %d", compile_stack.length));
}
for (int x = compile_stack.length - 1; x >= 0; x--) {
if (compile_stack[x] == null) {
throw new NullPointerException(String.format("Item %d in " + "compile_stack[] is null", x));
}
// look for and add expressions
for (final String var : compile_stack[x].getVariableNames()) {
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("Looking for variable %s for %s", var, compile_stack[x].getId()));
}
ExpressionIterator source = expressions.get(var);
if (source != null) {
compile_stack[x].addResults(var, source.getCopy());
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("Adding expression %s to %s", source.getId(), compile_stack[x].getId()));
}
}
}
compile_stack[x].compile();
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("Successfully compiled %s", compile_stack[x].getId()));
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("Finished compilations in " + (DateTime.currentTimeMillis() - intersect_start) + " ms");
}
return serialize().addCallback(new CompleteCB()).addErrback(new ErrorCB());
}
}
/**
* Callback executed after we have resolved the metric, tag names and tag
* values to their respective UIDs. This callback then runs the actual
* queries and fetches their results.
*/
class BuildCB implements Callback<Deferred<Object>, net.opentsdb.core.Query[]> {
@Override
public Deferred<Object> call(final net.opentsdb.core.Query[] queries) {
final ArrayList<Deferred<DataPoints[]>> deferreds = new ArrayList<Deferred<DataPoints[]>>(queries.length);
for (final net.opentsdb.core.Query query : queries) {
deferreds.add(query.runAsync());
}
return Deferred.groupInOrder(deferreds).addCallback(new QueriesCB()).addErrback(new ErrorCB());
}
}
// TODO - only run the ones that will be involved in an output. Folks WILL
// ask for stuff they don't need.... *sigh*
ts_query.buildQueriesAsync(tsdb).addCallback(new BuildCB()).addErrback(new ErrorCB());
}
Aggregations