use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.
the class TaskCommunicatorContextImpl method getVertexCompletedTaskCount.
@Override
public int getVertexCompletedTaskCount(String vertexName) {
Preconditions.checkArgument(vertexName != null, "VertexName must be specified");
DAG dag = getDag();
Vertex vertex = dag.getVertex(vertexName);
return vertex.getCompletedTasks();
}
use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.
the class TaskCommunicatorContextImpl method getInputVertexNames.
@Override
public Iterable<String> getInputVertexNames(String vertexName) {
Preconditions.checkNotNull(vertexName, "VertexName cannot be null: " + vertexName);
DAG dag = getDag();
Vertex vertex = dag.getVertex(vertexName);
Set<Vertex> sources = vertex.getInputVertices().keySet();
return Iterables.transform(sources, new Function<Vertex, String>() {
@Override
public String apply(Vertex input) {
return input.getName();
}
});
}
use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.
the class TaskCommunicatorContextImpl method getVertexRunningTaskCount.
@Override
public int getVertexRunningTaskCount(String vertexName) {
Preconditions.checkArgument(vertexName != null, "VertexName must be specified");
DAG dag = getDag();
Vertex vertex = dag.getVertex(vertexName);
return vertex.getRunningTasks();
}
use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.
the class TaskCommunicatorContextImpl method getVertexTotalTaskCount.
@Override
public int getVertexTotalTaskCount(String vertexName) {
Preconditions.checkArgument(vertexName != null, "VertexName must be specified");
DAG dag = getDag();
Vertex vertex = dag.getVertex(vertexName);
return vertex.getTotalTasks();
}
use of org.apache.tez.dag.app.dag.Vertex in project hive by apache.
the class LlapTaskSchedulerService method getTransitiveVertexOutputs.
private static Map<Integer, Set<Integer>> getTransitiveVertexOutputs(DagInfo info) {
if (!(info instanceof DAG)) {
LOG.warn("DAG info is not a DAG - cannot derive dependencies");
return null;
}
DAG dag = (DAG) info;
int vc = dag.getVertices().size();
// All the vertices belong to the same DAG, so we just use numbers.
Map<Integer, Set<Integer>> result = Maps.newHashMapWithExpectedSize(vc);
LinkedList<TezVertexID> queue = new LinkedList<>();
// We assume a DAG is a DAG, and that it's connected. Add direct dependencies.
for (Vertex v : dag.getVertices().values()) {
Map<Vertex, Edge> out = v.getOutputVertices();
if (out == null) {
result.put(v.getVertexId().getId(), Sets.newHashSet());
} else {
Set<Integer> set = Sets.newHashSetWithExpectedSize(vc);
for (Vertex outV : out.keySet()) {
set.add(outV.getVertexId().getId());
}
result.put(v.getVertexId().getId(), set);
}
if (v.getOutputVerticesCount() == 0) {
queue.add(v.getVertexId());
}
}
Set<Integer> processed = Sets.newHashSetWithExpectedSize(vc);
while (!queue.isEmpty()) {
TezVertexID id = queue.poll();
// Already processed. See backtracking.
if (processed.contains(id.getId()))
continue;
Vertex v = dag.getVertex(id);
Map<Vertex, Edge> out = v.getOutputVertices();
if (out != null) {
// Check that all the outputs have been processed; if not, insert them into queue
// before the current vertex and try again. It's possible e.g. in a structure like this:
// _1
// / 2
// 3 4 where 1 may be added to the queue before 2
boolean doBacktrack = false;
for (Vertex outV : out.keySet()) {
TezVertexID outId = outV.getVertexId();
int outNum = outId.getId();
if (!processed.contains(outNum)) {
if (!doBacktrack) {
queue.addFirst(id);
doBacktrack = true;
}
queue.addFirst(outId);
}
}
if (doBacktrack)
continue;
}
int num = id.getId();
processed.add(num);
Set<Integer> deps = result.get(num);
Map<Vertex, Edge> in = v.getInputVertices();
if (in != null) {
for (Vertex inV : in.keySet()) {
queue.add(inV.getVertexId());
// Our outputs are the transitive outputs of our inputs.
result.get(inV.getVertexId().getId()).addAll(deps);
}
}
}
return result;
}
Aggregations