use of org.apache.tez.dag.api.records.DAGProtos.RootInputLeafOutputProto in project tez by apache.
the class VertexImpl method setAdditionalOutputs.
@Override
public void setAdditionalOutputs(List<RootInputLeafOutputProto> outputs) {
LOG.info("Setting " + outputs.size() + " additional outputs for vertex " + this.logIdentifier);
this.additionalOutputs = Maps.newHashMapWithExpectedSize(outputs.size());
this.outputCommitters = Maps.newHashMapWithExpectedSize(outputs.size());
for (RootInputLeafOutputProto output : outputs) {
addIO(output.getName());
OutputDescriptor od = DagTypeConverters.convertOutputDescriptorFromDAGPlan(output.getIODescriptor());
this.additionalOutputs.put(output.getName(), new RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>(output.getName(), od, output.hasControllerDescriptor() ? DagTypeConverters.convertOutputCommitterDescriptorFromDAGPlan(output.getControllerDescriptor()) : null));
OutputSpec outputSpec = new OutputSpec(output.getName(), od, 0);
additionalOutputSpecs.add(outputSpec);
}
}
use of org.apache.tez.dag.api.records.DAGProtos.RootInputLeafOutputProto in project tez by apache.
the class VertexImpl method setAdditionalInputs.
@Override
public void setAdditionalInputs(List<RootInputLeafOutputProto> inputs) {
LOG.info("Setting " + inputs.size() + " additional inputs for vertex" + this.logIdentifier);
this.rootInputDescriptors = Maps.newHashMapWithExpectedSize(inputs.size());
for (RootInputLeafOutputProto input : inputs) {
addIO(input.getName());
InputDescriptor id = DagTypeConverters.convertInputDescriptorFromDAGPlan(input.getIODescriptor());
this.rootInputDescriptors.put(input.getName(), new RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>(input.getName(), id, input.hasControllerDescriptor() ? DagTypeConverters.convertInputInitializerDescriptorFromDAGPlan(input.getControllerDescriptor()) : null));
this.rootInputSpecs.put(input.getName(), DEFAULT_ROOT_INPUT_SPECS);
}
}
use of org.apache.tez.dag.api.records.DAGProtos.RootInputLeafOutputProto in project tez by apache.
the class TestVertexImpl method testBadCommitter.
@Test(timeout = 5000)
public void testBadCommitter() throws Exception {
VertexImpl v = vertices.get("vertex2");
List<RootInputLeafOutputProto> outputs = new ArrayList<RootInputLeafOutputProto>();
outputs.add(RootInputLeafOutputProto.newBuilder().setControllerDescriptor(TezEntityDescriptorProto.newBuilder().setClassName(CountingOutputCommitter.class.getName()).setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder().setUserPayload(ByteString.copyFrom(new CountingOutputCommitter.CountingOutputCommitterConfig(true, true, false).toUserPayload())).build())).setName("output_v2").setIODescriptor(TezEntityDescriptorProto.newBuilder().setClassName("output.class")).build());
v.setAdditionalOutputs(outputs);
initAllVertices(VertexState.INITED);
startVertex(v);
CountingOutputCommitter committer = (CountingOutputCommitter) v.getOutputCommitter("output_v2");
TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
TezTaskID t2 = TezTaskID.getInstance(v.getVertexId(), 1);
dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.FAILED, v.getState());
Assert.assertEquals(VertexTerminationCause.COMMIT_FAILURE, v.getTerminationCause());
Assert.assertEquals(1, committer.commitCounter);
Assert.assertEquals(1, committer.abortCounter);
Assert.assertEquals(1, committer.initCounter);
Assert.assertEquals(1, committer.setupCounter);
}
use of org.apache.tez.dag.api.records.DAGProtos.RootInputLeafOutputProto in project tez by apache.
the class TestVertexImpl method testVertexSuccessToFailedAfterTaskScheduler.
@Test(timeout = 5000)
public void testVertexSuccessToFailedAfterTaskScheduler() throws Exception {
// For downstream failures
VertexImpl v = vertices.get("vertex2");
List<RootInputLeafOutputProto> outputs = new ArrayList<RootInputLeafOutputProto>();
outputs.add(RootInputLeafOutputProto.newBuilder().setControllerDescriptor(TezEntityDescriptorProto.newBuilder().setClassName(CountingOutputCommitter.class.getName()).setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder().setUserPayload(ByteString.copyFrom(new CountingOutputCommitter.CountingOutputCommitterConfig().toUserPayload())).build())).setName("output_v2").setIODescriptor(TezEntityDescriptorProto.newBuilder().setClassName("output.class")).build());
v.setAdditionalOutputs(outputs);
initAllVertices(VertexState.INITED);
startVertex(v);
TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
TezTaskID t2 = TezTaskID.getInstance(v.getVertexId(), 1);
dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
Assert.assertEquals(1, dagEventDispatcher.eventCount.get(DAGEventType.DAG_VERTEX_COMPLETED).intValue());
dispatcher.getEventHandler().handle(new VertexEventTaskReschedule(t1));
dispatcher.await();
Assert.assertEquals(VertexState.FAILED, v.getState());
Assert.assertEquals(2, dagEventDispatcher.eventCount.get(DAGEventType.DAG_VERTEX_COMPLETED).intValue());
}
use of org.apache.tez.dag.api.records.DAGProtos.RootInputLeafOutputProto in project tez by apache.
the class TestVertexImpl method testBadCommitter2.
@Test(timeout = 5000)
public void testBadCommitter2() throws Exception {
VertexImpl v = vertices.get("vertex2");
List<RootInputLeafOutputProto> outputs = new ArrayList<RootInputLeafOutputProto>();
outputs.add(RootInputLeafOutputProto.newBuilder().setControllerDescriptor(TezEntityDescriptorProto.newBuilder().setClassName(CountingOutputCommitter.class.getName()).setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder().setUserPayload(ByteString.copyFrom(new CountingOutputCommitter.CountingOutputCommitterConfig(true, true, true).toUserPayload())).build())).setName("output_v2").setIODescriptor(TezEntityDescriptorProto.newBuilder().setClassName("output.class")).build());
v.setAdditionalOutputs(outputs);
initAllVertices(VertexState.INITED);
startVertex(v);
CountingOutputCommitter committer = (CountingOutputCommitter) v.getOutputCommitter("output_v2");
TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
TezTaskID t2 = TezTaskID.getInstance(v.getVertexId(), 1);
dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.FAILED, v.getState());
Assert.assertEquals(VertexTerminationCause.COMMIT_FAILURE, v.getTerminationCause());
Assert.assertEquals(1, committer.commitCounter);
Assert.assertEquals(1, committer.abortCounter);
Assert.assertEquals(1, committer.initCounter);
Assert.assertEquals(1, committer.setupCounter);
}
Aggregations