Examples with RootInputLeafOutputProto org.apache.tez.dag.api.records.DAGProtos.RootInputLeafOutputProto used on opensource projects

Example 6 with RootInputLeafOutputProto

use of org.apache.tez.dag.api.records.DAGProtos.RootInputLeafOutputProto in project tez by apache.

the class TestDAGImpl method testDAGCompletionWithCommitFailure.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testDAGCompletionWithCommitFailure() throws IOException {
    // all vertices completed -> DAG completion and commit
    initDAG(mrrDag);
    dispatcher.await();
    // committer for bad vertex will throw exception
    Vertex badVertex = mrrDag.getVertex("vertex3");
    List<RootInputLeafOutputProto> outputs = new ArrayList<RootInputLeafOutputProto>();
    outputs.add(RootInputLeafOutputProto.newBuilder().setControllerDescriptor(TezEntityDescriptorProto.newBuilder().setClassName(CountingOutputCommitter.class.getName()).setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder().setUserPayload(ByteString.copyFrom(new CountingOutputCommitter.CountingOutputCommitterConfig(true, false, false).toUserPayload())).build())).setName("output3").setIODescriptor(TezEntityDescriptorProto.newBuilder().setClassName("output.class")).build());
    badVertex.setAdditionalOutputs(outputs);
    startDAG(mrrDag);
    dispatcher.await();
    for (int i = 0; i < 2; ++i) {
        Vertex v = mrrDag.getVertex("vertex" + (i + 1));
        dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED));
        dispatcher.await();
        Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
        Assert.assertEquals(i + 1, mrrDag.getSuccessfulVertices());
    }
    // no commit yet
    for (Vertex v : mrrDag.vertices.values()) {
        for (OutputCommitter c : v.getOutputCommitters().values()) {
            CountingOutputCommitter committer = (CountingOutputCommitter) c;
            Assert.assertEquals(0, committer.abortCounter);
            Assert.assertEquals(0, committer.commitCounter);
            Assert.assertEquals(1, committer.initCounter);
            Assert.assertEquals(1, committer.setupCounter);
        }
    }
    // dag completion and commit. Exception causes all outputs to be aborted
    Vertex v = mrrDag.getVertex("vertex3");
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
    Assert.assertEquals(3, mrrDag.getSuccessfulVertices());
    Assert.assertEquals(DAGState.FAILED, mrrDag.getState());
    Assert.assertEquals(DAGTerminationCause.COMMIT_FAILURE, mrrDag.getTerminationCause());
    for (Vertex vertex : mrrDag.vertices.values()) {
        for (OutputCommitter c : vertex.getOutputCommitters().values()) {
            CountingOutputCommitter committer = (CountingOutputCommitter) c;
            Assert.assertEquals(1, committer.abortCounter);
            Assert.assertEquals(1, committer.initCounter);
            Assert.assertEquals(1, committer.setupCounter);
        }
    }
}

Example 7 with RootInputLeafOutputProto

use of org.apache.tez.dag.api.records.DAGProtos.RootInputLeafOutputProto in project tez by apache.

the class VertexInitializedEvent method fromProto.

public void fromProto(RecoveryProtos.VertexInitializedProto proto) throws IOException {
    this.vertexID = TezVertexID.fromString(proto.getVertexId());
    this.vertexName = proto.getVertexName();
    this.initRequestedTime = proto.getInitRequestedTime();
    this.initedTime = proto.getInitTime();
    this.numTasks = proto.getNumTasks();
    if (proto.getInputsCount() > 0) {
        this.additionalInputs = new LinkedHashMap<String, RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>>();
        for (RootInputLeafOutputProto inputProto : proto.getInputsList()) {
            RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input = new RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>(inputProto.getName(), DagTypeConverters.convertInputDescriptorFromDAGPlan(inputProto.getIODescriptor()), inputProto.hasControllerDescriptor() ? DagTypeConverters.convertInputInitializerDescriptorFromDAGPlan(inputProto.getControllerDescriptor()) : null);
            additionalInputs.put(input.getName(), input);
        }
    }
    int eventCount = proto.getInitGeneratedEventsCount();
    if (eventCount > 0) {
        this.initGeneratedEvents = Lists.newArrayListWithCapacity(eventCount);
    }
    for (TezEventProto eventProto : proto.getInitGeneratedEventsList()) {
        this.initGeneratedEvents.add(TezEventUtils.fromProto(eventProto));
    }
}