Examples with VertexManagerEvent org.apache.tez.runtime.api.events.VertexManagerEvent used on opensource projects

Example 11 with VertexManagerEvent

use of org.apache.tez.runtime.api.events.VertexManagerEvent in project tez by apache.

the class TestVertexImpl method getVertexManagerEvent.

private VertexManagerEvent getVertexManagerEvent(long[] sizes, long totalSize, Vertex vertex) throws IOException {
    ByteBuffer payload = null;
    if (sizes != null) {
        RoaringBitmap partitionStats = ShuffleUtils.getPartitionStatsForPhysicalOutput(sizes);
        DataOutputBuffer dout = new DataOutputBuffer();
        partitionStats.serialize(dout);
        ByteString partitionStatsBytes = TezCommonUtils.compressByteArrayToByteString(dout.getData());
        payload = ShuffleUserPayloads.VertexManagerEventPayloadProto.newBuilder().setOutputSize(totalSize).setPartitionStats(partitionStatsBytes).build().toByteString().asReadOnlyByteBuffer();
    } else {
        payload = ShuffleUserPayloads.VertexManagerEventPayloadProto.newBuilder().setOutputSize(totalSize).build().toByteString().asReadOnlyByteBuffer();
    }
    VertexManagerEvent vmEvent = VertexManagerEvent.create(vertex.getName(), payload);
    return vmEvent;
}

Example 12 with VertexManagerEvent

use of org.apache.tez.runtime.api.events.VertexManagerEvent in project tez by apache.

the class TestVertexImpl method testExceptionFromVM_OnVertexManagerEventReceived.

@Test(timeout = 5000)
public void testExceptionFromVM_OnVertexManagerEventReceived() throws Exception {
    useCustomInitializer = true;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithVMException("TestInputInitializer", VMExceptionLocation.OnVertexManagerEventReceived);
    setupPostDagCreation();
    VertexImplWithControlledInitializerManager v1 = (VertexImplWithControlledInitializerManager) vertices.get("vertex1");
    dispatcher.getEventHandler().handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_INIT));
    // wait to ensure init is completed, so that rootInitManager is not null
    dispatcher.await();
    RootInputInitializerManagerControlled initializerManager1 = v1.getRootInputInitializerManager();
    initializerManager1.completeInputInitialization();
    VertexManagerEvent vmEvent = VertexManagerEvent.create(v1.getName(), ByteBuffer.wrap(new byte[0]));
    TezTaskAttemptID taId1 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v1.getVertexId(), 0), 0);
    TezEvent tezEvent = new TezEvent(vmEvent, new EventMetaData(EventProducerConsumerType.OUTPUT, v1.getName(), null, taId1));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v1.getVertexId(), Lists.newArrayList(tezEvent)));
    dispatcher.await();
    Assert.assertEquals(VertexState.FAILED, v1.getState());
    String diagnostics = StringUtils.join(v1.getDiagnostics(), ",");
    Assert.assertTrue(diagnostics.contains(VMExceptionLocation.OnVertexManagerEventReceived.name()));
}

Example 13 with VertexManagerEvent

use of org.apache.tez.runtime.api.events.VertexManagerEvent in project tez by apache.

the class TestVertexImpl method testVMEventBeforeVertexInitialized.

@Test(timeout = 5000)
public void testVMEventBeforeVertexInitialized() throws Exception {
    useCustomInitializer = true;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithCountingVM();
    setupPostDagCreation();
    VertexImpl v1 = vertices.get("vertex1");
    VertexImpl v2 = vertices.get("vertex2");
    VertexImpl v3 = vertices.get("vertex3");
    dispatcher.getEventHandler().handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_INIT));
    dispatcher.await();
    assertEquals(VertexState.INITED, v1.getState());
    dispatcher.getEventHandler().handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_START));
    dispatcher.await();
    assertEquals(VertexState.RUNNING, v1.getState());
    assertEquals(VertexState.NEW, v3.getState());
    // Generate a VM event for v1, targeted at v3
    VertexManagerEvent vmEvent = VertexManagerEvent.create("vertex3", ByteBuffer.wrap(new byte[0]));
    TezEvent tezVmEvent = new TezEvent(vmEvent, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", null, TezTaskAttemptID.getInstance(TezTaskID.getInstance(v1.getVertexId(), 1), 1)));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v1.getVertexId(), Collections.singletonList(tezVmEvent)));
    dispatcher.await();
    assertEquals(1, v3.pendingVmEvents.size());
    assertEquals(0, InvocationCountingVertexManager.numVmEventsReceived.get());
    // Initialize v2, which will trigger initialization of v3
    dispatcher.getEventHandler().handle(new VertexEvent(v2.getVertexId(), VertexEventType.V_INIT));
    dispatcher.await();
    assertEquals(VertexState.INITED, v3.getState());
    // The VM event should have been processed.
    assertEquals(0, v3.pendingVmEvents.size());
    assertEquals(1, InvocationCountingVertexManager.numVmEventsReceived.get());
    // Send another VM event - make sure it's processed without additional events.
    vmEvent = VertexManagerEvent.create("vertex3", ByteBuffer.wrap(new byte[0]));
    tezVmEvent = new TezEvent(vmEvent, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", null, TezTaskAttemptID.getInstance(TezTaskID.getInstance(v1.getVertexId(), 1), 2)));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v1.getVertexId(), Collections.singletonList(tezVmEvent)));
    dispatcher.await();
    assertEquals(0, v3.pendingVmEvents.size());
    assertEquals(2, InvocationCountingVertexManager.numVmEventsReceived.get());
}

Example 14 with VertexManagerEvent

use of org.apache.tez.runtime.api.events.VertexManagerEvent in project tez by apache.

the class TestCommit method testVertexRouteEventErrorWhileCommitting.

@Test(timeout = 5000)
public void testVertexRouteEventErrorWhileCommitting() throws Exception {
    conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, false);
    setupDAG(createDAGPlan_SingleVertexWith2Committer(true, true, true));
    initDAG(dag);
    startDAG(dag);
    VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
    v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    Assert.assertEquals(VertexState.COMMITTING, v1.getState());
    // reschedule task
    VertexManagerEvent vmEvent = VertexManagerEvent.create("vertex1", ByteBuffer.wrap(new byte[0]));
    TezTaskAttemptID taId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v1.getVertexId(), 0), 0);
    TezEvent tezEvent = new TezEvent(vmEvent, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", null, taId));
    v1.handle(new VertexEventRouteEvent(v1.getVertexId(), Collections.singletonList(tezEvent)));
    waitUntil(dag, DAGState.FAILED);
    Assert.assertEquals(VertexState.FAILED, v1.getState());
    Assert.assertEquals(VertexTerminationCause.AM_USERCODE_FAILURE, v1.getTerminationCause());
    Assert.assertTrue(v1.commitFutures.isEmpty());
    Assert.assertEquals(DAGState.FAILED, dag.getState());
    Assert.assertEquals(DAGTerminationCause.VERTEX_FAILURE, dag.getTerminationCause());
    historyEventHandler.verifyVertexCommitStartedEvent(v1.getVertexId(), 1);
    historyEventHandler.verifyVertexFinishedEvent(v1.getVertexId(), 1);
    CountingOutputCommitter v1OutputCommitter_1 = (CountingOutputCommitter) v1.getOutputCommitter("v1Out_1");
    CountingOutputCommitter v1OutputCommitter_2 = (CountingOutputCommitter) v1.getOutputCommitter("v1Out_2");
    Assert.assertEquals(1, v1OutputCommitter_1.initCounter);
    Assert.assertEquals(1, v1OutputCommitter_1.setupCounter);
    // commit may not have started, so can't verify commitCounter
    Assert.assertEquals(1, v1OutputCommitter_1.abortCounter);
    Assert.assertEquals(1, v1OutputCommitter_2.initCounter);
    Assert.assertEquals(1, v1OutputCommitter_2.setupCounter);
    // commit may not have started, so can't verify commitCounter
    Assert.assertEquals(1, v1OutputCommitter_2.abortCounter);
}

Example 15 with VertexManagerEvent

use of org.apache.tez.runtime.api.events.VertexManagerEvent in project tez by apache.

the class ShuffleVertexManagerBase method onVertexStarted.

@Override
public synchronized void onVertexStarted(List<TaskAttemptIdentifier> completions) {
    // examine edges after vertex started because until then these may not have been defined
    Map<String, EdgeProperty> inputs = getContext().getInputVertexEdgeProperties();
    for (Map.Entry<String, EdgeProperty> entry : inputs.entrySet()) {
        srcVertexInfo.put(entry.getKey(), createSourceVertexInfo(entry.getValue(), getContext().getVertexNumTasks(getContext().getVertexName())));
        // TODO what if derived class has already called this
        // register for status update from all source vertices
        getContext().registerForVertexStateUpdates(entry.getKey(), EnumSet.of(VertexState.CONFIGURED));
        if (entry.getValue().getDataMovementType() == DataMovementType.SCATTER_GATHER) {
            bipartiteSources++;
        }
    }
    onVertexStartedCheck();
    for (VertexStateUpdate stateUpdate : pendingStateUpdates) {
        handleVertexStateUpdate(stateUpdate);
    }
    pendingStateUpdates.clear();
    // track the tasks in this vertex
    updatePendingTasks();
    for (VertexManagerEvent vmEvent : pendingVMEvents) {
        handleVertexManagerEvent(vmEvent);
    }
    pendingVMEvents.clear();
    LOG.info("OnVertexStarted vertex: {} with {} source tasks and {} pending" + " tasks", getContext().getVertexName(), totalNumBipartiteSourceTasks, totalTasksToSchedule);
    if (completions != null) {
        for (TaskAttemptIdentifier attempt : completions) {
            onSourceTaskCompleted(attempt);
        }
    }
    onVertexStartedDone.set(true);
    // for the special case when source has 0 tasks or min fraction == 0
    processPendingTasks(null);
}