Examples with TezEvent org.apache.tez.runtime.api.impl.TezEvent used on opensource projects

Example 31 with TezEvent

use of org.apache.tez.runtime.api.impl.TezEvent in project tez by apache.

the class TestCommit method testVertexRouteEventErrorWhileCommitting.

@Test(timeout = 5000)
public void testVertexRouteEventErrorWhileCommitting() throws Exception {
    conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, false);
    setupDAG(createDAGPlan_SingleVertexWith2Committer(true, true, true));
    initDAG(dag);
    startDAG(dag);
    VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
    v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskID(), TaskState.SUCCEEDED));
    Assert.assertEquals(VertexState.COMMITTING, v1.getState());
    // reschedule task
    VertexManagerEvent vmEvent = VertexManagerEvent.create("vertex1", ByteBuffer.wrap(new byte[0]));
    TezTaskAttemptID taId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v1.getVertexId(), 0), 0);
    TezEvent tezEvent = new TezEvent(vmEvent, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", null, taId));
    v1.handle(new VertexEventRouteEvent(v1.getVertexId(), Collections.singletonList(tezEvent)));
    waitUntil(dag, DAGState.FAILED);
    Assert.assertEquals(VertexState.FAILED, v1.getState());
    Assert.assertEquals(VertexTerminationCause.AM_USERCODE_FAILURE, v1.getTerminationCause());
    Assert.assertTrue(v1.commitFutures.isEmpty());
    Assert.assertEquals(DAGState.FAILED, dag.getState());
    Assert.assertEquals(DAGTerminationCause.VERTEX_FAILURE, dag.getTerminationCause());
    historyEventHandler.verifyVertexCommitStartedEvent(v1.getVertexId(), 1);
    historyEventHandler.verifyVertexFinishedEvent(v1.getVertexId(), 1);
    CountingOutputCommitter v1OutputCommitter_1 = (CountingOutputCommitter) v1.getOutputCommitter("v1Out_1");
    CountingOutputCommitter v1OutputCommitter_2 = (CountingOutputCommitter) v1.getOutputCommitter("v1Out_2");
    Assert.assertEquals(1, v1OutputCommitter_1.initCounter);
    Assert.assertEquals(1, v1OutputCommitter_1.setupCounter);
    // commit may not have started, so can't verify commitCounter
    Assert.assertEquals(1, v1OutputCommitter_1.abortCounter);
    Assert.assertEquals(1, v1OutputCommitter_2.initCounter);
    Assert.assertEquals(1, v1OutputCommitter_2.setupCounter);
    // commit may not have started, so can't verify commitCounter
    Assert.assertEquals(1, v1OutputCommitter_2.abortCounter);
}

Example 32 with TezEvent

use of org.apache.tez.runtime.api.impl.TezEvent in project tez by apache.

the class TestEdge method testInvalidConsumerNumber.

@Test(timeout = 5000)
public void testInvalidConsumerNumber() throws Exception {
    EventHandler mockEventHandler = mock(EventHandler.class);
    Edge edge = new Edge(EdgeProperty.create(EdgeManagerPluginDescriptor.create(CustomEdgeManagerWithInvalidReturnValue.class.getName()).setUserPayload(new CustomEdgeManagerWithInvalidReturnValue.EdgeManagerConfig(1, 1, 0, 1).toUserPayload()), DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create(""), InputDescriptor.create("")), mockEventHandler, new TezConfiguration());
    TezVertexID v1Id = createVertexID(1);
    TezVertexID v2Id = createVertexID(2);
    edge.setSourceVertex(mockVertex("v1", v1Id, new LinkedHashMap<TezTaskID, Task>()));
    edge.setDestinationVertex(mockVertex("v2", v2Id, new LinkedHashMap<TezTaskID, Task>()));
    edge.initialize();
    try {
        TezEvent ireEvent = new TezEvent(InputReadErrorEvent.create("diag", 0, 1), new EventMetaData(EventProducerConsumerType.INPUT, "v2", "v1", TezTaskAttemptID.getInstance(TezTaskID.getInstance(v2Id, 1), 1)));
        edge.sendTezEventToSourceTasks(ireEvent);
        Assert.fail();
    } catch (AMUserCodeException e) {
        e.printStackTrace();
        assertTrue(e.getCause().getMessage().contains("ConsumerTaskNum must be positive"));
    }
}

Example 33 with TezEvent

use of org.apache.tez.runtime.api.impl.TezEvent in project tez by apache.

the class TestEdge method verifyEvents.

private void verifyEvents(TezTaskAttemptID srcTAID, LinkedHashMap<TezTaskID, Task> destTasks) {
    int count = 0;
    for (Entry<TezTaskID, Task> taskEntry : destTasks.entrySet()) {
        Task mockTask = taskEntry.getValue();
        ArgumentCaptor<TezEvent> args = ArgumentCaptor.forClass(TezEvent.class);
        verify(mockTask, times(1)).registerTezEvent(args.capture());
        TezEvent capturedEvent = args.getValue();
        DataMovementEvent dmEvent = (DataMovementEvent) capturedEvent.getEvent();
        assertEquals(srcTAID.getId(), dmEvent.getVersion());
        assertEquals(count++, dmEvent.getSourceIndex());
        assertEquals(srcTAID.getTaskID().getId(), dmEvent.getTargetIndex());
        byte[] res = new byte[dmEvent.getUserPayload().limit() - dmEvent.getUserPayload().position()];
        dmEvent.getUserPayload().slice().get(res);
        assertTrue(Arrays.equals("bytes".getBytes(), res));
    }
}

Example 34 with TezEvent

use of org.apache.tez.runtime.api.impl.TezEvent in project tez by apache.

the class TestDAGRecovery method testVertexRecoverWithSetParallelismCalledFlag.

/**
 * RecoveryEvents:
 *  DAG:  DAGInitedEvent -> DAGStartedEvent
 *  V1:   VertexReconfigrationDoneEvent -> VertexInitializedEvent -> VertexStartedEvent -> setParallelismCalledFlag
 *
 * V1 skip initialization.
 */
@Test(timeout = 5000)
public void testVertexRecoverWithSetParallelismCalledFlag() {
    initMockDAGRecoveryDataForVertex();
    List<TezEvent> inputGeneratedTezEvents = new ArrayList<TezEvent>();
    VertexInitializedEvent v1InitedEvent = new VertexInitializedEvent(v1Id, "vertex1", 0L, v1InitedTime, v1NumTask, "", null, inputGeneratedTezEvents, null);
    VertexConfigurationDoneEvent v1ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v1Id, 0L, v1NumTask, null, null, null, true);
    VertexStartedEvent v1StartedEvent = new VertexStartedEvent(v1Id, 0L, v1StartedTime);
    VertexRecoveryData vertexRecoveryData = new VertexRecoveryData(v1InitedEvent, v1ReconfigureDoneEvent, v1StartedEvent, null, new HashMap<TezTaskID, TaskRecoveryData>(), false);
    doReturn(vertexRecoveryData).when(dagRecoveryData).getVertexRecoveryData(v1Id);
    DAGEventRecoverEvent recoveryEvent = new DAGEventRecoverEvent(dagId, dagRecoveryData);
    dag.handle(recoveryEvent);
    dispatcher.await();
    VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
    VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
    VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
    assertEquals(DAGState.RUNNING, dag.getState());
    // v1 skip initialization
    assertEquals(VertexState.RUNNING, v1.getState());
    assertEquals(v1InitedTime, v1.initedTime);
    assertEquals(v1StartedTime, v1.startedTime);
    assertEquals(v1NumTask, v1.getTotalTasks());
    assertEquals(VertexState.RUNNING, v2.getState());
    assertEquals(VertexState.RUNNING, v3.getState());
}

Example 35 with TezEvent

use of org.apache.tez.runtime.api.impl.TezEvent in project tez by apache.

the class TestDAGRecovery method testMultipleVertexRecoverFromVertexTaskStart.

/**
 * RecoveryEvents:
 *  DAG:  DAGInitedEvent -> DAGStartedEvent
 *  V1:   VertexReconfigrationDoneEvent -> VertexInitializedEvent -> VertexStartedEvent -> VertexTaskStart
 *  V2:   VertexReconfigrationDoneEvent -> VertexInitializedEvent -> VertexStartedEvent -> VertexTaskStart
 *  V3:   VertexReconfigrationDoneEvent -> VertexInitializedEvent -> VertexStartedEvent -> VertexTaskStart
 *
 * V1 skip initialization.
 * V2 skip initialization.
 * V3 skip initialization.
 */
@Test(timeout = 5000)
public void testMultipleVertexRecoverFromVertexTaskStart() {
    initMockDAGRecoveryDataForVertex();
    List<TezEvent> inputGeneratedTezEvents = new ArrayList<TezEvent>();
    VertexInitializedEvent v1InitedEvent = new VertexInitializedEvent(v1Id, "vertex1", 0L, v1InitedTime, v1NumTask, "", null, inputGeneratedTezEvents, null);
    VertexInitializedEvent v2InitedEvent = new VertexInitializedEvent(v2Id, "vertex2", 0L, v2InitedTime, v2NumTask, "", null, inputGeneratedTezEvents, null);
    VertexInitializedEvent v3InitedEvent = new VertexInitializedEvent(v3Id, "vertex3", 0L, v3InitedTime, v3NumTask, "", null, inputGeneratedTezEvents, null);
    VertexConfigurationDoneEvent v1ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v1Id, 0L, v1NumTask, null, null, null, true);
    VertexConfigurationDoneEvent v2ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v2Id, 0L, v2NumTask, null, null, null, true);
    VertexConfigurationDoneEvent v3ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v3Id, 0L, v3NumTask, null, null, null, true);
    VertexStartedEvent v1StartedEvent = new VertexStartedEvent(v1Id, 0L, v1StartedTime);
    VertexStartedEvent v2StartedEvent = new VertexStartedEvent(v2Id, 0L, v2StartedTime);
    VertexStartedEvent v3StartedEvent = new VertexStartedEvent(v3Id, 0L, v3StartedTime);
    TaskStartedEvent v1taskStartedEvent = new TaskStartedEvent(t1v1Id, "vertex1", 0L, 0L);
    TaskRecoveryData v1taskRecoveryData = new TaskRecoveryData(v1taskStartedEvent, null, null);
    Map<TezTaskID, TaskRecoveryData> v1taskRecoveryDataMap = new HashMap<>();
    // put dummy tasks
    v1taskRecoveryDataMap.put(t1v1Id, v1taskRecoveryData);
    TaskStartedEvent v2taskStartedEvent = new TaskStartedEvent(t1v2Id, "vertex2", 0L, 0L);
    TaskRecoveryData v2taskRecoveryData = new TaskRecoveryData(v2taskStartedEvent, null, null);
    Map<TezTaskID, TaskRecoveryData> v2taskRecoveryDataMap = new HashMap<>();
    // put dummy tasks
    v2taskRecoveryDataMap.put(t1v2Id, v2taskRecoveryData);
    TaskStartedEvent v3taskStartedEvent = new TaskStartedEvent(t1v3Id, "vertex3", 0L, 0L);
    TaskRecoveryData v3taskRecoveryData = new TaskRecoveryData(v3taskStartedEvent, null, null);
    Map<TezTaskID, TaskRecoveryData> v3taskRecoveryDataMap = new HashMap<>();
    // put dummy tasks
    v3taskRecoveryDataMap.put(t1v3Id, v3taskRecoveryData);
    VertexRecoveryData vertex1RecoveryData = new VertexRecoveryData(v1InitedEvent, v1ReconfigureDoneEvent, v1StartedEvent, null, v1taskRecoveryDataMap, false);
    VertexRecoveryData vertex2RecoveryData = new VertexRecoveryData(v2InitedEvent, v2ReconfigureDoneEvent, v2StartedEvent, null, v2taskRecoveryDataMap, false);
    VertexRecoveryData vertex3RecoveryData = new VertexRecoveryData(v3InitedEvent, v3ReconfigureDoneEvent, v3StartedEvent, null, v3taskRecoveryDataMap, false);
    doReturn(vertex1RecoveryData).when(dagRecoveryData).getVertexRecoveryData(v1Id);
    doReturn(vertex2RecoveryData).when(dagRecoveryData).getVertexRecoveryData(v2Id);
    doReturn(vertex3RecoveryData).when(dagRecoveryData).getVertexRecoveryData(v3Id);
    DAGEventRecoverEvent recoveryEvent = new DAGEventRecoverEvent(dagId, dagRecoveryData);
    dag.handle(recoveryEvent);
    dispatcher.await();
    VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
    VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
    VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
    assertEquals(DAGState.RUNNING, dag.getState());
    // v1 skip initialization
    assertEquals(VertexState.RUNNING, v1.getState());
    assertEquals(v1InitedTime, v1.initedTime);
    assertEquals(v1StartedTime, v1.startedTime);
    assertEquals(v1NumTask, v1.getTotalTasks());
    // v2 skip initialization
    assertEquals(VertexState.RUNNING, v2.getState());
    assertEquals(v2InitedTime, v2.initedTime);
    assertEquals(v2StartedTime, v2.startedTime);
    assertEquals(v2NumTask, v2.getTotalTasks());
    // v3 skip initialization
    assertEquals(VertexState.RUNNING, v3.getState());
    assertEquals(v3InitedTime, v3.initedTime);
    assertEquals(v3StartedTime, v3.startedTime);
    assertEquals(v3NumTask, v3.getTotalTasks());
}