Examples with TezEvent org.apache.tez.runtime.api.impl.TezEvent used on opensource projects

Example 36 with TezEvent

use of org.apache.tez.runtime.api.impl.TezEvent in project tez by apache.

the class TestDAGRecovery method testVertexRecoverFromStart.

/**
 * RecoveryEvents:
 *  DAG:  DAGInitedEvent -> DAGStartedEvent
 *  V1:   VertexReconfigrationDoneEvent -> VertexInitializedEvent -> VertexStartedEvent
 *
 * Reinitialize V1 again.
 */
@Test(timeout = 5000)
public void testVertexRecoverFromStart() {
    initMockDAGRecoveryDataForVertex();
    List<TezEvent> inputGeneratedTezEvents = new ArrayList<TezEvent>();
    VertexInitializedEvent v1InitedEvent = new VertexInitializedEvent(v1Id, "vertex1", 0L, v1InitedTime, v1NumTask, "", null, inputGeneratedTezEvents, null);
    VertexConfigurationDoneEvent v1ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v1Id, 0L, v1NumTask, null, null, null, false);
    VertexStartedEvent v1StartedEvent = new VertexStartedEvent(v1Id, 0L, v1StartedTime);
    VertexRecoveryData vertexRecoveryData = new VertexRecoveryData(v1InitedEvent, v1ReconfigureDoneEvent, v1StartedEvent, null, new HashMap<TezTaskID, TaskRecoveryData>(), false);
    doReturn(vertexRecoveryData).when(dagRecoveryData).getVertexRecoveryData(v1Id);
    DAGEventRecoverEvent recoveryEvent = new DAGEventRecoverEvent(dagId, dagRecoveryData);
    dag.handle(recoveryEvent);
    dispatcher.await();
    VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
    VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
    VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
    assertEquals(DAGState.RUNNING, dag.getState());
    // reinitialize v1
    assertEquals(VertexState.INITIALIZING, v1.getState());
    assertEquals(VertexState.RUNNING, v2.getState());
    assertEquals(VertexState.INITED, v3.getState());
}

Example 37 with TezEvent

use of org.apache.tez.runtime.api.impl.TezEvent in project tez by apache.

the class TestDAGRecovery method initMockDAGRecoveryDataForTask.

// ///////////////////////////// Task ////////////////////////////////////////////////////////////
private void initMockDAGRecoveryDataForTask() {
    List<TezEvent> inputGeneratedTezEvents = new ArrayList<TezEvent>();
    VertexInitializedEvent v1InitedEvent = new VertexInitializedEvent(v1Id, "vertex1", 0L, v1InitedTime, v1NumTask, "", null, inputGeneratedTezEvents, null);
    Map<String, InputSpecUpdate> rootInputSpecs = new HashMap<String, InputSpecUpdate>();
    VertexConfigurationDoneEvent v1ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v1Id, 0L, v1NumTask, null, null, rootInputSpecs, true);
    VertexStartedEvent v1StartedEvent = new VertexStartedEvent(v1Id, 0L, v1StartedTime);
    TaskStartedEvent v1taskStartedEvent = new TaskStartedEvent(t1v1Id, "vertex1", 0L, 0L);
    TaskRecoveryData v1taskRecoveryData = new TaskRecoveryData(v1taskStartedEvent, null, null);
    Map<TezTaskID, TaskRecoveryData> v1taskRecoveryDataMap = new HashMap<>();
    // put dummy tasks
    v1taskRecoveryDataMap.put(t1v1Id, v1taskRecoveryData);
    VertexRecoveryData v1RecoveryData = new VertexRecoveryData(v1InitedEvent, v1ReconfigureDoneEvent, v1StartedEvent, null, v1taskRecoveryDataMap, false);
    DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagId, dagInitedTime, "user", "dagName", null);
    DAGStartedEvent dagStartedEvent = new DAGStartedEvent(dagId, dagStartedTime, "user", "dagName");
    doReturn(v1RecoveryData).when(dagRecoveryData).getVertexRecoveryData(v1Id);
    doReturn(dagInitedEvent).when(dagRecoveryData).getDAGInitializedEvent();
    doReturn(dagStartedEvent).when(dagRecoveryData).getDAGStartedEvent();
}

Example 38 with TezEvent

use of org.apache.tez.runtime.api.impl.TezEvent in project tez by apache.

the class TestDAGRecovery method testTARecoverFromSucceeded.

/**
 * RecoveryEvents: TaskAttemptStartedEvent -> TaskAttemptFinishedEvent (SUCCEEDED)
 * Recover it to SUCCEEDED
 */
@Test(timeout = 5000)
public void testTARecoverFromSucceeded() {
    initMockDAGRecoveryDataForTaskAttempt();
    TaskAttemptStartedEvent taStartedEvent = new TaskAttemptStartedEvent(ta1t1v1Id, "v1", ta1LaunchTime, mock(ContainerId.class), mock(NodeId.class), "", "", "");
    List<TezEvent> taGeneratedEvents = new ArrayList<TezEvent>();
    EventMetaData sourceInfo = new EventMetaData(EventProducerConsumerType.INPUT, "vertex1", "vertex3", ta1t1v1Id);
    taGeneratedEvents.add(new TezEvent(DataMovementEvent.create(ByteBuffer.wrap(new byte[0])), sourceInfo));
    TaskAttemptFinishedEvent taFinishedEvent = new TaskAttemptFinishedEvent(ta1t1v1Id, "v1", ta1LaunchTime, ta1FinishedTime, TaskAttemptState.SUCCEEDED, null, null, "", null, null, taGeneratedEvents, 0L, null, 0L, null, null, null, null, null);
    TaskAttemptRecoveryData taRecoveryData = new TaskAttemptRecoveryData(taStartedEvent, taFinishedEvent);
    doReturn(taRecoveryData).when(dagRecoveryData).getTaskAttemptRecoveryData(ta1t1v1Id);
    dag.handle(new DAGEventRecoverEvent(dagId, dagRecoveryData));
    dispatcher.await();
    TaskImpl task = (TaskImpl) dag.getVertex(v1Id).getTask(t1v1Id);
    TaskAttemptImpl taskAttempt = (TaskAttemptImpl) task.getAttempt(ta1t1v1Id);
    assertEquals(TaskAttemptStateInternal.SUCCEEDED, taskAttempt.getInternalState());
    historyEventHandler.verifyHistoryEvent(0, HistoryEventType.TASK_ATTEMPT_FINISHED);
    assertEquals(TaskStateInternal.SUCCEEDED, task.getInternalState());
    assertEquals(ta1LaunchTime, taskAttempt.getLaunchTime());
    assertEquals(ta1FinishedTime, taskAttempt.getFinishTime());
}

Example 39 with TezEvent

use of org.apache.tez.runtime.api.impl.TezEvent in project tez by apache.

the class TestDAGRecovery method testVertexRecoverFromVertexTaskStart.

/**
 * RecoveryEvents:
 *  DAG:  DAGInitedEvent -> DAGStartedEvent
 *  V1:   VertexReconfigrationDoneEvent -> VertexInitializedEvent -> VertexStartedEvent -> VertexTaskStart
 *
 * V1 skip initialization.
 */
@Test(timeout = 5000)
public void testVertexRecoverFromVertexTaskStart() {
    initMockDAGRecoveryDataForVertex();
    List<TezEvent> inputGeneratedTezEvents = new ArrayList<TezEvent>();
    VertexInitializedEvent v1InitedEvent = new VertexInitializedEvent(v1Id, "vertex1", 0L, v1InitedTime, v1NumTask, "", null, inputGeneratedTezEvents, null);
    VertexConfigurationDoneEvent v1ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v1Id, 0L, v1NumTask, null, null, null, true);
    VertexStartedEvent v1StartedEvent = new VertexStartedEvent(v1Id, 0L, v1StartedTime);
    TaskStartedEvent taskStartedEvent = new TaskStartedEvent(t1v1Id, "v1", 0L, 0L);
    TaskRecoveryData taskRecoveryData = new TaskRecoveryData(taskStartedEvent, null, null);
    Map<TezTaskID, TaskRecoveryData> taskRecoveryDataMap = new HashMap<>();
    // put dummy tasks
    taskRecoveryDataMap.put(t1v2Id, taskRecoveryData);
    VertexRecoveryData vertexRecoveryData = new VertexRecoveryData(v1InitedEvent, v1ReconfigureDoneEvent, v1StartedEvent, null, taskRecoveryDataMap, false);
    doReturn(vertexRecoveryData).when(dagRecoveryData).getVertexRecoveryData(v1Id);
    DAGEventRecoverEvent recoveryEvent = new DAGEventRecoverEvent(dagId, dagRecoveryData);
    dag.handle(recoveryEvent);
    dispatcher.await();
    VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
    VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
    VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
    assertEquals(DAGState.RUNNING, dag.getState());
    // v1 skip initialization
    assertEquals(VertexState.RUNNING, v1.getState());
    assertEquals(v1InitedTime, v1.initedTime);
    assertEquals(v1StartedTime, v1.startedTime);
    assertEquals(v1NumTask, v1.getTotalTasks());
    assertEquals(VertexState.RUNNING, v2.getState());
    assertEquals(VertexState.RUNNING, v3.getState());
}

Example 40 with TezEvent

use of org.apache.tez.runtime.api.impl.TezEvent in project tez by apache.

the class TestDAGRecovery method testMultipleVertexRecoverFromVertex.

/**
 * RecoveryEvents:
 *  DAG:  DAGInitedEvent -> DAGStartedEvent
 *  V1:   VertexReconfigrationDoneEvent -> VertexInitializedEvent
 *  V2:   VertexReconfigrationDoneEvent -> VertexInitializedEvent -> VertexStartedEvent -> VertexTaskStart
 *  V3:   VertexReconfigrationDoneEvent -> VertexInitializedEvent -> VertexStartedEvent -> VertexTaskStart
 *
 *  Reinitialize V1 again.
 *  V2 skip initialization.
 *  Reinitialize V3 again. Since V3 is dependent on V1
 */
@Test(timeout = 5000)
public void testMultipleVertexRecoverFromVertex() {
    initMockDAGRecoveryDataForVertex();
    List<TezEvent> inputGeneratedTezEvents = new ArrayList<TezEvent>();
    VertexInitializedEvent v1InitedEvent = new VertexInitializedEvent(v1Id, "vertex1", 0L, v1InitedTime, v1NumTask, "", null, inputGeneratedTezEvents, null);
    VertexInitializedEvent v2InitedEvent = new VertexInitializedEvent(v2Id, "vertex2", 0L, v2InitedTime, v2NumTask, "", null, inputGeneratedTezEvents, null);
    VertexInitializedEvent v3InitedEvent = new VertexInitializedEvent(v3Id, "vertex3", 0L, v3InitedTime, v3NumTask, "", null, inputGeneratedTezEvents, null);
    VertexConfigurationDoneEvent v2ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v2Id, 0L, v2NumTask, null, null, null, true);
    VertexConfigurationDoneEvent v3ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v3Id, 0L, v3NumTask, null, null, null, true);
    VertexStartedEvent v2StartedEvent = new VertexStartedEvent(v2Id, 0L, v2StartedTime);
    VertexStartedEvent v3StartedEvent = new VertexStartedEvent(v3Id, 0L, v3StartedTime);
    TaskStartedEvent v2taskStartedEvent = new TaskStartedEvent(t1v2Id, "vertex2", 0L, 0L);
    TaskRecoveryData v2taskRecoveryData = new TaskRecoveryData(v2taskStartedEvent, null, null);
    Map<TezTaskID, TaskRecoveryData> v2taskRecoveryDataMap = new HashMap<>();
    // put dummy tasks
    v2taskRecoveryDataMap.put(t1v2Id, v2taskRecoveryData);
    TaskStartedEvent v3taskStartedEvent = new TaskStartedEvent(t1v3Id, "vertex3", 0L, 0L);
    TaskRecoveryData v3taskRecoveryData = new TaskRecoveryData(v3taskStartedEvent, null, null);
    Map<TezTaskID, TaskRecoveryData> v3taskRecoveryDataMap = new HashMap<>();
    // put dummy tasks
    v3taskRecoveryDataMap.put(t1v3Id, v3taskRecoveryData);
    VertexRecoveryData vertex1RecoveryData = new VertexRecoveryData(v1InitedEvent, null, null, null, null, false);
    VertexRecoveryData vertex2RecoveryData = new VertexRecoveryData(v2InitedEvent, v2ReconfigureDoneEvent, v2StartedEvent, null, v2taskRecoveryDataMap, false);
    VertexRecoveryData vertex3RecoveryData = new VertexRecoveryData(v3InitedEvent, v3ReconfigureDoneEvent, v3StartedEvent, null, v3taskRecoveryDataMap, false);
    doReturn(vertex1RecoveryData).when(dagRecoveryData).getVertexRecoveryData(v1Id);
    doReturn(vertex2RecoveryData).when(dagRecoveryData).getVertexRecoveryData(v2Id);
    doReturn(vertex3RecoveryData).when(dagRecoveryData).getVertexRecoveryData(v3Id);
    DAGEventRecoverEvent recoveryEvent = new DAGEventRecoverEvent(dagId, dagRecoveryData);
    dag.handle(recoveryEvent);
    dispatcher.await();
    VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
    VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
    VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
    assertEquals(DAGState.RUNNING, dag.getState());
    // reinitialize v1
    assertEquals(VertexState.INITIALIZING, v1.getState());
    // v2 skip initialization
    assertEquals(VertexState.RUNNING, v2.getState());
    assertEquals(v2InitedTime, v2.initedTime);
    assertEquals(v2StartedTime, v2.startedTime);
    assertEquals(v2NumTask, v2.getTotalTasks());
    // reinitialize v3
    assertEquals(VertexState.INITED, v3.getState());
}