use of org.apache.tez.dag.history.events.DAGStartedEvent in project tez by apache.
the class TestDAGRecovery method initMockDAGRecoveryDataForTask.
// ///////////////////////////// Task ////////////////////////////////////////////////////////////
private void initMockDAGRecoveryDataForTask() {
List<TezEvent> inputGeneratedTezEvents = new ArrayList<TezEvent>();
VertexInitializedEvent v1InitedEvent = new VertexInitializedEvent(v1Id, "vertex1", 0L, v1InitedTime, v1NumTask, "", null, inputGeneratedTezEvents, null);
Map<String, InputSpecUpdate> rootInputSpecs = new HashMap<String, InputSpecUpdate>();
VertexConfigurationDoneEvent v1ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v1Id, 0L, v1NumTask, null, null, rootInputSpecs, true);
VertexStartedEvent v1StartedEvent = new VertexStartedEvent(v1Id, 0L, v1StartedTime);
VertexRecoveryData v1RecoveryData = new VertexRecoveryData(v1InitedEvent, v1ReconfigureDoneEvent, v1StartedEvent, null, new HashMap<TezTaskID, TaskRecoveryData>(), false);
DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagId, dagInitedTime, "user", "dagName", null);
DAGStartedEvent dagStartedEvent = new DAGStartedEvent(dagId, dagStartedTime, "user", "dagName");
doReturn(v1RecoveryData).when(dagRecoveryData).getVertexRecoveryData(v1Id);
doReturn(dagInitedEvent).when(dagRecoveryData).getDAGInitializedEvent();
doReturn(dagStartedEvent).when(dagRecoveryData).getDAGStartedEvent();
}
use of org.apache.tez.dag.history.events.DAGStartedEvent in project tez by apache.
the class TestRecoveryParser method testSkipAllOtherEvents_1.
// skipAllOtherEvents due to non-recoverable (in the middle of commit)
@Test(timeout = 5000)
public void testSkipAllOtherEvents_1() throws IOException {
ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
TezDAGID dagID = TezDAGID.getInstance(appId, 1);
AppContext appContext = mock(AppContext.class);
when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1"));
when(appContext.getClock()).thenReturn(new SystemClock());
DAGPlan dagPlan = TestDAGImpl.createTestDAGPlan();
// write data in attempt_1
RecoveryService rService = new RecoveryService(appContext);
Configuration conf = new Configuration();
conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true);
rService.init(conf);
rService.start();
rService.handle(new DAGHistoryEvent(dagID, new DAGSubmittedEvent(dagID, 1L, dagPlan, ApplicationAttemptId.newInstance(appId, 1), null, "user", new Configuration(), null, null)));
rService.handle(new DAGHistoryEvent(dagID, new DAGInitializedEvent(dagID, 1L, "user", dagPlan.getName(), null)));
// only for testing, DAGCommitStartedEvent is not supposed to happen at this time.
rService.handle(new DAGHistoryEvent(dagID, new DAGCommitStartedEvent(dagID, System.currentTimeMillis())));
rService.stop();
// write data in attempt_2
when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/2"));
rService = new RecoveryService(appContext);
rService.init(conf);
rService.start();
// only for testing, DAGStartedEvent is not supposed to happen at this time.
rService.handle(new DAGHistoryEvent(dagID, new DAGStartedEvent(dagID, 1L, "user", "dag1")));
rService.stop();
DAGRecoveryData dagData = parser.parseRecoveryData();
assertEquals(true, dagData.nonRecoverable);
assertTrue(dagData.reason.contains("DAG Commit was in progress, not recoverable,"));
// DAGSubmittedEvent is handled but DAGInitializedEvent and DAGStartedEvent in the next attempt are both skipped
// due to the dag is not recoerable.
verify(mockAppMaster).createDAG(any(DAGPlan.class), any(TezDAGID.class));
assertNull(dagData.getDAGInitializedEvent());
assertNull(dagData.getDAGStartedEvent());
}
use of org.apache.tez.dag.history.events.DAGStartedEvent in project tez by apache.
the class TestRecoveryParser method testSkipAllOtherEvents_2.
// skipAllOtherEvents due to dag finished
@Test(timeout = 5000)
public void testSkipAllOtherEvents_2() throws IOException {
ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
TezDAGID dagID = TezDAGID.getInstance(appId, 1);
AppContext appContext = mock(AppContext.class);
when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1"));
when(appContext.getClock()).thenReturn(new SystemClock());
DAGPlan dagPlan = TestDAGImpl.createTestDAGPlan();
// write data in attempt_1
RecoveryService rService = new RecoveryService(appContext);
Configuration conf = new Configuration();
conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true);
rService.init(conf);
rService.start();
rService.handle(new DAGHistoryEvent(dagID, new DAGSubmittedEvent(dagID, 1L, dagPlan, ApplicationAttemptId.newInstance(appId, 1), null, "user", new Configuration(), null, null)));
rService.handle(new DAGHistoryEvent(dagID, new DAGInitializedEvent(dagID, 1L, "user", dagPlan.getName(), null)));
rService.handle(new DAGHistoryEvent(dagID, new DAGFinishedEvent(dagID, 1L, 2L, DAGState.FAILED, "diag", null, "user", "dag1", null, appAttemptId, dagPlan)));
rService.handle(new DAGHistoryEvent(dagID, new DAGStartedEvent(dagID, 1L, "user", "dag1")));
rService.stop();
// write data in attempt_2
when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/2"));
rService = new RecoveryService(appContext);
rService.init(conf);
rService.start();
rService.handle(new DAGHistoryEvent(dagID, new DAGStartedEvent(dagID, 1L, "user", "dag1")));
rService.stop();
DAGRecoveryData dagData = parser.parseRecoveryData();
assertEquals(false, dagData.nonRecoverable);
assertEquals(DAGState.FAILED, dagData.dagState);
assertEquals(true, dagData.isCompleted);
// DAGSubmittedEvent, DAGInitializedEvent and DAGFinishedEvent is handled
verify(mockAppMaster).createDAG(any(DAGPlan.class), any(TezDAGID.class));
// DAGInitializedEvent may not been handled before DAGFinishedEvent,
// because DAGFinishedEvent's writeToRecoveryImmediately is true
assertNotNull(dagData.getDAGFinishedEvent());
assertNull(dagData.getDAGStartedEvent());
}
use of org.apache.tez.dag.history.events.DAGStartedEvent in project tez by apache.
the class TestRecoveryParser method testRecoveryData.
@Test(timeout = 5000)
public void testRecoveryData() throws IOException {
ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
TezDAGID dagID = TezDAGID.getInstance(appId, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
AppContext appContext = mock(AppContext.class);
when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1"));
when(appContext.getClock()).thenReturn(new SystemClock());
when(mockDAGImpl.getID()).thenReturn(dagID);
when(appContext.getHadoopShim()).thenReturn(new DefaultHadoopShim());
when(appContext.getApplicationID()).thenReturn(appId);
RecoveryService rService = new RecoveryService(appContext);
Configuration conf = new Configuration();
conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true);
rService.init(conf);
rService.start();
DAGPlan dagPlan = TestDAGImpl.createTestDAGPlan();
// DAG DAGSubmittedEvent -> DAGInitializedEvent -> DAGStartedEvent
rService.handle(new DAGHistoryEvent(dagID, new DAGSubmittedEvent(dagID, 1L, dagPlan, ApplicationAttemptId.newInstance(appId, 1), null, "user", new Configuration(), null, null)));
DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagID, 100L, "user", "dagName", null);
DAGStartedEvent dagStartedEvent = new DAGStartedEvent(dagID, 0L, "user", "dagName");
rService.handle(new DAGHistoryEvent(dagID, dagInitedEvent));
rService.handle(new DAGHistoryEvent(dagID, dagStartedEvent));
// 3 vertices of this dag: v0, v1, v2
TezVertexID v0Id = TezVertexID.getInstance(dagID, 0);
TezVertexID v1Id = TezVertexID.getInstance(dagID, 1);
TezVertexID v2Id = TezVertexID.getInstance(dagID, 2);
// v0 VertexInitializedEvent
VertexInitializedEvent v0InitedEvent = new VertexInitializedEvent(v0Id, "v0", 200L, 400L, 2, null, null, null, null);
rService.handle(new DAGHistoryEvent(dagID, v0InitedEvent));
// v1 VertexFinishedEvent(KILLED)
VertexFinishedEvent v1FinishedEvent = new VertexFinishedEvent(v1Id, "v1", 2, 300L, 400L, 500L, 600L, 700L, VertexState.KILLED, "", null, null, null, null);
rService.handle(new DAGHistoryEvent(dagID, v1FinishedEvent));
// v2 VertexInitializedEvent -> VertexStartedEvent
List<TezEvent> initGeneratedEvents = Lists.newArrayList(new TezEvent(DataMovementEvent.create(ByteBuffer.wrap(new byte[0])), null));
VertexInitializedEvent v2InitedEvent = new VertexInitializedEvent(v2Id, "v2", 200L, 300L, 2, null, null, initGeneratedEvents, null);
VertexStartedEvent v2StartedEvent = new VertexStartedEvent(v2Id, 0L, 0L);
rService.handle(new DAGHistoryEvent(dagID, v2InitedEvent));
rService.handle(new DAGHistoryEvent(dagID, v2StartedEvent));
// 3 tasks of v2
TezTaskID t0v2Id = TezTaskID.getInstance(v2Id, 0);
TezTaskID t1v2Id = TezTaskID.getInstance(v2Id, 1);
TezTaskID t2v2Id = TezTaskID.getInstance(v2Id, 2);
// t0v2 TaskStartedEvent
TaskStartedEvent t0v2StartedEvent = new TaskStartedEvent(t0v2Id, "v2", 400L, 5000L);
rService.handle(new DAGHistoryEvent(dagID, t0v2StartedEvent));
// t1v2 TaskFinishedEvent
TaskFinishedEvent t1v2FinishedEvent = new TaskFinishedEvent(t1v2Id, "v1", 0L, 0L, null, TaskState.KILLED, "", null, 4);
rService.handle(new DAGHistoryEvent(dagID, t1v2FinishedEvent));
// t2v2 TaskStartedEvent -> TaskFinishedEvent
TaskStartedEvent t2v2StartedEvent = new TaskStartedEvent(t2v2Id, "v2", 400L, 500L);
rService.handle(new DAGHistoryEvent(dagID, t2v2StartedEvent));
TaskFinishedEvent t2v2FinishedEvent = new TaskFinishedEvent(t2v2Id, "v1", 0L, 0L, null, TaskState.SUCCEEDED, "", null, 4);
rService.handle(new DAGHistoryEvent(dagID, t2v2FinishedEvent));
// attempts under t0v2
ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
NodeId nodeId = NodeId.newInstance("localhost", 9999);
TezTaskAttemptID ta0t0v2Id = TezTaskAttemptID.getInstance(t0v2Id, 0);
TaskAttemptStartedEvent ta0t0v2StartedEvent = new TaskAttemptStartedEvent(ta0t0v2Id, "v1", 0L, containerId, nodeId, "", "", "");
rService.handle(new DAGHistoryEvent(dagID, ta0t0v2StartedEvent));
// attempts under t2v2
TezTaskAttemptID ta0t2v2Id = TezTaskAttemptID.getInstance(t2v2Id, 0);
TaskAttemptStartedEvent ta0t2v2StartedEvent = new TaskAttemptStartedEvent(ta0t2v2Id, "v1", 500L, containerId, nodeId, "", "", "");
rService.handle(new DAGHistoryEvent(dagID, ta0t2v2StartedEvent));
TaskAttemptFinishedEvent ta0t2v2FinishedEvent = new TaskAttemptFinishedEvent(ta0t2v2Id, "v1", 500L, 600L, TaskAttemptState.SUCCEEDED, null, null, "", null, null, null, 0L, null, 0L, null, null, null, null, null);
rService.handle(new DAGHistoryEvent(dagID, ta0t2v2FinishedEvent));
rService.stop();
DAGRecoveryData dagData = parser.parseRecoveryData();
assertFalse(dagData.nonRecoverable);
// There's no equals method for the history event, so here only verify the init/start/finish time of each event for simplicity
assertEquals(dagInitedEvent.getInitTime(), dagData.getDAGInitializedEvent().getInitTime());
assertEquals(dagStartedEvent.getStartTime(), dagData.getDAGStartedEvent().getStartTime());
assertNull(dagData.getDAGFinishedEvent());
VertexRecoveryData v0Data = dagData.getVertexRecoveryData(v0Id);
VertexRecoveryData v1Data = dagData.getVertexRecoveryData(v1Id);
VertexRecoveryData v2Data = dagData.getVertexRecoveryData(v2Id);
assertNotNull(v0Data);
assertNotNull(v1Data);
assertNotNull(v2Data);
assertEquals(v0InitedEvent.getInitedTime(), v0Data.getVertexInitedEvent().getInitedTime());
assertNull(v0Data.getVertexStartedEvent());
assertNull(v1Data.getVertexInitedEvent());
assertEquals(v1FinishedEvent.getFinishTime(), v1Data.getVertexFinishedEvent().getFinishTime());
assertEquals(v2InitedEvent.getInitedTime(), v2Data.getVertexInitedEvent().getInitedTime());
assertEquals(v2StartedEvent.getStartTime(), v2Data.getVertexStartedEvent().getStartTime());
TaskRecoveryData t0v2Data = dagData.getTaskRecoveryData(t0v2Id);
TaskRecoveryData t1v2Data = dagData.getTaskRecoveryData(t1v2Id);
TaskRecoveryData t2v2Data = dagData.getTaskRecoveryData(t2v2Id);
assertNotNull(t0v2Data);
assertNotNull(t1v2Data);
assertNotNull(t2v2Data);
assertEquals(t0v2StartedEvent.getStartTime(), t0v2Data.getTaskStartedEvent().getStartTime());
assertNull(t0v2Data.getTaskFinishedEvent());
assertEquals(t1v2FinishedEvent.getFinishTime(), t1v2Data.getTaskFinishedEvent().getFinishTime());
assertNull(t1v2Data.getTaskStartedEvent());
assertEquals(t2v2StartedEvent.getStartTime(), t2v2Data.getTaskStartedEvent().getStartTime());
assertEquals(t2v2FinishedEvent.getFinishTime(), t2v2Data.getTaskFinishedEvent().getFinishTime());
TaskAttemptRecoveryData ta0t0v2Data = dagData.getTaskAttemptRecoveryData(ta0t0v2Id);
TaskAttemptRecoveryData ta0t2v2Data = dagData.getTaskAttemptRecoveryData(ta0t2v2Id);
assertNotNull(ta0t0v2Data);
assertNotNull(ta0t2v2Data);
assertEquals(ta0t0v2StartedEvent.getStartTime(), ta0t0v2Data.getTaskAttemptStartedEvent().getStartTime());
assertNull(ta0t0v2Data.getTaskAttemptFinishedEvent());
assertEquals(ta0t2v2StartedEvent.getStartTime(), ta0t2v2Data.getTaskAttemptStartedEvent().getStartTime());
assertEquals(ta0t2v2FinishedEvent.getFinishTime(), ta0t2v2Data.getTaskAttemptFinishedEvent().getFinishTime());
}
use of org.apache.tez.dag.history.events.DAGStartedEvent in project tez by apache.
the class TestATSHistoryLoggingService method testATSEventBatching.
@Test(timeout = 20000)
public void testATSEventBatching() {
atsHistoryLoggingService.start();
TezDAGID tezDAGID = TezDAGID.getInstance(ApplicationId.newInstance(100l, 1), 1);
DAGHistoryEvent historyEvent = new DAGHistoryEvent(tezDAGID, new DAGStartedEvent(tezDAGID, 1001l, "user1", "dagName1"));
for (int i = 0; i < 100; ++i) {
atsHistoryLoggingService.handle(historyEvent);
}
try {
Thread.sleep(1000l);
} catch (InterruptedException e) {
// Do nothing
}
LOG.info("ATS entitiesSent=" + atsEntitiesCounter + ", timelineInvocations=" + atsInvokeCounter);
Assert.assertTrue(atsEntitiesCounter > atsInvokeCounter);
Assert.assertEquals(atsEntitiesCounter / 2, atsInvokeCounter);
}
Aggregations