use of org.apache.tez.dag.history.events.DAGCommitStartedEvent in project tez by apache.
the class TestRecoveryParser method testRecoverableSummary_DAGInCommitting.
@Test(timeout = 5000)
public void testRecoverableSummary_DAGInCommitting() throws IOException {
ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
TezDAGID dagID = TezDAGID.getInstance(appId, 1);
AppContext appContext = mock(AppContext.class);
when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1"));
when(appContext.getClock()).thenReturn(new SystemClock());
when(mockDAGImpl.getID()).thenReturn(dagID);
RecoveryService rService = new RecoveryService(appContext);
Configuration conf = new Configuration();
conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true);
rService.init(conf);
rService.start();
DAGPlan dagPlan = TestDAGImpl.createTestDAGPlan();
// write a DAGSubmittedEvent first to initialize summaryStream
rService.handle(new DAGHistoryEvent(dagID, new DAGSubmittedEvent(dagID, 1L, dagPlan, ApplicationAttemptId.newInstance(appId, 1), null, "user", new Configuration(), null, null)));
// It should be fine to skip other events, just for testing.
rService.handle(new DAGHistoryEvent(dagID, new DAGCommitStartedEvent(dagID, 0L)));
rService.stop();
DAGRecoveryData dagData = parser.parseRecoveryData();
assertEquals(dagID, dagData.recoveredDagID);
assertTrue(dagData.nonRecoverable);
assertTrue(dagData.reason.contains("DAG Commit was in progress"));
}
use of org.apache.tez.dag.history.events.DAGCommitStartedEvent in project tez by apache.
the class TestRecoveryParser method testSkipAllOtherEvents_1.
// skipAllOtherEvents due to non-recoverable (in the middle of commit)
@Test(timeout = 5000)
public void testSkipAllOtherEvents_1() throws IOException {
ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
TezDAGID dagID = TezDAGID.getInstance(appId, 1);
AppContext appContext = mock(AppContext.class);
when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1"));
when(appContext.getClock()).thenReturn(new SystemClock());
DAGPlan dagPlan = TestDAGImpl.createTestDAGPlan();
// write data in attempt_1
RecoveryService rService = new RecoveryService(appContext);
Configuration conf = new Configuration();
conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true);
rService.init(conf);
rService.start();
rService.handle(new DAGHistoryEvent(dagID, new DAGSubmittedEvent(dagID, 1L, dagPlan, ApplicationAttemptId.newInstance(appId, 1), null, "user", new Configuration(), null, null)));
rService.handle(new DAGHistoryEvent(dagID, new DAGInitializedEvent(dagID, 1L, "user", dagPlan.getName(), null)));
// only for testing, DAGCommitStartedEvent is not supposed to happen at this time.
rService.handle(new DAGHistoryEvent(dagID, new DAGCommitStartedEvent(dagID, System.currentTimeMillis())));
rService.stop();
// write data in attempt_2
when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/2"));
rService = new RecoveryService(appContext);
rService.init(conf);
rService.start();
// only for testing, DAGStartedEvent is not supposed to happen at this time.
rService.handle(new DAGHistoryEvent(dagID, new DAGStartedEvent(dagID, 1L, "user", "dag1")));
rService.stop();
DAGRecoveryData dagData = parser.parseRecoveryData();
assertEquals(true, dagData.nonRecoverable);
assertTrue(dagData.reason.contains("DAG Commit was in progress, not recoverable,"));
// DAGSubmittedEvent is handled but DAGInitializedEvent and DAGStartedEvent in the next attempt are both skipped
// due to the dag is not recoerable.
verify(mockAppMaster).createDAG(any(DAGPlan.class), any(TezDAGID.class));
assertNull(dagData.getDAGInitializedEvent());
assertNull(dagData.getDAGStartedEvent());
}
use of org.apache.tez.dag.history.events.DAGCommitStartedEvent in project tez by apache.
the class TestHistoryEventTimelineConversion method testHandlerExists.
@Test(timeout = 5000)
public void testHandlerExists() throws JSONException {
for (HistoryEventType eventType : HistoryEventType.values()) {
HistoryEvent event = null;
switch(eventType) {
case APP_LAUNCHED:
event = new AppLaunchedEvent(applicationId, random.nextInt(), random.nextInt(), user, new Configuration(false), null);
break;
case AM_LAUNCHED:
event = new AMLaunchedEvent(applicationAttemptId, random.nextInt(), random.nextInt(), user);
break;
case AM_STARTED:
event = new AMStartedEvent(applicationAttemptId, random.nextInt(), user);
break;
case DAG_SUBMITTED:
event = new DAGSubmittedEvent(tezDAGID, random.nextInt(), dagPlan, applicationAttemptId, null, user, null, containerLogs, null);
break;
case DAG_INITIALIZED:
event = new DAGInitializedEvent(tezDAGID, random.nextInt(), user, dagPlan.getName(), null);
break;
case DAG_STARTED:
event = new DAGStartedEvent(tezDAGID, random.nextInt(), user, dagPlan.getName());
break;
case DAG_FINISHED:
event = new DAGFinishedEvent(tezDAGID, random.nextInt(), random.nextInt(), DAGState.ERROR, null, null, user, dagPlan.getName(), null, applicationAttemptId, dagPlan);
break;
case VERTEX_INITIALIZED:
event = new VertexInitializedEvent(tezVertexID, "v1", random.nextInt(), random.nextInt(), random.nextInt(), "proc", null, null, null);
break;
case VERTEX_STARTED:
event = new VertexStartedEvent(tezVertexID, random.nextInt(), random.nextInt());
break;
case VERTEX_CONFIGURE_DONE:
event = new VertexConfigurationDoneEvent(tezVertexID, 0L, 1, null, null, null, true);
break;
case VERTEX_FINISHED:
event = new VertexFinishedEvent(tezVertexID, "v1", 1, random.nextInt(), random.nextInt(), random.nextInt(), random.nextInt(), random.nextInt(), VertexState.ERROR, null, null, null, null, null);
break;
case TASK_STARTED:
event = new TaskStartedEvent(tezTaskID, "v1", random.nextInt(), random.nextInt());
break;
case TASK_FINISHED:
event = new TaskFinishedEvent(tezTaskID, "v1", random.nextInt(), random.nextInt(), tezTaskAttemptID, TaskState.FAILED, null, null, 0);
break;
case TASK_ATTEMPT_STARTED:
event = new TaskAttemptStartedEvent(tezTaskAttemptID, "v1", random.nextInt(), containerId, nodeId, null, null, "nodeHttpAddress");
break;
case TASK_ATTEMPT_FINISHED:
event = new TaskAttemptFinishedEvent(tezTaskAttemptID, "v1", random.nextInt(), random.nextInt(), TaskAttemptState.FAILED, TaskFailureType.NON_FATAL, TaskAttemptTerminationCause.OUTPUT_LOST, null, null, null, null, 0, null, 0, containerId, nodeId, null, null, "nodeHttpAddress");
break;
case CONTAINER_LAUNCHED:
event = new ContainerLaunchedEvent(containerId, random.nextInt(), applicationAttemptId);
break;
case CONTAINER_STOPPED:
event = new ContainerStoppedEvent(containerId, random.nextInt(), -1, applicationAttemptId);
break;
case DAG_COMMIT_STARTED:
event = new DAGCommitStartedEvent();
break;
case VERTEX_COMMIT_STARTED:
event = new VertexCommitStartedEvent();
break;
case VERTEX_GROUP_COMMIT_STARTED:
event = new VertexGroupCommitStartedEvent();
break;
case VERTEX_GROUP_COMMIT_FINISHED:
event = new VertexGroupCommitFinishedEvent();
break;
case DAG_RECOVERED:
event = new DAGRecoveredEvent(applicationAttemptId, tezDAGID, dagPlan.getName(), user, random.nextLong(), containerLogs);
break;
case DAG_KILL_REQUEST:
event = new DAGKillRequestEvent();
break;
default:
Assert.fail("Unhandled event type " + eventType);
}
if (event == null || !event.isHistoryEvent()) {
continue;
}
HistoryEventTimelineConversion.convertToTimelineEntities(event);
}
}
use of org.apache.tez.dag.history.events.DAGCommitStartedEvent in project tez by apache.
the class TestRecoveryService method testRecoveryFlushOnSummaryEvent.
@Test(timeout = 5000)
public void testRecoveryFlushOnSummaryEvent() throws Exception {
setup(true, new String[][] { { TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, "-1" }, { TezConfiguration.DAG_RECOVERY_FLUSH_INTERVAL_SECS, "-1" } });
recoveryService.start();
DAGPlan dagPlan = DAGPlan.newBuilder().setName("test_dag").build();
// This writes to recovery immediately.
recoveryService.handle(new DAGHistoryEvent(dagId, new DAGSubmittedEvent(dagId, startTime, dagPlan, appAttemptId, null, "nobody", conf, null, "default")));
waitForDrain(-1);
verify(summaryFos, times(1)).hflush();
verify(dagFos, times(1)).hflush();
// This does not write to recovery immediately.
recoveryService.handle(new DAGHistoryEvent(dagId, new DAGCommitStartedEvent(dagId, startTime)));
waitForDrain(-1);
verify(summaryFos, times(2)).hflush();
verify(dagFos, times(1)).hflush();
// Does flush on stop.
recoveryService.stop();
verify(dagFos, times(2)).hflush();
}
use of org.apache.tez.dag.history.events.DAGCommitStartedEvent in project tez by apache.
the class TestRecoveryParser method testRecoverableSummary_DAGFinishCommitting.
@Test(timeout = 5000)
public void testRecoverableSummary_DAGFinishCommitting() throws IOException {
ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
TezDAGID dagID = TezDAGID.getInstance(appId, 1);
AppContext appContext = mock(AppContext.class);
when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1"));
when(appContext.getClock()).thenReturn(new SystemClock());
when(mockDAGImpl.getID()).thenReturn(dagID);
RecoveryService rService = new RecoveryService(appContext);
Configuration conf = new Configuration();
conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true);
rService.init(conf);
rService.start();
DAGPlan dagPlan = TestDAGImpl.createTestDAGPlan();
// write a DAGSubmittedEvent first to initialize summaryStream
rService.handle(new DAGHistoryEvent(dagID, new DAGSubmittedEvent(dagID, 1L, dagPlan, ApplicationAttemptId.newInstance(appId, 1), null, "user", new Configuration(), null, null)));
// It should be fine to skip other events, just for testing.
rService.handle(new DAGHistoryEvent(dagID, new DAGCommitStartedEvent(dagID, 0L)));
rService.handle(new DAGHistoryEvent(dagID, new DAGFinishedEvent(dagID, 1L, 2L, DAGState.FAILED, "diag", null, "user", "dag1", null, appAttemptId, dagPlan)));
rService.stop();
DAGRecoveryData dagData = parser.parseRecoveryData();
assertEquals(dagID, dagData.recoveredDagID);
assertEquals(DAGState.FAILED, dagData.dagState);
assertFalse(dagData.nonRecoverable);
assertNull(dagData.reason);
assertTrue(dagData.isCompleted);
}
Aggregations