use of org.apache.tez.dag.history.events.DAGInitializedEvent in project tez by apache.
the class DAGImpl method logJobHistoryInitedEvent.
void logJobHistoryInitedEvent() {
if (recoveryData == null || recoveryData.getDAGInitializedEvent() == null) {
DAGInitializedEvent initEvt = new DAGInitializedEvent(this.dagId, clock.getTime(), this.userName, this.dagName, this.getVertexNameIDMapping());
this.appContext.getHistoryHandler().handle(new DAGHistoryEvent(dagId, initEvt));
}
}
use of org.apache.tez.dag.history.events.DAGInitializedEvent in project tez by apache.
the class TestDAGRecovery method initMockDAGRecoveryDataForTask.
// ///////////////////////////// Task ////////////////////////////////////////////////////////////
private void initMockDAGRecoveryDataForTask() {
List<TezEvent> inputGeneratedTezEvents = new ArrayList<TezEvent>();
VertexInitializedEvent v1InitedEvent = new VertexInitializedEvent(v1Id, "vertex1", 0L, v1InitedTime, v1NumTask, "", null, inputGeneratedTezEvents, null);
Map<String, InputSpecUpdate> rootInputSpecs = new HashMap<String, InputSpecUpdate>();
VertexConfigurationDoneEvent v1ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v1Id, 0L, v1NumTask, null, null, rootInputSpecs, true);
VertexStartedEvent v1StartedEvent = new VertexStartedEvent(v1Id, 0L, v1StartedTime);
VertexRecoveryData v1RecoveryData = new VertexRecoveryData(v1InitedEvent, v1ReconfigureDoneEvent, v1StartedEvent, null, new HashMap<TezTaskID, TaskRecoveryData>(), false);
DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagId, dagInitedTime, "user", "dagName", null);
DAGStartedEvent dagStartedEvent = new DAGStartedEvent(dagId, dagStartedTime, "user", "dagName");
doReturn(v1RecoveryData).when(dagRecoveryData).getVertexRecoveryData(v1Id);
doReturn(dagInitedEvent).when(dagRecoveryData).getDAGInitializedEvent();
doReturn(dagStartedEvent).when(dagRecoveryData).getDAGStartedEvent();
}
use of org.apache.tez.dag.history.events.DAGInitializedEvent in project tez by apache.
the class TestRecoveryParser method testSkipAllOtherEvents_1.
// skipAllOtherEvents due to non-recoverable (in the middle of commit)
@Test(timeout = 5000)
public void testSkipAllOtherEvents_1() throws IOException {
ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
TezDAGID dagID = TezDAGID.getInstance(appId, 1);
AppContext appContext = mock(AppContext.class);
when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1"));
when(appContext.getClock()).thenReturn(new SystemClock());
DAGPlan dagPlan = TestDAGImpl.createTestDAGPlan();
// write data in attempt_1
RecoveryService rService = new RecoveryService(appContext);
Configuration conf = new Configuration();
conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true);
rService.init(conf);
rService.start();
rService.handle(new DAGHistoryEvent(dagID, new DAGSubmittedEvent(dagID, 1L, dagPlan, ApplicationAttemptId.newInstance(appId, 1), null, "user", new Configuration(), null, null)));
rService.handle(new DAGHistoryEvent(dagID, new DAGInitializedEvent(dagID, 1L, "user", dagPlan.getName(), null)));
// only for testing, DAGCommitStartedEvent is not supposed to happen at this time.
rService.handle(new DAGHistoryEvent(dagID, new DAGCommitStartedEvent(dagID, System.currentTimeMillis())));
rService.stop();
// write data in attempt_2
when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/2"));
rService = new RecoveryService(appContext);
rService.init(conf);
rService.start();
// only for testing, DAGStartedEvent is not supposed to happen at this time.
rService.handle(new DAGHistoryEvent(dagID, new DAGStartedEvent(dagID, 1L, "user", "dag1")));
rService.stop();
DAGRecoveryData dagData = parser.parseRecoveryData();
assertEquals(true, dagData.nonRecoverable);
assertTrue(dagData.reason.contains("DAG Commit was in progress, not recoverable,"));
// DAGSubmittedEvent is handled but DAGInitializedEvent and DAGStartedEvent in the next attempt are both skipped
// due to the dag is not recoerable.
verify(mockAppMaster).createDAG(any(DAGPlan.class), any(TezDAGID.class));
assertNull(dagData.getDAGInitializedEvent());
assertNull(dagData.getDAGStartedEvent());
}
use of org.apache.tez.dag.history.events.DAGInitializedEvent in project tez by apache.
the class TestRecoveryParser method testSkipAllOtherEvents_2.
// skipAllOtherEvents due to dag finished
@Test(timeout = 5000)
public void testSkipAllOtherEvents_2() throws IOException {
ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
TezDAGID dagID = TezDAGID.getInstance(appId, 1);
AppContext appContext = mock(AppContext.class);
when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1"));
when(appContext.getClock()).thenReturn(new SystemClock());
DAGPlan dagPlan = TestDAGImpl.createTestDAGPlan();
// write data in attempt_1
RecoveryService rService = new RecoveryService(appContext);
Configuration conf = new Configuration();
conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true);
rService.init(conf);
rService.start();
rService.handle(new DAGHistoryEvent(dagID, new DAGSubmittedEvent(dagID, 1L, dagPlan, ApplicationAttemptId.newInstance(appId, 1), null, "user", new Configuration(), null, null)));
rService.handle(new DAGHistoryEvent(dagID, new DAGInitializedEvent(dagID, 1L, "user", dagPlan.getName(), null)));
rService.handle(new DAGHistoryEvent(dagID, new DAGFinishedEvent(dagID, 1L, 2L, DAGState.FAILED, "diag", null, "user", "dag1", null, appAttemptId, dagPlan)));
rService.handle(new DAGHistoryEvent(dagID, new DAGStartedEvent(dagID, 1L, "user", "dag1")));
rService.stop();
// write data in attempt_2
when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/2"));
rService = new RecoveryService(appContext);
rService.init(conf);
rService.start();
rService.handle(new DAGHistoryEvent(dagID, new DAGStartedEvent(dagID, 1L, "user", "dag1")));
rService.stop();
DAGRecoveryData dagData = parser.parseRecoveryData();
assertEquals(false, dagData.nonRecoverable);
assertEquals(DAGState.FAILED, dagData.dagState);
assertEquals(true, dagData.isCompleted);
// DAGSubmittedEvent, DAGInitializedEvent and DAGFinishedEvent is handled
verify(mockAppMaster).createDAG(any(DAGPlan.class), any(TezDAGID.class));
// DAGInitializedEvent may not been handled before DAGFinishedEvent,
// because DAGFinishedEvent's writeToRecoveryImmediately is true
assertNotNull(dagData.getDAGFinishedEvent());
assertNull(dagData.getDAGStartedEvent());
}
use of org.apache.tez.dag.history.events.DAGInitializedEvent in project tez by apache.
the class TestRecoveryParser method testLastCorruptedRecoveryRecord.
@Test(timeout = 5000)
public void testLastCorruptedRecoveryRecord() throws IOException {
ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
TezDAGID dagID = TezDAGID.getInstance(appId, 1);
AppContext appContext = mock(AppContext.class);
when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1"));
when(appContext.getClock()).thenReturn(new SystemClock());
when(appContext.getHadoopShim()).thenReturn(new DefaultHadoopShim());
when(appContext.getApplicationID()).thenReturn(appId);
DAGPlan dagPlan = TestDAGImpl.createTestDAGPlan();
// write data in attempt_1
RecoveryService rService = new RecoveryService(appContext);
Configuration conf = new Configuration();
conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true);
rService.init(conf);
rService.start();
rService.handle(new DAGHistoryEvent(dagID, new DAGSubmittedEvent(dagID, 1L, dagPlan, ApplicationAttemptId.newInstance(appId, 1), null, "user", new Configuration(), null, null)));
// wait until DAGSubmittedEvent is handled in the RecoveryEventHandling thread
rService.await();
rService.outputStreamMap.get(dagID).writeUTF("INVALID_DATA");
rService.stop();
// write data in attempt_2
when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/2"));
rService = new RecoveryService(appContext);
rService.init(conf);
rService.start();
rService.handle(new DAGHistoryEvent(dagID, new DAGInitializedEvent(dagID, 1L, "user", dagPlan.getName(), null)));
rService.await();
rService.outputStreamMap.get(dagID).writeUTF("INVALID_DATA");
rService.stop();
// corrupted last records will be skipped but the whole recovery logs will be read
DAGRecoveryData dagData = parser.parseRecoveryData();
assertEquals(false, dagData.isCompleted);
assertEquals(null, dagData.reason);
assertEquals(false, dagData.nonRecoverable);
// verify DAGSubmitedEvent & DAGInititlizedEvent is handled.
verify(mockAppMaster).createDAG(any(DAGPlan.class), any(TezDAGID.class));
assertNotNull(dagData.getDAGInitializedEvent());
}
Aggregations