Examples with DAGHistoryEvent org.apache.tez.dag.history.DAGHistoryEvent used on opensource projects

Example 51 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class TestRecoveryParser method testLastCorruptedSummaryRecord.

@Test(timeout = 5000)
public void testLastCorruptedSummaryRecord() throws IOException {
    ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
    TezDAGID dagID = TezDAGID.getInstance(appId, 1);
    AppContext appContext = mock(AppContext.class);
    when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1"));
    when(appContext.getClock()).thenReturn(new SystemClock());
    // write data in attempt_1
    RecoveryService rService = new RecoveryService(appContext);
    Configuration conf = new Configuration();
    conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true);
    rService.init(conf);
    rService.start();
    DAGPlan dagPlan = TestDAGImpl.createTestDAGPlan();
    // write a DAGSubmittedEvent first to initialize summaryStream
    rService.handle(new DAGHistoryEvent(dagID, new DAGSubmittedEvent(dagID, 1L, dagPlan, ApplicationAttemptId.newInstance(appId, 1), null, "user", new Configuration(), null, null)));
    // write an corrupted SummaryEvent
    rService.summaryStream.writeChars("INVALID_DATA");
    rService.stop();
    try {
        // Corrupted SummaryEvent will cause recovery fail (throw exception here)
        parser.parseRecoveryData();
        fail();
    } catch (IOException e) {
        // exception when parsing protobuf object
        e.printStackTrace();
    }
}

Example 52 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class TestRecoveryParser method testRecoverableSummary_DAGFinishCommitting.

@Test(timeout = 5000)
public void testRecoverableSummary_DAGFinishCommitting() throws IOException {
    ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    TezDAGID dagID = TezDAGID.getInstance(appId, 1);
    AppContext appContext = mock(AppContext.class);
    when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1"));
    when(appContext.getClock()).thenReturn(new SystemClock());
    when(mockDAGImpl.getID()).thenReturn(dagID);
    RecoveryService rService = new RecoveryService(appContext);
    Configuration conf = new Configuration();
    conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true);
    rService.init(conf);
    rService.start();
    DAGPlan dagPlan = TestDAGImpl.createTestDAGPlan();
    // write a DAGSubmittedEvent first to initialize summaryStream
    rService.handle(new DAGHistoryEvent(dagID, new DAGSubmittedEvent(dagID, 1L, dagPlan, ApplicationAttemptId.newInstance(appId, 1), null, "user", new Configuration(), null, null)));
    // It should be fine to skip other events, just for testing.
    rService.handle(new DAGHistoryEvent(dagID, new DAGCommitStartedEvent(dagID, 0L)));
    rService.handle(new DAGHistoryEvent(dagID, new DAGFinishedEvent(dagID, 1L, 2L, DAGState.FAILED, "diag", null, "user", "dag1", null, appAttemptId, dagPlan)));
    rService.stop();
    DAGRecoveryData dagData = parser.parseRecoveryData();
    assertEquals(dagID, dagData.recoveredDagID);
    assertEquals(DAGState.FAILED, dagData.dagState);
    assertFalse(dagData.nonRecoverable);
    assertNull(dagData.reason);
    assertTrue(dagData.isCompleted);
}

Example 53 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class TestRecoveryParser method testRecoverableSummary_VertexInCommitting.

@Test(timeout = 5000)
public void testRecoverableSummary_VertexInCommitting() throws IOException {
    ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
    TezDAGID dagID = TezDAGID.getInstance(appId, 1);
    AppContext appContext = mock(AppContext.class);
    when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1"));
    when(appContext.getClock()).thenReturn(new SystemClock());
    when(mockDAGImpl.getID()).thenReturn(dagID);
    RecoveryService rService = new RecoveryService(appContext);
    Configuration conf = new Configuration();
    conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true);
    rService.init(conf);
    rService.start();
    DAGPlan dagPlan = TestDAGImpl.createTestDAGPlan();
    // write a DAGSubmittedEvent first to initialize summaryStream
    rService.handle(new DAGHistoryEvent(dagID, new DAGSubmittedEvent(dagID, 1L, dagPlan, ApplicationAttemptId.newInstance(appId, 1), null, "user", new Configuration(), null, null)));
    // It should be fine to skip other events, just for testing.
    rService.handle(new DAGHistoryEvent(dagID, new VertexCommitStartedEvent(TezVertexID.getInstance(dagID, 0), 0L)));
    rService.stop();
    DAGRecoveryData dagData = parser.parseRecoveryData();
    assertEquals(dagID, dagData.recoveredDagID);
    assertTrue(dagData.nonRecoverable);
    assertTrue(dagData.reason.contains("Vertex Commit was in progress"));
}

Example 54 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class TestRecoveryService method testDrainEvents.

@Test(timeout = 5000)
public void testDrainEvents() throws Exception {
    setup(false, null);
    recoveryService.start();
    int randEventCount = new Random().nextInt(100) + 100;
    for (int i = 0; i < randEventCount; ++i) {
        recoveryService.handle(new DAGHistoryEvent(dagId, new TaskStartedEvent(tezTaskId, "v1", 0L, 0L)));
    }
    recoveryService.stop();
    assertEquals(randEventCount, recoveryService.processedRecoveryEventCounter.get());
}

Example 55 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class TestRecoveryService method testRecoveryFlush.

@Test(timeout = 10000)
public void testRecoveryFlush() throws Exception {
    setup(true, new String[][] { { TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, "10" }, { TezConfiguration.DAG_RECOVERY_FLUSH_INTERVAL_SECS, "5" } });
    recoveryService.start();
    // 5 second flush
    recoveryService.handle(new DAGHistoryEvent(dagId, new DAGStartedEvent(dagId, startTime, "nobody", "test-dag")));
    Thread.sleep(5000);
    assertTrue(recoveryService.eventQueue.isEmpty());
    verify(fs, times(1)).create(eq(dagRecoveryPath), eq(false), anyInt());
    verify(dagFos, times(0)).hflush();
    recoveryService.handle(new DAGHistoryEvent(dagId, new DAGStartedEvent(dagId, startTime, "nobody", "test-dag")));
    waitForDrain(1000);
    verify(dagFos, times(1)).hflush();
    // Number of events flush.
    for (int i = 0; i < 9; ++i) {
        recoveryService.handle(new DAGHistoryEvent(dagId, new DAGStartedEvent(dagId, startTime, "nobody", "test-dag")));
    }
    waitForDrain(-1);
    verify(dagFos, times(1)).hflush();
    recoveryService.handle(new DAGHistoryEvent(dagId, new DAGStartedEvent(dagId, startTime, "nobody", "test-dag")));
    waitForDrain(-1);
    verify(dagFos, times(2)).hflush();
    recoveryService.handle(new DAGHistoryEvent(dagId, new DAGStartedEvent(dagId, startTime, "nobody", "test-dag")));
    recoveryService.stop();
}