Examples with DAGHistoryEvent org.apache.tez.dag.history.DAGHistoryEvent used on opensource projects

Example 41 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class AMContainerImpl method logStopped.

private void logStopped(int exitStatus) {
    final Clock clock = appContext.getClock();
    final HistoryEventHandler historyHandler = appContext.getHistoryHandler();
    ContainerStoppedEvent lEvt = new ContainerStoppedEvent(containerId, clock.getTime(), exitStatus, appContext.getApplicationAttemptId());
    historyHandler.handle(new DAGHistoryEvent(appContext.getCurrentDAGID(), lEvt));
}

Example 42 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class TestRecoveryService method testRecoveryFlushOnMaxEvents.

@Test(timeout = 5000)
public void testRecoveryFlushOnMaxEvents() throws Exception {
    setup(true, new String[][] { { TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, "10" }, { TezConfiguration.DAG_RECOVERY_FLUSH_INTERVAL_SECS, "-1" } });
    recoveryService.start();
    // Send 1 event less, wait for drain
    for (int i = 0; i < 9; ++i) {
        recoveryService.handle(new DAGHistoryEvent(dagId, new DAGStartedEvent(dagId, startTime, "nobody", "test-dag")));
    }
    waitForDrain(-1);
    verify(dagFos, times(0)).hflush();
    // This event should cause the flush.
    recoveryService.handle(new DAGHistoryEvent(dagId, new DAGStartedEvent(dagId, startTime, "nobody", "test-dag")));
    waitForDrain(-1);
    verify(dagFos, times(1)).hflush();
    recoveryService.stop();
}

Example 43 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class TestRecoveryService method testRecoveryFlushOnStop.

@Test(timeout = 50000)
public void testRecoveryFlushOnStop() throws Exception {
    setup(true, new String[][] { { TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, "-1" }, { TezConfiguration.DAG_RECOVERY_FLUSH_INTERVAL_SECS, "-1" } });
    recoveryService.start();
    // Does not flush on event counts.
    for (int i = 0; i < TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS_DEFAULT; ++i) {
        recoveryService.handle(new DAGHistoryEvent(dagId, new DAGStartedEvent(dagId, startTime, "nobody", "test-dag")));
    }
    waitForDrain(-1);
    verify(dagFos, times(0)).hflush();
    // Does not flush on timeout.
    Thread.sleep(TezConfiguration.DAG_RECOVERY_FLUSH_INTERVAL_SECS_DEFAULT * 1000);
    recoveryService.handle(new DAGHistoryEvent(dagId, new DAGStartedEvent(dagId, startTime, "nobody", "test-dag")));
    waitForDrain(-1);
    verify(dagFos, times(0)).hflush();
    // Does flush on stop.
    recoveryService.stop();
    verify(dagFos, times(1)).hflush();
}

Example 44 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class TestRecoveryService method testRecoveryFlushOnTimeoutEvents.

@Test(timeout = 10000)
public void testRecoveryFlushOnTimeoutEvents() throws Exception {
    setup(true, new String[][] { { TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, "-1" }, { TezConfiguration.DAG_RECOVERY_FLUSH_INTERVAL_SECS, "5" } });
    recoveryService.start();
    // Send lot of events.
    for (int i = 0; i < TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS_DEFAULT; ++i) {
        recoveryService.handle(new DAGHistoryEvent(dagId, new DAGStartedEvent(dagId, startTime, "nobody", "test-dag")));
    }
    // wait for timeout.
    Thread.sleep(5000);
    assertTrue(recoveryService.eventQueue.isEmpty());
    verify(fs, times(1)).create(eq(dagRecoveryPath), eq(false), anyInt());
    verify(dagFos, times(0)).hflush();
    // The flush is trigged by sending 1 event after the timeout.
    recoveryService.handle(new DAGHistoryEvent(dagId, new DAGStartedEvent(dagId, startTime, "nobody", "test-dag")));
    waitForDrain(1000);
    verify(dagFos, times(1)).hflush();
    recoveryService.stop();
}

Example 45 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class TestRecoveryService method testMultipleDAGFinishedEvent.

@Test(timeout = 5000)
public void testMultipleDAGFinishedEvent() throws Exception {
    setup(false, null);
    recoveryService.start();
    int randEventCount = new Random().nextInt(100) + 100;
    for (int i = 0; i < randEventCount; ++i) {
        recoveryService.handle(new DAGHistoryEvent(dagId, new TaskStartedEvent(tezTaskId, "v1", 0L, 0L)));
    }
    recoveryService.await();
    assertTrue(recoveryService.outputStreamMap.containsKey(dagId));
    // 2 DAGFinishedEvent
    recoveryService.handle(new DAGHistoryEvent(dagId, new DAGFinishedEvent(dagId, 1L, 2L, DAGState.FAILED, "diag", null, "user", "dag1", null, appAttemptId, null)));
    // outputStream removed
    assertFalse(recoveryService.outputStreamMap.containsKey(dagId));
    recoveryService.handle(new DAGHistoryEvent(dagId, new DAGFinishedEvent(dagId, 1L, 2L, DAGState.ERROR, "diag", null, "user", "dag1", null, appAttemptId, null)));
    // no new outputStream opened
    assertEquals(recoveryService.outputStreamMap.size(), 0);
    assertFalse(recoveryService.outputStreamMap.containsKey(dagId));
    recoveryService.stop();
}