Examples with DAGStartedEvent org.apache.tez.dag.history.events.DAGStartedEvent used on opensource projects

Example 16 with DAGStartedEvent

use of org.apache.tez.dag.history.events.DAGStartedEvent in project tez by apache.

the class TestRecoveryService method testRecoveryFlush.

@Test(timeout = 10000)
public void testRecoveryFlush() throws Exception {
    setup(true, new String[][] { { TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, "10" }, { TezConfiguration.DAG_RECOVERY_FLUSH_INTERVAL_SECS, "5" } });
    recoveryService.start();
    // 5 second flush
    recoveryService.handle(new DAGHistoryEvent(dagId, new DAGStartedEvent(dagId, startTime, "nobody", "test-dag")));
    Thread.sleep(5000);
    assertTrue(recoveryService.eventQueue.isEmpty());
    verify(fs, times(1)).create(eq(dagRecoveryPath), eq(false), anyInt());
    verify(dagFos, times(0)).hflush();
    recoveryService.handle(new DAGHistoryEvent(dagId, new DAGStartedEvent(dagId, startTime, "nobody", "test-dag")));
    waitForDrain(1000);
    verify(dagFos, times(1)).hflush();
    // Number of events flush.
    for (int i = 0; i < 9; ++i) {
        recoveryService.handle(new DAGHistoryEvent(dagId, new DAGStartedEvent(dagId, startTime, "nobody", "test-dag")));
    }
    waitForDrain(-1);
    verify(dagFos, times(1)).hflush();
    recoveryService.handle(new DAGHistoryEvent(dagId, new DAGStartedEvent(dagId, startTime, "nobody", "test-dag")));
    waitForDrain(-1);
    verify(dagFos, times(2)).hflush();
    recoveryService.handle(new DAGHistoryEvent(dagId, new DAGStartedEvent(dagId, startTime, "nobody", "test-dag")));
    recoveryService.stop();
}

Example 17 with DAGStartedEvent

use of org.apache.tez.dag.history.events.DAGStartedEvent in project tez by apache.

the class DAGImpl method logJobHistoryStartedEvent.

void logJobHistoryStartedEvent() {
    if (recoveryData == null || recoveryData.getDAGStartedEvent() == null) {
        DAGStartedEvent startEvt = new DAGStartedEvent(this.dagId, this.startTime, this.userName, this.dagName);
        this.appContext.getHistoryHandler().handle(new DAGHistoryEvent(dagId, startEvt));
    }
}

Example 18 with DAGStartedEvent

use of org.apache.tez.dag.history.events.DAGStartedEvent in project tez by apache.

the class RecoveryParser method getNextEvent.

private static HistoryEvent getNextEvent(FSDataInputStream inputStream) throws IOException {
    int eventTypeOrdinal = -1;
    try {
        eventTypeOrdinal = inputStream.readInt();
    } catch (EOFException eof) {
        return null;
    }
    if (eventTypeOrdinal < 0 || eventTypeOrdinal >= HistoryEventType.values().length) {
        // reached end
        throw new IOException("Corrupt data found when trying to read next event type" + ", eventTypeOrdinal=" + eventTypeOrdinal);
    }
    HistoryEventType eventType = HistoryEventType.values()[eventTypeOrdinal];
    HistoryEvent event;
    switch(eventType) {
        case AM_LAUNCHED:
            event = new AMLaunchedEvent();
            break;
        case AM_STARTED:
            event = new AMStartedEvent();
            break;
        case DAG_SUBMITTED:
            event = new DAGSubmittedEvent();
            break;
        case DAG_INITIALIZED:
            event = new DAGInitializedEvent();
            break;
        case DAG_STARTED:
            event = new DAGStartedEvent();
            break;
        case DAG_COMMIT_STARTED:
            event = new DAGCommitStartedEvent();
            break;
        case DAG_FINISHED:
            event = new DAGFinishedEvent();
            break;
        case DAG_KILL_REQUEST:
            event = new DAGKillRequestEvent();
            break;
        case CONTAINER_LAUNCHED:
            event = new ContainerLaunchedEvent();
            break;
        case CONTAINER_STOPPED:
            event = new ContainerStoppedEvent();
            break;
        case VERTEX_INITIALIZED:
            event = new VertexInitializedEvent();
            break;
        case VERTEX_CONFIGURE_DONE:
            event = new VertexConfigurationDoneEvent();
            break;
        case VERTEX_STARTED:
            event = new VertexStartedEvent();
            break;
        case VERTEX_COMMIT_STARTED:
            event = new VertexCommitStartedEvent();
            break;
        case VERTEX_GROUP_COMMIT_STARTED:
            event = new VertexGroupCommitStartedEvent();
            break;
        case VERTEX_GROUP_COMMIT_FINISHED:
            event = new VertexGroupCommitFinishedEvent();
            break;
        case VERTEX_FINISHED:
            event = new VertexFinishedEvent();
            break;
        case TASK_STARTED:
            event = new TaskStartedEvent();
            break;
        case TASK_FINISHED:
            event = new TaskFinishedEvent();
            break;
        case TASK_ATTEMPT_STARTED:
            event = new TaskAttemptStartedEvent();
            break;
        case TASK_ATTEMPT_FINISHED:
            event = new TaskAttemptFinishedEvent();
            break;
        default:
            throw new IOException("Invalid data found, unknown event type " + eventType);
    }
    try {
        event.fromProtoStream(inputStream);
    } catch (EOFException eof) {
        return null;
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Parsed event from input stream" + ", eventType=" + eventType + ", event=" + event.toString());
    }
    return event;
}

Example 19 with DAGStartedEvent

use of org.apache.tez.dag.history.events.DAGStartedEvent in project tez by apache.

the class TestRecovery method testTwoRoundsRecoverying.

@Test(timeout = 1800000)
public void testTwoRoundsRecoverying() throws Exception {
    ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
    TezDAGID dagId = TezDAGID.getInstance(appId, 1);
    TezVertexID vertexId0 = TezVertexID.getInstance(dagId, 0);
    TezVertexID vertexId1 = TezVertexID.getInstance(dagId, 1);
    TezVertexID vertexId2 = TezVertexID.getInstance(dagId, 2);
    ContainerId containerId = ContainerId.newInstance(ApplicationAttemptId.newInstance(appId, 1), 1);
    NodeId nodeId = NodeId.newInstance("localhost", 10);
    List<TezEvent> initGeneratedEvents = Lists.newArrayList(new TezEvent(InputDataInformationEvent.createWithObjectPayload(0, new Object()), null));
    List<SimpleShutdownCondition> shutdownConditions = Lists.newArrayList(new SimpleShutdownCondition(TIMING.POST, new DAGInitializedEvent(dagId, 0L, "username", "dagName", null)), new SimpleShutdownCondition(TIMING.POST, new DAGStartedEvent(dagId, 0L, "username", "dagName")), new SimpleShutdownCondition(TIMING.POST, new VertexInitializedEvent(vertexId0, "Tokenizer", 0L, 0L, 0, "", null, initGeneratedEvents, null)), new SimpleShutdownCondition(TIMING.POST, new VertexStartedEvent(vertexId0, 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new VertexConfigurationDoneEvent(vertexId0, 0L, 2, null, null, null, true)), new SimpleShutdownCondition(TIMING.POST, new TaskStartedEvent(TezTaskID.getInstance(vertexId0, 0), "vertexName", 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new TaskAttemptStartedEvent(TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId0, 0), 0), "vertexName", 0L, containerId, nodeId, "", "", "")), new SimpleShutdownCondition(TIMING.POST, new TaskFinishedEvent(TezTaskID.getInstance(vertexId0, 0), "vertexName", 0L, 0L, null, TaskState.SUCCEEDED, "", new TezCounters(), 0)), new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent(vertexId0, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, VertexState.SUCCEEDED, "", new TezCounters(), new VertexStats(), new HashMap<String, Integer>(), null)), new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent(vertexId1, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, VertexState.SUCCEEDED, "", new TezCounters(), new VertexStats(), new HashMap<String, Integer>(), null)), new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent(vertexId2, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, VertexState.SUCCEEDED, "", new TezCounters(), new VertexStats(), new HashMap<String, Integer>(), null)), new SimpleShutdownCondition(TIMING.POST, new DAGFinishedEvent(dagId, 0L, 0L, DAGState.SUCCEEDED, "", new TezCounters(), "username", "dagName", new HashMap<String, Integer>(), ApplicationAttemptId.newInstance(appId, 1), null)));
    Random rand = new Random();
    for (int i = 0; i < shutdownConditions.size() - 1; i++) {
        // timeout.
        if (rand.nextDouble() < 0.5) {
            int nextSimpleConditionIndex = i + 1 + rand.nextInt(shutdownConditions.size() - i - 1);
            if (nextSimpleConditionIndex == shutdownConditions.size() - 1) {
                testOrderedWordCountMultipleRoundRecoverying(new RecoveryServiceWithEventHandlingHook.MultipleRoundShutdownCondition(Lists.newArrayList(shutdownConditions.get(i), shutdownConditions.get(nextSimpleConditionIndex))), true, shutdownConditions.get(i).getHistoryEvent().getEventType() == HistoryEventType.VERTEX_STARTED);
            }
        }
    }
}

Example 20 with DAGStartedEvent

use of org.apache.tez.dag.history.events.DAGStartedEvent in project tez by apache.

the class TestATSHistoryLoggingService method testTimelineServiceDisable.

@Test(timeout = 20000)
public void testTimelineServiceDisable() throws Exception {
    atsHistoryLoggingService.start();
    ATSHistoryLoggingService atsHistoryLoggingService1;
    atsHistoryLoggingService1 = new ATSHistoryLoggingService();
    atsHistoryLoggingService1.setAppContext(appContext);
    atsHistoryLoggingService1.timelineClient = mock(TimelineClient.class);
    when(atsHistoryLoggingService1.timelineClient.putEntities(Matchers.<TimelineEntity[]>anyVararg())).thenAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            ++atsInvokeCounter;
            atsEntitiesCounter += invocation.getArguments().length;
            try {
                Thread.sleep(10l);
            } catch (InterruptedException e) {
            // do nothing
            }
            return null;
        }
    });
    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
    conf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSHistoryLoggingService.class.getName());
    atsHistoryLoggingService1.init(conf);
    atsHistoryLoggingService1.start();
    TezDAGID tezDAGID = TezDAGID.getInstance(ApplicationId.newInstance(100l, 1), 1);
    DAGHistoryEvent historyEvent = new DAGHistoryEvent(tezDAGID, new DAGStartedEvent(tezDAGID, 1001l, "user1", "dagName1"));
    for (int i = 0; i < 100; ++i) {
        atsHistoryLoggingService1.handle(historyEvent);
    }
    try {
        Thread.sleep(20l);
    } catch (InterruptedException e) {
    // Do nothing
    }
    LOG.info("ATS entitiesSent=" + atsEntitiesCounter + ", timelineInvocations=" + atsInvokeCounter);
    Assert.assertEquals(atsInvokeCounter, 0);
    Assert.assertEquals(atsEntitiesCounter, 0);
    Assert.assertNull(atsHistoryLoggingService1.timelineClient);
    atsHistoryLoggingService1.close();
}