Examples with DAGInitializedEvent org.apache.tez.dag.history.events.DAGInitializedEvent used on opensource projects

Example 16 with DAGInitializedEvent

use of org.apache.tez.dag.history.events.DAGInitializedEvent in project tez by apache.

the class RecoveryParser method getNextEvent.

private static HistoryEvent getNextEvent(FSDataInputStream inputStream) throws IOException {
    int eventTypeOrdinal = -1;
    try {
        eventTypeOrdinal = inputStream.readInt();
    } catch (EOFException eof) {
        return null;
    }
    if (eventTypeOrdinal < 0 || eventTypeOrdinal >= HistoryEventType.values().length) {
        // reached end
        throw new IOException("Corrupt data found when trying to read next event type" + ", eventTypeOrdinal=" + eventTypeOrdinal);
    }
    HistoryEventType eventType = HistoryEventType.values()[eventTypeOrdinal];
    HistoryEvent event;
    switch(eventType) {
        case AM_LAUNCHED:
            event = new AMLaunchedEvent();
            break;
        case AM_STARTED:
            event = new AMStartedEvent();
            break;
        case DAG_SUBMITTED:
            event = new DAGSubmittedEvent();
            break;
        case DAG_INITIALIZED:
            event = new DAGInitializedEvent();
            break;
        case DAG_STARTED:
            event = new DAGStartedEvent();
            break;
        case DAG_COMMIT_STARTED:
            event = new DAGCommitStartedEvent();
            break;
        case DAG_FINISHED:
            event = new DAGFinishedEvent();
            break;
        case DAG_KILL_REQUEST:
            event = new DAGKillRequestEvent();
            break;
        case CONTAINER_LAUNCHED:
            event = new ContainerLaunchedEvent();
            break;
        case CONTAINER_STOPPED:
            event = new ContainerStoppedEvent();
            break;
        case VERTEX_INITIALIZED:
            event = new VertexInitializedEvent();
            break;
        case VERTEX_CONFIGURE_DONE:
            event = new VertexConfigurationDoneEvent();
            break;
        case VERTEX_STARTED:
            event = new VertexStartedEvent();
            break;
        case VERTEX_COMMIT_STARTED:
            event = new VertexCommitStartedEvent();
            break;
        case VERTEX_GROUP_COMMIT_STARTED:
            event = new VertexGroupCommitStartedEvent();
            break;
        case VERTEX_GROUP_COMMIT_FINISHED:
            event = new VertexGroupCommitFinishedEvent();
            break;
        case VERTEX_FINISHED:
            event = new VertexFinishedEvent();
            break;
        case TASK_STARTED:
            event = new TaskStartedEvent();
            break;
        case TASK_FINISHED:
            event = new TaskFinishedEvent();
            break;
        case TASK_ATTEMPT_STARTED:
            event = new TaskAttemptStartedEvent();
            break;
        case TASK_ATTEMPT_FINISHED:
            event = new TaskAttemptFinishedEvent();
            break;
        default:
            throw new IOException("Invalid data found, unknown event type " + eventType);
    }
    try {
        event.fromProtoStream(inputStream);
    } catch (EOFException eof) {
        return null;
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Parsed event from input stream" + ", eventType=" + eventType + ", event=" + event.toString());
    }
    return event;
}

Example 17 with DAGInitializedEvent

use of org.apache.tez.dag.history.events.DAGInitializedEvent in project tez by apache.

the class TestRecovery method testTwoRoundsRecoverying.

@Test(timeout = 1800000)
public void testTwoRoundsRecoverying() throws Exception {
    ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
    TezDAGID dagId = TezDAGID.getInstance(appId, 1);
    TezVertexID vertexId0 = TezVertexID.getInstance(dagId, 0);
    TezVertexID vertexId1 = TezVertexID.getInstance(dagId, 1);
    TezVertexID vertexId2 = TezVertexID.getInstance(dagId, 2);
    ContainerId containerId = ContainerId.newInstance(ApplicationAttemptId.newInstance(appId, 1), 1);
    NodeId nodeId = NodeId.newInstance("localhost", 10);
    List<TezEvent> initGeneratedEvents = Lists.newArrayList(new TezEvent(InputDataInformationEvent.createWithObjectPayload(0, new Object()), null));
    List<SimpleShutdownCondition> shutdownConditions = Lists.newArrayList(new SimpleShutdownCondition(TIMING.POST, new DAGInitializedEvent(dagId, 0L, "username", "dagName", null)), new SimpleShutdownCondition(TIMING.POST, new DAGStartedEvent(dagId, 0L, "username", "dagName")), new SimpleShutdownCondition(TIMING.POST, new VertexInitializedEvent(vertexId0, "Tokenizer", 0L, 0L, 0, "", null, initGeneratedEvents, null)), new SimpleShutdownCondition(TIMING.POST, new VertexStartedEvent(vertexId0, 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new VertexConfigurationDoneEvent(vertexId0, 0L, 2, null, null, null, true)), new SimpleShutdownCondition(TIMING.POST, new TaskStartedEvent(TezTaskID.getInstance(vertexId0, 0), "vertexName", 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new TaskAttemptStartedEvent(TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId0, 0), 0), "vertexName", 0L, containerId, nodeId, "", "", "")), new SimpleShutdownCondition(TIMING.POST, new TaskFinishedEvent(TezTaskID.getInstance(vertexId0, 0), "vertexName", 0L, 0L, null, TaskState.SUCCEEDED, "", new TezCounters(), 0)), new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent(vertexId0, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, VertexState.SUCCEEDED, "", new TezCounters(), new VertexStats(), new HashMap<String, Integer>(), null)), new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent(vertexId1, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, VertexState.SUCCEEDED, "", new TezCounters(), new VertexStats(), new HashMap<String, Integer>(), null)), new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent(vertexId2, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, VertexState.SUCCEEDED, "", new TezCounters(), new VertexStats(), new HashMap<String, Integer>(), null)), new SimpleShutdownCondition(TIMING.POST, new DAGFinishedEvent(dagId, 0L, 0L, DAGState.SUCCEEDED, "", new TezCounters(), "username", "dagName", new HashMap<String, Integer>(), ApplicationAttemptId.newInstance(appId, 1), null)));
    Random rand = new Random();
    for (int i = 0; i < shutdownConditions.size() - 1; i++) {
        // timeout.
        if (rand.nextDouble() < 0.5) {
            int nextSimpleConditionIndex = i + 1 + rand.nextInt(shutdownConditions.size() - i - 1);
            if (nextSimpleConditionIndex == shutdownConditions.size() - 1) {
                testOrderedWordCountMultipleRoundRecoverying(new RecoveryServiceWithEventHandlingHook.MultipleRoundShutdownCondition(Lists.newArrayList(shutdownConditions.get(i), shutdownConditions.get(nextSimpleConditionIndex))), true, shutdownConditions.get(i).getHistoryEvent().getEventType() == HistoryEventType.VERTEX_STARTED);
            }
        }
    }
}