Search in sources :

Example 6 with VertexGroupCommitStartedEvent

use of org.apache.tez.dag.history.events.VertexGroupCommitStartedEvent in project tez by apache.

the class TestHistoryEventJsonConversion method testHandlerExists.

@Test(timeout = 5000)
public void testHandlerExists() throws JSONException {
    for (HistoryEventType eventType : HistoryEventType.values()) {
        HistoryEvent event = null;
        switch(eventType) {
            case APP_LAUNCHED:
                event = new AppLaunchedEvent(applicationId, random.nextInt(), random.nextInt(), user, new Configuration(false), null);
                break;
            case AM_LAUNCHED:
                event = new AMLaunchedEvent(applicationAttemptId, random.nextInt(), random.nextInt(), user);
                break;
            case AM_STARTED:
                event = new AMStartedEvent(applicationAttemptId, random.nextInt(), user);
                break;
            case DAG_SUBMITTED:
                event = new DAGSubmittedEvent(tezDAGID, random.nextInt(), dagPlan, applicationAttemptId, null, user, null, null, "Q_" + eventType.name());
                break;
            case DAG_INITIALIZED:
                event = new DAGInitializedEvent(tezDAGID, random.nextInt(), user, dagPlan.getName(), null);
                break;
            case DAG_STARTED:
                event = new DAGStartedEvent(tezDAGID, random.nextInt(), user, dagPlan.getName());
                break;
            case DAG_FINISHED:
                event = new DAGFinishedEvent(tezDAGID, random.nextInt(), random.nextInt(), DAGState.ERROR, null, null, user, dagPlan.getName(), null, applicationAttemptId, dagPlan);
                break;
            case VERTEX_INITIALIZED:
                event = new VertexInitializedEvent(tezVertexID, "v1", random.nextInt(), random.nextInt(), random.nextInt(), "proc", null, null, null);
                break;
            case VERTEX_STARTED:
                event = new VertexStartedEvent(tezVertexID, random.nextInt(), random.nextInt());
                break;
            case VERTEX_CONFIGURE_DONE:
                event = new VertexConfigurationDoneEvent(tezVertexID, 0L, 1, null, null, null, true);
                break;
            case VERTEX_FINISHED:
                event = new VertexFinishedEvent(tezVertexID, "v1", 1, random.nextInt(), random.nextInt(), random.nextInt(), random.nextInt(), random.nextInt(), VertexState.ERROR, null, null, null, null, null);
                break;
            case TASK_STARTED:
                event = new TaskStartedEvent(tezTaskID, "v1", random.nextInt(), random.nextInt());
                break;
            case TASK_FINISHED:
                event = new TaskFinishedEvent(tezTaskID, "v1", random.nextInt(), random.nextInt(), tezTaskAttemptID, TaskState.FAILED, null, null, 0);
                break;
            case TASK_ATTEMPT_STARTED:
                event = new TaskAttemptStartedEvent(tezTaskAttemptID, "v1", random.nextInt(), containerId, nodeId, null, null, "nodeHttpAddress");
                break;
            case TASK_ATTEMPT_FINISHED:
                event = new TaskAttemptFinishedEvent(tezTaskAttemptID, "v1", random.nextInt(), random.nextInt(), TaskAttemptState.KILLED, null, TaskAttemptTerminationCause.TERMINATED_BY_CLIENT, null, null, null, null, 0, null, 0, containerId, nodeId, null, null, "nodeHttpAddress");
                break;
            case CONTAINER_LAUNCHED:
                event = new ContainerLaunchedEvent(containerId, random.nextInt(), applicationAttemptId);
                break;
            case CONTAINER_STOPPED:
                event = new ContainerStoppedEvent(containerId, random.nextInt(), -1, applicationAttemptId);
                break;
            case DAG_COMMIT_STARTED:
                event = new DAGCommitStartedEvent();
                break;
            case VERTEX_COMMIT_STARTED:
                event = new VertexCommitStartedEvent();
                break;
            case VERTEX_GROUP_COMMIT_STARTED:
                event = new VertexGroupCommitStartedEvent();
                break;
            case VERTEX_GROUP_COMMIT_FINISHED:
                event = new VertexGroupCommitFinishedEvent();
                break;
            case DAG_RECOVERED:
                event = new DAGRecoveredEvent(applicationAttemptId, tezDAGID, dagPlan.getName(), user, 1l, null);
                break;
            case DAG_KILL_REQUEST:
                event = new DAGKillRequestEvent();
                break;
            default:
                Assert.fail("Unhandled event type " + eventType);
        }
        if (event == null || !event.isHistoryEvent()) {
            continue;
        }
        JSONObject json = HistoryEventJsonConversion.convertToJson(event);
        if (eventType == HistoryEventType.DAG_SUBMITTED) {
            try {
                Assert.assertEquals("Q_" + eventType.name(), json.getJSONObject(ATSConstants.OTHER_INFO).getString(ATSConstants.DAG_QUEUE_NAME));
                Assert.assertEquals("Q_" + eventType.name(), json.getJSONObject(ATSConstants.PRIMARY_FILTERS).getString(ATSConstants.DAG_QUEUE_NAME));
            } catch (JSONException ex) {
                Assert.fail("Exception: " + ex.getMessage() + " for type: " + eventType);
            }
        }
    }
}
Also used : DAGCommitStartedEvent(org.apache.tez.dag.history.events.DAGCommitStartedEvent) Configuration(org.apache.hadoop.conf.Configuration) VertexInitializedEvent(org.apache.tez.dag.history.events.VertexInitializedEvent) HistoryEventType(org.apache.tez.dag.history.HistoryEventType) DAGInitializedEvent(org.apache.tez.dag.history.events.DAGInitializedEvent) ContainerStoppedEvent(org.apache.tez.dag.history.events.ContainerStoppedEvent) DAGKillRequestEvent(org.apache.tez.dag.history.events.DAGKillRequestEvent) DAGStartedEvent(org.apache.tez.dag.history.events.DAGStartedEvent) VertexConfigurationDoneEvent(org.apache.tez.dag.history.events.VertexConfigurationDoneEvent) DAGRecoveredEvent(org.apache.tez.dag.history.events.DAGRecoveredEvent) TaskAttemptFinishedEvent(org.apache.tez.dag.history.events.TaskAttemptFinishedEvent) AMStartedEvent(org.apache.tez.dag.history.events.AMStartedEvent) VertexStartedEvent(org.apache.tez.dag.history.events.VertexStartedEvent) VertexGroupCommitStartedEvent(org.apache.tez.dag.history.events.VertexGroupCommitStartedEvent) JSONException(org.codehaus.jettison.json.JSONException) HistoryEvent(org.apache.tez.dag.history.HistoryEvent) TaskStartedEvent(org.apache.tez.dag.history.events.TaskStartedEvent) TaskAttemptStartedEvent(org.apache.tez.dag.history.events.TaskAttemptStartedEvent) AppLaunchedEvent(org.apache.tez.dag.history.events.AppLaunchedEvent) TaskFinishedEvent(org.apache.tez.dag.history.events.TaskFinishedEvent) JSONObject(org.codehaus.jettison.json.JSONObject) VertexGroupCommitFinishedEvent(org.apache.tez.dag.history.events.VertexGroupCommitFinishedEvent) AMLaunchedEvent(org.apache.tez.dag.history.events.AMLaunchedEvent) ContainerLaunchedEvent(org.apache.tez.dag.history.events.ContainerLaunchedEvent) DAGFinishedEvent(org.apache.tez.dag.history.events.DAGFinishedEvent) VertexFinishedEvent(org.apache.tez.dag.history.events.VertexFinishedEvent) DAGSubmittedEvent(org.apache.tez.dag.history.events.DAGSubmittedEvent) VertexCommitStartedEvent(org.apache.tez.dag.history.events.VertexCommitStartedEvent) Test(org.junit.Test)

Example 7 with VertexGroupCommitStartedEvent

use of org.apache.tez.dag.history.events.VertexGroupCommitStartedEvent in project tez by apache.

the class RecoveryParser method getNextEvent.

private static HistoryEvent getNextEvent(FSDataInputStream inputStream) throws IOException {
    int eventTypeOrdinal = -1;
    try {
        eventTypeOrdinal = inputStream.readInt();
    } catch (EOFException eof) {
        return null;
    }
    if (eventTypeOrdinal < 0 || eventTypeOrdinal >= HistoryEventType.values().length) {
        // reached end
        throw new IOException("Corrupt data found when trying to read next event type" + ", eventTypeOrdinal=" + eventTypeOrdinal);
    }
    HistoryEventType eventType = HistoryEventType.values()[eventTypeOrdinal];
    HistoryEvent event;
    switch(eventType) {
        case AM_LAUNCHED:
            event = new AMLaunchedEvent();
            break;
        case AM_STARTED:
            event = new AMStartedEvent();
            break;
        case DAG_SUBMITTED:
            event = new DAGSubmittedEvent();
            break;
        case DAG_INITIALIZED:
            event = new DAGInitializedEvent();
            break;
        case DAG_STARTED:
            event = new DAGStartedEvent();
            break;
        case DAG_COMMIT_STARTED:
            event = new DAGCommitStartedEvent();
            break;
        case DAG_FINISHED:
            event = new DAGFinishedEvent();
            break;
        case DAG_KILL_REQUEST:
            event = new DAGKillRequestEvent();
            break;
        case CONTAINER_LAUNCHED:
            event = new ContainerLaunchedEvent();
            break;
        case CONTAINER_STOPPED:
            event = new ContainerStoppedEvent();
            break;
        case VERTEX_INITIALIZED:
            event = new VertexInitializedEvent();
            break;
        case VERTEX_CONFIGURE_DONE:
            event = new VertexConfigurationDoneEvent();
            break;
        case VERTEX_STARTED:
            event = new VertexStartedEvent();
            break;
        case VERTEX_COMMIT_STARTED:
            event = new VertexCommitStartedEvent();
            break;
        case VERTEX_GROUP_COMMIT_STARTED:
            event = new VertexGroupCommitStartedEvent();
            break;
        case VERTEX_GROUP_COMMIT_FINISHED:
            event = new VertexGroupCommitFinishedEvent();
            break;
        case VERTEX_FINISHED:
            event = new VertexFinishedEvent();
            break;
        case TASK_STARTED:
            event = new TaskStartedEvent();
            break;
        case TASK_FINISHED:
            event = new TaskFinishedEvent();
            break;
        case TASK_ATTEMPT_STARTED:
            event = new TaskAttemptStartedEvent();
            break;
        case TASK_ATTEMPT_FINISHED:
            event = new TaskAttemptFinishedEvent();
            break;
        default:
            throw new IOException("Invalid data found, unknown event type " + eventType);
    }
    try {
        event.fromProtoStream(inputStream);
    } catch (EOFException eof) {
        return null;
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Parsed event from input stream" + ", eventType=" + eventType + ", event=" + event.toString());
    }
    return event;
}
Also used : DAGCommitStartedEvent(org.apache.tez.dag.history.events.DAGCommitStartedEvent) VertexInitializedEvent(org.apache.tez.dag.history.events.VertexInitializedEvent) HistoryEventType(org.apache.tez.dag.history.HistoryEventType) DAGInitializedEvent(org.apache.tez.dag.history.events.DAGInitializedEvent) DAGKillRequestEvent(org.apache.tez.dag.history.events.DAGKillRequestEvent) ContainerStoppedEvent(org.apache.tez.dag.history.events.ContainerStoppedEvent) EOFException(java.io.EOFException) DAGStartedEvent(org.apache.tez.dag.history.events.DAGStartedEvent) VertexConfigurationDoneEvent(org.apache.tez.dag.history.events.VertexConfigurationDoneEvent) TaskAttemptFinishedEvent(org.apache.tez.dag.history.events.TaskAttemptFinishedEvent) AMStartedEvent(org.apache.tez.dag.history.events.AMStartedEvent) VertexStartedEvent(org.apache.tez.dag.history.events.VertexStartedEvent) VertexGroupCommitStartedEvent(org.apache.tez.dag.history.events.VertexGroupCommitStartedEvent) IOException(java.io.IOException) HistoryEvent(org.apache.tez.dag.history.HistoryEvent) TaskStartedEvent(org.apache.tez.dag.history.events.TaskStartedEvent) TaskAttemptStartedEvent(org.apache.tez.dag.history.events.TaskAttemptStartedEvent) TaskFinishedEvent(org.apache.tez.dag.history.events.TaskFinishedEvent) VertexGroupCommitFinishedEvent(org.apache.tez.dag.history.events.VertexGroupCommitFinishedEvent) AMLaunchedEvent(org.apache.tez.dag.history.events.AMLaunchedEvent) ContainerLaunchedEvent(org.apache.tez.dag.history.events.ContainerLaunchedEvent) DAGFinishedEvent(org.apache.tez.dag.history.events.DAGFinishedEvent) VertexFinishedEvent(org.apache.tez.dag.history.events.VertexFinishedEvent) DAGSubmittedEvent(org.apache.tez.dag.history.events.DAGSubmittedEvent) VertexCommitStartedEvent(org.apache.tez.dag.history.events.VertexCommitStartedEvent)

Aggregations

VertexGroupCommitStartedEvent (org.apache.tez.dag.history.events.VertexGroupCommitStartedEvent)7 DAGSubmittedEvent (org.apache.tez.dag.history.events.DAGSubmittedEvent)6 Configuration (org.apache.hadoop.conf.Configuration)5 VertexFinishedEvent (org.apache.tez.dag.history.events.VertexFinishedEvent)5 VertexGroupCommitFinishedEvent (org.apache.tez.dag.history.events.VertexGroupCommitFinishedEvent)5 DAGHistoryEvent (org.apache.tez.dag.history.DAGHistoryEvent)4 Path (org.apache.hadoop.fs.Path)3 ApplicationId (org.apache.hadoop.yarn.api.records.ApplicationId)3 SystemClock (org.apache.hadoop.yarn.util.SystemClock)3 DAGPlan (org.apache.tez.dag.api.records.DAGProtos.DAGPlan)3 DAGRecoveryData (org.apache.tez.dag.app.RecoveryParser.DAGRecoveryData)3 HistoryEvent (org.apache.tez.dag.history.HistoryEvent)3 HistoryEventType (org.apache.tez.dag.history.HistoryEventType)3 AMLaunchedEvent (org.apache.tez.dag.history.events.AMLaunchedEvent)3 AMStartedEvent (org.apache.tez.dag.history.events.AMStartedEvent)3 ContainerLaunchedEvent (org.apache.tez.dag.history.events.ContainerLaunchedEvent)3 ContainerStoppedEvent (org.apache.tez.dag.history.events.ContainerStoppedEvent)3 DAGCommitStartedEvent (org.apache.tez.dag.history.events.DAGCommitStartedEvent)3 DAGFinishedEvent (org.apache.tez.dag.history.events.DAGFinishedEvent)3 DAGInitializedEvent (org.apache.tez.dag.history.events.DAGInitializedEvent)3