Examples with VertexStartedEvent org.apache.tez.dag.history.events.VertexStartedEvent used on opensource projects

Example 6 with VertexStartedEvent

use of org.apache.tez.dag.history.events.VertexStartedEvent in project tez by apache.

the class TestATSV15HistoryLoggingService method makeHistoryEvents.

private List<DAGHistoryEvent> makeHistoryEvents(TezDAGID dagId, ATSV15HistoryLoggingService service) {
    List<DAGHistoryEvent> historyEvents = new ArrayList<>();
    long time = System.currentTimeMillis();
    Configuration conf = new Configuration(service.getConfig());
    historyEvents.add(new DAGHistoryEvent(null, new AMStartedEvent(attemptId, time, user)));
    historyEvents.add(new DAGHistoryEvent(dagId, new DAGSubmittedEvent(dagId, time, DAGPlan.getDefaultInstance(), attemptId, null, user, conf, null, "default")));
    TezVertexID vertexID = TezVertexID.getInstance(dagId, 1);
    historyEvents.add(new DAGHistoryEvent(dagId, new VertexStartedEvent(vertexID, time, time)));
    TezTaskID tezTaskID = TezTaskID.getInstance(vertexID, 1);
    historyEvents.add(new DAGHistoryEvent(dagId, new TaskStartedEvent(tezTaskID, "test", time, time)));
    historyEvents.add(new DAGHistoryEvent(dagId, new TaskAttemptStartedEvent(TezTaskAttemptID.getInstance(tezTaskID, 1), "test", time, ContainerId.newContainerId(attemptId, 1), NodeId.newInstance("localhost", 8765), null, null, null)));
    return historyEvents;
}

Example 7 with VertexStartedEvent

use of org.apache.tez.dag.history.events.VertexStartedEvent in project tez by apache.

the class TestHistoryEventHandler method makeHistoryEvents.

private List<DAGHistoryEvent> makeHistoryEvents(TezDAGID dagId, Configuration inConf) {
    List<DAGHistoryEvent> historyEvents = new ArrayList<>();
    long time = System.currentTimeMillis();
    Configuration conf = new Configuration(inConf);
    historyEvents.add(new DAGHistoryEvent(null, new AMStartedEvent(attemptId, time, user)));
    historyEvents.add(new DAGHistoryEvent(dagId, new DAGSubmittedEvent(dagId, time, DAGPlan.getDefaultInstance(), attemptId, null, user, conf, null, "default")));
    TezVertexID vertexID = TezVertexID.getInstance(dagId, 1);
    historyEvents.add(new DAGHistoryEvent(dagId, new VertexStartedEvent(vertexID, time, time)));
    ContainerId containerId = ContainerId.newContainerId(attemptId, dagId.getId());
    TezTaskID tezTaskID = TezTaskID.getInstance(vertexID, 1);
    historyEvents.add(new DAGHistoryEvent(dagId, new TaskStartedEvent(tezTaskID, "test", time, time)));
    historyEvents.add(new DAGHistoryEvent(new ContainerLaunchedEvent(containerId, time, attemptId)));
    historyEvents.add(new DAGHistoryEvent(dagId, new TaskAttemptStartedEvent(TezTaskAttemptID.getInstance(tezTaskID, 1), "test", time, containerId, NodeId.newInstance("localhost", 8765), null, null, null)));
    historyEvents.add(new DAGHistoryEvent(dagId, new TaskAttemptFinishedEvent(TezTaskAttemptID.getInstance(tezTaskID, 1), "test", time, time + 1, TaskAttemptState.KILLED, null, TaskAttemptTerminationCause.EXTERNAL_PREEMPTION, "", null, null, null, time, null, time, containerId, NodeId.newInstance("localhost", 8765), null, null, null)));
    historyEvents.add(new DAGHistoryEvent(dagId, new TaskAttemptStartedEvent(TezTaskAttemptID.getInstance(tezTaskID, 2), "test", time, containerId, NodeId.newInstance("localhost", 8765), null, null, null)));
    historyEvents.add(new DAGHistoryEvent(dagId, new TaskAttemptFinishedEvent(TezTaskAttemptID.getInstance(tezTaskID, 2), "test", time + 2, time + 3, TaskAttemptState.KILLED, null, TaskAttemptTerminationCause.INTERRUPTED_BY_USER, "", null, null, null, time, null, time + 2, containerId, NodeId.newInstance("localhost", 8765), null, null, null)));
    historyEvents.add(new DAGHistoryEvent(dagId, new DAGFinishedEvent(dagId, time, time, DAGState.SUCCEEDED, null, null, user, "test", null, attemptId, DAGPlan.getDefaultInstance())));
    historyEvents.add(new DAGHistoryEvent(new ContainerStoppedEvent(containerId, time + 4, 0, attemptId)));
    return historyEvents;
}

Example 8 with VertexStartedEvent

use of org.apache.tez.dag.history.events.VertexStartedEvent in project tez by apache.

the class TestRecovery method testRecovery_HashJoin.

@Test(timeout = 1800000)
public void testRecovery_HashJoin() throws Exception {
    ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
    TezDAGID dagId = TezDAGID.getInstance(appId, 1);
    TezVertexID vertexId0 = TezVertexID.getInstance(dagId, 0);
    TezVertexID vertexId1 = TezVertexID.getInstance(dagId, 1);
    TezVertexID vertexId2 = TezVertexID.getInstance(dagId, 2);
    ContainerId containerId = ContainerId.newInstance(ApplicationAttemptId.newInstance(appId, 1), 1);
    NodeId nodeId = NodeId.newInstance("localhost", 10);
    List<TezEvent> initGeneratedEvents = Lists.newArrayList(new TezEvent(InputDataInformationEvent.createWithObjectPayload(0, new Object()), null));
    List<SimpleShutdownCondition> shutdownConditions = Lists.newArrayList(new SimpleShutdownCondition(TIMING.POST, new DAGInitializedEvent(dagId, 0L, "username", "dagName", null)), new SimpleShutdownCondition(TIMING.POST, new DAGStartedEvent(dagId, 0L, "username", "dagName")), new SimpleShutdownCondition(TIMING.POST, new DAGFinishedEvent(dagId, 0L, 0L, DAGState.SUCCEEDED, "", new TezCounters(), "username", "dagName", new HashMap<String, Integer>(), ApplicationAttemptId.newInstance(appId, 1), null)), new SimpleShutdownCondition(TIMING.POST, new VertexInitializedEvent(vertexId0, "hashSide", 0L, 0L, 0, "", null, initGeneratedEvents, null)), new SimpleShutdownCondition(TIMING.POST, new VertexInitializedEvent(vertexId1, "streamingSide", 0L, 0L, 0, "", null, null, null)), new SimpleShutdownCondition(TIMING.POST, new VertexInitializedEvent(vertexId2, "joiner", 0L, 0L, 0, "", null, null, null)), new SimpleShutdownCondition(TIMING.POST, new VertexStartedEvent(vertexId0, 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new VertexStartedEvent(vertexId1, 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new VertexStartedEvent(vertexId2, 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new VertexConfigurationDoneEvent(vertexId0, 0L, 2, null, null, null, true)), new SimpleShutdownCondition(TIMING.POST, new VertexConfigurationDoneEvent(vertexId1, 0L, 2, null, null, null, true)), new SimpleShutdownCondition(TIMING.POST, new VertexConfigurationDoneEvent(vertexId2, 0L, 2, null, null, null, true)), new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent(vertexId0, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, VertexState.SUCCEEDED, "", new TezCounters(), new VertexStats(), new HashMap<String, Integer>(), null)), new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent(vertexId1, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, VertexState.SUCCEEDED, "", new TezCounters(), new VertexStats(), new HashMap<String, Integer>(), null)), new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent(vertexId2, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, VertexState.SUCCEEDED, "", new TezCounters(), new VertexStats(), new HashMap<String, Integer>(), null)), new SimpleShutdownCondition(TIMING.POST, new TaskStartedEvent(TezTaskID.getInstance(vertexId0, 0), "vertexName", 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new TaskStartedEvent(TezTaskID.getInstance(vertexId1, 0), "vertexName", 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new TaskStartedEvent(TezTaskID.getInstance(vertexId2, 0), "vertexName", 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new TaskFinishedEvent(TezTaskID.getInstance(vertexId0, 0), "vertexName", 0L, 0L, null, TaskState.SUCCEEDED, "", new TezCounters(), 0)), new SimpleShutdownCondition(TIMING.POST, new TaskFinishedEvent(TezTaskID.getInstance(vertexId1, 0), "vertexName", 0L, 0L, null, TaskState.SUCCEEDED, "", new TezCounters(), 0)), new SimpleShutdownCondition(TIMING.POST, new TaskFinishedEvent(TezTaskID.getInstance(vertexId2, 0), "vertexName", 0L, 0L, null, TaskState.SUCCEEDED, "", new TezCounters(), 0)), new SimpleShutdownCondition(TIMING.POST, new TaskAttemptStartedEvent(TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId0, 0), 0), "vertexName", 0L, containerId, nodeId, "", "", "")), new SimpleShutdownCondition(TIMING.POST, new TaskAttemptStartedEvent(TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId1, 0), 0), "vertexName", 0L, containerId, nodeId, "", "", "")), new SimpleShutdownCondition(TIMING.POST, new TaskAttemptStartedEvent(TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId2, 0), 0), "vertexName", 0L, containerId, nodeId, "", "", "")));
    Random rand = new Random();
    for (int i = 0; i < shutdownConditions.size(); i++) {
        // timeout.
        if (rand.nextDouble() < 0.5) {
            // generate split in client side when HistoryEvent type is VERTEX_STARTED (TEZ-2976)
            testHashJoinExample(shutdownConditions.get(i), true, shutdownConditions.get(i).getHistoryEvent().getEventType() == HistoryEventType.VERTEX_STARTED);
        }
    }
}

Example 9 with VertexStartedEvent

use of org.apache.tez.dag.history.events.VertexStartedEvent in project tez by apache.

the class TestRecovery method testRecovery_OrderedWordCount.

@Test(timeout = 1800000)
public void testRecovery_OrderedWordCount() throws Exception {
    ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
    TezDAGID dagId = TezDAGID.getInstance(appId, 1);
    TezVertexID vertexId0 = TezVertexID.getInstance(dagId, 0);
    TezVertexID vertexId1 = TezVertexID.getInstance(dagId, 1);
    TezVertexID vertexId2 = TezVertexID.getInstance(dagId, 2);
    ContainerId containerId = ContainerId.newInstance(ApplicationAttemptId.newInstance(appId, 1), 1);
    NodeId nodeId = NodeId.newInstance("localhost", 10);
    List<TezEvent> initGeneratedEvents = Lists.newArrayList(new TezEvent(InputDataInformationEvent.createWithObjectPayload(0, new Object()), null));
    List<SimpleShutdownCondition> shutdownConditions = Lists.newArrayList(new SimpleShutdownCondition(TIMING.POST, new DAGInitializedEvent(dagId, 0L, "username", "dagName", null)), new SimpleShutdownCondition(TIMING.POST, new DAGStartedEvent(dagId, 0L, "username", "dagName")), new SimpleShutdownCondition(TIMING.POST, new DAGFinishedEvent(dagId, 0L, 0L, DAGState.SUCCEEDED, "", new TezCounters(), "username", "dagName", new HashMap<String, Integer>(), ApplicationAttemptId.newInstance(appId, 1), null)), new SimpleShutdownCondition(TIMING.POST, new VertexInitializedEvent(vertexId0, "Tokenizer", 0L, 0L, 0, "", null, initGeneratedEvents, null)), new SimpleShutdownCondition(TIMING.POST, new VertexInitializedEvent(vertexId1, "Summation", 0L, 0L, 0, "", null, null, null)), new SimpleShutdownCondition(TIMING.POST, new VertexInitializedEvent(vertexId2, "Sorter", 0L, 0L, 0, "", null, null, null)), new SimpleShutdownCondition(TIMING.POST, new VertexConfigurationDoneEvent(vertexId0, 0L, 2, null, null, null, true)), new SimpleShutdownCondition(TIMING.POST, new VertexConfigurationDoneEvent(vertexId1, 0L, 2, null, null, null, true)), new SimpleShutdownCondition(TIMING.POST, new VertexConfigurationDoneEvent(vertexId2, 0L, 2, null, null, null, true)), new SimpleShutdownCondition(TIMING.POST, new VertexStartedEvent(vertexId0, 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new VertexStartedEvent(vertexId1, 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new VertexStartedEvent(vertexId2, 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent(vertexId0, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, VertexState.SUCCEEDED, "", new TezCounters(), new VertexStats(), new HashMap<String, Integer>(), null)), new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent(vertexId1, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, VertexState.SUCCEEDED, "", new TezCounters(), new VertexStats(), new HashMap<String, Integer>(), null)), new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent(vertexId2, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, VertexState.SUCCEEDED, "", new TezCounters(), new VertexStats(), new HashMap<String, Integer>(), null)), new SimpleShutdownCondition(TIMING.POST, new TaskStartedEvent(TezTaskID.getInstance(vertexId0, 0), "vertexName", 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new TaskStartedEvent(TezTaskID.getInstance(vertexId1, 0), "vertexName", 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new TaskStartedEvent(TezTaskID.getInstance(vertexId2, 0), "vertexName", 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new TaskFinishedEvent(TezTaskID.getInstance(vertexId0, 0), "vertexName", 0L, 0L, null, TaskState.SUCCEEDED, "", new TezCounters(), 0)), new SimpleShutdownCondition(TIMING.POST, new TaskFinishedEvent(TezTaskID.getInstance(vertexId1, 0), "vertexName", 0L, 0L, null, TaskState.SUCCEEDED, "", new TezCounters(), 0)), new SimpleShutdownCondition(TIMING.POST, new TaskFinishedEvent(TezTaskID.getInstance(vertexId2, 0), "vertexName", 0L, 0L, null, TaskState.SUCCEEDED, "", new TezCounters(), 0)), new SimpleShutdownCondition(TIMING.POST, new TaskAttemptStartedEvent(TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId0, 0), 0), "vertexName", 0L, containerId, nodeId, "", "", "")), new SimpleShutdownCondition(TIMING.POST, new TaskAttemptStartedEvent(TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId1, 0), 0), "vertexName", 0L, containerId, nodeId, "", "", "")), new SimpleShutdownCondition(TIMING.POST, new TaskAttemptStartedEvent(TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId2, 0), 0), "vertexName", 0L, containerId, nodeId, "", "", "")));
    Random rand = new Random();
    for (int i = 0; i < shutdownConditions.size(); i++) {
        // timeout.
        if (rand.nextDouble() < 0.5) {
            // generate split in client side when HistoryEvent type is VERTEX_STARTED (TEZ-2976)
            testOrderedWordCount(shutdownConditions.get(i), true, shutdownConditions.get(i).getHistoryEvent().getEventType() == HistoryEventType.VERTEX_STARTED);
        }
    }
}

Example 10 with VertexStartedEvent

use of org.apache.tez.dag.history.events.VertexStartedEvent in project tez by apache.

the class TestDAGRecovery method initMockDAGRecoveryDataForTaskAttempt.

// ///////////////////////////// TaskAttempt Recovery /////////////////////////////////////////////////////
private void initMockDAGRecoveryDataForTaskAttempt() {
    TaskStartedEvent t1StartedEvent = new TaskStartedEvent(t1v1Id, "vertex1", 0L, t1StartedTime);
    TaskRecoveryData taskRecoveryData = new TaskRecoveryData(t1StartedEvent, null, null);
    Map<TezTaskID, TaskRecoveryData> taskRecoveryDataMap = new HashMap<TezTaskID, TaskRecoveryData>();
    taskRecoveryDataMap.put(t1v1Id, taskRecoveryData);
    List<TezEvent> inputGeneratedTezEvents = new ArrayList<TezEvent>();
    VertexInitializedEvent v1InitedEvent = new VertexInitializedEvent(v1Id, "vertex1", 0L, v1InitedTime, v1NumTask, "", null, inputGeneratedTezEvents, null);
    Map<String, InputSpecUpdate> rootInputSpecs = new HashMap<String, InputSpecUpdate>();
    VertexConfigurationDoneEvent v1ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v1Id, 0L, v1NumTask, null, null, rootInputSpecs, true);
    VertexStartedEvent v1StartedEvent = new VertexStartedEvent(v1Id, 0L, v1StartedTime);
    VertexRecoveryData v1RecoveryData = new VertexRecoveryData(v1InitedEvent, v1ReconfigureDoneEvent, v1StartedEvent, null, taskRecoveryDataMap, false);
    DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagId, dagInitedTime, "user", "dagName", null);
    DAGStartedEvent dagStartedEvent = new DAGStartedEvent(dagId, dagStartedTime, "user", "dagName");
    doReturn(v1RecoveryData).when(dagRecoveryData).getVertexRecoveryData(v1Id);
    doReturn(dagInitedEvent).when(dagRecoveryData).getDAGInitializedEvent();
    doReturn(dagStartedEvent).when(dagRecoveryData).getDAGStartedEvent();
}