Examples with TezVertexID org.apache.tez.dag.records.TezVertexID used on opensource projects

Example 61 with TezVertexID

use of org.apache.tez.dag.records.TezVertexID in project tez by apache.

the class TestPreemption method testPreemptionWithoutSession.

@Test(timeout = 5000)
public void testPreemptionWithoutSession() throws Exception {
    System.out.println("TestPreemptionWithoutSession");
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    tezconf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 0);
    AtomicBoolean mockAppLauncherGoFlag = new AtomicBoolean(false);
    MockTezClient tezClient = new MockTezClient("testPreemption", tezconf, false, null, null, null, mockAppLauncherGoFlag, false, false, 2, 2);
    tezClient.start();
    DAGClient dagClient = tezClient.submitDAG(createDAG(DataMovementType.SCATTER_GATHER));
    // now the MockApp has been started. sync with it to get the launcher
    syncWithMockAppLauncher(false, mockAppLauncherGoFlag, tezClient);
    DAGImpl dagImpl;
    do {
        // usually needs to sleep 2-3 times
        Thread.sleep(100);
    } while ((dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG()) == null);
    int vertexIndex = 0;
    int upToTaskVersion = 3;
    TezVertexID vertexId = TezVertexID.getInstance(dagImpl.getID(), vertexIndex);
    TezTaskAttemptID taId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 0);
    mockLauncher.preemptContainerForTask(taId.getTaskID(), upToTaskVersion);
    mockLauncher.startScheduling(true);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    for (int i = 0; i <= upToTaskVersion; ++i) {
        TezTaskAttemptID testTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), i);
        TaskAttemptImpl taImpl = dagImpl.getTaskAttempt(testTaId);
        Assert.assertEquals(TaskAttemptStateInternal.KILLED, taImpl.getInternalState());
    }
    tezClient.stop();
}

Example 62 with TezVertexID

use of org.apache.tez.dag.records.TezVertexID in project tez by apache.

the class AMWebController method getVertexFromIndex.

Vertex getVertexFromIndex(DAG dag, Integer vertexIndex) {
    final TezVertexID tezVertexID = TezVertexID.getInstance(dag.getID(), vertexIndex);
    Vertex vertex = dag.getVertex(tezVertexID);
    return vertex;
}

Example 63 with TezVertexID

use of org.apache.tez.dag.records.TezVertexID in project tez by apache.

the class AMWebController method getVerticesByIdx.

Collection<Vertex> getVerticesByIdx(DAG dag, Collection<Integer> indexes) {
    Collection<Vertex> vertices = new ArrayList<Vertex>(indexes.size());
    final TezDAGID tezDAGID = dag.getID();
    for (Integer idx : indexes) {
        final TezVertexID tezVertexID = TezVertexID.getInstance(tezDAGID, idx);
        if (tezVertexID == null) {
            continue;
        }
        final Vertex vertex = dag.getVertex(tezVertexID);
        if (vertex != null) {
            vertices.add(vertex);
        }
    }
    return vertices;
}

Example 64 with TezVertexID

use of org.apache.tez.dag.records.TezVertexID in project tez by apache.

the class AMWebController method getVertexProgress.

public void getVertexProgress() {
    int dagID;
    int vertexID;
    setCorsHeaders();
    if (!hasAccess()) {
        sendErrorResponse(HttpServletResponse.SC_UNAUTHORIZED, "Access denied for user: " + request().getRemoteUser(), null);
        return;
    }
    try {
        dagID = getQueryParamInt(WebUIService.DAG_ID);
        vertexID = getQueryParamInt(WebUIService.VERTEX_ID);
    } catch (NumberFormatException e) {
        sendErrorResponse(HttpServletResponse.SC_BAD_REQUEST, "Invalid dag or vertex id", e);
        return;
    }
    DAG currentDAG = appContext.getCurrentDAG();
    if (currentDAG == null || currentDAG.getID().getId() != dagID) {
        sendErrorResponse(HttpServletResponse.SC_NOT_FOUND, "Not current Dag: " + dagID, null);
        return;
    }
    final TezVertexID tezVertexID = TezVertexID.getInstance(currentDAG.getID(), vertexID);
    Vertex vertex = currentDAG.getVertex(tezVertexID);
    if (vertex == null) {
        sendErrorResponse(HttpServletResponse.SC_NOT_FOUND, "vertex not found: " + vertexID, null);
        return;
    }
    Map<String, ProgressInfo> result = new HashMap<String, ProgressInfo>();
    result.put(VERTEX_PROGRESS, new ProgressInfo(tezVertexID.toString(), vertex.getCompletedTaskProgress()));
    renderJSON(result);
}

Example 65 with TezVertexID

use of org.apache.tez.dag.records.TezVertexID in project tez by apache.

the class TestHistoryEventHandler method makeHistoryEvents.

private List<DAGHistoryEvent> makeHistoryEvents(TezDAGID dagId, Configuration inConf) {
    List<DAGHistoryEvent> historyEvents = new ArrayList<>();
    long time = System.currentTimeMillis();
    Configuration conf = new Configuration(inConf);
    historyEvents.add(new DAGHistoryEvent(null, new AMStartedEvent(attemptId, time, user)));
    historyEvents.add(new DAGHistoryEvent(dagId, new DAGSubmittedEvent(dagId, time, DAGPlan.getDefaultInstance(), attemptId, null, user, conf, null, "default")));
    TezVertexID vertexID = TezVertexID.getInstance(dagId, 1);
    historyEvents.add(new DAGHistoryEvent(dagId, new VertexStartedEvent(vertexID, time, time)));
    ContainerId containerId = ContainerId.newContainerId(attemptId, dagId.getId());
    TezTaskID tezTaskID = TezTaskID.getInstance(vertexID, 1);
    historyEvents.add(new DAGHistoryEvent(dagId, new TaskStartedEvent(tezTaskID, "test", time, time)));
    historyEvents.add(new DAGHistoryEvent(new ContainerLaunchedEvent(containerId, time, attemptId)));
    historyEvents.add(new DAGHistoryEvent(dagId, new TaskAttemptStartedEvent(TezTaskAttemptID.getInstance(tezTaskID, 1), "test", time, containerId, NodeId.newInstance("localhost", 8765), null, null, null)));
    historyEvents.add(new DAGHistoryEvent(dagId, new TaskAttemptFinishedEvent(TezTaskAttemptID.getInstance(tezTaskID, 1), "test", time, time + 1, TaskAttemptState.KILLED, null, TaskAttemptTerminationCause.EXTERNAL_PREEMPTION, "", null, null, null, time, null, time, containerId, NodeId.newInstance("localhost", 8765), null, null, null)));
    historyEvents.add(new DAGHistoryEvent(dagId, new TaskAttemptStartedEvent(TezTaskAttemptID.getInstance(tezTaskID, 2), "test", time, containerId, NodeId.newInstance("localhost", 8765), null, null, null)));
    historyEvents.add(new DAGHistoryEvent(dagId, new TaskAttemptFinishedEvent(TezTaskAttemptID.getInstance(tezTaskID, 2), "test", time + 2, time + 3, TaskAttemptState.KILLED, null, TaskAttemptTerminationCause.INTERRUPTED_BY_USER, "", null, null, null, time, null, time + 2, containerId, NodeId.newInstance("localhost", 8765), null, null, null)));
    historyEvents.add(new DAGHistoryEvent(dagId, new DAGFinishedEvent(dagId, time, time, DAGState.SUCCEEDED, null, null, user, "test", null, attemptId, DAGPlan.getDefaultInstance())));
    historyEvents.add(new DAGHistoryEvent(new ContainerStoppedEvent(containerId, time + 4, 0, attemptId)));
    return historyEvents;
}