Examples with Vertex org.apache.tez.dag.app.dag.Vertex used on opensource projects

Example 41 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class AMWebController method getVertexProgress.

public void getVertexProgress() {
    int dagID;
    int vertexID;
    setCorsHeaders();
    if (!hasAccess()) {
        sendErrorResponse(HttpServletResponse.SC_UNAUTHORIZED, "Access denied for user: " + request().getRemoteUser(), null);
        return;
    }
    try {
        dagID = getQueryParamInt(WebUIService.DAG_ID);
        vertexID = getQueryParamInt(WebUIService.VERTEX_ID);
    } catch (NumberFormatException e) {
        sendErrorResponse(HttpServletResponse.SC_BAD_REQUEST, "Invalid dag or vertex id", e);
        return;
    }
    DAG currentDAG = appContext.getCurrentDAG();
    if (currentDAG == null || currentDAG.getID().getId() != dagID) {
        sendErrorResponse(HttpServletResponse.SC_NOT_FOUND, "Not current Dag: " + dagID, null);
        return;
    }
    final TezVertexID tezVertexID = TezVertexID.getInstance(currentDAG.getID(), vertexID);
    Vertex vertex = currentDAG.getVertex(tezVertexID);
    if (vertex == null) {
        sendErrorResponse(HttpServletResponse.SC_NOT_FOUND, "vertex not found: " + vertexID, null);
        return;
    }
    Map<String, ProgressInfo> result = new HashMap<String, ProgressInfo>();
    result.put(VERTEX_PROGRESS, new ProgressInfo(tezVertexID.toString(), vertex.getCompletedTaskProgress()));
    renderJSON(result);
}

Example 42 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class TestVertexImpl method testSetCustomEdgeManager.

@Test(timeout = 5000)
public void testSetCustomEdgeManager() throws Exception {
    // Vertex5 linked to v3 (v3 src, v5 dest)
    VertexImpl v5 = vertices.get("vertex5");
    v5.vertexReconfigurationPlanned();
    initAllVertices(VertexState.INITED);
    Edge edge = edges.get("e4");
    EdgeManagerPlugin em = edge.getEdgeManager();
    EdgeManagerForTest originalEm = (EdgeManagerForTest) em;
    assertTrue(Arrays.equals(edgePayload, originalEm.getEdgeManagerContext().getUserPayload().deepCopyAsArray()));
    UserPayload userPayload = UserPayload.create(ByteBuffer.wrap(new String("foo").getBytes()));
    EdgeManagerPluginDescriptor edgeManagerDescriptor = EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName());
    edgeManagerDescriptor.setUserPayload(userPayload);
    EdgeProperty edgeProp = EdgeProperty.create(edgeManagerDescriptor, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"));
    Vertex v3 = vertices.get("vertex3");
    Map<String, EdgeProperty> edgeManagerDescriptors = Collections.singletonMap(v3.getName(), edgeProp);
    v5.reconfigureVertex(v5.getTotalTasks() - 1, null, edgeManagerDescriptors);
    v5.doneReconfiguringVertex();
    VertexImpl v5Impl = (VertexImpl) v5;
    EdgeManagerPlugin modifiedEdgeManager = v5Impl.sourceVertices.get(v3).getEdgeManager();
    Assert.assertNotNull(modifiedEdgeManager);
    assertTrue(modifiedEdgeManager instanceof EdgeManagerForTest);
    // Ensure initialize() is called with the correct payload
    assertTrue(Arrays.equals(userPayload.deepCopyAsArray(), ((EdgeManagerForTest) modifiedEdgeManager).getUserPayload().deepCopyAsArray()));
}

Example 43 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class TestVertexImpl method parseVertexEdges.

private void parseVertexEdges() {
    LOG.info("Parsing edges from dag plan, edgeCount=" + dagPlan.getEdgeCount());
    int vCnt = dagPlan.getVertexCount();
    Map<String, EdgePlan> edgePlans = DagTypeConverters.createEdgePlanMapFromDAGPlan(dagPlan.getEdgeList());
    // TODO - this test logic is tightly linked to impl DAGImpl code.
    for (int i = 0; i < vCnt; ++i) {
        VertexPlan vertexPlan = dagPlan.getVertex(i);
        Vertex vertex = vertices.get(vertexPlan.getName());
        Map<Vertex, Edge> inVertices = new HashMap<Vertex, Edge>();
        Map<Vertex, Edge> outVertices = new HashMap<Vertex, Edge>();
        for (String inEdgeId : vertexPlan.getInEdgeIdList()) {
            EdgePlan edgePlan = edgePlans.get(inEdgeId);
            Vertex inVertex = this.vertices.get(edgePlan.getInputVertexName());
            Edge edge = this.edges.get(inEdgeId);
            edge.setSourceVertex(inVertex);
            edge.setDestinationVertex(vertex);
            inVertices.put(inVertex, edge);
        }
        for (String outEdgeId : vertexPlan.getOutEdgeIdList()) {
            EdgePlan edgePlan = edgePlans.get(outEdgeId);
            Vertex outVertex = this.vertices.get(edgePlan.getOutputVertexName());
            Edge edge = this.edges.get(outEdgeId);
            edge.setSourceVertex(vertex);
            edge.setDestinationVertex(outVertex);
            outVertices.put(outVertex, edge);
        }
        LOG.info("Setting input vertices for vertex " + vertex.getName() + ", inputVerticesCnt=" + inVertices.size());
        vertex.setInputVertices(inVertices);
        LOG.info("Setting output vertices for vertex " + vertex.getName() + ", outputVerticesCnt=" + outVertices.size());
        vertex.setOutputVertices(outVertices);
    }
}

Example 44 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class TestDAGSchedulerNaturalOrderControlled method testSimpleFlow.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testSimpleFlow() {
    EventHandler eventHandler = mock(EventHandler.class);
    DAG dag = createMockDag();
    DAGSchedulerNaturalOrderControlled dagScheduler = new DAGSchedulerNaturalOrderControlled(dag, eventHandler);
    int numVertices = 5;
    Vertex[] vertices = new Vertex[numVertices];
    for (int i = 0; i < numVertices; i++) {
        vertices[i] = dag.getVertex("vertex" + i);
    }
    // Schedule all tasks belonging to v0
    for (int i = 0; i < vertices[0].getTotalTasks(); i++) {
        dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[0].getVertexId(), i, 0));
    }
    verify(eventHandler, times(vertices[0].getTotalTasks())).handle(any(Event.class));
    reset(eventHandler);
    // Schedule 3 tasks belonging to v2
    for (int i = 0; i < 3; i++) {
        dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[2].getVertexId(), i, 0));
    }
    verify(eventHandler, times(3)).handle(any(Event.class));
    reset(eventHandler);
    // Schedule 3 tasks belonging to v3
    for (int i = 0; i < 3; i++) {
        dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[3].getVertexId(), i, 0));
    }
    verify(eventHandler, times(3)).handle(any(Event.class));
    reset(eventHandler);
    // Schedule remaining tasks belonging to v2
    for (int i = 3; i < vertices[2].getTotalTasks(); i++) {
        dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[2].getVertexId(), i, 0));
    }
    verify(eventHandler, times(vertices[2].getTotalTasks() - 3)).handle(any(Event.class));
    reset(eventHandler);
    // Schedule remaining tasks belonging to v3
    for (int i = 3; i < vertices[3].getTotalTasks(); i++) {
        dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[3].getVertexId(), i, 0));
    }
    verify(eventHandler, times(vertices[3].getTotalTasks() - 3)).handle(any(Event.class));
    reset(eventHandler);
    // Schedule all tasks belonging to v4
    for (int i = 0; i < vertices[4].getTotalTasks(); i++) {
        dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[4].getVertexId(), i, 0));
    }
    verify(eventHandler, times(vertices[4].getTotalTasks())).handle(any(Event.class));
    reset(eventHandler);
}

Example 45 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class TestDAGSchedulerNaturalOrderControlled method testMultipleRequestsForSameTask.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testMultipleRequestsForSameTask() {
    EventHandler eventHandler = mock(EventHandler.class);
    DAG dag = createMockDag();
    DAGSchedulerNaturalOrderControlled dagScheduler = new DAGSchedulerNaturalOrderControlled(dag, eventHandler);
    int numVertices = 5;
    Vertex[] vertices = new Vertex[numVertices];
    for (int i = 0; i < numVertices; i++) {
        vertices[i] = dag.getVertex("vertex" + i);
    }
    // Schedule all but 1 task belonging to v0
    for (int i = 0; i < vertices[0].getTotalTasks() - 1; i++) {
        dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[0].getVertexId(), i, 0));
    }
    verify(eventHandler, times(vertices[0].getTotalTasks() - 1)).handle(any(Event.class));
    reset(eventHandler);
    // Schedule all tasks belonging to v2
    for (int i = 0; i < vertices[2].getTotalTasks(); i++) {
        dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[2].getVertexId(), i, 0));
    }
    // Nothing should be scheduled
    verify(eventHandler, never()).handle(any(Event.class));
    reset(eventHandler);
    // Schedule an extra attempt for all but 1 task belonging to v0
    for (int i = 0; i < vertices[0].getTotalTasks() - 1; i++) {
        dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[0].getVertexId(), i, 1));
    }
    // Only v0 requests should have gone out
    verify(eventHandler, times(vertices[0].getTotalTasks() - 1)).handle(any(Event.class));
    reset(eventHandler);
    // Schedule last task of v0, with attempt 1
    dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[0].getVertexId(), vertices[0].getTotalTasks() - 1, 1));
    // One v0 request and all of v2 should have gone out
    verify(eventHandler, times(1 + vertices[2].getTotalTasks())).handle(any(Event.class));
}