Examples with VertexPlan org.apache.tez.dag.api.records.DAGProtos.VertexPlan used on opensource projects

Example 11 with VertexPlan

use of org.apache.tez.dag.api.records.DAGProtos.VertexPlan in project tez by apache.

the class TestVertexImpl2 method createVertexPlanForExeuctionContextTests.

private VertexPlan createVertexPlanForExeuctionContextTests(ExecutionContextTestInfoHolder info) {
    ConfigurationProto confProto = ConfigurationProto.newBuilder().addConfKeyValues(PlanKeyValuePair.newBuilder().setKey("foo").setValue("bar").build()).addConfKeyValues(PlanKeyValuePair.newBuilder().setKey("foo1").setValue("bar2").build()).build();
    VertexPlan.Builder vertexPlanBuilder = VertexPlan.newBuilder().setName(info.vertexName).setVertexConf(confProto).setTaskConfig(DAGProtos.PlanTaskConfiguration.newBuilder().setNumTasks(10).setJavaOpts("dontcare").setMemoryMb(1024).setVirtualCores(1).setTaskModule("taskmodule").build()).setType(DAGProtos.PlanVertexType.NORMAL);
    if (info.vertexExecutionContext != null) {
        vertexPlanBuilder.setExecutionContext(DagTypeConverters.convertToProto(info.vertexExecutionContext));
    }
    return vertexPlanBuilder.build();
}

Example 12 with VertexPlan

use of org.apache.tez.dag.api.records.DAGProtos.VertexPlan in project tez by apache.

the class DAGImpl method createVertex.

private static VertexImpl createVertex(DAGImpl dag, String vertexName, int vId) {
    TezVertexID vertexId = TezBuilderUtils.newVertexID(dag.getID(), vId);
    VertexPlan vertexPlan = dag.getJobPlan().getVertex(vId);
    VertexLocationHint vertexLocationHint = DagTypeConverters.convertFromDAGPlan(vertexPlan.getTaskLocationHintList());
    VertexImpl v = new VertexImpl(vertexId, vertexPlan, vertexName, dag.dagConf, dag.eventHandler, dag.taskCommunicatorManagerInterface, dag.clock, dag.taskHeartbeatHandler, !dag.commitAllOutputsOnSuccess, dag.appContext, vertexLocationHint, dag.vertexGroups, dag.taskSpecificLaunchCmdOption, dag.entityUpdateTracker, dag.dagOnlyConf);
    return v;
}

Example 13 with VertexPlan

use of org.apache.tez.dag.api.records.DAGProtos.VertexPlan in project tez by apache.

the class DAGImpl method parseVertexEdges.

// hooks up this VertexImpl to input and output EdgeProperties
private static void parseVertexEdges(DAGImpl dag, Map<String, EdgePlan> edgePlans, Vertex vertex) {
    VertexPlan vertexPlan = vertex.getVertexPlan();
    Map<Vertex, Edge> inVertices = new HashMap<Vertex, Edge>();
    Map<Vertex, Edge> outVertices = new HashMap<Vertex, Edge>();
    for (String inEdgeId : vertexPlan.getInEdgeIdList()) {
        EdgePlan edgePlan = edgePlans.get(inEdgeId);
        Vertex inVertex = dag.vertexMap.get(edgePlan.getInputVertexName());
        Edge edge = dag.edges.get(inEdgeId);
        edge.setSourceVertex(inVertex);
        edge.setDestinationVertex(vertex);
        inVertices.put(inVertex, edge);
    }
    for (String outEdgeId : vertexPlan.getOutEdgeIdList()) {
        EdgePlan edgePlan = edgePlans.get(outEdgeId);
        Vertex outVertex = dag.vertexMap.get(edgePlan.getOutputVertexName());
        Edge edge = dag.edges.get(outEdgeId);
        edge.setSourceVertex(vertex);
        edge.setDestinationVertex(outVertex);
        outVertices.put(outVertex, edge);
    }
    vertex.setInputVertices(inVertices);
    vertex.setOutputVertices(outVertices);
}

Example 14 with VertexPlan

use of org.apache.tez.dag.api.records.DAGProtos.VertexPlan in project tez by apache.

the class DAGAppMaster method generateDAGVizFile.

private void generateDAGVizFile(TezDAGID dagId, DAGPlan dagPB, String[] logDirs) {
    Graph graph = new Graph(sanitizeLabelForViz(dagPB.getName()));
    for (VertexPlan v : dagPB.getVertexList()) {
        String nodeLabel = sanitizeLabelForViz(v.getName()) + "[" + getShortClassName(v.getProcessorDescriptor().getClassName() + "]");
        Graph.Node n = graph.newNode(sanitizeLabelForViz(v.getName()), nodeLabel);
        for (DAGProtos.RootInputLeafOutputProto input : v.getInputsList()) {
            Graph.Node inputNode = graph.getNode(sanitizeLabelForViz(v.getName()) + "_" + sanitizeLabelForViz(input.getName()));
            inputNode.setLabel(sanitizeLabelForViz(v.getName()) + "[" + sanitizeLabelForViz(input.getName()) + "]");
            inputNode.setShape("box");
            inputNode.addEdge(n, "Input" + " [inputClass=" + getShortClassName(input.getIODescriptor().getClassName()) + ", initializer=" + getShortClassName(input.getControllerDescriptor().getClassName()) + "]");
        }
        for (DAGProtos.RootInputLeafOutputProto output : v.getOutputsList()) {
            Graph.Node outputNode = graph.getNode(sanitizeLabelForViz(v.getName()) + "_" + sanitizeLabelForViz(output.getName()));
            outputNode.setLabel(sanitizeLabelForViz(v.getName()) + "[" + sanitizeLabelForViz(output.getName()) + "]");
            outputNode.setShape("box");
            n.addEdge(outputNode, "Output" + " [outputClass=" + getShortClassName(output.getIODescriptor().getClassName()) + ", committer=" + getShortClassName(output.getControllerDescriptor().getClassName()) + "]");
        }
    }
    for (DAGProtos.EdgePlan e : dagPB.getEdgeList()) {
        Graph.Node n = graph.getNode(sanitizeLabelForViz(e.getInputVertexName()));
        n.addEdge(graph.getNode(sanitizeLabelForViz(e.getOutputVertexName())), "[" + "input=" + getShortClassName(e.getEdgeSource().getClassName()) + ", output=" + getShortClassName(e.getEdgeDestination().getClassName()) + ", dataMovement=" + e.getDataMovementType().name().trim() + ", schedulingType=" + e.getSchedulingType().name().trim() + "]");
    }
    String outputFile = "";
    if (logDirs != null && logDirs.length != 0) {
        outputFile += logDirs[0];
        outputFile += File.separator;
    }
    outputFile += dagId.toString() + ".dot";
    try {
        LOG.info("Generating DAG graphviz file" + ", dagId=" + dagId.toString() + ", filePath=" + outputFile);
        graph.save(outputFile);
    } catch (Exception e) {
        LOG.warn("Error occurred when trying to save graph structure" + " for dag " + dagId.toString(), e);
    }
}

Example 15 with VertexPlan

use of org.apache.tez.dag.api.records.DAGProtos.VertexPlan in project tez by apache.

the class DAGAppMaster method startDAG.

private void startDAG(DAGPlan dagPlan, Map<String, LocalResource> additionalAMResources) throws TezException {
    long submitTime = this.clock.getTime();
    this.appName = dagPlan.getName();
    // /////////////////// Create the job itself.
    final DAG newDAG = createDAG(dagPlan);
    _updateLoggers(newDAG, "");
    if (LOG.isDebugEnabled()) {
        LOG.debug("Running a DAG with " + dagPlan.getVertexCount() + " vertices ");
        for (VertexPlan v : dagPlan.getVertexList()) {
            LOG.debug("DAG has vertex " + v.getName());
        }
    }
    Map<String, LocalResource> lrDiff = getAdditionalLocalResourceDiff(newDAG, additionalAMResources);
    if (lrDiff != null) {
        amResources.putAll(lrDiff);
        cumulativeAdditionalResources.putAll(lrDiff);
    }
    String callerContextStr = "";
    if (dagPlan.hasCallerContext()) {
        CallerContext callerContext = DagTypeConverters.convertCallerContextFromProto(dagPlan.getCallerContext());
        callerContextStr = ", callerContext=" + callerContext.contextAsSimpleString();
    }
    LOG.info("Running DAG: " + dagPlan.getName() + callerContextStr);
    String timeStamp = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(Calendar.getInstance().getTime());
    System.err.println(timeStamp + " Running Dag: " + newDAG.getID());
    System.out.println(timeStamp + " Running Dag: " + newDAG.getID());
    // Job name is the same as the app name until we support multiple dags
    // for an app later
    final DAGSubmittedEvent submittedEvent = new DAGSubmittedEvent(newDAG.getID(), submitTime, dagPlan, this.appAttemptID, cumulativeAdditionalResources, newDAG.getUserName(), newDAG.getConf(), containerLogs, getContext().getQueueName());
    boolean dagLoggingEnabled = newDAG.getConf().getBoolean(TezConfiguration.TEZ_DAG_HISTORY_LOGGING_ENABLED, TezConfiguration.TEZ_DAG_HISTORY_LOGGING_ENABLED_DEFAULT);
    submittedEvent.setHistoryLoggingEnabled(dagLoggingEnabled);
    try {
        appMasterUgi.doAs(new PrivilegedExceptionAction<Void>() {

            @Override
            public Void run() throws Exception {
                historyEventHandler.handleCriticalEvent(new DAGHistoryEvent(newDAG.getID(), submittedEvent));
                return null;
            }
        });
    } catch (IOException e) {
        throw new TezUncheckedException(e);
    } catch (InterruptedException e) {
        throw new TezUncheckedException(e);
    }
    startDAGExecution(newDAG, lrDiff);
    // set state after curDag is set
    this.state = DAGAppMasterState.RUNNING;
}