Examples with VertexPlan org.apache.tez.dag.api.records.DAGProtos.VertexPlan used on opensource projects

Example 6 with VertexPlan

use of org.apache.tez.dag.api.records.DAGProtos.VertexPlan in project tez by apache.

the class TestDAGPlan method userVertexOrderingIsMaintained.

@Test(timeout = 5000)
public void userVertexOrderingIsMaintained() {
    DAG dag = DAG.create("testDag");
    ProcessorDescriptor pd1 = ProcessorDescriptor.create("processor1").setUserPayload(UserPayload.create(ByteBuffer.wrap("processor1Bytes".getBytes())));
    ProcessorDescriptor pd2 = ProcessorDescriptor.create("processor2").setUserPayload(UserPayload.create(ByteBuffer.wrap("processor2Bytes".getBytes())));
    ProcessorDescriptor pd3 = ProcessorDescriptor.create("processor3").setUserPayload(UserPayload.create(ByteBuffer.wrap("processor3Bytes".getBytes())));
    Vertex v1 = Vertex.create("v1", pd1, 10, Resource.newInstance(1024, 1));
    Vertex v2 = Vertex.create("v2", pd2, 1, Resource.newInstance(1024, 1));
    Vertex v3 = Vertex.create("v3", pd3, 1, Resource.newInstance(1024, 1));
    v1.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>()).addTaskLocalFiles(new HashMap<String, LocalResource>());
    v2.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>()).addTaskLocalFiles(new HashMap<String, LocalResource>());
    v3.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>()).addTaskLocalFiles(new HashMap<String, LocalResource>());
    InputDescriptor inputDescriptor = InputDescriptor.create("input").setUserPayload(UserPayload.create(ByteBuffer.wrap("inputBytes".getBytes())));
    OutputDescriptor outputDescriptor = OutputDescriptor.create("output").setUserPayload(UserPayload.create(ByteBuffer.wrap("outputBytes".getBytes())));
    Edge edge = Edge.create(v1, v2, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, outputDescriptor, inputDescriptor));
    dag.addVertex(v1).addVertex(v2).addEdge(edge).addVertex(v3);
    DAGPlan dagProto = dag.createDag(new TezConfiguration(), null, null, null, true);
    assertEquals(3, dagProto.getVertexCount());
    assertEquals(1, dagProto.getEdgeCount());
    VertexPlan v1Proto = dagProto.getVertex(0);
    VertexPlan v2Proto = dagProto.getVertex(1);
    VertexPlan v3Proto = dagProto.getVertex(2);
    EdgePlan edgeProto = dagProto.getEdge(0);
    // either v1 or v2 will be on top based on topological order
    String v1ProtoPayload = new String(v1Proto.getProcessorDescriptor().getTezUserPayload().getUserPayload().toByteArray());
    String v2ProtoPayload = new String(v2Proto.getProcessorDescriptor().getTezUserPayload().getUserPayload().toByteArray());
    assertTrue(v1ProtoPayload.equals("processor1Bytes") || v1ProtoPayload.equals("processor3Bytes"));
    assertTrue(v2ProtoPayload.equals("processor1Bytes") || v2ProtoPayload.equals("processor3Bytes"));
    assertTrue(v1Proto.getProcessorDescriptor().getClassName().equals("processor1") || v1Proto.getProcessorDescriptor().getClassName().equals("processor3"));
    assertTrue(v2Proto.getProcessorDescriptor().getClassName().equals("processor1") || v2Proto.getProcessorDescriptor().getClassName().equals("processor3"));
    assertEquals("processor2Bytes", new String(v3Proto.getProcessorDescriptor().getTezUserPayload().getUserPayload().toByteArray()));
    assertEquals("processor2", v3Proto.getProcessorDescriptor().getClassName());
    assertEquals("inputBytes", new String(edgeProto.getEdgeDestination().getTezUserPayload().getUserPayload().toByteArray()));
    assertEquals("input", edgeProto.getEdgeDestination().getClassName());
    assertEquals("outputBytes", new String(edgeProto.getEdgeSource().getTezUserPayload().getUserPayload().toByteArray()));
    assertEquals("output", edgeProto.getEdgeSource().getClassName());
    EdgeProperty edgeProperty = DagTypeConverters.createEdgePropertyMapFromDAGPlan(dagProto.getEdgeList().get(0));
    byte[] ib = edgeProperty.getEdgeDestination().getUserPayload().deepCopyAsArray();
    assertEquals("inputBytes", new String(ib));
    assertEquals("input", edgeProperty.getEdgeDestination().getClassName());
    byte[] ob = edgeProperty.getEdgeSource().getUserPayload().deepCopyAsArray();
    assertEquals("outputBytes", new String(ob));
    assertEquals("output", edgeProperty.getEdgeSource().getClassName());
}

Example 7 with VertexPlan

use of org.apache.tez.dag.api.records.DAGProtos.VertexPlan in project tez by apache.

the class TestDAGVerify method testDAGCreateDataInference.

@Test(timeout = 5000)
public void testDAGCreateDataInference() {
    Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create(dummyProcessorClassName));
    Map<String, LocalResource> lrs1 = Maps.newHashMap();
    String lrName1 = "LR1";
    lrs1.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
    Map<String, LocalResource> lrs2 = Maps.newHashMap();
    String lrName2 = "LR2";
    lrs2.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
    Set<String> hosts = Sets.newHashSet();
    hosts.add("h1");
    hosts.add("h2");
    List<TaskLocationHint> taskLocationHints = Lists.newLinkedList();
    taskLocationHints.add(TaskLocationHint.createTaskLocationHint(hosts, null));
    taskLocationHints.add(TaskLocationHint.createTaskLocationHint(hosts, null));
    VertexLocationHint vLoc = VertexLocationHint.create(taskLocationHints);
    DataSourceDescriptor ds = DataSourceDescriptor.create(InputDescriptor.create("I.class"), InputInitializerDescriptor.create(dummyInputInitClassName), dummyTaskCount, null, vLoc, lrs2);
    v1.addDataSource("i1", ds);
    DAG dag = DAG.create("testDag");
    dag.addVertex(v1);
    dag.addTaskLocalFiles(lrs1);
    DAGPlan dagPlan = dag.createDag(new TezConfiguration(), null, null, null, true);
    Assert.assertEquals(lrName1, dagPlan.getLocalResource(0).getName());
    VertexPlan vPlan = dagPlan.getVertex(0);
    PlanTaskConfiguration taskPlan = vPlan.getTaskConfig();
    Assert.assertEquals(dummyTaskCount, taskPlan.getNumTasks());
    Assert.assertEquals(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB_DEFAULT, taskPlan.getMemoryMb());
    Assert.assertEquals(lrName2, taskPlan.getLocalResource(0).getName());
    Assert.assertEquals(dummyTaskCount, vPlan.getTaskLocationHintCount());
}

Example 8 with VertexPlan

use of org.apache.tez.dag.api.records.DAGProtos.VertexPlan in project tez by apache.

the class TestVertexImpl method createDAGPlanForOneToOneSplit.

private DAGPlan createDAGPlanForOneToOneSplit(String initializerClassName, int numTasks, boolean addNullEdge) {
    VertexPlan.Builder v1Builder = VertexPlan.newBuilder();
    v1Builder.setName("vertex1").setType(PlanVertexType.NORMAL).addOutEdgeId("e1").addOutEdgeId("e2");
    if (addNullEdge) {
        v1Builder.addOutEdgeId("e5");
    }
    if (initializerClassName != null) {
        numTasks = -1;
        v1Builder.addInputs(RootInputLeafOutputProto.newBuilder().setControllerDescriptor(TezEntityDescriptorProto.newBuilder().setClassName(initializerClassName)).setName("input1").setIODescriptor(TezEntityDescriptorProto.newBuilder().setClassName("InputClazz").build()).build()).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(-1).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("x1.y1").build());
    } else {
        v1Builder.setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(numTasks).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("x1.y1").build());
    }
    VertexPlan v1Plan = v1Builder.build();
    VertexPlan.Builder v4Builder = VertexPlan.newBuilder();
    v4Builder.setName("vertex4").setType(PlanVertexType.NORMAL).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(numTasks).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("x4.y4").build()).addInEdgeId("e3").addInEdgeId("e4");
    if (addNullEdge) {
        v4Builder.addOutEdgeId("e6");
    }
    VertexPlan v4Plan = v4Builder.build();
    LOG.info("Setting up one to one dag plan");
    DAGPlan.Builder dagBuilder = DAGPlan.newBuilder().setName("testVertexOneToOneSplit").addVertex(v1Plan).addVertex(VertexPlan.newBuilder().setName("vertex2").setType(PlanVertexType.NORMAL).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(numTasks).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("x2.y2").build()).addInEdgeId("e1").addOutEdgeId("e3").build()).addVertex(VertexPlan.newBuilder().setName("vertex3").setType(PlanVertexType.NORMAL).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(numTasks).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("x3.y3").build()).addInEdgeId("e2").addOutEdgeId("e4").build()).addVertex(v4Plan).addEdge(EdgePlan.newBuilder().setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("v1_v2")).setInputVertexName("vertex1").setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o2")).setOutputVertexName("vertex2").setDataMovementType(PlanEdgeDataMovementType.ONE_TO_ONE).setId("e1").setDataSourceType(PlanEdgeDataSourceType.PERSISTED).setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL).build()).addEdge(EdgePlan.newBuilder().setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("v1_v3")).setInputVertexName("vertex1").setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o2")).setOutputVertexName("vertex3").setDataMovementType(PlanEdgeDataMovementType.ONE_TO_ONE).setId("e2").setDataSourceType(PlanEdgeDataSourceType.PERSISTED).setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL).build()).addEdge(EdgePlan.newBuilder().setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("v2_v4")).setInputVertexName("vertex2").setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o2")).setOutputVertexName("vertex4").setDataMovementType(PlanEdgeDataMovementType.ONE_TO_ONE).setId("e3").setDataSourceType(PlanEdgeDataSourceType.PERSISTED).setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL).build()).addEdge(EdgePlan.newBuilder().setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("v3_v4")).setInputVertexName("vertex3").setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o2")).setOutputVertexName("vertex4").setDataMovementType(PlanEdgeDataMovementType.ONE_TO_ONE).setId("e4").setDataSourceType(PlanEdgeDataSourceType.PERSISTED).setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL));
    if (addNullEdge) {
        dagBuilder.addVertex(VertexPlan.newBuilder().setName("vertex5").setType(PlanVertexType.NORMAL).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(1).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("x4.y4").build()).addInEdgeId("e5").build()).addVertex(VertexPlan.newBuilder().setName("vertex6").setType(PlanVertexType.NORMAL).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(1).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("x4.y4").build()).addInEdgeId("e6").build()).addEdge(EdgePlan.newBuilder().setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("v1_v5")).setInputVertexName("vertex1").setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o2")).setOutputVertexName("vertex5").setDataMovementType(PlanEdgeDataMovementType.CUSTOM).setId("e5").setDataSourceType(PlanEdgeDataSourceType.PERSISTED).setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL).build()).addEdge(EdgePlan.newBuilder().setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("v4_v6")).setInputVertexName("vertex4").setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o2")).setOutputVertexName("vertex6").setDataMovementType(PlanEdgeDataMovementType.CUSTOM).setId("e6").setDataSourceType(PlanEdgeDataSourceType.PERSISTED).setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL).build());
    }
    DAGPlan dag = dagBuilder.build();
    return dag;
}

Example 9 with VertexPlan

use of org.apache.tez.dag.api.records.DAGProtos.VertexPlan in project tez by apache.

the class TestVertexImpl method setupVertices.

private void setupVertices() {
    int vCnt = dagPlan.getVertexCount();
    LOG.info("Setting up vertices from dag plan, verticesCnt=" + vCnt);
    vertices = new HashMap<String, VertexImpl>();
    vertexIdMap = new HashMap<TezVertexID, VertexImpl>();
    Configuration dagConf = new Configuration(false);
    dagConf.set("abc", "foobar");
    for (int i = 0; i < vCnt; ++i) {
        VertexPlan vPlan = dagPlan.getVertex(i);
        String vName = vPlan.getName();
        TezVertexID vertexId = TezVertexID.getInstance(dagId, i + 1);
        VertexImpl v = null;
        VertexLocationHint locationHint = DagTypeConverters.convertFromDAGPlan(vPlan.getTaskLocationHintList());
        if (useCustomInitializer) {
            if (customInitializer == null) {
                v = new VertexImplWithControlledInitializerManager(vertexId, vPlan, vPlan.getName(), conf, dispatcher.getEventHandler(), taskCommunicatorManagerInterface, clock, thh, appContext, locationHint, dispatcher, updateTracker, dagConf);
            } else {
                v = new VertexImplWithRunningInputInitializer(vertexId, vPlan, vPlan.getName(), conf, dispatcher.getEventHandler(), taskCommunicatorManagerInterface, clock, thh, appContext, locationHint, dispatcher, customInitializer, updateTracker, dagConf);
            }
        } else {
            v = new VertexImpl(vertexId, vPlan, vPlan.getName(), conf, dispatcher.getEventHandler(), taskCommunicatorManagerInterface, clock, thh, true, appContext, locationHint, vertexGroups, taskSpecificLaunchCmdOption, updateTracker, dagConf);
        }
        vertices.put(vName, v);
        vertexIdMap.put(vertexId, v);
    }
}

Example 10 with VertexPlan

use of org.apache.tez.dag.api.records.DAGProtos.VertexPlan in project tez by apache.

the class TestVertexImpl method parseVertexEdges.

private void parseVertexEdges() {
    LOG.info("Parsing edges from dag plan, edgeCount=" + dagPlan.getEdgeCount());
    int vCnt = dagPlan.getVertexCount();
    Map<String, EdgePlan> edgePlans = DagTypeConverters.createEdgePlanMapFromDAGPlan(dagPlan.getEdgeList());
    // TODO - this test logic is tightly linked to impl DAGImpl code.
    for (int i = 0; i < vCnt; ++i) {
        VertexPlan vertexPlan = dagPlan.getVertex(i);
        Vertex vertex = vertices.get(vertexPlan.getName());
        Map<Vertex, Edge> inVertices = new HashMap<Vertex, Edge>();
        Map<Vertex, Edge> outVertices = new HashMap<Vertex, Edge>();
        for (String inEdgeId : vertexPlan.getInEdgeIdList()) {
            EdgePlan edgePlan = edgePlans.get(inEdgeId);
            Vertex inVertex = this.vertices.get(edgePlan.getInputVertexName());
            Edge edge = this.edges.get(inEdgeId);
            edge.setSourceVertex(inVertex);
            edge.setDestinationVertex(vertex);
            inVertices.put(inVertex, edge);
        }
        for (String outEdgeId : vertexPlan.getOutEdgeIdList()) {
            EdgePlan edgePlan = edgePlans.get(outEdgeId);
            Vertex outVertex = this.vertices.get(edgePlan.getOutputVertexName());
            Edge edge = this.edges.get(outEdgeId);
            edge.setSourceVertex(vertex);
            edge.setDestinationVertex(outVertex);
            outVertices.put(outVertex, edge);
        }
        LOG.info("Setting input vertices for vertex " + vertex.getName() + ", inputVerticesCnt=" + inVertices.size());
        vertex.setInputVertices(inVertices);
        LOG.info("Setting output vertices for vertex " + vertex.getName() + ", outputVerticesCnt=" + outVertices.size());
        vertex.setOutputVertices(outVertices);
    }
}