Examples with DAGPlan org.apache.tez.dag.api.records.DAGProtos.DAGPlan used on opensource projects

Example 6 with DAGPlan

use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.

the class TestVertexImpl method createDAGWithCustomVertexManager.

private DAGPlan createDAGWithCustomVertexManager() {
    LOG.info("Setting up custom vertex manager dag plan");
    DAGPlan dag = DAGPlan.newBuilder().setName("TestCustomVMDAG").addVertex(VertexPlan.newBuilder().setName("v1").setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("A.class")).setType(PlanVertexType.NORMAL).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(-1).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("A.class").build()).setVertexManagerPlugin(TezEntityDescriptorProto.newBuilder().setClassName(VertexManagerPluginForTest.class.getName())).build()).addVertex(VertexPlan.newBuilder().setName("v2").setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("A.class")).setType(PlanVertexType.NORMAL).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(-1).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("A.class").build()).setVertexManagerPlugin(TezEntityDescriptorProto.newBuilder().setClassName(VertexManagerPluginForTest.class.getName())).addOutEdgeId("2_3").build()).addVertex(VertexPlan.newBuilder().setName("v3").setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("A.class")).setType(PlanVertexType.NORMAL).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(-1).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("A.class").build()).setVertexManagerPlugin(TezEntityDescriptorProto.newBuilder().setClassName(VertexManagerPluginForTest.class.getName())).addInEdgeId("2_3").build()).addEdge(EdgePlan.newBuilder().setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("2_3")).setInputVertexName("v2").setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("2_3.class")).setOutputVertexName("v3").setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER).setId("2_3").setDataSourceType(PlanEdgeDataSourceType.PERSISTED).setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL).build()).build();
    return dag;
}

Example 7 with DAGPlan

use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.

the class TestVertexImpl method createDAGPlanWithInitializer0Tasks.

private DAGPlan createDAGPlanWithInitializer0Tasks(String initializerClassName) {
    LOG.info("Setting up dag plan with input initializer and 0 tasks");
    DAGPlan dag = DAGPlan.newBuilder().setName("initializerWith0Tasks").addVertex(VertexPlan.newBuilder().setName("vertex1").setType(PlanVertexType.NORMAL).addInputs(RootInputLeafOutputProto.newBuilder().setControllerDescriptor(TezEntityDescriptorProto.newBuilder().setClassName(initializerClassName)).setName("input1").setIODescriptor(TezEntityDescriptorProto.newBuilder().setClassName("InputClazz").build()).build()).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(-1).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("x1.y1").build()).addInEdgeId("e1").build()).addVertex(VertexPlan.newBuilder().setName("vertex2").setType(PlanVertexType.NORMAL).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(1).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("x2.y2").build()).addOutEdgeId("e1").build()).addEdge(EdgePlan.newBuilder().setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("v1_v2")).setInputVertexName("vertex2").setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o2")).setOutputVertexName("vertex1").setDataMovementType(PlanEdgeDataMovementType.BROADCAST).setId("e1").setDataSourceType(PlanEdgeDataSourceType.PERSISTED).setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL).build()).build();
    return dag;
}

Example 8 with DAGPlan

use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.

the class TestVertexImpl method createSamplerDAGPlan.

// Create a plan with 3 vertices: A, B, C
// A -> B, A -> C, B -> C
private DAGPlan createSamplerDAGPlan(boolean customEdge) {
    LOG.info("Setting up dag plan");
    VertexPlan.Builder vCBuilder = VertexPlan.newBuilder();
    vCBuilder.setName("C").setType(PlanVertexType.NORMAL).setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("C.class")).addTaskLocationHint(PlanTaskLocationHint.newBuilder().addHost("host3").addRack("rack3").build()).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(customEdge ? -1 : 2).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("foo").setTaskModule("x3.y3").build()).setVertexManagerPlugin(TezEntityDescriptorProto.newBuilder().setClassName(VertexManagerPluginForTest.class.getName())).addInEdgeId("A_C").addInEdgeId("B_C");
    if (customEdge) {
        vCBuilder.setVertexManagerPlugin(TezEntityDescriptorProto.newBuilder().setClassName(VertexManagerPluginForTest.class.getName()));
    }
    VertexPlan vCPlan = vCBuilder.build();
    DAGPlan dag = DAGPlan.newBuilder().setName("TestSamplerDAG").addVertex(VertexPlan.newBuilder().setName("A").setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("A.class")).setType(PlanVertexType.NORMAL).addTaskLocationHint(PlanTaskLocationHint.newBuilder().addHost("host1").addRack("rack1").build()).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(1).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("A.class").build()).addOutEdgeId("A_B").addOutEdgeId("A_C").build()).addVertex(VertexPlan.newBuilder().setName("B").setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("B.class")).setType(PlanVertexType.NORMAL).addTaskLocationHint(PlanTaskLocationHint.newBuilder().addHost("host2").addRack("rack2").build()).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(2).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("").build()).addInEdgeId("A_B").addOutEdgeId("B_C").build()).addVertex(vCPlan).addEdge(EdgePlan.newBuilder().setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("A_B.class")).setInputVertexName("A").setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("A_B.class")).setOutputVertexName("B").setDataMovementType(PlanEdgeDataMovementType.BROADCAST).setId("A_B").setDataSourceType(PlanEdgeDataSourceType.PERSISTED).setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL).build()).addEdge(EdgePlan.newBuilder().setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("A_C")).setInputVertexName("A").setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("A_C.class")).setOutputVertexName("C").setDataMovementType(customEdge ? PlanEdgeDataMovementType.CUSTOM : PlanEdgeDataMovementType.SCATTER_GATHER).setId("A_C").setDataSourceType(PlanEdgeDataSourceType.PERSISTED).setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL).build()).addEdge(EdgePlan.newBuilder().setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("B_C.class")).setInputVertexName("B").setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("B_C.class")).setOutputVertexName("C").setDataMovementType(customEdge ? PlanEdgeDataMovementType.CUSTOM : PlanEdgeDataMovementType.SCATTER_GATHER).setId("B_C").setDataSourceType(PlanEdgeDataSourceType.PERSISTED).setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL).build()).build();
    return dag;
}

Example 9 with DAGPlan

use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.

the class TestVertexImpl method createDAGPlanWithCountingVM.

// Simple dag with a CountingVM on v3 (which has v1, v2 as inputs)
// v1, v2 -> v3
private DAGPlan createDAGPlanWithCountingVM() {
    LOG.info("Setting up dag plan with coutning VertexManager");
    DAGPlan dag = DAGPlan.newBuilder().setName("dagWithCountingVM").addVertex(VertexPlan.newBuilder().setName("vertex1").setType(PlanVertexType.NORMAL).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(1).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("x1.y1").build()).addOutEdgeId("e1").build()).addVertex(VertexPlan.newBuilder().setName("vertex2").setType(PlanVertexType.NORMAL).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(1).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("x2.y2").build()).addOutEdgeId("e2").build()).addVertex(VertexPlan.newBuilder().setName("vertex3").setType(PlanVertexType.NORMAL).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(1).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("x3.y3").build()).addInEdgeId("e1").addInEdgeId("e2").setVertexManagerPlugin(TezEntityDescriptorProto.newBuilder().setClassName(InvocationCountingVertexManager.class.getName())).build()).addEdge(EdgePlan.newBuilder().setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("v1_v2")).setInputVertexName("vertex1").setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o2")).setOutputVertexName("vertex3").setDataMovementType(PlanEdgeDataMovementType.BROADCAST).setId("e1").setDataSourceType(PlanEdgeDataSourceType.PERSISTED).setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL).build()).addEdge(EdgePlan.newBuilder().setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("v1_v3")).setInputVertexName("vertex2").setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o2")).setOutputVertexName("vertex3").setDataMovementType(PlanEdgeDataMovementType.BROADCAST).setId("e2").setDataSourceType(PlanEdgeDataSourceType.PERSISTED).setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL).build()).build();
    return dag;
}

Example 10 with DAGPlan

use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.

the class TestVertexImpl method createDAGPlanWithInputDistributor.

private DAGPlan createDAGPlanWithInputDistributor(String initializerClassName) {
    LOG.info("Setting up invalid dag plan with input distributor");
    DAGPlan dag = DAGPlan.newBuilder().setName("testVertexWithInitializer").addVertex(VertexPlan.newBuilder().setName("vertex1").setType(PlanVertexType.NORMAL).addInputs(RootInputLeafOutputProto.newBuilder().setControllerDescriptor(TezEntityDescriptorProto.newBuilder().setClassName(initializerClassName)).setName("input1").setIODescriptor(TezEntityDescriptorProto.newBuilder().setClassName("InputClazz").build()).build()).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(2).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("x3.y3").build()).addOutEdgeId("e1").build()).addVertex(VertexPlan.newBuilder().setName("vertex2").setType(PlanVertexType.NORMAL).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(2).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("x1.y1").build()).addInEdgeId("e1").build()).addEdge(EdgePlan.newBuilder().setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("v1_v5")).setInputVertexName("vertex1").setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o2")).setOutputVertexName("vertex2").setDataMovementType(PlanEdgeDataMovementType.CUSTOM).setId("e1").setDataSourceType(PlanEdgeDataSourceType.PERSISTED).setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL).build()).build();
    return dag;
}