Examples with DAGPlan org.apache.tez.dag.api.records.DAGProtos.DAGPlan used on opensource projects

Example 56 with DAGPlan

use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.

the class TestDAGPlan method testServiceDescriptorPropagation.

@Test(timeout = 5000)
public void testServiceDescriptorPropagation() {
    DAG dag = DAG.create("testDag");
    ProcessorDescriptor pd1 = ProcessorDescriptor.create("processor1").setUserPayload(UserPayload.create(ByteBuffer.wrap("processor1Bytes".getBytes())));
    ProcessorDescriptor pd2 = ProcessorDescriptor.create("processor2").setUserPayload(UserPayload.create(ByteBuffer.wrap("processor2Bytes".getBytes())));
    VertexExecutionContext defaultExecutionContext = VertexExecutionContext.create("plugin", "plugin", "plugin");
    VertexExecutionContext v1Context = VertexExecutionContext.createExecuteInAm(true);
    ServicePluginsDescriptor servicePluginsDescriptor = ServicePluginsDescriptor.create(true, new TaskSchedulerDescriptor[] { TaskSchedulerDescriptor.create("plugin", null) }, new ContainerLauncherDescriptor[] { ContainerLauncherDescriptor.create("plugin", null) }, new TaskCommunicatorDescriptor[] { TaskCommunicatorDescriptor.create("plugin", null) });
    Vertex v1 = Vertex.create("v1", pd1, 10, Resource.newInstance(1024, 1)).setExecutionContext(v1Context);
    Vertex v2 = Vertex.create("v2", pd2, 1, Resource.newInstance(1024, 1));
    v1.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>()).addTaskLocalFiles(new HashMap<String, LocalResource>());
    v2.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>()).addTaskLocalFiles(new HashMap<String, LocalResource>());
    InputDescriptor inputDescriptor = InputDescriptor.create("input").setUserPayload(UserPayload.create(ByteBuffer.wrap("inputBytes".getBytes())));
    OutputDescriptor outputDescriptor = OutputDescriptor.create("output").setUserPayload(UserPayload.create(ByteBuffer.wrap("outputBytes".getBytes())));
    Edge edge = Edge.create(v1, v2, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, outputDescriptor, inputDescriptor));
    dag.addVertex(v1).addVertex(v2).addEdge(edge);
    dag.setExecutionContext(defaultExecutionContext);
    DAGPlan dagProto = dag.createDag(new TezConfiguration(), null, null, null, true, servicePluginsDescriptor, null);
    assertEquals(2, dagProto.getVertexCount());
    assertEquals(1, dagProto.getEdgeCount());
    assertTrue(dagProto.hasDefaultExecutionContext());
    VertexExecutionContextProto defaultContextProto = dagProto.getDefaultExecutionContext();
    assertFalse(defaultContextProto.getExecuteInContainers());
    assertFalse(defaultContextProto.getExecuteInAm());
    assertEquals("plugin", defaultContextProto.getTaskSchedulerName());
    assertEquals("plugin", defaultContextProto.getContainerLauncherName());
    assertEquals("plugin", defaultContextProto.getTaskCommName());
    VertexPlan v1Proto = dagProto.getVertex(0);
    assertTrue(v1Proto.hasExecutionContext());
    VertexExecutionContextProto v1ContextProto = v1Proto.getExecutionContext();
    assertFalse(v1ContextProto.getExecuteInContainers());
    assertTrue(v1ContextProto.getExecuteInAm());
    assertFalse(v1ContextProto.hasTaskSchedulerName());
    assertFalse(v1ContextProto.hasContainerLauncherName());
    assertFalse(v1ContextProto.hasTaskCommName());
    VertexPlan v2Proto = dagProto.getVertex(1);
    assertFalse(v2Proto.hasExecutionContext());
}

Example 57 with DAGPlan

use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.

the class TestDAGPlan method testUserPayloadSerde.

@Test(timeout = 5000)
public void testUserPayloadSerde() {
    DAG dag = DAG.create("testDag");
    ProcessorDescriptor pd1 = ProcessorDescriptor.create("processor1").setUserPayload(UserPayload.create(ByteBuffer.wrap("processor1Bytes".getBytes())));
    ProcessorDescriptor pd2 = ProcessorDescriptor.create("processor2").setUserPayload(UserPayload.create(ByteBuffer.wrap("processor2Bytes".getBytes())));
    Vertex v1 = Vertex.create("v1", pd1, 10, Resource.newInstance(1024, 1));
    Vertex v2 = Vertex.create("v2", pd2, 1, Resource.newInstance(1024, 1));
    v1.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>()).addTaskLocalFiles(new HashMap<String, LocalResource>());
    v2.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>()).addTaskLocalFiles(new HashMap<String, LocalResource>());
    InputDescriptor inputDescriptor = InputDescriptor.create("input").setUserPayload(UserPayload.create(ByteBuffer.wrap("inputBytes".getBytes())));
    OutputDescriptor outputDescriptor = OutputDescriptor.create("output").setUserPayload(UserPayload.create(ByteBuffer.wrap("outputBytes".getBytes())));
    Edge edge = Edge.create(v1, v2, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, outputDescriptor, inputDescriptor));
    dag.addVertex(v1).addVertex(v2).addEdge(edge);
    DAGPlan dagProto = dag.createDag(new TezConfiguration(), null, null, null, true);
    assertEquals(2, dagProto.getVertexCount());
    assertEquals(1, dagProto.getEdgeCount());
    VertexPlan v1Proto = dagProto.getVertex(0);
    VertexPlan v2Proto = dagProto.getVertex(1);
    EdgePlan edgeProto = dagProto.getEdge(0);
    assertEquals("processor1Bytes", new String(v1Proto.getProcessorDescriptor().getTezUserPayload().getUserPayload().toByteArray()));
    assertEquals("processor1", v1Proto.getProcessorDescriptor().getClassName());
    assertEquals("processor2Bytes", new String(v2Proto.getProcessorDescriptor().getTezUserPayload().getUserPayload().toByteArray()));
    assertEquals("processor2", v2Proto.getProcessorDescriptor().getClassName());
    assertEquals("inputBytes", new String(edgeProto.getEdgeDestination().getTezUserPayload().getUserPayload().toByteArray()));
    assertEquals("input", edgeProto.getEdgeDestination().getClassName());
    assertEquals("outputBytes", new String(edgeProto.getEdgeSource().getTezUserPayload().getUserPayload().toByteArray()));
    assertEquals("output", edgeProto.getEdgeSource().getClassName());
    EdgeProperty edgeProperty = DagTypeConverters.createEdgePropertyMapFromDAGPlan(dagProto.getEdgeList().get(0));
    byte[] ib = edgeProperty.getEdgeDestination().getUserPayload().deepCopyAsArray();
    assertEquals("inputBytes", new String(ib));
    assertEquals("input", edgeProperty.getEdgeDestination().getClassName());
    byte[] ob = edgeProperty.getEdgeSource().getUserPayload().deepCopyAsArray();
    assertEquals("outputBytes", new String(ob));
    assertEquals("output", edgeProperty.getEdgeSource().getClassName());
}

Example 58 with DAGPlan

use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.

the class TestDAGPlan method testBasicJobPlanSerde.

@Test(timeout = 5000)
public void testBasicJobPlanSerde() throws IOException {
    DAGPlan job = DAGPlan.newBuilder().setName("test").addVertex(VertexPlan.newBuilder().setName("vertex1").setType(PlanVertexType.NORMAL).addTaskLocationHint(PlanTaskLocationHint.newBuilder().addHost("machineName").addRack("rack1").build()).setTaskConfig(PlanTaskConfiguration.newBuilder().setNumTasks(2).setVirtualCores(4).setMemoryMb(1024).setJavaOpts("").setTaskModule("x.y").build()).build()).build();
    File file = tempFolder.newFile("jobPlan");
    FileOutputStream outStream = null;
    try {
        outStream = new FileOutputStream(file);
        job.writeTo(outStream);
    } finally {
        if (outStream != null) {
            outStream.close();
        }
    }
    DAGPlan inJob;
    FileInputStream inputStream;
    try {
        inputStream = new FileInputStream(file);
        inJob = DAGPlan.newBuilder().mergeFrom(inputStream).build();
    } finally {
        outStream.close();
    }
    Assert.assertEquals(job, inJob);
}

Example 59 with DAGPlan

use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.

the class TestDAG method testCreateDAGForHistoryLogLevel.

@Test
public void testCreateDAGForHistoryLogLevel() {
    Map<String, LocalResource> lrDAG = Collections.singletonMap("LR1", LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
    Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create("dummyProcessor1"), 1, Resource.newInstance(1, 1));
    Vertex v2 = Vertex.create("v2", ProcessorDescriptor.create("dummyProcessor2"), 1, Resource.newInstance(1, 1));
    DAG dag = DAG.create("dag1").addVertex(v1).addVertex(v2).addTaskLocalFiles(lrDAG);
    TezConfiguration tezConf = new TezConfiguration();
    // Expect null when history log level is not set in both dag and tezConf
    DAGPlan dagPlan = dag.createDag(tezConf, null, null, null, false);
    Builder builder = DAGPlan.newBuilder(dagPlan);
    Assert.assertNull(findKVP(builder.getDagConf(), TezConfiguration.TEZ_HISTORY_LOGGING_LOGLEVEL));
    // Set tezConf but not dag, expect value in tezConf.
    tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_LOGLEVEL, "TASK");
    dagPlan = dag.createDag(tezConf, null, null, null, false);
    Assert.assertEquals("TASK", findKVP(DAGPlan.newBuilder(dagPlan).getDagConf(), TezConfiguration.TEZ_HISTORY_LOGGING_LOGLEVEL));
    // Set invalid value in tezConf, expect exception.
    tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_LOGLEVEL, "invalid");
    try {
        dagPlan = dag.createDag(tezConf, null, null, null, false);
        Assert.fail("Expected illegal argument exception");
    } catch (IllegalArgumentException e) {
        Assert.assertEquals("Config: " + TezConfiguration.TEZ_HISTORY_LOGGING_LOGLEVEL + " is set to invalid value: invalid", e.getMessage());
    }
    // Set value in dag, should override tez conf value.
    dag.setHistoryLogLevel(HistoryLogLevel.VERTEX);
    dagPlan = dag.createDag(tezConf, null, null, null, false);
    Assert.assertEquals("VERTEX", findKVP(DAGPlan.newBuilder(dagPlan).getDagConf(), TezConfiguration.TEZ_HISTORY_LOGGING_LOGLEVEL));
    // Set value directly into dagConf.
    dag.setConf(TezConfiguration.TEZ_HISTORY_LOGGING_LOGLEVEL, HistoryLogLevel.DAG.name());
    dagPlan = dag.createDag(tezConf, null, null, null, false);
    Assert.assertEquals("DAG", findKVP(DAGPlan.newBuilder(dagPlan).getDagConf(), TezConfiguration.TEZ_HISTORY_LOGGING_LOGLEVEL));
    // Set value invalid directly into dagConf and expect exception.
    dag.setConf(TezConfiguration.TEZ_HISTORY_LOGGING_LOGLEVEL, "invalid");
    try {
        dagPlan = dag.createDag(tezConf, null, null, null, false);
        Assert.fail("Expected illegal argument exception");
    } catch (IllegalArgumentException e) {
        Assert.assertEquals("Config: " + TezConfiguration.TEZ_HISTORY_LOGGING_LOGLEVEL + " is set to invalid value: invalid", e.getMessage());
    }
}

Example 60 with DAGPlan

use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.

the class TestDAG method testRecreateDAG.

@Test
public void testRecreateDAG() {
    Map<String, LocalResource> lrDAG = Collections.singletonMap("LR1", LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
    Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create("dummyProcessor1"), 1, Resource.newInstance(1, 1));
    Vertex v2 = Vertex.create("v2", ProcessorDescriptor.create("dummyProcessor2"), 1, Resource.newInstance(1, 1));
    DAG dag = DAG.create("dag1").addVertex(v1).addVertex(v2).addTaskLocalFiles(lrDAG);
    TezConfiguration tezConf = new TezConfiguration();
    DAGPlan firstPlan = dag.createDag(tezConf, null, null, null, false);
    for (int i = 0; i < 3; ++i) {
        DAGPlan dagPlan = dag.createDag(tezConf, null, null, null, false);
        Assert.assertEquals(dagPlan, firstPlan);
    }
}