Examples with DAGPlan org.apache.tez.dag.api.records.DAGProtos.DAGPlan used on opensource projects

Example 21 with DAGPlan

use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.

the class TestDAGPlan method testCredentialsSerde.

@Test(timeout = 5000)
public void testCredentialsSerde() {
    DAG dag = DAG.create("testDag");
    ProcessorDescriptor pd1 = ProcessorDescriptor.create("processor1").setUserPayload(UserPayload.create(ByteBuffer.wrap("processor1Bytes".getBytes())));
    ProcessorDescriptor pd2 = ProcessorDescriptor.create("processor2").setUserPayload(UserPayload.create(ByteBuffer.wrap("processor2Bytes".getBytes())));
    Vertex v1 = Vertex.create("v1", pd1, 10, Resource.newInstance(1024, 1));
    Vertex v2 = Vertex.create("v2", pd2, 1, Resource.newInstance(1024, 1));
    v1.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>()).addTaskLocalFiles(new HashMap<String, LocalResource>());
    v2.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>()).addTaskLocalFiles(new HashMap<String, LocalResource>());
    InputDescriptor inputDescriptor = InputDescriptor.create("input").setUserPayload(UserPayload.create(ByteBuffer.wrap("inputBytes".getBytes())));
    OutputDescriptor outputDescriptor = OutputDescriptor.create("output").setUserPayload(UserPayload.create(ByteBuffer.wrap("outputBytes".getBytes())));
    Edge edge = Edge.create(v1, v2, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, outputDescriptor, inputDescriptor));
    dag.addVertex(v1).addVertex(v2).addEdge(edge);
    Credentials dagCredentials = new Credentials();
    Token<TokenIdentifier> token1 = new Token<TokenIdentifier>();
    Token<TokenIdentifier> token2 = new Token<TokenIdentifier>();
    dagCredentials.addToken(new Text("Token1"), token1);
    dagCredentials.addToken(new Text("Token2"), token2);
    dag.setCredentials(dagCredentials);
    DAGPlan dagProto = dag.createDag(new TezConfiguration(), null, null, null, true);
    assertTrue(dagProto.hasCredentialsBinary());
    Credentials fetchedCredentials = DagTypeConverters.convertByteStringToCredentials(dagProto.getCredentialsBinary());
    assertEquals(2, fetchedCredentials.numberOfTokens());
    assertNotNull(fetchedCredentials.getToken(new Text("Token1")));
    assertNotNull(fetchedCredentials.getToken(new Text("Token2")));
}

Example 22 with DAGPlan

use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.

the class TestDAGPlan method userVertexOrderingIsMaintained.

@Test(timeout = 5000)
public void userVertexOrderingIsMaintained() {
    DAG dag = DAG.create("testDag");
    ProcessorDescriptor pd1 = ProcessorDescriptor.create("processor1").setUserPayload(UserPayload.create(ByteBuffer.wrap("processor1Bytes".getBytes())));
    ProcessorDescriptor pd2 = ProcessorDescriptor.create("processor2").setUserPayload(UserPayload.create(ByteBuffer.wrap("processor2Bytes".getBytes())));
    ProcessorDescriptor pd3 = ProcessorDescriptor.create("processor3").setUserPayload(UserPayload.create(ByteBuffer.wrap("processor3Bytes".getBytes())));
    Vertex v1 = Vertex.create("v1", pd1, 10, Resource.newInstance(1024, 1));
    Vertex v2 = Vertex.create("v2", pd2, 1, Resource.newInstance(1024, 1));
    Vertex v3 = Vertex.create("v3", pd3, 1, Resource.newInstance(1024, 1));
    v1.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>()).addTaskLocalFiles(new HashMap<String, LocalResource>());
    v2.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>()).addTaskLocalFiles(new HashMap<String, LocalResource>());
    v3.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>()).addTaskLocalFiles(new HashMap<String, LocalResource>());
    InputDescriptor inputDescriptor = InputDescriptor.create("input").setUserPayload(UserPayload.create(ByteBuffer.wrap("inputBytes".getBytes())));
    OutputDescriptor outputDescriptor = OutputDescriptor.create("output").setUserPayload(UserPayload.create(ByteBuffer.wrap("outputBytes".getBytes())));
    Edge edge = Edge.create(v1, v2, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, outputDescriptor, inputDescriptor));
    dag.addVertex(v1).addVertex(v2).addEdge(edge).addVertex(v3);
    DAGPlan dagProto = dag.createDag(new TezConfiguration(), null, null, null, true);
    assertEquals(3, dagProto.getVertexCount());
    assertEquals(1, dagProto.getEdgeCount());
    VertexPlan v1Proto = dagProto.getVertex(0);
    VertexPlan v2Proto = dagProto.getVertex(1);
    VertexPlan v3Proto = dagProto.getVertex(2);
    EdgePlan edgeProto = dagProto.getEdge(0);
    // either v1 or v2 will be on top based on topological order
    String v1ProtoPayload = new String(v1Proto.getProcessorDescriptor().getTezUserPayload().getUserPayload().toByteArray());
    String v2ProtoPayload = new String(v2Proto.getProcessorDescriptor().getTezUserPayload().getUserPayload().toByteArray());
    assertTrue(v1ProtoPayload.equals("processor1Bytes") || v1ProtoPayload.equals("processor3Bytes"));
    assertTrue(v2ProtoPayload.equals("processor1Bytes") || v2ProtoPayload.equals("processor3Bytes"));
    assertTrue(v1Proto.getProcessorDescriptor().getClassName().equals("processor1") || v1Proto.getProcessorDescriptor().getClassName().equals("processor3"));
    assertTrue(v2Proto.getProcessorDescriptor().getClassName().equals("processor1") || v2Proto.getProcessorDescriptor().getClassName().equals("processor3"));
    assertEquals("processor2Bytes", new String(v3Proto.getProcessorDescriptor().getTezUserPayload().getUserPayload().toByteArray()));
    assertEquals("processor2", v3Proto.getProcessorDescriptor().getClassName());
    assertEquals("inputBytes", new String(edgeProto.getEdgeDestination().getTezUserPayload().getUserPayload().toByteArray()));
    assertEquals("input", edgeProto.getEdgeDestination().getClassName());
    assertEquals("outputBytes", new String(edgeProto.getEdgeSource().getTezUserPayload().getUserPayload().toByteArray()));
    assertEquals("output", edgeProto.getEdgeSource().getClassName());
    EdgeProperty edgeProperty = DagTypeConverters.createEdgePropertyMapFromDAGPlan(dagProto.getEdgeList().get(0));
    byte[] ib = edgeProperty.getEdgeDestination().getUserPayload().deepCopyAsArray();
    assertEquals("inputBytes", new String(ib));
    assertEquals("input", edgeProperty.getEdgeDestination().getClassName());
    byte[] ob = edgeProperty.getEdgeSource().getUserPayload().deepCopyAsArray();
    assertEquals("outputBytes", new String(ob));
    assertEquals("output", edgeProperty.getEdgeSource().getClassName());
}

Example 23 with DAGPlan

use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.

the class TestDAGPlan method testEdgeManagerSerde.

@Test(timeout = 5000)
public void testEdgeManagerSerde() {
    DAG dag = DAG.create("testDag");
    ProcessorDescriptor pd1 = ProcessorDescriptor.create("processor1").setUserPayload(UserPayload.create(ByteBuffer.wrap("processor1Bytes".getBytes())));
    ProcessorDescriptor pd2 = ProcessorDescriptor.create("processor2").setUserPayload(UserPayload.create(ByteBuffer.wrap("processor2Bytes".getBytes())));
    Vertex v1 = Vertex.create("v1", pd1, 10, Resource.newInstance(1024, 1));
    Vertex v2 = Vertex.create("v2", pd2, 1, Resource.newInstance(1024, 1));
    v1.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>()).addTaskLocalFiles(new HashMap<String, LocalResource>());
    v2.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>()).addTaskLocalFiles(new HashMap<String, LocalResource>());
    InputDescriptor inputDescriptor = InputDescriptor.create("input").setUserPayload(UserPayload.create(ByteBuffer.wrap("inputBytes".getBytes())));
    OutputDescriptor outputDescriptor = OutputDescriptor.create("output").setUserPayload(UserPayload.create(ByteBuffer.wrap("outputBytes".getBytes())));
    Edge edge = Edge.create(v1, v2, EdgeProperty.create(EdgeManagerPluginDescriptor.create("emClass").setUserPayload(UserPayload.create(ByteBuffer.wrap("emPayload".getBytes()))), DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, outputDescriptor, inputDescriptor));
    dag.addVertex(v1).addVertex(v2).addEdge(edge);
    DAGPlan dagProto = dag.createDag(new TezConfiguration(), null, null, null, true);
    EdgeProperty edgeProperty = DagTypeConverters.createEdgePropertyMapFromDAGPlan(dagProto.getEdgeList().get(0));
    EdgeManagerPluginDescriptor emDesc = edgeProperty.getEdgeManagerDescriptor();
    Assert.assertNotNull(emDesc);
    Assert.assertEquals("emClass", emDesc.getClassName());
    Assert.assertTrue(Arrays.equals("emPayload".getBytes(), emDesc.getUserPayload().deepCopyAsArray()));
}

Example 24 with DAGPlan

use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.

the class TestDAGVerify method testDAGCreateDataInference.

@Test(timeout = 5000)
public void testDAGCreateDataInference() {
    Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create(dummyProcessorClassName));
    Map<String, LocalResource> lrs1 = Maps.newHashMap();
    String lrName1 = "LR1";
    lrs1.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
    Map<String, LocalResource> lrs2 = Maps.newHashMap();
    String lrName2 = "LR2";
    lrs2.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
    Set<String> hosts = Sets.newHashSet();
    hosts.add("h1");
    hosts.add("h2");
    List<TaskLocationHint> taskLocationHints = Lists.newLinkedList();
    taskLocationHints.add(TaskLocationHint.createTaskLocationHint(hosts, null));
    taskLocationHints.add(TaskLocationHint.createTaskLocationHint(hosts, null));
    VertexLocationHint vLoc = VertexLocationHint.create(taskLocationHints);
    DataSourceDescriptor ds = DataSourceDescriptor.create(InputDescriptor.create("I.class"), InputInitializerDescriptor.create(dummyInputInitClassName), dummyTaskCount, null, vLoc, lrs2);
    v1.addDataSource("i1", ds);
    DAG dag = DAG.create("testDag");
    dag.addVertex(v1);
    dag.addTaskLocalFiles(lrs1);
    DAGPlan dagPlan = dag.createDag(new TezConfiguration(), null, null, null, true);
    Assert.assertEquals(lrName1, dagPlan.getLocalResource(0).getName());
    VertexPlan vPlan = dagPlan.getVertex(0);
    PlanTaskConfiguration taskPlan = vPlan.getTaskConfig();
    Assert.assertEquals(dummyTaskCount, taskPlan.getNumTasks());
    Assert.assertEquals(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB_DEFAULT, taskPlan.getMemoryMb());
    Assert.assertEquals(lrName2, taskPlan.getLocalResource(0).getName());
    Assert.assertEquals(dummyTaskCount, vPlan.getTaskLocationHintCount());
}

Example 25 with DAGPlan

use of org.apache.tez.dag.api.records.DAGProtos.DAGPlan in project tez by apache.

the class TestDAGVerify method testDAGAccessControls.

@Test(timeout = 5000)
public void testDAGAccessControls() {
    DAG dag = DAG.create("testDag");
    ProcessorDescriptor pd1 = ProcessorDescriptor.create("processor1").setUserPayload(UserPayload.create(ByteBuffer.wrap("processor1Bytes".getBytes())));
    Vertex v1 = Vertex.create("v1", pd1, 10, Resource.newInstance(1024, 1));
    dag.addVertex(v1);
    DAGAccessControls dagAccessControls = new DAGAccessControls();
    dagAccessControls.setUsersWithViewACLs(Arrays.asList("u1")).setUsersWithModifyACLs(Arrays.asList("*")).setGroupsWithViewACLs(Arrays.asList("g1")).setGroupsWithModifyACLs(Arrays.asList("g2"));
    dag.setAccessControls(dagAccessControls);
    Configuration conf = new Configuration(false);
    DAGPlan dagPlan = dag.createDag(conf, null, null, null, true);
    Assert.assertNull(conf.get(TezConstants.TEZ_DAG_VIEW_ACLS));
    Assert.assertNull(conf.get(TezConstants.TEZ_DAG_MODIFY_ACLS));
    ACLInfo aclInfo = dagPlan.getAclInfo();
    Assert.assertEquals(Collections.singletonList("u1"), aclInfo.getUsersWithViewAccessList());
    Assert.assertEquals(Collections.singletonList("g1"), aclInfo.getGroupsWithViewAccessList());
    Assert.assertEquals(Collections.singletonList("*"), aclInfo.getUsersWithModifyAccessList());
    Assert.assertEquals(Collections.singletonList("g2"), aclInfo.getGroupsWithModifyAccessList());
}