Examples with NamedEntityDescriptor org.apache.tez.dag.api.NamedEntityDescriptor used on opensource projects

Example 11 with NamedEntityDescriptor

use of org.apache.tez.dag.api.NamedEntityDescriptor in project tez by apache.

the class TestDAGAppMaster method testParseAllPluginsNoneSpecified.

@Test(timeout = 5000)
public void testParseAllPluginsNoneSpecified() throws IOException {
    Configuration conf = new Configuration(false);
    conf.set(TEST_KEY, TEST_VAL);
    UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(conf);
    List<NamedEntityDescriptor> tsDescriptors;
    BiMap<String, Integer> tsMap;
    List<NamedEntityDescriptor> clDescriptors;
    BiMap<String, Integer> clMap;
    List<NamedEntityDescriptor> tcDescriptors;
    BiMap<String, Integer> tcMap;
    // No plugins. Non local
    tsDescriptors = Lists.newLinkedList();
    tsMap = HashBiMap.create();
    clDescriptors = Lists.newLinkedList();
    clMap = HashBiMap.create();
    tcDescriptors = Lists.newLinkedList();
    tcMap = HashBiMap.create();
    DAGAppMaster.parseAllPlugins(tsDescriptors, tsMap, clDescriptors, clMap, tcDescriptors, tcMap, null, false, defaultPayload);
    verifyDescAndMap(tsDescriptors, tsMap, 1, true, TezConstants.getTezYarnServicePluginName());
    verifyDescAndMap(clDescriptors, clMap, 1, true, TezConstants.getTezYarnServicePluginName());
    verifyDescAndMap(tcDescriptors, tcMap, 1, true, TezConstants.getTezYarnServicePluginName());
    // No plugins. Local
    tsDescriptors = Lists.newLinkedList();
    tsMap = HashBiMap.create();
    clDescriptors = Lists.newLinkedList();
    clMap = HashBiMap.create();
    tcDescriptors = Lists.newLinkedList();
    tcMap = HashBiMap.create();
    DAGAppMaster.parseAllPlugins(tsDescriptors, tsMap, clDescriptors, clMap, tcDescriptors, tcMap, null, true, defaultPayload);
    verifyDescAndMap(tsDescriptors, tsMap, 1, true, TezConstants.getTezUberServicePluginName());
    verifyDescAndMap(clDescriptors, clMap, 1, true, TezConstants.getTezUberServicePluginName());
    verifyDescAndMap(tcDescriptors, tcMap, 1, true, TezConstants.getTezUberServicePluginName());
}

Example 12 with NamedEntityDescriptor

use of org.apache.tez.dag.api.NamedEntityDescriptor in project tez by apache.

the class TestDAGAppMaster method testParseAllPluginsCustomAndYarnSpecified.

@Test(timeout = 5000)
public void testParseAllPluginsCustomAndYarnSpecified() throws IOException {
    Configuration conf = new Configuration(false);
    conf.set(TEST_KEY, TEST_VAL);
    UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(conf);
    TezUserPayloadProto payloadProto = TezUserPayloadProto.newBuilder().setUserPayload(ByteString.copyFrom(defaultPayload.getPayload())).build();
    AMPluginDescriptorProto proto = createAmPluginDescriptor(true, false, true, payloadProto);
    List<NamedEntityDescriptor> tsDescriptors;
    BiMap<String, Integer> tsMap;
    List<NamedEntityDescriptor> clDescriptors;
    BiMap<String, Integer> clMap;
    List<NamedEntityDescriptor> tcDescriptors;
    BiMap<String, Integer> tcMap;
    // Only plugin, Yarn.
    tsDescriptors = Lists.newLinkedList();
    tsMap = HashBiMap.create();
    clDescriptors = Lists.newLinkedList();
    clMap = HashBiMap.create();
    tcDescriptors = Lists.newLinkedList();
    tcMap = HashBiMap.create();
    DAGAppMaster.parseAllPlugins(tsDescriptors, tsMap, clDescriptors, clMap, tcDescriptors, tcMap, proto, false, defaultPayload);
    verifyDescAndMap(tsDescriptors, tsMap, 2, true, TezConstants.getTezYarnServicePluginName(), TS_NAME);
    verifyDescAndMap(clDescriptors, clMap, 2, true, TezConstants.getTezYarnServicePluginName(), CL_NAME);
    verifyDescAndMap(tcDescriptors, tcMap, 2, true, TezConstants.getTezYarnServicePluginName(), TC_NAME);
    assertNull(tsDescriptors.get(0).getClassName());
    assertNull(clDescriptors.get(0).getClassName());
    assertNull(tcDescriptors.get(0).getClassName());
    assertEquals(TS_NAME + CLASS_SUFFIX, tsDescriptors.get(1).getClassName());
    assertEquals(CL_NAME + CLASS_SUFFIX, clDescriptors.get(1).getClassName());
    assertEquals(TC_NAME + CLASS_SUFFIX, tcDescriptors.get(1).getClassName());
}

Example 13 with NamedEntityDescriptor

use of org.apache.tez.dag.api.NamedEntityDescriptor in project tez by apache.

the class TestTaskCommunicatorManager method testCustomTaskCommSpecified.

@Test(timeout = 5000)
public void testCustomTaskCommSpecified() throws IOException, TezException {
    AppContext appContext = mock(AppContext.class);
    TaskHeartbeatHandler thh = mock(TaskHeartbeatHandler.class);
    ContainerHeartbeatHandler chh = mock(ContainerHeartbeatHandler.class);
    String customTaskCommName = "customTaskComm";
    List<NamedEntityDescriptor> taskCommDescriptors = new LinkedList<>();
    ByteBuffer bb = ByteBuffer.allocate(4);
    bb.putInt(0, 3);
    UserPayload customPayload = UserPayload.create(bb);
    taskCommDescriptors.add(new NamedEntityDescriptor(customTaskCommName, FakeTaskComm.class.getName()).setUserPayload(customPayload));
    TaskCommManagerForMultipleCommTest tcm = new TaskCommManagerForMultipleCommTest(appContext, thh, chh, taskCommDescriptors);
    try {
        tcm.init(new Configuration(false));
        tcm.start();
        assertEquals(1, tcm.getNumTaskComms());
        assertFalse(tcm.getYarnTaskCommCreated());
        assertFalse(tcm.getUberTaskCommCreated());
        assertEquals(customTaskCommName, tcm.getTaskCommName(0));
        assertEquals(bb, tcm.getTaskCommContext(0).getInitialUserPayload().getPayload());
    } finally {
        tcm.stop();
    }
}

Example 14 with NamedEntityDescriptor

use of org.apache.tez.dag.api.NamedEntityDescriptor in project tez by apache.

the class TestTaskCommunicatorManager method testReportFailureFromTaskCommunicator.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testReportFailureFromTaskCommunicator() throws TezException {
    String dagName = DAG_NAME;
    EventHandler eventHandler = mock(EventHandler.class);
    AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
    doReturn("testTaskCommunicator").when(appContext).getTaskCommunicatorName(0);
    doReturn(eventHandler).when(appContext).getEventHandler();
    DAG dag = mock(DAG.class);
    TezDAGID dagId = TezDAGID.getInstance(ApplicationId.newInstance(1, 0), DAG_INDEX);
    doReturn(dagName).when(dag).getName();
    doReturn(dagId).when(dag).getID();
    doReturn(dag).when(appContext).getCurrentDAG();
    NamedEntityDescriptor<TaskCommunicatorDescriptor> namedEntityDescriptor = new NamedEntityDescriptor<>("testTaskCommunicator", TaskCommForFailureTest.class.getName());
    List<NamedEntityDescriptor> list = new LinkedList<>();
    list.add(namedEntityDescriptor);
    TaskCommunicatorManager taskCommManager = new TaskCommunicatorManager(appContext, mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class), list);
    try {
        taskCommManager.init(new Configuration());
        taskCommManager.start();
        taskCommManager.registerRunningContainer(mock(ContainerId.class), 0);
        ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        Event rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGEventTerminateDag);
        DAGEventTerminateDag killEvent = (DAGEventTerminateDag) rawEvent;
        assertTrue(killEvent.getDiagnosticInfo().contains("ReportError"));
        assertTrue(killEvent.getDiagnosticInfo().contains(ServicePluginErrorDefaults.SERVICE_UNAVAILABLE.name()));
        assertTrue(killEvent.getDiagnosticInfo().contains("[0:testTaskCommunicator]"));
        reset(eventHandler);
        taskCommManager.dagComplete(dag);
        argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGAppMasterEventUserServiceFatalError);
        DAGAppMasterEventUserServiceFatalError event = (DAGAppMasterEventUserServiceFatalError) rawEvent;
        assertEquals(DAGAppMasterEventType.TASK_COMMUNICATOR_SERVICE_FATAL_ERROR, event.getType());
        assertTrue(event.getDiagnosticInfo().contains("ReportedFatalError"));
        assertTrue(event.getDiagnosticInfo().contains(ServicePluginErrorDefaults.INCONSISTENT_STATE.name()));
        assertTrue(event.getDiagnosticInfo().contains("[0:testTaskCommunicator]"));
    } finally {
        taskCommManager.stop();
    }
}

Example 15 with NamedEntityDescriptor

use of org.apache.tez.dag.api.NamedEntityDescriptor in project tez by apache.

the class TestTaskCommunicatorManager method testEventRouting.

@Test(timeout = 5000)
public void testEventRouting() throws Exception {
    AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
    NodeId nodeId = NodeId.newInstance("host1", 3131);
    when(appContext.getAllContainers().get(any(ContainerId.class)).getContainer().getNodeId()).thenReturn(nodeId);
    TaskHeartbeatHandler thh = mock(TaskHeartbeatHandler.class);
    ContainerHeartbeatHandler chh = mock(ContainerHeartbeatHandler.class);
    Configuration conf = new Configuration(false);
    conf.set("testkey", "testvalue");
    UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(conf);
    String customTaskCommName = "customTaskComm";
    List<NamedEntityDescriptor> taskCommDescriptors = new LinkedList<>();
    ByteBuffer bb = ByteBuffer.allocate(4);
    bb.putInt(0, 3);
    UserPayload customPayload = UserPayload.create(bb);
    taskCommDescriptors.add(new NamedEntityDescriptor(customTaskCommName, FakeTaskComm.class.getName()).setUserPayload(customPayload));
    taskCommDescriptors.add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null).setUserPayload(defaultPayload));
    TaskCommManagerForMultipleCommTest tcm = new TaskCommManagerForMultipleCommTest(appContext, thh, chh, taskCommDescriptors);
    try {
        tcm.init(new Configuration(false));
        tcm.start();
        assertEquals(2, tcm.getNumTaskComms());
        assertTrue(tcm.getYarnTaskCommCreated());
        assertFalse(tcm.getUberTaskCommCreated());
        verify(tcm.getTestTaskComm(0)).initialize();
        verify(tcm.getTestTaskComm(0)).start();
        verify(tcm.getTestTaskComm(1)).initialize();
        verify(tcm.getTestTaskComm(1)).start();
        ContainerId containerId1 = mock(ContainerId.class);
        tcm.registerRunningContainer(containerId1, 0);
        verify(tcm.getTestTaskComm(0)).registerRunningContainer(eq(containerId1), eq("host1"), eq(3131));
        ContainerId containerId2 = mock(ContainerId.class);
        tcm.registerRunningContainer(containerId2, 1);
        verify(tcm.getTestTaskComm(1)).registerRunningContainer(eq(containerId2), eq("host1"), eq(3131));
    } finally {
        tcm.stop();
        verify(tcm.getTaskCommunicator(0).getTaskCommunicator()).shutdown();
        verify(tcm.getTaskCommunicator(1).getTaskCommunicator()).shutdown();
    }
}