Examples with NamedEntityDescriptor org.apache.tez.dag.api.NamedEntityDescriptor used on opensource projects

Example 1 with NamedEntityDescriptor

use of org.apache.tez.dag.api.NamedEntityDescriptor in project tez by apache.

the class TestContainerLauncherManager method testReportFailureFromContainerLauncher.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testReportFailureFromContainerLauncher() throws ServicePluginException, TezException {
    final String dagName = DAG_NAME;
    final int dagIndex = DAG_INDEX;
    TezDAGID dagId = TezDAGID.getInstance(ApplicationId.newInstance(0, 0), dagIndex);
    DAG dag = mock(DAG.class);
    doReturn(dagName).when(dag).getName();
    doReturn(dagId).when(dag).getID();
    EventHandler eventHandler = mock(EventHandler.class);
    AppContext appContext = mock(AppContext.class);
    doReturn(eventHandler).when(appContext).getEventHandler();
    doReturn(dag).when(appContext).getCurrentDAG();
    doReturn("testlauncher").when(appContext).getContainerLauncherName(0);
    NamedEntityDescriptor<TaskCommunicatorDescriptor> taskCommDescriptor = new NamedEntityDescriptor<>("testlauncher", ContainerLauncherForTest.class.getName());
    List<NamedEntityDescriptor> list = new LinkedList<>();
    list.add(taskCommDescriptor);
    ContainerLauncherManager containerLauncherManager = new ContainerLauncherManager(appContext, mock(TaskCommunicatorManagerInterface.class), "", list, false);
    try {
        ContainerLaunchContext clc1 = mock(ContainerLaunchContext.class);
        Container container1 = mock(Container.class);
        ContainerLauncherLaunchRequestEvent launchRequestEvent = new ContainerLauncherLaunchRequestEvent(clc1, container1, 0, 0, 0);
        containerLauncherManager.handle(launchRequestEvent);
        ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        Event rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGAppMasterEventUserServiceFatalError);
        DAGAppMasterEventUserServiceFatalError event = (DAGAppMasterEventUserServiceFatalError) rawEvent;
        assertEquals(DAGAppMasterEventType.CONTAINER_LAUNCHER_SERVICE_FATAL_ERROR, event.getType());
        assertTrue(event.getDiagnosticInfo().contains("ReportedFatalError"));
        assertTrue(event.getDiagnosticInfo().contains(ServicePluginErrorDefaults.INCONSISTENT_STATE.name()));
        assertTrue(event.getDiagnosticInfo().contains("[0:testlauncher]"));
        reset(eventHandler);
        // stop container
        ContainerId containerId2 = mock(ContainerId.class);
        NodeId nodeId2 = mock(NodeId.class);
        ContainerLauncherStopRequestEvent stopRequestEvent = new ContainerLauncherStopRequestEvent(containerId2, nodeId2, null, 0, 0, 0);
        argumentCaptor = ArgumentCaptor.forClass(Event.class);
        containerLauncherManager.handle(stopRequestEvent);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGEventTerminateDag);
        DAGEventTerminateDag killEvent = (DAGEventTerminateDag) rawEvent;
        assertTrue(killEvent.getDiagnosticInfo().contains("ReportError"));
        assertTrue(killEvent.getDiagnosticInfo().contains(ServicePluginErrorDefaults.SERVICE_UNAVAILABLE.name()));
        assertTrue(killEvent.getDiagnosticInfo().contains("[0:testlauncher]"));
    } finally {
        containerLauncherManager.stop();
    }
}

Example 2 with NamedEntityDescriptor

use of org.apache.tez.dag.api.NamedEntityDescriptor in project tez by apache.

the class TestContainerLauncherManager method testEventRouting.

@Test(timeout = 5000)
public void testEventRouting() throws Exception {
    Configuration conf = new Configuration(false);
    UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
    AppContext appContext = mock(AppContext.class);
    TaskCommunicatorManagerInterface tal = mock(TaskCommunicatorManagerInterface.class);
    String customLauncherName = "customLauncher";
    List<NamedEntityDescriptor> launcherDescriptors = new LinkedList<>();
    ByteBuffer bb = ByteBuffer.allocate(4);
    bb.putInt(0, 3);
    UserPayload customPayload = UserPayload.create(bb);
    launcherDescriptors.add(new NamedEntityDescriptor(customLauncherName, FakeContainerLauncher.class.getName()).setUserPayload(customPayload));
    launcherDescriptors.add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null).setUserPayload(userPayload));
    ContainerLaucherRouterForMultipleLauncherTest clr = new ContainerLaucherRouterForMultipleLauncherTest(appContext, tal, null, launcherDescriptors, true);
    try {
        clr.init(conf);
        clr.start();
        assertEquals(2, clr.getNumContainerLaunchers());
        assertTrue(clr.getYarnContainerLauncherCreated());
        assertFalse(clr.getUberContainerLauncherCreated());
        assertEquals(customLauncherName, clr.getContainerLauncherName(0));
        assertEquals(TezConstants.getTezYarnServicePluginName(), clr.getContainerLauncherName(1));
        verify(clr.getTestContainerLauncher(0)).initialize();
        verify(clr.getTestContainerLauncher(0)).start();
        verify(clr.getTestContainerLauncher(1)).initialize();
        verify(clr.getTestContainerLauncher(1)).start();
        ContainerLaunchContext clc1 = mock(ContainerLaunchContext.class);
        Container container1 = mock(Container.class);
        ContainerLaunchContext clc2 = mock(ContainerLaunchContext.class);
        Container container2 = mock(Container.class);
        ContainerLauncherLaunchRequestEvent launchRequestEvent1 = new ContainerLauncherLaunchRequestEvent(clc1, container1, 0, 0, 0);
        ContainerLauncherLaunchRequestEvent launchRequestEvent2 = new ContainerLauncherLaunchRequestEvent(clc2, container2, 1, 0, 0);
        clr.handle(launchRequestEvent1);
        ArgumentCaptor<ContainerLaunchRequest> captor = ArgumentCaptor.forClass(ContainerLaunchRequest.class);
        verify(clr.getTestContainerLauncher(0)).launchContainer(captor.capture());
        assertEquals(1, captor.getAllValues().size());
        ContainerLaunchRequest launchRequest1 = captor.getValue();
        assertEquals(clc1, launchRequest1.getContainerLaunchContext());
        clr.handle(launchRequestEvent2);
        captor = ArgumentCaptor.forClass(ContainerLaunchRequest.class);
        verify(clr.getTestContainerLauncher(1)).launchContainer(captor.capture());
        assertEquals(1, captor.getAllValues().size());
        ContainerLaunchRequest launchRequest2 = captor.getValue();
        assertEquals(clc2, launchRequest2.getContainerLaunchContext());
    } finally {
        clr.stop();
        verify(clr.getTestContainerLauncher(0)).shutdown();
        verify(clr.getTestContainerLauncher(1)).shutdown();
    }
}

Example 3 with NamedEntityDescriptor

use of org.apache.tez.dag.api.NamedEntityDescriptor in project tez by apache.

the class TestTaskCommunicatorManager method testMultipleTaskComms.

@Test(timeout = 5000)
public void testMultipleTaskComms() throws IOException, TezException {
    AppContext appContext = mock(AppContext.class);
    TaskHeartbeatHandler thh = mock(TaskHeartbeatHandler.class);
    ContainerHeartbeatHandler chh = mock(ContainerHeartbeatHandler.class);
    Configuration conf = new Configuration(false);
    conf.set("testkey", "testvalue");
    UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(conf);
    String customTaskCommName = "customTaskComm";
    List<NamedEntityDescriptor> taskCommDescriptors = new LinkedList<>();
    ByteBuffer bb = ByteBuffer.allocate(4);
    bb.putInt(0, 3);
    UserPayload customPayload = UserPayload.create(bb);
    taskCommDescriptors.add(new NamedEntityDescriptor(customTaskCommName, FakeTaskComm.class.getName()).setUserPayload(customPayload));
    taskCommDescriptors.add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null).setUserPayload(defaultPayload));
    TaskCommManagerForMultipleCommTest tcm = new TaskCommManagerForMultipleCommTest(appContext, thh, chh, taskCommDescriptors);
    try {
        tcm.init(new Configuration(false));
        tcm.start();
        assertEquals(2, tcm.getNumTaskComms());
        assertTrue(tcm.getYarnTaskCommCreated());
        assertFalse(tcm.getUberTaskCommCreated());
        assertEquals(customTaskCommName, tcm.getTaskCommName(0));
        assertEquals(bb, tcm.getTaskCommContext(0).getInitialUserPayload().getPayload());
        assertEquals(TezConstants.getTezYarnServicePluginName(), tcm.getTaskCommName(1));
        Configuration confParsed = TezUtils.createConfFromUserPayload(tcm.getTaskCommContext(1).getInitialUserPayload());
        assertEquals("testvalue", confParsed.get("testkey"));
    } finally {
        tcm.stop();
    }
}

Example 4 with NamedEntityDescriptor

use of org.apache.tez.dag.api.NamedEntityDescriptor in project tez by apache.

the class TestTaskSchedulerManager method testTaskSchedulerRouting.

@Test(timeout = 5000)
public void testTaskSchedulerRouting() throws Exception {
    Configuration conf = new Configuration(false);
    UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(conf);
    String customSchedulerName = "fakeScheduler";
    List<NamedEntityDescriptor> taskSchedulers = new LinkedList<>();
    ByteBuffer bb = ByteBuffer.allocate(4);
    bb.putInt(0, 3);
    UserPayload userPayload = UserPayload.create(bb);
    taskSchedulers.add(new NamedEntityDescriptor(customSchedulerName, FakeTaskScheduler.class.getName()).setUserPayload(userPayload));
    taskSchedulers.add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null).setUserPayload(defaultPayload));
    TSEHForMultipleSchedulersTest tseh = new TSEHForMultipleSchedulersTest(mockAppContext, mockClientService, mockEventHandler, mockSigMatcher, mockWebUIService, taskSchedulers, false);
    tseh.init(conf);
    tseh.start();
    // Verify that the YARN task scheduler is installed by default
    assertTrue(tseh.getYarnSchedulerCreated());
    assertFalse(tseh.getUberSchedulerCreated());
    assertEquals(2, tseh.getNumCreateInvocations());
    // Verify the order of the schedulers
    assertEquals(customSchedulerName, tseh.getTaskSchedulerName(0));
    assertEquals(TezConstants.getTezYarnServicePluginName(), tseh.getTaskSchedulerName(1));
    verify(tseh.getTestTaskScheduler(0)).initialize();
    verify(tseh.getTestTaskScheduler(0)).start();
    ApplicationId appId = ApplicationId.newInstance(1000, 1);
    TezDAGID dagId = TezDAGID.getInstance(appId, 1);
    TezVertexID vertexID = TezVertexID.getInstance(dagId, 1);
    TezTaskID taskId1 = TezTaskID.getInstance(vertexID, 1);
    TezTaskAttemptID attemptId11 = TezTaskAttemptID.getInstance(taskId1, 1);
    TezTaskID taskId2 = TezTaskID.getInstance(vertexID, 2);
    TezTaskAttemptID attemptId21 = TezTaskAttemptID.getInstance(taskId2, 1);
    Resource resource = Resource.newInstance(1024, 1);
    TaskAttempt mockTaskAttempt1 = mock(TaskAttempt.class);
    TaskAttempt mockTaskAttempt2 = mock(TaskAttempt.class);
    AMSchedulerEventTALaunchRequest launchRequest1 = new AMSchedulerEventTALaunchRequest(attemptId11, resource, mock(TaskSpec.class), mockTaskAttempt1, mock(TaskLocationHint.class), 1, mock(ContainerContext.class), 0, 0, 0);
    tseh.handle(launchRequest1);
    verify(tseh.getTestTaskScheduler(0)).allocateTask(eq(mockTaskAttempt1), eq(resource), any(String[].class), any(String[].class), any(Priority.class), any(Object.class), eq(launchRequest1));
    AMSchedulerEventTALaunchRequest launchRequest2 = new AMSchedulerEventTALaunchRequest(attemptId21, resource, mock(TaskSpec.class), mockTaskAttempt2, mock(TaskLocationHint.class), 1, mock(ContainerContext.class), 1, 0, 0);
    tseh.handle(launchRequest2);
    verify(tseh.getTestTaskScheduler(1)).allocateTask(eq(mockTaskAttempt2), eq(resource), any(String[].class), any(String[].class), any(Priority.class), any(Object.class), eq(launchRequest2));
}

Example 5 with NamedEntityDescriptor

use of org.apache.tez.dag.api.NamedEntityDescriptor in project tez by apache.

the class TestTaskSchedulerManager method testCustomTaskSchedulerSetup.

// Verified via statics
@Test(timeout = 5000)
public void testCustomTaskSchedulerSetup() throws IOException {
    Configuration conf = new Configuration(false);
    conf.set("testkey", "testval");
    UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(conf);
    String customSchedulerName = "fakeScheduler";
    List<NamedEntityDescriptor> taskSchedulers = new LinkedList<>();
    ByteBuffer bb = ByteBuffer.allocate(4);
    bb.putInt(0, 3);
    UserPayload userPayload = UserPayload.create(bb);
    taskSchedulers.add(new NamedEntityDescriptor(customSchedulerName, FakeTaskScheduler.class.getName()).setUserPayload(userPayload));
    taskSchedulers.add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null).setUserPayload(defaultPayload));
    TSEHForMultipleSchedulersTest tseh = new TSEHForMultipleSchedulersTest(mockAppContext, mockClientService, mockEventHandler, mockSigMatcher, mockWebUIService, taskSchedulers, false);
    tseh.init(conf);
    tseh.start();
    // Verify that the YARN task scheduler is installed by default
    assertTrue(tseh.getYarnSchedulerCreated());
    assertFalse(tseh.getUberSchedulerCreated());
    assertEquals(2, tseh.getNumCreateInvocations());
    // Verify the order of the schedulers
    assertEquals(customSchedulerName, tseh.getTaskSchedulerName(0));
    assertEquals(TezConstants.getTezYarnServicePluginName(), tseh.getTaskSchedulerName(1));
    // Verify the payload setup for the custom task scheduler
    assertNotNull(tseh.getTaskSchedulerContext(0));
    assertEquals(bb, tseh.getTaskSchedulerContext(0).getInitialUserPayload().getPayload());
    // Verify the payload on the yarn scheduler
    assertNotNull(tseh.getTaskSchedulerContext(1));
    Configuration parsed = TezUtils.createConfFromUserPayload(tseh.getTaskSchedulerContext(1).getInitialUserPayload());
    assertEquals("testval", parsed.get("testkey"));
}