Examples with TaskCommunicatorManagerInterface org.apache.tez.dag.app.TaskCommunicatorManagerInterface used on opensource projects

Example 21 with TaskCommunicatorManagerInterface

use of org.apache.tez.dag.app.TaskCommunicatorManagerInterface in project tez by apache.

the class TestTaskAttempt method testKilledInNew.

@SuppressWarnings("deprecation")
@Test(timeout = 5000)
public void testKilledInNew() throws ServicePluginException {
    ApplicationId appId = ApplicationId.newInstance(1, 2);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 0);
    TezDAGID dagID = TezDAGID.getInstance(appId, 1);
    TezVertexID vertexID = TezVertexID.getInstance(dagID, 1);
    TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
    MockEventHandler eventHandler = spy(new MockEventHandler());
    TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();
    Configuration taskConf = new Configuration();
    taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
    taskConf.setBoolean("fs.file.impl.disable.cache", true);
    locationHint = TaskLocationHint.createTaskLocationHint(new HashSet<String>(Arrays.asList(new String[] { "127.0.0.1" })), null);
    Resource resource = Resource.newInstance(1024, 1);
    NodeId nid = NodeId.newInstance("127.0.0.1", 0);
    ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
    Container container = mock(Container.class);
    when(container.getId()).thenReturn(contId);
    when(container.getNodeId()).thenReturn(nid);
    when(container.getNodeHttpAddress()).thenReturn("localhost:0");
    AMContainerMap containers = new AMContainerMap(mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class), new ContainerContextMatcher(), appCtx);
    containers.addContainerIfNew(container, 0, 0, 0);
    doReturn(new ClusterInfo()).when(appCtx).getClusterInfo();
    doReturn(containers).when(appCtx).getAllContainers();
    TaskHeartbeatHandler mockHeartbeatHandler = mock(TaskHeartbeatHandler.class);
    MockTaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler, taListener, taskConf, new SystemClock(), mockHeartbeatHandler, appCtx, false, resource, createFakeContainerContext(), true);
    Assert.assertEquals(TaskAttemptStateInternal.NEW, taImpl.getInternalState());
    taImpl.handle(new TaskAttemptEventKillRequest(taImpl.getID(), "kill it", TaskAttemptTerminationCause.TERMINATED_BY_CLIENT));
    Assert.assertEquals(TaskAttemptStateInternal.KILLED, taImpl.getInternalState());
    Assert.assertEquals(0, taImpl.taskAttemptStartedEventLogged);
    Assert.assertEquals(1, taImpl.taskAttemptFinishedEventLogged);
}

Example 22 with TaskCommunicatorManagerInterface

use of org.apache.tez.dag.app.TaskCommunicatorManagerInterface in project tez by apache.

the class TestTaskAttempt method testNodeFailedLeafVertex.

@Test(timeout = 5000)
public // Ensure node failure on Successful Leaf tasks do not cause them to be marked as KILLED
void testNodeFailedLeafVertex() throws Exception {
    ApplicationId appId = ApplicationId.newInstance(1, 2);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 0);
    TezDAGID dagID = TezDAGID.getInstance(appId, 1);
    TezVertexID vertexID = TezVertexID.getInstance(dagID, 1);
    TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
    MockEventHandler eventHandler = spy(new MockEventHandler());
    TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();
    Configuration taskConf = new Configuration();
    taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
    taskConf.setBoolean("fs.file.impl.disable.cache", true);
    locationHint = TaskLocationHint.createTaskLocationHint(new HashSet<String>(Arrays.asList(new String[] { "127.0.0.1" })), null);
    Resource resource = Resource.newInstance(1024, 1);
    NodeId nid = NodeId.newInstance("127.0.0.1", 0);
    @SuppressWarnings("deprecation") ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
    Container container = mock(Container.class);
    when(container.getId()).thenReturn(contId);
    when(container.getNodeId()).thenReturn(nid);
    when(container.getNodeHttpAddress()).thenReturn("localhost:0");
    AMContainerMap containers = new AMContainerMap(mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class), new ContainerContextMatcher(), appCtx);
    containers.addContainerIfNew(container, 0, 0, 0);
    doReturn(new ClusterInfo()).when(appCtx).getClusterInfo();
    doReturn(containers).when(appCtx).getAllContainers();
    TaskHeartbeatHandler mockHeartbeatHandler = mock(TaskHeartbeatHandler.class);
    TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler, taListener, taskConf, new SystemClock(), mockHeartbeatHandler, appCtx, false, resource, createFakeContainerContext(), true);
    TezTaskAttemptID taskAttemptID = taImpl.getID();
    ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
    taImpl.handle(new TaskAttemptEventSchedule(taskAttemptID, 0, 0));
    taImpl.handle(new TaskAttemptEventSubmitted(taskAttemptID, contId));
    taImpl.handle(new TaskAttemptEventStartedRemotely(taskAttemptID));
    assertEquals("Task attempt is not in the RUNNING state", TaskAttemptState.RUNNING, taImpl.getState());
    verify(mockHeartbeatHandler).register(taskAttemptID);
    int expectedEventsAtRunning = 5;
    verify(eventHandler, times(expectedEventsAtRunning)).handle(arg.capture());
    taImpl.handle(new TaskAttemptEvent(taskAttemptID, TaskAttemptEventType.TA_DONE));
    assertEquals("Task attempt is not in the  SUCCEEDED state", TaskAttemptState.SUCCEEDED, taImpl.getState());
    verify(mockHeartbeatHandler).unregister(taskAttemptID);
    assertEquals(0, taImpl.getDiagnostics().size());
    int expectedEvenstAfterTerminating = expectedEventsAtRunning + 3;
    arg = ArgumentCaptor.forClass(Event.class);
    verify(eventHandler, times(expectedEvenstAfterTerminating)).handle(arg.capture());
    verifyEventType(arg.getAllValues().subList(expectedEventsAtRunning, expectedEvenstAfterTerminating), TaskEventTASucceeded.class, 1);
    verifyEventType(arg.getAllValues().subList(expectedEventsAtRunning, expectedEvenstAfterTerminating), AMSchedulerEventTAEnded.class, 1);
    verifyEventType(arg.getAllValues().subList(expectedEventsAtRunning, expectedEvenstAfterTerminating), DAGEventCounterUpdate.class, 1);
    // Send out a Node Failure.
    taImpl.handle(new TaskAttemptEventNodeFailed(taskAttemptID, "NodeDecomissioned", TaskAttemptTerminationCause.NODE_FAILED));
    // Verify no additional events
    int expectedEventsNodeFailure = expectedEvenstAfterTerminating + 0;
    arg = ArgumentCaptor.forClass(Event.class);
    verify(eventHandler, times(expectedEventsNodeFailure)).handle(arg.capture());
    // Verify still in SUCCEEDED state
    assertEquals("Task attempt is not in the  SUCCEEDED state", TaskAttemptState.SUCCEEDED, taImpl.getState());
    // verify unregister is not invoked again
    verify(mockHeartbeatHandler, times(1)).unregister(taskAttemptID);
    // error cause remains as default value
    assertEquals(TaskAttemptTerminationCause.UNKNOWN_ERROR, taImpl.getTerminationCause());
}

Example 23 with TaskCommunicatorManagerInterface

use of org.apache.tez.dag.app.TaskCommunicatorManagerInterface in project tez by apache.

the class TestContainerLauncherManager method testCustomLauncherSpecified.

@Test(timeout = 5000)
public void testCustomLauncherSpecified() throws IOException, TezException {
    Configuration conf = new Configuration(false);
    AppContext appContext = mock(AppContext.class);
    TaskCommunicatorManagerInterface tal = mock(TaskCommunicatorManagerInterface.class);
    String customLauncherName = "customLauncher";
    List<NamedEntityDescriptor> launcherDescriptors = new LinkedList<>();
    ByteBuffer bb = ByteBuffer.allocate(4);
    bb.putInt(0, 3);
    UserPayload customPayload = UserPayload.create(bb);
    launcherDescriptors.add(new NamedEntityDescriptor(customLauncherName, FakeContainerLauncher.class.getName()).setUserPayload(customPayload));
    ContainerLaucherRouterForMultipleLauncherTest clr = new ContainerLaucherRouterForMultipleLauncherTest(appContext, tal, null, launcherDescriptors, true);
    try {
        clr.init(conf);
        clr.start();
        assertEquals(1, clr.getNumContainerLaunchers());
        assertFalse(clr.getYarnContainerLauncherCreated());
        assertFalse(clr.getUberContainerLauncherCreated());
        assertEquals(customLauncherName, clr.getContainerLauncherName(0));
        assertEquals(bb, clr.getContainerLauncherContext(0).getInitialUserPayload().getPayload());
    } finally {
        clr.stop();
    }
}

Example 24 with TaskCommunicatorManagerInterface

use of org.apache.tez.dag.app.TaskCommunicatorManagerInterface in project tez by apache.

the class TestContainerLauncherManager method testMultipleContainerLaunchers.

@Test(timeout = 5000)
public void testMultipleContainerLaunchers() throws IOException, TezException {
    Configuration conf = new Configuration(false);
    conf.set("testkey", "testvalue");
    UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
    AppContext appContext = mock(AppContext.class);
    TaskCommunicatorManagerInterface tal = mock(TaskCommunicatorManagerInterface.class);
    String customLauncherName = "customLauncher";
    List<NamedEntityDescriptor> launcherDescriptors = new LinkedList<>();
    ByteBuffer bb = ByteBuffer.allocate(4);
    bb.putInt(0, 3);
    UserPayload customPayload = UserPayload.create(bb);
    launcherDescriptors.add(new NamedEntityDescriptor(customLauncherName, FakeContainerLauncher.class.getName()).setUserPayload(customPayload));
    launcherDescriptors.add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null).setUserPayload(userPayload));
    ContainerLaucherRouterForMultipleLauncherTest clr = new ContainerLaucherRouterForMultipleLauncherTest(appContext, tal, null, launcherDescriptors, true);
    try {
        clr.init(conf);
        clr.start();
        assertEquals(2, clr.getNumContainerLaunchers());
        assertTrue(clr.getYarnContainerLauncherCreated());
        assertFalse(clr.getUberContainerLauncherCreated());
        assertEquals(customLauncherName, clr.getContainerLauncherName(0));
        assertEquals(bb, clr.getContainerLauncherContext(0).getInitialUserPayload().getPayload());
        assertEquals(TezConstants.getTezYarnServicePluginName(), clr.getContainerLauncherName(1));
        Configuration confParsed = TezUtils.createConfFromUserPayload(clr.getContainerLauncherContext(1).getInitialUserPayload());
        assertEquals("testvalue", confParsed.get("testkey"));
    } finally {
        clr.stop();
    }
}