Examples with TaskCommunicatorManagerInterface org.apache.tez.dag.app.TaskCommunicatorManagerInterface used on opensource projects

Example 11 with TaskCommunicatorManagerInterface

use of org.apache.tez.dag.app.TaskCommunicatorManagerInterface in project tez by apache.

the class TestTaskAttempt method testCompletedAtSubmitted.

@Test(timeout = 5000)
public void testCompletedAtSubmitted() throws ServicePluginException {
    ApplicationId appId = ApplicationId.newInstance(1, 2);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 0);
    TezDAGID dagID = TezDAGID.getInstance(appId, 1);
    TezVertexID vertexID = TezVertexID.getInstance(dagID, 1);
    TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
    MockEventHandler eventHandler = spy(new MockEventHandler());
    TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();
    Configuration taskConf = new Configuration();
    taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
    taskConf.setBoolean("fs.file.impl.disable.cache", true);
    locationHint = TaskLocationHint.createTaskLocationHint(new HashSet<String>(Arrays.asList(new String[] { "127.0.0.1" })), null);
    Resource resource = Resource.newInstance(1024, 1);
    NodeId nid = NodeId.newInstance("127.0.0.1", 0);
    @SuppressWarnings("deprecation") ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
    Container container = mock(Container.class);
    when(container.getId()).thenReturn(contId);
    when(container.getNodeId()).thenReturn(nid);
    when(container.getNodeHttpAddress()).thenReturn("localhost:0");
    AMContainerMap containers = new AMContainerMap(mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class), new ContainerContextMatcher(), appCtx);
    containers.addContainerIfNew(container, 0, 0, 0);
    doReturn(new ClusterInfo()).when(appCtx).getClusterInfo();
    doReturn(containers).when(appCtx).getAllContainers();
    TaskHeartbeatHandler mockHeartbeatHandler = mock(TaskHeartbeatHandler.class);
    TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler, taListener, taskConf, new SystemClock(), mockHeartbeatHandler, appCtx, false, resource, createFakeContainerContext(), false);
    TezTaskAttemptID taskAttemptID = taImpl.getID();
    ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
    taImpl.handle(new TaskAttemptEventSchedule(taskAttemptID, 0, 0));
    taImpl.handle(new TaskAttemptEventSubmitted(taskAttemptID, contId));
    assertEquals("Task attempt is not in the RUNNING state", taImpl.getState(), TaskAttemptState.STARTING);
    verify(mockHeartbeatHandler).register(taskAttemptID);
    int expectedEventsAtStarting = 4;
    verify(eventHandler, times(expectedEventsAtStarting)).handle(arg.capture());
    // Ensure status_updates are handled in the submitted state.
    taImpl.handle(new TaskAttemptEventStatusUpdate(taskAttemptID, new TaskStatusUpdateEvent(null, 0.1f, null, false)));
    taImpl.handle(new TaskAttemptEvent(taskAttemptID, TaskAttemptEventType.TA_DONE));
    assertEquals("Task attempt is not in the  SUCCEEDED state", taImpl.getState(), TaskAttemptState.SUCCEEDED);
    verify(mockHeartbeatHandler).unregister(taskAttemptID);
    assertEquals(0, taImpl.getDiagnostics().size());
    int expectedEvenstAfterTerminating = expectedEventsAtStarting + 3;
    arg = ArgumentCaptor.forClass(Event.class);
    verify(eventHandler, times(expectedEvenstAfterTerminating)).handle(arg.capture());
    Event e = verifyEventType(arg.getAllValues().subList(expectedEventsAtStarting, expectedEvenstAfterTerminating), TaskEventTASucceeded.class, 1);
    assertEquals(TaskEventType.T_ATTEMPT_SUCCEEDED, e.getType());
    verifyEventType(arg.getAllValues().subList(expectedEventsAtStarting, expectedEvenstAfterTerminating), AMSchedulerEventTAEnded.class, 1);
    verifyEventType(arg.getAllValues().subList(expectedEventsAtStarting, expectedEvenstAfterTerminating), DAGEventCounterUpdate.class, 1);
}

Example 12 with TaskCommunicatorManagerInterface

use of org.apache.tez.dag.app.TaskCommunicatorManagerInterface in project tez by apache.

the class TestTaskImpl method setup.

@Before
public void setup() {
    conf = new Configuration();
    conf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 4);
    taskCommunicatorManagerInterface = mock(TaskCommunicatorManagerInterface.class);
    taskHeartbeatHandler = mock(TaskHeartbeatHandler.class);
    credentials = new Credentials();
    clock = new SystemClock();
    locationHint = TaskLocationHint.createTaskLocationHint(null, null);
    appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
    dagId = TezDAGID.getInstance(appId, 1);
    vertexId = TezVertexID.getInstance(dagId, 1);
    appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
    when(appContext.getDAGRecoveryData()).thenReturn(null);
    appContext.setDAGRecoveryData(null);
    mockContainerId = mock(ContainerId.class);
    mockContainer = mock(Container.class);
    mockAMContainer = mock(AMContainer.class);
    when(mockAMContainer.getContainer()).thenReturn(mockContainer);
    when(mockContainer.getNodeHttpAddress()).thenReturn("localhost:1234");
    mockNodeId = mock(NodeId.class);
    mockHistoryHandler = mock(HistoryEventHandler.class);
    when(mockContainer.getId()).thenReturn(mockContainerId);
    when(mockContainer.getNodeId()).thenReturn(mockNodeId);
    when(mockAMContainer.getContainer()).thenReturn(mockContainer);
    when(appContext.getAllContainers().get(mockContainerId)).thenReturn(mockAMContainer);
    when(appContext.getHistoryHandler()).thenReturn(mockHistoryHandler);
    taskResource = Resource.newInstance(1024, 1);
    localResources = new HashMap<String, LocalResource>();
    environment = new HashMap<String, String>();
    javaOpts = "";
    leafVertex = false;
    containerContext = new ContainerContext(localResources, credentials, environment, javaOpts);
    Vertex vertex = mock(Vertex.class);
    doReturn(new VertexImpl.VertexConfigImpl(conf)).when(vertex).getVertexConfig();
    eventHandler = new TestEventHandler();
    mockTask = new MockTaskImpl(vertexId, partition, eventHandler, conf, taskCommunicatorManagerInterface, clock, taskHeartbeatHandler, appContext, leafVertex, taskResource, containerContext, vertex);
    mockTaskSpec = mock(TaskSpec.class);
    mockVertex = mock(Vertex.class);
    ServicePluginInfo servicePluginInfo = new ServicePluginInfo().setContainerLauncherName(TezConstants.getTezYarnServicePluginName());
    when(mockVertex.getServicePluginInfo()).thenReturn(servicePluginInfo);
    when(mockVertex.getVertexConfig()).thenReturn(new VertexImpl.VertexConfigImpl(conf));
}

Example 13 with TaskCommunicatorManagerInterface

use of org.apache.tez.dag.app.TaskCommunicatorManagerInterface in project tez by apache.

the class TestContainerLauncherManager method testNoLaunchersSpecified.

@Test(timeout = 5000)
public void testNoLaunchersSpecified() throws IOException, TezException {
    AppContext appContext = mock(AppContext.class);
    TaskCommunicatorManagerInterface tal = mock(TaskCommunicatorManagerInterface.class);
    try {
        new ContainerLaucherRouterForMultipleLauncherTest(appContext, tal, null, null, false);
        fail("Expecting a failure without any launchers being specified");
    } catch (IllegalArgumentException e) {
    }
}

Example 14 with TaskCommunicatorManagerInterface

use of org.apache.tez.dag.app.TaskCommunicatorManagerInterface in project tez by apache.

the class TestContainerLauncherManager method testEventRouting.

@Test(timeout = 5000)
public void testEventRouting() throws Exception {
    Configuration conf = new Configuration(false);
    UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
    AppContext appContext = mock(AppContext.class);
    TaskCommunicatorManagerInterface tal = mock(TaskCommunicatorManagerInterface.class);
    String customLauncherName = "customLauncher";
    List<NamedEntityDescriptor> launcherDescriptors = new LinkedList<>();
    ByteBuffer bb = ByteBuffer.allocate(4);
    bb.putInt(0, 3);
    UserPayload customPayload = UserPayload.create(bb);
    launcherDescriptors.add(new NamedEntityDescriptor(customLauncherName, FakeContainerLauncher.class.getName()).setUserPayload(customPayload));
    launcherDescriptors.add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null).setUserPayload(userPayload));
    ContainerLaucherRouterForMultipleLauncherTest clr = new ContainerLaucherRouterForMultipleLauncherTest(appContext, tal, null, launcherDescriptors, true);
    try {
        clr.init(conf);
        clr.start();
        assertEquals(2, clr.getNumContainerLaunchers());
        assertTrue(clr.getYarnContainerLauncherCreated());
        assertFalse(clr.getUberContainerLauncherCreated());
        assertEquals(customLauncherName, clr.getContainerLauncherName(0));
        assertEquals(TezConstants.getTezYarnServicePluginName(), clr.getContainerLauncherName(1));
        verify(clr.getTestContainerLauncher(0)).initialize();
        verify(clr.getTestContainerLauncher(0)).start();
        verify(clr.getTestContainerLauncher(1)).initialize();
        verify(clr.getTestContainerLauncher(1)).start();
        ContainerLaunchContext clc1 = mock(ContainerLaunchContext.class);
        Container container1 = mock(Container.class);
        ContainerLaunchContext clc2 = mock(ContainerLaunchContext.class);
        Container container2 = mock(Container.class);
        ContainerLauncherLaunchRequestEvent launchRequestEvent1 = new ContainerLauncherLaunchRequestEvent(clc1, container1, 0, 0, 0);
        ContainerLauncherLaunchRequestEvent launchRequestEvent2 = new ContainerLauncherLaunchRequestEvent(clc2, container2, 1, 0, 0);
        clr.handle(launchRequestEvent1);
        ArgumentCaptor<ContainerLaunchRequest> captor = ArgumentCaptor.forClass(ContainerLaunchRequest.class);
        verify(clr.getTestContainerLauncher(0)).launchContainer(captor.capture());
        assertEquals(1, captor.getAllValues().size());
        ContainerLaunchRequest launchRequest1 = captor.getValue();
        assertEquals(clc1, launchRequest1.getContainerLaunchContext());
        clr.handle(launchRequestEvent2);
        captor = ArgumentCaptor.forClass(ContainerLaunchRequest.class);
        verify(clr.getTestContainerLauncher(1)).launchContainer(captor.capture());
        assertEquals(1, captor.getAllValues().size());
        ContainerLaunchRequest launchRequest2 = captor.getValue();
        assertEquals(clc2, launchRequest2.getContainerLaunchContext());
    } finally {
        clr.stop();
        verify(clr.getTestContainerLauncher(0)).shutdown();
        verify(clr.getTestContainerLauncher(1)).shutdown();
    }
}

Example 15 with TaskCommunicatorManagerInterface

use of org.apache.tez.dag.app.TaskCommunicatorManagerInterface in project tez by apache.

the class TestAMContainerMap method testCleanupOnDagComplete.

@Test(timeout = 10000)
public void testCleanupOnDagComplete() {
    ContainerHeartbeatHandler chh = mock(ContainerHeartbeatHandler.class);
    TaskCommunicatorManagerInterface tal = mock(TaskCommunicatorManagerInterface.class);
    AppContext appContext = mock(AppContext.class);
    when(appContext.getAMConf()).thenReturn(new Configuration());
    int numContainers = 7;
    WrappedContainer[] wContainers = new WrappedContainer[numContainers];
    for (int i = 0; i < numContainers; i++) {
        WrappedContainer wc = new WrappedContainer(false, null, i);
        wContainers[i] = wc;
    }
    AMContainerMap amContainerMap = new AMContainerMapForTest(chh, tal, mock(ContainerSignatureMatcher.class), appContext, wContainers);
    for (int i = 0; i < numContainers; i++) {
        amContainerMap.addContainerIfNew(wContainers[i].container, 0, 0, 0);
    }
    // Container 1 in LAUNCHING state
    wContainers[0].launchContainer();
    wContainers[0].verifyState(AMContainerState.LAUNCHING);
    // Container 2 in IDLE state
    wContainers[1].launchContainer();
    wContainers[1].containerLaunched();
    wContainers[1].verifyState(AMContainerState.IDLE);
    // Container 3 RUNNING state
    wContainers[2].launchContainer();
    wContainers[2].containerLaunched();
    wContainers[2].assignTaskAttempt(wContainers[2].taskAttemptID);
    wContainers[2].verifyState(AMContainerState.RUNNING);
    // Cointainer 4 STOP_REQUESTED
    wContainers[3].launchContainer();
    wContainers[3].containerLaunched();
    wContainers[3].stopRequest();
    wContainers[3].verifyState(AMContainerState.STOP_REQUESTED);
    // Container 5 STOPPING
    wContainers[4].launchContainer();
    wContainers[4].containerLaunched();
    wContainers[4].stopRequest();
    wContainers[4].nmStopSent();
    wContainers[4].verifyState(AMContainerState.STOPPING);
    // Container 6 COMPLETED
    wContainers[5].launchContainer();
    wContainers[5].containerLaunched();
    wContainers[5].stopRequest();
    wContainers[5].nmStopSent();
    wContainers[5].containerCompleted();
    wContainers[5].verifyState(AMContainerState.COMPLETED);
    // Container 7 STOP_REQUESTED + ERROR
    wContainers[6].launchContainer();
    wContainers[6].containerLaunched();
    wContainers[6].containerLaunched();
    assertTrue(wContainers[6].amContainer.isInErrorState());
    wContainers[6].verifyState(AMContainerState.STOP_REQUESTED);
    // 7 containers present, and registered with AMContainerMap at this point.
    assertEquals(7, amContainerMap.containerMap.size());
    amContainerMap.dagComplete(mock(DAG.class));
    assertEquals(5, amContainerMap.containerMap.size());
}