Examples with AppContext org.apache.tez.dag.app.AppContext used on opensource projects

Example 16 with AppContext

use of org.apache.tez.dag.app.AppContext in project tez by apache.

the class TestAMContainerMap method testCleanupOnDagComplete.

@Test(timeout = 10000)
public void testCleanupOnDagComplete() {
    ContainerHeartbeatHandler chh = mock(ContainerHeartbeatHandler.class);
    TaskCommunicatorManagerInterface tal = mock(TaskCommunicatorManagerInterface.class);
    AppContext appContext = mock(AppContext.class);
    when(appContext.getAMConf()).thenReturn(new Configuration());
    int numContainers = 7;
    WrappedContainer[] wContainers = new WrappedContainer[numContainers];
    for (int i = 0; i < numContainers; i++) {
        WrappedContainer wc = new WrappedContainer(false, null, i);
        wContainers[i] = wc;
    }
    AMContainerMap amContainerMap = new AMContainerMapForTest(chh, tal, mock(ContainerSignatureMatcher.class), appContext, wContainers);
    for (int i = 0; i < numContainers; i++) {
        amContainerMap.addContainerIfNew(wContainers[i].container, 0, 0, 0);
    }
    // Container 1 in LAUNCHING state
    wContainers[0].launchContainer();
    wContainers[0].verifyState(AMContainerState.LAUNCHING);
    // Container 2 in IDLE state
    wContainers[1].launchContainer();
    wContainers[1].containerLaunched();
    wContainers[1].verifyState(AMContainerState.IDLE);
    // Container 3 RUNNING state
    wContainers[2].launchContainer();
    wContainers[2].containerLaunched();
    wContainers[2].assignTaskAttempt(wContainers[2].taskAttemptID);
    wContainers[2].verifyState(AMContainerState.RUNNING);
    // Cointainer 4 STOP_REQUESTED
    wContainers[3].launchContainer();
    wContainers[3].containerLaunched();
    wContainers[3].stopRequest();
    wContainers[3].verifyState(AMContainerState.STOP_REQUESTED);
    // Container 5 STOPPING
    wContainers[4].launchContainer();
    wContainers[4].containerLaunched();
    wContainers[4].stopRequest();
    wContainers[4].nmStopSent();
    wContainers[4].verifyState(AMContainerState.STOPPING);
    // Container 6 COMPLETED
    wContainers[5].launchContainer();
    wContainers[5].containerLaunched();
    wContainers[5].stopRequest();
    wContainers[5].nmStopSent();
    wContainers[5].containerCompleted();
    wContainers[5].verifyState(AMContainerState.COMPLETED);
    // Container 7 STOP_REQUESTED + ERROR
    wContainers[6].launchContainer();
    wContainers[6].containerLaunched();
    wContainers[6].containerLaunched();
    assertTrue(wContainers[6].amContainer.isInErrorState());
    wContainers[6].verifyState(AMContainerState.STOP_REQUESTED);
    // 7 containers present, and registered with AMContainerMap at this point.
    assertEquals(7, amContainerMap.containerMap.size());
    amContainerMap.dagComplete(mock(DAG.class));
    assertEquals(5, amContainerMap.containerMap.size());
}

Example 17 with AppContext

use of org.apache.tez.dag.app.AppContext in project tez by apache.

the class TestAMNodeTracker method testSingleNodeNotBlacklistedAlternateScheduler.

@Test(timeout = 5000)
public void testSingleNodeNotBlacklistedAlternateScheduler() {
    AppContext appContext = mock(AppContext.class);
    Configuration conf = new Configuration(false);
    conf.setInt(TezConfiguration.TEZ_AM_MAX_TASK_FAILURES_PER_NODE, 2);
    conf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, true);
    conf.setInt(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_IGNORE_THRESHOLD, 33);
    TestEventHandler handler = new TestEventHandler();
    AMNodeTracker amNodeTracker = new AMNodeTracker(handler, appContext);
    doReturn(amNodeTracker).when(appContext).getNodeTracker();
    AMContainerMap amContainerMap = mock(AMContainerMap.class);
    TaskSchedulerManager taskSchedulerManager = mock(TaskSchedulerManager.class);
    dispatcher.register(AMNodeEventType.class, amNodeTracker);
    dispatcher.register(AMContainerEventType.class, amContainerMap);
    dispatcher.register(AMSchedulerEventType.class, taskSchedulerManager);
    amNodeTracker.init(conf);
    amNodeTracker.start();
    _testSingleNodeNotBlacklisted(amNodeTracker, handler, 1);
}

Example 18 with AppContext

use of org.apache.tez.dag.app.AppContext in project tez by apache.

the class TestAMNodeTracker method testMultipleSourcesNodeRegistration.

@Test(timeout = 5000)
public void testMultipleSourcesNodeRegistration() {
    AppContext appContext = mock(AppContext.class);
    AMNodeTracker amNodeTracker = new AMNodeTracker(eventHandler, appContext);
    doReturn(amNodeTracker).when(appContext).getNodeTracker();
    amNodeTracker.init(new Configuration(false));
    amNodeTracker.start();
    NodeId nodeId1 = NodeId.newInstance("source01", 3333);
    NodeId nodeId2 = NodeId.newInstance("source02", 3333);
    amNodeTracker.nodeSeen(nodeId1, 0);
    amNodeTracker.nodeSeen(nodeId2, 1);
    assertEquals(1, amNodeTracker.getNumNodes(0));
    assertEquals(1, amNodeTracker.getNumNodes(1));
    assertNotNull(amNodeTracker.get(nodeId1, 0));
    assertNull(amNodeTracker.get(nodeId2, 0));
    assertNull(amNodeTracker.get(nodeId1, 1));
    assertNotNull(amNodeTracker.get(nodeId2, 1));
}

Example 19 with AppContext

use of org.apache.tez.dag.app.AppContext in project tez by apache.

the class TestAMNodeTracker method testNodeSelfBlacklistAlternateScheduler2.

@Test(timeout = 10000)
public void testNodeSelfBlacklistAlternateScheduler2() {
    AppContext appContext = mock(AppContext.class);
    Configuration conf = new Configuration(false);
    conf.setInt(TezConfiguration.TEZ_AM_MAX_TASK_FAILURES_PER_NODE, 2);
    TestEventHandler handler = new TestEventHandler();
    AMNodeTracker amNodeTracker = new AMNodeTracker(handler, appContext);
    doReturn(amNodeTracker).when(appContext).getNodeTracker();
    AMContainerMap amContainerMap = mock(AMContainerMap.class);
    TaskSchedulerManager taskSchedulerManager = mock(TaskSchedulerManager.class);
    dispatcher.register(AMNodeEventType.class, amNodeTracker);
    dispatcher.register(AMContainerEventType.class, amContainerMap);
    dispatcher.register(AMSchedulerEventType.class, taskSchedulerManager);
    amNodeTracker.init(conf);
    amNodeTracker.start();
    try {
        // This should not affect the blacklisting behaviour
        for (int i = 0; i < 100; i++) {
            amNodeTracker.nodeSeen(NodeId.newInstance("fakenode" + i, 3333), 0);
        }
        _testNodeSelfBlacklist(amNodeTracker, handler, 1);
        assertFalse(amNodeTracker.isBlacklistingIgnored(0));
    } finally {
        amNodeTracker.stop();
    }
}

Example 20 with AppContext

use of org.apache.tez.dag.app.AppContext in project tez by apache.

the class TestAMNodeTracker method testSingleNodeNotBlacklisted.

@Test(timeout = 5000)
public void testSingleNodeNotBlacklisted() {
    AppContext appContext = mock(AppContext.class);
    Configuration conf = new Configuration(false);
    conf.setInt(TezConfiguration.TEZ_AM_MAX_TASK_FAILURES_PER_NODE, 2);
    conf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, true);
    conf.setInt(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_IGNORE_THRESHOLD, 33);
    TestEventHandler handler = new TestEventHandler();
    AMNodeTracker amNodeTracker = new AMNodeTracker(handler, appContext);
    doReturn(amNodeTracker).when(appContext).getNodeTracker();
    AMContainerMap amContainerMap = mock(AMContainerMap.class);
    TaskSchedulerManager taskSchedulerManager = mock(TaskSchedulerManager.class);
    dispatcher.register(AMNodeEventType.class, amNodeTracker);
    dispatcher.register(AMContainerEventType.class, amContainerMap);
    dispatcher.register(AMSchedulerEventType.class, taskSchedulerManager);
    amNodeTracker.init(conf);
    amNodeTracker.start();
    _testSingleNodeNotBlacklisted(amNodeTracker, handler, 0);
}