Examples with ContainerContext org.apache.tez.dag.app.ContainerContext used on opensource projects

Example 1 with ContainerContext

use of org.apache.tez.dag.app.ContainerContext in project tez by apache.

the class TestTaskImpl method setup.

@Before
public void setup() {
    conf = new Configuration();
    conf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 4);
    taskCommunicatorManagerInterface = mock(TaskCommunicatorManagerInterface.class);
    taskHeartbeatHandler = mock(TaskHeartbeatHandler.class);
    credentials = new Credentials();
    clock = new SystemClock();
    locationHint = TaskLocationHint.createTaskLocationHint(null, null);
    appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
    dagId = TezDAGID.getInstance(appId, 1);
    vertexId = TezVertexID.getInstance(dagId, 1);
    appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
    when(appContext.getDAGRecoveryData()).thenReturn(null);
    appContext.setDAGRecoveryData(null);
    mockContainerId = mock(ContainerId.class);
    mockContainer = mock(Container.class);
    mockAMContainer = mock(AMContainer.class);
    when(mockAMContainer.getContainer()).thenReturn(mockContainer);
    when(mockContainer.getNodeHttpAddress()).thenReturn("localhost:1234");
    mockNodeId = mock(NodeId.class);
    mockHistoryHandler = mock(HistoryEventHandler.class);
    when(mockContainer.getId()).thenReturn(mockContainerId);
    when(mockContainer.getNodeId()).thenReturn(mockNodeId);
    when(mockAMContainer.getContainer()).thenReturn(mockContainer);
    when(appContext.getAllContainers().get(mockContainerId)).thenReturn(mockAMContainer);
    when(appContext.getHistoryHandler()).thenReturn(mockHistoryHandler);
    taskResource = Resource.newInstance(1024, 1);
    localResources = new HashMap<String, LocalResource>();
    environment = new HashMap<String, String>();
    javaOpts = "";
    leafVertex = false;
    containerContext = new ContainerContext(localResources, credentials, environment, javaOpts);
    Vertex vertex = mock(Vertex.class);
    doReturn(new VertexImpl.VertexConfigImpl(conf)).when(vertex).getVertexConfig();
    eventHandler = new TestEventHandler();
    mockTask = new MockTaskImpl(vertexId, partition, eventHandler, conf, taskCommunicatorManagerInterface, clock, taskHeartbeatHandler, appContext, leafVertex, taskResource, containerContext, vertex);
    mockTaskSpec = mock(TaskSpec.class);
    mockVertex = mock(Vertex.class);
    ServicePluginInfo servicePluginInfo = new ServicePluginInfo().setContainerLauncherName(TezConstants.getTezYarnServicePluginName());
    when(mockVertex.getServicePluginInfo()).thenReturn(servicePluginInfo);
    when(mockVertex.getVertexConfig()).thenReturn(new VertexImpl.VertexConfigImpl(conf));
}

Example 2 with ContainerContext

use of org.apache.tez.dag.app.ContainerContext in project tez by apache.

the class TestVertexImpl2 method testTaskLoggingOptsPerLogger.

@Test(timeout = 5000)
public void testTaskLoggingOptsPerLogger() {
    Configuration conf = new TezConfiguration();
    conf.set(TezConfiguration.TEZ_TASK_LOG_LEVEL, "DEBUG;org.apache.hadoop.ipc=INFO;org.apache.hadoop.server=INFO");
    LogTestInfoHolder testInfo = new LogTestInfoHolder();
    VertexWrapper vertexWrapper = createVertexWrapperForLogTests(testInfo, conf);
    List<String> expectedCommands = new LinkedList<String>();
    expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
    expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "DEBUG" + "," + TezConstants.TEZ_CONTAINER_LOGGER_NAME);
    for (int i = 0; i < testInfo.numTasks; i++) {
        ContainerContext containerContext = vertexWrapper.vertex.getContainerContext(i);
        String javaOpts = containerContext.getJavaOpts();
        assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
        for (String expectedCmd : expectedCommands) {
            assertTrue(javaOpts.contains(expectedCmd));
        }
        Map<String, String> env = containerContext.getEnvironment();
        String val = env.get(testInfo.envKey);
        assertEquals(testInfo.envVal, val);
        String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
        assertEquals("org.apache.hadoop.ipc=INFO;org.apache.hadoop.server=INFO", logEnvVal);
    }
}

Example 3 with ContainerContext

use of org.apache.tez.dag.app.ContainerContext in project tez by apache.

the class TestVertexImpl2 method testTaskSpecificLoggingOpts2.

@Test(timeout = 5000)
public void testTaskSpecificLoggingOpts2() {
    String vertexName = "testvertex";
    String customJavaOpts = "-Xmx128m";
    Configuration conf = new TezConfiguration();
    conf.set(TezConfiguration.TEZ_TASK_LOG_LEVEL, "WARN;org.apache.tez=INFO");
    conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS_LIST, vertexName + "[0,1,2]");
    conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LOG_LEVEL, "DEBUG");
    conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS, customJavaOpts);
    LogTestInfoHolder testInfo = new LogTestInfoHolder();
    VertexWrapper vertexWrapper = createVertexWrapperForLogTests(testInfo, conf);
    // Expected command opts for regular tasks
    List<String> expectedCommands = new LinkedList<String>();
    expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
    expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "WARN" + "," + TezConstants.TEZ_CONTAINER_LOGGER_NAME);
    for (int i = 3; i < testInfo.numTasks; i++) {
        ContainerContext containerContext = vertexWrapper.vertex.getContainerContext(i);
        String javaOpts = containerContext.getJavaOpts();
        assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
        for (String expectedCmd : expectedCommands) {
            assertTrue(javaOpts.contains(expectedCmd));
        }
        Map<String, String> env = containerContext.getEnvironment();
        String val = env.get(testInfo.envKey);
        assertEquals(testInfo.envVal, val);
        String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
        assertEquals("org.apache.tez=INFO", logEnvVal);
    }
    // Expected command opts for instrumented tasks.
    expectedCommands = new LinkedList<String>();
    expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
    expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "DEBUG" + "," + TezConstants.TEZ_CONTAINER_LOGGER_NAME);
    for (int i = 0; i < 3; i++) {
        ContainerContext containerContext = vertexWrapper.vertex.getContainerContext(i);
        String javaOpts = containerContext.getJavaOpts();
        assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
        for (String expectedCmd : expectedCommands) {
            assertTrue(javaOpts.contains(expectedCmd));
        }
        Map<String, String> env = containerContext.getEnvironment();
        String val = env.get(testInfo.envKey);
        assertEquals(testInfo.envVal, val);
        String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
        assertNull(logEnvVal);
    }
}

Example 4 with ContainerContext

use of org.apache.tez.dag.app.ContainerContext in project tez by apache.

the class ContainerContextMatcher method isSuperSet.

@Override
public boolean isSuperSet(Object cs1, Object cs2) {
    checkArguments(cs1, cs2);
    ContainerContext context1 = (ContainerContext) cs1;
    ContainerContext context2 = (ContainerContext) cs2;
    return context1.isSuperSet(context2);
}

Example 5 with ContainerContext

use of org.apache.tez.dag.app.ContainerContext in project tez by apache.

the class TestTaskSchedulerManager method testSimpleAllocate.

@Test(timeout = 5000)
public void testSimpleAllocate() throws Exception {
    Configuration conf = new Configuration(false);
    schedulerHandler.init(conf);
    schedulerHandler.start();
    TaskAttemptImpl mockTaskAttempt = mock(TaskAttemptImpl.class);
    TezTaskAttemptID mockAttemptId = mock(TezTaskAttemptID.class);
    when(mockAttemptId.getId()).thenReturn(0);
    when(mockTaskAttempt.getID()).thenReturn(mockAttemptId);
    Resource resource = Resource.newInstance(1024, 1);
    ContainerContext containerContext = new ContainerContext(new HashMap<String, LocalResource>(), new Credentials(), new HashMap<String, String>(), "");
    int priority = 10;
    TaskLocationHint locHint = TaskLocationHint.createTaskLocationHint(new HashSet<String>(), null);
    ContainerId mockCId = mock(ContainerId.class);
    Container container = mock(Container.class);
    when(container.getId()).thenReturn(mockCId);
    AMContainer mockAMContainer = mock(AMContainer.class);
    when(mockAMContainer.getContainerId()).thenReturn(mockCId);
    when(mockAMContainer.getState()).thenReturn(AMContainerState.IDLE);
    when(mockAMContainerMap.get(mockCId)).thenReturn(mockAMContainer);
    AMSchedulerEventTALaunchRequest lr = new AMSchedulerEventTALaunchRequest(mockAttemptId, resource, null, mockTaskAttempt, locHint, priority, containerContext, 0, 0, 0);
    schedulerHandler.taskAllocated(0, mockTaskAttempt, lr, container);
    assertEquals(1, mockEventHandler.events.size());
    assertTrue(mockEventHandler.events.get(0) instanceof AMContainerEventAssignTA);
    AMContainerEventAssignTA assignEvent = (AMContainerEventAssignTA) mockEventHandler.events.get(0);
    assertEquals(priority, assignEvent.getPriority());
    assertEquals(mockAttemptId, assignEvent.getTaskAttemptId());
}