Examples with ContainerId org.apache.hadoop.yarn.api.records.ContainerId used on opensource projects

Example 36 with ContainerId

use of org.apache.hadoop.yarn.api.records.ContainerId in project hadoop by apache.

the class TestTimelineServiceClientIntegration method testPutExtendedEntities.

@Test
public void testPutExtendedEntities() throws Exception {
    ApplicationId appId = ApplicationId.newInstance(0, 1);
    TimelineV2Client client = TimelineV2Client.createTimelineClient(appId);
    try {
        // set the timeline service address manually
        client.setTimelineServiceAddress(collectorManager.getRestServerBindAddress());
        client.init(conf);
        client.start();
        ClusterEntity cluster = new ClusterEntity();
        cluster.setId(YarnConfiguration.DEFAULT_RM_CLUSTER_ID);
        FlowRunEntity flow = new FlowRunEntity();
        flow.setUser(UserGroupInformation.getCurrentUser().getShortUserName());
        flow.setName("test_flow_name");
        flow.setVersion("test_flow_version");
        flow.setRunId(1L);
        flow.setParent(cluster.getType(), cluster.getId());
        ApplicationEntity app = new ApplicationEntity();
        app.setId(appId.toString());
        flow.addChild(app.getType(), app.getId());
        ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
        ApplicationAttemptEntity appAttempt = new ApplicationAttemptEntity();
        appAttempt.setId(attemptId.toString());
        ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
        ContainerEntity container = new ContainerEntity();
        container.setId(containerId.toString());
        UserEntity user = new UserEntity();
        user.setId(UserGroupInformation.getCurrentUser().getShortUserName());
        QueueEntity queue = new QueueEntity();
        queue.setId("default_queue");
        client.putEntities(cluster, flow, app, appAttempt, container, user, queue);
        client.putEntitiesAsync(cluster, flow, app, appAttempt, container, user, queue);
    } finally {
        client.stop();
    }
}

Example 37 with ContainerId

use of org.apache.hadoop.yarn.api.records.ContainerId in project hive by apache.

the class TestLlapTaskCommunicator method testEntityTracker1.

@Test(timeout = 5000)
public void testEntityTracker1() {
    LlapTaskCommunicator.EntityTracker entityTracker = new LlapTaskCommunicator.EntityTracker();
    String host1 = "host1";
    int port = 1451;
    // Simple container registration and un-registration without any task attempt being involved.
    ContainerId containerId101 = constructContainerId(101);
    entityTracker.registerContainer(containerId101, host1, port);
    assertEquals(LlapNodeId.getInstance(host1, port), entityTracker.getNodeIdForContainer(containerId101));
    entityTracker.unregisterContainer(containerId101);
    assertNull(entityTracker.getContainerAttemptMapForNode(LlapNodeId.getInstance(host1, port)));
    assertNull(entityTracker.getNodeIdForContainer(containerId101));
    assertEquals(0, entityTracker.nodeMap.size());
    assertEquals(0, entityTracker.attemptToNodeMap.size());
    assertEquals(0, entityTracker.containerToNodeMap.size());
    // Simple task registration and un-registration.
    ContainerId containerId1 = constructContainerId(1);
    TezTaskAttemptID taskAttemptId1 = constructTaskAttemptId(1);
    entityTracker.registerTaskAttempt(containerId1, taskAttemptId1, host1, port);
    assertEquals(LlapNodeId.getInstance(host1, port), entityTracker.getNodeIdForContainer(containerId1));
    assertEquals(LlapNodeId.getInstance(host1, port), entityTracker.getNodeIdForTaskAttempt(taskAttemptId1));
    entityTracker.unregisterTaskAttempt(taskAttemptId1);
    assertNull(entityTracker.getContainerAttemptMapForNode(LlapNodeId.getInstance(host1, port)));
    assertNull(entityTracker.getNodeIdForContainer(containerId1));
    assertNull(entityTracker.getNodeIdForTaskAttempt(taskAttemptId1));
    assertEquals(0, entityTracker.nodeMap.size());
    assertEquals(0, entityTracker.attemptToNodeMap.size());
    assertEquals(0, entityTracker.containerToNodeMap.size());
    // Register taskAttempt, unregister container. TaskAttempt should also be unregistered
    ContainerId containerId201 = constructContainerId(201);
    TezTaskAttemptID taskAttemptId201 = constructTaskAttemptId(201);
    entityTracker.registerTaskAttempt(containerId201, taskAttemptId201, host1, port);
    assertEquals(LlapNodeId.getInstance(host1, port), entityTracker.getNodeIdForContainer(containerId201));
    assertEquals(LlapNodeId.getInstance(host1, port), entityTracker.getNodeIdForTaskAttempt(taskAttemptId201));
    entityTracker.unregisterContainer(containerId201);
    assertNull(entityTracker.getContainerAttemptMapForNode(LlapNodeId.getInstance(host1, port)));
    assertNull(entityTracker.getNodeIdForContainer(containerId201));
    assertNull(entityTracker.getNodeIdForTaskAttempt(taskAttemptId201));
    assertEquals(0, entityTracker.nodeMap.size());
    assertEquals(0, entityTracker.attemptToNodeMap.size());
    assertEquals(0, entityTracker.containerToNodeMap.size());
    // No errors
    entityTracker.unregisterTaskAttempt(taskAttemptId201);
}

Example 38 with ContainerId

use of org.apache.hadoop.yarn.api.records.ContainerId in project hive by apache.

the class TestLlapTaskCommunicator method constructContainerId.

private ContainerId constructContainerId(int id) {
    ContainerId containerId = mock(ContainerId.class);
    doReturn(id).when(containerId).getId();
    doReturn((long) id).when(containerId).getContainerId();
    return containerId;
}

Example 39 with ContainerId

use of org.apache.hadoop.yarn.api.records.ContainerId in project hadoop by apache.

the class TestDefaultContainerExecutor method testContainerLaunchError.

@Test
public void testContainerLaunchError() throws IOException, InterruptedException {
    if (Shell.WINDOWS) {
        BASE_TMP_PATH = new Path(new File("target").getAbsolutePath(), TestDefaultContainerExecutor.class.getSimpleName());
    }
    Path localDir = new Path(BASE_TMP_PATH, "localDir");
    List<String> localDirs = new ArrayList<String>();
    localDirs.add(localDir.toString());
    List<String> logDirs = new ArrayList<String>();
    Path logDir = new Path(BASE_TMP_PATH, "logDir");
    logDirs.add(logDir.toString());
    Configuration conf = new Configuration();
    conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.toString());
    conf.set(YarnConfiguration.NM_LOG_DIRS, logDir.toString());
    FileContext lfs = FileContext.getLocalFSFileContext(conf);
    DefaultContainerExecutor mockExec = spy(new DefaultContainerExecutor(lfs));
    mockExec.setConf(conf);
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            String diagnostics = (String) invocationOnMock.getArguments()[0];
            assertTrue("Invalid Diagnostics message: " + diagnostics, diagnostics.contains("No such file or directory"));
            return null;
        }
    }).when(mockExec).logOutput(any(String.class));
    String appSubmitter = "nobody";
    String appId = "APP_ID";
    String containerId = "CONTAINER_ID";
    Container container = mock(Container.class);
    ContainerId cId = mock(ContainerId.class);
    ContainerLaunchContext context = mock(ContainerLaunchContext.class);
    HashMap<String, String> env = new HashMap<String, String>();
    env.put("LANG", "C");
    when(container.getContainerId()).thenReturn(cId);
    when(container.getLaunchContext()).thenReturn(context);
    try {
        doAnswer(new Answer() {

            @Override
            public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
                ContainerDiagnosticsUpdateEvent event = (ContainerDiagnosticsUpdateEvent) invocationOnMock.getArguments()[0];
                assertTrue("Invalid Diagnostics message: " + event.getDiagnosticsUpdate(), event.getDiagnosticsUpdate().contains("No such file or directory"));
                return null;
            }
        }).when(container).handle(any(ContainerDiagnosticsUpdateEvent.class));
        when(cId.toString()).thenReturn(containerId);
        when(cId.getApplicationAttemptId()).thenReturn(ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 0));
        when(context.getEnvironment()).thenReturn(env);
        mockExec.createUserLocalDirs(localDirs, appSubmitter);
        mockExec.createUserCacheDirs(localDirs, appSubmitter);
        mockExec.createAppDirs(localDirs, appSubmitter, appId);
        mockExec.createAppLogDirs(appId, logDirs, appSubmitter);
        Path scriptPath = new Path("file:///bin/echo");
        Path tokensPath = new Path("file:///dev/null");
        if (Shell.WINDOWS) {
            File tmp = new File(BASE_TMP_PATH.toString(), "test_echo.cmd");
            BufferedWriter output = new BufferedWriter(new FileWriter(tmp));
            output.write("Exit 1");
            output.write("Echo No such file or directory 1>&2");
            output.close();
            scriptPath = new Path(tmp.getAbsolutePath());
            tmp = new File(BASE_TMP_PATH.toString(), "tokens");
            tmp.createNewFile();
            tokensPath = new Path(tmp.getAbsolutePath());
        }
        Path workDir = localDir;
        Path pidFile = new Path(workDir, "pid.txt");
        mockExec.init();
        mockExec.activateContainer(cId, pidFile);
        int ret = mockExec.launchContainer(new ContainerStartContext.Builder().setContainer(container).setNmPrivateContainerScriptPath(scriptPath).setNmPrivateTokensPath(tokensPath).setUser(appSubmitter).setAppId(appId).setContainerWorkDir(workDir).setLocalDirs(localDirs).setLogDirs(logDirs).build());
        Assert.assertNotSame(0, ret);
    } finally {
        mockExec.deleteAsUser(new DeletionAsUserContext.Builder().setUser(appSubmitter).setSubDir(localDir).build());
        mockExec.deleteAsUser(new DeletionAsUserContext.Builder().setUser(appSubmitter).setSubDir(logDir).build());
    }
}

Example 40 with ContainerId

use of org.apache.hadoop.yarn.api.records.ContainerId in project hadoop by apache.

the class NMLeveldbStateStoreService method loadContainersState.

@Override
public List<RecoveredContainerState> loadContainersState() throws IOException {
    ArrayList<RecoveredContainerState> containers = new ArrayList<RecoveredContainerState>();
    ArrayList<ContainerId> containersToRemove = new ArrayList<ContainerId>();
    LeveldbIterator iter = null;
    try {
        iter = new LeveldbIterator(db);
        iter.seek(bytes(CONTAINERS_KEY_PREFIX));
        while (iter.hasNext()) {
            Entry<byte[], byte[]> entry = iter.peekNext();
            String key = asString(entry.getKey());
            if (!key.startsWith(CONTAINERS_KEY_PREFIX)) {
                break;
            }
            int idEndPos = key.indexOf('/', CONTAINERS_KEY_PREFIX.length());
            if (idEndPos < 0) {
                throw new IOException("Unable to determine container in key: " + key);
            }
            ContainerId containerId = ContainerId.fromString(key.substring(CONTAINERS_KEY_PREFIX.length(), idEndPos));
            String keyPrefix = key.substring(0, idEndPos + 1);
            RecoveredContainerState rcs = loadContainerState(containerId, iter, keyPrefix);
            // Don't load container without StartContainerRequest
            if (rcs.startRequest != null) {
                containers.add(rcs);
            } else {
                containersToRemove.add(containerId);
            }
        }
    } catch (DBException e) {
        throw new IOException(e);
    } finally {
        if (iter != null) {
            iter.close();
        }
    }
    // remove container without StartContainerRequest
    for (ContainerId containerId : containersToRemove) {
        LOG.warn("Remove container " + containerId + " with incomplete records");
        try {
            removeContainer(containerId);
        // TODO: kill and cleanup the leaked container
        } catch (IOException e) {
            LOG.error("Unable to remove container " + containerId + " in store", e);
        }
    }
    return containers;
}