Examples with ContainerId org.apache.hadoop.yarn.api.records.ContainerId used on opensource projects

Example 41 with ContainerId

use of org.apache.hadoop.yarn.api.records.ContainerId in project hadoop by apache.

the class ContainersMonitorImpl method handle.

@Override
@SuppressWarnings("unchecked")
public void handle(ContainersMonitorEvent monitoringEvent) {
    ContainerId containerId = monitoringEvent.getContainerId();
    if (!containersMonitorEnabled) {
        if (monitoringEvent.getType() == ContainersMonitorEventType.CHANGE_MONITORING_CONTAINER_RESOURCE) {
            // Nothing to enforce. Update container resource immediately.
            ChangeMonitoringContainerResourceEvent changeEvent = (ChangeMonitoringContainerResourceEvent) monitoringEvent;
            changeContainerResource(containerId, changeEvent.getResource());
        }
        return;
    }
    switch(monitoringEvent.getType()) {
        case START_MONITORING_CONTAINER:
            onStartMonitoringContainer(monitoringEvent, containerId);
            break;
        case STOP_MONITORING_CONTAINER:
            onStopMonitoringContainer(monitoringEvent, containerId);
            break;
        case CHANGE_MONITORING_CONTAINER_RESOURCE:
            onChangeMonitoringContainerResource(monitoringEvent, containerId);
            break;
        default:
    }
}

Example 42 with ContainerId

use of org.apache.hadoop.yarn.api.records.ContainerId in project hadoop by apache.

the class ContainersMonitorImpl method updateContainerMetrics.

private void updateContainerMetrics(ContainersMonitorEvent monitoringEvent) {
    if (!containerMetricsEnabled || monitoringEvent == null) {
        return;
    }
    ContainerId containerId = monitoringEvent.getContainerId();
    ContainerMetrics usageMetrics;
    int vmemLimitMBs;
    int pmemLimitMBs;
    int cpuVcores;
    switch(monitoringEvent.getType()) {
        case START_MONITORING_CONTAINER:
            usageMetrics = ContainerMetrics.forContainer(containerId, containerMetricsPeriodMs, containerMetricsUnregisterDelayMs);
            ContainerStartMonitoringEvent startEvent = (ContainerStartMonitoringEvent) monitoringEvent;
            usageMetrics.recordStateChangeDurations(startEvent.getLaunchDuration(), startEvent.getLocalizationDuration());
            cpuVcores = startEvent.getCpuVcores();
            vmemLimitMBs = (int) (startEvent.getVmemLimit() >> 20);
            pmemLimitMBs = (int) (startEvent.getPmemLimit() >> 20);
            usageMetrics.recordResourceLimit(vmemLimitMBs, pmemLimitMBs, cpuVcores);
            break;
        case STOP_MONITORING_CONTAINER:
            usageMetrics = ContainerMetrics.getContainerMetrics(containerId);
            if (usageMetrics != null) {
                usageMetrics.finished();
            }
            break;
        case CHANGE_MONITORING_CONTAINER_RESOURCE:
            usageMetrics = ContainerMetrics.forContainer(containerId, containerMetricsPeriodMs, containerMetricsUnregisterDelayMs);
            ChangeMonitoringContainerResourceEvent changeEvent = (ChangeMonitoringContainerResourceEvent) monitoringEvent;
            Resource resource = changeEvent.getResource();
            pmemLimitMBs = (int) resource.getMemorySize();
            vmemLimitMBs = (int) (pmemLimitMBs * vmemRatio);
            cpuVcores = resource.getVirtualCores();
            usageMetrics.recordResourceLimit(vmemLimitMBs, pmemLimitMBs, cpuVcores);
            break;
        default:
            break;
    }
}

Example 43 with ContainerId

use of org.apache.hadoop.yarn.api.records.ContainerId in project hadoop by apache.

the class NMLeveldbStateStoreService method loadContainerTokensState.

@Override
public RecoveredContainerTokensState loadContainerTokensState() throws IOException {
    RecoveredContainerTokensState state = new RecoveredContainerTokensState();
    state.activeTokens = new HashMap<ContainerId, Long>();
    LeveldbIterator iter = null;
    try {
        iter = new LeveldbIterator(db);
        iter.seek(bytes(CONTAINER_TOKENS_KEY_PREFIX));
        final int containerTokensKeyPrefixLength = CONTAINER_TOKENS_KEY_PREFIX.length();
        while (iter.hasNext()) {
            Entry<byte[], byte[]> entry = iter.next();
            String fullKey = asString(entry.getKey());
            if (!fullKey.startsWith(CONTAINER_TOKENS_KEY_PREFIX)) {
                break;
            }
            String key = fullKey.substring(containerTokensKeyPrefixLength);
            if (key.equals(CURRENT_MASTER_KEY_SUFFIX)) {
                state.currentMasterKey = parseMasterKey(entry.getValue());
            } else if (key.equals(PREV_MASTER_KEY_SUFFIX)) {
                state.previousMasterKey = parseMasterKey(entry.getValue());
            } else if (key.startsWith(ConverterUtils.CONTAINER_PREFIX)) {
                loadContainerToken(state, fullKey, key, entry.getValue());
            }
        }
    } catch (DBException e) {
        throw new IOException(e);
    } finally {
        if (iter != null) {
            iter.close();
        }
    }
    return state;
}

Example 44 with ContainerId

use of org.apache.hadoop.yarn.api.records.ContainerId in project hadoop by apache.

the class TestLinuxContainerExecutorWithMocks method testContainerLaunch.

@Test
public void testContainerLaunch() throws IOException {
    String appSubmitter = "nobody";
    String cmd = String.valueOf(PrivilegedOperation.RunAsUserCommand.LAUNCH_CONTAINER.getValue());
    String appId = "APP_ID";
    String containerId = "CONTAINER_ID";
    Container container = mock(Container.class);
    ContainerId cId = mock(ContainerId.class);
    ContainerLaunchContext context = mock(ContainerLaunchContext.class);
    HashMap<String, String> env = new HashMap<String, String>();
    when(container.getContainerId()).thenReturn(cId);
    when(container.getLaunchContext()).thenReturn(context);
    when(cId.toString()).thenReturn(containerId);
    when(context.getEnvironment()).thenReturn(env);
    Path scriptPath = new Path("file:///bin/echo");
    Path tokensPath = new Path("file:///dev/null");
    Path workDir = new Path("/tmp");
    Path pidFile = new Path(workDir, "pid.txt");
    mockExec.activateContainer(cId, pidFile);
    int ret = mockExec.launchContainer(new ContainerStartContext.Builder().setContainer(container).setNmPrivateContainerScriptPath(scriptPath).setNmPrivateTokensPath(tokensPath).setUser(appSubmitter).setAppId(appId).setContainerWorkDir(workDir).setLocalDirs(dirsHandler.getLocalDirs()).setLogDirs(dirsHandler.getLogDirs()).setFilecacheDirs(new ArrayList<>()).setUserLocalDirs(new ArrayList<>()).setContainerLocalDirs(new ArrayList<>()).setContainerLogDirs(new ArrayList<>()).build());
    assertEquals(0, ret);
    assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER, appSubmitter, cmd, appId, containerId, workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(), StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, dirsHandler.getLocalDirs()), StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, dirsHandler.getLogDirs()), "cgroups=none"), readMockParams());
}

Example 45 with ContainerId

use of org.apache.hadoop.yarn.api.records.ContainerId in project hadoop by apache.

the class TestLinuxContainerExecutorWithMocks method testContainerKill.

@Test
public void testContainerKill() throws IOException {
    String appSubmitter = "nobody";
    String cmd = String.valueOf(PrivilegedOperation.RunAsUserCommand.SIGNAL_CONTAINER.getValue());
    ContainerExecutor.Signal signal = ContainerExecutor.Signal.QUIT;
    String sigVal = String.valueOf(signal.getValue());
    Container container = mock(Container.class);
    ContainerId cId = mock(ContainerId.class);
    ContainerLaunchContext context = mock(ContainerLaunchContext.class);
    when(container.getContainerId()).thenReturn(cId);
    when(container.getLaunchContext()).thenReturn(context);
    mockExec.signalContainer(new ContainerSignalContext.Builder().setContainer(container).setUser(appSubmitter).setPid("1000").setSignal(signal).build());
    assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER, appSubmitter, cmd, "1000", sigVal), readMockParams());
}