Examples with ContainerStatus org.apache.hadoop.yarn.api.records.ContainerStatus used on opensource projects

Example 71 with ContainerStatus

use of org.apache.hadoop.yarn.api.records.ContainerStatus in project flink by apache.

the class YarnResourceManagerDriverTest method testGetContainerCompletedCauseForUnknown.

@Test
public void testGetContainerCompletedCauseForUnknown() {
    ContainerStatus containerStatus = createCompletedContainerStatusForUnknownCause();
    testingGetContainerCompletedCause(containerStatus, String.format("Container %s marked as failed.\n Exit code:%s.", containerStatus.getContainerId(), containerStatus.getExitStatus()));
}

Example 72 with ContainerStatus

use of org.apache.hadoop.yarn.api.records.ContainerStatus in project flink by apache.

the class YarnResourceManagerDriverTest method testStartWorkerVariousSpec.

@Test
public void testStartWorkerVariousSpec() throws Exception {
    final TaskExecutorProcessSpec taskExecutorProcessSpec1 = new TaskExecutorProcessSpec(new CPUResource(1), MemorySize.ZERO, MemorySize.ZERO, MemorySize.ofMebiBytes(50), MemorySize.ofMebiBytes(50), MemorySize.ofMebiBytes(50), MemorySize.ofMebiBytes(50), MemorySize.ZERO, MemorySize.ZERO, Collections.emptyList());
    final TaskExecutorProcessSpec taskExecutorProcessSpec2 = new TaskExecutorProcessSpec(new CPUResource(2), MemorySize.ZERO, MemorySize.ZERO, MemorySize.ofMebiBytes(500), MemorySize.ofMebiBytes(500), MemorySize.ofMebiBytes(500), MemorySize.ofMebiBytes(500), MemorySize.ZERO, MemorySize.ZERO, Collections.emptyList());
    new Context() {

        {
            final String startCommand1 = TaskManagerOptions.TASK_HEAP_MEMORY.key() + "=" + (50L << 20);
            final String startCommand2 = TaskManagerOptions.TASK_HEAP_MEMORY.key() + "=" + (100L << 20);
            final CompletableFuture<Void> startContainerAsyncCommandFuture1 = new CompletableFuture<>();
            final CompletableFuture<Void> startContainerAsyncCommandFuture2 = new CompletableFuture<>();
            prepareForTestStartTaskExecutorProcessVariousSpec(startCommand1, startCommand2, startContainerAsyncCommandFuture1, startContainerAsyncCommandFuture2, taskExecutorProcessSpec1);
            testingYarnAMRMClientAsyncBuilder.setGetMatchingRequestsFunction(tuple -> {
                final Priority priority = tuple.f0;
                final List<AMRMClient.ContainerRequest> matchingRequests = new ArrayList<>();
                for (CompletableFuture<AMRMClient.ContainerRequest> addContainerRequestFuture : addContainerRequestFutures) {
                    final AMRMClient.ContainerRequest request = addContainerRequestFuture.getNow(null);
                    if (request != null && priority.equals(request.getPriority())) {
                        assertThat(tuple.f2, is(request.getCapability()));
                        matchingRequests.add(request);
                    }
                }
                return Collections.singletonList(matchingRequests);
            });
            runTest(() -> {
                final Resource containerResource1 = ((YarnResourceManagerDriver) getDriver()).getContainerResource(taskExecutorProcessSpec1).get();
                final Resource containerResource2 = ((YarnResourceManagerDriver) getDriver()).getContainerResource(taskExecutorProcessSpec2).get();
                // Make sure two worker resource spec will be normalized to different
                // container resources
                assertNotEquals(containerResource1, containerResource2);
                runInMainThread(() -> getDriver().requestResource(taskExecutorProcessSpec1));
                runInMainThread(() -> getDriver().requestResource(taskExecutorProcessSpec2));
                // Verify both containers requested
                verifyFutureCompleted(addContainerRequestFutures.get(0));
                verifyFutureCompleted(addContainerRequestFutures.get(1));
                // Mock that container 1 is allocated
                Container container1 = createTestingContainerWithResource(containerResource1);
                resourceManagerClientCallbackHandler.onContainersAllocated(Collections.singletonList(container1));
                // Verify that only worker with spec1 is started.
                verifyFutureCompleted(startContainerAsyncCommandFuture1);
                assertFalse(startContainerAsyncCommandFuture2.isDone());
                // Mock that container 1 is completed, while the worker is still pending
                ContainerStatus testingContainerStatus = createTestingContainerCompletedStatus(container1.getId());
                resourceManagerClientCallbackHandler.onContainersCompleted(Collections.singletonList(testingContainerStatus));
                // Verify that only container 1 is requested again
                verifyFutureCompleted(addContainerRequestFutures.get(2));
                assertThat(addContainerRequestFutures.get(2).get().getCapability(), is(containerResource1));
                assertFalse(addContainerRequestFutures.get(3).isDone());
            });
        }
    };
}

Example 73 with ContainerStatus

use of org.apache.hadoop.yarn.api.records.ContainerStatus in project hadoop by apache.

the class MockRM method waitForContainerToComplete.

public void waitForContainerToComplete(RMAppAttempt attempt, NMContainerStatus completedContainer) throws InterruptedException {
    drainEventsImplicitly();
    int timeWaiting = 0;
    while (timeWaiting < TIMEOUT_MS_FOR_CONTAINER_AND_NODE) {
        List<ContainerStatus> containers = attempt.getJustFinishedContainers();
        LOG.info("Received completed containers " + containers);
        for (ContainerStatus container : containers) {
            if (container.getContainerId().equals(completedContainer.getContainerId())) {
                return;
            }
        }
        Thread.sleep(WAIT_MS_PER_LOOP);
        timeWaiting += WAIT_MS_PER_LOOP;
    }
}

Example 74 with ContainerStatus

use of org.apache.hadoop.yarn.api.records.ContainerStatus in project hadoop by apache.

the class NodeManager method stopContainers.

@Override
public synchronized StopContainersResponse stopContainers(StopContainersRequest request) throws YarnException {
    for (ContainerId containerID : request.getContainerIds()) {
        String applicationId = String.valueOf(containerID.getApplicationAttemptId().getApplicationId().getId());
        // Mark the container as COMPLETE
        List<Container> applicationContainers = containers.get(containerID.getApplicationAttemptId().getApplicationId());
        for (Container c : applicationContainers) {
            if (c.getId().compareTo(containerID) == 0) {
                ContainerStatus containerStatus = containerStatusMap.get(c);
                containerStatus.setState(ContainerState.COMPLETE);
                containerStatusMap.put(c, containerStatus);
            }
        }
        // Send a heartbeat
        try {
            heartbeat();
        } catch (IOException ioe) {
            throw RPCUtil.getRemoteException(ioe);
        }
        // Remove container and update status
        int ctr = 0;
        Container container = null;
        for (Iterator<Container> i = applicationContainers.iterator(); i.hasNext(); ) {
            container = i.next();
            if (container.getId().compareTo(containerID) == 0) {
                i.remove();
                ++ctr;
            }
        }
        if (ctr != 1) {
            throw new IllegalStateException("Container " + containerID + " stopped " + ctr + " times!");
        }
        Resources.addTo(available, container.getResource());
        Resources.subtractFrom(used, container.getResource());
        if (LOG.isDebugEnabled()) {
            LOG.debug("stopContainer:" + " node=" + containerManagerAddress + " application=" + applicationId + " container=" + containerID + " available=" + available + " used=" + used);
        }
    }
    return StopContainersResponse.newInstance(null, null);
}

Example 75 with ContainerStatus

use of org.apache.hadoop.yarn.api.records.ContainerStatus in project hadoop by apache.

the class MockNM method nodeHeartbeat.

public NodeHeartbeatResponse nodeHeartbeat(List<ContainerStatus> updatedStats, List<Container> increasedConts, boolean isHealthy, int resId) throws Exception {
    NodeHeartbeatRequest req = Records.newRecord(NodeHeartbeatRequest.class);
    NodeStatus status = Records.newRecord(NodeStatus.class);
    status.setResponseId(resId);
    status.setNodeId(nodeId);
    ArrayList<ContainerId> completedContainers = new ArrayList<ContainerId>();
    for (ContainerStatus stat : updatedStats) {
        if (stat.getState() == ContainerState.COMPLETE) {
            completedContainers.add(stat.getContainerId());
        }
        containerStats.put(stat.getContainerId(), stat);
    }
    status.setContainersStatuses(new ArrayList<ContainerStatus>(containerStats.values()));
    for (ContainerId cid : completedContainers) {
        containerStats.remove(cid);
    }
    status.setIncreasedContainers(increasedConts);
    NodeHealthStatus healthStatus = Records.newRecord(NodeHealthStatus.class);
    healthStatus.setHealthReport("");
    healthStatus.setIsNodeHealthy(isHealthy);
    healthStatus.setLastHealthReportTime(1);
    status.setNodeHealthStatus(healthStatus);
    req.setNodeStatus(status);
    req.setLastKnownContainerTokenMasterKey(this.currentContainerTokenMasterKey);
    req.setLastKnownNMTokenMasterKey(this.currentNMTokenMasterKey);
    NodeHeartbeatResponse heartbeatResponse = resourceTracker.nodeHeartbeat(req);
    MasterKey masterKeyFromRM = heartbeatResponse.getContainerTokenMasterKey();
    if (masterKeyFromRM != null && masterKeyFromRM.getKeyId() != this.currentContainerTokenMasterKey.getKeyId()) {
        this.currentContainerTokenMasterKey = masterKeyFromRM;
    }
    masterKeyFromRM = heartbeatResponse.getNMTokenMasterKey();
    if (masterKeyFromRM != null && masterKeyFromRM.getKeyId() != this.currentNMTokenMasterKey.getKeyId()) {
        this.currentNMTokenMasterKey = masterKeyFromRM;
    }
    Resource newResource = heartbeatResponse.getResource();
    if (newResource != null) {
        memory = newResource.getMemorySize();
        vCores = newResource.getVirtualCores();
    }
    return heartbeatResponse;
}