Examples with ContainerStatus org.apache.hadoop.yarn.api.records.ContainerStatus used on opensource projects

Example 51 with ContainerStatus

use of org.apache.hadoop.yarn.api.records.ContainerStatus in project hadoop by apache.

the class TestNodeManagerShutdown method startContainer.

public static void startContainer(NodeManager nm, ContainerId cId, FileContext localFS, File scriptFileDir, File processStartFile, final int port) throws IOException, YarnException {
    File scriptFile = createUnhaltingScriptFile(cId, scriptFileDir, processStartFile);
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    NodeId nodeId = BuilderUtils.newNodeId(InetAddress.getByName("localhost").getCanonicalHostName(), port);
    URL localResourceUri = URL.fromPath(localFS.makeQualified(new Path(scriptFile.getAbsolutePath())));
    LocalResource localResource = recordFactory.newRecordInstance(LocalResource.class);
    localResource.setResource(localResourceUri);
    localResource.setSize(-1);
    localResource.setVisibility(LocalResourceVisibility.APPLICATION);
    localResource.setType(LocalResourceType.FILE);
    localResource.setTimestamp(scriptFile.lastModified());
    String destinationFile = "dest_file";
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    localResources.put(destinationFile, localResource);
    containerLaunchContext.setLocalResources(localResources);
    List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
    containerLaunchContext.setCommands(commands);
    final InetSocketAddress containerManagerBindAddress = NetUtils.createSocketAddrForHost("127.0.0.1", port);
    UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(cId.toString());
    org.apache.hadoop.security.token.Token<NMTokenIdentifier> nmToken = ConverterUtils.convertFromYarn(nm.getNMContext().getNMTokenSecretManager().createNMToken(cId.getApplicationAttemptId(), nodeId, user), containerManagerBindAddress);
    currentUser.addToken(nmToken);
    ContainerManagementProtocol containerManager = currentUser.doAs(new PrivilegedAction<ContainerManagementProtocol>() {

        @Override
        public ContainerManagementProtocol run() {
            Configuration conf = new Configuration();
            YarnRPC rpc = YarnRPC.create(conf);
            InetSocketAddress containerManagerBindAddress = NetUtils.createSocketAddrForHost("127.0.0.1", port);
            return (ContainerManagementProtocol) rpc.getProxy(ContainerManagementProtocol.class, containerManagerBindAddress, conf);
        }
    });
    StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, TestContainerManager.createContainerToken(cId, 0, nodeId, user, nm.getNMContext().getContainerTokenSecretManager()));
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    List<ContainerId> containerIds = new ArrayList<ContainerId>();
    containerIds.add(cId);
    GetContainerStatusesRequest request = GetContainerStatusesRequest.newInstance(containerIds);
    ContainerStatus containerStatus = containerManager.getContainerStatuses(request).getContainerStatuses().get(0);
    Assert.assertTrue(EnumSet.of(ContainerState.RUNNING, ContainerState.SCHEDULED).contains(containerStatus.getState()));
}

Example 52 with ContainerStatus

use of org.apache.hadoop.yarn.api.records.ContainerStatus in project hadoop by apache.

the class TestNodeStatusUpdater method createContainerStatus.

public static ContainerStatus createContainerStatus(int id, ContainerState containerState) {
    ApplicationId applicationId = ApplicationId.newInstance(0, 1);
    ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 1);
    ContainerId contaierId = ContainerId.newContainerId(applicationAttemptId, id);
    ContainerStatus containerStatus = BuilderUtils.newContainerStatus(contaierId, containerState, "test_containerStatus: id=" + id + ", containerState: " + containerState, 0, Resource.newInstance(1024, 1));
    return containerStatus;
}

Example 53 with ContainerStatus

use of org.apache.hadoop.yarn.api.records.ContainerStatus in project hadoop by apache.

the class TestNodeManagerResync method testNMSentContainerStatusOnResync.

// This is to test when NM gets the resync response from last heart beat, it
// should be able to send the already-sent-via-last-heart-beat container
// statuses again when it re-register with RM.
@Test
public void testNMSentContainerStatusOnResync() throws Exception {
    final ContainerStatus testCompleteContainer = TestNodeStatusUpdater.createContainerStatus(2, ContainerState.COMPLETE);
    final Container container = TestNodeStatusUpdater.getMockContainer(testCompleteContainer);
    NMContainerStatus report = createNMContainerStatus(2, ContainerState.COMPLETE);
    when(container.getNMContainerStatus()).thenReturn(report);
    NodeManager nm = new NodeManager() {

        int registerCount = 0;

        @Override
        protected NodeStatusUpdater createNodeStatusUpdater(Context context, Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
            return new TestNodeStatusUpdaterResync(context, dispatcher, healthChecker, metrics) {

                @Override
                protected ResourceTracker createResourceTracker() {
                    return new MockResourceTracker() {

                        @Override
                        public RegisterNodeManagerResponse registerNodeManager(RegisterNodeManagerRequest request) throws YarnException, IOException {
                            if (registerCount == 0) {
                                // first register, no containers info.
                                try {
                                    Assert.assertEquals(0, request.getNMContainerStatuses().size());
                                } catch (AssertionError error) {
                                    error.printStackTrace();
                                    assertionFailedInThread.set(true);
                                }
                                // put the completed container into the context
                                getNMContext().getContainers().put(testCompleteContainer.getContainerId(), container);
                                getNMContext().getApplications().put(testCompleteContainer.getContainerId().getApplicationAttemptId().getApplicationId(), mock(Application.class));
                            } else {
                                // second register contains the completed container info.
                                List<NMContainerStatus> statuses = request.getNMContainerStatuses();
                                try {
                                    Assert.assertEquals(1, statuses.size());
                                    Assert.assertEquals(testCompleteContainer.getContainerId(), statuses.get(0).getContainerId());
                                } catch (AssertionError error) {
                                    error.printStackTrace();
                                    assertionFailedInThread.set(true);
                                }
                            }
                            registerCount++;
                            return super.registerNodeManager(request);
                        }

                        @Override
                        public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) {
                            // first heartBeat contains the completed container info
                            List<ContainerStatus> statuses = request.getNodeStatus().getContainersStatuses();
                            try {
                                Assert.assertEquals(1, statuses.size());
                                Assert.assertEquals(testCompleteContainer.getContainerId(), statuses.get(0).getContainerId());
                            } catch (AssertionError error) {
                                error.printStackTrace();
                                assertionFailedInThread.set(true);
                            }
                            // notify RESYNC on first heartbeat.
                            return YarnServerBuilderUtils.newNodeHeartbeatResponse(1, NodeAction.RESYNC, null, null, null, null, 1000L);
                        }
                    };
                }
            };
        }
    };
    YarnConfiguration conf = createNMConfig();
    nm.init(conf);
    nm.start();
    try {
        syncBarrier.await();
    } catch (BrokenBarrierException e) {
    }
    Assert.assertFalse(assertionFailedInThread.get());
    nm.stop();
}

Example 54 with ContainerStatus

use of org.apache.hadoop.yarn.api.records.ContainerStatus in project hadoop by apache.

the class RMNodeImpl method findLostContainers.

private List<ContainerStatus> findLostContainers(int numRemoteRunning, List<ContainerStatus> containerStatuses) {
    if (numRemoteRunning >= launchedContainers.size()) {
        return Collections.emptyList();
    }
    Set<ContainerId> nodeContainers = new HashSet<ContainerId>(numRemoteRunning);
    List<ContainerStatus> lostContainers = new ArrayList<ContainerStatus>(launchedContainers.size() - numRemoteRunning);
    for (ContainerStatus remoteContainer : containerStatuses) {
        if (remoteContainer.getState() == ContainerState.RUNNING && remoteContainer.getExecutionType() == ExecutionType.GUARANTEED) {
            nodeContainers.add(remoteContainer.getContainerId());
        }
    }
    Iterator<ContainerId> iter = launchedContainers.iterator();
    while (iter.hasNext()) {
        ContainerId containerId = iter.next();
        if (!nodeContainers.contains(containerId)) {
            String diag = "Container " + containerId + " was running but not reported from " + nodeId;
            LOG.warn(diag);
            lostContainers.add(SchedulerUtils.createAbnormalContainerStatus(containerId, diag));
            iter.remove();
        }
    }
    return lostContainers;
}

Example 55 with ContainerStatus

use of org.apache.hadoop.yarn.api.records.ContainerStatus in project hadoop by apache.

the class RMNodeImpl method handleContainerStatus.

private void handleContainerStatus(List<ContainerStatus> containerStatuses) {
    // Filter the map to only obtain just launched containers and finished
    // containers.
    List<ContainerStatus> newlyLaunchedContainers = new ArrayList<ContainerStatus>();
    List<ContainerStatus> newlyCompletedContainers = new ArrayList<ContainerStatus>();
    int numRemoteRunningContainers = 0;
    for (ContainerStatus remoteContainer : containerStatuses) {
        ContainerId containerId = remoteContainer.getContainerId();
        // more about this container
        if (containersToClean.contains(containerId)) {
            LOG.info("Container " + containerId + " already scheduled for " + "cleanup, no further processing");
            continue;
        }
        ApplicationId containerAppId = containerId.getApplicationAttemptId().getApplicationId();
        if (finishedApplications.contains(containerAppId)) {
            LOG.info("Container " + containerId + " belongs to an application that is already killed," + " no further processing");
            continue;
        } else if (!runningApplications.contains(containerAppId)) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Container " + containerId + " is the first container get launched for application " + containerAppId);
            }
            handleRunningAppOnNode(this, context, containerAppId, nodeId);
        }
        // Process running containers
        if (remoteContainer.getState() == ContainerState.RUNNING || remoteContainer.getState() == ContainerState.SCHEDULED) {
            ++numRemoteRunningContainers;
            if (!launchedContainers.contains(containerId)) {
                // Just launched container. RM knows about it the first time.
                launchedContainers.add(containerId);
                newlyLaunchedContainers.add(remoteContainer);
                // Unregister from containerAllocationExpirer.
                containerAllocationExpirer.unregister(new AllocationExpirationInfo(containerId));
            }
        } else {
            // A finished container
            launchedContainers.remove(containerId);
            if (completedContainers.add(containerId)) {
                newlyCompletedContainers.add(remoteContainer);
            }
            // Unregister from containerAllocationExpirer.
            containerAllocationExpirer.unregister(new AllocationExpirationInfo(containerId));
        }
    }
    List<ContainerStatus> lostContainers = findLostContainers(numRemoteRunningContainers, containerStatuses);
    for (ContainerStatus remoteContainer : lostContainers) {
        ContainerId containerId = remoteContainer.getContainerId();
        if (completedContainers.add(containerId)) {
            newlyCompletedContainers.add(remoteContainer);
        }
    }
    if (newlyLaunchedContainers.size() != 0 || newlyCompletedContainers.size() != 0) {
        nodeUpdateQueue.add(new UpdatedContainerInfo(newlyLaunchedContainers, newlyCompletedContainers));
    }
}