Examples with StartContainerRequest org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest used on opensource projects

Example 1 with StartContainerRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest in project hadoop by apache.

the class Application method assign.

private synchronized void assign(SchedulerRequestKey schedulerKey, NodeType type, List<Container> containers) throws IOException, YarnException {
    for (Iterator<Container> i = containers.iterator(); i.hasNext(); ) {
        Container container = i.next();
        String host = container.getNodeId().toString();
        if (Resources.equals(requestSpec.get(schedulerKey), container.getResource())) {
            // See which task can use this container
            for (Iterator<Task> t = tasks.get(schedulerKey).iterator(); t.hasNext(); ) {
                Task task = t.next();
                if (task.getState() == State.PENDING && task.canSchedule(type, host)) {
                    NodeManager nodeManager = getNodeManager(host);
                    task.start(nodeManager, container.getId());
                    i.remove();
                    // Track application resource usage
                    Resources.addTo(used, container.getResource());
                    LOG.info("Assigned container (" + container + ") of type " + type + " to task " + task.getTaskId() + " at priority " + schedulerKey.getPriority() + " on node " + nodeManager.getHostName() + ", currently using " + used + " resources");
                    // Update resource requests
                    updateResourceRequests(requests.get(schedulerKey), type, task);
                    // Launch the container
                    StartContainerRequest scRequest = StartContainerRequest.newInstance(createCLC(), container.getContainerToken());
                    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
                    list.add(scRequest);
                    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
                    nodeManager.startContainers(allRequests);
                    break;
                }
            }
        }
    }
}

Example 2 with StartContainerRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest in project hadoop by apache.

the class NodeManager method startContainers.

@Override
public synchronized StartContainersResponse startContainers(StartContainersRequest requests) throws YarnException {
    for (StartContainerRequest request : requests.getStartContainerRequests()) {
        Token containerToken = request.getContainerToken();
        ContainerTokenIdentifier tokenId = null;
        try {
            tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken);
        } catch (IOException e) {
            throw RPCUtil.getRemoteException(e);
        }
        ContainerId containerID = tokenId.getContainerID();
        ApplicationId applicationId = containerID.getApplicationAttemptId().getApplicationId();
        List<Container> applicationContainers = containers.get(applicationId);
        if (applicationContainers == null) {
            applicationContainers = new ArrayList<Container>();
            containers.put(applicationId, applicationContainers);
        }
        // Sanity check
        for (Container container : applicationContainers) {
            if (container.getId().compareTo(containerID) == 0) {
                throw new IllegalStateException("Container " + containerID + " already setup on node " + containerManagerAddress);
            }
        }
        Container container = BuilderUtils.newContainer(containerID, this.nodeId, nodeHttpAddress, // DKDC - Doesn't matter
        tokenId.getResource(), // DKDC - Doesn't matter
        null, // DKDC - Doesn't matter
        null);
        ContainerStatus containerStatus = BuilderUtils.newContainerStatus(container.getId(), ContainerState.NEW, "", -1000, container.getResource());
        applicationContainers.add(container);
        containerStatusMap.put(container, containerStatus);
        Resources.subtractFrom(available, tokenId.getResource());
        Resources.addTo(used, tokenId.getResource());
        if (LOG.isDebugEnabled()) {
            LOG.debug("startContainer:" + " node=" + containerManagerAddress + " application=" + applicationId + " container=" + container + " available=" + available + " used=" + used);
        }
    }
    StartContainersResponse response = StartContainersResponse.newInstance(null, null, null);
    return response;
}

Example 3 with StartContainerRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest in project hadoop by apache.

the class StartContainersRequestPBImpl method addLocalRequestsToProto.

private void addLocalRequestsToProto() {
    maybeInitBuilder();
    builder.clearStartContainerRequest();
    List<StartContainerRequestProto> protoList = new ArrayList<StartContainerRequestProto>();
    for (StartContainerRequest r : this.requests) {
        protoList.add(convertToProtoFormat(r));
    }
    builder.addAllStartContainerRequest(protoList);
}

Example 4 with StartContainerRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest in project hadoop by apache.

the class TestEventFlow method testSuccessfulContainerLaunch.

@Test
public void testSuccessfulContainerLaunch() throws InterruptedException, IOException, YarnException {
    FileContext localFS = FileContext.getLocalFSFileContext();
    localFS.delete(new Path(localDir.getAbsolutePath()), true);
    localFS.delete(new Path(localLogDir.getAbsolutePath()), true);
    localFS.delete(new Path(remoteLogDir.getAbsolutePath()), true);
    localDir.mkdir();
    localLogDir.mkdir();
    remoteLogDir.mkdir();
    YarnConfiguration conf = new YarnConfiguration();
    Context context = new NMContext(new NMContainerTokenSecretManager(conf), new NMTokenSecretManagerInNM(), null, null, new NMNullStateStoreService(), false, conf) {

        @Override
        public int getHttpPort() {
            return 1234;
        }
    };
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "0.0.0.0:" + ServerSocketUtil.getPort(8040, 10));
    ContainerExecutor exec = new DefaultContainerExecutor();
    exec.setConf(conf);
    DeletionService del = new DeletionService(exec);
    Dispatcher dispatcher = new AsyncDispatcher();
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(NodeManager.getNodeHealthScriptRunner(conf), dirsHandler);
    healthChecker.init(conf);
    NodeManagerMetrics metrics = NodeManagerMetrics.create();
    NodeStatusUpdater nodeStatusUpdater = new NodeStatusUpdaterImpl(context, dispatcher, healthChecker, metrics) {

        @Override
        protected ResourceTracker getRMClient() {
            return new LocalRMInterface();
        }

        ;

        @Override
        protected void stopRMProxy() {
            return;
        }

        @Override
        protected void startStatusUpdater() {
            // Don't start any updating thread.
            return;
        }

        @Override
        public long getRMIdentifier() {
            return SIMULATED_RM_IDENTIFIER;
        }
    };
    DummyContainerManager containerManager = new DummyContainerManager(context, exec, del, nodeStatusUpdater, metrics, dirsHandler);
    nodeStatusUpdater.init(conf);
    ((NMContext) context).setContainerManager(containerManager);
    nodeStatusUpdater.start();
    ((NMContext) context).setNodeStatusUpdater(nodeStatusUpdater);
    containerManager.init(conf);
    containerManager.start();
    ContainerLaunchContext launchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    ApplicationId applicationId = ApplicationId.newInstance(0, 0);
    ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 0);
    ContainerId cID = ContainerId.newContainerId(applicationAttemptId, 0);
    String user = "testing";
    StartContainerRequest scRequest = StartContainerRequest.newInstance(launchContext, TestContainerManager.createContainerToken(cID, SIMULATED_RM_IDENTIFIER, context.getNodeId(), user, context.getContainerTokenSecretManager()));
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    BaseContainerManagerTest.waitForContainerState(containerManager, cID, Arrays.asList(ContainerState.RUNNING, ContainerState.SCHEDULED), 20);
    List<ContainerId> containerIds = new ArrayList<ContainerId>();
    containerIds.add(cID);
    StopContainersRequest stopRequest = StopContainersRequest.newInstance(containerIds);
    containerManager.stopContainers(stopRequest);
    BaseContainerManagerTest.waitForContainerState(containerManager, cID, ContainerState.COMPLETE);
    containerManager.stop();
}

Example 5 with StartContainerRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest in project hadoop by apache.

the class AMLauncher method launch.

private void launch() throws IOException, YarnException {
    connect();
    ContainerId masterContainerID = masterContainer.getId();
    ApplicationSubmissionContext applicationContext = application.getSubmissionContext();
    LOG.info("Setting up container " + masterContainer + " for AM " + application.getAppAttemptId());
    ContainerLaunchContext launchContext = createAMContainerLaunchContext(applicationContext, masterContainerID);
    StartContainerRequest scRequest = StartContainerRequest.newInstance(launchContext, masterContainer.getContainerToken());
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    StartContainersResponse response = containerMgrProxy.startContainers(allRequests);
    if (response.getFailedRequests() != null && response.getFailedRequests().containsKey(masterContainerID)) {
        Throwable t = response.getFailedRequests().get(masterContainerID).deSerialize();
        parseAndThrowException(t);
    } else {
        LOG.info("Done launching container " + masterContainer + " for AM " + application.getAppAttemptId());
    }
}