Examples with Container org.apache.hadoop.yarn.api.records.Container used on opensource projects

Example 91 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class DistributedSchedulingAllocateRequestPBImpl method initAllocatedContainers.

private void initAllocatedContainers() {
    DistributedSchedulingAllocateRequestProtoOrBuilder p = viaProto ? proto : builder;
    List<ContainerProto> list = p.getAllocatedContainersList();
    this.containers = new ArrayList<Container>();
    for (ContainerProto c : list) {
        this.containers.add(convertFromProtoFormat(c));
    }
}

Example 92 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class OpportunisticContainerAllocator method allocateContainersInternal.

private void allocateContainersInternal(long rmIdentifier, AllocationParams appParams, ContainerIdGenerator idCounter, Set<String> blacklist, ApplicationAttemptId id, Map<String, RemoteNode> allNodes, String userName, Map<Resource, List<Container>> containers, ResourceRequest anyAsk) throws YarnException {
    int toAllocate = anyAsk.getNumContainers() - (containers.isEmpty() ? 0 : containers.get(anyAsk.getCapability()).size());
    List<RemoteNode> nodesForScheduling = new ArrayList<>();
    for (Entry<String, RemoteNode> nodeEntry : allNodes.entrySet()) {
        // Do not use blacklisted nodes for scheduling.
        if (blacklist.contains(nodeEntry.getKey())) {
            continue;
        }
        nodesForScheduling.add(nodeEntry.getValue());
    }
    if (nodesForScheduling.isEmpty()) {
        LOG.warn("No nodes available for allocating opportunistic containers. [" + "allNodes=" + allNodes + ", " + "blacklist=" + blacklist + "]");
        return;
    }
    int numAllocated = 0;
    int nextNodeToSchedule = 0;
    for (int numCont = 0; numCont < toAllocate; numCont++) {
        nextNodeToSchedule++;
        nextNodeToSchedule %= nodesForScheduling.size();
        RemoteNode node = nodesForScheduling.get(nextNodeToSchedule);
        Container container = buildContainer(rmIdentifier, appParams, idCounter, anyAsk, id, userName, node);
        List<Container> cList = containers.get(anyAsk.getCapability());
        if (cList == null) {
            cList = new ArrayList<>();
            containers.put(anyAsk.getCapability(), cList);
        }
        cList.add(container);
        numAllocated++;
        LOG.info("Allocated [" + container.getId() + "] as opportunistic.");
    }
    LOG.info("Allocated " + numAllocated + " opportunistic containers.");
}

Example 93 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class OpportunisticContainerContext method matchAllocationToOutstandingRequest.

/**
   * This method matches a returned list of Container Allocations to any
   * outstanding OPPORTUNISTIC ResourceRequest.
   * @param capability Capability
   * @param allocatedContainers Allocated Containers
   */
public void matchAllocationToOutstandingRequest(Resource capability, List<Container> allocatedContainers) {
    for (Container c : allocatedContainers) {
        SchedulerRequestKey schedulerKey = SchedulerRequestKey.extractFrom(c);
        Map<Resource, ResourceRequest> asks = outstandingOpReqs.get(schedulerKey);
        if (asks == null) {
            continue;
        }
        ResourceRequest rr = asks.get(capability);
        if (rr != null) {
            rr.setNumContainers(rr.getNumContainers() - 1);
            if (rr.getNumContainers() == 0) {
                asks.remove(capability);
            }
        }
    }
}

Example 94 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class MockResourceManagerFacade method allocate.

@SuppressWarnings("deprecation")
@Override
public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException {
    if (request.getAskList() != null && request.getAskList().size() > 0 && request.getReleaseList() != null && request.getReleaseList().size() > 0) {
        Assert.fail("The mock RM implementation does not support receiving " + "askList and releaseList in the same heartbeat");
    }
    String amrmToken = getAppIdentifier();
    ArrayList<Container> containerList = new ArrayList<Container>();
    if (request.getAskList() != null) {
        for (ResourceRequest rr : request.getAskList()) {
            for (int i = 0; i < rr.getNumContainers(); i++) {
                ContainerId containerId = ContainerId.newInstance(getApplicationAttemptId(1), containerIndex.incrementAndGet());
                Container container = Records.newRecord(Container.class);
                container.setId(containerId);
                container.setPriority(rr.getPriority());
                // We don't use the node for running containers in the test cases. So
                // it is OK to hard code it to some dummy value
                NodeId nodeId = NodeId.newInstance(!Strings.isNullOrEmpty(rr.getResourceName()) ? rr.getResourceName() : "dummy", 1000);
                container.setNodeId(nodeId);
                container.setResource(rr.getCapability());
                containerList.add(container);
                synchronized (applicationContainerIdMap) {
                    // Keep track of the containers returned to this application. We
                    // will need it in future
                    Assert.assertTrue("The application id is Not registered before allocate(): " + amrmToken, applicationContainerIdMap.containsKey(amrmToken));
                    List<ContainerId> ids = applicationContainerIdMap.get(amrmToken);
                    ids.add(containerId);
                    this.allocatedContainerMap.put(containerId, container);
                }
            }
        }
    }
    if (request.getReleaseList() != null && request.getReleaseList().size() > 0) {
        Log.getLog().info("Releasing containers: " + request.getReleaseList().size());
        synchronized (applicationContainerIdMap) {
            Assert.assertTrue("The application id is not registered before allocate(): " + amrmToken, applicationContainerIdMap.containsKey(amrmToken));
            List<ContainerId> ids = applicationContainerIdMap.get(amrmToken);
            for (ContainerId id : request.getReleaseList()) {
                boolean found = false;
                for (ContainerId c : ids) {
                    if (c.equals(id)) {
                        found = true;
                        break;
                    }
                }
                Assert.assertTrue("ContainerId " + id + " being released is not valid for application: " + conf.get("AMRMTOKEN"), found);
                ids.remove(id);
                // Return the released container back to the AM with new fake Ids. The
                // test case does not care about the IDs. The IDs are faked because
                // otherwise the LRM will throw duplication identifier exception. This
                // returning of fake containers is ONLY done for testing purpose - for
                // the test code to get confirmation that the sub-cluster resource
                // managers received the release request
                ContainerId fakeContainerId = ContainerId.newInstance(getApplicationAttemptId(1), containerIndex.incrementAndGet());
                Container fakeContainer = allocatedContainerMap.get(id);
                fakeContainer.setId(fakeContainerId);
                containerList.add(fakeContainer);
            }
        }
    }
    Log.getLog().info("Allocating containers: " + containerList.size() + " for application attempt: " + conf.get("AMRMTOKEN"));
    // Always issue a new AMRMToken as if RM rolled master key
    Token newAMRMToken = Token.newInstance(new byte[0], "", new byte[0], "");
    return AllocateResponse.newInstance(0, new ArrayList<ContainerStatus>(), containerList, new ArrayList<NodeReport>(), null, AMCommand.AM_RESYNC, 1, null, new ArrayList<NMToken>(), newAMRMToken, new ArrayList<UpdatedContainer>());
}

Example 95 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class TestAMRMProxyService method getContainersAndAssert.

private List<Container> getContainersAndAssert(int appId, int numberOfResourceRequests) throws Exception {
    AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class);
    allocateRequest.setResponseId(1);
    List<Container> containers = new ArrayList<Container>(numberOfResourceRequests);
    List<ResourceRequest> askList = new ArrayList<ResourceRequest>(numberOfResourceRequests);
    for (int testAppId = 0; testAppId < numberOfResourceRequests; testAppId++) {
        askList.add(createResourceRequest("test-node-" + Integer.toString(testAppId), 6000, 2, testAppId % 5, 1));
    }
    allocateRequest.setAskList(askList);
    AllocateResponse allocateResponse = allocate(appId, allocateRequest);
    Assert.assertNotNull("allocate() returned null response", allocateResponse);
    Assert.assertNull("new AMRMToken from RM should have been nulled by AMRMProxyService", allocateResponse.getAMRMToken());
    containers.addAll(allocateResponse.getAllocatedContainers());
    // Send max 10 heart beats to receive all the containers. If not, we will
    // fail the test
    int numHeartbeat = 0;
    while (containers.size() < askList.size() && numHeartbeat++ < 10) {
        allocateResponse = allocate(appId, Records.newRecord(AllocateRequest.class));
        Assert.assertNotNull("allocate() returned null response", allocateResponse);
        Assert.assertNull("new AMRMToken from RM should have been nulled by AMRMProxyService", allocateResponse.getAMRMToken());
        containers.addAll(allocateResponse.getAllocatedContainers());
        LOG.info("Number of allocated containers in this request: " + Integer.toString(allocateResponse.getAllocatedContainers().size()));
        LOG.info("Total number of allocated containers: " + Integer.toString(containers.size()));
        Thread.sleep(10);
    }
    // We broadcast the request, the number of containers we received will be
    // higher than we ask
    Assert.assertTrue("The asklist count is not same as response", askList.size() <= containers.size());
    return containers;
}