Examples with ContainerRequest org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest used on opensource projects

Example 46 with ContainerRequest

use of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest in project hadoop by apache.

the class TestAMRMClient method testAllocRequestId.

private void testAllocRequestId(final AMRMClientImpl<ContainerRequest> amClient) throws YarnException, IOException {
    // setup container request
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority, 1));
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority, 1));
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority, 2));
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority, 2));
    amClient.removeContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    amClient.removeContainerRequest(new ContainerRequest(capability, nodes, racks, priority, 1));
    amClient.removeContainerRequest(new ContainerRequest(capability, nodes, racks, priority, 2));
    assertNumContainers(amClient, 0, 1, 1, 1, 9, 0);
    assertNumContainers(amClient, 1, 1, 1, 1, 9, 0);
    assertNumContainers(amClient, 2, 1, 1, 1, 9, 0);
    int containersRequestedAny = 3;
    // RM should allocate container within 2 calls to allocate()
    List<Container> allocatedContainers = new ArrayList<>();
    int iterationsLeft = 5;
    Set<ContainerId> releases = new TreeSet<ContainerId>();
    while (allocatedContainers.size() < containersRequestedAny && iterationsLeft-- > 0) {
        AllocateResponse allocResponse = amClient.allocate(0.1f);
        assertEquals(0, amClient.ask.size());
        assertEquals(0, amClient.release.size());
        allocatedContainers.addAll(allocResponse.getAllocatedContainers());
        for (Container container : allocResponse.getAllocatedContainers()) {
            ContainerId rejectContainerId = container.getId();
            releases.add(rejectContainerId);
            amClient.releaseAssignedContainer(rejectContainerId);
        }
        if (allocatedContainers.size() < containersRequestedAny) {
            // let NM heartbeat to RM and trigger allocations
            triggerSchedulingWithNMHeartBeat();
        }
    }
    assertEquals(containersRequestedAny, allocatedContainers.size());
    Set<Long> expAllocIds = new HashSet<>(Arrays.asList(Long.valueOf(0), Long.valueOf(1), Long.valueOf(2)));
    Set<Long> actAllocIds = new HashSet<>();
    for (Container ac : allocatedContainers) {
        actAllocIds.add(Long.valueOf(ac.getAllocationRequestId()));
    }
    assertEquals(expAllocIds, actAllocIds);
    assertEquals(3, amClient.release.size());
    assertEquals(0, amClient.ask.size());
    waitForContainerCompletion(3, amClient, releases);
}

Example 47 with ContainerRequest

use of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest in project hadoop by apache.

the class TestAMRMClient method initAMRMClientAndTest.

private void initAMRMClientAndTest(boolean useAllocReqId) throws YarnException, IOException {
    AMRMClient<ContainerRequest> amClient = null;
    try {
        // start am rm client
        amClient = AMRMClient.<ContainerRequest>createAMRMClient();
        //setting an instance NMTokenCache
        amClient.setNMTokenCache(new NMTokenCache());
        //asserting we are not using the singleton instance cache
        Assert.assertNotSame(NMTokenCache.getSingleton(), amClient.getNMTokenCache());
        amClient.init(conf);
        amClient.start();
        amClient.registerApplicationMaster("Host", 10000, "");
        if (useAllocReqId) {
            testAllocRequestId((AMRMClientImpl<ContainerRequest>) amClient);
        } else {
            testAllocation((AMRMClientImpl<ContainerRequest>) amClient);
        }
        amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null);
    } finally {
        if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
            amClient.stop();
        }
    }
}

Example 48 with ContainerRequest

use of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest in project asterixdb by apache.

the class AsterixApplicationMaster method requestResources.

/**
     * @param c
     *            The cluster exception to attempt to alocate with the RM
     * @throws YarnException
     */
private void requestResources(Cluster c) throws YarnException, UnknownHostException {
    //set memory
    if (c.getCcContainerMem() != null) {
        ccMem = Integer.parseInt(c.getCcContainerMem());
    } else {
        ccMem = CC_MEMORY_MBS_DEFAULT;
    }
    if (c.getNcContainerMem() != null) {
        ncMem = Integer.parseInt(c.getNcContainerMem());
    } else {
        ncMem = CC_MEMORY_MBS_DEFAULT;
    }
    //request CC
    int numNodes = 0;
    ContainerRequest ccAsk = hostToRequest(cC.getClusterIp(), true);
    resourceManager.addContainerRequest(ccAsk);
    LOG.info("Asked for CC: " + Arrays.toString(ccAsk.getNodes().toArray()));
    numNodes++;
    //now we wait to be given the CC before starting the NCs...
    //we will wait a minute.
    int deathClock = 60;
    while (ccUp.get() == false && deathClock > 0) {
        try {
            Thread.sleep(1000);
        } catch (InterruptedException ex) {
            LOG.debug(ex);
        }
        --deathClock;
    }
    if (deathClock == 0 && ccUp.get() == false) {
        throw new YarnException("Couldn't allocate container for CC. Abort!");
    }
    LOG.info("Waiting for CC process to start");
    // is there a good way to do this? maybe try opening a socket to it...
    try {
        Thread.sleep(10000);
    } catch (InterruptedException ex) {
        LOG.debug(ex);
    }
    //request NCs
    for (Node n : c.getNode()) {
        resourceManager.addContainerRequest(hostToRequest(n.getClusterIp(), false));
        LOG.info("Asked for NC: " + n.getClusterIp());
        numNodes++;
        synchronized (pendingNCs) {
            pendingNCs.add(n);
        }
    }
    LOG.info("Requested all NCs and CCs. Wait for things to settle!");
    numRequestedContainers.set(numNodes);
    numTotalContainers = numNodes;
    doneAllocating = true;
}

Example 49 with ContainerRequest

use of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest in project apex-core by apache.

the class BlacklistBasedResourceRequestHandler method recreateContainerRequest.

private void recreateContainerRequest(Map<StreamingContainerAgent.ContainerStartRequest, MutablePair<Integer, ContainerRequest>> requestedResources, int loopCounter, ResourceRequestHandler resourceRequestor, List<ContainerRequest> removedContainerRequests) {
    for (Map.Entry<StreamingContainerAgent.ContainerStartRequest, MutablePair<Integer, ContainerRequest>> entry : requestedResources.entrySet()) {
        if ((loopCounter - entry.getValue().getKey()) > NUMBER_MISSED_HEARTBEATS) {
            StreamingContainerAgent.ContainerStartRequest csr = entry.getKey();
            removedContainerRequests.add(entry.getValue().getRight());
            ContainerRequest cr = resourceRequestor.createContainerRequest(csr, false);
            if (cr.getNodes() != null && !cr.getNodes().isEmpty()) {
                addHostSpecificRequest(csr, cr);
            } else {
                otherContainerRequests.put(cr, csr);
            }
        }
    }
}

Example 50 with ContainerRequest

use of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest in project apex-core by apache.

the class BlacklistBasedResourceRequestHandler method reissueContainerRequests.

@Override
public void reissueContainerRequests(AMRMClient<ContainerRequest> amRmClient, Map<StreamingContainerAgent.ContainerStartRequest, MutablePair<Integer, ContainerRequest>> requestedResources, int loopCounter, ResourceRequestHandler resourceRequestor, List<ContainerRequest> containerRequests, List<ContainerRequest> removedContainerRequests) {
    if (!requestedResources.isEmpty()) {
        // Check if any requests timed out, create new requests in that case
        recreateContainerRequest(requestedResources, loopCounter, resourceRequestor, removedContainerRequests);
    }
    // Issue all host specific requests first
    if (!hostSpecificRequestsMap.isEmpty()) {
        LOG.info("Issue Host specific requests first");
        // Blacklist all the nodes and issue request for host specific
        Entry<String, List<ContainerRequest>> set = hostSpecificRequestsMap.entrySet().iterator().next();
        List<ContainerRequest> requests = set.getValue();
        List<String> blacklistNodes = resourceRequestor.getNodesExceptHost(requests.get(0).getNodes());
        amRmClient.updateBlacklist(blacklistNodes, requests.get(0).getNodes());
        blacklistedNodesForHostSpecificRequests = blacklistNodes;
        LOG.info("Sending {} request(s) after blacklisting all nodes other than {}", requests.size(), requests.get(0).getNodes());
        for (ContainerRequest cr : requests) {
            ContainerStartRequest csr = hostSpecificRequests.get(cr);
            ContainerRequest newCr = new ContainerRequest(cr.getCapability(), null, null, cr.getPriority());
            MutablePair<Integer, ContainerRequest> pair = new MutablePair<>(loopCounter, newCr);
            requestedResources.put(csr, pair);
            containerRequests.add(newCr);
            hostSpecificRequests.remove(cr);
        }
        hostSpecificRequestsMap.remove(set.getKey());
    } else {
        if (blacklistedNodesForHostSpecificRequests != null) {
            // Remove the blacklisted nodes during host specific requests
            LOG.debug("All requests done.. Removing nodes from blacklist {}", blacklistedNodesForHostSpecificRequests);
            amRmClient.updateBlacklist(null, blacklistedNodesForHostSpecificRequests);
            blacklistedNodesForHostSpecificRequests = null;
        }
        // Proceed with other requests after host specific requests are done
        if (!otherContainerRequests.isEmpty()) {
            for (Entry<ContainerRequest, ContainerStartRequest> entry : otherContainerRequests.entrySet()) {
                ContainerRequest cr = entry.getKey();
                ContainerStartRequest csr = entry.getValue();
                MutablePair<Integer, ContainerRequest> pair = new MutablePair<>(loopCounter, cr);
                requestedResources.put(csr, pair);
                containerRequests.add(cr);
            }
            otherContainerRequests.clear();
        }
    }
}