Examples with UpdatedContainer org.apache.hadoop.yarn.api.records.UpdatedContainer used on opensource projects

Example 1 with UpdatedContainer

use of org.apache.hadoop.yarn.api.records.UpdatedContainer in project hadoop by apache.

the class TestOpportunisticContainerAllocation method testPromotionFromAcquired.

@Test(timeout = 60000)
public void testPromotionFromAcquired() throws YarnException, IOException {
    // setup container request
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, null, null, priority2, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
    int oppContainersRequestedAny = amClient.getTable(0).get(priority2, ResourceRequest.ANY, ExecutionType.OPPORTUNISTIC, capability).remoteRequest.getNumContainers();
    assertEquals(1, oppContainersRequestedAny);
    assertEquals(1, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    // RM should allocate container within 2 calls to allocate()
    int allocatedContainerCount = 0;
    Map<ContainerId, Container> allocatedOpportContainers = new HashMap<>();
    int iterationsLeft = 50;
    amClient.getNMTokenCache().clearCache();
    Assert.assertEquals(0, amClient.getNMTokenCache().numberOfTokensInCache());
    HashMap<String, Token> receivedNMTokens = new HashMap<>();
    updateMetrics("Before Opp Allocation");
    while (allocatedContainerCount < oppContainersRequestedAny && iterationsLeft-- > 0) {
        AllocateResponse allocResponse = amClient.allocate(0.1f);
        assertEquals(0, amClient.ask.size());
        assertEquals(0, amClient.release.size());
        allocatedContainerCount += allocResponse.getAllocatedContainers().size();
        for (Container container : allocResponse.getAllocatedContainers()) {
            if (container.getExecutionType() == ExecutionType.OPPORTUNISTIC) {
                allocatedOpportContainers.put(container.getId(), container);
                removeCR(container);
            }
        }
        for (NMToken token : allocResponse.getNMTokens()) {
            String nodeID = token.getNodeId().toString();
            receivedNMTokens.put(nodeID, token.getToken());
        }
        if (allocatedContainerCount < oppContainersRequestedAny) {
            // sleep to let NM's heartbeat to RM and trigger allocations
            sleep(100);
        }
    }
    assertEquals(oppContainersRequestedAny, allocatedContainerCount);
    assertEquals(oppContainersRequestedAny, allocatedOpportContainers.size());
    updateMetrics("After Opp Allocation / Before Promotion");
    try {
        Container c = allocatedOpportContainers.values().iterator().next();
        amClient.requestContainerUpdate(c, UpdateContainerRequest.newInstance(c.getVersion(), c.getId(), ContainerUpdateType.PROMOTE_EXECUTION_TYPE, null, ExecutionType.OPPORTUNISTIC));
        Assert.fail("Should throw Exception..");
    } catch (IllegalArgumentException e) {
        System.out.println("## " + e.getMessage());
        Assert.assertTrue(e.getMessage().contains("target should be GUARANTEED and original should be OPPORTUNISTIC"));
    }
    Container c = allocatedOpportContainers.values().iterator().next();
    amClient.requestContainerUpdate(c, UpdateContainerRequest.newInstance(c.getVersion(), c.getId(), ContainerUpdateType.PROMOTE_EXECUTION_TYPE, null, ExecutionType.GUARANTEED));
    iterationsLeft = 120;
    Map<ContainerId, UpdatedContainer> updatedContainers = new HashMap<>();
    // do a few iterations to ensure RM is not going to send new containers
    while (iterationsLeft-- > 0 && updatedContainers.isEmpty()) {
        // inform RM of rejection
        AllocateResponse allocResponse = amClient.allocate(0.1f);
        // RM did not send new containers because AM does not need any
        if (allocResponse.getUpdatedContainers() != null) {
            for (UpdatedContainer updatedContainer : allocResponse.getUpdatedContainers()) {
                System.out.println("Got update..");
                updatedContainers.put(updatedContainer.getContainer().getId(), updatedContainer);
            }
        }
        if (iterationsLeft > 0) {
            // sleep to make sure NM's heartbeat
            sleep(100);
        }
    }
    updateMetrics("After Promotion");
    assertEquals(1, updatedContainers.size());
    for (ContainerId cId : allocatedOpportContainers.keySet()) {
        Container orig = allocatedOpportContainers.get(cId);
        UpdatedContainer updatedContainer = updatedContainers.get(cId);
        assertNotNull(updatedContainer);
        assertEquals(ExecutionType.GUARANTEED, updatedContainer.getContainer().getExecutionType());
        assertEquals(orig.getResource(), updatedContainer.getContainer().getResource());
        assertEquals(orig.getNodeId(), updatedContainer.getContainer().getNodeId());
        assertEquals(orig.getVersion() + 1, updatedContainer.getContainer().getVersion());
    }
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    amClient.ask.clear();
}

Example 2 with UpdatedContainer

use of org.apache.hadoop.yarn.api.records.UpdatedContainer in project hadoop by apache.

the class ApplicationMasterService method addToUpdatedContainers.

protected void addToUpdatedContainers(AllocateResponse allocateResponse, ContainerUpdateType updateType, List<Container> updatedContainers) {
    if (updatedContainers != null && updatedContainers.size() > 0) {
        ArrayList<UpdatedContainer> containersToSet = new ArrayList<>();
        if (allocateResponse.getUpdatedContainers() != null && !allocateResponse.getUpdatedContainers().isEmpty()) {
            containersToSet.addAll(allocateResponse.getUpdatedContainers());
        }
        for (Container updatedContainer : updatedContainers) {
            containersToSet.add(UpdatedContainer.newInstance(updateType, updatedContainer));
        }
        allocateResponse.setUpdatedContainers(containersToSet);
    }
}

Example 3 with UpdatedContainer

use of org.apache.hadoop.yarn.api.records.UpdatedContainer in project hadoop by apache.

the class AMRMClientImpl method removePendingChangeRequests.

protected void removePendingChangeRequests(List<UpdatedContainer> changedContainers) {
    for (UpdatedContainer changedContainer : changedContainers) {
        ContainerId containerId = changedContainer.getContainer().getId();
        if (pendingChange.get(containerId) == null) {
            continue;
        }
        if (LOG.isDebugEnabled()) {
            LOG.debug("RM has confirmed changed resource allocation for " + "container " + containerId + ". Current resource allocation:" + changedContainer.getContainer().getResource() + ". Remove pending change request:" + pendingChange.get(containerId).getValue());
        }
        pendingChange.remove(containerId);
    }
}

Example 4 with UpdatedContainer

use of org.apache.hadoop.yarn.api.records.UpdatedContainer in project hadoop by apache.

the class AMRMClientImpl method allocate.

@Override
public AllocateResponse allocate(float progressIndicator) throws YarnException, IOException {
    Preconditions.checkArgument(progressIndicator >= 0, "Progress indicator should not be negative");
    AllocateResponse allocateResponse = null;
    List<ResourceRequest> askList = null;
    List<ContainerId> releaseList = null;
    AllocateRequest allocateRequest = null;
    List<String> blacklistToAdd = new ArrayList<String>();
    List<String> blacklistToRemove = new ArrayList<String>();
    Map<ContainerId, SimpleEntry<Container, UpdateContainerRequest>> oldChange = new HashMap<>();
    try {
        synchronized (this) {
            askList = cloneAsks();
            // Save the current change for recovery
            oldChange.putAll(change);
            List<UpdateContainerRequest> updateList = createUpdateList();
            releaseList = new ArrayList<ContainerId>(release);
            // optimistically clear this collection assuming no RPC failure
            ask.clear();
            release.clear();
            change.clear();
            blacklistToAdd.addAll(blacklistAdditions);
            blacklistToRemove.addAll(blacklistRemovals);
            ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(blacklistToAdd, blacklistToRemove);
            allocateRequest = AllocateRequest.newBuilder().responseId(lastResponseId).progress(progressIndicator).askList(askList).resourceBlacklistRequest(blacklistRequest).releaseList(releaseList).updateRequests(updateList).build();
            // clear blacklistAdditions and blacklistRemovals before
            // unsynchronized part
            blacklistAdditions.clear();
            blacklistRemovals.clear();
        }
        try {
            allocateResponse = rmClient.allocate(allocateRequest);
        } catch (ApplicationMasterNotRegisteredException e) {
            LOG.warn("ApplicationMaster is out of sync with ResourceManager," + " hence resyncing.");
            synchronized (this) {
                release.addAll(this.pendingRelease);
                blacklistAdditions.addAll(this.blacklistedNodes);
                for (RemoteRequestsTable remoteRequestsTable : remoteRequests.values()) {
                    @SuppressWarnings("unchecked") Iterator<ResourceRequestInfo<T>> reqIter = remoteRequestsTable.iterator();
                    while (reqIter.hasNext()) {
                        addResourceRequestToAsk(reqIter.next().remoteRequest);
                    }
                }
                change.putAll(this.pendingChange);
            }
            // re register with RM
            registerApplicationMaster();
            allocateResponse = allocate(progressIndicator);
            return allocateResponse;
        }
        synchronized (this) {
            // update these on successful RPC
            clusterNodeCount = allocateResponse.getNumClusterNodes();
            lastResponseId = allocateResponse.getResponseId();
            clusterAvailableResources = allocateResponse.getAvailableResources();
            if (!allocateResponse.getNMTokens().isEmpty()) {
                populateNMTokens(allocateResponse.getNMTokens());
            }
            if (allocateResponse.getAMRMToken() != null) {
                updateAMRMToken(allocateResponse.getAMRMToken());
            }
            if (!pendingRelease.isEmpty() && !allocateResponse.getCompletedContainersStatuses().isEmpty()) {
                removePendingReleaseRequests(allocateResponse.getCompletedContainersStatuses());
            }
            if (!pendingChange.isEmpty()) {
                List<ContainerStatus> completed = allocateResponse.getCompletedContainersStatuses();
                List<UpdatedContainer> changed = new ArrayList<>();
                changed.addAll(allocateResponse.getUpdatedContainers());
                // containers
                for (ContainerStatus status : completed) {
                    ContainerId containerId = status.getContainerId();
                    pendingChange.remove(containerId);
                }
                // remove all pending change requests that have been satisfied
                if (!changed.isEmpty()) {
                    removePendingChangeRequests(changed);
                }
            }
        }
    } finally {
        // TODO how to differentiate remote yarn exception vs error in rpc
        if (allocateResponse == null) {
            // preserve ask and release for next call to allocate()
            synchronized (this) {
                release.addAll(releaseList);
                // synchronized block at the beginning of this method.
                for (ResourceRequest oldAsk : askList) {
                    if (!ask.contains(oldAsk)) {
                        ask.add(oldAsk);
                    }
                }
                // that do not exist in the current change map:
                for (Map.Entry<ContainerId, SimpleEntry<Container, UpdateContainerRequest>> entry : oldChange.entrySet()) {
                    ContainerId oldContainerId = entry.getKey();
                    Container oldContainer = entry.getValue().getKey();
                    UpdateContainerRequest oldupdate = entry.getValue().getValue();
                    if (change.get(oldContainerId) == null) {
                        change.put(oldContainerId, new SimpleEntry<>(oldContainer, oldupdate));
                    }
                }
                blacklistAdditions.addAll(blacklistToAdd);
                blacklistRemovals.addAll(blacklistToRemove);
            }
        }
    }
    return allocateResponse;
}

Example 5 with UpdatedContainer

use of org.apache.hadoop.yarn.api.records.UpdatedContainer in project hadoop by apache.

the class MockResourceManagerFacade method allocate.

@SuppressWarnings("deprecation")
@Override
public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException {
    if (request.getAskList() != null && request.getAskList().size() > 0 && request.getReleaseList() != null && request.getReleaseList().size() > 0) {
        Assert.fail("The mock RM implementation does not support receiving " + "askList and releaseList in the same heartbeat");
    }
    String amrmToken = getAppIdentifier();
    ArrayList<Container> containerList = new ArrayList<Container>();
    if (request.getAskList() != null) {
        for (ResourceRequest rr : request.getAskList()) {
            for (int i = 0; i < rr.getNumContainers(); i++) {
                ContainerId containerId = ContainerId.newInstance(getApplicationAttemptId(1), containerIndex.incrementAndGet());
                Container container = Records.newRecord(Container.class);
                container.setId(containerId);
                container.setPriority(rr.getPriority());
                // We don't use the node for running containers in the test cases. So
                // it is OK to hard code it to some dummy value
                NodeId nodeId = NodeId.newInstance(!Strings.isNullOrEmpty(rr.getResourceName()) ? rr.getResourceName() : "dummy", 1000);
                container.setNodeId(nodeId);
                container.setResource(rr.getCapability());
                containerList.add(container);
                synchronized (applicationContainerIdMap) {
                    // Keep track of the containers returned to this application. We
                    // will need it in future
                    Assert.assertTrue("The application id is Not registered before allocate(): " + amrmToken, applicationContainerIdMap.containsKey(amrmToken));
                    List<ContainerId> ids = applicationContainerIdMap.get(amrmToken);
                    ids.add(containerId);
                    this.allocatedContainerMap.put(containerId, container);
                }
            }
        }
    }
    if (request.getReleaseList() != null && request.getReleaseList().size() > 0) {
        Log.getLog().info("Releasing containers: " + request.getReleaseList().size());
        synchronized (applicationContainerIdMap) {
            Assert.assertTrue("The application id is not registered before allocate(): " + amrmToken, applicationContainerIdMap.containsKey(amrmToken));
            List<ContainerId> ids = applicationContainerIdMap.get(amrmToken);
            for (ContainerId id : request.getReleaseList()) {
                boolean found = false;
                for (ContainerId c : ids) {
                    if (c.equals(id)) {
                        found = true;
                        break;
                    }
                }
                Assert.assertTrue("ContainerId " + id + " being released is not valid for application: " + conf.get("AMRMTOKEN"), found);
                ids.remove(id);
                // Return the released container back to the AM with new fake Ids. The
                // test case does not care about the IDs. The IDs are faked because
                // otherwise the LRM will throw duplication identifier exception. This
                // returning of fake containers is ONLY done for testing purpose - for
                // the test code to get confirmation that the sub-cluster resource
                // managers received the release request
                ContainerId fakeContainerId = ContainerId.newInstance(getApplicationAttemptId(1), containerIndex.incrementAndGet());
                Container fakeContainer = allocatedContainerMap.get(id);
                fakeContainer.setId(fakeContainerId);
                containerList.add(fakeContainer);
            }
        }
    }
    Log.getLog().info("Allocating containers: " + containerList.size() + " for application attempt: " + conf.get("AMRMTOKEN"));
    // Always issue a new AMRMToken as if RM rolled master key
    Token newAMRMToken = Token.newInstance(new byte[0], "", new byte[0], "");
    return AllocateResponse.newInstance(0, new ArrayList<ContainerStatus>(), containerList, new ArrayList<NodeReport>(), null, AMCommand.AM_RESYNC, 1, null, new ArrayList<NMToken>(), newAMRMToken, new ArrayList<UpdatedContainer>());
}