Examples with AllocateRequest org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest used on opensource projects

Example 6 with AllocateRequest

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest in project hadoop by apache.

the class TestApplicationCleanup method testProcessingNMContainerStatusesOnNMRestart.

// The test verifies processing of NMContainerStatuses which are sent during
// NM registration.
// 1. Start the cluster-RM,NM,Submit app with 1024MB,Launch & register AM
// 2. AM sends ResourceRequest for 1 container with memory 2048MB.
// 3. Verify for number of container allocated by RM
// 4. Verify Memory Usage by cluster, it should be 3072. AM memory + requested
// memory. 1024 + 2048=3072
// 5. Re-register NM by sending completed container status
// 6. Verify for Memory Used, it should be 1024
// 7. Send AM heatbeat to RM. Allocated response should contain completed
// container.
@Test(timeout = 60000)
public void testProcessingNMContainerStatusesOnNMRestart() throws Exception {
    conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    // 1. Start the cluster-RM,NM,Submit app with 1024MB,Launch & register AM
    MockRM rm1 = new MockRM(conf, memStore);
    rm1.start();
    int nmMemory = 8192;
    int amMemory = 1024;
    int containerMemory = 2048;
    MockNM nm1 = new MockNM("127.0.0.1:1234", nmMemory, rm1.getResourceTrackerService());
    nm1.registerNode();
    RMApp app0 = rm1.submitApp(amMemory);
    MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1);
    // 2. AM sends ResourceRequest for 1 container with memory 2048MB.
    int noOfContainers = 1;
    List<Container> allocateContainers = am0.allocateAndWaitForContainers(noOfContainers, containerMemory, nm1);
    // 3. Verify for number of container allocated by RM
    Assert.assertEquals(noOfContainers, allocateContainers.size());
    Container container = allocateContainers.get(0);
    nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1, ContainerState.RUNNING);
    nm1.nodeHeartbeat(am0.getApplicationAttemptId(), container.getId().getContainerId(), ContainerState.RUNNING);
    rm1.waitForState(app0.getApplicationId(), RMAppState.RUNNING);
    // 4. Verify Memory Usage by cluster, it should be 3072. AM memory +
    // requested memory. 1024 + 2048=3072
    ResourceScheduler rs = rm1.getRMContext().getScheduler();
    long allocatedMB = rs.getRootQueueMetrics().getAllocatedMB();
    Assert.assertEquals(amMemory + containerMemory, allocatedMB);
    // 5. Re-register NM by sending completed container status
    List<NMContainerStatus> nMContainerStatusForApp = createNMContainerStatusForApp(am0);
    nm1.registerNode(nMContainerStatusForApp, Arrays.asList(app0.getApplicationId()));
    waitForClusterMemory(nm1, rs, amMemory);
    // 6. Verify for Memory Used, it should be 1024
    Assert.assertEquals(amMemory, rs.getRootQueueMetrics().getAllocatedMB());
    // 7. Send AM heatbeat to RM. Allocated response should contain completed
    // container
    AllocateRequest req = AllocateRequest.newInstance(0, 0F, new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>(), null);
    AllocateResponse allocate = am0.allocate(req);
    List<ContainerStatus> completedContainersStatuses = allocate.getCompletedContainersStatuses();
    Assert.assertEquals(noOfContainers, completedContainersStatuses.size());
    // Application clean up should happen Cluster memory used is 0
    nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
    waitForClusterMemory(nm1, rs, 0);
    rm1.stop();
}

Example 7 with AllocateRequest

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest in project hadoop by apache.

the class TestRecordFactory method testPbRecordFactory.

@Test
public void testPbRecordFactory() {
    RecordFactory pbRecordFactory = RecordFactoryPBImpl.get();
    try {
        AllocateResponse response = pbRecordFactory.newRecordInstance(AllocateResponse.class);
        Assert.assertEquals(AllocateResponsePBImpl.class, response.getClass());
    } catch (YarnRuntimeException e) {
        e.printStackTrace();
        Assert.fail("Failed to crete record");
    }
    try {
        AllocateRequest response = pbRecordFactory.newRecordInstance(AllocateRequest.class);
        Assert.assertEquals(AllocateRequestPBImpl.class, response.getClass());
    } catch (YarnRuntimeException e) {
        e.printStackTrace();
        Assert.fail("Failed to crete record");
    }
}

Example 8 with AllocateRequest

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest in project hadoop by apache.

the class TestDistributedScheduler method testDistributedScheduler.

@Test
public void testDistributedScheduler() throws Exception {
    Configuration conf = new Configuration();
    DistributedScheduler distributedScheduler = new DistributedScheduler();
    RequestInterceptor finalReqIntcptr = setup(conf, distributedScheduler);
    registerAM(distributedScheduler, finalReqIntcptr, Arrays.asList(RemoteNode.newInstance(NodeId.newInstance("a", 1), "http://a:1"), RemoteNode.newInstance(NodeId.newInstance("b", 2), "http://b:2")));
    final AtomicBoolean flipFlag = new AtomicBoolean(true);
    Mockito.when(finalReqIntcptr.allocateForDistributedScheduling(Mockito.any(DistributedSchedulingAllocateRequest.class))).thenAnswer(new Answer<DistributedSchedulingAllocateResponse>() {

        @Override
        public DistributedSchedulingAllocateResponse answer(InvocationOnMock invocationOnMock) throws Throwable {
            flipFlag.set(!flipFlag.get());
            if (flipFlag.get()) {
                return createAllocateResponse(Arrays.asList(RemoteNode.newInstance(NodeId.newInstance("c", 3), "http://c:3"), RemoteNode.newInstance(NodeId.newInstance("d", 4), "http://d:4")));
            } else {
                return createAllocateResponse(Arrays.asList(RemoteNode.newInstance(NodeId.newInstance("d", 4), "http://d:4"), RemoteNode.newInstance(NodeId.newInstance("c", 3), "http://c:3")));
            }
        }
    });
    AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class);
    ResourceRequest guaranteedReq = createResourceRequest(ExecutionType.GUARANTEED, 5, "*");
    ResourceRequest opportunisticReq = createResourceRequest(ExecutionType.OPPORTUNISTIC, 4, "*");
    allocateRequest.setAskList(Arrays.asList(guaranteedReq, opportunisticReq));
    // Verify 4 containers were allocated
    AllocateResponse allocateResponse = distributedScheduler.allocate(allocateRequest);
    Assert.assertEquals(4, allocateResponse.getAllocatedContainers().size());
    // Verify equal distribution on hosts a and b, and none on c or d
    Map<NodeId, List<ContainerId>> allocs = mapAllocs(allocateResponse, 4);
    Assert.assertEquals(2, allocs.get(NodeId.newInstance("a", 1)).size());
    Assert.assertEquals(2, allocs.get(NodeId.newInstance("b", 2)).size());
    Assert.assertNull(allocs.get(NodeId.newInstance("c", 3)));
    Assert.assertNull(allocs.get(NodeId.newInstance("d", 4)));
    // New Allocate request
    allocateRequest = Records.newRecord(AllocateRequest.class);
    opportunisticReq = createResourceRequest(ExecutionType.OPPORTUNISTIC, 6, "*");
    allocateRequest.setAskList(Arrays.asList(guaranteedReq, opportunisticReq));
    // Verify 6 containers were allocated
    allocateResponse = distributedScheduler.allocate(allocateRequest);
    Assert.assertEquals(6, allocateResponse.getAllocatedContainers().size());
    // Verify new containers are equally distribution on hosts c and d,
    // and none on a or b
    allocs = mapAllocs(allocateResponse, 6);
    Assert.assertEquals(3, allocs.get(NodeId.newInstance("c", 3)).size());
    Assert.assertEquals(3, allocs.get(NodeId.newInstance("d", 4)).size());
    Assert.assertNull(allocs.get(NodeId.newInstance("a", 1)));
    Assert.assertNull(allocs.get(NodeId.newInstance("b", 2)));
    // Ensure the DistributedScheduler respects the list order..
    // The first request should be allocated to "d" since it is ranked higher
    // The second request should be allocated to "c" since the ranking is
    // flipped on every allocate response.
    allocateRequest = Records.newRecord(AllocateRequest.class);
    opportunisticReq = createResourceRequest(ExecutionType.OPPORTUNISTIC, 1, "*");
    allocateRequest.setAskList(Arrays.asList(guaranteedReq, opportunisticReq));
    allocateResponse = distributedScheduler.allocate(allocateRequest);
    allocs = mapAllocs(allocateResponse, 1);
    Assert.assertEquals(1, allocs.get(NodeId.newInstance("d", 4)).size());
    allocateRequest = Records.newRecord(AllocateRequest.class);
    opportunisticReq = createResourceRequest(ExecutionType.OPPORTUNISTIC, 1, "*");
    allocateRequest.setAskList(Arrays.asList(guaranteedReq, opportunisticReq));
    allocateResponse = distributedScheduler.allocate(allocateRequest);
    allocs = mapAllocs(allocateResponse, 1);
    Assert.assertEquals(1, allocs.get(NodeId.newInstance("c", 3)).size());
    allocateRequest = Records.newRecord(AllocateRequest.class);
    opportunisticReq = createResourceRequest(ExecutionType.OPPORTUNISTIC, 1, "*");
    allocateRequest.setAskList(Arrays.asList(guaranteedReq, opportunisticReq));
    allocateResponse = distributedScheduler.allocate(allocateRequest);
    allocs = mapAllocs(allocateResponse, 1);
    Assert.assertEquals(1, allocs.get(NodeId.newInstance("d", 4)).size());
}

Example 9 with AllocateRequest

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest in project hadoop by apache.

the class AMRMClientImpl method allocate.

@Override
public AllocateResponse allocate(float progressIndicator) throws YarnException, IOException {
    Preconditions.checkArgument(progressIndicator >= 0, "Progress indicator should not be negative");
    AllocateResponse allocateResponse = null;
    List<ResourceRequest> askList = null;
    List<ContainerId> releaseList = null;
    AllocateRequest allocateRequest = null;
    List<String> blacklistToAdd = new ArrayList<String>();
    List<String> blacklistToRemove = new ArrayList<String>();
    Map<ContainerId, SimpleEntry<Container, UpdateContainerRequest>> oldChange = new HashMap<>();
    try {
        synchronized (this) {
            askList = cloneAsks();
            // Save the current change for recovery
            oldChange.putAll(change);
            List<UpdateContainerRequest> updateList = createUpdateList();
            releaseList = new ArrayList<ContainerId>(release);
            // optimistically clear this collection assuming no RPC failure
            ask.clear();
            release.clear();
            change.clear();
            blacklistToAdd.addAll(blacklistAdditions);
            blacklistToRemove.addAll(blacklistRemovals);
            ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(blacklistToAdd, blacklistToRemove);
            allocateRequest = AllocateRequest.newBuilder().responseId(lastResponseId).progress(progressIndicator).askList(askList).resourceBlacklistRequest(blacklistRequest).releaseList(releaseList).updateRequests(updateList).build();
            // clear blacklistAdditions and blacklistRemovals before
            // unsynchronized part
            blacklistAdditions.clear();
            blacklistRemovals.clear();
        }
        try {
            allocateResponse = rmClient.allocate(allocateRequest);
        } catch (ApplicationMasterNotRegisteredException e) {
            LOG.warn("ApplicationMaster is out of sync with ResourceManager," + " hence resyncing.");
            synchronized (this) {
                release.addAll(this.pendingRelease);
                blacklistAdditions.addAll(this.blacklistedNodes);
                for (RemoteRequestsTable remoteRequestsTable : remoteRequests.values()) {
                    @SuppressWarnings("unchecked") Iterator<ResourceRequestInfo<T>> reqIter = remoteRequestsTable.iterator();
                    while (reqIter.hasNext()) {
                        addResourceRequestToAsk(reqIter.next().remoteRequest);
                    }
                }
                change.putAll(this.pendingChange);
            }
            // re register with RM
            registerApplicationMaster();
            allocateResponse = allocate(progressIndicator);
            return allocateResponse;
        }
        synchronized (this) {
            // update these on successful RPC
            clusterNodeCount = allocateResponse.getNumClusterNodes();
            lastResponseId = allocateResponse.getResponseId();
            clusterAvailableResources = allocateResponse.getAvailableResources();
            if (!allocateResponse.getNMTokens().isEmpty()) {
                populateNMTokens(allocateResponse.getNMTokens());
            }
            if (allocateResponse.getAMRMToken() != null) {
                updateAMRMToken(allocateResponse.getAMRMToken());
            }
            if (!pendingRelease.isEmpty() && !allocateResponse.getCompletedContainersStatuses().isEmpty()) {
                removePendingReleaseRequests(allocateResponse.getCompletedContainersStatuses());
            }
            if (!pendingChange.isEmpty()) {
                List<ContainerStatus> completed = allocateResponse.getCompletedContainersStatuses();
                List<UpdatedContainer> changed = new ArrayList<>();
                changed.addAll(allocateResponse.getUpdatedContainers());
                // containers
                for (ContainerStatus status : completed) {
                    ContainerId containerId = status.getContainerId();
                    pendingChange.remove(containerId);
                }
                // remove all pending change requests that have been satisfied
                if (!changed.isEmpty()) {
                    removePendingChangeRequests(changed);
                }
            }
        }
    } finally {
        // TODO how to differentiate remote yarn exception vs error in rpc
        if (allocateResponse == null) {
            // preserve ask and release for next call to allocate()
            synchronized (this) {
                release.addAll(releaseList);
                // synchronized block at the beginning of this method.
                for (ResourceRequest oldAsk : askList) {
                    if (!ask.contains(oldAsk)) {
                        ask.add(oldAsk);
                    }
                }
                // that do not exist in the current change map:
                for (Map.Entry<ContainerId, SimpleEntry<Container, UpdateContainerRequest>> entry : oldChange.entrySet()) {
                    ContainerId oldContainerId = entry.getKey();
                    Container oldContainer = entry.getValue().getKey();
                    UpdateContainerRequest oldupdate = entry.getValue().getValue();
                    if (change.get(oldContainerId) == null) {
                        change.put(oldContainerId, new SimpleEntry<>(oldContainer, oldupdate));
                    }
                }
                blacklistAdditions.addAll(blacklistToAdd);
                blacklistRemovals.addAll(blacklistToRemove);
            }
        }
    }
    return allocateResponse;
}

Example 10 with AllocateRequest

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest in project hadoop by apache.

the class BaseAMRMProxyTest method allocate.

protected AllocateResponse allocate(final int testAppId) throws Exception, YarnException, IOException {
    final AllocateRequest req = Records.newRecord(AllocateRequest.class);
    req.setResponseId(testAppId);
    return allocate(testAppId, req);
}