Examples with AllocateResponse org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse used on opensource projects

Example 46 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project weave by continuuity.

the class AMRMClientImpl method allocate.

@Override
public AllocationResponse allocate(float progressIndicator) throws YarnRemoteException {
    AllocateResponse allocateResponse = null;
    ArrayList<ResourceRequest> askList = null;
    ArrayList<ContainerId> releaseList = null;
    AllocateRequest allocateRequest = null;
    try {
        synchronized (this) {
            askList = new ArrayList<ResourceRequest>(ask);
            releaseList = new ArrayList<ContainerId>(release);
            // optimistically clear this collection assuming no RPC failure
            ask.clear();
            release.clear();
            allocateRequest = BuilderUtils.newAllocateRequest(appAttemptId, lastResponseId, progressIndicator, askList, releaseList);
        }
        allocateResponse = rmClient.allocate(allocateRequest);
        AllocationResponse response = AllocationResponses.create(allocateResponse);
        synchronized (this) {
            // update these on successful RPC
            clusterNodeCount = allocateResponse.getNumClusterNodes();
            lastResponseId = response.getResponseId();
            clusterAvailableResources = response.getAvailableResources();
        }
        return response;
    } finally {
        // TODO how to differentiate remote YARN exception vs error in RPC
        if (allocateResponse == null) {
            // Preserve ask and release for next call to allocate()
            synchronized (this) {
                release.addAll(releaseList);
                // synchronized block at the beginning of this method.
                for (ResourceRequest oldAsk : askList) {
                    if (!ask.contains(oldAsk)) {
                        ask.add(oldAsk);
                    }
                }
            }
        }
    }
}

Example 47 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class TestApplicationCleanup method testAppCleanupWhenRMRestartedBeforeAppFinished.

@SuppressWarnings("resource")
@Test(timeout = 60000)
public void testAppCleanupWhenRMRestartedBeforeAppFinished() throws Exception {
    conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    // start RM
    MockRM rm1 = new MockRM(conf, memStore);
    rm1.start();
    MockNM nm1 = new MockNM("127.0.0.1:1234", 1024, rm1.getResourceTrackerService());
    nm1.registerNode();
    MockNM nm2 = new MockNM("127.0.0.1:5678", 1024, rm1.getResourceTrackerService());
    nm2.registerNode();
    // create app and launch the AM
    RMApp app0 = rm1.submitApp(200);
    MockAM am0 = launchAM(app0, rm1, nm1);
    // alloc another container on nm2
    AllocateResponse allocResponse = am0.allocate(Arrays.asList(ResourceRequest.newInstance(Priority.newInstance(1), "*", Resource.newInstance(1024, 0), 1)), null);
    while (null == allocResponse.getAllocatedContainers() || allocResponse.getAllocatedContainers().isEmpty()) {
        nm2.nodeHeartbeat(true);
        allocResponse = am0.allocate(null, null);
        Thread.sleep(1000);
    }
    // start new RM
    MockRM rm2 = new MockRM(conf, memStore);
    rm2.start();
    // nm1/nm2 register to rm2, and do a heartbeat
    nm1.setResourceTrackerService(rm2.getResourceTrackerService());
    nm1.registerNode(Arrays.asList(NMContainerStatus.newInstance(ContainerId.newContainerId(am0.getApplicationAttemptId(), 1), 0, ContainerState.COMPLETE, Resource.newInstance(1024, 1), "", 0, Priority.newInstance(0), 1234)), Arrays.asList(app0.getApplicationId()));
    nm2.setResourceTrackerService(rm2.getResourceTrackerService());
    nm2.registerNode(Arrays.asList(app0.getApplicationId()));
    // assert app state has been saved.
    rm2.waitForState(app0.getApplicationId(), RMAppState.FAILED);
    // wait for application cleanup message received on NM1
    waitForAppCleanupMessageRecved(nm1, app0.getApplicationId());
    // wait for application cleanup message received on NM2
    waitForAppCleanupMessageRecved(nm2, app0.getApplicationId());
    rm1.stop();
    rm2.stop();
}

Example 48 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class TestApplicationMasterLauncher method testallocateBeforeAMRegistration.

@SuppressWarnings("unused")
@Test(timeout = 100000)
public void testallocateBeforeAMRegistration() throws Exception {
    Logger rootLogger = LogManager.getRootLogger();
    boolean thrown = false;
    rootLogger.setLevel(Level.DEBUG);
    MockRM rm = new MockRM();
    rm.start();
    MockNM nm1 = rm.registerNode("h1:1234", 5000);
    RMApp app = rm.submitApp(2000);
    // kick the scheduling
    nm1.nodeHeartbeat(true);
    RMAppAttempt attempt = app.getCurrentAppAttempt();
    MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());
    // request for containers
    int request = 2;
    AllocateResponse ar = null;
    try {
        ar = am.allocate("h1", 1000, request, new ArrayList<ContainerId>());
        Assert.fail();
    } catch (ApplicationMasterNotRegisteredException e) {
    }
    // kick the scheduler
    nm1.nodeHeartbeat(true);
    AllocateResponse amrs = null;
    try {
        amrs = am.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>());
        Assert.fail();
    } catch (ApplicationMasterNotRegisteredException e) {
    }
    am.registerAppAttempt();
    try {
        am.registerAppAttempt(false);
        Assert.fail();
    } catch (Exception e) {
        Assert.assertEquals("Application Master is already registered : " + attempt.getAppAttemptId().getApplicationId(), e.getMessage());
    }
    // Simulate an AM that was disconnected and app attempt was removed
    // (responseMap does not contain attemptid)
    am.unregisterAppAttempt();
    nm1.nodeHeartbeat(attempt.getAppAttemptId(), 1, ContainerState.COMPLETE);
    rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FINISHED);
    try {
        amrs = am.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>());
        Assert.fail();
    } catch (ApplicationAttemptNotFoundException e) {
    }
}

Example 49 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class TestApplicationMasterService method testPriorityInAllocatedResponse.

@Test(timeout = 300000)
public void testPriorityInAllocatedResponse() throws Exception {
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    // Set Max Application Priority as 10
    conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY, 10);
    MockRM rm = new MockRM(conf);
    rm.start();
    // Register node1
    MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
    // Submit an application
    Priority appPriority1 = Priority.newInstance(5);
    RMApp app1 = rm.submitApp(2048, appPriority1);
    nm1.nodeHeartbeat(true);
    RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
    MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
    am1.registerAppAttempt();
    AllocateRequestPBImpl allocateRequest = new AllocateRequestPBImpl();
    List<ContainerId> release = new ArrayList<ContainerId>();
    List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
    allocateRequest.setReleaseList(release);
    allocateRequest.setAskList(ask);
    AllocateResponse response1 = am1.allocate(allocateRequest);
    Assert.assertEquals(appPriority1, response1.getApplicationPriority());
    // Change the priority of App1 to 8
    Priority appPriority2 = Priority.newInstance(8);
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(app1.getUser());
    rm.getRMAppManager().updateApplicationPriority(ugi, app1.getApplicationId(), appPriority2);
    AllocateResponse response2 = am1.allocate(allocateRequest);
    Assert.assertEquals(appPriority2, response2.getApplicationPriority());
    rm.stop();
}

Example 50 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class TestApplicationMasterService method testAllocateAfterUnregister.

@Test(timeout = 1200000)
public void testAllocateAfterUnregister() throws Exception {
    MyResourceManager rm = new MyResourceManager(conf);
    rm.start();
    DrainDispatcher rmDispatcher = (DrainDispatcher) rm.getRMContext().getDispatcher();
    // Register node1
    MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
    // Submit an application
    RMApp app1 = rm.submitApp(2048);
    nm1.nodeHeartbeat(true);
    RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
    MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
    am1.registerAppAttempt();
    // unregister app attempt
    FinishApplicationMasterRequest req = FinishApplicationMasterRequest.newInstance(FinalApplicationStatus.KILLED, "", "");
    am1.unregisterAppAttempt(req, false);
    // request container after unregister
    am1.addRequests(new String[] { "127.0.0.1" }, GB, 1, 1);
    AllocateResponse alloc1Response = am1.schedule();
    nm1.nodeHeartbeat(true);
    rmDispatcher.await();
    alloc1Response = am1.schedule();
    Assert.assertEquals(0, alloc1Response.getAllocatedContainers().size());
}