Examples with ResourceRequest org.apache.hadoop.yarn.api.records.ResourceRequest used on opensource projects

Example 16 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class FiCaSchedulerApp method getAllocation.

/**
   * This method produces an Allocation that includes the current view
   * of the resources that will be allocated to and preempted from this
   * application.
   *
   * @param resourceCalculator resourceCalculator
   * @param clusterResource clusterResource
   * @param minimumAllocation minimumAllocation
   * @return an allocation
   */
public Allocation getAllocation(ResourceCalculator resourceCalculator, Resource clusterResource, Resource minimumAllocation) {
    try {
        writeLock.lock();
        Set<ContainerId> currentContPreemption = Collections.unmodifiableSet(new HashSet<ContainerId>(containersToPreempt));
        containersToPreempt.clear();
        Resource tot = Resource.newInstance(0, 0);
        for (ContainerId c : currentContPreemption) {
            Resources.addTo(tot, liveContainers.get(c).getContainer().getResource());
        }
        int numCont = (int) Math.ceil(Resources.divide(rc, clusterResource, tot, minimumAllocation));
        ResourceRequest rr = ResourceRequest.newBuilder().priority(Priority.UNDEFINED).resourceName(ResourceRequest.ANY).capability(minimumAllocation).numContainers(numCont).build();
        List<Container> newlyAllocatedContainers = pullNewlyAllocatedContainers();
        List<Container> newlyIncreasedContainers = pullNewlyIncreasedContainers();
        List<Container> newlyDecreasedContainers = pullNewlyDecreasedContainers();
        List<Container> newlyPromotedContainers = pullNewlyPromotedContainers();
        List<Container> newlyDemotedContainers = pullNewlyDemotedContainers();
        List<NMToken> updatedNMTokens = pullUpdatedNMTokens();
        Resource headroom = getHeadroom();
        setApplicationHeadroomForMetrics(headroom);
        return new Allocation(newlyAllocatedContainers, headroom, null, currentContPreemption, Collections.singletonList(rr), updatedNMTokens, newlyIncreasedContainers, newlyDecreasedContainers, newlyPromotedContainers, newlyDemotedContainers);
    } finally {
        writeLock.unlock();
    }
}

Example 17 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class TestWorkPreservingRMRestart method testUAMRecoveryOnRMWorkPreservingRestart.

@Test(timeout = 600000)
public void testUAMRecoveryOnRMWorkPreservingRestart() throws Exception {
    conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    // start RM
    rm1 = new MockRM(conf, memStore);
    rm1.start();
    MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
    nm1.registerNode();
    // create app and launch the UAM
    RMApp app0 = rm1.submitApp(200, true);
    MockAM am0 = MockRM.launchUAM(app0, rm1, nm1);
    am0.registerAppAttempt();
    // Allocate containers to UAM
    int numContainers = 2;
    am0.allocate("127.0.0.1", 1000, numContainers, new ArrayList<ContainerId>());
    nm1.nodeHeartbeat(true);
    List<Container> conts = am0.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers();
    Assert.assertTrue(conts.isEmpty());
    while (conts.size() == 0) {
        nm1.nodeHeartbeat(true);
        conts.addAll(am0.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers());
        Thread.sleep(500);
    }
    Assert.assertFalse(conts.isEmpty());
    // start new RM
    rm2 = new MockRM(conf, memStore);
    rm2.start();
    rm2.waitForState(app0.getApplicationId(), RMAppState.ACCEPTED);
    rm2.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.LAUNCHED);
    // recover app
    nm1.setResourceTrackerService(rm2.getResourceTrackerService());
    RMApp recoveredApp = rm2.getRMContext().getRMApps().get(app0.getApplicationId());
    NMContainerStatus container1 = TestRMRestart.createNMContainerStatus(am0.getApplicationAttemptId(), 1, ContainerState.RUNNING);
    NMContainerStatus container2 = TestRMRestart.createNMContainerStatus(am0.getApplicationAttemptId(), 2, ContainerState.RUNNING);
    nm1.registerNode(Arrays.asList(container1, container2), null);
    // Wait for RM to settle down on recovering containers;
    waitForNumContainersToRecover(2, rm2, am0.getApplicationAttemptId());
    // retry registerApplicationMaster() after RM restart.
    am0.setAMRMProtocol(rm2.getApplicationMasterService(), rm2.getRMContext());
    am0.registerAppAttempt(true);
    // Check if UAM is correctly recovered on restart
    rm2.waitForState(app0.getApplicationId(), RMAppState.RUNNING);
    rm2.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.RUNNING);
    // Check if containers allocated to UAM are recovered
    Map<ApplicationId, SchedulerApplication> schedulerApps = ((AbstractYarnScheduler) rm2.getResourceScheduler()).getSchedulerApplications();
    SchedulerApplication schedulerApp = schedulerApps.get(recoveredApp.getApplicationId());
    SchedulerApplicationAttempt schedulerAttempt = schedulerApp.getCurrentAppAttempt();
    Assert.assertEquals(numContainers, schedulerAttempt.getLiveContainers().size());
    // Check if UAM is able to heart beat
    Assert.assertNotNull(am0.doHeartbeat());
    // Complete the UAM
    am0.unregisterAppAttempt(false);
    rm2.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.FINISHED);
    rm2.waitForState(app0.getApplicationId(), RMAppState.FINISHED);
    Assert.assertEquals(FinalApplicationStatus.SUCCEEDED, recoveredApp.getFinalApplicationStatus());
    // Restart RM once more to check UAM is not re-run
    MockRM rm3 = new MockRM(conf, memStore);
    rm3.start();
    recoveredApp = rm3.getRMContext().getRMApps().get(app0.getApplicationId());
    Assert.assertEquals(RMAppState.FINISHED, recoveredApp.getState());
}

Example 18 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class TestRMAppAttemptTransitions method testScheduleTransitionReplaceAMContainerRequestWithDefaults.

@SuppressWarnings("unchecked")
@Test
public void testScheduleTransitionReplaceAMContainerRequestWithDefaults() {
    YarnScheduler mockScheduler = mock(YarnScheduler.class);
    when(mockScheduler.allocate(any(ApplicationAttemptId.class), any(List.class), any(List.class), any(List.class), any(List.class), any(ContainerUpdates.class))).thenAnswer(new Answer<Allocation>() {

        @SuppressWarnings("rawtypes")
        @Override
        public Allocation answer(InvocationOnMock invocation) throws Throwable {
            ResourceRequest rr = (ResourceRequest) ((List) invocation.getArguments()[1]).get(0);
            // capacity shouldn't changed
            assertEquals(Resource.newInstance(3333, 1), rr.getCapability());
            assertEquals("label-expression", rr.getNodeLabelExpression());
            // priority, #container, relax-locality will be changed
            assertEquals(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, rr.getPriority());
            assertEquals(1, rr.getNumContainers());
            assertEquals(ResourceRequest.ANY, rr.getResourceName());
            // just return an empty allocation
            List l = new ArrayList();
            Set s = new HashSet();
            return new Allocation(l, Resources.none(), s, s, l);
        }
    });
    // create an attempt.
    applicationAttempt = new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), spyRMContext, scheduler, masterService, submissionContext, new Configuration(), ResourceRequest.newInstance(Priority.UNDEFINED, "host1", Resource.newInstance(3333, 1), 3, false, "label-expression"), application);
    new RMAppAttemptImpl.ScheduleTransition().transition((RMAppAttemptImpl) applicationAttempt, null);
}

Example 19 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class TestFifoScheduler method testBlackListNodes.

@Test(timeout = 50000)
public void testBlackListNodes() throws Exception {
    Configuration conf = new Configuration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
    MockRM rm = new MockRM(conf);
    rm.start();
    FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler();
    int rack_num_0 = 0;
    int rack_num_1 = 1;
    // Add 4 nodes in 2 racks
    // host_0_0 in rack0
    String host_0_0 = "127.0.0.1";
    RMNode n1 = MockNodes.newNodeInfo(rack_num_0, MockNodes.newResource(4 * GB), 1, host_0_0);
    fs.handle(new NodeAddedSchedulerEvent(n1));
    // host_0_1 in rack0
    String host_0_1 = "127.0.0.2";
    RMNode n2 = MockNodes.newNodeInfo(rack_num_0, MockNodes.newResource(4 * GB), 1, host_0_1);
    fs.handle(new NodeAddedSchedulerEvent(n2));
    // host_1_0 in rack1
    String host_1_0 = "127.0.0.3";
    RMNode n3 = MockNodes.newNodeInfo(rack_num_1, MockNodes.newResource(4 * GB), 1, host_1_0);
    fs.handle(new NodeAddedSchedulerEvent(n3));
    // host_1_1 in rack1
    String host_1_1 = "127.0.0.4";
    RMNode n4 = MockNodes.newNodeInfo(rack_num_1, MockNodes.newResource(4 * GB), 1, host_1_1);
    fs.handle(new NodeAddedSchedulerEvent(n4));
    // Add one application
    ApplicationId appId1 = BuilderUtils.newApplicationId(100, 1);
    ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId(appId1, 1);
    createMockRMApp(appAttemptId1, rm.getRMContext());
    SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId1, "queue", "user");
    fs.handle(appEvent);
    SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId1, false);
    fs.handle(attemptEvent);
    List<ContainerId> emptyId = new ArrayList<ContainerId>();
    List<ResourceRequest> emptyAsk = new ArrayList<ResourceRequest>();
    // Allow rack-locality for rack_1, but blacklist host_1_0
    // Set up resource requests
    // Ask for a 1 GB container for app 1
    List<ResourceRequest> ask1 = new ArrayList<ResourceRequest>();
    ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), "rack1", BuilderUtils.newResource(GB, 1), 1, RMNodeLabelsManager.NO_LABEL));
    ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), ResourceRequest.ANY, BuilderUtils.newResource(GB, 1), 1, RMNodeLabelsManager.NO_LABEL));
    fs.allocate(appAttemptId1, ask1, emptyId, Collections.singletonList(host_1_0), null, NULL_UPDATE_REQUESTS);
    // Trigger container assignment
    fs.handle(new NodeUpdateSchedulerEvent(n3));
    // Get the allocation for the application and verify no allocation on
    // blacklist node
    Allocation allocation1 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, NULL_UPDATE_REQUESTS);
    Assert.assertEquals("allocation1", 0, allocation1.getContainers().size());
    // verify host_1_1 can get allocated as not in blacklist
    fs.handle(new NodeUpdateSchedulerEvent(n4));
    Allocation allocation2 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, NULL_UPDATE_REQUESTS);
    Assert.assertEquals("allocation2", 1, allocation2.getContainers().size());
    List<Container> containerList = allocation2.getContainers();
    for (Container container : containerList) {
        Assert.assertEquals("Container is allocated on n4", container.getNodeId(), n4.getNodeID());
    }
    // Ask for a 1 GB container again for app 1
    List<ResourceRequest> ask2 = new ArrayList<ResourceRequest>();
    // this time, rack0 is also in blacklist, so only host_1_1 is available to
    // be assigned
    ask2.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), ResourceRequest.ANY, BuilderUtils.newResource(GB, 1), 1));
    fs.allocate(appAttemptId1, ask2, emptyId, Collections.singletonList("rack0"), null, NULL_UPDATE_REQUESTS);
    // verify n1 is not qualified to be allocated
    fs.handle(new NodeUpdateSchedulerEvent(n1));
    Allocation allocation3 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, NULL_UPDATE_REQUESTS);
    Assert.assertEquals("allocation3", 0, allocation3.getContainers().size());
    // verify n2 is not qualified to be allocated
    fs.handle(new NodeUpdateSchedulerEvent(n2));
    Allocation allocation4 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, NULL_UPDATE_REQUESTS);
    Assert.assertEquals("allocation4", 0, allocation4.getContainers().size());
    // verify n3 is not qualified to be allocated
    fs.handle(new NodeUpdateSchedulerEvent(n3));
    Allocation allocation5 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, NULL_UPDATE_REQUESTS);
    Assert.assertEquals("allocation5", 0, allocation5.getContainers().size());
    fs.handle(new NodeUpdateSchedulerEvent(n4));
    Allocation allocation6 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, NULL_UPDATE_REQUESTS);
    Assert.assertEquals("allocation6", 1, allocation6.getContainers().size());
    containerList = allocation6.getContainers();
    for (Container container : containerList) {
        Assert.assertEquals("Container is allocated on n4", container.getNodeId(), n4.getNodeID());
    }
    rm.stop();
}

Example 20 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class TestFifoScheduler method createResourceRequest.

private ResourceRequest createResourceRequest(int memory, String host, int priority, int numContainers) {
    ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class);
    request.setCapability(Resources.createResource(memory));
    request.setResourceName(host);
    request.setNumContainers(numContainers);
    Priority prio = recordFactory.newRecordInstance(Priority.class);
    prio.setPriority(priority);
    request.setPriority(prio);
    return request;
}