use of org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode in project hadoop by apache.
the class TestFairScheduler method testNoMoreCpuOnNode.
@Test
public void testNoMoreCpuOnNode() throws IOException {
scheduler.init(conf);
scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 1), 1, "127.0.0.1");
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
scheduler.handle(nodeEvent1);
ApplicationAttemptId attId = createSchedulingRequest(1024, 1, "default", "user1", 2);
FSAppAttempt app = scheduler.getSchedulerApp(attId);
scheduler.update();
NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1);
scheduler.handle(updateEvent);
assertEquals(1, app.getLiveContainers().size());
scheduler.handle(updateEvent);
assertEquals(1, app.getLiveContainers().size());
}
use of org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode in project hadoop by apache.
the class TestFairScheduler method testAggregateCapacityTracking.
@Test
public void testAggregateCapacityTracking() throws Exception {
scheduler.init(conf);
scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
// Add a node
RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1");
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
scheduler.handle(nodeEvent1);
assertEquals(1024, scheduler.getClusterResource().getMemorySize());
// Add another node
RMNode node2 = MockNodes.newNodeInfo(1, Resources.createResource(512), 2, "127.0.0.2");
NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
scheduler.handle(nodeEvent2);
assertEquals(1536, scheduler.getClusterResource().getMemorySize());
// Remove the first node
NodeRemovedSchedulerEvent nodeEvent3 = new NodeRemovedSchedulerEvent(node1);
scheduler.handle(nodeEvent3);
assertEquals(512, scheduler.getClusterResource().getMemorySize());
}
use of org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode in project hadoop by apache.
the class TestFairScheduler method testMultipleContainersWaitingForReservation.
@Test(timeout = 5000)
public void testMultipleContainersWaitingForReservation() throws IOException {
scheduler.init(conf);
scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
// Add a node
RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1");
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
scheduler.handle(nodeEvent1);
// Request full capacity of node
createSchedulingRequest(1024, "queue1", "user1", 1);
scheduler.update();
NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1);
scheduler.handle(updateEvent);
ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue2", "user2", 1);
ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue3", "user3", 1);
scheduler.update();
scheduler.handle(updateEvent);
// One container should get reservation and the other should get nothing
assertEquals(1024, scheduler.getSchedulerApp(attId1).getCurrentReservation().getMemorySize());
assertEquals(0, scheduler.getSchedulerApp(attId2).getCurrentReservation().getMemorySize());
}
use of org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode in project hadoop by apache.
the class TestFairScheduler method testSimpleContainerAllocation.
@Test(timeout = 5000)
public void testSimpleContainerAllocation() throws IOException {
scheduler.init(conf);
scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
// Add a node
RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024, 4), 1, "127.0.0.1");
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
scheduler.handle(nodeEvent1);
// Add another node
RMNode node2 = MockNodes.newNodeInfo(1, Resources.createResource(512, 2), 2, "127.0.0.2");
NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
scheduler.handle(nodeEvent2);
createSchedulingRequest(512, 2, "queue1", "user1", 2);
scheduler.update();
NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1);
scheduler.handle(updateEvent);
// Asked for less than increment allocation.
assertEquals(FairSchedulerConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB, scheduler.getQueueManager().getQueue("queue1").getResourceUsage().getMemorySize());
NodeUpdateSchedulerEvent updateEvent2 = new NodeUpdateSchedulerEvent(node2);
scheduler.handle(updateEvent2);
assertEquals(1024, scheduler.getQueueManager().getQueue("queue1").getResourceUsage().getMemorySize());
assertEquals(2, scheduler.getQueueManager().getQueue("queue1").getResourceUsage().getVirtualCores());
// verify metrics
QueueMetrics queue1Metrics = scheduler.getQueueManager().getQueue("queue1").getMetrics();
assertEquals(1024, queue1Metrics.getAllocatedMB());
assertEquals(2, queue1Metrics.getAllocatedVirtualCores());
assertEquals(1024, scheduler.getRootQueueMetrics().getAllocatedMB());
assertEquals(2, scheduler.getRootQueueMetrics().getAllocatedVirtualCores());
assertEquals(512, scheduler.getRootQueueMetrics().getAvailableMB());
assertEquals(4, scheduler.getRootQueueMetrics().getAvailableVirtualCores());
}
use of org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode in project hadoop by apache.
the class TestFairScheduler method testChildMaxResources.
/**
* Test the child max resource settings.
*
* @throws IOException if scheduler reinitialization fails
*/
@Test
public void testChildMaxResources() throws IOException {
PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
out.println("<?xml version=\"1.0\"?>");
out.println("<allocations>");
out.println(" <queue name=\"queueA\" type=\"parent\">");
out.println(" <maxChildResources>2048mb,2vcores</maxChildResources>");
out.println(" </queue>");
out.println("</allocations>");
out.close();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
// Add one big node (only care about aggregate capacity)
RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(8 * 1024, 8), 1, "127.0.0.1");
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
scheduler.handle(nodeEvent1);
ApplicationAttemptId attId1 = createSchedulingRequest(1024, 1, "queueA.queueB", "user1", 8);
ApplicationAttemptId attId2 = createSchedulingRequest(1024, 1, "queueA.queueC", "user1", 8);
scheduler.update();
NodeUpdateSchedulerEvent nodeEvent = new NodeUpdateSchedulerEvent(node1);
scheduler.handle(nodeEvent);
scheduler.handle(nodeEvent);
scheduler.handle(nodeEvent);
scheduler.handle(nodeEvent);
scheduler.handle(nodeEvent);
scheduler.handle(nodeEvent);
scheduler.handle(nodeEvent);
scheduler.handle(nodeEvent);
// Apps should be running with 2 containers
assertEquals("App 1 is not running with the correct number of containers", 2, scheduler.getSchedulerApp(attId1).getLiveContainers().size());
assertEquals("App 2 is not running with the correct number of containers", 2, scheduler.getSchedulerApp(attId2).getLiveContainers().size());
out = new PrintWriter(new FileWriter(ALLOC_FILE));
out.println("<?xml version=\"1.0\"?>");
out.println("<allocations>");
out.println(" <queue name=\"queueA\" type=\"parent\">");
out.println(" <maxChildResources>3072mb,3vcores</maxChildResources>");
out.println(" </queue>");
out.println("</allocations>");
out.close();
scheduler.reinitialize(conf, resourceManager.getRMContext());
scheduler.update();
scheduler.handle(nodeEvent);
scheduler.handle(nodeEvent);
scheduler.handle(nodeEvent);
scheduler.handle(nodeEvent);
// Apps should be running with 3 containers now
assertEquals("App 1 is not running with the correct number of containers", 3, scheduler.getSchedulerApp(attId1).getLiveContainers().size());
assertEquals("App 2 is not running with the correct number of containers", 3, scheduler.getSchedulerApp(attId2).getLiveContainers().size());
out = new PrintWriter(new FileWriter(ALLOC_FILE));
out.println("<?xml version=\"1.0\"?>");
out.println("<allocations>");
out.println(" <queue name=\"queueA\" type=\"parent\">");
out.println(" <maxChildResources>1024mb,1vcores</maxChildResources>");
out.println(" </queue>");
out.println("</allocations>");
out.close();
scheduler.reinitialize(conf, resourceManager.getRMContext());
scheduler.update();
scheduler.handle(nodeEvent);
// Apps still should be running with 3 containers because we don't preempt
assertEquals("App 1 is not running with the correct number of containers", 3, scheduler.getSchedulerApp(attId1).getLiveContainers().size());
assertEquals("App 2 is not running with the correct number of containers", 3, scheduler.getSchedulerApp(attId2).getLiveContainers().size());
}
Aggregations