Examples with QueueInfo org.apache.hadoop.yarn.api.records.QueueInfo used on opensource projects

Example 21 with QueueInfo

use of org.apache.hadoop.yarn.api.records.QueueInfo in project hadoop by apache.

the class TestFairScheduler method testQueueInfo.

@Test
public void testQueueInfo() throws IOException {
    conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
    out.println("<?xml version=\"1.0\"?>");
    out.println("<allocations>");
    out.println("<queue name=\"queueA\">");
    out.println("<weight>.25</weight>");
    out.println("</queue>");
    out.println("<queue name=\"queueB\">");
    out.println("<weight>.75</weight>");
    out.println("</queue>");
    out.println("</allocations>");
    out.close();
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(conf, resourceManager.getRMContext());
    // Add one big node (only care about aggregate capacity)
    RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(8 * 1024, 8), 1, "127.0.0.1");
    NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
    scheduler.handle(nodeEvent1);
    // Queue A wants 1 * 1024.
    createSchedulingRequest(1 * 1024, "queueA", "user1");
    // Queue B wants 6 * 1024
    createSchedulingRequest(6 * 1024, "queueB", "user1");
    scheduler.update();
    // Capacity should be the same as weight of Queue,
    // because the sum of all active Queues' weight are 1.
    // Before NodeUpdate Event, CurrentCapacity should be 0
    QueueInfo queueInfo = scheduler.getQueueInfo("queueA", false, false);
    Assert.assertEquals(0.25f, queueInfo.getCapacity(), 0.0f);
    Assert.assertEquals(0.0f, queueInfo.getCurrentCapacity(), 0.0f);
    queueInfo = scheduler.getQueueInfo("queueB", false, false);
    Assert.assertEquals(0.75f, queueInfo.getCapacity(), 0.0f);
    Assert.assertEquals(0.0f, queueInfo.getCurrentCapacity(), 0.0f);
    // Each NodeUpdate Event will only assign one container.
    // To assign two containers, call handle NodeUpdate Event twice.
    NodeUpdateSchedulerEvent nodeEvent2 = new NodeUpdateSchedulerEvent(node1);
    scheduler.handle(nodeEvent2);
    scheduler.handle(nodeEvent2);
    // After NodeUpdate Event, CurrentCapacity for queueA should be 1/2=0.5
    // and CurrentCapacity for queueB should be 6/6=1.
    queueInfo = scheduler.getQueueInfo("queueA", false, false);
    Assert.assertEquals(0.25f, queueInfo.getCapacity(), 0.0f);
    Assert.assertEquals(0.5f, queueInfo.getCurrentCapacity(), 0.0f);
    queueInfo = scheduler.getQueueInfo("queueB", false, false);
    Assert.assertEquals(0.75f, queueInfo.getCapacity(), 0.0f);
    Assert.assertEquals(1.0f, queueInfo.getCurrentCapacity(), 0.0f);
}

Example 22 with QueueInfo

use of org.apache.hadoop.yarn.api.records.QueueInfo in project hadoop by apache.

the class TestSchedulerUtils method testNormalizeNodeLabelExpression.

@Test(timeout = 30000)
public void testNormalizeNodeLabelExpression() throws IOException {
    // mock queue and scheduler
    YarnScheduler scheduler = mock(YarnScheduler.class);
    Set<String> queueAccessibleNodeLabels = Sets.newHashSet();
    QueueInfo queueInfo = mock(QueueInfo.class);
    when(queueInfo.getQueueName()).thenReturn("queue");
    when(queueInfo.getAccessibleNodeLabels()).thenReturn(queueAccessibleNodeLabels);
    when(queueInfo.getDefaultNodeLabelExpression()).thenReturn(" x ");
    when(scheduler.getQueueInfo(any(String.class), anyBoolean(), anyBoolean())).thenReturn(queueInfo);
    Resource maxResource = Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
    // queue has labels, success cases
    try {
        // set queue accessible node labels to [x, y]
        queueAccessibleNodeLabels.clear();
        queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
        rmContext.getNodeLabelManager().addToCluserNodeLabels(ImmutableSet.of(NodeLabel.newInstance("x"), NodeLabel.newInstance("y")));
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        Assert.assertTrue(resReq.getNodeLabelExpression().equals("x"));
        resReq.setNodeLabelExpression(" y ");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        Assert.assertTrue(resReq.getNodeLabelExpression().equals("y"));
    } catch (InvalidResourceRequestException e) {
        e.printStackTrace();
        fail("Should be valid when request labels is a subset of queue labels");
    } finally {
        rmContext.getNodeLabelManager().removeFromClusterNodeLabels(Arrays.asList("x", "y"));
    }
}

Example 23 with QueueInfo

use of org.apache.hadoop.yarn.api.records.QueueInfo in project hadoop by apache.

the class TestSchedulerUtils method testValidateResourceRequestWithErrorLabelsPermission.

@Test(timeout = 30000)
public void testValidateResourceRequestWithErrorLabelsPermission() throws IOException {
    // mock queue and scheduler
    YarnScheduler scheduler = mock(YarnScheduler.class);
    Set<String> queueAccessibleNodeLabels = Sets.newHashSet();
    QueueInfo queueInfo = mock(QueueInfo.class);
    when(queueInfo.getQueueName()).thenReturn("queue");
    when(queueInfo.getAccessibleNodeLabels()).thenReturn(queueAccessibleNodeLabels);
    when(scheduler.getQueueInfo(any(String.class), anyBoolean(), anyBoolean())).thenReturn(queueInfo);
    Resource maxResource = Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
    // queue has labels, success cases
    try {
        // set queue accessible node labesl to [x, y]
        queueAccessibleNodeLabels.clear();
        queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
        rmContext.getNodeLabelManager().addToCluserNodeLabels(ImmutableSet.of(NodeLabel.newInstance("x"), NodeLabel.newInstance("y")));
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        resReq.setNodeLabelExpression("x");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        resReq.setNodeLabelExpression("y");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        resReq.setNodeLabelExpression("");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        resReq.setNodeLabelExpression(" ");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
    } catch (InvalidResourceRequestException e) {
        e.printStackTrace();
        fail("Should be valid when request labels is a subset of queue labels");
    } finally {
        rmContext.getNodeLabelManager().removeFromClusterNodeLabels(Arrays.asList("x", "y"));
    }
    // requested. should fail
    try {
        // set queue accessible node labesl to [x, y]
        queueAccessibleNodeLabels.clear();
        queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        resReq.setNodeLabelExpression("x");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        fail("Should fail");
    } catch (InvalidResourceRequestException e) {
    }
    // queue has labels, failed cases (when ask a label not included by queue)
    try {
        // set queue accessible node labesl to [x, y]
        queueAccessibleNodeLabels.clear();
        queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
        rmContext.getNodeLabelManager().addToCluserNodeLabels(ImmutableSet.of(NodeLabel.newInstance("x"), NodeLabel.newInstance("y")));
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        resReq.setNodeLabelExpression("z");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        fail("Should fail");
    } catch (InvalidResourceRequestException e) {
    } finally {
        rmContext.getNodeLabelManager().removeFromClusterNodeLabels(Arrays.asList("x", "y"));
    }
    // now
    try {
        // set queue accessible node labesl to [x, y]
        queueAccessibleNodeLabels.clear();
        queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
        rmContext.getNodeLabelManager().addToCluserNodeLabels(ImmutableSet.of(NodeLabel.newInstance("x"), NodeLabel.newInstance("y")));
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        resReq.setNodeLabelExpression("x && y");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        fail("Should fail");
    } catch (InvalidResourceRequestException e) {
    } finally {
        rmContext.getNodeLabelManager().removeFromClusterNodeLabels(Arrays.asList("x", "y"));
    }
    // queue doesn't have label, succeed (when request no label)
    queueAccessibleNodeLabels.clear();
    try {
        // set queue accessible node labels to empty
        queueAccessibleNodeLabels.clear();
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        resReq.setNodeLabelExpression("");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        resReq.setNodeLabelExpression("  ");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
    } catch (InvalidResourceRequestException e) {
        e.printStackTrace();
        fail("Should be valid when request labels is empty");
    }
    boolean invalidlabelexception = false;
    // queue doesn't have label, failed (when request any label)
    try {
        // set queue accessible node labels to empty
        queueAccessibleNodeLabels.clear();
        rmContext.getNodeLabelManager().addToCluserNodeLabels(ImmutableSet.of(NodeLabel.newInstance("x")));
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        resReq.setNodeLabelExpression("x");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        fail("Should fail");
    } catch (InvalidLabelResourceRequestException e) {
        invalidlabelexception = true;
    } catch (InvalidResourceRequestException e) {
    } finally {
        rmContext.getNodeLabelManager().removeFromClusterNodeLabels(Arrays.asList("x"));
    }
    Assert.assertTrue("InvalidLabelResourceRequestException excpeted", invalidlabelexception);
    // queue is "*", always succeeded
    try {
        // set queue accessible node labels to empty
        queueAccessibleNodeLabels.clear();
        queueAccessibleNodeLabels.add(RMNodeLabelsManager.ANY);
        rmContext.getNodeLabelManager().addToCluserNodeLabels(ImmutableSet.of(NodeLabel.newInstance("x"), NodeLabel.newInstance("y"), NodeLabel.newInstance("z")));
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        resReq.setNodeLabelExpression("x");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        resReq.setNodeLabelExpression("y");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        resReq.setNodeLabelExpression("z");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
    } catch (InvalidResourceRequestException e) {
        e.printStackTrace();
        fail("Should be valid when queue can access any labels");
    } finally {
        rmContext.getNodeLabelManager().removeFromClusterNodeLabels(Arrays.asList("x", "y", "z"));
    }
    // same as above, but cluster node labels don't contains label, should fail
    try {
        // set queue accessible node labels to empty
        queueAccessibleNodeLabels.clear();
        queueAccessibleNodeLabels.add(RMNodeLabelsManager.ANY);
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        resReq.setNodeLabelExpression("x");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        fail("Should fail");
    } catch (InvalidResourceRequestException e) {
    }
    // we don't allow resource name other than ANY and specify label
    try {
        // set queue accessible node labesl to [x, y]
        queueAccessibleNodeLabels.clear();
        queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
        rmContext.getNodeLabelManager().addToCluserNodeLabels(ImmutableSet.of(NodeLabel.newInstance("x"), NodeLabel.newInstance("y")));
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), "rack", resource, 1);
        resReq.setNodeLabelExpression("x");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        fail("Should fail");
    } catch (InvalidResourceRequestException e) {
    } finally {
        rmContext.getNodeLabelManager().removeFromClusterNodeLabels(Arrays.asList("x", "y"));
    }
    // queue has accessible label = *
    try {
        // set queue accessible node labesl to *
        queueAccessibleNodeLabels.clear();
        queueAccessibleNodeLabels.addAll(Arrays.asList(CommonNodeLabelsManager.ANY));
        rmContext.getNodeLabelManager().addToCluserNodeLabels(ImmutableSet.of(NodeLabel.newInstance("x")));
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), "rack", resource, 1);
        resReq.setNodeLabelExpression("x");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        fail("Should fail");
    } catch (InvalidResourceRequestException e) {
    } finally {
        rmContext.getNodeLabelManager().removeFromClusterNodeLabels(Arrays.asList("x"));
    }
    try {
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq1 = BuilderUtils.newResourceRequest(mock(Priority.class), "*", resource, 1, "x");
        SchedulerUtils.normalizeAndvalidateRequest(resReq1, maxResource, "queue", scheduler, rmContext);
        fail("Should fail");
    } catch (InvalidResourceRequestException e) {
        assertEquals("Invalid label resource request, cluster do not contain , " + "label= x", e.getMessage());
    }
    try {
        rmContext.getYarnConfiguration().set(YarnConfiguration.NODE_LABELS_ENABLED, "false");
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq1 = BuilderUtils.newResourceRequest(mock(Priority.class), "*", resource, 1, "x");
        SchedulerUtils.normalizeAndvalidateRequest(resReq1, maxResource, "queue", scheduler, rmContext);
        Assert.assertEquals(RMNodeLabelsManager.NO_LABEL, resReq1.getNodeLabelExpression());
    } catch (InvalidResourceRequestException e) {
        assertEquals("Invalid resource request, node label not enabled but " + "request contains label expression", e.getMessage());
    }
}

Example 24 with QueueInfo

use of org.apache.hadoop.yarn.api.records.QueueInfo in project hadoop by apache.

the class ClientRMService method getQueueInfo.

@Override
public GetQueueInfoResponse getQueueInfo(GetQueueInfoRequest request) throws YarnException {
    UserGroupInformation callerUGI;
    try {
        callerUGI = UserGroupInformation.getCurrentUser();
    } catch (IOException ie) {
        LOG.info("Error getting UGI ", ie);
        throw RPCUtil.getRemoteException(ie);
    }
    GetQueueInfoResponse response = recordFactory.newRecordInstance(GetQueueInfoResponse.class);
    try {
        QueueInfo queueInfo = scheduler.getQueueInfo(request.getQueueName(), request.getIncludeChildQueues(), request.getRecursive());
        List<ApplicationReport> appReports = EMPTY_APPS_REPORT;
        if (request.getIncludeApplications()) {
            List<ApplicationAttemptId> apps = scheduler.getAppsInQueue(request.getQueueName());
            appReports = new ArrayList<ApplicationReport>(apps.size());
            for (ApplicationAttemptId app : apps) {
                RMApp rmApp = rmContext.getRMApps().get(app.getApplicationId());
                if (rmApp != null) {
                    // Check if user is allowed access to this app
                    if (!checkAccess(callerUGI, rmApp.getUser(), ApplicationAccessType.VIEW_APP, rmApp)) {
                        continue;
                    }
                    appReports.add(rmApp.createAndGetApplicationReport(callerUGI.getUserName(), true));
                }
            }
        }
        queueInfo.setApplications(appReports);
        response.setQueueInfo(queueInfo);
    } catch (IOException ioe) {
        LOG.info("Failed to getQueueInfo for " + request.getQueueName(), ioe);
    }
    return response;
}

Example 25 with QueueInfo

use of org.apache.hadoop.yarn.api.records.QueueInfo in project hadoop by apache.

the class FifoSchedulerMetrics method trackQueue.

@Override
public void trackQueue(String queueName) {
    trackedQueues.add(queueName);
    FifoScheduler fifo = (FifoScheduler) scheduler;
    // for FifoScheduler, only DEFAULT_QUEUE
    // here the three parameters doesn't affect results
    final QueueInfo queue = fifo.getQueueInfo(queueName, false, false);
    // track currentCapacity, maximumCapacity (always 1.0f)
    metrics.register("variable.queue." + queueName + ".currentcapacity", new Gauge<Float>() {

        @Override
        public Float getValue() {
            return queue.getCurrentCapacity();
        }
    });
    metrics.register("variable.queue." + queueName + ".", new Gauge<Float>() {

        @Override
        public Float getValue() {
            return queue.getCurrentCapacity();
        }
    });
}