Examples with NullRMNodeLabelsManager org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager used on opensource projects

Example 16 with NullRMNodeLabelsManager

use of org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager in project hadoop by apache.

the class TestCapacityScheduler method testMoveAppViolateQueueState.

@Test(expected = YarnException.class)
public void testMoveAppViolateQueueState() throws Exception {
    resourceManager = new ResourceManager() {

        @Override
        protected RMNodeLabelsManager createNodeLabelManager() {
            RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
            mgr.init(getConfig());
            return mgr;
        }
    };
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
    setupQueueConfiguration(csConf);
    StringBuilder qState = new StringBuilder();
    qState.append(CapacitySchedulerConfiguration.PREFIX).append(B).append(CapacitySchedulerConfiguration.DOT).append(CapacitySchedulerConfiguration.STATE);
    csConf.set(qState.toString(), QueueState.STOPPED.name());
    YarnConfiguration conf = new YarnConfiguration(csConf);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    resourceManager.init(conf);
    resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
    resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
    ((AsyncDispatcher) resourceManager.getRMContext().getDispatcher()).start();
    mockContext = mock(RMContext.class);
    when(mockContext.getConfigurationProvider()).thenReturn(new LocalConfigurationProvider());
    ResourceScheduler scheduler = resourceManager.getResourceScheduler();
    // Register node1
    String host_0 = "host_0";
    NodeManager nm_0 = registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(6 * GB, 1));
    // ResourceRequest priorities
    Priority priority_0 = Priority.newInstance(0);
    Priority priority_1 = Priority.newInstance(1);
    // Submit application_0
    Application application_0 = new Application("user_0", "a1", resourceManager);
    // app + app attempt event sent to scheduler
    application_0.submit();
    application_0.addNodeManager(host_0, 1234, nm_0);
    Resource capability_0_0 = Resources.createResource(3 * GB, 1);
    application_0.addResourceRequestSpec(priority_1, capability_0_0);
    Resource capability_0_1 = Resources.createResource(2 * GB, 1);
    application_0.addResourceRequestSpec(priority_0, capability_0_1);
    Task task_0_0 = new Task(application_0, priority_1, new String[] { host_0 });
    application_0.addTask(task_0_0);
    // Send resource requests to the scheduler
    // allocate
    application_0.schedule();
    // task_0_0 allocated
    nodeUpdate(nm_0);
    // Get allocations from the scheduler
    // task_0_0
    application_0.schedule();
    checkApplicationResourceUsage(3 * GB, application_0);
    checkNodeResourceUsage(3 * GB, nm_0);
    // b2 queue contains 3GB consumption app,
    // add another 3GB will hit max capacity limit on queue b
    scheduler.moveApplication(application_0.getApplicationId(), "b1");
}

Example 17 with NullRMNodeLabelsManager

use of org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager in project hadoop by apache.

the class TestCapacityScheduler method testPendingResourceUpdatedAccordingToIncreaseRequestChanges.

@Test
public void testPendingResourceUpdatedAccordingToIncreaseRequestChanges() throws Exception {
    Configuration conf = TestUtils.getConfigurationWithQueueLabels(new Configuration(false));
    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
    final RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
    mgr.init(conf);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    MockRM rm = new MockRM(conf, memStore) {

        protected RMNodeLabelsManager createNodeLabelManager() {
            return mgr;
        }
    };
    rm.start();
    // label = ""
    MockNM nm1 = new MockNM("h1:1234", 200 * GB, rm.getResourceTrackerService());
    nm1.registerNode();
    // Launch app1 in queue=a1
    RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a1");
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
    // Allocate two more containers
    am1.allocate(Arrays.asList(ResourceRequest.newInstance(Priority.newInstance(1), "*", Resources.createResource(2 * GB), 2)), null);
    ContainerId containerId1 = ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);
    ContainerId containerId2 = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
    ContainerId containerId3 = ContainerId.newContainerId(am1.getApplicationAttemptId(), 3);
    Assert.assertTrue(rm.waitForState(nm1, containerId3, RMContainerState.ALLOCATED));
    // Acquire them
    am1.allocate(null, null);
    sentRMContainerLaunched(rm, ContainerId.newContainerId(am1.getApplicationAttemptId(), 1L));
    sentRMContainerLaunched(rm, ContainerId.newContainerId(am1.getApplicationAttemptId(), 2L));
    sentRMContainerLaunched(rm, ContainerId.newContainerId(am1.getApplicationAttemptId(), 3L));
    // am1 asks to change its AM container from 1GB to 3GB
    am1.sendContainerResizingRequest(Arrays.asList(UpdateContainerRequest.newInstance(0, containerId1, ContainerUpdateType.INCREASE_RESOURCE, Resources.createResource(3 * GB), null)));
    FiCaSchedulerApp app = getFiCaSchedulerApp(rm, app1.getApplicationId());
    Assert.assertEquals(2 * GB, app.getAppAttemptResourceUsage().getPending().getMemorySize());
    checkPendingResource(rm, "a1", 2 * GB, null);
    checkPendingResource(rm, "a", 2 * GB, null);
    checkPendingResource(rm, "root", 2 * GB, null);
    // am1 asks to change containerId2 (2G -> 3G) and containerId3 (2G -> 5G)
    am1.sendContainerResizingRequest(Arrays.asList(UpdateContainerRequest.newInstance(0, containerId2, ContainerUpdateType.INCREASE_RESOURCE, Resources.createResource(3 * GB), null), UpdateContainerRequest.newInstance(0, containerId3, ContainerUpdateType.INCREASE_RESOURCE, Resources.createResource(5 * GB), null)));
    Assert.assertEquals(6 * GB, app.getAppAttemptResourceUsage().getPending().getMemorySize());
    checkPendingResource(rm, "a1", 6 * GB, null);
    checkPendingResource(rm, "a", 6 * GB, null);
    checkPendingResource(rm, "root", 6 * GB, null);
    // am1 asks to change containerId1 (1G->3G), containerId2 (2G -> 4G) and
    // containerId3 (2G -> 2G)
    am1.sendContainerResizingRequest(Arrays.asList(UpdateContainerRequest.newInstance(0, containerId1, ContainerUpdateType.INCREASE_RESOURCE, Resources.createResource(3 * GB), null), UpdateContainerRequest.newInstance(0, containerId2, ContainerUpdateType.INCREASE_RESOURCE, Resources.createResource(4 * GB), null), UpdateContainerRequest.newInstance(0, containerId3, ContainerUpdateType.INCREASE_RESOURCE, Resources.createResource(2 * GB), null)));
    Assert.assertEquals(4 * GB, app.getAppAttemptResourceUsage().getPending().getMemorySize());
    checkPendingResource(rm, "a1", 4 * GB, null);
    checkPendingResource(rm, "a", 4 * GB, null);
    checkPendingResource(rm, "root", 4 * GB, null);
}

Example 18 with NullRMNodeLabelsManager

use of org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager in project hadoop by apache.

the class TestNodeLabelContainerAllocation method setUp.

@Before
public void setUp() throws Exception {
    conf = new YarnConfiguration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    mgr = new NullRMNodeLabelsManager();
    mgr.init(conf);
}

Example 19 with NullRMNodeLabelsManager

use of org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager in project hadoop by apache.

the class TestQueueParsing method testQueueParsingWhenLabelsNotExist.

@Test
public void testQueueParsingWhenLabelsNotExist() throws IOException {
    YarnConfiguration conf = new YarnConfiguration();
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf);
    setupQueueConfigurationWithLabels(csConf);
    CapacityScheduler capacityScheduler = new CapacityScheduler();
    RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null);
    RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
    nodeLabelsManager.init(conf);
    nodeLabelsManager.start();
    rmContext.setNodeLabelManager(nodeLabelsManager);
    capacityScheduler.setConf(csConf);
    capacityScheduler.setRMContext(rmContext);
    capacityScheduler.init(csConf);
    capacityScheduler.start();
    ServiceOperations.stopQuietly(capacityScheduler);
    ServiceOperations.stopQuietly(nodeLabelsManager);
}

Example 20 with NullRMNodeLabelsManager

use of org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager in project hadoop by apache.

the class TestQueueParsing method testRMStartWrongNodeCapacity.

@Test(timeout = 60000, expected = java.lang.IllegalArgumentException.class)
public void testRMStartWrongNodeCapacity() throws Exception {
    YarnConfiguration config = new YarnConfiguration();
    nodeLabelManager = new NullRMNodeLabelsManager();
    nodeLabelManager.init(config);
    config.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(config);
    // Define top-level queues
    conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "a" });
    conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
    conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100);
    conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "z", 100);
    final String A = CapacitySchedulerConfiguration.ROOT + ".a";
    conf.setCapacity(A, 100);
    conf.setAccessibleNodeLabels(A, ImmutableSet.of("x", "y", "z"));
    conf.setCapacityByLabel(A, "x", 100);
    conf.setCapacityByLabel(A, "y", 100);
    conf.setCapacityByLabel(A, "z", 70);
    MockRM rm = null;
    try {
        rm = new MockRM(conf) {

            @Override
            public RMNodeLabelsManager createNodeLabelManager() {
                return nodeLabelManager;
            }
        };
    } finally {
        IOUtils.closeStream(rm);
    }
}