Examples with DynamicResourceConfiguration org.apache.hadoop.yarn.server.resourcemanager.resource.DynamicResourceConfiguration used on opensource projects

Example 1 with DynamicResourceConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.resource.DynamicResourceConfiguration in project hadoop by apache.

the class AdminService method refreshNodesResources.

@Override
public RefreshNodesResourcesResponse refreshNodesResources(RefreshNodesResourcesRequest request) throws YarnException, StandbyException {
    final String operation = "refreshNodesResources";
    UserGroupInformation user = checkAcls(operation);
    final String msg = "refresh nodes.";
    checkRMStatus(user.getShortUserName(), operation, msg);
    RefreshNodesResourcesResponse response = recordFactory.newRecordInstance(RefreshNodesResourcesResponse.class);
    try {
        Configuration conf = getConfig();
        Configuration configuration = new Configuration(conf);
        DynamicResourceConfiguration newConf;
        InputStream drInputStream = this.rmContext.getConfigurationProvider().getConfigurationInputStream(configuration, YarnConfiguration.DR_CONFIGURATION_FILE);
        if (drInputStream != null) {
            newConf = new DynamicResourceConfiguration(configuration, drInputStream);
        } else {
            newConf = new DynamicResourceConfiguration(configuration);
        }
        if (newConf.getNodes() != null && newConf.getNodes().length != 0) {
            Map<NodeId, ResourceOption> nodeResourceMap = newConf.getNodeResourceMap();
            UpdateNodeResourceRequest updateRequest = UpdateNodeResourceRequest.newInstance(nodeResourceMap);
            updateNodeResource(updateRequest);
        }
        // refresh dynamic resource in ResourceTrackerService
        this.rmContext.getResourceTrackerService().updateDynamicResourceConfiguration(newConf);
        RMAuditLogger.logSuccess(user.getShortUserName(), operation, "AdminService");
        return response;
    } catch (IOException ioe) {
        throw logAndWrapException(ioe, user.getShortUserName(), operation, msg);
    }
}

Example 2 with DynamicResourceConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.resource.DynamicResourceConfiguration in project hadoop by apache.

the class TestRMAdminService method testResourcePersistentForNMRegistrationWithNewResource.

@Test
public void testResourcePersistentForNMRegistrationWithNewResource() throws IOException, YarnException {
    configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
    //upload default configurations
    uploadDefaultConfiguration();
    try {
        rm = new MockRM(configuration);
        rm.init(configuration);
        rm.start();
        rm.registerNode("h1:1234", 5120);
    } catch (Exception ex) {
        fail("Should not get any exceptions");
    }
    NodeId nid = NodeId.fromString("h1:1234");
    RMNode ni = rm.getRMContext().getRMNodes().get(nid);
    Resource resource = ni.getTotalCapability();
    Assert.assertEquals("<memory:5120, vCores:5>", resource.toString());
    DynamicResourceConfiguration drConf = new DynamicResourceConfiguration();
    drConf.set(PREFIX + NODES, "h1:1234");
    drConf.set(PREFIX + "h1:1234.vcores", "4");
    drConf.set(PREFIX + "h1:1234.memory", "4096");
    uploadConfiguration(drConf, "dynamic-resources.xml");
    rm.adminService.refreshNodesResources(RefreshNodesResourcesRequest.newInstance());
    try {
        // register the same node again with a different resource.
        // validate this won't work as resource cached in RM side.
        rm.registerNode("h1:1234", 8192, 8);
    } catch (Exception ex) {
        fail("Should not get any exceptions");
    }
    RMNode niAfter = rm.getRMContext().getRMNodes().get(nid);
    Resource resourceAfter = niAfter.getTotalCapability();
    Assert.assertEquals("<memory:4096, vCores:4>", resourceAfter.toString());
    // Replace original dr file with an empty dr file, and validate node
    // registration with new resources will take effective now.
    deleteOnRemoteFileSystem("dynamic-resources.xml");
    DynamicResourceConfiguration emptyDRConf = new DynamicResourceConfiguration();
    uploadConfiguration(emptyDRConf, "dynamic-resources.xml");
    rm.adminService.refreshNodesResources(RefreshNodesResourcesRequest.newInstance());
    try {
        // register the same node third time, this time the register resource
        // should work.
        rm.registerNode("h1:1234", 8192, 8);
    } catch (Exception ex) {
        fail("Should not get any exceptions");
    }
    niAfter = rm.getRMContext().getRMNodes().get(nid);
    resourceAfter = niAfter.getTotalCapability();
    // new resource in registration should take effective as we empty
    // dynamic resource file already.
    Assert.assertEquals("<memory:8192, vCores:8>", resourceAfter.toString());
}

Example 3 with DynamicResourceConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.resource.DynamicResourceConfiguration in project hadoop by apache.

the class TestRMAdminService method testRefreshNodesResourceWithResourceReturnInRegistration.

@Test
public void testRefreshNodesResourceWithResourceReturnInRegistration() throws IOException, YarnException {
    configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
    //upload default configurations
    uploadDefaultConfiguration();
    MockNM nm = null;
    try {
        rm = new MockRM(configuration);
        rm.init(configuration);
        rm.start();
        nm = rm.registerNode("h1:1234", 2048, 2);
    } catch (Exception ex) {
        fail("Should not get any exceptions");
    }
    NodeId nid = NodeId.fromString("h1:1234");
    RMNode ni = rm.getRMContext().getRMNodes().get(nid);
    Resource resource = ni.getTotalCapability();
    Assert.assertEquals("<memory:2048, vCores:2>", resource.toString());
    DynamicResourceConfiguration drConf = new DynamicResourceConfiguration();
    drConf.set(PREFIX + NODES, "h1:1234");
    drConf.set(PREFIX + "h1:1234.vcores", "4");
    drConf.set(PREFIX + "h1:1234.memory", "4096");
    uploadConfiguration(drConf, "dynamic-resources.xml");
    rm.adminService.refreshNodesResources(RefreshNodesResourcesRequest.newInstance());
    try {
        // register the same node again with original resource.
        // validate this will get new resource back;
        nm.registerNode();
    } catch (Exception ex) {
        fail("Should not get any exceptions");
    }
    RMNode niAfter = rm.getRMContext().getRMNodes().get(nid);
    Resource resourceAfter = niAfter.getTotalCapability();
    Assert.assertEquals("<memory:4096, vCores:4>", resourceAfter.toString());
    Assert.assertEquals(4096, nm.getMemory());
    Assert.assertEquals(4, nm.getvCores());
}

Example 4 with DynamicResourceConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.resource.DynamicResourceConfiguration in project hadoop by apache.

the class TestRMAdminService method testRefreshNodesResourceWithFileSystemBasedConfigurationProvider.

@Test
public void testRefreshNodesResourceWithFileSystemBasedConfigurationProvider() throws IOException, YarnException {
    configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
    //upload default configurations
    uploadDefaultConfiguration();
    try {
        rm = new MockRM(configuration);
        rm.init(configuration);
        rm.start();
        rm.registerNode("h1:1234", 5120);
    } catch (Exception ex) {
        fail("Should not get any exceptions");
    }
    NodeId nid = NodeId.fromString("h1:1234");
    RMNode ni = rm.getRMContext().getRMNodes().get(nid);
    Resource resource = ni.getTotalCapability();
    Assert.assertEquals("<memory:5120, vCores:5>", resource.toString());
    DynamicResourceConfiguration drConf = new DynamicResourceConfiguration();
    drConf.set(PREFIX + NODES, "h1:1234");
    drConf.set(PREFIX + "h1:1234.vcores", "4");
    drConf.set(PREFIX + "h1:1234.memory", "4096");
    uploadConfiguration(drConf, "dynamic-resources.xml");
    rm.adminService.refreshNodesResources(RefreshNodesResourcesRequest.newInstance());
    rm.drainEvents();
    RMNode niAfter = rm.getRMContext().getRMNodes().get(nid);
    Resource resourceAfter = niAfter.getTotalCapability();
    Assert.assertEquals("<memory:4096, vCores:4>", resourceAfter.toString());
}

Example 5 with DynamicResourceConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.resource.DynamicResourceConfiguration in project hadoop by apache.

the class TestRMAdminService method testRefreshNodesResourceWithResourceReturnInHeartbeat.

@Test
public void testRefreshNodesResourceWithResourceReturnInHeartbeat() throws IOException, YarnException {
    configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
    //upload default configurations
    uploadDefaultConfiguration();
    MockNM nm = null;
    try {
        rm = new MockRM(configuration);
        rm.init(configuration);
        rm.start();
        nm = rm.registerNode("h1:1234", 2048, 2);
    } catch (Exception ex) {
        fail("Should not get any exceptions");
    }
    NodeId nid = NodeId.fromString("h1:1234");
    RMNode ni = rm.getRMContext().getRMNodes().get(nid);
    Resource resource = ni.getTotalCapability();
    Assert.assertEquals("<memory:2048, vCores:2>", resource.toString());
    DynamicResourceConfiguration drConf = new DynamicResourceConfiguration();
    drConf.set(PREFIX + NODES, "h1:1234");
    drConf.set(PREFIX + "h1:1234.vcores", "4");
    drConf.set(PREFIX + "h1:1234.memory", "4096");
    uploadConfiguration(drConf, "dynamic-resources.xml");
    rm.adminService.refreshNodesResources(RefreshNodesResourcesRequest.newInstance());
    try {
        // NM-RM heartbeat, validate that this will get new resource back.
        nm.nodeHeartbeat(true);
    } catch (Exception ex) {
        fail("Should not get any exceptions");
    }
    RMNode niAfter = rm.getRMContext().getRMNodes().get(nid);
    Resource resourceAfter = niAfter.getTotalCapability();
    Assert.assertEquals("<memory:4096, vCores:4>", resourceAfter.toString());
    Assert.assertEquals(4096, nm.getMemory());
    Assert.assertEquals(4, nm.getvCores());
}