Examples with CapacitySchedulerConfiguration org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration used on opensource projects

Example 16 with CapacitySchedulerConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration in project hadoop by apache.

the class TestRMWebServicesAppsModification method testAppSubmit.

public void testAppSubmit(String acceptMedia, String contentMedia) throws Exception {
    // create a test app and submit it via rest(after getting an app-id) then
    // get the app details from the rmcontext and check that everything matches
    client().addFilter(new LoggingFilter(System.out));
    String lrKey = "example";
    String queueName = "testqueue";
    // create the queue
    String[] queues = { "default", "testqueue" };
    CapacitySchedulerConfiguration csconf = new CapacitySchedulerConfiguration();
    csconf.setQueues("root", queues);
    csconf.setCapacity("root.default", 50.0f);
    csconf.setCapacity("root.testqueue", 50.0f);
    rm.getResourceScheduler().reinitialize(csconf, rm.getRMContext());
    String appName = "test";
    String appType = "test-type";
    String urlPath = "apps";
    String appId = testGetNewApplication(acceptMedia);
    List<String> commands = new ArrayList<>();
    commands.add("/bin/sleep 5");
    HashMap<String, String> environment = new HashMap<>();
    environment.put("APP_VAR", "ENV_SETTING");
    HashMap<ApplicationAccessType, String> acls = new HashMap<>();
    acls.put(ApplicationAccessType.MODIFY_APP, "testuser1, testuser2");
    acls.put(ApplicationAccessType.VIEW_APP, "testuser3, testuser4");
    Set<String> tags = new HashSet<>();
    tags.add("tag1");
    tags.add("tag 2");
    CredentialsInfo credentials = new CredentialsInfo();
    HashMap<String, String> tokens = new HashMap<>();
    HashMap<String, String> secrets = new HashMap<>();
    secrets.put("secret1", Base64.encodeBase64String("mysecret".getBytes("UTF8")));
    credentials.setSecrets(secrets);
    credentials.setTokens(tokens);
    ApplicationSubmissionContextInfo appInfo = new ApplicationSubmissionContextInfo();
    appInfo.setApplicationId(appId);
    appInfo.setApplicationName(appName);
    appInfo.setMaxAppAttempts(2);
    appInfo.setQueue(queueName);
    appInfo.setApplicationType(appType);
    appInfo.setPriority(0);
    HashMap<String, LocalResourceInfo> lr = new HashMap<>();
    LocalResourceInfo y = new LocalResourceInfo();
    y.setUrl(new URI("http://www.test.com/file.txt"));
    y.setSize(100);
    y.setTimestamp(System.currentTimeMillis());
    y.setType(LocalResourceType.FILE);
    y.setVisibility(LocalResourceVisibility.APPLICATION);
    lr.put(lrKey, y);
    appInfo.getContainerLaunchContextInfo().setResources(lr);
    appInfo.getContainerLaunchContextInfo().setCommands(commands);
    appInfo.getContainerLaunchContextInfo().setEnvironment(environment);
    appInfo.getContainerLaunchContextInfo().setAcls(acls);
    appInfo.getContainerLaunchContextInfo().getAuxillaryServiceData().put("test", Base64.encodeBase64URLSafeString("value12".getBytes("UTF8")));
    appInfo.getContainerLaunchContextInfo().setCredentials(credentials);
    appInfo.getResource().setMemory(1024);
    appInfo.getResource().setvCores(1);
    appInfo.setApplicationTags(tags);
    // Set LogAggregationContextInfo
    String includePattern = "file1";
    String excludePattern = "file2";
    String rolledLogsIncludePattern = "file3";
    String rolledLogsExcludePattern = "file4";
    String className = "policy_class";
    String parameters = "policy_parameter";
    LogAggregationContextInfo logAggregationContextInfo = new LogAggregationContextInfo();
    logAggregationContextInfo.setIncludePattern(includePattern);
    logAggregationContextInfo.setExcludePattern(excludePattern);
    logAggregationContextInfo.setRolledLogsIncludePattern(rolledLogsIncludePattern);
    logAggregationContextInfo.setRolledLogsExcludePattern(rolledLogsExcludePattern);
    logAggregationContextInfo.setLogAggregationPolicyClassName(className);
    logAggregationContextInfo.setLogAggregationPolicyParameters(parameters);
    appInfo.setLogAggregationContextInfo(logAggregationContextInfo);
    // Set attemptFailuresValidityInterval
    long attemptFailuresValidityInterval = 5000;
    appInfo.setAttemptFailuresValidityInterval(attemptFailuresValidityInterval);
    // Set ReservationId
    String reservationId = ReservationId.newInstance(System.currentTimeMillis(), 1).toString();
    appInfo.setReservationId(reservationId);
    ClientResponse response = this.constructWebResource(urlPath).accept(acceptMedia).entity(appInfo, contentMedia).post(ClientResponse.class);
    if (!this.isAuthenticationEnabled()) {
        assertResponseStatusCode(Status.UNAUTHORIZED, response.getStatusInfo());
        return;
    }
    assertResponseStatusCode(Status.ACCEPTED, response.getStatusInfo());
    assertTrue(!response.getHeaders().getFirst(HttpHeaders.LOCATION).isEmpty());
    String locURL = response.getHeaders().getFirst(HttpHeaders.LOCATION);
    assertTrue(locURL.contains("/apps/application"));
    appId = locURL.substring(locURL.indexOf("/apps/") + "/apps/".length());
    WebResource res = resource().uri(new URI(locURL));
    res = res.queryParam("user.name", webserviceUserName);
    response = res.get(ClientResponse.class);
    assertResponseStatusCode(Status.OK, response.getStatusInfo());
    RMApp app = rm.getRMContext().getRMApps().get(ApplicationId.fromString(appId));
    assertEquals(appName, app.getName());
    assertEquals(webserviceUserName, app.getUser());
    assertEquals(2, app.getMaxAppAttempts());
    if (app.getQueue().contains("root.")) {
        queueName = "root." + queueName;
    }
    assertEquals(queueName, app.getQueue());
    assertEquals(appType, app.getApplicationType());
    assertEquals(tags, app.getApplicationTags());
    ContainerLaunchContext ctx = app.getApplicationSubmissionContext().getAMContainerSpec();
    assertEquals(commands, ctx.getCommands());
    assertEquals(environment, ctx.getEnvironment());
    assertEquals(acls, ctx.getApplicationACLs());
    Map<String, LocalResource> appLRs = ctx.getLocalResources();
    assertTrue(appLRs.containsKey(lrKey));
    LocalResource exampleLR = appLRs.get(lrKey);
    assertEquals(URL.fromURI(y.getUrl()), exampleLR.getResource());
    assertEquals(y.getSize(), exampleLR.getSize());
    assertEquals(y.getTimestamp(), exampleLR.getTimestamp());
    assertEquals(y.getType(), exampleLR.getType());
    assertEquals(y.getPattern(), exampleLR.getPattern());
    assertEquals(y.getVisibility(), exampleLR.getVisibility());
    Credentials cs = new Credentials();
    ByteArrayInputStream str = new ByteArrayInputStream(app.getApplicationSubmissionContext().getAMContainerSpec().getTokens().array());
    DataInputStream di = new DataInputStream(str);
    cs.readTokenStorageStream(di);
    Text key = new Text("secret1");
    assertTrue("Secrets missing from credentials object", cs.getAllSecretKeys().contains(key));
    assertEquals("mysecret", new String(cs.getSecretKey(key), "UTF-8"));
    // Check LogAggregationContext
    ApplicationSubmissionContext asc = app.getApplicationSubmissionContext();
    LogAggregationContext lac = asc.getLogAggregationContext();
    assertEquals(includePattern, lac.getIncludePattern());
    assertEquals(excludePattern, lac.getExcludePattern());
    assertEquals(rolledLogsIncludePattern, lac.getRolledLogsIncludePattern());
    assertEquals(rolledLogsExcludePattern, lac.getRolledLogsExcludePattern());
    assertEquals(className, lac.getLogAggregationPolicyClassName());
    assertEquals(parameters, lac.getLogAggregationPolicyParameters());
    // Check attemptFailuresValidityInterval
    assertEquals(attemptFailuresValidityInterval, asc.getAttemptFailuresValidityInterval());
    // Check ReservationId
    assertEquals(reservationId, app.getReservationId().toString());
    response = this.constructWebResource("apps", appId).accept(acceptMedia).get(ClientResponse.class);
    assertResponseStatusCode(Status.OK, response.getStatusInfo());
}

Example 17 with CapacitySchedulerConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration in project hadoop by apache.

the class TestRMWebServicesAppsModification method testSingleAppKillUnauthorized.

@Test(timeout = 60000)
public void testSingleAppKillUnauthorized() throws Exception {
    boolean isCapacityScheduler = rm.getResourceScheduler() instanceof CapacityScheduler;
    boolean isFairScheduler = rm.getResourceScheduler() instanceof FairScheduler;
    assumeTrue("This test is only supported on Capacity and Fair Scheduler", isCapacityScheduler || isFairScheduler);
    // FairScheduler use ALLOCATION_FILE to configure ACL
    if (isCapacityScheduler) {
        // default root queue allows anyone to have admin acl
        CapacitySchedulerConfiguration csconf = new CapacitySchedulerConfiguration();
        csconf.setAcl("root", QueueACL.ADMINISTER_QUEUE, "someuser");
        csconf.setAcl("root.default", QueueACL.ADMINISTER_QUEUE, "someuser");
        rm.getResourceScheduler().reinitialize(csconf, rm.getRMContext());
    }
    rm.start();
    MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
    String[] mediaTypes = { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML };
    for (String mediaType : mediaTypes) {
        RMApp app = rm.submitApp(CONTAINER_MB, "test", "someuser");
        amNodeManager.nodeHeartbeat(true);
        ClientResponse response = this.constructWebResource("apps", app.getApplicationId().toString(), "state").accept(mediaType).get(ClientResponse.class);
        AppState info = response.getEntity(AppState.class);
        info.setState(YarnApplicationState.KILLED.toString());
        response = this.constructWebResource("apps", app.getApplicationId().toString(), "state").accept(mediaType).entity(info, MediaType.APPLICATION_XML).put(ClientResponse.class);
        validateResponseStatus(response, Status.FORBIDDEN);
    }
    rm.stop();
}

Example 18 with CapacitySchedulerConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration in project hadoop by apache.

the class TestRMAdminService method uploadDefaultConfiguration.

private void uploadDefaultConfiguration() throws IOException {
    Configuration conf = new Configuration();
    uploadConfiguration(conf, "core-site.xml");
    YarnConfiguration yarnConf = new YarnConfiguration();
    yarnConf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
    uploadConfiguration(yarnConf, "yarn-site.xml");
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
    uploadConfiguration(csConf, "capacity-scheduler.xml");
    Configuration hadoopPolicyConf = new Configuration(false);
    hadoopPolicyConf.addResource(YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
    uploadConfiguration(hadoopPolicyConf, "hadoop-policy.xml");
}

Example 19 with CapacitySchedulerConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration in project hadoop by apache.

the class TestRMAdminService method testRMInitialsWithFileSystemBasedConfigurationProvider.

@Test
public void testRMInitialsWithFileSystemBasedConfigurationProvider() throws Exception {
    configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
    // upload configurations
    final File excludeHostsFile = new File(tmpDir.toString(), "excludeHosts");
    if (excludeHostsFile.exists()) {
        excludeHostsFile.delete();
    }
    if (!excludeHostsFile.createNewFile()) {
        Assert.fail("Can not create " + "excludeHosts");
    }
    PrintWriter fileWriter = new PrintWriter(excludeHostsFile);
    fileWriter.write("0.0.0.0:123");
    fileWriter.close();
    uploadToRemoteFileSystem(new Path(excludeHostsFile.getAbsolutePath()));
    YarnConfiguration yarnConf = new YarnConfiguration();
    yarnConf.set(YarnConfiguration.YARN_ADMIN_ACL, "world:anyone:rwcda");
    yarnConf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, this.workingPath + "/excludeHosts");
    uploadConfiguration(yarnConf, "yarn-site.xml");
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
    csConf.set(CapacitySchedulerConfiguration.MAXIMUM_SYSTEM_APPLICATIONS, "5000");
    uploadConfiguration(csConf, "capacity-scheduler.xml");
    String aclsString = "alice,bob users,wheel";
    Configuration newConf = new Configuration();
    newConf.set("security.applicationclient.protocol.acl", aclsString);
    uploadConfiguration(newConf, "hadoop-policy.xml");
    Configuration conf = new Configuration();
    conf.setBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true);
    conf.set("hadoop.proxyuser.test.groups", "test_groups");
    conf.set("hadoop.proxyuser.test.hosts", "test_hosts");
    conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, MockUnixGroupsMapping.class, GroupMappingServiceProvider.class);
    uploadConfiguration(conf, "core-site.xml");
    // update the groups
    MockUnixGroupsMapping.updateGroups();
    ResourceManager resourceManager = null;
    try {
        try {
            resourceManager = new ResourceManager();
            resourceManager.init(configuration);
            resourceManager.start();
        } catch (Exception ex) {
            fail("Should not get any exceptions");
        }
        // validate values for excludeHosts
        Set<String> excludeHosts = resourceManager.getRMContext().getNodesListManager().getHostsReader().getExcludedHosts();
        Assert.assertTrue(excludeHosts.size() == 1);
        Assert.assertTrue(excludeHosts.contains("0.0.0.0:123"));
        // validate values for admin-acls
        String aclStringAfter = resourceManager.adminService.getAccessControlList().getAclString().trim();
        Assert.assertEquals(aclStringAfter, "world:anyone:rwcda," + UserGroupInformation.getCurrentUser().getShortUserName());
        // validate values for queue configuration
        CapacityScheduler cs = (CapacityScheduler) resourceManager.getRMContext().getScheduler();
        int maxAppsAfter = cs.getConfiguration().getMaximumSystemApplications();
        Assert.assertEquals(maxAppsAfter, 5000);
        // verify service Acls for AdminService
        ServiceAuthorizationManager adminServiceServiceManager = resourceManager.adminService.getServer().getServiceAuthorizationManager();
        verifyServiceACLsRefresh(adminServiceServiceManager, org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class, aclsString);
        // verify service ACLs for ClientRMService
        ServiceAuthorizationManager clientRMServiceServiceManager = resourceManager.getRMContext().getClientRMService().getServer().getServiceAuthorizationManager();
        verifyServiceACLsRefresh(clientRMServiceServiceManager, org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class, aclsString);
        // verify service ACLs for ApplicationMasterService
        ServiceAuthorizationManager appMasterService = resourceManager.getRMContext().getApplicationMasterService().getServer().getServiceAuthorizationManager();
        verifyServiceACLsRefresh(appMasterService, org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class, aclsString);
        // verify service ACLs for ResourceTrackerService
        ServiceAuthorizationManager RTService = resourceManager.getRMContext().getResourceTrackerService().getServer().getServiceAuthorizationManager();
        verifyServiceACLsRefresh(RTService, org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class, aclsString);
        // verify ProxyUsers and ProxyHosts
        ProxyUsers.refreshSuperUserGroupsConfiguration(configuration);
        Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyGroups().get("hadoop.proxyuser.test.groups").size() == 1);
        Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyGroups().get("hadoop.proxyuser.test.groups").contains("test_groups"));
        Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyHosts().get("hadoop.proxyuser.test.hosts").size() == 1);
        Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyHosts().get("hadoop.proxyuser.test.hosts").contains("test_hosts"));
        // verify UserToGroupsMappings
        List<String> groupAfter = Groups.getUserToGroupsMappingService(configuration).getGroups(UserGroupInformation.getCurrentUser().getUserName());
        Assert.assertTrue(groupAfter.contains("test_group_D") && groupAfter.contains("test_group_E") && groupAfter.contains("test_group_F") && groupAfter.size() == 3);
    } finally {
        if (resourceManager != null) {
            resourceManager.stop();
        }
    }
}

Example 20 with CapacitySchedulerConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration in project hadoop by apache.

the class TestCapacitySchedulerPlanFollower method setupPlanFollower.

private void setupPlanFollower() throws Exception {
    mClock = mock(Clock.class);
    mAgent = mock(ReservationAgent.class);
    String reservationQ = ReservationSystemTestUtil.getFullReservationQueueName();
    CapacitySchedulerConfiguration csConf = cs.getConfiguration();
    csConf.setReservationWindow(reservationQ, 20L);
    csConf.setMaximumCapacity(reservationQ, 40);
    csConf.setAverageCapacity(reservationQ, 20);
    policy.init(reservationQ, csConf);
}