Examples with UserGroupInformation org.apache.hadoop.security.UserGroupInformation used on opensource projects

Example 91 with UserGroupInformation

use of org.apache.hadoop.security.UserGroupInformation in project hbase by apache.

the class SecureBulkLoadManager method secureBulkLoadHFiles.

public Map<byte[], List<Path>> secureBulkLoadHFiles(final Region region, final BulkLoadHFileRequest request) throws IOException {
    final List<Pair<byte[], String>> familyPaths = new ArrayList<>(request.getFamilyPathCount());
    for (ClientProtos.BulkLoadHFileRequest.FamilyPath el : request.getFamilyPathList()) {
        familyPaths.add(new Pair<>(el.getFamily().toByteArray(), el.getPath()));
    }
    Token userToken = null;
    if (userProvider.isHadoopSecurityEnabled()) {
        userToken = new Token(request.getFsToken().getIdentifier().toByteArray(), request.getFsToken().getPassword().toByteArray(), new Text(request.getFsToken().getKind()), new Text(request.getFsToken().getService()));
    }
    final String bulkToken = request.getBulkToken();
    User user = getActiveUser();
    final UserGroupInformation ugi = user.getUGI();
    if (userProvider.isHadoopSecurityEnabled()) {
        try {
            Token tok = TokenUtil.obtainToken(conn);
            if (tok != null) {
                boolean b = ugi.addToken(tok);
                LOG.debug("token added " + tok + " for user " + ugi + " return=" + b);
            }
        } catch (IOException ioe) {
            LOG.warn("unable to add token", ioe);
        }
    }
    if (userToken != null) {
        ugi.addToken(userToken);
    } else if (userProvider.isHadoopSecurityEnabled()) {
        //for mini cluster testing
        throw new DoNotRetryIOException("User token cannot be null");
    }
    boolean bypass = false;
    if (region.getCoprocessorHost() != null) {
        bypass = region.getCoprocessorHost().preBulkLoadHFile(familyPaths);
    }
    boolean loaded = false;
    Map<byte[], List<Path>> map = null;
    try {
        if (!bypass) {
            // ('request user'), another for the target fs (HBase region server principal).
            if (userProvider.isHadoopSecurityEnabled()) {
                FsDelegationToken targetfsDelegationToken = new FsDelegationToken(userProvider, "renewer");
                targetfsDelegationToken.acquireDelegationToken(fs);
                Token<?> targetFsToken = targetfsDelegationToken.getUserToken();
                if (targetFsToken != null && (userToken == null || !targetFsToken.getService().equals(userToken.getService()))) {
                    ugi.addToken(targetFsToken);
                }
            }
            map = ugi.doAs(new PrivilegedAction<Map<byte[], List<Path>>>() {

                @Override
                public Map<byte[], List<Path>> run() {
                    FileSystem fs = null;
                    try {
                        fs = FileSystem.get(conf);
                        for (Pair<byte[], String> el : familyPaths) {
                            Path stageFamily = new Path(bulkToken, Bytes.toString(el.getFirst()));
                            if (!fs.exists(stageFamily)) {
                                fs.mkdirs(stageFamily);
                                fs.setPermission(stageFamily, PERM_ALL_ACCESS);
                            }
                        }
                        //To enable access prior to staging
                        return region.bulkLoadHFiles(familyPaths, true, new SecureBulkLoadListener(fs, bulkToken, conf), request.getCopyFile());
                    } catch (Exception e) {
                        LOG.error("Failed to complete bulk load", e);
                    }
                    return null;
                }
            });
            if (map != null) {
                loaded = true;
            }
        }
    } finally {
        if (region.getCoprocessorHost() != null) {
            region.getCoprocessorHost().postBulkLoadHFile(familyPaths, map, loaded);
        }
    }
    return map;
}

Example 92 with UserGroupInformation

use of org.apache.hadoop.security.UserGroupInformation in project hadoop by apache.

the class TestSchedulerUtils method testValidateResourceBlacklistRequest.

@Test
public void testValidateResourceBlacklistRequest() throws Exception {
    MyContainerManager containerManager = new MyContainerManager();
    final MockRMWithAMS rm = new MockRMWithAMS(new YarnConfiguration(), containerManager);
    rm.start();
    MockNM nm1 = rm.registerNode("localhost:1234", 5120);
    Map<ApplicationAccessType, String> acls = new HashMap<ApplicationAccessType, String>(2);
    acls.put(ApplicationAccessType.VIEW_APP, "*");
    RMApp app = rm.submitApp(1024, "appname", "appuser", acls);
    nm1.nodeHeartbeat(true);
    RMAppAttempt attempt = app.getCurrentAppAttempt();
    ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
    waitForLaunchedState(attempt);
    // Create a client to the RM.
    final Configuration conf = rm.getConfig();
    final YarnRPC rpc = YarnRPC.create(conf);
    UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(applicationAttemptId.toString());
    Credentials credentials = containerManager.getContainerCredentials();
    final InetSocketAddress rmBindAddress = rm.getApplicationMasterService().getBindAddress();
    Token<? extends TokenIdentifier> amRMToken = MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress, credentials.getAllTokens());
    currentUser.addToken(amRMToken);
    ApplicationMasterProtocol client = currentUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {

        @Override
        public ApplicationMasterProtocol run() {
            return (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol.class, rmBindAddress, conf);
        }
    });
    RegisterApplicationMasterRequest request = Records.newRecord(RegisterApplicationMasterRequest.class);
    client.registerApplicationMaster(request);
    ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(Collections.singletonList(ResourceRequest.ANY), null);
    AllocateRequest allocateRequest = AllocateRequest.newInstance(0, 0.0f, null, null, blacklistRequest);
    boolean error = false;
    try {
        client.allocate(allocateRequest);
    } catch (InvalidResourceBlacklistRequestException e) {
        error = true;
    }
    rm.stop();
    Assert.assertTrue("Didn't not catch InvalidResourceBlacklistRequestException", error);
}

Example 93 with UserGroupInformation

use of org.apache.hadoop.security.UserGroupInformation in project hadoop by apache.

the class TestApplicationPriority method testUpdateInvalidPriorityAtRuntime.

@Test
public void testUpdateInvalidPriorityAtRuntime() throws Exception {
    Configuration conf = new Configuration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    // Set Max Application Priority as 10
    conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY, 10);
    MockRM rm = new MockRM(conf);
    rm.start();
    Priority appPriority1 = Priority.newInstance(5);
    MockNM nm1 = rm.registerNode("127.0.0.1:1234", 16 * GB);
    RMApp app1 = rm.submitApp(1 * GB, appPriority1);
    // kick the scheduler, 1 GB given to AM1, remaining 15GB on nm1
    MockAM am1 = MockRM.launchAM(app1, rm, nm1);
    am1.registerAppAttempt();
    // get scheduler
    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
    // Change the priority of App1 to 15
    Priority appPriority2 = Priority.newInstance(15);
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(app1.getUser());
    cs.updateApplicationPriority(appPriority2, app1.getApplicationId(), null, ugi);
    // get scheduler app
    FiCaSchedulerApp schedulerAppAttempt = cs.getSchedulerApplications().get(app1.getApplicationId()).getCurrentAppAttempt();
    // Verify whether priority 15 is reset to 10
    Priority appPriority3 = Priority.newInstance(10);
    Assert.assertEquals(appPriority3, schedulerAppAttempt.getPriority());
    rm.stop();
}

Example 94 with UserGroupInformation

use of org.apache.hadoop.security.UserGroupInformation in project hadoop by apache.

the class TestApplicationPriority method testRMRestartWithChangeInPriority.

@Test(timeout = 180000)
public void testRMRestartWithChangeInPriority() throws Exception {
    conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
    conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, false);
    conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
    conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
    conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY, 10);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    RMState rmState = memStore.getState();
    Map<ApplicationId, ApplicationStateData> rmAppState = rmState.getApplicationState();
    // PHASE 1: create state in an RM
    // start RM
    MockRM rm1 = new MockRM(conf, memStore);
    rm1.start();
    MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
    nm1.registerNode();
    Priority appPriority1 = Priority.newInstance(5);
    RMApp app1 = rm1.submitApp(1 * GB, appPriority1);
    // kick the scheduler, 1 GB given to AM1, remaining 15GB on nm1
    MockAM am1 = MockRM.launchAM(app1, rm1, nm1);
    am1.registerAppAttempt();
    // get scheduler
    CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
    // Change the priority of App1 to 8
    Priority appPriority2 = Priority.newInstance(8);
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(app1.getUser());
    cs.updateApplicationPriority(appPriority2, app1.getApplicationId(), null, ugi);
    // let things settle down
    Thread.sleep(1000);
    // create new RM to represent restart and recover state
    MockRM rm2 = new MockRM(conf, memStore);
    // start new RM
    rm2.start();
    // change NM to point to new RM
    nm1.setResourceTrackerService(rm2.getResourceTrackerService());
    // Verify RM Apps after this restart
    Assert.assertEquals(1, rm2.getRMContext().getRMApps().size());
    // get scheduler app
    RMApp loadedApp = rm2.getRMContext().getRMApps().get(app1.getApplicationId());
    // Verify whether priority 15 is reset to 10
    Assert.assertEquals(appPriority2, loadedApp.getApplicationPriority());
    rm2.stop();
    rm1.stop();
}

Example 95 with UserGroupInformation

use of org.apache.hadoop.security.UserGroupInformation in project hadoop by apache.

the class TestApplicationPriority method testUpdatePriorityAtRuntime.

@Test
public void testUpdatePriorityAtRuntime() throws Exception {
    Configuration conf = new Configuration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    // Set Max Application Priority as 10
    conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY, 10);
    MockRM rm = new MockRM(conf);
    rm.start();
    Priority appPriority1 = Priority.newInstance(5);
    MockNM nm1 = rm.registerNode("127.0.0.1:1234", 16 * GB);
    RMApp app1 = rm.submitApp(1 * GB, appPriority1);
    // kick the scheduler, 1 GB given to AM1, remaining 15GB on nm1
    MockAM am1 = MockRM.launchAM(app1, rm, nm1);
    am1.registerAppAttempt();
    // get scheduler
    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
    // Change the priority of App1 to 8
    Priority appPriority2 = Priority.newInstance(8);
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(app1.getUser());
    cs.updateApplicationPriority(appPriority2, app1.getApplicationId(), null, ugi);
    // get scheduler app
    FiCaSchedulerApp schedulerAppAttempt = cs.getSchedulerApplications().get(app1.getApplicationId()).getCurrentAppAttempt();
    // Verify whether the new priority is updated
    Assert.assertEquals(appPriority2, schedulerAppAttempt.getPriority());
}