Examples with Credentials org.apache.hadoop.security.Credentials used on opensource projects

Example 16 with Credentials

use of org.apache.hadoop.security.Credentials in project storm by apache.

the class AutoHDFS method addTokensToUGI.

public void addTokensToUGI(Subject subject) {
    if (subject != null) {
        Set<Credentials> privateCredentials = subject.getPrivateCredentials(Credentials.class);
        if (privateCredentials != null) {
            for (Credentials cred : privateCredentials) {
                Collection<Token<? extends TokenIdentifier>> allTokens = cred.getAllTokens();
                if (allTokens != null) {
                    for (Token<? extends TokenIdentifier> token : allTokens) {
                        try {
                            UserGroupInformation.getCurrentUser().addToken(token);
                            LOG.info("Added delegation tokens to UGI.");
                        } catch (IOException e) {
                            LOG.error("Exception while trying to add tokens to ugi", e);
                        }
                    }
                }
            }
        }
    }
}

Example 17 with Credentials

use of org.apache.hadoop.security.Credentials in project hadoop by apache.

the class TestAMRMTokens method testMasterKeyRollOver.

/**
   * Validate master-key-roll-over and that tokens are usable even after
   * master-key-roll-over.
   * 
   * @throws Exception
   */
@Test
public void testMasterKeyRollOver() throws Exception {
    conf.setLong(YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, rolling_interval_sec);
    conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms);
    conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, "0.0.0.0:0");
    MyContainerManager containerManager = new MyContainerManager();
    final MockRMWithAMS rm = new MockRMWithAMS(conf, containerManager);
    rm.start();
    Long startTime = System.currentTimeMillis();
    final Configuration conf = rm.getConfig();
    final YarnRPC rpc = YarnRPC.create(conf);
    ApplicationMasterProtocol rmClient = null;
    AMRMTokenSecretManager appTokenSecretManager = rm.getRMContext().getAMRMTokenSecretManager();
    MasterKeyData oldKey = appTokenSecretManager.getMasterKey();
    Assert.assertNotNull(oldKey);
    try {
        MockNM nm1 = rm.registerNode("localhost:1234", 5120);
        RMApp app = rm.submitApp(1024);
        nm1.nodeHeartbeat(true);
        int waitCount = 0;
        while (containerManager.containerTokens == null && waitCount++ < maxWaitAttempts) {
            LOG.info("Waiting for AM Launch to happen..");
            Thread.sleep(1000);
        }
        Assert.assertNotNull(containerManager.containerTokens);
        RMAppAttempt attempt = app.getCurrentAppAttempt();
        ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
        // Create a client to the RM.
        UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(applicationAttemptId.toString());
        Credentials credentials = containerManager.getContainerCredentials();
        final InetSocketAddress rmBindAddress = rm.getApplicationMasterService().getBindAddress();
        Token<? extends TokenIdentifier> amRMToken = MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress, credentials.getAllTokens());
        currentUser.addToken(amRMToken);
        rmClient = createRMClient(rm, conf, rpc, currentUser);
        RegisterApplicationMasterRequest request = Records.newRecord(RegisterApplicationMasterRequest.class);
        rmClient.registerApplicationMaster(request);
        // One allocate call.
        AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class);
        Assert.assertTrue(rmClient.allocate(allocateRequest).getAMCommand() == null);
        // At mean time, the old AMRMToken should continue to work
        while (System.currentTimeMillis() - startTime < rolling_interval_sec * 1000) {
            rmClient.allocate(allocateRequest);
            Thread.sleep(500);
        }
        MasterKeyData newKey = appTokenSecretManager.getMasterKey();
        Assert.assertNotNull(newKey);
        Assert.assertFalse("Master key should have changed!", oldKey.equals(newKey));
        // Another allocate call with old AMRMToken. Should continue to work.
        // To avoid using cached client
        rpc.stopProxy(rmClient, conf);
        rmClient = createRMClient(rm, conf, rpc, currentUser);
        Assert.assertTrue(rmClient.allocate(allocateRequest).getAMCommand() == null);
        waitCount = 0;
        while (waitCount++ <= maxWaitAttempts) {
            if (appTokenSecretManager.getCurrnetMasterKeyData() != oldKey) {
                break;
            }
            try {
                rmClient.allocate(allocateRequest);
            } catch (Exception ex) {
                break;
            }
            Thread.sleep(200);
        }
        // active the nextMasterKey, and replace the currentMasterKey
        Assert.assertTrue(appTokenSecretManager.getCurrnetMasterKeyData().equals(newKey));
        Assert.assertTrue(appTokenSecretManager.getMasterKey().equals(newKey));
        Assert.assertTrue(appTokenSecretManager.getNextMasterKeyData() == null);
        // Create a new Token
        Token<AMRMTokenIdentifier> newToken = appTokenSecretManager.createAndGetAMRMToken(applicationAttemptId);
        SecurityUtil.setTokenService(newToken, rmBindAddress);
        currentUser.addToken(newToken);
        // Another allocate call. Should continue to work.
        // To avoid using cached client
        rpc.stopProxy(rmClient, conf);
        rmClient = createRMClient(rm, conf, rpc, currentUser);
        allocateRequest = Records.newRecord(AllocateRequest.class);
        Assert.assertTrue(rmClient.allocate(allocateRequest).getAMCommand() == null);
        // Should not work by using the old AMRMToken.
        // To avoid using cached client
        rpc.stopProxy(rmClient, conf);
        try {
            currentUser.addToken(amRMToken);
            rmClient = createRMClient(rm, conf, rpc, currentUser);
            allocateRequest = Records.newRecord(AllocateRequest.class);
            Assert.assertTrue(rmClient.allocate(allocateRequest).getAMCommand() == null);
            Assert.fail("The old Token should not work");
        } catch (Exception ex) {
        // expect exception
        }
    } finally {
        rm.stop();
        if (rmClient != null) {
            // To avoid using cached client
            rpc.stopProxy(rmClient, conf);
        }
    }
}

Example 18 with Credentials

use of org.apache.hadoop.security.Credentials in project hadoop by apache.

the class TestDelegationTokenRenewer method testFSLeakInObtainSystemTokensForUser.

// Test FileSystem memory leak in obtainSystemTokensForUser.
@Test
public void testFSLeakInObtainSystemTokensForUser() throws Exception {
    Credentials credentials = new Credentials();
    String user = "test";
    int oldCounter = MyFS.getInstanceCounter();
    delegationTokenRenewer.obtainSystemTokensForUser(user, credentials);
    delegationTokenRenewer.obtainSystemTokensForUser(user, credentials);
    delegationTokenRenewer.obtainSystemTokensForUser(user, credentials);
    Assert.assertEquals(oldCounter, MyFS.getInstanceCounter());
}

Example 19 with Credentials

use of org.apache.hadoop.security.Credentials in project hadoop by apache.

the class TestDelegationTokenRenewer method testRMRestartWithExpiredToken.

// 1. token is expired before app completes.
// 2. RM shutdown.
// 3. When RM recovers the app, token renewal will fail as token expired.
//    RM should request a new token and sent it to NM for log-aggregation.
@Test
public void testRMRestartWithExpiredToken() throws Exception {
    Configuration yarnConf = new YarnConfiguration();
    yarnConf.setBoolean(YarnConfiguration.RM_PROXY_USER_PRIVILEGES_ENABLED, true);
    yarnConf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    yarnConf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
    yarnConf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
    UserGroupInformation.setConfiguration(yarnConf);
    // create Token1:
    Text userText1 = new Text("user1");
    DelegationTokenIdentifier dtId1 = new DelegationTokenIdentifier(userText1, new Text("renewer1"), userText1);
    final Token<DelegationTokenIdentifier> originalToken = new Token<>(dtId1.getBytes(), "password1".getBytes(), dtId1.getKind(), new Text("service1"));
    Credentials credentials = new Credentials();
    credentials.addToken(userText1, originalToken);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(yarnConf);
    MockRM rm1 = new TestSecurityMockRM(yarnConf, memStore);
    rm1.start();
    RMApp app = rm1.submitApp(200, "name", "user", new HashMap<ApplicationAccessType, String>(), false, "default", 1, credentials);
    // create token2
    Text userText2 = new Text("user1");
    DelegationTokenIdentifier dtId2 = new DelegationTokenIdentifier(userText1, new Text("renewer2"), userText2);
    final Token<DelegationTokenIdentifier> updatedToken = new Token<DelegationTokenIdentifier>(dtId2.getBytes(), "password2".getBytes(), dtId2.getKind(), new Text("service2"));
    AtomicBoolean firstRenewInvoked = new AtomicBoolean(false);
    AtomicBoolean secondRenewInvoked = new AtomicBoolean(false);
    MockRM rm2 = new TestSecurityMockRM(yarnConf, memStore) {

        @Override
        protected DelegationTokenRenewer createDelegationTokenRenewer() {
            return new DelegationTokenRenewer() {

                @Override
                protected void renewToken(final DelegationTokenToRenew dttr) throws IOException {
                    if (dttr.token.equals(updatedToken)) {
                        secondRenewInvoked.set(true);
                        super.renewToken(dttr);
                    } else if (dttr.token.equals(originalToken)) {
                        firstRenewInvoked.set(true);
                        throw new InvalidToken("Failed to renew");
                    } else {
                        throw new IOException("Unexpected");
                    }
                }

                @Override
                protected Token<?>[] obtainSystemTokensForUser(String user, final Credentials credentials) throws IOException {
                    credentials.addToken(updatedToken.getService(), updatedToken);
                    return new Token<?>[] { updatedToken };
                }
            };
        }
    };
    // simulating restart the rm
    rm2.start();
    // check nm can retrieve the token
    final MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm2.getResourceTrackerService());
    nm1.registerNode();
    NodeHeartbeatResponse response = nm1.nodeHeartbeat(true);
    ByteBuffer tokenBuffer = response.getSystemCredentialsForApps().get(app.getApplicationId());
    Assert.assertNotNull(tokenBuffer);
    Credentials appCredentials = new Credentials();
    DataInputByteBuffer buf = new DataInputByteBuffer();
    tokenBuffer.rewind();
    buf.reset(tokenBuffer);
    appCredentials.readTokenStorageStream(buf);
    Assert.assertTrue(firstRenewInvoked.get() && secondRenewInvoked.get());
    Assert.assertTrue(appCredentials.getAllTokens().contains(updatedToken));
}

Example 20 with Credentials

use of org.apache.hadoop.security.Credentials in project hadoop by apache.

the class TestDelegationTokenRenewer method testCancelWithMultipleAppSubmissions.

// Test submitting an application with the token obtained by a previously
// submitted application that is set to be cancelled.  Token should be
// renewed while all apps are running, and then cancelled when all apps
// complete
@Test(timeout = 30000)
public void testCancelWithMultipleAppSubmissions() throws Exception {
    MockRM rm = new TestSecurityMockRM(conf, null);
    rm.start();
    final MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
    nm1.registerNode();
    // create Token1:
    Text userText1 = new Text("user");
    DelegationTokenIdentifier dtId1 = new DelegationTokenIdentifier(userText1, new Text("renewer1"), userText1);
    final Token<DelegationTokenIdentifier> token1 = new Token<DelegationTokenIdentifier>(dtId1.getBytes(), "password1".getBytes(), dtId1.getKind(), new Text("service1"));
    Credentials credentials = new Credentials();
    credentials.addToken(token1.getService(), token1);
    DelegationTokenRenewer renewer = rm.getRMContext().getDelegationTokenRenewer();
    Assert.assertTrue(renewer.getAllTokens().isEmpty());
    Assert.assertFalse(Renewer.cancelled);
    Resource resource = Records.newRecord(Resource.class);
    resource.setMemorySize(200);
    RMApp app1 = rm.submitApp(resource, "name", "user", null, false, null, 2, credentials, null, true, false, false, null, 0, null, true, null);
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
    rm.waitForState(app1.getApplicationId(), RMAppState.RUNNING);
    DelegationTokenToRenew dttr = renewer.getAllTokens().get(token1);
    Assert.assertNotNull(dttr);
    Assert.assertTrue(dttr.referringAppIds.contains(app1.getApplicationId()));
    RMApp app2 = rm.submitApp(resource, "name", "user", null, false, null, 2, credentials, null, true, false, false, null, 0, null, true, null);
    MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1);
    rm.waitForState(app2.getApplicationId(), RMAppState.RUNNING);
    Assert.assertTrue(renewer.getAllTokens().containsKey(token1));
    Assert.assertTrue(dttr.referringAppIds.contains(app2.getApplicationId()));
    Assert.assertTrue(dttr.referringAppIds.contains(app2.getApplicationId()));
    Assert.assertFalse(Renewer.cancelled);
    finishAMAndWaitForComplete(app2, rm, nm1, am2, dttr);
    // app2 completes, app1 is still running, check the token is not cancelled
    Assert.assertTrue(renewer.getAllTokens().containsKey(token1));
    Assert.assertTrue(dttr.referringAppIds.contains(app1.getApplicationId()));
    Assert.assertFalse(dttr.referringAppIds.contains(app2.getApplicationId()));
    Assert.assertFalse(dttr.isTimerCancelled());
    Assert.assertFalse(Renewer.cancelled);
    RMApp app3 = rm.submitApp(resource, "name", "user", null, false, null, 2, credentials, null, true, false, false, null, 0, null, true, null);
    MockAM am3 = MockRM.launchAndRegisterAM(app3, rm, nm1);
    rm.waitForState(app3.getApplicationId(), RMAppState.RUNNING);
    Assert.assertTrue(renewer.getAllTokens().containsKey(token1));
    Assert.assertTrue(dttr.referringAppIds.contains(app1.getApplicationId()));
    Assert.assertTrue(dttr.referringAppIds.contains(app3.getApplicationId()));
    Assert.assertFalse(dttr.isTimerCancelled());
    Assert.assertFalse(Renewer.cancelled);
    finishAMAndWaitForComplete(app1, rm, nm1, am1, dttr);
    Assert.assertTrue(renewer.getAllTokens().containsKey(token1));
    Assert.assertFalse(dttr.referringAppIds.contains(app1.getApplicationId()));
    Assert.assertTrue(dttr.referringAppIds.contains(app3.getApplicationId()));
    Assert.assertFalse(dttr.isTimerCancelled());
    Assert.assertFalse(Renewer.cancelled);
    finishAMAndWaitForComplete(app3, rm, nm1, am3, dttr);
    Assert.assertFalse(renewer.getAllTokens().containsKey(token1));
    Assert.assertTrue(dttr.referringAppIds.isEmpty());
    Assert.assertTrue(dttr.isTimerCancelled());
    Assert.assertTrue(Renewer.cancelled);
    // make sure the token also has been removed from appTokens
    Assert.assertFalse(renewer.getDelegationTokens().contains(token1));
}