Examples with InvalidToken org.apache.hadoop.security.token.SecretManager.InvalidToken used on opensource projects

Example 6 with InvalidToken

use of org.apache.hadoop.security.token.SecretManager.InvalidToken in project hadoop by apache.

the class TestAMRMClientOnRMRestart method testAMRMClientOnAMRMTokenRollOverOnRMRestart.

// Test verify for AM issued with rolled-over AMRMToken
// is still able to communicate with restarted RM.
@Test(timeout = 30000)
public void testAMRMClientOnAMRMTokenRollOverOnRMRestart() throws Exception {
    conf.setLong(YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, rolling_interval_sec);
    conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    // start first RM
    MyResourceManager2 rm1 = new MyResourceManager2(conf, memStore);
    rm1.start();
    DrainDispatcher dispatcher = (DrainDispatcher) rm1.getRMContext().getDispatcher();
    Long startTime = System.currentTimeMillis();
    // Submit the application
    RMApp app = rm1.submitApp(1024);
    dispatcher.await();
    MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
    nm1.registerNode();
    // Node heartbeat
    nm1.nodeHeartbeat(true);
    dispatcher.await();
    ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt().getAppAttemptId();
    rm1.sendAMLaunched(appAttemptId);
    dispatcher.await();
    AMRMTokenSecretManager amrmTokenSecretManagerForRM1 = rm1.getRMContext().getAMRMTokenSecretManager();
    org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> token = amrmTokenSecretManagerForRM1.createAndGetAMRMToken(appAttemptId);
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    ugi.addTokenIdentifier(token.decodeIdentifier());
    AMRMClient<ContainerRequest> amClient = new MyAMRMClientImpl(rm1);
    amClient.init(conf);
    amClient.start();
    amClient.registerApplicationMaster("h1", 10000, "");
    amClient.allocate(0.1f);
    // At mean time, the old AMRMToken should continue to work
    while (System.currentTimeMillis() - startTime < rolling_interval_sec * 1000) {
        amClient.allocate(0.1f);
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
        // DO NOTHING
        }
    }
    Assert.assertTrue(amrmTokenSecretManagerForRM1.getMasterKey().getMasterKey().getKeyId() != token.decodeIdentifier().getKeyId());
    amClient.allocate(0.1f);
    // active the nextMasterKey, and replace the currentMasterKey
    org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> newToken = amrmTokenSecretManagerForRM1.createAndGetAMRMToken(appAttemptId);
    int waitCount = 0;
    while (waitCount++ <= 50) {
        if (amrmTokenSecretManagerForRM1.getCurrnetMasterKeyData().getMasterKey().getKeyId() != token.decodeIdentifier().getKeyId()) {
            break;
        }
        try {
            amClient.allocate(0.1f);
        } catch (Exception ex) {
            break;
        }
        Thread.sleep(500);
    }
    Assert.assertTrue(amrmTokenSecretManagerForRM1.getNextMasterKeyData() == null);
    Assert.assertTrue(amrmTokenSecretManagerForRM1.getCurrnetMasterKeyData().getMasterKey().getKeyId() == newToken.decodeIdentifier().getKeyId());
    // start 2nd RM
    conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, "0.0.0.0:" + ServerSocketUtil.getPort(45020, 10));
    final MyResourceManager2 rm2 = new MyResourceManager2(conf, memStore);
    rm2.start();
    nm1.setResourceTrackerService(rm2.getResourceTrackerService());
    ((MyAMRMClientImpl) amClient).updateRMProxy(rm2);
    dispatcher = (DrainDispatcher) rm2.getRMContext().getDispatcher();
    AMRMTokenSecretManager amrmTokenSecretManagerForRM2 = rm2.getRMContext().getAMRMTokenSecretManager();
    Assert.assertTrue(amrmTokenSecretManagerForRM2.getCurrnetMasterKeyData().getMasterKey().getKeyId() == newToken.decodeIdentifier().getKeyId());
    Assert.assertTrue(amrmTokenSecretManagerForRM2.getNextMasterKeyData() == null);
    try {
        UserGroupInformation testUser = UserGroupInformation.createRemoteUser("testUser");
        SecurityUtil.setTokenService(token, rm2.getApplicationMasterService().getBindAddress());
        testUser.addToken(token);
        testUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {

            @Override
            public ApplicationMasterProtocol run() {
                return (ApplicationMasterProtocol) YarnRPC.create(conf).getProxy(ApplicationMasterProtocol.class, rm2.getApplicationMasterService().getBindAddress(), conf);
            }
        }).allocate(Records.newRecord(AllocateRequest.class));
        Assert.fail("The old Token should not work");
    } catch (Exception ex) {
        Assert.assertTrue(ex instanceof InvalidToken);
        Assert.assertTrue(ex.getMessage().contains("Invalid AMRMToken from " + token.decodeIdentifier().getApplicationAttemptId()));
    }
    // make sure the recovered AMRMToken works for new RM
    amClient.allocate(0.1f);
    amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null);
    amClient.stop();
    rm1.stop();
    rm2.stop();
}

Example 7 with InvalidToken

use of org.apache.hadoop.security.token.SecretManager.InvalidToken in project hadoop by apache.

the class RMWebServices method renewDelegationToken.

private Response renewDelegationToken(DelegationToken tokenData, HttpServletRequest hsr, UserGroupInformation callerUGI) throws AuthorizationException, IOException, InterruptedException, Exception {
    Token<RMDelegationTokenIdentifier> token = extractToken(tokenData.getToken());
    org.apache.hadoop.yarn.api.records.Token dToken = BuilderUtils.newDelegationToken(token.getIdentifier(), token.getKind().toString(), token.getPassword(), token.getService().toString());
    final RenewDelegationTokenRequest req = RenewDelegationTokenRequest.newInstance(dToken);
    RenewDelegationTokenResponse resp;
    try {
        resp = callerUGI.doAs(new PrivilegedExceptionAction<RenewDelegationTokenResponse>() {

            @Override
            public RenewDelegationTokenResponse run() throws IOException, YarnException {
                return rm.getClientRMService().renewDelegationToken(req);
            }
        });
    } catch (UndeclaredThrowableException ue) {
        if (ue.getCause() instanceof YarnException) {
            if (ue.getCause().getCause() instanceof InvalidToken) {
                throw new BadRequestException(ue.getCause().getCause().getMessage());
            } else if (ue.getCause().getCause() instanceof org.apache.hadoop.security.AccessControlException) {
                return Response.status(Status.FORBIDDEN).entity(ue.getCause().getCause().getMessage()).build();
            }
            LOG.info("Renew delegation token request failed", ue);
            throw ue;
        }
        LOG.info("Renew delegation token request failed", ue);
        throw ue;
    } catch (Exception e) {
        LOG.info("Renew delegation token request failed", e);
        throw e;
    }
    long renewTime = resp.getNextExpirationTime();
    DelegationToken respToken = new DelegationToken();
    respToken.setNextExpirationTime(renewTime);
    return Response.status(Status.OK).entity(respToken).build();
}

Example 8 with InvalidToken

use of org.apache.hadoop.security.token.SecretManager.InvalidToken in project hadoop by apache.

the class TestNMContainerTokenSecretManager method testRecovery.

@Test
public void testRecovery() throws IOException {
    YarnConfiguration conf = new YarnConfiguration();
    conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
    final NodeId nodeId = NodeId.newInstance("somehost", 1234);
    final ContainerId cid1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    final ContainerId cid2 = BuilderUtils.newContainerId(2, 2, 2, 2);
    ContainerTokenKeyGeneratorForTest keygen = new ContainerTokenKeyGeneratorForTest(conf);
    NMMemoryStateStoreService stateStore = new NMMemoryStateStoreService();
    stateStore.init(conf);
    stateStore.start();
    NMContainerTokenSecretManager secretMgr = new NMContainerTokenSecretManager(conf, stateStore);
    secretMgr.setNodeId(nodeId);
    MasterKey currentKey = keygen.generateKey();
    secretMgr.setMasterKey(currentKey);
    ContainerTokenIdentifier tokenId1 = createContainerTokenId(cid1, nodeId, "user1", secretMgr);
    ContainerTokenIdentifier tokenId2 = createContainerTokenId(cid2, nodeId, "user2", secretMgr);
    assertNotNull(secretMgr.retrievePassword(tokenId1));
    assertNotNull(secretMgr.retrievePassword(tokenId2));
    // restart and verify tokens still valid
    secretMgr = new NMContainerTokenSecretManager(conf, stateStore);
    secretMgr.setNodeId(nodeId);
    secretMgr.recover();
    assertEquals(currentKey, secretMgr.getCurrentKey());
    assertTrue(secretMgr.isValidStartContainerRequest(tokenId1));
    assertTrue(secretMgr.isValidStartContainerRequest(tokenId2));
    assertNotNull(secretMgr.retrievePassword(tokenId1));
    assertNotNull(secretMgr.retrievePassword(tokenId2));
    // roll master key and start a container
    secretMgr.startContainerSuccessful(tokenId2);
    currentKey = keygen.generateKey();
    secretMgr.setMasterKey(currentKey);
    // restart and verify tokens still valid due to prev key persist
    secretMgr = new NMContainerTokenSecretManager(conf, stateStore);
    secretMgr.setNodeId(nodeId);
    secretMgr.recover();
    assertEquals(currentKey, secretMgr.getCurrentKey());
    assertTrue(secretMgr.isValidStartContainerRequest(tokenId1));
    assertFalse(secretMgr.isValidStartContainerRequest(tokenId2));
    assertNotNull(secretMgr.retrievePassword(tokenId1));
    assertNotNull(secretMgr.retrievePassword(tokenId2));
    // roll master key again, restart, and verify keys no longer valid
    currentKey = keygen.generateKey();
    secretMgr.setMasterKey(currentKey);
    secretMgr = new NMContainerTokenSecretManager(conf, stateStore);
    secretMgr.setNodeId(nodeId);
    secretMgr.recover();
    assertEquals(currentKey, secretMgr.getCurrentKey());
    assertTrue(secretMgr.isValidStartContainerRequest(tokenId1));
    assertFalse(secretMgr.isValidStartContainerRequest(tokenId2));
    try {
        secretMgr.retrievePassword(tokenId1);
        fail("token should not be valid");
    } catch (InvalidToken e) {
    // expected
    }
    try {
        secretMgr.retrievePassword(tokenId2);
        fail("token should not be valid");
    } catch (InvalidToken e) {
    // expected
    }
    stateStore.close();
}

Example 9 with InvalidToken

use of org.apache.hadoop.security.token.SecretManager.InvalidToken in project hadoop by apache.

the class TestNMTokenSecretManagerInNM method testRecovery.

@Test
public void testRecovery() throws IOException {
    YarnConfiguration conf = new YarnConfiguration();
    conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
    final NodeId nodeId = NodeId.newInstance("somehost", 1234);
    final ApplicationAttemptId attempt1 = ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1);
    final ApplicationAttemptId attempt2 = ApplicationAttemptId.newInstance(ApplicationId.newInstance(2, 2), 2);
    NMTokenKeyGeneratorForTest keygen = new NMTokenKeyGeneratorForTest();
    NMMemoryStateStoreService stateStore = new NMMemoryStateStoreService();
    stateStore.init(conf);
    stateStore.start();
    NMTokenSecretManagerInNM secretMgr = new NMTokenSecretManagerInNM(stateStore);
    secretMgr.setNodeId(nodeId);
    MasterKey currentKey = keygen.generateKey();
    secretMgr.setMasterKey(currentKey);
    NMTokenIdentifier attemptToken1 = getNMTokenId(secretMgr.createNMToken(attempt1, nodeId, "user1"));
    NMTokenIdentifier attemptToken2 = getNMTokenId(secretMgr.createNMToken(attempt2, nodeId, "user2"));
    secretMgr.appAttemptStartContainer(attemptToken1);
    secretMgr.appAttemptStartContainer(attemptToken2);
    assertTrue(secretMgr.isAppAttemptNMTokenKeyPresent(attempt1));
    assertTrue(secretMgr.isAppAttemptNMTokenKeyPresent(attempt2));
    assertNotNull(secretMgr.retrievePassword(attemptToken1));
    assertNotNull(secretMgr.retrievePassword(attemptToken2));
    // restart and verify key is still there and token still valid
    secretMgr = new NMTokenSecretManagerInNM(stateStore);
    secretMgr.recover();
    secretMgr.setNodeId(nodeId);
    assertEquals(currentKey, secretMgr.getCurrentKey());
    assertTrue(secretMgr.isAppAttemptNMTokenKeyPresent(attempt1));
    assertTrue(secretMgr.isAppAttemptNMTokenKeyPresent(attempt2));
    assertNotNull(secretMgr.retrievePassword(attemptToken1));
    assertNotNull(secretMgr.retrievePassword(attemptToken2));
    // roll master key and remove an app
    currentKey = keygen.generateKey();
    secretMgr.setMasterKey(currentKey);
    secretMgr.appFinished(attempt1.getApplicationId());
    // restart and verify attempt1 key is still valid due to prev key persist
    secretMgr = new NMTokenSecretManagerInNM(stateStore);
    secretMgr.recover();
    secretMgr.setNodeId(nodeId);
    assertEquals(currentKey, secretMgr.getCurrentKey());
    assertFalse(secretMgr.isAppAttemptNMTokenKeyPresent(attempt1));
    assertTrue(secretMgr.isAppAttemptNMTokenKeyPresent(attempt2));
    assertNotNull(secretMgr.retrievePassword(attemptToken1));
    assertNotNull(secretMgr.retrievePassword(attemptToken2));
    // roll master key again, restart, and verify attempt1 key is bad but
    // attempt2 is still good due to app key persist
    currentKey = keygen.generateKey();
    secretMgr.setMasterKey(currentKey);
    secretMgr = new NMTokenSecretManagerInNM(stateStore);
    secretMgr.recover();
    secretMgr.setNodeId(nodeId);
    assertEquals(currentKey, secretMgr.getCurrentKey());
    assertFalse(secretMgr.isAppAttemptNMTokenKeyPresent(attempt1));
    assertTrue(secretMgr.isAppAttemptNMTokenKeyPresent(attempt2));
    try {
        secretMgr.retrievePassword(attemptToken1);
        fail("attempt token should not still be valid");
    } catch (InvalidToken e) {
    // expected
    }
    assertNotNull(secretMgr.retrievePassword(attemptToken2));
    // remove last attempt, restart, verify both tokens are now bad
    secretMgr.appFinished(attempt2.getApplicationId());
    secretMgr = new NMTokenSecretManagerInNM(stateStore);
    secretMgr.recover();
    secretMgr.setNodeId(nodeId);
    assertEquals(currentKey, secretMgr.getCurrentKey());
    assertFalse(secretMgr.isAppAttemptNMTokenKeyPresent(attempt1));
    assertFalse(secretMgr.isAppAttemptNMTokenKeyPresent(attempt2));
    try {
        secretMgr.retrievePassword(attemptToken1);
        fail("attempt token should not still be valid");
    } catch (InvalidToken e) {
    // expected
    }
    try {
        secretMgr.retrievePassword(attemptToken2);
        fail("attempt token should not still be valid");
    } catch (InvalidToken e) {
    // expected
    }
    stateStore.close();
}

Example 10 with InvalidToken

use of org.apache.hadoop.security.token.SecretManager.InvalidToken in project hadoop by apache.

the class TestDelegationTokenRenewer method testRMRestartWithExpiredToken.

// 1. token is expired before app completes.
// 2. RM shutdown.
// 3. When RM recovers the app, token renewal will fail as token expired.
//    RM should request a new token and sent it to NM for log-aggregation.
@Test
public void testRMRestartWithExpiredToken() throws Exception {
    Configuration yarnConf = new YarnConfiguration();
    yarnConf.setBoolean(YarnConfiguration.RM_PROXY_USER_PRIVILEGES_ENABLED, true);
    yarnConf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    yarnConf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
    yarnConf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
    UserGroupInformation.setConfiguration(yarnConf);
    // create Token1:
    Text userText1 = new Text("user1");
    DelegationTokenIdentifier dtId1 = new DelegationTokenIdentifier(userText1, new Text("renewer1"), userText1);
    final Token<DelegationTokenIdentifier> originalToken = new Token<>(dtId1.getBytes(), "password1".getBytes(), dtId1.getKind(), new Text("service1"));
    Credentials credentials = new Credentials();
    credentials.addToken(userText1, originalToken);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(yarnConf);
    MockRM rm1 = new TestSecurityMockRM(yarnConf, memStore);
    rm1.start();
    RMApp app = rm1.submitApp(200, "name", "user", new HashMap<ApplicationAccessType, String>(), false, "default", 1, credentials);
    // create token2
    Text userText2 = new Text("user1");
    DelegationTokenIdentifier dtId2 = new DelegationTokenIdentifier(userText1, new Text("renewer2"), userText2);
    final Token<DelegationTokenIdentifier> updatedToken = new Token<DelegationTokenIdentifier>(dtId2.getBytes(), "password2".getBytes(), dtId2.getKind(), new Text("service2"));
    AtomicBoolean firstRenewInvoked = new AtomicBoolean(false);
    AtomicBoolean secondRenewInvoked = new AtomicBoolean(false);
    MockRM rm2 = new TestSecurityMockRM(yarnConf, memStore) {

        @Override
        protected DelegationTokenRenewer createDelegationTokenRenewer() {
            return new DelegationTokenRenewer() {

                @Override
                protected void renewToken(final DelegationTokenToRenew dttr) throws IOException {
                    if (dttr.token.equals(updatedToken)) {
                        secondRenewInvoked.set(true);
                        super.renewToken(dttr);
                    } else if (dttr.token.equals(originalToken)) {
                        firstRenewInvoked.set(true);
                        throw new InvalidToken("Failed to renew");
                    } else {
                        throw new IOException("Unexpected");
                    }
                }

                @Override
                protected Token<?>[] obtainSystemTokensForUser(String user, final Credentials credentials) throws IOException {
                    credentials.addToken(updatedToken.getService(), updatedToken);
                    return new Token<?>[] { updatedToken };
                }
            };
        }
    };
    // simulating restart the rm
    rm2.start();
    // check nm can retrieve the token
    final MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm2.getResourceTrackerService());
    nm1.registerNode();
    NodeHeartbeatResponse response = nm1.nodeHeartbeat(true);
    ByteBuffer tokenBuffer = response.getSystemCredentialsForApps().get(app.getApplicationId());
    Assert.assertNotNull(tokenBuffer);
    Credentials appCredentials = new Credentials();
    DataInputByteBuffer buf = new DataInputByteBuffer();
    tokenBuffer.rewind();
    buf.reset(tokenBuffer);
    appCredentials.readTokenStorageStream(buf);
    Assert.assertTrue(firstRenewInvoked.get() && secondRenewInvoked.get());
    Assert.assertTrue(appCredentials.getAllTokens().contains(updatedToken));
}