Examples with RMDelegationTokenIdentifier org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier used on opensource projects

Example 16 with RMDelegationTokenIdentifier

use of org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier in project hadoop by apache.

the class TestRMRestart method testRMDelegationTokenRestoredOnRMRestart.

@Test(timeout = 60000)
public void testRMDelegationTokenRestoredOnRMRestart() throws Exception {
    conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    conf.set(YarnConfiguration.RM_ADDRESS, "localhost:8032");
    UserGroupInformation.setConfiguration(conf);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    RMState rmState = memStore.getState();
    Map<ApplicationId, ApplicationStateData> rmAppState = rmState.getApplicationState();
    Map<RMDelegationTokenIdentifier, Long> rmDTState = rmState.getRMDTSecretManagerState().getTokenState();
    Set<DelegationKey> rmDTMasterKeyState = rmState.getRMDTSecretManagerState().getMasterKeyState();
    MockRM rm1 = new TestSecurityMockRM(conf, memStore);
    rm1.start();
    // create an empty credential
    Credentials ts = new Credentials();
    // request a token and add into credential
    GetDelegationTokenRequest request1 = GetDelegationTokenRequest.newInstance("renewer1");
    UserGroupInformation.getCurrentUser().setAuthenticationMethod(AuthMethod.KERBEROS);
    GetDelegationTokenResponse response1 = rm1.getClientRMService().getDelegationToken(request1);
    org.apache.hadoop.yarn.api.records.Token delegationToken1 = response1.getRMDelegationToken();
    Token<RMDelegationTokenIdentifier> token1 = ConverterUtils.convertFromYarn(delegationToken1, rmAddr);
    RMDelegationTokenIdentifier dtId1 = token1.decodeIdentifier();
    HashSet<RMDelegationTokenIdentifier> tokenIdentSet = new HashSet<RMDelegationTokenIdentifier>();
    ts.addToken(token1.getService(), token1);
    tokenIdentSet.add(dtId1);
    // submit an app with customized credential
    RMApp app = rm1.submitApp(200, "name", "user", new HashMap<ApplicationAccessType, String>(), false, "default", 1, ts);
    // assert app info is saved
    ApplicationStateData appState = rmAppState.get(app.getApplicationId());
    Assert.assertNotNull(appState);
    // assert all master keys are saved
    Set<DelegationKey> allKeysRM1 = rm1.getRMContext().getRMDelegationTokenSecretManager().getAllMasterKeys();
    Assert.assertEquals(allKeysRM1, rmDTMasterKeyState);
    // assert all tokens are saved
    Map<RMDelegationTokenIdentifier, Long> allTokensRM1 = rm1.getRMContext().getRMDelegationTokenSecretManager().getAllTokens();
    Assert.assertEquals(tokenIdentSet, allTokensRM1.keySet());
    Assert.assertEquals(allTokensRM1, rmDTState);
    // assert sequence number is saved
    Assert.assertEquals(rm1.getRMContext().getRMDelegationTokenSecretManager().getLatestDTSequenceNumber(), rmState.getRMDTSecretManagerState().getDTSequenceNumber());
    // request one more token
    GetDelegationTokenRequest request2 = GetDelegationTokenRequest.newInstance("renewer2");
    GetDelegationTokenResponse response2 = rm1.getClientRMService().getDelegationToken(request2);
    org.apache.hadoop.yarn.api.records.Token delegationToken2 = response2.getRMDelegationToken();
    Token<RMDelegationTokenIdentifier> token2 = ConverterUtils.convertFromYarn(delegationToken2, rmAddr);
    RMDelegationTokenIdentifier dtId2 = token2.decodeIdentifier();
    // cancel token2
    try {
        rm1.getRMContext().getRMDelegationTokenSecretManager().cancelToken(token2, UserGroupInformation.getCurrentUser().getUserName());
    } catch (Exception e) {
        Assert.fail();
    }
    // Assert the token which has the latest delegationTokenSequenceNumber is removed
    Assert.assertEquals(rm1.getRMContext().getRMDelegationTokenSecretManager().getLatestDTSequenceNumber(), dtId2.getSequenceNumber());
    Assert.assertFalse(rmDTState.containsKey(dtId2));
    // start new RM
    MockRM rm2 = new TestSecurityMockRM(conf, memStore);
    rm2.start();
    // assert master keys and tokens are populated back to DTSecretManager
    Map<RMDelegationTokenIdentifier, Long> allTokensRM2 = rm2.getRMContext().getRMDelegationTokenSecretManager().getAllTokens();
    Assert.assertEquals(allTokensRM2.keySet(), allTokensRM1.keySet());
    // rm2 has its own master keys when it starts, we use containsAll here
    Assert.assertTrue(rm2.getRMContext().getRMDelegationTokenSecretManager().getAllMasterKeys().containsAll(allKeysRM1));
    // assert sequenceNumber is properly recovered,
    // even though the token which has max sequenceNumber is not stored
    Assert.assertEquals(rm1.getRMContext().getRMDelegationTokenSecretManager().getLatestDTSequenceNumber(), rm2.getRMContext().getRMDelegationTokenSecretManager().getLatestDTSequenceNumber());
    // renewDate before renewing
    Long renewDateBeforeRenew = allTokensRM2.get(dtId1);
    try {
        // Sleep for one millisecond to make sure renewDataAfterRenew is greater
        Thread.sleep(1);
        // renew recovered token
        rm2.getRMContext().getRMDelegationTokenSecretManager().renewToken(token1, "renewer1");
    } catch (Exception e) {
        Assert.fail();
    }
    allTokensRM2 = rm2.getRMContext().getRMDelegationTokenSecretManager().getAllTokens();
    Long renewDateAfterRenew = allTokensRM2.get(dtId1);
    // assert token is renewed
    Assert.assertTrue(renewDateAfterRenew > renewDateBeforeRenew);
    // assert new token is added into state store
    Assert.assertTrue(rmDTState.containsValue(renewDateAfterRenew));
    // assert old token is removed from state store
    Assert.assertFalse(rmDTState.containsValue(renewDateBeforeRenew));
    try {
        rm2.getRMContext().getRMDelegationTokenSecretManager().cancelToken(token1, UserGroupInformation.getCurrentUser().getUserName());
    } catch (Exception e) {
        Assert.fail();
    }
    // assert token is removed from state after its cancelled
    allTokensRM2 = rm2.getRMContext().getRMDelegationTokenSecretManager().getAllTokens();
    Assert.assertFalse(allTokensRM2.containsKey(dtId1));
    Assert.assertFalse(rmDTState.containsKey(dtId1));
}

Example 17 with RMDelegationTokenIdentifier

use of org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier in project hadoop by apache.

the class TestRMRestart method testDelegationTokenRestoredInDelegationTokenRenewer.

@Test(timeout = 60000)
public void testDelegationTokenRestoredInDelegationTokenRenewer() throws Exception {
    conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    UserGroupInformation.setConfiguration(conf);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    RMState rmState = memStore.getState();
    Map<ApplicationId, ApplicationStateData> rmAppState = rmState.getApplicationState();
    MockRM rm1 = new TestSecurityMockRM(conf, memStore);
    rm1.start();
    HashSet<Token<RMDelegationTokenIdentifier>> tokenSet = new HashSet<Token<RMDelegationTokenIdentifier>>();
    // create an empty credential
    Credentials ts = new Credentials();
    // create tokens and add into credential
    Text userText1 = new Text("user1");
    RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier(userText1, new Text("renewer1"), userText1);
    Token<RMDelegationTokenIdentifier> token1 = new Token<RMDelegationTokenIdentifier>(dtId1, rm1.getRMContext().getRMDelegationTokenSecretManager());
    SecurityUtil.setTokenService(token1, rmAddr);
    ts.addToken(userText1, token1);
    tokenSet.add(token1);
    Text userText2 = new Text("user2");
    RMDelegationTokenIdentifier dtId2 = new RMDelegationTokenIdentifier(userText2, new Text("renewer2"), userText2);
    Token<RMDelegationTokenIdentifier> token2 = new Token<RMDelegationTokenIdentifier>(dtId2, rm1.getRMContext().getRMDelegationTokenSecretManager());
    SecurityUtil.setTokenService(token2, rmAddr);
    ts.addToken(userText2, token2);
    tokenSet.add(token2);
    // submit an app with customized credential
    RMApp app = rm1.submitApp(200, "name", "user", new HashMap<ApplicationAccessType, String>(), false, "default", 1, ts);
    // assert app info is saved
    ApplicationStateData appState = rmAppState.get(app.getApplicationId());
    Assert.assertNotNull(appState);
    // assert delegation tokens exist in rm1 DelegationTokenRenewr
    Assert.assertEquals(tokenSet, rm1.getRMContext().getDelegationTokenRenewer().getDelegationTokens());
    // assert delegation tokens are saved
    DataOutputBuffer dob = new DataOutputBuffer();
    ts.writeTokenStorageToStream(dob);
    ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
    securityTokens.rewind();
    Assert.assertEquals(securityTokens, appState.getApplicationSubmissionContext().getAMContainerSpec().getTokens());
    // start new RM
    MockRM rm2 = new TestSecurityMockRM(conf, memStore);
    rm2.start();
    // Need to wait for a while as now token renewal happens on another thread
    // and is asynchronous in nature.
    waitForTokensToBeRenewed(rm2, tokenSet);
    // verify tokens are properly populated back to rm2 DelegationTokenRenewer
    Assert.assertEquals(tokenSet, rm2.getRMContext().getDelegationTokenRenewer().getDelegationTokens());
}

Example 18 with RMDelegationTokenIdentifier

use of org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier in project hadoop by apache.

the class RMDelegationTokenSecretManager method recover.

@Override
public void recover(RMState rmState) throws Exception {
    LOG.info("recovering RMDelegationTokenSecretManager.");
    // recover RMDTMasterKeys
    for (DelegationKey dtKey : rmState.getRMDTSecretManagerState().getMasterKeyState()) {
        addKey(dtKey);
    }
    // recover RMDelegationTokens
    Map<RMDelegationTokenIdentifier, Long> rmDelegationTokens = rmState.getRMDTSecretManagerState().getTokenState();
    this.delegationTokenSequenceNumber = rmState.getRMDTSecretManagerState().getDTSequenceNumber();
    for (Map.Entry<RMDelegationTokenIdentifier, Long> entry : rmDelegationTokens.entrySet()) {
        addPersistedDelegationToken(entry.getKey(), entry.getValue());
    }
}

Example 19 with RMDelegationTokenIdentifier

use of org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier in project hadoop by apache.

the class RMWebServices method cancelDelegationToken.

// For cancelling tokens, the encoded token is passed as a header
// There are two reasons for this -
// 1. Passing a request body as part of a DELETE request is not
// allowed by Jetty
// 2. Passing the encoded token as part of the url is not ideal
// since urls tend to get logged and anyone with access to
// the logs can extract tokens which are meant to be secret
@DELETE
@Path("/delegation-token")
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
public Response cancelDelegationToken(@Context HttpServletRequest hsr) throws AuthorizationException, IOException, InterruptedException, Exception {
    init();
    UserGroupInformation callerUGI;
    try {
        callerUGI = createKerberosUserGroupInformation(hsr);
    } catch (YarnException ye) {
        return Response.status(Status.FORBIDDEN).entity(ye.getMessage()).build();
    }
    Token<RMDelegationTokenIdentifier> token = extractToken(hsr);
    org.apache.hadoop.yarn.api.records.Token dToken = BuilderUtils.newDelegationToken(token.getIdentifier(), token.getKind().toString(), token.getPassword(), token.getService().toString());
    final CancelDelegationTokenRequest req = CancelDelegationTokenRequest.newInstance(dToken);
    try {
        callerUGI.doAs(new PrivilegedExceptionAction<CancelDelegationTokenResponse>() {

            @Override
            public CancelDelegationTokenResponse run() throws IOException, YarnException {
                return rm.getClientRMService().cancelDelegationToken(req);
            }
        });
    } catch (UndeclaredThrowableException ue) {
        if (ue.getCause() instanceof YarnException) {
            if (ue.getCause().getCause() instanceof InvalidToken) {
                throw new BadRequestException(ue.getCause().getCause().getMessage());
            } else if (ue.getCause().getCause() instanceof org.apache.hadoop.security.AccessControlException) {
                return Response.status(Status.FORBIDDEN).entity(ue.getCause().getCause().getMessage()).build();
            }
            LOG.info("Renew delegation token request failed", ue);
            throw ue;
        }
        LOG.info("Renew delegation token request failed", ue);
        throw ue;
    } catch (Exception e) {
        LOG.info("Renew delegation token request failed", e);
        throw e;
    }
    return Response.status(Status.OK).build();
}

Example 20 with RMDelegationTokenIdentifier

use of org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier in project hadoop by apache.

the class RMStateStoreTestBase method testRMDTSecretManagerStateStore.

public void testRMDTSecretManagerStateStore(RMStateStoreHelper stateStoreHelper) throws Exception {
    RMStateStore store = stateStoreHelper.getRMStateStore();
    TestDispatcher dispatcher = new TestDispatcher();
    store.setRMDispatcher(dispatcher);
    // store RM delegation token;
    RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier(new Text("owner1"), new Text("renewer1"), new Text("realuser1"));
    int sequenceNumber = 1111;
    dtId1.setSequenceNumber(sequenceNumber);
    byte[] tokenBeforeStore = dtId1.getBytes();
    Long renewDate1 = new Long(System.currentTimeMillis());
    store.storeRMDelegationToken(dtId1, renewDate1);
    modifyRMDelegationTokenState();
    Map<RMDelegationTokenIdentifier, Long> token1 = new HashMap<RMDelegationTokenIdentifier, Long>();
    token1.put(dtId1, renewDate1);
    // store delegation key;
    DelegationKey key = new DelegationKey(1234, 4321, "keyBytes".getBytes());
    HashSet<DelegationKey> keySet = new HashSet<DelegationKey>();
    keySet.add(key);
    store.storeRMDTMasterKey(key);
    RMDTSecretManagerState secretManagerState = store.loadState().getRMDTSecretManagerState();
    Assert.assertEquals(token1, secretManagerState.getTokenState());
    Assert.assertEquals(keySet, secretManagerState.getMasterKeyState());
    Assert.assertEquals(sequenceNumber, secretManagerState.getDTSequenceNumber());
    RMDelegationTokenIdentifier tokenAfterStore = secretManagerState.getTokenState().keySet().iterator().next();
    Assert.assertTrue(Arrays.equals(tokenBeforeStore, tokenAfterStore.getBytes()));
    // update RM delegation token;
    renewDate1 = new Long(System.currentTimeMillis());
    store.updateRMDelegationToken(dtId1, renewDate1);
    token1.put(dtId1, renewDate1);
    RMDTSecretManagerState updateSecretManagerState = store.loadState().getRMDTSecretManagerState();
    Assert.assertEquals(token1, updateSecretManagerState.getTokenState());
    Assert.assertEquals(keySet, updateSecretManagerState.getMasterKeyState());
    Assert.assertEquals(sequenceNumber, updateSecretManagerState.getDTSequenceNumber());
    // check to delete delegationKey
    store.removeRMDTMasterKey(key);
    keySet.clear();
    RMDTSecretManagerState noKeySecretManagerState = store.loadState().getRMDTSecretManagerState();
    Assert.assertEquals(token1, noKeySecretManagerState.getTokenState());
    Assert.assertEquals(keySet, noKeySecretManagerState.getMasterKeyState());
    Assert.assertEquals(sequenceNumber, noKeySecretManagerState.getDTSequenceNumber());
    // check to delete delegationToken
    store.removeRMDelegationToken(dtId1);
    RMDTSecretManagerState noKeyAndTokenSecretManagerState = store.loadState().getRMDTSecretManagerState();
    token1.clear();
    Assert.assertEquals(token1, noKeyAndTokenSecretManagerState.getTokenState());
    Assert.assertEquals(keySet, noKeyAndTokenSecretManagerState.getMasterKeyState());
    Assert.assertEquals(sequenceNumber, noKeySecretManagerState.getDTSequenceNumber());
    store.close();
}