Examples with GetDelegationTokenResponse org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse used on opensource projects

Example 1 with GetDelegationTokenResponse

use of org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse in project hadoop by apache.

the class RMWebServices method createDelegationToken.

private Response createDelegationToken(DelegationToken tokenData, HttpServletRequest hsr, UserGroupInformation callerUGI) throws AuthorizationException, IOException, InterruptedException, Exception {
    final String renewer = tokenData.getRenewer();
    GetDelegationTokenResponse resp;
    try {
        resp = callerUGI.doAs(new PrivilegedExceptionAction<GetDelegationTokenResponse>() {

            @Override
            public GetDelegationTokenResponse run() throws IOException, YarnException {
                GetDelegationTokenRequest createReq = GetDelegationTokenRequest.newInstance(renewer);
                return rm.getClientRMService().getDelegationToken(createReq);
            }
        });
    } catch (Exception e) {
        LOG.info("Create delegation token request failed", e);
        throw e;
    }
    Token<RMDelegationTokenIdentifier> tk = new Token<RMDelegationTokenIdentifier>(resp.getRMDelegationToken().getIdentifier().array(), resp.getRMDelegationToken().getPassword().array(), new Text(resp.getRMDelegationToken().getKind()), new Text(resp.getRMDelegationToken().getService()));
    RMDelegationTokenIdentifier identifier = tk.decodeIdentifier();
    long currentExpiration = rm.getRMContext().getRMDelegationTokenSecretManager().getRenewDate(identifier);
    DelegationToken respToken = new DelegationToken(tk.encodeToUrlString(), renewer, identifier.getOwner().toString(), tk.getKind().toString(), currentExpiration, identifier.getMaxDate());
    return Response.status(Status.OK).entity(respToken).build();
}

Example 2 with GetDelegationTokenResponse

use of org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse in project hadoop by apache.

the class TestYarnClientProtocolProvider method testClusterGetDelegationToken.

@Test
public void testClusterGetDelegationToken() throws Exception {
    Configuration conf = new Configuration(false);
    Cluster cluster = null;
    try {
        conf = new Configuration();
        conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
        cluster = new Cluster(conf);
        YARNRunner yrunner = (YARNRunner) cluster.getClient();
        GetDelegationTokenResponse getDTResponse = recordFactory.newRecordInstance(GetDelegationTokenResponse.class);
        org.apache.hadoop.yarn.api.records.Token rmDTToken = recordFactory.newRecordInstance(org.apache.hadoop.yarn.api.records.Token.class);
        rmDTToken.setIdentifier(ByteBuffer.wrap(new byte[2]));
        rmDTToken.setKind("Testclusterkind");
        rmDTToken.setPassword(ByteBuffer.wrap("testcluster".getBytes()));
        rmDTToken.setService("0.0.0.0:8032");
        getDTResponse.setRMDelegationToken(rmDTToken);
        final ApplicationClientProtocol cRMProtocol = mock(ApplicationClientProtocol.class);
        when(cRMProtocol.getDelegationToken(any(GetDelegationTokenRequest.class))).thenReturn(getDTResponse);
        ResourceMgrDelegate rmgrDelegate = new ResourceMgrDelegate(new YarnConfiguration(conf)) {

            @Override
            protected void serviceStart() throws Exception {
                assertTrue(this.client instanceof YarnClientImpl);
                this.client = spy(this.client);
                doNothing().when(this.client).close();
                ((YarnClientImpl) this.client).setRMClient(cRMProtocol);
            }
        };
        yrunner.setResourceMgrDelegate(rmgrDelegate);
        Token t = cluster.getDelegationToken(new Text(" "));
        assertTrue("Token kind is instead " + t.getKind().toString(), "Testclusterkind".equals(t.getKind().toString()));
    } finally {
        if (cluster != null) {
            cluster.close();
        }
    }
}

Example 3 with GetDelegationTokenResponse

use of org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse in project hadoop by apache.

the class TestRMRestart method testAppSubmissionWithOldDelegationTokenAfterRMRestart.

// This is to test submit an application to the new RM with the old delegation
// token got from previous RM.
@Test(timeout = 60000)
public void testAppSubmissionWithOldDelegationTokenAfterRMRestart() throws Exception {
    conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    conf.set(YarnConfiguration.RM_ADDRESS, "localhost:8032");
    UserGroupInformation.setConfiguration(conf);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    MockRM rm1 = new TestSecurityMockRM(conf, memStore);
    rm1.start();
    GetDelegationTokenRequest request1 = GetDelegationTokenRequest.newInstance("renewer1");
    UserGroupInformation.getCurrentUser().setAuthenticationMethod(AuthMethod.KERBEROS);
    GetDelegationTokenResponse response1 = rm1.getClientRMService().getDelegationToken(request1);
    Token<RMDelegationTokenIdentifier> token1 = ConverterUtils.convertFromYarn(response1.getRMDelegationToken(), rmAddr);
    // start new RM
    MockRM rm2 = new TestSecurityMockRM(conf, memStore);
    rm2.start();
    // submit an app with the old delegation token got from previous RM.
    Credentials ts = new Credentials();
    ts.addToken(token1.getService(), token1);
    RMApp app = rm2.submitApp(200, "name", "user", new HashMap<ApplicationAccessType, String>(), false, "default", 1, ts);
    rm2.waitForState(app.getApplicationId(), RMAppState.ACCEPTED);
}

Example 4 with GetDelegationTokenResponse

use of org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse in project hadoop by apache.

the class TestRMRestart method testRMDelegationTokenRestoredOnRMRestart.

@Test(timeout = 60000)
public void testRMDelegationTokenRestoredOnRMRestart() throws Exception {
    conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    conf.set(YarnConfiguration.RM_ADDRESS, "localhost:8032");
    UserGroupInformation.setConfiguration(conf);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    RMState rmState = memStore.getState();
    Map<ApplicationId, ApplicationStateData> rmAppState = rmState.getApplicationState();
    Map<RMDelegationTokenIdentifier, Long> rmDTState = rmState.getRMDTSecretManagerState().getTokenState();
    Set<DelegationKey> rmDTMasterKeyState = rmState.getRMDTSecretManagerState().getMasterKeyState();
    MockRM rm1 = new TestSecurityMockRM(conf, memStore);
    rm1.start();
    // create an empty credential
    Credentials ts = new Credentials();
    // request a token and add into credential
    GetDelegationTokenRequest request1 = GetDelegationTokenRequest.newInstance("renewer1");
    UserGroupInformation.getCurrentUser().setAuthenticationMethod(AuthMethod.KERBEROS);
    GetDelegationTokenResponse response1 = rm1.getClientRMService().getDelegationToken(request1);
    org.apache.hadoop.yarn.api.records.Token delegationToken1 = response1.getRMDelegationToken();
    Token<RMDelegationTokenIdentifier> token1 = ConverterUtils.convertFromYarn(delegationToken1, rmAddr);
    RMDelegationTokenIdentifier dtId1 = token1.decodeIdentifier();
    HashSet<RMDelegationTokenIdentifier> tokenIdentSet = new HashSet<RMDelegationTokenIdentifier>();
    ts.addToken(token1.getService(), token1);
    tokenIdentSet.add(dtId1);
    // submit an app with customized credential
    RMApp app = rm1.submitApp(200, "name", "user", new HashMap<ApplicationAccessType, String>(), false, "default", 1, ts);
    // assert app info is saved
    ApplicationStateData appState = rmAppState.get(app.getApplicationId());
    Assert.assertNotNull(appState);
    // assert all master keys are saved
    Set<DelegationKey> allKeysRM1 = rm1.getRMContext().getRMDelegationTokenSecretManager().getAllMasterKeys();
    Assert.assertEquals(allKeysRM1, rmDTMasterKeyState);
    // assert all tokens are saved
    Map<RMDelegationTokenIdentifier, Long> allTokensRM1 = rm1.getRMContext().getRMDelegationTokenSecretManager().getAllTokens();
    Assert.assertEquals(tokenIdentSet, allTokensRM1.keySet());
    Assert.assertEquals(allTokensRM1, rmDTState);
    // assert sequence number is saved
    Assert.assertEquals(rm1.getRMContext().getRMDelegationTokenSecretManager().getLatestDTSequenceNumber(), rmState.getRMDTSecretManagerState().getDTSequenceNumber());
    // request one more token
    GetDelegationTokenRequest request2 = GetDelegationTokenRequest.newInstance("renewer2");
    GetDelegationTokenResponse response2 = rm1.getClientRMService().getDelegationToken(request2);
    org.apache.hadoop.yarn.api.records.Token delegationToken2 = response2.getRMDelegationToken();
    Token<RMDelegationTokenIdentifier> token2 = ConverterUtils.convertFromYarn(delegationToken2, rmAddr);
    RMDelegationTokenIdentifier dtId2 = token2.decodeIdentifier();
    // cancel token2
    try {
        rm1.getRMContext().getRMDelegationTokenSecretManager().cancelToken(token2, UserGroupInformation.getCurrentUser().getUserName());
    } catch (Exception e) {
        Assert.fail();
    }
    // Assert the token which has the latest delegationTokenSequenceNumber is removed
    Assert.assertEquals(rm1.getRMContext().getRMDelegationTokenSecretManager().getLatestDTSequenceNumber(), dtId2.getSequenceNumber());
    Assert.assertFalse(rmDTState.containsKey(dtId2));
    // start new RM
    MockRM rm2 = new TestSecurityMockRM(conf, memStore);
    rm2.start();
    // assert master keys and tokens are populated back to DTSecretManager
    Map<RMDelegationTokenIdentifier, Long> allTokensRM2 = rm2.getRMContext().getRMDelegationTokenSecretManager().getAllTokens();
    Assert.assertEquals(allTokensRM2.keySet(), allTokensRM1.keySet());
    // rm2 has its own master keys when it starts, we use containsAll here
    Assert.assertTrue(rm2.getRMContext().getRMDelegationTokenSecretManager().getAllMasterKeys().containsAll(allKeysRM1));
    // assert sequenceNumber is properly recovered,
    // even though the token which has max sequenceNumber is not stored
    Assert.assertEquals(rm1.getRMContext().getRMDelegationTokenSecretManager().getLatestDTSequenceNumber(), rm2.getRMContext().getRMDelegationTokenSecretManager().getLatestDTSequenceNumber());
    // renewDate before renewing
    Long renewDateBeforeRenew = allTokensRM2.get(dtId1);
    try {
        // Sleep for one millisecond to make sure renewDataAfterRenew is greater
        Thread.sleep(1);
        // renew recovered token
        rm2.getRMContext().getRMDelegationTokenSecretManager().renewToken(token1, "renewer1");
    } catch (Exception e) {
        Assert.fail();
    }
    allTokensRM2 = rm2.getRMContext().getRMDelegationTokenSecretManager().getAllTokens();
    Long renewDateAfterRenew = allTokensRM2.get(dtId1);
    // assert token is renewed
    Assert.assertTrue(renewDateAfterRenew > renewDateBeforeRenew);
    // assert new token is added into state store
    Assert.assertTrue(rmDTState.containsValue(renewDateAfterRenew));
    // assert old token is removed from state store
    Assert.assertFalse(rmDTState.containsValue(renewDateBeforeRenew));
    try {
        rm2.getRMContext().getRMDelegationTokenSecretManager().cancelToken(token1, UserGroupInformation.getCurrentUser().getUserName());
    } catch (Exception e) {
        Assert.fail();
    }
    // assert token is removed from state after its cancelled
    allTokensRM2 = rm2.getRMContext().getRMDelegationTokenSecretManager().getAllTokens();
    Assert.assertFalse(allTokensRM2.containsKey(dtId1));
    Assert.assertFalse(rmDTState.containsKey(dtId1));
}

Example 5 with GetDelegationTokenResponse

use of org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse in project hadoop by apache.

the class TestRMDelegationTokens method testRMDTMasterKeyStateOnRollingMasterKey.

// Test the DT mast key in the state-store when the mast key is being rolled.
@Test(timeout = 15000)
public void testRMDTMasterKeyStateOnRollingMasterKey() throws Exception {
    Configuration conf = new Configuration(testConf);
    conf.set("hadoop.security.authentication", "kerberos");
    UserGroupInformation.setConfiguration(conf);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    RMState rmState = memStore.getState();
    Map<RMDelegationTokenIdentifier, Long> rmDTState = rmState.getRMDTSecretManagerState().getTokenState();
    Set<DelegationKey> rmDTMasterKeyState = rmState.getRMDTSecretManagerState().getMasterKeyState();
    MockRM rm1 = new MyMockRM(conf, memStore);
    rm1.start();
    // on rm start, two master keys are created.
    // One is created at RMDTSecretMgr.startThreads.updateCurrentKey();
    // the other is created on the first run of
    // tokenRemoverThread.rollMasterKey()
    RMDelegationTokenSecretManager dtSecretManager = rm1.getRMContext().getRMDelegationTokenSecretManager();
    // assert all master keys are saved
    Assert.assertEquals(dtSecretManager.getAllMasterKeys(), rmDTMasterKeyState);
    // request to generate a RMDelegationToken
    GetDelegationTokenRequest request = mock(GetDelegationTokenRequest.class);
    when(request.getRenewer()).thenReturn("renewer1");
    GetDelegationTokenResponse response = rm1.getClientRMService().getDelegationToken(request);
    org.apache.hadoop.yarn.api.records.Token delegationToken = response.getRMDelegationToken();
    Token<RMDelegationTokenIdentifier> token1 = ConverterUtils.convertFromYarn(delegationToken, (Text) null);
    RMDelegationTokenIdentifier dtId1 = token1.decodeIdentifier();
    // in state-store also.
    while (((TestRMDelegationTokenSecretManager) dtSecretManager).numUpdatedKeys.get() < 3) {
        ((TestRMDelegationTokenSecretManager) dtSecretManager).checkCurrentKeyInStateStore(rmDTMasterKeyState);
        Thread.sleep(100);
    }
    // wait for token to expire and remove from state-store
    // rollMasterKey is called every 1 second.
    int count = 0;
    while (rmDTState.containsKey(dtId1) && count < 100) {
        Thread.sleep(100);
        count++;
    }
    rm1.stop();
}