Examples with RMDelegationTokenIdentifier org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier used on opensource projects

Example 21 with RMDelegationTokenIdentifier

use of org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier in project hadoop by apache.

the class TestRMDelegationTokens method testRMDTMasterKeyStateOnRollingMasterKey.

// Test the DT mast key in the state-store when the mast key is being rolled.
@Test(timeout = 15000)
public void testRMDTMasterKeyStateOnRollingMasterKey() throws Exception {
    Configuration conf = new Configuration(testConf);
    conf.set("hadoop.security.authentication", "kerberos");
    UserGroupInformation.setConfiguration(conf);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    RMState rmState = memStore.getState();
    Map<RMDelegationTokenIdentifier, Long> rmDTState = rmState.getRMDTSecretManagerState().getTokenState();
    Set<DelegationKey> rmDTMasterKeyState = rmState.getRMDTSecretManagerState().getMasterKeyState();
    MockRM rm1 = new MyMockRM(conf, memStore);
    rm1.start();
    // on rm start, two master keys are created.
    // One is created at RMDTSecretMgr.startThreads.updateCurrentKey();
    // the other is created on the first run of
    // tokenRemoverThread.rollMasterKey()
    RMDelegationTokenSecretManager dtSecretManager = rm1.getRMContext().getRMDelegationTokenSecretManager();
    // assert all master keys are saved
    Assert.assertEquals(dtSecretManager.getAllMasterKeys(), rmDTMasterKeyState);
    // request to generate a RMDelegationToken
    GetDelegationTokenRequest request = mock(GetDelegationTokenRequest.class);
    when(request.getRenewer()).thenReturn("renewer1");
    GetDelegationTokenResponse response = rm1.getClientRMService().getDelegationToken(request);
    org.apache.hadoop.yarn.api.records.Token delegationToken = response.getRMDelegationToken();
    Token<RMDelegationTokenIdentifier> token1 = ConverterUtils.convertFromYarn(delegationToken, (Text) null);
    RMDelegationTokenIdentifier dtId1 = token1.decodeIdentifier();
    // in state-store also.
    while (((TestRMDelegationTokenSecretManager) dtSecretManager).numUpdatedKeys.get() < 3) {
        ((TestRMDelegationTokenSecretManager) dtSecretManager).checkCurrentKeyInStateStore(rmDTMasterKeyState);
        Thread.sleep(100);
    }
    // wait for token to expire and remove from state-store
    // rollMasterKey is called every 1 second.
    int count = 0;
    while (rmDTState.containsKey(dtId1) && count < 100) {
        Thread.sleep(100);
        count++;
    }
    rm1.stop();
}

Example 22 with RMDelegationTokenIdentifier

use of org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier in project hadoop by apache.

the class TestTokenClientRMService method checkTokenCancellation.

private void checkTokenCancellation(ClientRMService rmService, UserGroupInformation owner, UserGroupInformation renewer) throws IOException, YarnException {
    RMDelegationTokenIdentifier tokenIdentifier = new RMDelegationTokenIdentifier(new Text(owner.getUserName()), new Text(renewer.getUserName()), null);
    Token<?> token = new Token<RMDelegationTokenIdentifier>(tokenIdentifier, dtsm);
    org.apache.hadoop.yarn.api.records.Token dToken = BuilderUtils.newDelegationToken(token.getIdentifier(), token.getKind().toString(), token.getPassword(), token.getService().toString());
    CancelDelegationTokenRequest request = Records.newRecord(CancelDelegationTokenRequest.class);
    request.setDelegationToken(dToken);
    rmService.cancelDelegationToken(request);
}

Example 23 with RMDelegationTokenIdentifier

use of org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier in project hadoop by apache.

the class TestClientRMTokens method testDelegationToken.

@Test
public void testDelegationToken() throws IOException, InterruptedException {
    final YarnConfiguration conf = new YarnConfiguration();
    conf.set(YarnConfiguration.RM_PRINCIPAL, "testuser/localhost@apache.org");
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    UserGroupInformation.setConfiguration(conf);
    ResourceScheduler scheduler = createMockScheduler(conf);
    long initialInterval = 10000l;
    long maxLifetime = 20000l;
    long renewInterval = 10000l;
    RMDelegationTokenSecretManager rmDtSecretManager = createRMDelegationTokenSecretManager(initialInterval, maxLifetime, renewInterval);
    rmDtSecretManager.startThreads();
    LOG.info("Creating DelegationTokenSecretManager with initialInterval: " + initialInterval + ", maxLifetime: " + maxLifetime + ", renewInterval: " + renewInterval);
    final ClientRMService clientRMService = new ClientRMServiceForTest(conf, scheduler, rmDtSecretManager);
    clientRMService.init(conf);
    clientRMService.start();
    ApplicationClientProtocol clientRMWithDT = null;
    try {
        // Create a user for the renewr and fake the authentication-method
        UserGroupInformation loggedInUser = UserGroupInformation.createRemoteUser("testrenewer@APACHE.ORG");
        Assert.assertEquals("testrenewer", loggedInUser.getShortUserName());
        // Default realm is APACHE.ORG
        loggedInUser.setAuthenticationMethod(AuthenticationMethod.KERBEROS);
        org.apache.hadoop.yarn.api.records.Token token = getDelegationToken(loggedInUser, clientRMService, loggedInUser.getShortUserName());
        long tokenFetchTime = System.currentTimeMillis();
        LOG.info("Got delegation token at: " + tokenFetchTime);
        // Now try talking to RMService using the delegation token
        clientRMWithDT = getClientRMProtocolWithDT(token, clientRMService.getBindAddress(), "loginuser1", conf);
        GetNewApplicationRequest request = Records.newRecord(GetNewApplicationRequest.class);
        try {
            clientRMWithDT.getNewApplication(request);
        } catch (IOException e) {
            fail("Unexpected exception" + e);
        } catch (YarnException e) {
            fail("Unexpected exception" + e);
        }
        // Renew after 50% of token age.
        while (System.currentTimeMillis() < tokenFetchTime + initialInterval / 2) {
            Thread.sleep(500l);
        }
        long nextExpTime = renewDelegationToken(loggedInUser, clientRMService, token);
        long renewalTime = System.currentTimeMillis();
        LOG.info("Renewed token at: " + renewalTime + ", NextExpiryTime: " + nextExpTime);
        // Wait for first expiry, but before renewed expiry.
        while (System.currentTimeMillis() > tokenFetchTime + initialInterval && System.currentTimeMillis() < nextExpTime) {
            Thread.sleep(500l);
        }
        Thread.sleep(50l);
        // Valid token because of renewal.
        try {
            clientRMWithDT.getNewApplication(request);
        } catch (IOException e) {
            fail("Unexpected exception" + e);
        } catch (YarnException e) {
            fail("Unexpected exception" + e);
        }
        // Wait for expiry.
        while (System.currentTimeMillis() < renewalTime + renewInterval) {
            Thread.sleep(500l);
        }
        Thread.sleep(50l);
        LOG.info("At time: " + System.currentTimeMillis() + ", token should be invalid");
        // Token should have expired.      
        try {
            clientRMWithDT.getNewApplication(request);
            fail("Should not have succeeded with an expired token");
        } catch (Exception e) {
            assertEquals(InvalidToken.class.getName(), e.getClass().getName());
            assertTrue(e.getMessage().contains("is expired"));
        }
        // Stop the existing proxy, start another.
        if (clientRMWithDT != null) {
            RPC.stopProxy(clientRMWithDT);
            clientRMWithDT = null;
        }
        token = getDelegationToken(loggedInUser, clientRMService, loggedInUser.getShortUserName());
        tokenFetchTime = System.currentTimeMillis();
        LOG.info("Got delegation token at: " + tokenFetchTime);
        // Now try talking to RMService using the delegation token
        clientRMWithDT = getClientRMProtocolWithDT(token, clientRMService.getBindAddress(), "loginuser2", conf);
        request = Records.newRecord(GetNewApplicationRequest.class);
        try {
            clientRMWithDT.getNewApplication(request);
        } catch (IOException e) {
            fail("Unexpected exception" + e);
        } catch (YarnException e) {
            fail("Unexpected exception" + e);
        }
        cancelDelegationToken(loggedInUser, clientRMService, token);
        if (clientRMWithDT != null) {
            RPC.stopProxy(clientRMWithDT);
            clientRMWithDT = null;
        }
        // Creating a new connection.
        clientRMWithDT = getClientRMProtocolWithDT(token, clientRMService.getBindAddress(), "loginuser2", conf);
        LOG.info("Cancelled delegation token at: " + System.currentTimeMillis());
        // Verify cancellation worked.
        try {
            clientRMWithDT.getNewApplication(request);
            fail("Should not have succeeded with a cancelled delegation token");
        } catch (IOException e) {
        } catch (YarnException e) {
        }
        // Stop the existing proxy, start another.
        if (clientRMWithDT != null) {
            RPC.stopProxy(clientRMWithDT);
            clientRMWithDT = null;
        }
        token = getDelegationToken(loggedInUser, clientRMService, loggedInUser.getShortUserName());
        byte[] tokenIdentifierContent = token.getIdentifier().array();
        RMDelegationTokenIdentifier tokenIdentifier = new RMDelegationTokenIdentifier();
        DataInputBuffer dib = new DataInputBuffer();
        dib.reset(tokenIdentifierContent, tokenIdentifierContent.length);
        tokenIdentifier.readFields(dib);
        // Construct new version RMDelegationTokenIdentifier with additional field
        RMDelegationTokenIdentifierForTest newVersionTokenIdentifier = new RMDelegationTokenIdentifierForTest(tokenIdentifier, "message");
        Token<RMDelegationTokenIdentifier> newRMDTtoken = new Token<RMDelegationTokenIdentifier>(newVersionTokenIdentifier, rmDtSecretManager);
        org.apache.hadoop.yarn.api.records.Token newToken = BuilderUtils.newDelegationToken(newRMDTtoken.getIdentifier(), newRMDTtoken.getKind().toString(), newRMDTtoken.getPassword(), newRMDTtoken.getService().toString());
        // Now try talking to RMService using the new version delegation token
        clientRMWithDT = getClientRMProtocolWithDT(newToken, clientRMService.getBindAddress(), "loginuser3", conf);
        request = Records.newRecord(GetNewApplicationRequest.class);
        try {
            clientRMWithDT.getNewApplication(request);
        } catch (IOException e) {
            fail("Unexpected exception" + e);
        } catch (YarnException e) {
            fail("Unexpected exception" + e);
        }
    } finally {
        rmDtSecretManager.stopThreads();
        // TODO PRECOMMIT Close proxies.
        if (clientRMWithDT != null) {
            RPC.stopProxy(clientRMWithDT);
        }
    }
}

Example 24 with RMDelegationTokenIdentifier

use of org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier in project hadoop by apache.

the class TestYARNTokenIdentifier method testRMDelegationTokenIdentifier.

@Test
public void testRMDelegationTokenIdentifier() throws IOException {
    Text owner = new Text("user1");
    Text renewer = new Text("user2");
    Text realUser = new Text("user3");
    long issueDate = 1;
    long maxDate = 2;
    int sequenceNumber = 3;
    int masterKeyId = 4;
    RMDelegationTokenIdentifier token = new RMDelegationTokenIdentifier(owner, renewer, realUser);
    token.setIssueDate(issueDate);
    token.setMaxDate(maxDate);
    token.setSequenceNumber(sequenceNumber);
    token.setMasterKeyId(masterKeyId);
    RMDelegationTokenIdentifier anotherToken = new RMDelegationTokenIdentifier();
    byte[] tokenContent = token.getBytes();
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(tokenContent, tokenContent.length);
    anotherToken.readFields(dib);
    dib.close();
    // verify the whole record equals with original record
    Assert.assertEquals("Token is not the same after serialization " + "and deserialization.", token, anotherToken);
    Assert.assertEquals("owner from proto is not the same with original token", anotherToken.getOwner(), owner);
    Assert.assertEquals("renewer from proto is not the same with original token", anotherToken.getRenewer(), renewer);
    Assert.assertEquals("realUser from proto is not the same with original token", anotherToken.getRealUser(), realUser);
    Assert.assertEquals("issueDate from proto is not the same with original token", anotherToken.getIssueDate(), issueDate);
    Assert.assertEquals("maxDate from proto is not the same with original token", anotherToken.getMaxDate(), maxDate);
    Assert.assertEquals("sequenceNumber from proto is not the same with original token", anotherToken.getSequenceNumber(), sequenceNumber);
    Assert.assertEquals("masterKeyId from proto is not the same with original token", anotherToken.getMasterKeyId(), masterKeyId);
    // Test getProto    
    RMDelegationTokenIdentifier token1 = new RMDelegationTokenIdentifier(owner, renewer, realUser);
    token1.setIssueDate(issueDate);
    token1.setMaxDate(maxDate);
    token1.setSequenceNumber(sequenceNumber);
    token1.setMasterKeyId(masterKeyId);
    YARNDelegationTokenIdentifierProto tokenProto = token1.getProto();
    // Write token proto to stream
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream out = new DataOutputStream(baos);
    tokenProto.writeTo(out);
    // Read token
    byte[] tokenData = baos.toByteArray();
    RMDelegationTokenIdentifier readToken = new RMDelegationTokenIdentifier();
    DataInputBuffer db = new DataInputBuffer();
    db.reset(tokenData, tokenData.length);
    readToken.readFields(db);
    // Verify if read token equals with original token
    Assert.assertEquals("Token from getProto is not the same after " + "serialization and deserialization.", token1, readToken);
    db.close();
    out.close();
}

Example 25 with RMDelegationTokenIdentifier

use of org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier in project hadoop by apache.

the class TestLogAggregationService method testAddNewTokenSentFromRMForLogAggregation.

@Test(timeout = 20000)
public void testAddNewTokenSentFromRMForLogAggregation() throws Exception {
    Configuration conf = new YarnConfiguration();
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    UserGroupInformation.setConfiguration(conf);
    ApplicationId application1 = BuilderUtils.newApplicationId(1234, 1);
    Application mockApp = mock(Application.class);
    when(mockApp.getContainers()).thenReturn(new HashMap<ContainerId, Container>());
    this.context.getApplications().put(application1, mockApp);
    @SuppressWarnings("resource") LogAggregationService logAggregationService = new LogAggregationService(dispatcher, this.context, this.delSrvc, super.dirsHandler);
    logAggregationService.init(this.conf);
    logAggregationService.start();
    logAggregationService.handle(new LogHandlerAppStartedEvent(application1, this.user, null, this.acls, Records.newRecord(LogAggregationContext.class)));
    // Inject new token for log-aggregation after app log-aggregator init
    Text userText1 = new Text("user1");
    RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier(userText1, new Text("renewer1"), userText1);
    final Token<RMDelegationTokenIdentifier> token1 = new Token<RMDelegationTokenIdentifier>(dtId1.getBytes(), "password1".getBytes(), dtId1.getKind(), new Text("service1"));
    Credentials credentials = new Credentials();
    credentials.addToken(userText1, token1);
    this.context.getSystemCredentialsForApps().put(application1, credentials);
    logAggregationService.handle(new LogHandlerAppFinishedEvent(application1));
    final UserGroupInformation ugi = ((AppLogAggregatorImpl) logAggregationService.getAppLogAggregators().get(application1)).getUgi();
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        public Boolean get() {
            boolean hasNewToken = false;
            for (Token<?> token : ugi.getCredentials().getAllTokens()) {
                if (token.equals(token1)) {
                    hasNewToken = true;
                }
            }
            return hasNewToken;
        }
    }, 1000, 20000);
    logAggregationService.stop();
}