Examples with DelegationTokenIdentifier org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier used on opensource projects

Example 1 with DelegationTokenIdentifier

use of org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier in project hadoop by apache.

the class TestKMS method testDelegationTokensOps.

private void testDelegationTokensOps(final boolean ssl, final boolean kerb) throws Exception {
    final File confDir = getTestDir();
    final Configuration conf;
    if (kerb) {
        conf = setupConfForKerberos(confDir);
    } else {
        conf = createBaseKMSConf(confDir, null);
    }
    final String keystore;
    final String password;
    if (ssl) {
        final String sslConfDir = KeyStoreTestUtil.getClasspathDir(TestKMS.class);
        KeyStoreTestUtil.setupSSLConfig(confDir.getAbsolutePath(), sslConfDir, conf, false);
        keystore = confDir.getAbsolutePath() + "/serverKS.jks";
        password = "serverP";
    } else {
        keystore = null;
        password = null;
    }
    writeConf(confDir, conf);
    runServer(keystore, password, confDir, new KMSCallable<Void>() {

        @Override
        public Void call() throws Exception {
            final Configuration clientConf = new Configuration();
            final URI uri = createKMSUri(getKMSUrl());
            clientConf.set(KeyProviderFactory.KEY_PROVIDER_PATH, createKMSUri(getKMSUrl()).toString());
            doAs("client", new PrivilegedExceptionAction<Void>() {

                @Override
                public Void run() throws Exception {
                    KeyProvider kp = createProvider(uri, clientConf);
                    // test delegation token retrieval
                    KeyProviderDelegationTokenExtension kpdte = KeyProviderDelegationTokenExtension.createKeyProviderDelegationTokenExtension(kp);
                    final Credentials credentials = new Credentials();
                    final Token<?>[] tokens = kpdte.addDelegationTokens("client1", credentials);
                    Assert.assertEquals(1, credentials.getAllTokens().size());
                    InetSocketAddress kmsAddr = new InetSocketAddress(getKMSUrl().getHost(), getKMSUrl().getPort());
                    Assert.assertEquals(KMSDelegationToken.TOKEN_KIND, credentials.getToken(SecurityUtil.buildTokenService(kmsAddr)).getKind());
                    // Test non-renewer user cannot renew.
                    for (Token<?> token : tokens) {
                        if (!(token.getKind().equals(KMSDelegationToken.TOKEN_KIND))) {
                            LOG.info("Skipping token {}", token);
                            continue;
                        }
                        LOG.info("Got dt for " + uri + "; " + token);
                        try {
                            token.renew(clientConf);
                            Assert.fail("client should not be allowed to renew token with" + "renewer=client1");
                        } catch (Exception e) {
                            final DelegationTokenIdentifier identifier = (DelegationTokenIdentifier) token.decodeIdentifier();
                            GenericTestUtils.assertExceptionContains("tries to renew a token (" + identifier + ") with non-matching renewer", e);
                        }
                    }
                    final UserGroupInformation otherUgi;
                    if (kerb) {
                        UserGroupInformation.loginUserFromKeytab("client1", keytab.getAbsolutePath());
                        otherUgi = UserGroupInformation.getLoginUser();
                    } else {
                        otherUgi = UserGroupInformation.createUserForTesting("client1", new String[] { "other group" });
                        UserGroupInformation.setLoginUser(otherUgi);
                    }
                    try {
                        // test delegation token renewal via renewer
                        otherUgi.doAs(new PrivilegedExceptionAction<Void>() {

                            @Override
                            public Void run() throws Exception {
                                boolean renewed = false;
                                for (Token<?> token : tokens) {
                                    if (!(token.getKind().equals(KMSDelegationToken.TOKEN_KIND))) {
                                        LOG.info("Skipping token {}", token);
                                        continue;
                                    }
                                    LOG.info("Got dt for " + uri + "; " + token);
                                    long tokenLife = token.renew(clientConf);
                                    LOG.info("Renewed token of kind {}, new lifetime:{}", token.getKind(), tokenLife);
                                    Thread.sleep(100);
                                    long newTokenLife = token.renew(clientConf);
                                    LOG.info("Renewed token of kind {}, new lifetime:{}", token.getKind(), newTokenLife);
                                    Assert.assertTrue(newTokenLife > tokenLife);
                                    renewed = true;
                                }
                                Assert.assertTrue(renewed);
                                // test delegation token cancellation
                                for (Token<?> token : tokens) {
                                    if (!(token.getKind().equals(KMSDelegationToken.TOKEN_KIND))) {
                                        LOG.info("Skipping token {}", token);
                                        continue;
                                    }
                                    LOG.info("Got dt for " + uri + "; " + token);
                                    token.cancel(clientConf);
                                    LOG.info("Cancelled token of kind {}", token.getKind());
                                    try {
                                        token.renew(clientConf);
                                        Assert.fail("should not be able to renew a canceled token");
                                    } catch (Exception e) {
                                        LOG.info("Expected exception when renewing token", e);
                                    }
                                }
                                return null;
                            }
                        });
                        // Close the client provider. We will verify all providers'
                        // Truststore reloader threads are closed later.
                        kp.close();
                        return null;
                    } finally {
                        otherUgi.logoutUserFromKeytab();
                    }
                }
            });
            return null;
        }
    });
    // verify that providers created by KMSTokenRenewer are closed.
    if (ssl) {
        GenericTestUtils.waitFor(new Supplier<Boolean>() {

            @Override
            public Boolean get() {
                final Set<Thread> threadSet = Thread.getAllStackTraces().keySet();
                for (Thread t : threadSet) {
                    if (t.getName().contains(SSL_RELOADER_THREAD_NAME)) {
                        return false;
                    }
                }
                return true;
            }
        }, 1000, 10000);
    }
}

Example 2 with DelegationTokenIdentifier

use of org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier in project hadoop by apache.

the class TestKMS method testDelegationTokensUpdatedInUGI.

@Test
public void testDelegationTokensUpdatedInUGI() throws Exception {
    Configuration conf = new Configuration();
    File confDir = getTestDir();
    conf = createBaseKMSConf(confDir, conf);
    conf.set("hadoop.kms.authentication.delegation-token.max-lifetime.sec", "5");
    conf.set("hadoop.kms.authentication.delegation-token.renew-interval.sec", "5");
    writeConf(confDir, conf);
    // Running as a service (e.g. Yarn in practice).
    runServer(null, null, confDir, new KMSCallable<Void>() {

        @Override
        public Void call() throws Exception {
            final Configuration clientConf = new Configuration();
            final URI uri = createKMSUri(getKMSUrl());
            clientConf.set(KeyProviderFactory.KEY_PROVIDER_PATH, createKMSUri(getKMSUrl()).toString());
            final KeyProvider kp = createProvider(uri, clientConf);
            final KeyProviderDelegationTokenExtension kpdte = KeyProviderDelegationTokenExtension.createKeyProviderDelegationTokenExtension(kp);
            final InetSocketAddress kmsAddr = new InetSocketAddress(getKMSUrl().getHost(), getKMSUrl().getPort());
            // Job 1 (e.g. Yarn log aggregation job), with user DT.
            final Collection<Token<?>> job1Token = new HashSet<>();
            doAs("client", new PrivilegedExceptionAction<Void>() {

                @Override
                public Void run() throws Exception {
                    // Get a DT and use it.
                    final Credentials credentials = new Credentials();
                    kpdte.addDelegationTokens("client", credentials);
                    Assert.assertEquals(1, credentials.getAllTokens().size());
                    Assert.assertEquals(KMSDelegationToken.TOKEN_KIND, credentials.getToken(SecurityUtil.buildTokenService(kmsAddr)).getKind());
                    UserGroupInformation.getCurrentUser().addCredentials(credentials);
                    LOG.info("Added kms dt to credentials: {}", UserGroupInformation.getCurrentUser().getCredentials().getAllTokens());
                    Token<?> token = UserGroupInformation.getCurrentUser().getCredentials().getToken(SecurityUtil.buildTokenService(kmsAddr));
                    Assert.assertNotNull(token);
                    job1Token.add(token);
                    // Decode the token to get max time.
                    ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier());
                    DataInputStream dis = new DataInputStream(buf);
                    DelegationTokenIdentifier id = new DelegationTokenIdentifier(token.getKind());
                    id.readFields(dis);
                    dis.close();
                    final long maxTime = id.getMaxDate();
                    // wait for token to expire.
                    Thread.sleep(5100);
                    Assert.assertTrue("maxTime " + maxTime + " is not less than now.", maxTime > 0 && maxTime < Time.now());
                    try {
                        kp.getKeys();
                        Assert.fail("Operation should fail since dt is expired.");
                    } catch (Exception e) {
                        LOG.info("Expected error.", e);
                    }
                    return null;
                }
            });
            Assert.assertFalse(job1Token.isEmpty());
            // job 2 (e.g. Another Yarn log aggregation job, with user DT.
            doAs("client", new PrivilegedExceptionAction<Void>() {

                @Override
                public Void run() throws Exception {
                    // Get a new DT, but don't use it yet.
                    final Credentials newCreds = new Credentials();
                    kpdte.addDelegationTokens("client", newCreds);
                    Assert.assertEquals(1, newCreds.getAllTokens().size());
                    Assert.assertEquals(KMSDelegationToken.TOKEN_KIND, newCreds.getToken(SecurityUtil.buildTokenService(kmsAddr)).getKind());
                    // Using job 1's DT should fail.
                    final Credentials oldCreds = new Credentials();
                    for (Token<?> token : job1Token) {
                        if (token.getKind().equals(KMSDelegationToken.TOKEN_KIND)) {
                            oldCreds.addToken(SecurityUtil.buildTokenService(kmsAddr), token);
                        }
                    }
                    UserGroupInformation.getCurrentUser().addCredentials(oldCreds);
                    LOG.info("Added old kms dt to credentials: {}", UserGroupInformation.getCurrentUser().getCredentials().getAllTokens());
                    try {
                        kp.getKeys();
                        Assert.fail("Operation should fail since dt is expired.");
                    } catch (Exception e) {
                        LOG.info("Expected error.", e);
                    }
                    // Using the new DT should succeed.
                    Assert.assertEquals(1, newCreds.getAllTokens().size());
                    Assert.assertEquals(KMSDelegationToken.TOKEN_KIND, newCreds.getToken(SecurityUtil.buildTokenService(kmsAddr)).getKind());
                    UserGroupInformation.getCurrentUser().addCredentials(newCreds);
                    LOG.info("Credetials now are: {}", UserGroupInformation.getCurrentUser().getCredentials().getAllTokens());
                    kp.getKeys();
                    return null;
                }
            });
            return null;
        }
    });
}

Example 3 with DelegationTokenIdentifier

use of org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier in project hadoop by apache.

the class TestZKDelegationTokenSecretManager method testMultiNodeOperations.

@SuppressWarnings("unchecked")
@Test
public void testMultiNodeOperations() throws Exception {
    for (int i = 0; i < TEST_RETRIES; i++) {
        DelegationTokenManager tm1, tm2 = null;
        String connectString = zkServer.getConnectString();
        Configuration conf = getSecretConf(connectString);
        tm1 = new DelegationTokenManager(conf, new Text("bla"));
        tm1.init();
        tm2 = new DelegationTokenManager(conf, new Text("bla"));
        tm2.init();
        Token<DelegationTokenIdentifier> token = (Token<DelegationTokenIdentifier>) tm1.createToken(UserGroupInformation.getCurrentUser(), "foo");
        Assert.assertNotNull(token);
        tm2.verifyToken(token);
        tm2.renewToken(token, "foo");
        tm1.verifyToken(token);
        tm1.cancelToken(token, "foo");
        try {
            verifyTokenFail(tm2, token);
            fail("Expected InvalidToken");
        } catch (SecretManager.InvalidToken it) {
        // Ignore
        }
        token = (Token<DelegationTokenIdentifier>) tm2.createToken(UserGroupInformation.getCurrentUser(), "bar");
        Assert.assertNotNull(token);
        tm1.verifyToken(token);
        tm1.renewToken(token, "bar");
        tm2.verifyToken(token);
        tm2.cancelToken(token, "bar");
        try {
            verifyTokenFail(tm1, token);
            fail("Expected InvalidToken");
        } catch (SecretManager.InvalidToken it) {
        // Ignore
        }
        verifyDestroy(tm1, conf);
        verifyDestroy(tm2, conf);
    }
}

Example 4 with DelegationTokenIdentifier

use of org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier in project hadoop by apache.

the class TestZKDelegationTokenSecretManager method testStopThreads.

@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testStopThreads() throws Exception {
    DelegationTokenManager tm1 = null;
    String connectString = zkServer.getConnectString();
    // let's make the update interval short and the shutdown interval
    // comparatively longer, so if the update thread runs after shutdown,
    // it will cause an error.
    final long updateIntervalSeconds = 1;
    final long shutdownTimeoutMillis = updateIntervalSeconds * 1000 * 5;
    Configuration conf = getSecretConf(connectString);
    conf.setLong(DelegationTokenManager.UPDATE_INTERVAL, updateIntervalSeconds);
    conf.setLong(DelegationTokenManager.REMOVAL_SCAN_INTERVAL, updateIntervalSeconds);
    conf.setLong(DelegationTokenManager.RENEW_INTERVAL, updateIntervalSeconds);
    conf.setLong(ZKDelegationTokenSecretManager.ZK_DTSM_ZK_SHUTDOWN_TIMEOUT, shutdownTimeoutMillis);
    tm1 = new DelegationTokenManager(conf, new Text("foo"));
    tm1.init();
    Token<DelegationTokenIdentifier> token = (Token<DelegationTokenIdentifier>) tm1.createToken(UserGroupInformation.getCurrentUser(), "foo");
    Assert.assertNotNull(token);
    AbstractDelegationTokenSecretManager sm = tm1.getDelegationTokenSecretManager();
    ZKDelegationTokenSecretManager zksm = (ZKDelegationTokenSecretManager) sm;
    ExecutorService es = zksm.getListenerThreadPool();
    es.submit(new Callable<Void>() {

        public Void call() throws Exception {
            // force this to be shutdownNow
            Thread.sleep(shutdownTimeoutMillis * 2);
            return null;
        }
    });
    tm1.destroy();
}

Example 5 with DelegationTokenIdentifier

use of org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier in project hadoop by apache.

the class TestZKDelegationTokenSecretManager method testNodesLoadedAfterRestart.

@SuppressWarnings({ "unchecked" })
@Test
public void testNodesLoadedAfterRestart() throws Exception {
    final String connectString = zkServer.getConnectString();
    final Configuration conf = getSecretConf(connectString);
    final int removeScan = 1;
    // Set the remove scan interval to remove expired tokens
    conf.setLong(DelegationTokenManager.REMOVAL_SCAN_INTERVAL, removeScan);
    // Set the update interval to trigger background thread to run. The thread
    // is hard-coded to sleep at least 5 seconds.
    conf.setLong(DelegationTokenManager.UPDATE_INTERVAL, 5);
    // Set token expire time to 5 seconds.
    conf.setLong(DelegationTokenManager.RENEW_INTERVAL, 5);
    DelegationTokenManager tm = new DelegationTokenManager(conf, new Text("bla"));
    tm.init();
    Token<DelegationTokenIdentifier> token = (Token<DelegationTokenIdentifier>) tm.createToken(UserGroupInformation.getCurrentUser(), "good");
    Assert.assertNotNull(token);
    Token<DelegationTokenIdentifier> cancelled = (Token<DelegationTokenIdentifier>) tm.createToken(UserGroupInformation.getCurrentUser(), "cancelled");
    Assert.assertNotNull(cancelled);
    tm.verifyToken(token);
    tm.verifyToken(cancelled);
    // Cancel one token, verify it's gone
    tm.cancelToken(cancelled, "cancelled");
    final AbstractDelegationTokenSecretManager sm = tm.getDelegationTokenSecretManager();
    final ZKDelegationTokenSecretManager zksm = (ZKDelegationTokenSecretManager) sm;
    final AbstractDelegationTokenIdentifier idCancelled = sm.decodeTokenIdentifier(cancelled);
    LOG.info("Waiting for the cancelled token to be removed");
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        @Override
        public Boolean get() {
            AbstractDelegationTokenSecretManager.DelegationTokenInformation dtinfo = zksm.getTokenInfo(idCancelled);
            return dtinfo == null;
        }
    }, 100, 5000);
    // Fake a restart which launches a new tm
    tm.destroy();
    tm = new DelegationTokenManager(conf, new Text("bla"));
    tm.init();
    final AbstractDelegationTokenSecretManager smNew = tm.getDelegationTokenSecretManager();
    final ZKDelegationTokenSecretManager zksmNew = (ZKDelegationTokenSecretManager) smNew;
    // The cancelled token should be gone, and not loaded.
    AbstractDelegationTokenIdentifier id = smNew.decodeTokenIdentifier(cancelled);
    AbstractDelegationTokenSecretManager.DelegationTokenInformation dtinfo = zksmNew.getTokenInfo(id);
    Assert.assertNull("canceled dt should be gone!", dtinfo);
    // The good token should be loaded on startup, and removed after expiry.
    id = smNew.decodeTokenIdentifier(token);
    dtinfo = zksmNew.getTokenInfoFromMemory(id);
    Assert.assertNotNull("good dt should be in memory!", dtinfo);
    // Wait for the good token to expire.
    Thread.sleep(5000);
    final ZKDelegationTokenSecretManager zksm1 = zksmNew;
    final AbstractDelegationTokenIdentifier id1 = id;
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        @Override
        public Boolean get() {
            LOG.info("Waiting for the expired token to be removed...");
            return zksm1.getTokenInfo(id1) == null;
        }
    }, 1000, 5000);
}