Examples with KeyProviderDelegationTokenExtension org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension used on opensource projects

Example 1 with KeyProviderDelegationTokenExtension

use of org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension in project hadoop by apache.

the class DistributedFileSystem method addDelegationTokens.

@Override
public Token<?>[] addDelegationTokens(final String renewer, Credentials credentials) throws IOException {
    Token<?>[] tokens = super.addDelegationTokens(renewer, credentials);
    if (dfs.isHDFSEncryptionEnabled()) {
        KeyProviderDelegationTokenExtension keyProviderDelegationTokenExtension = KeyProviderDelegationTokenExtension.createKeyProviderDelegationTokenExtension(dfs.getKeyProvider());
        Token<?>[] kpTokens = keyProviderDelegationTokenExtension.addDelegationTokens(renewer, credentials);
        if (tokens != null && kpTokens != null) {
            Token<?>[] all = new Token<?>[tokens.length + kpTokens.length];
            System.arraycopy(tokens, 0, all, 0, tokens.length);
            System.arraycopy(kpTokens, 0, all, tokens.length, kpTokens.length);
            tokens = all;
        } else {
            tokens = (tokens != null) ? tokens : kpTokens;
        }
    }
    return tokens;
}

Example 2 with KeyProviderDelegationTokenExtension

use of org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension in project hadoop by apache.

the class TestKMS method testDelegationTokensOps.

private void testDelegationTokensOps(final boolean ssl, final boolean kerb) throws Exception {
    final File confDir = getTestDir();
    final Configuration conf;
    if (kerb) {
        conf = setupConfForKerberos(confDir);
    } else {
        conf = createBaseKMSConf(confDir, null);
    }
    final String keystore;
    final String password;
    if (ssl) {
        final String sslConfDir = KeyStoreTestUtil.getClasspathDir(TestKMS.class);
        KeyStoreTestUtil.setupSSLConfig(confDir.getAbsolutePath(), sslConfDir, conf, false);
        keystore = confDir.getAbsolutePath() + "/serverKS.jks";
        password = "serverP";
    } else {
        keystore = null;
        password = null;
    }
    writeConf(confDir, conf);
    runServer(keystore, password, confDir, new KMSCallable<Void>() {

        @Override
        public Void call() throws Exception {
            final Configuration clientConf = new Configuration();
            final URI uri = createKMSUri(getKMSUrl());
            clientConf.set(KeyProviderFactory.KEY_PROVIDER_PATH, createKMSUri(getKMSUrl()).toString());
            doAs("client", new PrivilegedExceptionAction<Void>() {

                @Override
                public Void run() throws Exception {
                    KeyProvider kp = createProvider(uri, clientConf);
                    // test delegation token retrieval
                    KeyProviderDelegationTokenExtension kpdte = KeyProviderDelegationTokenExtension.createKeyProviderDelegationTokenExtension(kp);
                    final Credentials credentials = new Credentials();
                    final Token<?>[] tokens = kpdte.addDelegationTokens("client1", credentials);
                    Assert.assertEquals(1, credentials.getAllTokens().size());
                    InetSocketAddress kmsAddr = new InetSocketAddress(getKMSUrl().getHost(), getKMSUrl().getPort());
                    Assert.assertEquals(KMSDelegationToken.TOKEN_KIND, credentials.getToken(SecurityUtil.buildTokenService(kmsAddr)).getKind());
                    // Test non-renewer user cannot renew.
                    for (Token<?> token : tokens) {
                        if (!(token.getKind().equals(KMSDelegationToken.TOKEN_KIND))) {
                            LOG.info("Skipping token {}", token);
                            continue;
                        }
                        LOG.info("Got dt for " + uri + "; " + token);
                        try {
                            token.renew(clientConf);
                            Assert.fail("client should not be allowed to renew token with" + "renewer=client1");
                        } catch (Exception e) {
                            final DelegationTokenIdentifier identifier = (DelegationTokenIdentifier) token.decodeIdentifier();
                            GenericTestUtils.assertExceptionContains("tries to renew a token (" + identifier + ") with non-matching renewer", e);
                        }
                    }
                    final UserGroupInformation otherUgi;
                    if (kerb) {
                        UserGroupInformation.loginUserFromKeytab("client1", keytab.getAbsolutePath());
                        otherUgi = UserGroupInformation.getLoginUser();
                    } else {
                        otherUgi = UserGroupInformation.createUserForTesting("client1", new String[] { "other group" });
                        UserGroupInformation.setLoginUser(otherUgi);
                    }
                    try {
                        // test delegation token renewal via renewer
                        otherUgi.doAs(new PrivilegedExceptionAction<Void>() {

                            @Override
                            public Void run() throws Exception {
                                boolean renewed = false;
                                for (Token<?> token : tokens) {
                                    if (!(token.getKind().equals(KMSDelegationToken.TOKEN_KIND))) {
                                        LOG.info("Skipping token {}", token);
                                        continue;
                                    }
                                    LOG.info("Got dt for " + uri + "; " + token);
                                    long tokenLife = token.renew(clientConf);
                                    LOG.info("Renewed token of kind {}, new lifetime:{}", token.getKind(), tokenLife);
                                    Thread.sleep(100);
                                    long newTokenLife = token.renew(clientConf);
                                    LOG.info("Renewed token of kind {}, new lifetime:{}", token.getKind(), newTokenLife);
                                    Assert.assertTrue(newTokenLife > tokenLife);
                                    renewed = true;
                                }
                                Assert.assertTrue(renewed);
                                // test delegation token cancellation
                                for (Token<?> token : tokens) {
                                    if (!(token.getKind().equals(KMSDelegationToken.TOKEN_KIND))) {
                                        LOG.info("Skipping token {}", token);
                                        continue;
                                    }
                                    LOG.info("Got dt for " + uri + "; " + token);
                                    token.cancel(clientConf);
                                    LOG.info("Cancelled token of kind {}", token.getKind());
                                    try {
                                        token.renew(clientConf);
                                        Assert.fail("should not be able to renew a canceled token");
                                    } catch (Exception e) {
                                        LOG.info("Expected exception when renewing token", e);
                                    }
                                }
                                return null;
                            }
                        });
                        // Close the client provider. We will verify all providers'
                        // Truststore reloader threads are closed later.
                        kp.close();
                        return null;
                    } finally {
                        otherUgi.logoutUserFromKeytab();
                    }
                }
            });
            return null;
        }
    });
    // verify that providers created by KMSTokenRenewer are closed.
    if (ssl) {
        GenericTestUtils.waitFor(new Supplier<Boolean>() {

            @Override
            public Boolean get() {
                final Set<Thread> threadSet = Thread.getAllStackTraces().keySet();
                for (Thread t : threadSet) {
                    if (t.getName().contains(SSL_RELOADER_THREAD_NAME)) {
                        return false;
                    }
                }
                return true;
            }
        }, 1000, 10000);
    }
}

Example 3 with KeyProviderDelegationTokenExtension

use of org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension in project hadoop by apache.

the class TestKMS method testDelegationTokensUpdatedInUGI.

@Test
public void testDelegationTokensUpdatedInUGI() throws Exception {
    Configuration conf = new Configuration();
    File confDir = getTestDir();
    conf = createBaseKMSConf(confDir, conf);
    conf.set("hadoop.kms.authentication.delegation-token.max-lifetime.sec", "5");
    conf.set("hadoop.kms.authentication.delegation-token.renew-interval.sec", "5");
    writeConf(confDir, conf);
    // Running as a service (e.g. Yarn in practice).
    runServer(null, null, confDir, new KMSCallable<Void>() {

        @Override
        public Void call() throws Exception {
            final Configuration clientConf = new Configuration();
            final URI uri = createKMSUri(getKMSUrl());
            clientConf.set(KeyProviderFactory.KEY_PROVIDER_PATH, createKMSUri(getKMSUrl()).toString());
            final KeyProvider kp = createProvider(uri, clientConf);
            final KeyProviderDelegationTokenExtension kpdte = KeyProviderDelegationTokenExtension.createKeyProviderDelegationTokenExtension(kp);
            final InetSocketAddress kmsAddr = new InetSocketAddress(getKMSUrl().getHost(), getKMSUrl().getPort());
            // Job 1 (e.g. Yarn log aggregation job), with user DT.
            final Collection<Token<?>> job1Token = new HashSet<>();
            doAs("client", new PrivilegedExceptionAction<Void>() {

                @Override
                public Void run() throws Exception {
                    // Get a DT and use it.
                    final Credentials credentials = new Credentials();
                    kpdte.addDelegationTokens("client", credentials);
                    Assert.assertEquals(1, credentials.getAllTokens().size());
                    Assert.assertEquals(KMSDelegationToken.TOKEN_KIND, credentials.getToken(SecurityUtil.buildTokenService(kmsAddr)).getKind());
                    UserGroupInformation.getCurrentUser().addCredentials(credentials);
                    LOG.info("Added kms dt to credentials: {}", UserGroupInformation.getCurrentUser().getCredentials().getAllTokens());
                    Token<?> token = UserGroupInformation.getCurrentUser().getCredentials().getToken(SecurityUtil.buildTokenService(kmsAddr));
                    Assert.assertNotNull(token);
                    job1Token.add(token);
                    // Decode the token to get max time.
                    ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier());
                    DataInputStream dis = new DataInputStream(buf);
                    DelegationTokenIdentifier id = new DelegationTokenIdentifier(token.getKind());
                    id.readFields(dis);
                    dis.close();
                    final long maxTime = id.getMaxDate();
                    // wait for token to expire.
                    Thread.sleep(5100);
                    Assert.assertTrue("maxTime " + maxTime + " is not less than now.", maxTime > 0 && maxTime < Time.now());
                    try {
                        kp.getKeys();
                        Assert.fail("Operation should fail since dt is expired.");
                    } catch (Exception e) {
                        LOG.info("Expected error.", e);
                    }
                    return null;
                }
            });
            Assert.assertFalse(job1Token.isEmpty());
            // job 2 (e.g. Another Yarn log aggregation job, with user DT.
            doAs("client", new PrivilegedExceptionAction<Void>() {

                @Override
                public Void run() throws Exception {
                    // Get a new DT, but don't use it yet.
                    final Credentials newCreds = new Credentials();
                    kpdte.addDelegationTokens("client", newCreds);
                    Assert.assertEquals(1, newCreds.getAllTokens().size());
                    Assert.assertEquals(KMSDelegationToken.TOKEN_KIND, newCreds.getToken(SecurityUtil.buildTokenService(kmsAddr)).getKind());
                    // Using job 1's DT should fail.
                    final Credentials oldCreds = new Credentials();
                    for (Token<?> token : job1Token) {
                        if (token.getKind().equals(KMSDelegationToken.TOKEN_KIND)) {
                            oldCreds.addToken(SecurityUtil.buildTokenService(kmsAddr), token);
                        }
                    }
                    UserGroupInformation.getCurrentUser().addCredentials(oldCreds);
                    LOG.info("Added old kms dt to credentials: {}", UserGroupInformation.getCurrentUser().getCredentials().getAllTokens());
                    try {
                        kp.getKeys();
                        Assert.fail("Operation should fail since dt is expired.");
                    } catch (Exception e) {
                        LOG.info("Expected error.", e);
                    }
                    // Using the new DT should succeed.
                    Assert.assertEquals(1, newCreds.getAllTokens().size());
                    Assert.assertEquals(KMSDelegationToken.TOKEN_KIND, newCreds.getToken(SecurityUtil.buildTokenService(kmsAddr)).getKind());
                    UserGroupInformation.getCurrentUser().addCredentials(newCreds);
                    LOG.info("Credetials now are: {}", UserGroupInformation.getCurrentUser().getCredentials().getAllTokens());
                    kp.getKeys();
                    return null;
                }
            });
            return null;
        }
    });
}

Example 4 with KeyProviderDelegationTokenExtension

use of org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension in project hadoop by apache.

the class TestKMS method testDelegationTokenAccess.

@Test
public void testDelegationTokenAccess() throws Exception {
    Configuration conf = new Configuration();
    conf.set("hadoop.security.authentication", "kerberos");
    final File testDir = getTestDir();
    conf = createBaseKMSConf(testDir, conf);
    conf.set("hadoop.kms.authentication.type", "kerberos");
    conf.set("hadoop.kms.authentication.kerberos.keytab", keytab.getAbsolutePath());
    conf.set("hadoop.kms.authentication.kerberos.principal", "HTTP/localhost");
    conf.set("hadoop.kms.authentication.kerberos.name.rules", "DEFAULT");
    final String keyA = "key_a";
    final String keyD = "key_d";
    conf.set(KeyAuthorizationKeyProvider.KEY_ACL + keyA + ".ALL", "*");
    conf.set(KeyAuthorizationKeyProvider.KEY_ACL + keyD + ".ALL", "*");
    writeConf(testDir, conf);
    runServer(null, null, testDir, new KMSCallable<Void>() {

        @Override
        public Void call() throws Exception {
            final Configuration conf = new Configuration();
            conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 128);
            final URI uri = createKMSUri(getKMSUrl());
            final Credentials credentials = new Credentials();
            final UserGroupInformation nonKerberosUgi = UserGroupInformation.getCurrentUser();
            try {
                KeyProvider kp = createProvider(uri, conf);
                kp.createKey(keyA, new KeyProvider.Options(conf));
            } catch (IOException ex) {
                System.out.println(ex.getMessage());
            }
            doAs("client", new PrivilegedExceptionAction<Void>() {

                @Override
                public Void run() throws Exception {
                    KeyProvider kp = createProvider(uri, conf);
                    KeyProviderDelegationTokenExtension kpdte = KeyProviderDelegationTokenExtension.createKeyProviderDelegationTokenExtension(kp);
                    kpdte.addDelegationTokens("foo", credentials);
                    return null;
                }
            });
            nonKerberosUgi.addCredentials(credentials);
            try {
                KeyProvider kp = createProvider(uri, conf);
                kp.createKey(keyA, new KeyProvider.Options(conf));
            } catch (IOException ex) {
                System.out.println(ex.getMessage());
            }
            nonKerberosUgi.doAs(new PrivilegedExceptionAction<Void>() {

                @Override
                public Void run() throws Exception {
                    KeyProvider kp = createProvider(uri, conf);
                    kp.createKey(keyD, new KeyProvider.Options(conf));
                    return null;
                }
            });
            return null;
        }
    });
}