Examples with KeyProvider org.apache.hadoop.crypto.key.KeyProvider used on opensource projects

Example 26 with KeyProvider

use of org.apache.hadoop.crypto.key.KeyProvider in project hadoop by apache.

the class TestLoadBalancingKMSClientProvider method testLoadBalancingWithAllBadNodes.

@Test
public void testLoadBalancingWithAllBadNodes() throws Exception {
    Configuration conf = new Configuration();
    KMSClientProvider p1 = mock(KMSClientProvider.class);
    when(p1.createKey(Mockito.anyString(), Mockito.any(Options.class))).thenThrow(new IOException("p1"));
    KMSClientProvider p2 = mock(KMSClientProvider.class);
    when(p2.createKey(Mockito.anyString(), Mockito.any(Options.class))).thenThrow(new IOException("p2"));
    KMSClientProvider p3 = mock(KMSClientProvider.class);
    when(p3.createKey(Mockito.anyString(), Mockito.any(Options.class))).thenThrow(new IOException("p3"));
    KMSClientProvider p4 = mock(KMSClientProvider.class);
    when(p4.createKey(Mockito.anyString(), Mockito.any(Options.class))).thenThrow(new IOException("p4"));
    when(p1.getKMSUrl()).thenReturn("p1");
    when(p2.getKMSUrl()).thenReturn("p2");
    when(p3.getKMSUrl()).thenReturn("p3");
    when(p4.getKMSUrl()).thenReturn("p4");
    KeyProvider kp = new LoadBalancingKMSClientProvider(new KMSClientProvider[] { p1, p2, p3, p4 }, 0, conf);
    try {
        kp.createKey("test3", new Options(conf)).getName();
        fail("Should fail since all providers threw an IOException");
    } catch (Exception e) {
        assertTrue(e instanceof IOException);
    }
}

Example 27 with KeyProvider

use of org.apache.hadoop.crypto.key.KeyProvider in project hadoop by apache.

the class TestKMS method testTGTRenewal.

@Test
public void testTGTRenewal() throws Exception {
    tearDownMiniKdc();
    Properties kdcConf = MiniKdc.createConf();
    kdcConf.setProperty(MiniKdc.MAX_TICKET_LIFETIME, "3");
    kdcConf.setProperty(MiniKdc.MIN_TICKET_LIFETIME, "3");
    setUpMiniKdc(kdcConf);
    Configuration conf = new Configuration();
    conf.set("hadoop.security.authentication", "kerberos");
    final File testDir = getTestDir();
    conf = createBaseKMSConf(testDir, conf);
    conf.set("hadoop.kms.authentication.type", "kerberos");
    conf.set("hadoop.kms.authentication.kerberos.keytab", keytab.getAbsolutePath());
    conf.set("hadoop.kms.authentication.kerberos.principal", "HTTP/localhost");
    conf.set("hadoop.kms.authentication.kerberos.name.rules", "DEFAULT");
    conf.set("hadoop.kms.proxyuser.client.users", "*");
    conf.set("hadoop.kms.proxyuser.client.hosts", "*");
    writeConf(testDir, conf);
    runServer(null, null, testDir, new KMSCallable<Void>() {

        @Override
        public Void call() throws Exception {
            final Configuration conf = new Configuration();
            final URI uri = createKMSUri(getKMSUrl());
            UserGroupInformation.setShouldRenewImmediatelyForTests(true);
            UserGroupInformation.loginUserFromKeytab("client", keytab.getAbsolutePath());
            final UserGroupInformation clientUgi = UserGroupInformation.getCurrentUser();
            clientUgi.doAs(new PrivilegedExceptionAction<Void>() {

                @Override
                public Void run() throws Exception {
                    // Verify getKeys can relogin
                    Thread.sleep(3100);
                    KeyProvider kp = createProvider(uri, conf);
                    kp.getKeys();
                    // Verify addDelegationTokens can relogin
                    // (different code path inside KMSClientProvider than getKeys)
                    Thread.sleep(3100);
                    kp = createProvider(uri, conf);
                    ((KeyProviderDelegationTokenExtension.DelegationTokenExtension) kp).addDelegationTokens("myuser", new Credentials());
                    // Verify getKeys can relogin with proxy user
                    UserGroupInformation anotherUgi = UserGroupInformation.createProxyUser("client1", clientUgi);
                    anotherUgi.doAs(new PrivilegedExceptionAction<Void>() {

                        @Override
                        public Void run() throws Exception {
                            Thread.sleep(3100);
                            KeyProvider kp = createProvider(uri, conf);
                            kp.getKeys();
                            return null;
                        }
                    });
                    return null;
                }
            });
            return null;
        }
    });
}

Example 28 with KeyProvider

use of org.apache.hadoop.crypto.key.KeyProvider in project hadoop by apache.

the class TestKMS method testKMSBlackList.

@Test
public void testKMSBlackList() throws Exception {
    Configuration conf = new Configuration();
    conf.set("hadoop.security.authentication", "kerberos");
    File testDir = getTestDir();
    conf = createBaseKMSConf(testDir, conf);
    conf.set("hadoop.kms.authentication.type", "kerberos");
    conf.set("hadoop.kms.authentication.kerberos.keytab", keytab.getAbsolutePath());
    conf.set("hadoop.kms.authentication.kerberos.principal", "HTTP/localhost");
    conf.set("hadoop.kms.authentication.kerberos.name.rules", "DEFAULT");
    for (KMSACLs.Type type : KMSACLs.Type.values()) {
        conf.set(type.getAclConfigKey(), " ");
    }
    conf.set(KMSACLs.Type.CREATE.getAclConfigKey(), "client,hdfs,otheradmin");
    conf.set(KMSACLs.Type.GENERATE_EEK.getAclConfigKey(), "client,hdfs,otheradmin");
    conf.set(KMSACLs.Type.DECRYPT_EEK.getAclConfigKey(), "client,hdfs,otheradmin");
    conf.set(KMSACLs.Type.DECRYPT_EEK.getBlacklistConfigKey(), "hdfs,otheradmin");
    conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "ck0.ALL", "*");
    conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "ck1.ALL", "*");
    writeConf(testDir, conf);
    runServer(null, null, testDir, new KMSCallable<Void>() {

        @Override
        public Void call() throws Exception {
            final Configuration conf = new Configuration();
            conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 128);
            final URI uri = createKMSUri(getKMSUrl());
            doAs("client", new PrivilegedExceptionAction<Void>() {

                @Override
                public Void run() throws Exception {
                    try {
                        KeyProvider kp = createProvider(uri, conf);
                        KeyProvider.KeyVersion kv = kp.createKey("ck0", new KeyProvider.Options(conf));
                        EncryptedKeyVersion eek = ((CryptoExtension) kp).generateEncryptedKey("ck0");
                        ((CryptoExtension) kp).decryptEncryptedKey(eek);
                        Assert.assertNull(kv.getMaterial());
                    } catch (Exception ex) {
                        Assert.fail(ex.getMessage());
                    }
                    return null;
                }
            });
            doAs("hdfs", new PrivilegedExceptionAction<Void>() {

                @Override
                public Void run() throws Exception {
                    try {
                        KeyProvider kp = createProvider(uri, conf);
                        KeyProvider.KeyVersion kv = kp.createKey("ck1", new KeyProvider.Options(conf));
                        EncryptedKeyVersion eek = ((CryptoExtension) kp).generateEncryptedKey("ck1");
                        ((CryptoExtension) kp).decryptEncryptedKey(eek);
                        Assert.fail("admin user must not be allowed to decrypt !!");
                    } catch (Exception ex) {
                    }
                    return null;
                }
            });
            doAs("otheradmin", new PrivilegedExceptionAction<Void>() {

                @Override
                public Void run() throws Exception {
                    try {
                        KeyProvider kp = createProvider(uri, conf);
                        KeyProvider.KeyVersion kv = kp.createKey("ck2", new KeyProvider.Options(conf));
                        EncryptedKeyVersion eek = ((CryptoExtension) kp).generateEncryptedKey("ck2");
                        ((CryptoExtension) kp).decryptEncryptedKey(eek);
                        Assert.fail("admin user must not be allowed to decrypt !!");
                    } catch (Exception ex) {
                    }
                    return null;
                }
            });
            return null;
        }
    });
}

Example 29 with KeyProvider

use of org.apache.hadoop.crypto.key.KeyProvider in project hadoop by apache.

the class TestKMS method testACLs.

@Test
@SuppressWarnings("checkstyle:methodlength")
public void testACLs() throws Exception {
    Configuration conf = new Configuration();
    conf.set("hadoop.security.authentication", "kerberos");
    final File testDir = getTestDir();
    conf = createBaseKMSConf(testDir, conf);
    conf.set("hadoop.kms.authentication.type", "kerberos");
    conf.set("hadoop.kms.authentication.kerberos.keytab", keytab.getAbsolutePath());
    conf.set("hadoop.kms.authentication.kerberos.principal", "HTTP/localhost");
    conf.set("hadoop.kms.authentication.kerberos.name.rules", "DEFAULT");
    for (KMSACLs.Type type : KMSACLs.Type.values()) {
        conf.set(type.getAclConfigKey(), type.toString());
    }
    conf.set(KMSACLs.Type.CREATE.getAclConfigKey(), KMSACLs.Type.CREATE.toString() + ",SET_KEY_MATERIAL");
    conf.set(KMSACLs.Type.ROLLOVER.getAclConfigKey(), KMSACLs.Type.ROLLOVER.toString() + ",SET_KEY_MATERIAL");
    conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k0.ALL", "*");
    conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k1.ALL", "*");
    writeConf(testDir, conf);
    runServer(null, null, testDir, new KMSCallable<Void>() {

        @Override
        public Void call() throws Exception {
            final Configuration conf = new Configuration();
            conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 128);
            final URI uri = createKMSUri(getKMSUrl());
            //nothing allowed
            doAs("client", new PrivilegedExceptionAction<Void>() {

                @Override
                public Void run() throws Exception {
                    KeyProvider kp = createProvider(uri, conf);
                    try {
                        kp.createKey("k", new KeyProvider.Options(conf));
                        Assert.fail();
                    } catch (AuthorizationException ex) {
                    //NOP
                    } catch (Exception ex) {
                        Assert.fail(ex.getMessage());
                    }
                    try {
                        kp.createKey("k", new byte[16], new KeyProvider.Options(conf));
                        Assert.fail();
                    } catch (AuthorizationException ex) {
                    //NOP
                    } catch (Exception ex) {
                        Assert.fail(ex.getMessage());
                    }
                    try {
                        kp.rollNewVersion("k");
                        Assert.fail();
                    } catch (AuthorizationException ex) {
                    //NOP
                    } catch (Exception ex) {
                        Assert.fail(ex.getMessage());
                    }
                    try {
                        kp.rollNewVersion("k", new byte[16]);
                        Assert.fail();
                    } catch (AuthorizationException ex) {
                    //NOP
                    } catch (Exception ex) {
                        Assert.fail(ex.getMessage());
                    }
                    try {
                        kp.getKeys();
                        Assert.fail();
                    } catch (AuthorizationException ex) {
                    //NOP
                    } catch (Exception ex) {
                        Assert.fail(ex.getMessage());
                    }
                    try {
                        kp.getKeysMetadata("k");
                        Assert.fail();
                    } catch (AuthorizationException ex) {
                    //NOP
                    } catch (Exception ex) {
                        Assert.fail(ex.getMessage());
                    }
                    try {
                        // we are using JavaKeyStoreProvider for testing, so we know how
                        // the keyversion is created.
                        kp.getKeyVersion("k@0");
                        Assert.fail();
                    } catch (AuthorizationException ex) {
                    //NOP
                    } catch (Exception ex) {
                        Assert.fail(ex.getMessage());
                    }
                    try {
                        kp.getCurrentKey("k");
                        Assert.fail();
                    } catch (AuthorizationException ex) {
                    //NOP
                    } catch (Exception ex) {
                        Assert.fail(ex.getMessage());
                    }
                    try {
                        kp.getMetadata("k");
                        Assert.fail();
                    } catch (AuthorizationException ex) {
                    //NOP
                    } catch (Exception ex) {
                        Assert.fail(ex.getMessage());
                    }
                    try {
                        kp.getKeyVersions("k");
                        Assert.fail();
                    } catch (AuthorizationException ex) {
                    //NOP
                    } catch (Exception ex) {
                        Assert.fail(ex.getMessage());
                    }
                    return null;
                }
            });
            doAs("CREATE", new PrivilegedExceptionAction<Void>() {

                @Override
                public Void run() throws Exception {
                    KeyProvider kp = createProvider(uri, conf);
                    try {
                        KeyProvider.KeyVersion kv = kp.createKey("k0", new KeyProvider.Options(conf));
                        Assert.assertNull(kv.getMaterial());
                    } catch (Exception ex) {
                        Assert.fail(ex.getMessage());
                    }
                    return null;
                }
            });
            doAs("DELETE", new PrivilegedExceptionAction<Void>() {

                @Override
                public Void run() throws Exception {
                    KeyProvider kp = createProvider(uri, conf);
                    try {
                        kp.deleteKey("k0");
                    } catch (Exception ex) {
                        Assert.fail(ex.getMessage());
                    }
                    return null;
                }
            });
            doAs("SET_KEY_MATERIAL", new PrivilegedExceptionAction<Void>() {

                @Override
                public Void run() throws Exception {
                    KeyProvider kp = createProvider(uri, conf);
                    try {
                        KeyProvider.KeyVersion kv = kp.createKey("k1", new byte[16], new KeyProvider.Options(conf));
                        Assert.assertNull(kv.getMaterial());
                    } catch (Exception ex) {
                        Assert.fail(ex.getMessage());
                    }
                    return null;
                }
            });
            doAs("ROLLOVER", new PrivilegedExceptionAction<Void>() {

                @Override
                public Void run() throws Exception {
                    KeyProvider kp = createProvider(uri, conf);
                    try {
                        KeyProvider.KeyVersion kv = kp.rollNewVersion("k1");
                        Assert.assertNull(kv.getMaterial());
                    } catch (Exception ex) {
                        Assert.fail(ex.getMessage());
                    }
                    return null;
                }
            });
            doAs("SET_KEY_MATERIAL", new PrivilegedExceptionAction<Void>() {

                @Override
                public Void run() throws Exception {
                    KeyProvider kp = createProvider(uri, conf);
                    try {
                        KeyProvider.KeyVersion kv = kp.rollNewVersion("k1", new byte[16]);
                        Assert.assertNull(kv.getMaterial());
                    } catch (Exception ex) {
                        Assert.fail(ex.getMessage());
                    }
                    return null;
                }
            });
            final KeyVersion currKv = doAs("GET", new PrivilegedExceptionAction<KeyVersion>() {

                @Override
                public KeyVersion run() throws Exception {
                    KeyProvider kp = createProvider(uri, conf);
                    try {
                        kp.getKeyVersion("k1@0");
                        KeyVersion kv = kp.getCurrentKey("k1");
                        return kv;
                    } catch (Exception ex) {
                        Assert.fail(ex.toString());
                    }
                    return null;
                }
            });
            final EncryptedKeyVersion encKv = doAs("GENERATE_EEK", new PrivilegedExceptionAction<EncryptedKeyVersion>() {

                @Override
                public EncryptedKeyVersion run() throws Exception {
                    KeyProvider kp = createProvider(uri, conf);
                    try {
                        KeyProviderCryptoExtension kpCE = KeyProviderCryptoExtension.createKeyProviderCryptoExtension(kp);
                        EncryptedKeyVersion ek1 = kpCE.generateEncryptedKey(currKv.getName());
                        return ek1;
                    } catch (Exception ex) {
                        Assert.fail(ex.toString());
                    }
                    return null;
                }
            });
            doAs("GENERATE_EEK", new PrivilegedExceptionAction<Void>() {

                @Override
                public Void run() throws Exception {
                    KeyProvider kp = createProvider(uri, conf);
                    KeyProviderCryptoExtension kpCE = KeyProviderCryptoExtension.createKeyProviderCryptoExtension(kp);
                    kpCE.reencryptEncryptedKey(encKv);
                    return null;
                }
            });
            doAs("DECRYPT_EEK", new PrivilegedExceptionAction<Void>() {

                @Override
                public Void run() throws Exception {
                    KeyProvider kp = createProvider(uri, conf);
                    try {
                        KeyProviderCryptoExtension kpCE = KeyProviderCryptoExtension.createKeyProviderCryptoExtension(kp);
                        kpCE.decryptEncryptedKey(encKv);
                    } catch (Exception ex) {
                        Assert.fail(ex.getMessage());
                    }
                    return null;
                }
            });
            doAs("GET_KEYS", new PrivilegedExceptionAction<Void>() {

                @Override
                public Void run() throws Exception {
                    KeyProvider kp = createProvider(uri, conf);
                    try {
                        kp.getKeys();
                    } catch (Exception ex) {
                        Assert.fail(ex.getMessage());
                    }
                    return null;
                }
            });
            doAs("GET_METADATA", new PrivilegedExceptionAction<Void>() {

                @Override
                public Void run() throws Exception {
                    KeyProvider kp = createProvider(uri, conf);
                    try {
                        kp.getMetadata("k1");
                        kp.getKeysMetadata("k1");
                    } catch (Exception ex) {
                        Assert.fail(ex.getMessage());
                    }
                    return null;
                }
            });
            //stop the reloader, to avoid running while we are writing the new file
            KMSWebApp.getACLs().stopReloader();
            // test ACL reloading
            // to ensure the ACLs file modifiedTime is newer
            Thread.sleep(10);
            conf.set(KMSACLs.Type.CREATE.getAclConfigKey(), "foo");
            conf.set(KMSACLs.Type.GENERATE_EEK.getAclConfigKey(), "foo");
            writeConf(testDir, conf);
            Thread.sleep(1000);
            // forcing a reload by hand.
            KMSWebApp.getACLs().run();
            // should not be able to create a key now
            doAs("CREATE", new PrivilegedExceptionAction<Void>() {

                @Override
                public Void run() throws Exception {
                    try {
                        KeyProvider kp = createProvider(uri, conf);
                        KeyProvider.KeyVersion kv = kp.createKey("k2", new KeyProvider.Options(conf));
                        Assert.fail();
                    } catch (AuthorizationException ex) {
                    //NOP
                    } catch (Exception ex) {
                        Assert.fail(ex.getMessage());
                    }
                    return null;
                }
            });
            doAs("GENERATE_EEK", new PrivilegedExceptionAction<Void>() {

                @Override
                public Void run() throws Exception {
                    KeyProvider kp = createProvider(uri, conf);
                    try {
                        KeyProviderCryptoExtension kpCE = KeyProviderCryptoExtension.createKeyProviderCryptoExtension(kp);
                        kpCE.generateEncryptedKey("k1");
                    } catch (IOException ex) {
                        // through the ValueQueue. See KMSCP#generateEncryptedKey.
                        if (ex.getCause().getCause() instanceof AuthorizationException) {
                            LOG.info("Caught expected exception.", ex);
                        } else {
                            throw ex;
                        }
                    }
                    return null;
                }
            });
            doAs("GENERATE_EEK", new PrivilegedExceptionAction<Void>() {

                @Override
                public Void run() throws Exception {
                    KeyProvider kp = createProvider(uri, conf);
                    try {
                        KeyProviderCryptoExtension kpCE = KeyProviderCryptoExtension.createKeyProviderCryptoExtension(kp);
                        kpCE.reencryptEncryptedKey(encKv);
                    } catch (AuthorizationException ex) {
                        LOG.info("Caught expected exception.", ex);
                    }
                    return null;
                }
            });
            return null;
        }
    });
}

Example 30 with KeyProvider

use of org.apache.hadoop.crypto.key.KeyProvider in project hadoop by apache.

the class TestKMS method doKMSRestart.

public void doKMSRestart(boolean useKrb) throws Exception {
    Configuration conf = new Configuration();
    conf.set("hadoop.security.authentication", "kerberos");
    final File testDir = getTestDir();
    conf = createBaseKMSConf(testDir, conf);
    if (useKrb) {
        conf.set("hadoop.kms.authentication.type", "kerberos");
    }
    conf.set("hadoop.kms.authentication.kerberos.keytab", keytab.getAbsolutePath());
    conf.set("hadoop.kms.authentication.kerberos.principal", "HTTP/localhost");
    conf.set("hadoop.kms.authentication.kerberos.name.rules", "DEFAULT");
    for (KMSACLs.Type type : KMSACLs.Type.values()) {
        conf.set(type.getAclConfigKey(), type.toString());
    }
    conf.set(KMSACLs.Type.CREATE.getAclConfigKey(), KMSACLs.Type.CREATE.toString() + ",SET_KEY_MATERIAL");
    conf.set(KMSACLs.Type.ROLLOVER.getAclConfigKey(), KMSACLs.Type.ROLLOVER.toString() + ",SET_KEY_MATERIAL");
    conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k0.ALL", "*");
    conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k1.ALL", "*");
    conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k2.ALL", "*");
    conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k3.ALL", "*");
    writeConf(testDir, conf);
    KMSCallable<KeyProvider> c = new KMSCallable<KeyProvider>() {

        @Override
        public KeyProvider call() throws Exception {
            final Configuration conf = new Configuration();
            conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 128);
            final URI uri = createKMSUri(getKMSUrl());
            final KeyProvider kp = doAs("SET_KEY_MATERIAL", new PrivilegedExceptionAction<KeyProvider>() {

                @Override
                public KeyProvider run() throws Exception {
                    KeyProvider kp = createProvider(uri, conf);
                    kp.createKey("k1", new byte[16], new KeyProvider.Options(conf));
                    return kp;
                }
            });
            return kp;
        }
    };
    final KeyProvider retKp = runServer(null, null, testDir, c);
    // Restart server (using the same port)
    runServer(c.getKMSUrl().getPort(), null, null, testDir, new KMSCallable<Void>() {

        @Override
        public Void call() throws Exception {
            final Configuration conf = new Configuration();
            conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 128);
            doAs("SET_KEY_MATERIAL", new PrivilegedExceptionAction<Void>() {

                @Override
                public Void run() throws Exception {
                    retKp.createKey("k2", new byte[16], new KeyProvider.Options(conf));
                    retKp.createKey("k3", new byte[16], new KeyProvider.Options(conf));
                    return null;
                }
            });
            return null;
        }
    });
}