Examples with Encryptor org.apache.hadoop.crypto.Encryptor used on opensource projects

Example 1 with Encryptor

use of org.apache.hadoop.crypto.Encryptor in project hbase by apache.

the class FanOutOneBlockAsyncDFSOutputSaslHelper method createTransparentCryptoHelper.

private static TransparentCryptoHelper createTransparentCryptoHelper() throws NoSuchMethodException {
    Method decryptEncryptedDataEncryptionKeyMethod = DFSClient.class.getDeclaredMethod("decryptEncryptedDataEncryptionKey", FileEncryptionInfo.class);
    decryptEncryptedDataEncryptionKeyMethod.setAccessible(true);
    return new TransparentCryptoHelper() {

        @Override
        public Encryptor createEncryptor(Configuration conf, FileEncryptionInfo feInfo, DFSClient client) throws IOException {
            try {
                KeyVersion decryptedKey = (KeyVersion) decryptEncryptedDataEncryptionKeyMethod.invoke(client, feInfo);
                CryptoCodec cryptoCodec = CryptoCodec.getInstance(conf, feInfo.getCipherSuite());
                Encryptor encryptor = cryptoCodec.createEncryptor();
                encryptor.init(decryptedKey.getMaterial(), feInfo.getIV());
                return encryptor;
            } catch (InvocationTargetException e) {
                Throwables.propagateIfPossible(e.getTargetException(), IOException.class);
                throw new RuntimeException(e.getTargetException());
            } catch (GeneralSecurityException e) {
                throw new IOException(e);
            } catch (IllegalAccessException e) {
                throw new RuntimeException(e);
            }
        }
    };
}

Example 2 with Encryptor

use of org.apache.hadoop.crypto.Encryptor in project hbase by apache.

the class FanOutOneBlockAsyncDFSOutputHelper method createOutput.

private static FanOutOneBlockAsyncDFSOutput createOutput(DistributedFileSystem dfs, String src, boolean overwrite, boolean createParent, short replication, long blockSize, EventLoopGroup eventLoopGroup, Class<? extends Channel> channelClass, StreamSlowMonitor monitor) throws IOException {
    Configuration conf = dfs.getConf();
    DFSClient client = dfs.getClient();
    String clientName = client.getClientName();
    ClientProtocol namenode = client.getNamenode();
    int createMaxRetries = conf.getInt(ASYNC_DFS_OUTPUT_CREATE_MAX_RETRIES, DEFAULT_ASYNC_DFS_OUTPUT_CREATE_MAX_RETRIES);
    ExcludeDatanodeManager excludeDatanodeManager = monitor.getExcludeDatanodeManager();
    Set<DatanodeInfo> toExcludeNodes = new HashSet<>(excludeDatanodeManager.getExcludeDNs().keySet());
    for (int retry = 0; ; retry++) {
        LOG.debug("When create output stream for {}, exclude list is {}, retry={}", src, toExcludeNodes, retry);
        HdfsFileStatus stat;
        try {
            stat = FILE_CREATOR.create(namenode, src, FsPermission.getFileDefault().applyUMask(FsPermission.getUMask(conf)), clientName, getCreateFlags(overwrite), createParent, replication, blockSize, CryptoProtocolVersion.supported());
        } catch (Exception e) {
            if (e instanceof RemoteException) {
                throw (RemoteException) e;
            } else {
                throw new NameNodeException(e);
            }
        }
        beginFileLease(client, stat.getFileId());
        boolean succ = false;
        LocatedBlock locatedBlock = null;
        List<Future<Channel>> futureList = null;
        try {
            DataChecksum summer = createChecksum(client);
            locatedBlock = namenode.addBlock(src, client.getClientName(), null, toExcludeNodes.toArray(new DatanodeInfo[0]), stat.getFileId(), null, null);
            Map<Channel, DatanodeInfo> datanodes = new IdentityHashMap<>();
            futureList = connectToDataNodes(conf, client, clientName, locatedBlock, 0L, 0L, PIPELINE_SETUP_CREATE, summer, eventLoopGroup, channelClass);
            for (int i = 0, n = futureList.size(); i < n; i++) {
                DatanodeInfo datanodeInfo = locatedBlock.getLocations()[i];
                try {
                    datanodes.put(futureList.get(i).syncUninterruptibly().getNow(), datanodeInfo);
                } catch (Exception e) {
                    // exclude the broken DN next time
                    toExcludeNodes.add(datanodeInfo);
                    excludeDatanodeManager.tryAddExcludeDN(datanodeInfo, "connect error");
                    throw e;
                }
            }
            Encryptor encryptor = createEncryptor(conf, stat, client);
            FanOutOneBlockAsyncDFSOutput output = new FanOutOneBlockAsyncDFSOutput(conf, dfs, client, namenode, clientName, src, stat.getFileId(), locatedBlock, encryptor, datanodes, summer, ALLOC, monitor);
            succ = true;
            return output;
        } catch (RemoteException e) {
            LOG.warn("create fan-out dfs output {} failed, retry = {}", src, retry, e);
            if (shouldRetryCreate(e)) {
                if (retry >= createMaxRetries) {
                    throw e.unwrapRemoteException();
                }
            } else {
                throw e.unwrapRemoteException();
            }
        } catch (IOException e) {
            LOG.warn("create fan-out dfs output {} failed, retry = {}", src, retry, e);
            if (retry >= createMaxRetries) {
                throw e;
            }
            // overwrite the old broken file.
            overwrite = true;
            try {
                Thread.sleep(ConnectionUtils.getPauseTime(100, retry));
            } catch (InterruptedException ie) {
                throw new InterruptedIOException();
            }
        } finally {
            if (!succ) {
                if (futureList != null) {
                    for (Future<Channel> f : futureList) {
                        f.addListener(new FutureListener<Channel>() {

                            @Override
                            public void operationComplete(Future<Channel> future) throws Exception {
                                if (future.isSuccess()) {
                                    future.getNow().close();
                                }
                            }
                        });
                    }
                }
                endFileLease(client, stat.getFileId());
            }
        }
    }
}

Example 3 with Encryptor

use of org.apache.hadoop.crypto.Encryptor in project hbase by apache.

the class FanOutOneBlockAsyncDFSOutputHelper method createOutput.

private static FanOutOneBlockAsyncDFSOutput createOutput(DistributedFileSystem dfs, String src, boolean overwrite, boolean createParent, short replication, long blockSize, EventLoop eventLoop) throws IOException {
    Configuration conf = dfs.getConf();
    FSUtils fsUtils = FSUtils.getInstance(dfs, conf);
    DFSClient client = dfs.getClient();
    String clientName = client.getClientName();
    ClientProtocol namenode = client.getNamenode();
    HdfsFileStatus stat;
    try {
        stat = namenode.create(src, FsPermission.getFileDefault().applyUMask(FsPermission.getUMask(conf)), clientName, new EnumSetWritable<>(overwrite ? EnumSet.of(CREATE, OVERWRITE) : EnumSet.of(CREATE)), createParent, replication, blockSize, CryptoProtocolVersion.supported());
    } catch (Exception e) {
        if (e instanceof RemoteException) {
            throw (RemoteException) e;
        } else {
            throw new NameNodeException(e);
        }
    }
    beginFileLease(client, stat.getFileId());
    boolean succ = false;
    LocatedBlock locatedBlock = null;
    List<Future<Channel>> futureList = null;
    try {
        DataChecksum summer = createChecksum(client);
        locatedBlock = BLOCK_ADDER.addBlock(namenode, src, client.getClientName(), null, null, stat.getFileId(), null);
        List<Channel> datanodeList = new ArrayList<>();
        futureList = connectToDataNodes(conf, client, clientName, locatedBlock, 0L, 0L, PIPELINE_SETUP_CREATE, summer, eventLoop);
        for (Future<Channel> future : futureList) {
            // fail the creation if there are connection failures since we are fail-fast. The upper
            // layer should retry itself if needed.
            datanodeList.add(future.syncUninterruptibly().getNow());
        }
        Encryptor encryptor = createEncryptor(conf, stat, client);
        FanOutOneBlockAsyncDFSOutput output = new FanOutOneBlockAsyncDFSOutput(conf, fsUtils, dfs, client, namenode, clientName, src, stat.getFileId(), locatedBlock, encryptor, eventLoop, datanodeList, summer, ALLOC);
        succ = true;
        return output;
    } finally {
        if (!succ) {
            if (futureList != null) {
                for (Future<Channel> f : futureList) {
                    f.addListener(new FutureListener<Channel>() {

                        @Override
                        public void operationComplete(Future<Channel> future) throws Exception {
                            if (future.isSuccess()) {
                                future.getNow().close();
                            }
                        }
                    });
                }
            }
            endFileLease(client, stat.getFileId());
            fsUtils.recoverFileLease(dfs, new Path(src), conf, new CancelOnClose(client));
        }
    }
}

Example 4 with Encryptor

use of org.apache.hadoop.crypto.Encryptor in project hbase by apache.

the class FanOutOneBlockAsyncDFSOutputSaslHelper method createTransparentCryptoHelperWithHDFS12396.

private static TransparentCryptoHelper createTransparentCryptoHelperWithHDFS12396() throws ClassNotFoundException, NoSuchMethodException {
    Class<?> hdfsKMSUtilCls = Class.forName("org.apache.hadoop.hdfs.HdfsKMSUtil");
    Method decryptEncryptedDataEncryptionKeyMethod = hdfsKMSUtilCls.getDeclaredMethod("decryptEncryptedDataEncryptionKey", FileEncryptionInfo.class, KeyProvider.class);
    decryptEncryptedDataEncryptionKeyMethod.setAccessible(true);
    return new TransparentCryptoHelper() {

        @Override
        public Encryptor createEncryptor(Configuration conf, FileEncryptionInfo feInfo, DFSClient client) throws IOException {
            try {
                KeyVersion decryptedKey = (KeyVersion) decryptEncryptedDataEncryptionKeyMethod.invoke(null, feInfo, client.getKeyProvider());
                CryptoCodec cryptoCodec = CryptoCodec.getInstance(conf, feInfo.getCipherSuite());
                Encryptor encryptor = cryptoCodec.createEncryptor();
                encryptor.init(decryptedKey.getMaterial(), feInfo.getIV());
                return encryptor;
            } catch (InvocationTargetException e) {
                Throwables.propagateIfPossible(e.getTargetException(), IOException.class);
                throw new RuntimeException(e.getTargetException());
            } catch (GeneralSecurityException e) {
                throw new IOException(e);
            } catch (IllegalAccessException e) {
                throw new RuntimeException(e);
            }
        }
    };
}

Example 5 with Encryptor

use of org.apache.hadoop.crypto.Encryptor in project hbase by apache.

the class FanOutOneBlockAsyncDFSOutputSaslHelper method createTransparentCryptoHelperWithoutHDFS12396.

private static TransparentCryptoHelper createTransparentCryptoHelperWithoutHDFS12396() throws NoSuchMethodException {
    Method decryptEncryptedDataEncryptionKeyMethod = DFSClient.class.getDeclaredMethod("decryptEncryptedDataEncryptionKey", FileEncryptionInfo.class);
    decryptEncryptedDataEncryptionKeyMethod.setAccessible(true);
    return new TransparentCryptoHelper() {

        @Override
        public Encryptor createEncryptor(Configuration conf, FileEncryptionInfo feInfo, DFSClient client) throws IOException {
            try {
                KeyVersion decryptedKey = (KeyVersion) decryptEncryptedDataEncryptionKeyMethod.invoke(client, feInfo);
                CryptoCodec cryptoCodec = CryptoCodec.getInstance(conf, feInfo.getCipherSuite());
                Encryptor encryptor = cryptoCodec.createEncryptor();
                encryptor.init(decryptedKey.getMaterial(), feInfo.getIV());
                return encryptor;
            } catch (InvocationTargetException e) {
                Throwables.propagateIfPossible(e.getTargetException(), IOException.class);
                throw new RuntimeException(e.getTargetException());
            } catch (GeneralSecurityException e) {
                throw new IOException(e);
            } catch (IllegalAccessException e) {
                throw new RuntimeException(e);
            }
        }
    };
}