Examples with DFSClient org.apache.hadoop.hdfs.DFSClient used on opensource projects

Example 1 with DFSClient

use of org.apache.hadoop.hdfs.DFSClient in project hbase by apache.

the class FanOutOneBlockAsyncDFSOutputSaslHelper method createTransparentCryptoHelper.

private static TransparentCryptoHelper createTransparentCryptoHelper() throws NoSuchMethodException {
    Method decryptEncryptedDataEncryptionKeyMethod = DFSClient.class.getDeclaredMethod("decryptEncryptedDataEncryptionKey", FileEncryptionInfo.class);
    decryptEncryptedDataEncryptionKeyMethod.setAccessible(true);
    return new TransparentCryptoHelper() {

        @Override
        public Encryptor createEncryptor(Configuration conf, FileEncryptionInfo feInfo, DFSClient client) throws IOException {
            try {
                KeyVersion decryptedKey = (KeyVersion) decryptEncryptedDataEncryptionKeyMethod.invoke(client, feInfo);
                CryptoCodec cryptoCodec = CryptoCodec.getInstance(conf, feInfo.getCipherSuite());
                Encryptor encryptor = cryptoCodec.createEncryptor();
                encryptor.init(decryptedKey.getMaterial(), feInfo.getIV());
                return encryptor;
            } catch (InvocationTargetException e) {
                Throwables.propagateIfPossible(e.getTargetException(), IOException.class);
                throw new RuntimeException(e.getTargetException());
            } catch (GeneralSecurityException e) {
                throw new IOException(e);
            } catch (IllegalAccessException e) {
                throw new RuntimeException(e);
            }
        }
    };
}

Example 2 with DFSClient

use of org.apache.hadoop.hdfs.DFSClient in project hadoop by apache.

the class TestBlockReplacement method testDeletedBlockWhenAddBlockIsInEdit.

/**
   * Standby namenode doesn't queue Delete block request when the add block
   * request is in the edit log which are yet to be read.
   * @throws Exception
   */
@Test
public void testDeletedBlockWhenAddBlockIsInEdit() throws Exception {
    Configuration conf = new HdfsConfiguration();
    cluster = new MiniDFSCluster.Builder(conf).nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(1).build();
    DFSClient client = null;
    try {
        cluster.waitActive();
        assertEquals("Number of namenodes is not 2", 2, cluster.getNumNameNodes());
        // Transitioning the namenode 0 to active.
        cluster.transitionToActive(0);
        assertTrue("Namenode 0 should be in active state", cluster.getNameNode(0).isActiveState());
        assertTrue("Namenode 1 should be in standby state", cluster.getNameNode(1).isStandbyState());
        // Trigger heartbeat to mark DatanodeStorageInfo#heartbeatedSinceFailover
        // to true.
        DataNodeTestUtils.triggerHeartbeat(cluster.getDataNodes().get(0));
        FileSystem fs = cluster.getFileSystem(0);
        // Trigger blockReport to mark DatanodeStorageInfo#blockContentsStale
        // to false.
        cluster.getDataNodes().get(0).triggerBlockReport(new BlockReportOptions.Factory().setIncremental(false).build());
        Path fileName = new Path("/tmp.txt");
        // create a file with one block
        DFSTestUtil.createFile(fs, fileName, 10L, (short) 1, 1234L);
        DFSTestUtil.waitReplication(fs, fileName, (short) 1);
        client = new DFSClient(cluster.getFileSystem(0).getUri(), conf);
        List<LocatedBlock> locatedBlocks = client.getNamenode().getBlockLocations("/tmp.txt", 0, 10L).getLocatedBlocks();
        assertTrue(locatedBlocks.size() == 1);
        assertTrue(locatedBlocks.get(0).getLocations().length == 1);
        // add a second datanode to the cluster
        cluster.startDataNodes(conf, 1, true, null, null, null, null);
        assertEquals("Number of datanodes should be 2", 2, cluster.getDataNodes().size());
        DataNode dn0 = cluster.getDataNodes().get(0);
        DataNode dn1 = cluster.getDataNodes().get(1);
        String activeNNBPId = cluster.getNamesystem(0).getBlockPoolId();
        DatanodeDescriptor sourceDnDesc = NameNodeAdapter.getDatanode(cluster.getNamesystem(0), dn0.getDNRegistrationForBP(activeNNBPId));
        DatanodeDescriptor destDnDesc = NameNodeAdapter.getDatanode(cluster.getNamesystem(0), dn1.getDNRegistrationForBP(activeNNBPId));
        ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, fileName);
        LOG.info("replaceBlock:  " + replaceBlock(block, (DatanodeInfo) sourceDnDesc, (DatanodeInfo) sourceDnDesc, (DatanodeInfo) destDnDesc));
        // Waiting for the FsDatasetAsyncDsikService to delete the block
        for (int tries = 0; tries < 20; tries++) {
            Thread.sleep(1000);
            // Triggering the deletion block report to report the deleted block
            // to namnemode
            DataNodeTestUtils.triggerDeletionReport(cluster.getDataNodes().get(0));
            locatedBlocks = client.getNamenode().getBlockLocations("/tmp.txt", 0, 10L).getLocatedBlocks();
            // If block was deleted and only on 1 datanode then break out
            if (locatedBlocks.get(0).getLocations().length == 1) {
                break;
            }
        }
        cluster.transitionToStandby(0);
        cluster.transitionToActive(1);
        assertTrue("Namenode 1 should be in active state", cluster.getNameNode(1).isActiveState());
        assertTrue("Namenode 0 should be in standby state", cluster.getNameNode(0).isStandbyState());
        client.close();
        // Opening a new client for new active  namenode
        client = new DFSClient(cluster.getFileSystem(1).getUri(), conf);
        List<LocatedBlock> locatedBlocks1 = client.getNamenode().getBlockLocations("/tmp.txt", 0, 10L).getLocatedBlocks();
        assertEquals(1, locatedBlocks1.size());
        assertEquals("The block should be only on 1 datanode ", 1, locatedBlocks1.get(0).getLocations().length);
    } finally {
        IOUtils.cleanup(null, client);
        cluster.shutdown();
    }
}

Example 3 with DFSClient

use of org.apache.hadoop.hdfs.DFSClient in project hadoop by apache.

the class TestRetryCacheWithHA method testDeleteSnapshot.

@Test(timeout = 60000)
public void testDeleteSnapshot() throws Exception {
    final DFSClient client = genClientWithDummyHandler();
    AtMostOnceOp op = new DeleteSnapshotOp(client, "/test", "s1");
    testClientRetryWithFailover(op);
}

Example 4 with DFSClient

use of org.apache.hadoop.hdfs.DFSClient in project hadoop by apache.

the class TestRetryCacheWithHA method testAddCachePool.

@Test(timeout = 60000)
public void testAddCachePool() throws Exception {
    DFSClient client = genClientWithDummyHandler();
    AtMostOnceOp op = new AddCachePoolOp(client, "pool");
    testClientRetryWithFailover(op);
}

Example 5 with DFSClient

use of org.apache.hadoop.hdfs.DFSClient in project hadoop by apache.

the class TestRetryCacheWithHA method testModifyCachePool.

@Test(timeout = 60000)
public void testModifyCachePool() throws Exception {
    DFSClient client = genClientWithDummyHandler();
    AtMostOnceOp op = new ModifyCachePoolOp(client, "pool");
    testClientRetryWithFailover(op);
}