Examples with HdfsAdmin org.apache.hadoop.hdfs.client.HdfsAdmin used on opensource projects

Example 6 with HdfsAdmin

use of org.apache.hadoop.hdfs.client.HdfsAdmin in project hadoop by apache.

the class TestEncryptionZones method testRootDirEZTrash.

@Test
public void testRootDirEZTrash() throws Exception {
    final HdfsAdmin dfsAdmin = new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
    final String currentUser = UserGroupInformation.getCurrentUser().getShortUserName();
    final Path rootDir = new Path("/");
    dfsAdmin.createEncryptionZone(rootDir, TEST_KEY, NO_TRASH);
    final Path encFile = new Path("/encFile");
    final int len = 8192;
    DFSTestUtil.createFile(fs, encFile, len, (short) 1, 0xFEED);
    Configuration clientConf = new Configuration(conf);
    clientConf.setLong(FS_TRASH_INTERVAL_KEY, 1);
    FsShell shell = new FsShell(clientConf);
    verifyShellDeleteWithTrash(shell, encFile);
    // Trash path should be consistent
    // if root path is an encryption zone
    Path encFileCurrentTrash = shell.getCurrentTrashDir(encFile);
    Path rootDirCurrentTrash = shell.getCurrentTrashDir(rootDir);
    assertEquals("Root trash should be equal with ezFile trash", encFileCurrentTrash, rootDirCurrentTrash);
    // Use webHDFS client to test trash root path
    final WebHdfsFileSystem webFS = WebHdfsTestUtil.getWebHdfsFileSystem(conf, WebHdfsConstants.WEBHDFS_SCHEME);
    final Path expectedTrash = new Path(rootDir, new Path(FileSystem.TRASH_PREFIX, currentUser));
    Path webHDFSTrash = webFS.getTrashRoot(encFile);
    assertEquals(expectedTrash.toUri().getPath(), webHDFSTrash.toUri().getPath());
    assertEquals(encFileCurrentTrash.getParent().toUri().getPath(), webHDFSTrash.toUri().getPath());
}

Example 7 with HdfsAdmin

use of org.apache.hadoop.hdfs.client.HdfsAdmin in project hadoop by apache.

the class TestEncryptionZones method testBasicOperations.

// CHECKSTYLE:OFF:MethodLengthCheck
@Test
public void testBasicOperations() throws Exception {
    int numZones = 0;
    /* Number of EZs should be 0 if no EZ is created */
    assertEquals("Unexpected number of encryption zones!", numZones, cluster.getNamesystem().getNumEncryptionZones());
    /* Test failure of create EZ on a directory that doesn't exist. */
    final Path zoneParent = new Path("/zones");
    final Path zone1 = new Path(zoneParent, "zone1");
    try {
        dfsAdmin.createEncryptionZone(zone1, TEST_KEY, NO_TRASH);
        fail("expected /test doesn't exist");
    } catch (IOException e) {
        assertExceptionContains("cannot find", e);
    }
    /* Normal creation of an EZ */
    fsWrapper.mkdir(zone1, FsPermission.getDirDefault(), true);
    dfsAdmin.createEncryptionZone(zone1, TEST_KEY, NO_TRASH);
    assertNumZones(++numZones);
    assertZonePresent(null, zone1.toString());
    /* Test failure of create EZ on a directory which is already an EZ. */
    try {
        dfsAdmin.createEncryptionZone(zone1, TEST_KEY, NO_TRASH);
    } catch (IOException e) {
        assertExceptionContains("is already an encryption zone", e);
    }
    /* create EZ on parent of an EZ should fail */
    try {
        dfsAdmin.createEncryptionZone(zoneParent, TEST_KEY, NO_TRASH);
        fail("EZ over an EZ");
    } catch (IOException e) {
        assertExceptionContains("encryption zone for a non-empty directory", e);
    }
    /* create EZ on a folder with a folder fails */
    final Path notEmpty = new Path("/notEmpty");
    final Path notEmptyChild = new Path(notEmpty, "child");
    fsWrapper.mkdir(notEmptyChild, FsPermission.getDirDefault(), true);
    try {
        dfsAdmin.createEncryptionZone(notEmpty, TEST_KEY, NO_TRASH);
        fail("Created EZ on an non-empty directory with folder");
    } catch (IOException e) {
        assertExceptionContains("create an encryption zone", e);
    }
    fsWrapper.delete(notEmptyChild, false);
    /* create EZ on a folder with a file fails */
    fsWrapper.createFile(notEmptyChild);
    try {
        dfsAdmin.createEncryptionZone(notEmpty, TEST_KEY, NO_TRASH);
        fail("Created EZ on an non-empty directory with file");
    } catch (IOException e) {
        assertExceptionContains("create an encryption zone", e);
    }
    /* Test failure of create EZ on a file. */
    try {
        dfsAdmin.createEncryptionZone(notEmptyChild, TEST_KEY, NO_TRASH);
        fail("Created EZ on a file");
    } catch (IOException e) {
        assertExceptionContains("create an encryption zone for a file.", e);
    }
    /* Test failure of creating an EZ passing a key that doesn't exist. */
    final Path zone2 = new Path("/zone2");
    fsWrapper.mkdir(zone2, FsPermission.getDirDefault(), false);
    final String myKeyName = "mykeyname";
    try {
        dfsAdmin.createEncryptionZone(zone2, myKeyName, NO_TRASH);
        fail("expected key doesn't exist");
    } catch (IOException e) {
        assertExceptionContains("doesn't exist.", e);
    }
    /* Test failure of empty and null key name */
    try {
        dfsAdmin.createEncryptionZone(zone2, "", NO_TRASH);
        fail("created a zone with empty key name");
    } catch (IOException e) {
        assertExceptionContains("Must specify a key name when creating", e);
    }
    try {
        dfsAdmin.createEncryptionZone(zone2, null, NO_TRASH);
        fail("created a zone with null key name");
    } catch (IOException e) {
        assertExceptionContains("Must specify a key name when creating", e);
    }
    assertNumZones(1);
    /* Test success of creating an EZ when they key exists. */
    DFSTestUtil.createKey(myKeyName, cluster, conf);
    dfsAdmin.createEncryptionZone(zone2, myKeyName, NO_TRASH);
    assertNumZones(++numZones);
    assertZonePresent(myKeyName, zone2.toString());
    /* Test failure of create encryption zones as a non super user. */
    final UserGroupInformation user = UserGroupInformation.createUserForTesting("user", new String[] { "mygroup" });
    final Path nonSuper = new Path("/nonSuper");
    fsWrapper.mkdir(nonSuper, FsPermission.getDirDefault(), false);
    user.doAs(new PrivilegedExceptionAction<Object>() {

        @Override
        public Object run() throws Exception {
            final HdfsAdmin userAdmin = new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
            try {
                userAdmin.createEncryptionZone(nonSuper, TEST_KEY, NO_TRASH);
                fail("createEncryptionZone is superuser-only operation");
            } catch (AccessControlException e) {
                assertExceptionContains("Superuser privilege is required", e);
            }
            return null;
        }
    });
    // Test success of creating an encryption zone a few levels down.
    Path deepZone = new Path("/d/e/e/p/zone");
    fsWrapper.mkdir(deepZone, FsPermission.getDirDefault(), true);
    dfsAdmin.createEncryptionZone(deepZone, TEST_KEY, NO_TRASH);
    assertNumZones(++numZones);
    assertZonePresent(null, deepZone.toString());
    // Create and list some zones to test batching of listEZ
    for (int i = 1; i < 6; i++) {
        final Path zonePath = new Path("/listZone" + i);
        fsWrapper.mkdir(zonePath, FsPermission.getDirDefault(), false);
        dfsAdmin.createEncryptionZone(zonePath, TEST_KEY, NO_TRASH);
        numZones++;
        assertNumZones(numZones);
        assertZonePresent(null, zonePath.toString());
    }
    fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    fs.saveNamespace();
    fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
    cluster.restartNameNode(true);
    assertNumZones(numZones);
    assertEquals("Unexpected number of encryption zones!", numZones, cluster.getNamesystem().getNumEncryptionZones());
    assertGauge("NumEncryptionZones", numZones, getMetrics(NS_METRICS));
    assertZonePresent(null, zone1.toString());
    // Verify newly added ez is present after restarting the NameNode
    // without persisting the namespace.
    Path nonpersistZone = new Path("/nonpersistZone");
    fsWrapper.mkdir(nonpersistZone, FsPermission.getDirDefault(), false);
    dfsAdmin.createEncryptionZone(nonpersistZone, TEST_KEY, NO_TRASH);
    numZones++;
    cluster.restartNameNode(true);
    assertNumZones(numZones);
    assertZonePresent(null, nonpersistZone.toString());
}

Example 8 with HdfsAdmin

use of org.apache.hadoop.hdfs.client.HdfsAdmin in project hadoop by apache.

the class TestEncryptionZones method setup.

@Before
public void setup() throws Exception {
    conf = new HdfsConfiguration();
    fsHelper = new FileSystemTestHelper();
    // Set up java key store
    String testRoot = fsHelper.getTestRootDir();
    testRootDir = new File(testRoot).getAbsoluteFile();
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, getKeyProviderURI());
    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
    // Lower the batch size for testing
    conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES, 2);
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    Logger.getLogger(EncryptionZoneManager.class).setLevel(Level.TRACE);
    fs = cluster.getFileSystem();
    fsWrapper = new FileSystemTestWrapper(fs);
    fcWrapper = new FileContextTestWrapper(FileContext.getFileContext(cluster.getURI(), conf));
    dfsAdmin = new HdfsAdmin(cluster.getURI(), conf);
    setProvider();
    // Create a test key
    DFSTestUtil.createKey(TEST_KEY, cluster, conf);
}

Example 9 with HdfsAdmin

use of org.apache.hadoop.hdfs.client.HdfsAdmin in project hadoop by apache.

the class TestErasureCodingPolicies method testGetErasureCodingPolicyOnANonExistentFile.

@Test
public void testGetErasureCodingPolicyOnANonExistentFile() throws Exception {
    Path path = new Path("/ecDir");
    try {
        fs.getErasureCodingPolicy(path);
        fail("FileNotFoundException should be thrown for a non-existent" + " file path");
    } catch (FileNotFoundException e) {
        assertExceptionContains("Path not found: " + path, e);
    }
    HdfsAdmin dfsAdmin = new HdfsAdmin(cluster.getURI(), conf);
    try {
        dfsAdmin.getErasureCodingPolicy(path);
        fail("FileNotFoundException should be thrown for a non-existent" + " file path");
    } catch (FileNotFoundException e) {
        assertExceptionContains("Path not found: " + path, e);
    }
}

Example 10 with HdfsAdmin

use of org.apache.hadoop.hdfs.client.HdfsAdmin in project hadoop by apache.

the class TestNameNodeMetrics method testGenerateEDEKTime.

@Test
public void testGenerateEDEKTime() throws IOException, NoSuchAlgorithmException {
    //Create new MiniDFSCluster with EncryptionZone configurations
    Configuration conf = new HdfsConfiguration();
    FileSystemTestHelper fsHelper = new FileSystemTestHelper();
    // Set up java key store
    String testRoot = fsHelper.getTestRootDir();
    File testRootDir = new File(testRoot).getAbsoluteFile();
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, JavaKeyStoreProvider.SCHEME_NAME + "://file" + new Path(testRootDir.toString(), "test.jks").toUri());
    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
    conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES, 2);
    try (MiniDFSCluster clusterEDEK = new MiniDFSCluster.Builder(conf).numDataNodes(1).build()) {
        DistributedFileSystem fsEDEK = clusterEDEK.getFileSystem();
        FileSystemTestWrapper fsWrapper = new FileSystemTestWrapper(fsEDEK);
        HdfsAdmin dfsAdmin = new HdfsAdmin(clusterEDEK.getURI(), conf);
        fsEDEK.getClient().setKeyProvider(clusterEDEK.getNameNode().getNamesystem().getProvider());
        String testKey = "test_key";
        DFSTestUtil.createKey(testKey, clusterEDEK, conf);
        final Path zoneParent = new Path("/zones");
        final Path zone1 = new Path(zoneParent, "zone1");
        fsWrapper.mkdir(zone1, FsPermission.getDirDefault(), true);
        dfsAdmin.createEncryptionZone(zone1, "test_key", EnumSet.of(CreateEncryptionZoneFlag.NO_TRASH));
        MetricsRecordBuilder rb = getMetrics(NN_METRICS);
        for (int i = 0; i < 3; i++) {
            Path filePath = new Path("/zones/zone1/testfile-" + i);
            DFSTestUtil.createFile(fsEDEK, filePath, 1024, (short) 3, 1L);
            assertQuantileGauges("GenerateEDEKTime1s", rb);
        }
    }
}