Examples with MiniDFSCluster org.apache.hadoop.hdfs.MiniDFSCluster used on opensource projects

Example 31 with MiniDFSCluster

use of org.apache.hadoop.hdfs.MiniDFSCluster in project hadoop by apache.

the class TestOfflineImageViewer method createOriginalFSImage.

// Create a populated namespace for later testing. Save its contents to a
// data structure and store its fsimage location.
// We only want to generate the fsimage file once and use it for
// multiple tests.
@BeforeClass
public static void createOriginalFSImage() throws IOException {
    tempDir = Files.createTempDir();
    MiniDFSCluster cluster = null;
    try {
        final ErasureCodingPolicy ecPolicy = ErasureCodingPolicyManager.getPolicyByID(HdfsConstants.XOR_2_1_POLICY_ID);
        Configuration conf = new Configuration();
        conf.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, 10000);
        conf.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, 5000);
        conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
        conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
        conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL, "RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//" + "DEFAULT");
        conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, ecPolicy.getName());
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
        cluster.waitActive();
        DistributedFileSystem hdfs = cluster.getFileSystem();
        // Create a reasonable namespace
        for (int i = 0; i < NUM_DIRS; i++, dirCount++) {
            Path dir = new Path("/dir" + i);
            hdfs.mkdirs(dir);
            writtenFiles.put(dir.toString(), pathToFileEntry(hdfs, dir.toString()));
            for (int j = 0; j < FILES_PER_DIR; j++) {
                Path file = new Path(dir, "file" + j);
                FSDataOutputStream o = hdfs.create(file);
                o.write(23);
                o.close();
                writtenFiles.put(file.toString(), pathToFileEntry(hdfs, file.toString()));
            }
        }
        // Create an empty directory
        Path emptydir = new Path("/emptydir");
        hdfs.mkdirs(emptydir);
        dirCount++;
        writtenFiles.put(emptydir.toString(), hdfs.getFileStatus(emptydir));
        //Create a directory whose name should be escaped in XML
        Path invalidXMLDir = new Path("/dirContainingInvalidXMLCharhere");
        hdfs.mkdirs(invalidXMLDir);
        dirCount++;
        //Create a directory with sticky bits
        Path stickyBitDir = new Path("/stickyBit");
        hdfs.mkdirs(stickyBitDir);
        hdfs.setPermission(stickyBitDir, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL, true));
        dirCount++;
        writtenFiles.put(stickyBitDir.toString(), hdfs.getFileStatus(stickyBitDir));
        // Get delegation tokens so we log the delegation token op
        Token<?>[] delegationTokens = hdfs.addDelegationTokens(TEST_RENEWER, null);
        for (Token<?> t : delegationTokens) {
            LOG.debug("got token " + t);
        }
        // Create INodeReference
        final Path src = new Path("/src");
        hdfs.mkdirs(src);
        dirCount++;
        writtenFiles.put(src.toString(), hdfs.getFileStatus(src));
        // Create snapshot and snapshotDiff.
        final Path orig = new Path("/src/orig");
        hdfs.mkdirs(orig);
        final Path file1 = new Path("/src/file");
        FSDataOutputStream o = hdfs.create(file1);
        o.write(23);
        o.write(45);
        o.close();
        hdfs.allowSnapshot(src);
        hdfs.createSnapshot(src, "snapshot");
        final Path dst = new Path("/dst");
        // Rename a directory in the snapshot directory to add snapshotCopy
        // field to the dirDiff entry.
        hdfs.rename(orig, dst);
        dirCount++;
        writtenFiles.put(dst.toString(), hdfs.getFileStatus(dst));
        // Truncate a file in the snapshot directory to add snapshotCopy and
        // blocks fields to the fileDiff entry.
        hdfs.truncate(file1, 1);
        writtenFiles.put(file1.toString(), hdfs.getFileStatus(file1));
        // Set XAttrs so the fsimage contains XAttr ops
        final Path xattr = new Path("/xattr");
        hdfs.mkdirs(xattr);
        dirCount++;
        hdfs.setXAttr(xattr, "user.a1", new byte[] { 0x31, 0x32, 0x33 });
        hdfs.setXAttr(xattr, "user.a2", new byte[] { 0x37, 0x38, 0x39 });
        // OIV should be able to handle empty value XAttrs
        hdfs.setXAttr(xattr, "user.a3", null);
        // OIV should be able to handle XAttr values that can't be expressed
        // as UTF8
        hdfs.setXAttr(xattr, "user.a4", new byte[] { -0x3d, 0x28 });
        writtenFiles.put(xattr.toString(), hdfs.getFileStatus(xattr));
        // Set ACLs
        hdfs.setAcl(xattr, Lists.newArrayList(aclEntry(ACCESS, USER, ALL), aclEntry(ACCESS, USER, "foo", ALL), aclEntry(ACCESS, GROUP, READ_EXECUTE), aclEntry(ACCESS, GROUP, "bar", READ_EXECUTE), aclEntry(ACCESS, OTHER, EXECUTE)));
        // Create an Erasure Coded dir
        Path ecDir = new Path("/ec");
        hdfs.mkdirs(ecDir);
        dirCount++;
        hdfs.getClient().setErasureCodingPolicy(ecDir.toString(), ecPolicy.getName());
        writtenFiles.put(ecDir.toString(), hdfs.getFileStatus(ecDir));
        // Create an empty Erasure Coded file
        Path emptyECFile = new Path(ecDir, "EmptyECFile.txt");
        hdfs.create(emptyECFile).close();
        writtenFiles.put(emptyECFile.toString(), pathToFileEntry(hdfs, emptyECFile.toString()));
        filesECCount++;
        // Create a small Erasure Coded file
        Path smallECFile = new Path(ecDir, "SmallECFile.txt");
        FSDataOutputStream out = hdfs.create(smallECFile);
        Random r = new Random();
        byte[] bytes = new byte[1024 * 10];
        r.nextBytes(bytes);
        out.write(bytes);
        writtenFiles.put(smallECFile.toString(), pathToFileEntry(hdfs, smallECFile.toString()));
        filesECCount++;
        // Write results to the fsimage file
        hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER, false);
        hdfs.saveNamespace();
        hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE, false);
        // Determine location of fsimage file
        originalFsimage = FSImageTestUtil.findLatestImageFile(FSImageTestUtil.getFSImage(cluster.getNameNode()).getStorage().getStorageDir(0));
        if (originalFsimage == null) {
            throw new RuntimeException("Didn't generate or can't find fsimage");
        }
        LOG.debug("original FS image file is " + originalFsimage);
    } finally {
        if (cluster != null)
            cluster.shutdown();
    }
}

Example 32 with MiniDFSCluster

use of org.apache.hadoop.hdfs.MiniDFSCluster in project hadoop by apache.

the class TestOfflineImageViewerForAcl method createOriginalFSImage.

/**
   * Create a populated namespace for later testing. Save its contents to a
   * data structure and store its fsimage location.
   * We only want to generate the fsimage file once and use it for
   * multiple tests.
   */
@BeforeClass
public static void createOriginalFSImage() throws IOException {
    MiniDFSCluster cluster = null;
    try {
        Configuration conf = new Configuration();
        conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
        cluster = new MiniDFSCluster.Builder(conf).build();
        cluster.waitActive();
        DistributedFileSystem hdfs = cluster.getFileSystem();
        // Create a reasonable namespace with ACLs
        Path dir = new Path("/dirWithNoAcl");
        hdfs.mkdirs(dir);
        writtenAcls.put(dir.toString(), hdfs.getAclStatus(dir));
        dir = new Path("/dirWithDefaultAcl");
        hdfs.mkdirs(dir);
        hdfs.setAcl(dir, Lists.newArrayList(aclEntry(DEFAULT, USER, ALL), aclEntry(DEFAULT, USER, "foo", ALL), aclEntry(DEFAULT, GROUP, READ_EXECUTE), aclEntry(DEFAULT, OTHER, NONE)));
        writtenAcls.put(dir.toString(), hdfs.getAclStatus(dir));
        Path file = new Path("/noAcl");
        FSDataOutputStream o = hdfs.create(file);
        o.write(23);
        o.close();
        writtenAcls.put(file.toString(), hdfs.getAclStatus(file));
        file = new Path("/withAcl");
        o = hdfs.create(file);
        o.write(23);
        o.close();
        hdfs.setAcl(file, Lists.newArrayList(aclEntry(ACCESS, USER, READ_WRITE), aclEntry(ACCESS, USER, "foo", READ), aclEntry(ACCESS, GROUP, READ), aclEntry(ACCESS, OTHER, NONE)));
        writtenAcls.put(file.toString(), hdfs.getAclStatus(file));
        file = new Path("/withSeveralAcls");
        o = hdfs.create(file);
        o.write(23);
        o.close();
        hdfs.setAcl(file, Lists.newArrayList(aclEntry(ACCESS, USER, READ_WRITE), aclEntry(ACCESS, USER, "foo", READ_WRITE), aclEntry(ACCESS, USER, "bar", READ), aclEntry(ACCESS, GROUP, READ), aclEntry(ACCESS, GROUP, "group", READ), aclEntry(ACCESS, OTHER, NONE)));
        writtenAcls.put(file.toString(), hdfs.getAclStatus(file));
        // Write results to the fsimage file
        hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER, false);
        hdfs.saveNamespace();
        // Determine the location of the fsimage file
        originalFsimage = FSImageTestUtil.findLatestImageFile(FSImageTestUtil.getFSImage(cluster.getNameNode()).getStorage().getStorageDir(0));
        if (originalFsimage == null) {
            throw new RuntimeException("Didn't generate or can't find fsimage");
        }
        LOG.debug("original FS image file is " + originalFsimage);
    } finally {
        if (cluster != null)
            cluster.shutdown();
    }
}

Example 33 with MiniDFSCluster

use of org.apache.hadoop.hdfs.MiniDFSCluster in project hadoop by apache.

the class TestWebHDFS method testWebHdfsOffsetAndLength.

@Test
public void testWebHdfsOffsetAndLength() throws Exception {
    MiniDFSCluster cluster = null;
    final Configuration conf = WebHdfsTestUtil.createConf();
    final int OFFSET = 42;
    final int LENGTH = 512;
    final String PATH = "/foo";
    byte[] CONTENTS = new byte[1024];
    RANDOM.nextBytes(CONTENTS);
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
        final WebHdfsFileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystem(conf, WebHdfsConstants.WEBHDFS_SCHEME);
        try (OutputStream os = fs.create(new Path(PATH))) {
            os.write(CONTENTS);
        }
        InetSocketAddress addr = cluster.getNameNode().getHttpAddress();
        URL url = new URL("http", addr.getHostString(), addr.getPort(), WebHdfsFileSystem.PATH_PREFIX + PATH + "?op=OPEN" + Param.toSortedString("&", new OffsetParam((long) OFFSET), new LengthParam((long) LENGTH)));
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        conn.setInstanceFollowRedirects(true);
        Assert.assertEquals(LENGTH, conn.getContentLength());
        byte[] subContents = new byte[LENGTH];
        byte[] realContents = new byte[LENGTH];
        System.arraycopy(CONTENTS, OFFSET, subContents, 0, LENGTH);
        IOUtils.readFully(conn.getInputStream(), realContents);
        Assert.assertArrayEquals(subContents, realContents);
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}

Example 34 with MiniDFSCluster

use of org.apache.hadoop.hdfs.MiniDFSCluster in project hadoop by apache.

the class TestWebHDFS method testWebHdfsGetBlockLocationsWithStorageType.

@Test
public void testWebHdfsGetBlockLocationsWithStorageType() throws Exception {
    MiniDFSCluster cluster = null;
    final Configuration conf = WebHdfsTestUtil.createConf();
    final int OFFSET = 42;
    final int LENGTH = 512;
    final Path PATH = new Path("/foo");
    byte[] CONTENTS = new byte[1024];
    RANDOM.nextBytes(CONTENTS);
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
        final WebHdfsFileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystem(conf, WebHdfsConstants.WEBHDFS_SCHEME);
        try (OutputStream os = fs.create(PATH)) {
            os.write(CONTENTS);
        }
        BlockLocation[] locations = fs.getFileBlockLocations(PATH, OFFSET, LENGTH);
        for (BlockLocation location : locations) {
            StorageType[] storageTypes = location.getStorageTypes();
            Assert.assertTrue(storageTypes != null && storageTypes.length > 0 && storageTypes[0] == StorageType.DISK);
        }
        // Query webhdfs REST API to get block locations
        InetSocketAddress addr = cluster.getNameNode().getHttpAddress();
        // Case 1
        // URL without length or offset parameters
        URL url1 = new URL("http", addr.getHostString(), addr.getPort(), WebHdfsFileSystem.PATH_PREFIX + "/foo?op=GETFILEBLOCKLOCATIONS");
        LOG.info("Sending GETFILEBLOCKLOCATIONS request " + url1);
        String response1 = getResponse(url1, "GET");
        LOG.info("The output of GETFILEBLOCKLOCATIONS request " + response1);
        // Parse BlockLocation array from json output using object mapper
        BlockLocation[] locationArray1 = toBlockLocationArray(response1);
        // Verify the result from rest call is same as file system api
        verifyEquals(locations, locationArray1);
        // Case 2
        // URL contains length and offset parameters
        URL url2 = new URL("http", addr.getHostString(), addr.getPort(), WebHdfsFileSystem.PATH_PREFIX + "/foo?op=GETFILEBLOCKLOCATIONS" + "&length=" + LENGTH + "&offset=" + OFFSET);
        LOG.info("Sending GETFILEBLOCKLOCATIONS request " + url2);
        String response2 = getResponse(url2, "GET");
        LOG.info("The output of GETFILEBLOCKLOCATIONS request " + response2);
        BlockLocation[] locationArray2 = toBlockLocationArray(response2);
        verifyEquals(locations, locationArray2);
        // Case 3
        // URL contains length parameter but without offset parameters
        URL url3 = new URL("http", addr.getHostString(), addr.getPort(), WebHdfsFileSystem.PATH_PREFIX + "/foo?op=GETFILEBLOCKLOCATIONS" + "&length=" + LENGTH);
        LOG.info("Sending GETFILEBLOCKLOCATIONS request " + url3);
        String response3 = getResponse(url3, "GET");
        LOG.info("The output of GETFILEBLOCKLOCATIONS request " + response3);
        BlockLocation[] locationArray3 = toBlockLocationArray(response3);
        verifyEquals(locations, locationArray3);
        // Case 4
        // URL contains offset parameter but without length parameter
        URL url4 = new URL("http", addr.getHostString(), addr.getPort(), WebHdfsFileSystem.PATH_PREFIX + "/foo?op=GETFILEBLOCKLOCATIONS" + "&offset=" + OFFSET);
        LOG.info("Sending GETFILEBLOCKLOCATIONS request " + url4);
        String response4 = getResponse(url4, "GET");
        LOG.info("The output of GETFILEBLOCKLOCATIONS request " + response4);
        BlockLocation[] locationArray4 = toBlockLocationArray(response4);
        verifyEquals(locations, locationArray4);
        // Case 5
        // URL specifies offset exceeds the file length
        URL url5 = new URL("http", addr.getHostString(), addr.getPort(), WebHdfsFileSystem.PATH_PREFIX + "/foo?op=GETFILEBLOCKLOCATIONS" + "&offset=1200");
        LOG.info("Sending GETFILEBLOCKLOCATIONS request " + url5);
        String response5 = getResponse(url5, "GET");
        LOG.info("The output of GETFILEBLOCKLOCATIONS request " + response5);
        BlockLocation[] locationArray5 = toBlockLocationArray(response5);
        // Expected an empty array of BlockLocation
        verifyEquals(new BlockLocation[] {}, locationArray5);
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}

Example 35 with MiniDFSCluster

use of org.apache.hadoop.hdfs.MiniDFSCluster in project hadoop by apache.

the class TestWebHDFS method testCreateWithNoDN.

/**
   * Test for catching "no datanode" IOException, when to create a file
   * but datanode is not running for some reason.
   */
@Test(timeout = 300000)
public void testCreateWithNoDN() throws Exception {
    MiniDFSCluster cluster = null;
    final Configuration conf = WebHdfsTestUtil.createConf();
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
        conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
        cluster.waitActive();
        FileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystem(conf, WebHdfsConstants.WEBHDFS_SCHEME);
        fs.create(new Path("/testnodatanode"));
        Assert.fail("No exception was thrown");
    } catch (IOException ex) {
        GenericTestUtils.assertExceptionContains("Failed to find datanode", ex);
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}