Examples with BlockLocation org.apache.hadoop.fs.BlockLocation used on opensource projects

Example 16 with BlockLocation

use of org.apache.hadoop.fs.BlockLocation in project hadoop by apache.

the class TestDistributedFileSystem method testLocatedFileStatusStorageIdsTypes.

@Test(timeout = 120000)
public void testLocatedFileStatusStorageIdsTypes() throws Exception {
    final Configuration conf = getTestConfiguration();
    final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
    try {
        final DistributedFileSystem fs = cluster.getFileSystem();
        final Path testFile = new Path("/testListLocatedStatus");
        final int blockSize = 4096;
        final int numBlocks = 10;
        // Create a test file
        final int repl = 2;
        DFSTestUtil.createFile(fs, testFile, blockSize, numBlocks * blockSize, blockSize, (short) repl, 0xADDED);
        DFSTestUtil.waitForReplication(fs, testFile, (short) repl, 30000);
        // Get the listing
        RemoteIterator<LocatedFileStatus> it = fs.listLocatedStatus(testFile);
        assertTrue("Expected file to be present", it.hasNext());
        LocatedFileStatus stat = it.next();
        BlockLocation[] locs = stat.getBlockLocations();
        assertEquals("Unexpected number of locations", numBlocks, locs.length);
        Set<String> dnStorageIds = new HashSet<>();
        for (DataNode d : cluster.getDataNodes()) {
            try (FsDatasetSpi.FsVolumeReferences volumes = d.getFSDataset().getFsVolumeReferences()) {
                for (FsVolumeSpi vol : volumes) {
                    dnStorageIds.add(vol.getStorageID());
                }
            }
        }
        for (BlockLocation loc : locs) {
            String[] ids = loc.getStorageIds();
            // Run it through a set to deduplicate, since there should be no dupes
            Set<String> storageIds = new HashSet<>();
            Collections.addAll(storageIds, ids);
            assertEquals("Unexpected num storage ids", repl, storageIds.size());
            // Make sure these are all valid storage IDs
            assertTrue("Unknown storage IDs found!", dnStorageIds.containsAll(storageIds));
            // Check storage types are the default, since we didn't set any
            StorageType[] types = loc.getStorageTypes();
            assertEquals("Unexpected num storage types", repl, types.length);
            for (StorageType t : types) {
                assertEquals("Unexpected storage type", StorageType.DEFAULT, t);
            }
        }
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}

Example 17 with BlockLocation

use of org.apache.hadoop.fs.BlockLocation in project hadoop by apache.

the class TestDatanodeDeath method checkFile.

//
// verify that the data written are sane
// 
private static void checkFile(FileSystem fileSys, Path name, int repl, int numblocks, int filesize, long seed) throws IOException {
    boolean done = false;
    int attempt = 0;
    long len = fileSys.getFileStatus(name).getLen();
    assertTrue(name + " should be of size " + filesize + " but found to be of size " + len, len == filesize);
    // wait till all full blocks are confirmed by the datanodes.
    while (!done) {
        attempt++;
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
        }
        done = true;
        BlockLocation[] locations = fileSys.getFileBlockLocations(fileSys.getFileStatus(name), 0, filesize);
        if (locations.length < numblocks) {
            if (attempt > 100) {
                System.out.println("File " + name + " has only " + locations.length + " blocks, " + " but is expected to have " + numblocks + " blocks.");
            }
            done = false;
            continue;
        }
        for (int idx = 0; idx < locations.length; idx++) {
            if (locations[idx].getHosts().length < repl) {
                if (attempt > 100) {
                    System.out.println("File " + name + " has " + locations.length + " blocks: " + " The " + idx + " block has only " + locations[idx].getHosts().length + " replicas but is expected to have " + repl + " replicas.");
                }
                done = false;
                break;
            }
        }
    }
    FSDataInputStream stm = fileSys.open(name);
    final byte[] expected = AppendTestUtil.randomBytes(seed, fileSize);
    // do a sanity check. Read the file
    byte[] actual = new byte[filesize];
    stm.readFully(0, actual);
    checkData(actual, 0, expected, "Read 1");
}

Example 18 with BlockLocation

use of org.apache.hadoop.fs.BlockLocation in project hadoop by apache.

the class ViewFsBaseTest method testGetBlockLocations.

// override for HDFS
@Test
public void testGetBlockLocations() throws IOException {
    Path targetFilePath = new Path(targetTestRoot, "data/largeFile");
    FileContextTestHelper.createFile(fcTarget, targetFilePath, 10, 1024);
    Path viewFilePath = new Path("/data/largeFile");
    checkFileStatus(fcView, viewFilePath.toString(), fileType.isFile);
    BlockLocation[] viewBL = fcView.getFileBlockLocations(viewFilePath, 0, 10240 + 100);
    Assert.assertEquals(SupportsBlocks ? 10 : 1, viewBL.length);
    BlockLocation[] targetBL = fcTarget.getFileBlockLocations(targetFilePath, 0, 10240 + 100);
    compareBLs(viewBL, targetBL);
    // Same test but now get it via the FileStatus Parameter
    fcView.getFileBlockLocations(viewFilePath, 0, 10240 + 100);
    targetBL = fcTarget.getFileBlockLocations(targetFilePath, 0, 10240 + 100);
    compareBLs(viewBL, targetBL);
}

Example 19 with BlockLocation

use of org.apache.hadoop.fs.BlockLocation in project hadoop by apache.

the class TestSwiftFileSystemPartitionedUploads method testFilePartUploadNoLengthCheck.

/**
   * tests functionality for big files ( > 5Gb) upload
   */
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testFilePartUploadNoLengthCheck() throws IOException, URISyntaxException {
    final Path path = new Path("/test/testFilePartUploadLengthCheck");
    int len = 8192;
    final byte[] src = SwiftTestUtils.dataset(len, 32, 144);
    FSDataOutputStream out = fs.create(path, false, getBufferSize(), (short) 1, BLOCK_SIZE);
    try {
        int totalPartitionsToWrite = len / PART_SIZE_BYTES;
        assertPartitionsWritten("Startup", out, 0);
        //write 2048
        int firstWriteLen = 2048;
        out.write(src, 0, firstWriteLen);
        //assert
        long expected = getExpectedPartitionsWritten(firstWriteLen, PART_SIZE_BYTES, false);
        SwiftUtils.debug(LOG, "First write: predict %d partitions written", expected);
        assertPartitionsWritten("First write completed", out, expected);
        //write the rest
        int remainder = len - firstWriteLen;
        SwiftUtils.debug(LOG, "remainder: writing: %d bytes", remainder);
        out.write(src, firstWriteLen, remainder);
        expected = getExpectedPartitionsWritten(len, PART_SIZE_BYTES, false);
        assertPartitionsWritten("Remaining data", out, expected);
        out.close();
        expected = getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true);
        assertPartitionsWritten("Stream closed", out, expected);
        Header[] headers = fs.getStore().getObjectHeaders(path, true);
        for (Header header : headers) {
            LOG.info(header.toString());
        }
        byte[] dest = readDataset(fs, path, len);
        LOG.info("Read dataset from " + path + ": data length =" + len);
        //compare data
        SwiftTestUtils.compareByteArrays(src, dest, len);
        FileStatus status = fs.getFileStatus(path);
        //now see what block location info comes back.
        //This will vary depending on the Swift version, so the results
        //aren't checked -merely that the test actually worked
        BlockLocation[] locations = fs.getFileBlockLocations(status, 0, len);
        assertNotNull("Null getFileBlockLocations()", locations);
        assertTrue("empty array returned for getFileBlockLocations()", locations.length > 0);
    } finally {
        IOUtils.closeStream(out);
    }
}

Example 20 with BlockLocation

use of org.apache.hadoop.fs.BlockLocation in project hadoop by apache.

the class TestSwiftFileSystemPartitionedUploads method testFilePartUpload.

/**
   * tests functionality for big files ( > 5Gb) upload
   */
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testFilePartUpload() throws Throwable {
    final Path path = new Path("/test/testFilePartUpload");
    int len = 8192;
    final byte[] src = SwiftTestUtils.dataset(len, 32, 144);
    FSDataOutputStream out = fs.create(path, false, getBufferSize(), (short) 1, BLOCK_SIZE);
    try {
        int totalPartitionsToWrite = len / PART_SIZE_BYTES;
        assertPartitionsWritten("Startup", out, 0);
        //write 2048
        int firstWriteLen = 2048;
        out.write(src, 0, firstWriteLen);
        //assert
        long expected = getExpectedPartitionsWritten(firstWriteLen, PART_SIZE_BYTES, false);
        SwiftUtils.debug(LOG, "First write: predict %d partitions written", expected);
        assertPartitionsWritten("First write completed", out, expected);
        //write the rest
        int remainder = len - firstWriteLen;
        SwiftUtils.debug(LOG, "remainder: writing: %d bytes", remainder);
        out.write(src, firstWriteLen, remainder);
        expected = getExpectedPartitionsWritten(len, PART_SIZE_BYTES, false);
        assertPartitionsWritten("Remaining data", out, expected);
        out.close();
        expected = getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true);
        assertPartitionsWritten("Stream closed", out, expected);
        Header[] headers = fs.getStore().getObjectHeaders(path, true);
        for (Header header : headers) {
            LOG.info(header.toString());
        }
        byte[] dest = readDataset(fs, path, len);
        LOG.info("Read dataset from " + path + ": data length =" + len);
        //compare data
        SwiftTestUtils.compareByteArrays(src, dest, len);
        FileStatus status;
        final Path qualifiedPath = path.makeQualified(fs);
        status = fs.getFileStatus(qualifiedPath);
        //now see what block location info comes back.
        //This will vary depending on the Swift version, so the results
        //aren't checked -merely that the test actually worked
        BlockLocation[] locations = fs.getFileBlockLocations(status, 0, len);
        assertNotNull("Null getFileBlockLocations()", locations);
        assertTrue("empty array returned for getFileBlockLocations()", locations.length > 0);
        //to a skip
        try {
            validatePathLen(path, len);
        } catch (AssertionError e) {
            //downgrade to a skip
            throw new AssumptionViolatedException(e, null);
        }
    } finally {
        IOUtils.closeStream(out);
    }
}