Examples with BlockLocation org.apache.hadoop.fs.BlockLocation used on opensource projects

Example 61 with BlockLocation

use of org.apache.hadoop.fs.BlockLocation in project mongo-hadoop by mongodb.

the class BSONSplitter method createFileSplit.

public BSONFileSplit createFileSplit(final FileStatus inFile, final FileSystem fs, final long splitStart, final long splitLen) {
    BSONFileSplit split;
    try {
        BlockLocation[] blkLocations;
        // This code is based off of org.apache.hadoop.mapreduce.lib
        // .input.FileInputFormat.getSplits()
        boolean isLocatedFileStatus = CompatUtils.isInstance(inFile, "org.apache.hadoop.fs.LocatedFileStatus", getConf(), FileStatus.class);
        if (isLocatedFileStatus) {
            blkLocations = (BlockLocation[]) CompatUtils.invokeMethod(FileStatus.class, inFile, "getBlockLocations", new Object[] {}, new Class[] {});
        } else {
            blkLocations = fs.getFileBlockLocations(inFile, splitStart, splitLen);
        }
        int blockIndex = getBlockIndex(blkLocations, splitStart);
        split = new BSONFileSplit(inFile.getPath(), splitStart, splitLen, blkLocations[blockIndex].getHosts());
    } catch (IOException e) {
        LOG.warn("Couldn't find block locations when constructing input split from byte offset. Using non-block-aware input split; " + e.getMessage());
        split = new BSONFileSplit(inFile.getPath(), splitStart, splitLen, null);
    }
    split.setKeyField(MongoConfigUtil.getInputKey(getConf()));
    return split;
}

Example 62 with BlockLocation

use of org.apache.hadoop.fs.BlockLocation in project hadoop by apache.

the class ViewFsBaseTest method compareBLs.

void compareBLs(BlockLocation[] viewBL, BlockLocation[] targetBL) {
    Assert.assertEquals(targetBL.length, viewBL.length);
    int i = 0;
    for (BlockLocation vbl : viewBL) {
        Assert.assertEquals(vbl.toString(), targetBL[i].toString());
        Assert.assertEquals(targetBL[i].getOffset(), vbl.getOffset());
        Assert.assertEquals(targetBL[i].getLength(), vbl.getLength());
        i++;
    }
}

Example 63 with BlockLocation

use of org.apache.hadoop.fs.BlockLocation in project hadoop by apache.

the class ViewFileSystemBaseTest method compareBLs.

void compareBLs(BlockLocation[] viewBL, BlockLocation[] targetBL) {
    Assert.assertEquals(targetBL.length, viewBL.length);
    int i = 0;
    for (BlockLocation vbl : viewBL) {
        Assert.assertEquals(vbl.toString(), targetBL[i].toString());
        Assert.assertEquals(targetBL[i].getOffset(), vbl.getOffset());
        Assert.assertEquals(targetBL[i].getLength(), vbl.getLength());
        i++;
    }
}

Example 64 with BlockLocation

use of org.apache.hadoop.fs.BlockLocation in project hadoop by apache.

the class ViewFileSystemBaseTest method testGetBlockLocations.

// override for HDFS
@Test
public void testGetBlockLocations() throws IOException {
    Path targetFilePath = new Path(targetTestRoot, "data/largeFile");
    FileSystemTestHelper.createFile(fsTarget, targetFilePath, 10, 1024);
    Path viewFilePath = new Path("/data/largeFile");
    Assert.assertTrue("Created File should be type File", fsView.isFile(viewFilePath));
    BlockLocation[] viewBL = fsView.getFileBlockLocations(fsView.getFileStatus(viewFilePath), 0, 10240 + 100);
    Assert.assertEquals(SupportsBlocks ? 10 : 1, viewBL.length);
    BlockLocation[] targetBL = fsTarget.getFileBlockLocations(fsTarget.getFileStatus(targetFilePath), 0, 10240 + 100);
    compareBLs(viewBL, targetBL);
    // Same test but now get it via the FileStatus Parameter
    fsView.getFileBlockLocations(fsView.getFileStatus(viewFilePath), 0, 10240 + 100);
    targetBL = fsTarget.getFileBlockLocations(fsTarget.getFileStatus(targetFilePath), 0, 10240 + 100);
    compareBLs(viewBL, targetBL);
}

Example 65 with BlockLocation

use of org.apache.hadoop.fs.BlockLocation in project hadoop by apache.

the class DFSClient method getBlockLocations.

/**
   * Get block location info about file
   *
   * getBlockLocations() returns a list of hostnames that store
   * data for a specific file region.  It returns a set of hostnames
   * for every block within the indicated region.
   *
   * This function is very useful when writing code that considers
   * data-placement when performing operations.  For example, the
   * MapReduce system tries to schedule tasks on the same machines
   * as the data-block the task processes.
   */
public BlockLocation[] getBlockLocations(String src, long start, long length) throws IOException {
    checkOpen();
    try (TraceScope ignored = newPathTraceScope("getBlockLocations", src)) {
        LocatedBlocks blocks = getLocatedBlocks(src, start, length);
        BlockLocation[] locations = DFSUtilClient.locatedBlocks2Locations(blocks);
        HdfsBlockLocation[] hdfsLocations = new HdfsBlockLocation[locations.length];
        for (int i = 0; i < locations.length; i++) {
            hdfsLocations[i] = new HdfsBlockLocation(locations[i], blocks.get(i));
        }
        return hdfsLocations;
    }
}