Examples with BlockLocation org.apache.hadoop.fs.BlockLocation used on opensource projects

Example 36 with BlockLocation

use of org.apache.hadoop.fs.BlockLocation in project brisk by riptano.

the class CassandraFileSystemThriftStore method getBlockLocation.

public BlockLocation[] getBlockLocation(List<Block> blocks, long start, long len) throws IOException {
    if (blocks.isEmpty())
        return null;
    List<ByteBuffer> blockKeys = new ArrayList<ByteBuffer>(blocks.size());
    for (Block b : blocks) blockKeys.add(uuidToByteBuffer(b.id));
    BlockLocation[] locations = new BlockLocation[blocks.size()];
    try {
        List<List<String>> blockEndpoints = ((Brisk.Iface) client).describe_keys(keySpace, blockKeys);
        for (int i = 0; i < blockEndpoints.size(); i++) {
            List<String> endpoints = blockEndpoints.get(i);
            Block b = blocks.get(i);
            long offset = (i == 0 && b.offset > start) ? start : b.offset;
            // TODO: Add topology info if at all possible?
            locations[i] = new BlockLocation(null, endpoints.toArray(new String[0]), offset, b.length);
        }
        return locations;
    } catch (Exception e) {
        throw new IOException(e);
    }
}

Example 37 with BlockLocation

use of org.apache.hadoop.fs.BlockLocation in project cdap by caskdata.

the class StreamDataFileSplitter method getBlockIndex.

/**
   * Returns the array index of the given blockLocations that contains the given offset.
   *
   * @param blockLocations Array of {@link BlockLocation} to search for.
   * @param offset File offset.
   * @param startIdx Starting index for the search in the array.
   * @return The array index of the {@link BlockLocation} that contains the given offset.
   */
private int getBlockIndex(BlockLocation[] blockLocations, long offset, int startIdx) {
    if (blockLocations == null) {
        return -1;
    }
    for (int i = startIdx; i < blockLocations.length; i++) {
        BlockLocation blockLocation = blockLocations[i];
        long endOffset = blockLocation.getOffset() + blockLocation.getLength();
        if (blockLocation.getOffset() <= offset && offset < endOffset) {
            return i;
        }
    }
    return -1;
}

Example 38 with BlockLocation

use of org.apache.hadoop.fs.BlockLocation in project cdap by caskdata.

the class StreamDataFileSplitter method computeSplits.

/**
   * Computes splits for the event file.
   */
<T> void computeSplits(FileSystem fs, long minSplitSize, long maxSplitSize, long startTime, long endTime, List<T> splits, StreamInputSplitFactory<T> splitFactory) throws IOException {
    // Compute the splits based on the min/max size
    Path eventFile = eventFileStatus.getPath();
    Path indexFile = getIndexFile(eventFile);
    BlockLocation[] blockLocations = fs.getFileBlockLocations(eventFile, 0, eventFileStatus.getLen());
    long length = eventFileStatus.getLen();
    long offset = 0;
    int blockIndex = 0;
    while (offset < length) {
        blockIndex = getBlockIndex(blockLocations, offset, blockIndex);
        String[] hosts = null;
        if (blockIndex >= 0) {
            hosts = blockLocations[blockIndex].getHosts();
        } else {
            blockIndex = 0;
        }
        long splitSize = computeSplitSize(eventFileStatus, offset, minSplitSize, maxSplitSize);
        splits.add(splitFactory.createSplit(eventFile, indexFile, startTime, endTime, offset, splitSize, hosts));
        offset += splitSize;
    }
    // One extra split for the tail of the file.
    splits.add(splitFactory.createSplit(eventFile, indexFile, startTime, endTime, offset, Long.MAX_VALUE, null));
}

Example 39 with BlockLocation

use of org.apache.hadoop.fs.BlockLocation in project SSM by Intel-bigdata.

the class CheckStorageAction method execute.

@Override
protected void execute() {
    ActionStatus actionStatus = getActionStatus();
    actionStatus.begin();
    try {
        HdfsFileStatus fileStatus = dfsClient.getFileInfo(fileName);
        long length = fileStatus.getLen();
        BlockLocation[] blockLocations = dfsClient.getBlockLocations(fileName, 0, length);
        for (BlockLocation blockLocation : blockLocations) {
            StringBuilder hosts = new StringBuilder();
            hosts.append("{");
            for (String host : blockLocation.getHosts()) {
                hosts.append(host + " ");
            }
            hosts.append("}");
            this.resultOut.println(hosts);
        }
        actionStatus.setSuccessful(true);
    } catch (Exception e) {
        actionStatus.setSuccessful(false);
        throw new RuntimeException(e);
    } finally {
        actionStatus.end();
    }
}

Example 40 with BlockLocation

use of org.apache.hadoop.fs.BlockLocation in project drill by apache.

the class Metadata method getHostAffinity.

/**
   * Get the host affinity for a row group
   *
   * @param fileStatus the parquet file
   * @param start      the start of the row group
   * @param length     the length of the row group
   * @return
   * @throws IOException
   */
private Map<String, Float> getHostAffinity(FileStatus fileStatus, long start, long length) throws IOException {
    BlockLocation[] blockLocations = fs.getFileBlockLocations(fileStatus, start, length);
    Map<String, Float> hostAffinityMap = Maps.newHashMap();
    for (BlockLocation blockLocation : blockLocations) {
        for (String host : blockLocation.getHosts()) {
            Float currentAffinity = hostAffinityMap.get(host);
            float blockStart = blockLocation.getOffset();
            float blockEnd = blockStart + blockLocation.getLength();
            float rowGroupEnd = start + length;
            Float newAffinity = (blockLocation.getLength() - (blockStart < start ? start - blockStart : 0) - (blockEnd > rowGroupEnd ? blockEnd - rowGroupEnd : 0)) / length;
            if (currentAffinity != null) {
                hostAffinityMap.put(host, currentAffinity + newAffinity);
            } else {
                hostAffinityMap.put(host, newAffinity);
            }
        }
    }
    return hostAffinityMap;
}