Examples with ImmutableRangeMap org.apache.drill.shaded.guava.com.google.common.collect.ImmutableRangeMap used on opensource projects

Example 1 with ImmutableRangeMap

use of org.apache.drill.shaded.guava.com.google.common.collect.ImmutableRangeMap in project drill by apache.

the class BlockMapBuilder method getEndpointByteMap.

/**
 * For a given FileWork, calculate how many bytes are available on each on drillbit endpoint
 *
 * @param work the FileWork to calculate endpoint bytes for
 * @throws IOException
 */
public EndpointByteMap getEndpointByteMap(Set<String> noDrillbitHosts, FileWork work) throws IOException {
    Stopwatch watch = Stopwatch.createStarted();
    Path fileName = work.getPath();
    ImmutableRangeMap<Long, BlockLocation> blockMap = getBlockMap(fileName);
    EndpointByteMapImpl endpointByteMap = new EndpointByteMapImpl();
    long start = work.getStart();
    long end = start + work.getLength();
    Range<Long> rowGroupRange = Range.closedOpen(start, end);
    // Find submap of ranges that intersect with the rowGroup
    ImmutableRangeMap<Long, BlockLocation> subRangeMap = blockMap.subRangeMap(rowGroupRange);
    // Iterate through each block in this submap and get the host for the block location
    for (Map.Entry<Range<Long>, BlockLocation> block : subRangeMap.asMapOfRanges().entrySet()) {
        String[] hosts;
        Range<Long> blockRange = block.getKey();
        try {
            hosts = block.getValue().getHosts();
        } catch (IOException ioe) {
            throw new RuntimeException("Failed to get hosts for block location", ioe);
        }
        Range<Long> intersection = rowGroupRange.intersection(blockRange);
        long bytes = intersection.upperEndpoint() - intersection.lowerEndpoint();
        // For each host in the current block location, add the intersecting bytes to the corresponding endpoint
        for (String host : hosts) {
            DrillbitEndpoint endpoint = getDrillBitEndpoint(host);
            if (endpoint != null) {
                endpointByteMap.add(endpoint, bytes);
            } else if (noDrillbitHosts != null) {
                noDrillbitHosts.add(host);
            }
        }
    }
    logger.debug("FileWork group ({},{}) max bytes {}", work.getPath(), work.getStart(), endpointByteMap.getMaxBytes());
    logger.debug("Took {} ms to set endpoint bytes", watch.stop().elapsed(TimeUnit.MILLISECONDS));
    return endpointByteMap;
}

Example 2 with ImmutableRangeMap

use of org.apache.drill.shaded.guava.com.google.common.collect.ImmutableRangeMap in project drill by apache.

the class BlockMapBuilder method buildBlockMap.

/**
 * Builds a mapping of block locations to file byte range
 */
private ImmutableRangeMap<Long, BlockLocation> buildBlockMap(FileStatus status) throws IOException {
    final Timer.Context context = metrics.timer(BLOCK_MAP_BUILDER_TIMER).time();
    BlockLocation[] blocks;
    ImmutableRangeMap<Long, BlockLocation> blockMap;
    blocks = fs.getFileBlockLocations(status, 0, status.getLen());
    ImmutableRangeMap.Builder<Long, BlockLocation> blockMapBuilder = new ImmutableRangeMap.Builder<Long, BlockLocation>();
    for (BlockLocation block : blocks) {
        long start = block.getOffset();
        long end = start + block.getLength();
        Range<Long> range = Range.closedOpen(start, end);
        blockMapBuilder = blockMapBuilder.put(range, block);
    }
    blockMap = blockMapBuilder.build();
    blockMapMap.put(status.getPath(), blockMap);
    context.stop();
    return blockMap;
}

Example 3 with ImmutableRangeMap

use of org.apache.drill.shaded.guava.com.google.common.collect.ImmutableRangeMap in project drill by apache.

the class TestAffinityCalculator method testBuildRangeMap.

@Test
public void testBuildRangeMap() {
    BlockLocation[] blocks = buildBlockLocations(new String[4], 256 * 1024 * 1024);
    long tA = System.nanoTime();
    ImmutableRangeMap.Builder<Long, BlockLocation> blockMapBuilder = new ImmutableRangeMap.Builder<Long, BlockLocation>();
    for (BlockLocation block : blocks) {
        long start = block.getOffset();
        long end = start + block.getLength();
        Range<Long> range = Range.closedOpen(start, end);
        blockMapBuilder = blockMapBuilder.put(range, block);
    }
    ImmutableRangeMap<Long, BlockLocation> map = blockMapBuilder.build();
    long tB = System.nanoTime();
    logger.info(String.format("Took %f ms to build range map", (tB - tA) / 1e6));
}