Examples with BlockLocation org.apache.hadoop.fs.BlockLocation used on opensource projects

Example 26 with BlockLocation

use of org.apache.hadoop.fs.BlockLocation in project hadoop by apache.

the class AdlFileSystem method getFileBlockLocations.

@Override
public BlockLocation[] getFileBlockLocations(final FileStatus status, final long offset, final long length) throws IOException {
    if (status == null) {
        return null;
    }
    if ((offset < 0) || (length < 0)) {
        throw new IllegalArgumentException("Invalid start or len parameter");
    }
    if (status.getLen() < offset) {
        return new BlockLocation[0];
    }
    final String[] name = { "localhost" };
    final String[] host = { "localhost" };
    long blockSize = ADL_BLOCK_SIZE;
    int numberOfLocations = (int) (length / blockSize) + ((length % blockSize == 0) ? 0 : 1);
    BlockLocation[] locations = new BlockLocation[numberOfLocations];
    for (int i = 0; i < locations.length; i++) {
        long currentOffset = offset + (i * blockSize);
        long currentLength = Math.min(blockSize, offset + length - currentOffset);
        locations[i] = new BlockLocation(name, host, currentOffset, currentLength);
    }
    return locations;
}

Example 27 with BlockLocation

use of org.apache.hadoop.fs.BlockLocation in project hbase by apache.

the class FSRegionScanner method run.

@Override
public void run() {
    try {
        // empty the map for each region
        Map<String, AtomicInteger> blockCountMap = new HashMap<>();
        //get table name
        String tableName = regionPath.getParent().getName();
        int totalBlkCount = 0;
        // ignore null
        FileStatus[] cfList = fs.listStatus(regionPath, new FSUtils.FamilyDirFilter(fs));
        if (null == cfList) {
            return;
        }
        // for each cf, get all the blocks information
        for (FileStatus cfStatus : cfList) {
            if (!cfStatus.isDirectory()) {
                // skip because this is not a CF directory
                continue;
            }
            FileStatus[] storeFileLists = fs.listStatus(cfStatus.getPath());
            if (null == storeFileLists) {
                continue;
            }
            for (FileStatus storeFile : storeFileLists) {
                BlockLocation[] blkLocations = fs.getFileBlockLocations(storeFile, 0, storeFile.getLen());
                if (null == blkLocations) {
                    continue;
                }
                totalBlkCount += blkLocations.length;
                for (BlockLocation blk : blkLocations) {
                    for (String host : blk.getHosts()) {
                        AtomicInteger count = blockCountMap.get(host);
                        if (count == null) {
                            count = new AtomicInteger(0);
                            blockCountMap.put(host, count);
                        }
                        count.incrementAndGet();
                    }
                }
            }
        }
        if (regionToBestLocalityRSMapping != null) {
            int largestBlkCount = 0;
            String hostToRun = null;
            for (Map.Entry<String, AtomicInteger> entry : blockCountMap.entrySet()) {
                String host = entry.getKey();
                int tmp = entry.getValue().get();
                if (tmp > largestBlkCount) {
                    largestBlkCount = tmp;
                    hostToRun = host;
                }
            }
            // empty regions could make this null
            if (null == hostToRun) {
                return;
            }
            if (hostToRun.endsWith(".")) {
                hostToRun = hostToRun.substring(0, hostToRun.length() - 1);
            }
            String name = tableName + ":" + regionPath.getName();
            synchronized (regionToBestLocalityRSMapping) {
                regionToBestLocalityRSMapping.put(name, hostToRun);
            }
        }
        if (regionDegreeLocalityMapping != null && totalBlkCount > 0) {
            Map<String, Float> hostLocalityMap = new HashMap<>();
            for (Map.Entry<String, AtomicInteger> entry : blockCountMap.entrySet()) {
                String host = entry.getKey();
                if (host.endsWith(".")) {
                    host = host.substring(0, host.length() - 1);
                }
                // Locality is fraction of blocks local to this host.
                float locality = ((float) entry.getValue().get()) / totalBlkCount;
                hostLocalityMap.put(host, locality);
            }
            // Put the locality map into the result map, keyed by the encoded name
            // of the region.
            regionDegreeLocalityMapping.put(regionPath.getName(), hostLocalityMap);
        }
    } catch (IOException e) {
        LOG.warn("Problem scanning file system", e);
    } catch (RuntimeException e) {
        LOG.warn("Problem scanning file system", e);
    }
}

Example 28 with BlockLocation

use of org.apache.hadoop.fs.BlockLocation in project hbase by apache.

the class FSUtils method computeHDFSBlocksDistribution.

/**
   * Compute HDFS blocks distribution of a given file, or a portion of the file
   * @param fs file system
   * @param status file status of the file
   * @param start start position of the portion
   * @param length length of the portion
   * @return The HDFS blocks distribution
   */
public static HDFSBlocksDistribution computeHDFSBlocksDistribution(final FileSystem fs, FileStatus status, long start, long length) throws IOException {
    HDFSBlocksDistribution blocksDistribution = new HDFSBlocksDistribution();
    BlockLocation[] blockLocations = fs.getFileBlockLocations(status, start, length);
    for (BlockLocation bl : blockLocations) {
        String[] hosts = bl.getHosts();
        long len = bl.getLength();
        blocksDistribution.addHostsAndBlockWeight(hosts, len);
    }
    return blocksDistribution;
}

Example 29 with BlockLocation

use of org.apache.hadoop.fs.BlockLocation in project hbase by apache.

the class FSUtils method addToHDFSBlocksDistribution.

/**
   * Update blocksDistribution with blockLocations
   * @param blocksDistribution the hdfs blocks distribution
   * @param blockLocations an array containing block location
   */
public static void addToHDFSBlocksDistribution(HDFSBlocksDistribution blocksDistribution, BlockLocation[] blockLocations) throws IOException {
    for (BlockLocation bl : blockLocations) {
        String[] hosts = bl.getHosts();
        long len = bl.getLength();
        blocksDistribution.addHostsAndBlockWeight(hosts, len);
    }
}

Example 30 with BlockLocation

use of org.apache.hadoop.fs.BlockLocation in project hbase by apache.

the class TestRegionFavoredNodes method testFavoredNodes.

@Test
public void testFavoredNodes() throws Exception {
    Assume.assumeTrue(createWithFavoredNode != null);
    // Get the addresses of the datanodes in the cluster.
    InetSocketAddress[] nodes = new InetSocketAddress[REGION_SERVERS];
    List<DataNode> datanodes = TEST_UTIL.getDFSCluster().getDataNodes();
    Method selfAddress;
    try {
        selfAddress = DataNode.class.getMethod("getSelfAddr");
    } catch (NoSuchMethodException ne) {
        selfAddress = DataNode.class.getMethod("getXferAddress");
    }
    for (int i = 0; i < REGION_SERVERS; i++) {
        nodes[i] = (InetSocketAddress) selfAddress.invoke(datanodes.get(i));
    }
    String[] nodeNames = new String[REGION_SERVERS];
    for (int i = 0; i < REGION_SERVERS; i++) {
        nodeNames[i] = nodes[i].getAddress().getHostAddress() + ":" + nodes[i].getPort();
    }
    // them as favored nodes through the region.
    for (int i = 0; i < REGION_SERVERS; i++) {
        HRegionServer server = TEST_UTIL.getHBaseCluster().getRegionServer(i);
        List<Region> regions = server.getOnlineRegions(TABLE_NAME);
        for (Region region : regions) {
            List<org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.ServerName> favoredNodes = new ArrayList<>(3);
            String encodedRegionName = region.getRegionInfo().getEncodedName();
            for (int j = 0; j < FAVORED_NODES_NUM; j++) {
                org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.ServerName.Builder b = org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.ServerName.newBuilder();
                b.setHostName(nodes[(i + j) % REGION_SERVERS].getAddress().getHostAddress());
                b.setPort(nodes[(i + j) % REGION_SERVERS].getPort());
                b.setStartCode(-1);
                favoredNodes.add(b.build());
            }
            server.updateRegionFavoredNodesMapping(encodedRegionName, favoredNodes);
        }
    }
    // get multiple files for each region.
    for (int i = 0; i < FLUSHES; i++) {
        TEST_UTIL.loadTable(table, COLUMN_FAMILY, false);
        TEST_UTIL.flush();
    }
    // they are consistent with the favored nodes for that region.
    for (int i = 0; i < REGION_SERVERS; i++) {
        HRegionServer server = TEST_UTIL.getHBaseCluster().getRegionServer(i);
        List<Region> regions = server.getOnlineRegions(TABLE_NAME);
        for (Region region : regions) {
            List<String> files = region.getStoreFileList(new byte[][] { COLUMN_FAMILY });
            for (String file : files) {
                FileStatus status = TEST_UTIL.getDFSCluster().getFileSystem().getFileStatus(new Path(new URI(file).getPath()));
                BlockLocation[] lbks = ((DistributedFileSystem) TEST_UTIL.getDFSCluster().getFileSystem()).getFileBlockLocations(status, 0, Long.MAX_VALUE);
                for (BlockLocation lbk : lbks) {
                    locations: for (String info : lbk.getNames()) {
                        for (int j = 0; j < FAVORED_NODES_NUM; j++) {
                            if (info.equals(nodeNames[(i + j) % REGION_SERVERS])) {
                                continue locations;
                            }
                        }
                        // This block was at a location that was not a favored location.
                        fail("Block location " + info + " not a favored node");
                    }
                }
            }
        }
    }
}