Examples with BlockLocationInfo alluxio.wire.BlockLocationInfo used on opensource projects

Example 1 with BlockLocationInfo

use of alluxio.wire.BlockLocationInfo in project alluxio by Alluxio.

the class BaseFileSystem method getBlockLocations.

@Override
public List<BlockLocationInfo> getBlockLocations(AlluxioURI path) throws IOException, AlluxioException {
    List<BlockLocationInfo> blockLocations = new ArrayList<>();
    // Don't need to checkUri here because we call other client operations
    List<FileBlockInfo> blocks = getStatus(path).getFileBlockInfos();
    for (FileBlockInfo fileBlockInfo : blocks) {
        // add the existing in-Alluxio block locations
        List<WorkerNetAddress> locations = fileBlockInfo.getBlockInfo().getLocations().stream().map(BlockLocation::getWorkerAddress).collect(toList());
        if (locations.isEmpty()) {
            // No in-Alluxio location
            if (!fileBlockInfo.getUfsLocations().isEmpty()) {
                // Case 1: Fallback to use under file system locations with co-located workers.
                // This maps UFS locations to a worker which is co-located.
                Map<String, WorkerNetAddress> finalWorkerHosts = getHostWorkerMap();
                locations = fileBlockInfo.getUfsLocations().stream().map(location -> finalWorkerHosts.get(HostAndPort.fromString(location).getHost())).filter(Objects::nonNull).collect(toList());
            }
            if (locations.isEmpty() && mFsContext.getPathConf(path).getBoolean(PropertyKey.USER_UFS_BLOCK_LOCATION_ALL_FALLBACK_ENABLED)) {
                // Case 2: Fallback to add all workers to locations so some apps (Impala) won't panic.
                locations.addAll(getHostWorkerMap().values());
                Collections.shuffle(locations);
            }
        }
        blockLocations.add(new BlockLocationInfo(fileBlockInfo, locations));
    }
    return blockLocations;
}

Example 2 with BlockLocationInfo

use of alluxio.wire.BlockLocationInfo in project alluxio by Alluxio.

the class FileSystemIntegrationTest method getBlockLocations.

@Test
public void getBlockLocations() throws Exception {
    // Test not in alluxio
    AlluxioURI testFile = new AlluxioURI("/test1");
    FileSystemTestUtils.createByteFile(mFileSystem, testFile, CreateFilePOptions.newBuilder().setWriteType(WritePType.THROUGH).setBlockSizeBytes(4).build(), 100);
    List<BlockLocationInfo> locations = mFileSystem.getBlockLocations(testFile);
    assertEquals("should have 25 blocks", 25, locations.size());
    long lastOffset = -1;
    for (BlockLocationInfo location : locations) {
        assertEquals("block " + location.getBlockInfo() + " should have single worker", 1, location.getLocations().size());
        assertTrue("block " + location.getBlockInfo() + " should have offset larger than " + lastOffset, location.getBlockInfo().getOffset() > lastOffset);
        lastOffset = location.getBlockInfo().getOffset();
    }
    // Test in alluxio
    testFile = new AlluxioURI("/test2");
    FileSystemTestUtils.createByteFile(mFileSystem, testFile, CreateFilePOptions.newBuilder().setWriteType(WritePType.CACHE_THROUGH).setBlockSizeBytes(100).build(), 500);
    locations = mFileSystem.getBlockLocations(testFile);
    assertEquals("Should have 5 blocks", 5, locations.size());
    lastOffset = -1;
    for (BlockLocationInfo location : locations) {
        assertEquals("block " + location.getBlockInfo() + " should have single worker", 1, location.getLocations().size());
        assertTrue("block " + location.getBlockInfo() + " should have offset larger than " + lastOffset, location.getBlockInfo().getOffset() > lastOffset);
        lastOffset = location.getBlockInfo().getOffset();
    }
}

Example 3 with BlockLocationInfo

use of alluxio.wire.BlockLocationInfo in project alluxio by Alluxio.

the class AbstractFileSystem method getFileBlockLocations.

@Nullable
@Override
public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len) throws IOException {
    LOG.debug("getFileBlockLocations({}, {}, {})", (file == null) ? null : file.getPath().getName(), start, len);
    if (file == null) {
        LOG.debug("getFileBlockLocations({}, {}, {}) returned null", null, start, len);
        return null;
    }
    if (mStatistics != null) {
        mStatistics.incrementReadOps(1);
    }
    List<BlockLocation> blockLocations = new ArrayList<>();
    AlluxioURI path = getAlluxioPath(file.getPath());
    try {
        List<BlockLocationInfo> locations = mFileSystem.getBlockLocations(path);
        locations.forEach(location -> {
            FileBlockInfo info = location.getBlockInfo();
            List<WorkerNetAddress> workers = location.getLocations();
            long offset = location.getBlockInfo().getOffset();
            long end = offset + info.getBlockInfo().getLength();
            if (end >= start && offset <= start + len) {
                List<HostAndPort> addresses = workers.stream().map(worker -> HostAndPort.fromParts(worker.getHost(), worker.getDataPort())).collect(toList());
                String[] names = addresses.stream().map(HostAndPort::toString).toArray(String[]::new);
                String[] hosts = addresses.stream().map(HostAndPort::getHost).toArray(String[]::new);
                blockLocations.add(new BlockLocation(names, hosts, offset, info.getBlockInfo().getLength()));
            }
        });
        BlockLocation[] ret = blockLocations.toArray(new BlockLocation[blockLocations.size()]);
        if (LOG.isDebugEnabled()) {
            LOG.debug("getFileBlockLocations({}, {}, {}) returned {}", file.getPath().getName(), start, len, Arrays.toString(ret));
        }
        return ret;
    } catch (AlluxioException e) {
        throw new IOException(e);
    }
}