Examples with BlockLocation alluxio.wire.BlockLocation used on opensource projects

Example 6 with BlockLocation

use of alluxio.wire.BlockLocation in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method getInStreamInAlluxioWhenCreateStreamIsFailed.

@Test
public void getInStreamInAlluxioWhenCreateStreamIsFailed() throws Exception {
    int workerCount = 5;
    boolean persisted = false;
    int[] blockLocations = new int[] { 2, 3, 4 };
    Map<Integer, Long> failedWorkers = ImmutableMap.of(0, 3L, 1, 1L, 3, 2L);
    int expectedWorker = 2;
    WorkerNetAddress[] workers = new WorkerNetAddress[workerCount];
    for (int i = 0; i < workers.length - 1; i++) {
        workers[i] = new WorkerNetAddress().setHost(String.format("worker-%d", i));
    }
    workers[workers.length - 1] = new WorkerNetAddress().setHost(WORKER_HOSTNAME_LOCAL);
    when(mContext.acquireBlockWorkerClient(WORKER_NET_ADDRESS_LOCAL)).thenThrow(new UnavailableException("failed to connect to " + WORKER_NET_ADDRESS_LOCAL.getHost()));
    BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays.stream(blockLocations).mapToObj(x -> new BlockLocation().setWorkerAddress(workers[x])).collect(Collectors.toList()));
    URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(persisted).setBlockIds(Collections.singletonList(BLOCK_ID)).setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info))));
    BlockLocationPolicy mockPolicy = mock(BlockLocationPolicy.class);
    when(mockPolicy.getWorker(any())).thenAnswer(arg -> arg.getArgument(0, GetWorkerOptions.class).getBlockWorkerInfos().iterator().next().getNetAddress());
    InStreamOptions options = new InStreamOptions(dummyStatus, FileSystemOptions.openFileDefaults(sConf), sConf);
    options.setUfsReadLocationPolicy(mockPolicy);
    when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info);
    when(mContext.getCachedWorkers()).thenReturn(Arrays.stream(workers).map(x -> new BlockWorkerInfo(x, -1, -1)).collect((Collectors.toList())));
    Map<WorkerNetAddress, Long> failedWorkerAddresses = failedWorkers.entrySet().stream().map(x -> new AbstractMap.SimpleImmutableEntry<>(workers[x.getKey()], x.getValue())).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
    BlockInStream inStream = null;
    int i = 2;
    while (i-- > 0) {
        try {
            inStream = mBlockStore.getInStream(BLOCK_ID, options, failedWorkerAddresses);
        } catch (Exception e) {
        // do nothing
        }
    }
    assertEquals(workers[expectedWorker], inStream.getAddress());
}

Example 7 with BlockLocation

use of alluxio.wire.BlockLocation in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method getInStreamProcessLocal.

@Test
public void getInStreamProcessLocal() throws Exception {
    WorkerNetAddress remote = new WorkerNetAddress().setHost("remote");
    WorkerNetAddress local = new WorkerNetAddress().setHost(WORKER_HOSTNAME_LOCAL);
    BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays.asList(new BlockLocation().setWorkerAddress(remote), new BlockLocation().setWorkerAddress(local)));
    when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info);
    when(mContext.hasProcessLocalWorker()).thenReturn(true);
    BlockWorker blockWorker = Mockito.mock(BlockWorker.class);
    when(mContext.getProcessLocalWorker()).thenReturn(blockWorker);
    BlockInStream stream = mBlockStore.getInStream(BLOCK_ID, new InStreamOptions(new URIStatus(new FileInfo().setBlockIds(Lists.newArrayList(BLOCK_ID))), sConf));
    assertEquals(local, stream.getAddress());
    assertEquals(BlockWorkerDataReader.Factory.class.getName(), stream.getDataReaderFactory().getClass().getName());
}

Example 8 with BlockLocation

use of alluxio.wire.BlockLocation in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method testGetInStreamFallback.

private void testGetInStreamFallback(int workerCount, boolean isPersisted, int[] blockLocations, Map<Integer, Long> failedWorkers, int expectedWorker) throws Exception {
    WorkerNetAddress[] workers = new WorkerNetAddress[workerCount];
    Arrays.setAll(workers, i -> new WorkerNetAddress().setHost(String.format("worker-%d", i)));
    BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays.stream(blockLocations).mapToObj(x -> new BlockLocation().setWorkerAddress(workers[x])).collect(Collectors.toList()));
    URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(isPersisted).setBlockIds(Collections.singletonList(BLOCK_ID)).setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info))));
    BlockLocationPolicy mockPolicy = mock(BlockLocationPolicy.class);
    when(mockPolicy.getWorker(any())).thenAnswer(arg -> arg.getArgument(0, GetWorkerOptions.class).getBlockWorkerInfos().iterator().next().getNetAddress());
    InStreamOptions options = new InStreamOptions(dummyStatus, FileSystemOptions.openFileDefaults(sConf), sConf);
    options.setUfsReadLocationPolicy(mockPolicy);
    when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info);
    when(mContext.getCachedWorkers()).thenReturn(Arrays.stream(workers).map(x -> new BlockWorkerInfo(x, -1, -1)).collect((Collectors.toList())));
    Map<WorkerNetAddress, Long> failedWorkerAddresses = failedWorkers.entrySet().stream().map(x -> new AbstractMap.SimpleImmutableEntry<>(workers[x.getKey()], x.getValue())).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
    BlockInStream inStream = mBlockStore.getInStream(BLOCK_ID, options, failedWorkerAddresses);
    assertEquals(workers[expectedWorker], inStream.getAddress());
}

Example 9 with BlockLocation

use of alluxio.wire.BlockLocation in project alluxio by Alluxio.

the class BaseFileSystem method getBlockLocations.

@Override
public List<BlockLocationInfo> getBlockLocations(AlluxioURI path) throws IOException, AlluxioException {
    List<BlockLocationInfo> blockLocations = new ArrayList<>();
    // Don't need to checkUri here because we call other client operations
    List<FileBlockInfo> blocks = getStatus(path).getFileBlockInfos();
    for (FileBlockInfo fileBlockInfo : blocks) {
        // add the existing in-Alluxio block locations
        List<WorkerNetAddress> locations = fileBlockInfo.getBlockInfo().getLocations().stream().map(BlockLocation::getWorkerAddress).collect(toList());
        if (locations.isEmpty()) {
            // No in-Alluxio location
            if (!fileBlockInfo.getUfsLocations().isEmpty()) {
                // Case 1: Fallback to use under file system locations with co-located workers.
                // This maps UFS locations to a worker which is co-located.
                Map<String, WorkerNetAddress> finalWorkerHosts = getHostWorkerMap();
                locations = fileBlockInfo.getUfsLocations().stream().map(location -> finalWorkerHosts.get(HostAndPort.fromString(location).getHost())).filter(Objects::nonNull).collect(toList());
            }
            if (locations.isEmpty() && mFsContext.getPathConf(path).getBoolean(PropertyKey.USER_UFS_BLOCK_LOCATION_ALL_FALLBACK_ENABLED)) {
                // Case 2: Fallback to add all workers to locations so some apps (Impala) won't panic.
                locations.addAll(getHostWorkerMap().values());
                Collections.shuffle(locations);
            }
        }
        blockLocations.add(new BlockLocationInfo(fileBlockInfo, locations));
    }
    return blockLocations;
}

Example 10 with BlockLocation

use of alluxio.wire.BlockLocation in project alluxio by Alluxio.

the class GrpcUtils method fromProto.

/**
 * Converts a proto type to a wire type.
 *
 * @param blockPLocation the proto type to convert
 * @return the converted wire type
 */
public static BlockLocation fromProto(alluxio.grpc.BlockLocation blockPLocation) {
    BlockLocation blockLocation = new BlockLocation();
    blockLocation.setWorkerId(blockPLocation.getWorkerId());
    blockLocation.setWorkerAddress(fromProto(blockPLocation.getWorkerAddress()));
    blockLocation.setTierAlias(blockPLocation.getTierAlias());
    blockLocation.setMediumType(blockPLocation.getMediumType());
    return blockLocation;
}