Examples with BlockLocation alluxio.wire.BlockLocation used on opensource projects

Example 31 with BlockLocation

use of alluxio.wire.BlockLocation in project alluxio by Alluxio.

the class EvictDefinitionTest method selectExecutorsBothWorkers.

@Test
public void selectExecutorsBothWorkers() throws Exception {
    Set<Pair<WorkerInfo, SerializableVoid>> result = selectExecutorsTestHelper(Lists.newArrayList(new BlockLocation().setWorkerAddress(ADDRESS_1), new BlockLocation().setWorkerAddress(ADDRESS_2)), 3, Lists.newArrayList(WORKER_INFO_1, WORKER_INFO_2, WORKER_INFO_3));
    Set<Pair<WorkerInfo, SerializableVoid>> expected = Sets.newHashSet();
    expected.add(new Pair<>(WORKER_INFO_1, null));
    expected.add(new Pair<>(WORKER_INFO_2, null));
    // Expect both workers having this block should be selected
    Assert.assertEquals(expected, result);
}

Example 32 with BlockLocation

use of alluxio.wire.BlockLocation in project alluxio by Alluxio.

the class ReplicateDefinitionTest method selectExecutorsInsufficientWorkerValid.

@Test
public void selectExecutorsInsufficientWorkerValid() throws Exception {
    mTestBlockInfo.setLocations(Lists.newArrayList(new BlockLocation().setWorkerAddress(ADDRESS_1)));
    Set<Pair<WorkerInfo, SerializableVoid>> result = selectExecutorsTestHelper(2, Lists.newArrayList(WORKER_INFO_1, WORKER_INFO_2));
    Set<Pair<WorkerInfo, SerializableVoid>> expected = Sets.newHashSet();
    expected.add(new Pair<>(WORKER_INFO_2, null));
    // select the only worker left though more copies are requested
    assertEquals(expected, result);
}

Example 33 with BlockLocation

use of alluxio.wire.BlockLocation in project alluxio by Alluxio.

the class ReplicateDefinitionTest method runTaskReplicateTestHelper.

/**
 * Helper function to run a replicate task.
 *
 * @param blockWorkers available block workers
 * @param mockInStream mock blockInStream returned by the Block Store
 * @param mockOutStream mock blockOutStream returned by the Block Store
 */
private void runTaskReplicateTestHelper(List<BlockWorkerInfo> blockWorkers, BlockInStream mockInStream, BlockOutStream mockOutStream) throws Exception {
    when(mMockFileSystem.getStatus(any(AlluxioURI.class))).thenReturn(mTestStatus);
    when(mMockFileSystemContext.getCachedWorkers()).thenReturn(blockWorkers);
    when(mMockBlockStore.getInStream(anyLong(), any(InStreamOptions.class))).thenReturn(mockInStream);
    when(mMockBlockStore.getInStream(any(BlockInfo.class), any(InStreamOptions.class), any(Map.class))).thenReturn(mockInStream);
    PowerMockito.mockStatic(BlockInStream.class);
    when(BlockInStream.create(any(FileSystemContext.class), any(BlockInfo.class), any(WorkerNetAddress.class), any(BlockInStreamSource.class), any(InStreamOptions.class))).thenReturn(mockInStream);
    when(mMockBlockStore.getOutStream(eq(TEST_BLOCK_ID), eq(TEST_BLOCK_SIZE), eq(LOCAL_ADDRESS), any(OutStreamOptions.class))).thenReturn(mockOutStream);
    when(mMockBlockStore.getInfo(TEST_BLOCK_ID)).thenReturn(mTestBlockInfo.setLocations(Lists.newArrayList(new BlockLocation().setWorkerAddress(ADDRESS_1))));
    PowerMockito.mockStatic(AlluxioBlockStore.class);
    when(AlluxioBlockStore.create(any(FileSystemContext.class))).thenReturn(mMockBlockStore);
    ReplicateConfig config = new ReplicateConfig(TEST_PATH, TEST_BLOCK_ID, 1);
    ReplicateDefinition definition = new ReplicateDefinition();
    definition.runTask(config, null, new RunTaskContext(1, 1, mMockJobServerContext));
}

Example 34 with BlockLocation

use of alluxio.wire.BlockLocation in project alluxio by Alluxio.

the class AlluxioBlockStore method getDataSourceAndType.

/**
 * Gets the data source and type of data source of a block. This method is primarily responsible
 * for determining the data source and type of data source. It takes a map of failed workers and
 * their most recently failed time and tries to update it when BlockInStream created failed,
 * attempting to avoid reading from a recently failed worker.
 *
 * @param info the info of the block to read
 * @param status the URIStatus associated with the read request
 * @param policy the policy determining the Alluxio worker location
 * @param failedWorkers the map of workers address to most recent failure time
 * @return the data source and type of data source of the block
 */
public Pair<WorkerNetAddress, BlockInStreamSource> getDataSourceAndType(BlockInfo info, URIStatus status, BlockLocationPolicy policy, Map<WorkerNetAddress, Long> failedWorkers) throws IOException {
    List<BlockLocation> locations = info.getLocations();
    List<BlockWorkerInfo> blockWorkerInfo = Collections.EMPTY_LIST;
    // Initial target workers to read the block given the block locations.
    Set<WorkerNetAddress> workerPool;
    // Note that, it is possible that the blocks have been written as UFS blocks
    if (status.isPersisted() || status.getPersistenceState().equals("TO_BE_PERSISTED")) {
        blockWorkerInfo = mContext.getCachedWorkers();
        if (blockWorkerInfo.isEmpty()) {
            throw new UnavailableException(ExceptionMessage.NO_WORKER_AVAILABLE.getMessage());
        }
        workerPool = blockWorkerInfo.stream().map(BlockWorkerInfo::getNetAddress).collect(toSet());
    } else {
        if (locations.isEmpty()) {
            blockWorkerInfo = mContext.getCachedWorkers();
            if (blockWorkerInfo.isEmpty()) {
                throw new UnavailableException(ExceptionMessage.NO_WORKER_AVAILABLE.getMessage());
            }
            throw new UnavailableException(ExceptionMessage.BLOCK_UNAVAILABLE.getMessage(info.getBlockId()));
        }
        workerPool = locations.stream().map(BlockLocation::getWorkerAddress).collect(toSet());
    }
    // Workers to read the block, after considering failed workers.
    Set<WorkerNetAddress> workers = handleFailedWorkers(workerPool, failedWorkers);
    // TODO(calvin, jianjian): Consider containing these two variables in one object
    BlockInStreamSource dataSourceType = null;
    WorkerNetAddress dataSource = null;
    locations = locations.stream().filter(location -> workers.contains(location.getWorkerAddress())).collect(toList());
    // First try to read data from Alluxio
    if (!locations.isEmpty()) {
        // TODO(calvin): Get location via a policy
        List<WorkerNetAddress> tieredLocations = locations.stream().map(location -> location.getWorkerAddress()).collect(toList());
        Collections.shuffle(tieredLocations);
        Optional<Pair<WorkerNetAddress, Boolean>> nearest = BlockLocationUtils.nearest(mTieredIdentity, tieredLocations, mContext.getClusterConf());
        if (nearest.isPresent()) {
            dataSource = nearest.get().getFirst();
            dataSourceType = nearest.get().getSecond() ? mContext.hasProcessLocalWorker() ? BlockInStreamSource.PROCESS_LOCAL : BlockInStreamSource.NODE_LOCAL : BlockInStreamSource.REMOTE;
        }
    }
    // Can't get data from Alluxio, get it from the UFS instead
    if (dataSource == null) {
        dataSourceType = BlockInStreamSource.UFS;
        Preconditions.checkNotNull(policy, PreconditionMessage.UFS_READ_LOCATION_POLICY_UNSPECIFIED);
        blockWorkerInfo = blockWorkerInfo.stream().filter(workerInfo -> workers.contains(workerInfo.getNetAddress())).collect(toList());
        GetWorkerOptions getWorkerOptions = GetWorkerOptions.defaults().setBlockInfo(new BlockInfo().setBlockId(info.getBlockId()).setLength(info.getLength()).setLocations(locations)).setBlockWorkerInfos(blockWorkerInfo);
        dataSource = policy.getWorker(getWorkerOptions);
        if (dataSource != null) {
            if (mContext.hasProcessLocalWorker() && dataSource.equals(mContext.getNodeLocalWorker())) {
                dataSourceType = BlockInStreamSource.PROCESS_LOCAL;
                LOG.debug("Create BlockInStream to read data from UFS through process local worker {}", dataSource);
            } else {
                LOG.debug("Create BlockInStream to read data from UFS through worker {} " + "(client embedded in local worker process: {}," + "client co-located with worker in different processes: {}, " + "local worker address: {})", dataSource, mContext.hasProcessLocalWorker(), mContext.hasNodeLocalWorker(), mContext.hasNodeLocalWorker() ? mContext.getNodeLocalWorker() : "N/A");
            }
        }
    }
    if (dataSource == null) {
        throw new UnavailableException(ExceptionMessage.NO_WORKER_AVAILABLE.getMessage());
    }
    return new Pair<>(dataSource, dataSourceType);
}

Example 35 with BlockLocation

use of alluxio.wire.BlockLocation in project alluxio by Alluxio.

the class UIFileBlockInfo method addLocations.

private void addLocations(FileBlockInfo fileBlockInfo) {
    Set<String> locations = new HashSet<>();
    // add alluxio locations
    for (BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) {
        locations.add(location.getWorkerAddress().getHost());
    }
    // add underFS locations
    for (String location : fileBlockInfo.getUfsLocations()) {
        locations.add(HostAndPort.fromString(location).getHost());
    }
    mLocations.addAll(locations);
}