Examples with WorkerNetAddress alluxio.wire.WorkerNetAddress used on opensource projects

Example 96 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class CapacityCommandTest method prepareLongWorkerNameInfoList.

/**
 * @return long worker name info list to test
 */
private List<WorkerInfo> prepareLongWorkerNameInfoList() {
    List<WorkerInfo> infoList = new ArrayList<>();
    Map<String, Long> capacityBytesOnTiersOne = new HashMap<>();
    capacityBytesOnTiersOne.put(Constants.MEDIUM_MEM, 600000000L);
    capacityBytesOnTiersOne.put(Constants.MEDIUM_HDD, 200000000L);
    Map<String, Long> usedBytesOnTiersOne = new HashMap<>();
    usedBytesOnTiersOne.put(Constants.MEDIUM_MEM, 300000000L);
    usedBytesOnTiersOne.put(Constants.MEDIUM_HDD, 200000000L);
    WorkerInfo firstInfo = new WorkerInfo().setAddress(new WorkerNetAddress().setHost("org.alluxio.long.host1")).setCapacityBytes(1200000000L).setCapacityBytesOnTiers(capacityBytesOnTiersOne).setId(1).setLastContactSec(6211).setStartTimeMs(1529222699127L).setState("In Service").setUsedBytes(1000000000L).setUsedBytesOnTiers(usedBytesOnTiersOne);
    Map<String, Long> capacityBytesOnTiersSec = new HashMap<>();
    capacityBytesOnTiersSec.put(Constants.MEDIUM_MEM, 600000000L);
    capacityBytesOnTiersSec.put(Constants.MEDIUM_SSD, 600000000L);
    Map<String, Long> usedBytesOnTiersSec = new HashMap<>();
    usedBytesOnTiersSec.put(Constants.MEDIUM_MEM, 20000000L);
    usedBytesOnTiersSec.put(Constants.MEDIUM_SSD, 30000000L);
    WorkerInfo secondInfo = new WorkerInfo().setAddress(new WorkerNetAddress().setHost("org.apache.hdp1")).setCapacityBytes(2000000000L).setCapacityBytesOnTiers(capacityBytesOnTiersSec).setId(2).setLastContactSec(681).setStartTimeMs(1529222699127L).setState("In Service").setUsedBytes(100000000L).setUsedBytesOnTiers(usedBytesOnTiersSec);
    infoList.add(firstInfo);
    infoList.add(secondInfo);
    return infoList;
}

Example 97 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class AlluxioBlockStore method getInStream.

/**
 * {@link #getInStream(long, InStreamOptions, Map)}.
 *
 * @param info the block info
 * @param options the options associated with the read request
 * @param failedWorkers the map of workers address to most recent failure time
 * @return a stream which reads from the beginning of the block
 */
public BlockInStream getInStream(BlockInfo info, InStreamOptions options, Map<WorkerNetAddress, Long> failedWorkers) throws IOException {
    Pair<WorkerNetAddress, BlockInStreamSource> dataSourceAndType = getDataSourceAndType(info, options.getStatus(), options.getUfsReadLocationPolicy(), failedWorkers);
    WorkerNetAddress dataSource = dataSourceAndType.getFirst();
    BlockInStreamSource dataSourceType = dataSourceAndType.getSecond();
    try {
        return BlockInStream.create(mContext, info, dataSource, dataSourceType, options);
    } catch (UnavailableException e) {
        // When BlockInStream created failed, it will update the passed-in failedWorkers
        // to attempt to avoid reading from this failed worker in next try.
        failedWorkers.put(dataSource, System.currentTimeMillis());
        throw e;
    }
}

Example 98 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class AlluxioBlockStore method getDataSourceAndType.

/**
 * Gets the data source and type of data source of a block. This method is primarily responsible
 * for determining the data source and type of data source. It takes a map of failed workers and
 * their most recently failed time and tries to update it when BlockInStream created failed,
 * attempting to avoid reading from a recently failed worker.
 *
 * @param info the info of the block to read
 * @param status the URIStatus associated with the read request
 * @param policy the policy determining the Alluxio worker location
 * @param failedWorkers the map of workers address to most recent failure time
 * @return the data source and type of data source of the block
 */
public Pair<WorkerNetAddress, BlockInStreamSource> getDataSourceAndType(BlockInfo info, URIStatus status, BlockLocationPolicy policy, Map<WorkerNetAddress, Long> failedWorkers) throws IOException {
    List<BlockLocation> locations = info.getLocations();
    List<BlockWorkerInfo> blockWorkerInfo = Collections.EMPTY_LIST;
    // Initial target workers to read the block given the block locations.
    Set<WorkerNetAddress> workerPool;
    // Note that, it is possible that the blocks have been written as UFS blocks
    if (status.isPersisted() || status.getPersistenceState().equals("TO_BE_PERSISTED")) {
        blockWorkerInfo = mContext.getCachedWorkers();
        if (blockWorkerInfo.isEmpty()) {
            throw new UnavailableException(ExceptionMessage.NO_WORKER_AVAILABLE.getMessage());
        }
        workerPool = blockWorkerInfo.stream().map(BlockWorkerInfo::getNetAddress).collect(toSet());
    } else {
        if (locations.isEmpty()) {
            blockWorkerInfo = mContext.getCachedWorkers();
            if (blockWorkerInfo.isEmpty()) {
                throw new UnavailableException(ExceptionMessage.NO_WORKER_AVAILABLE.getMessage());
            }
            throw new UnavailableException(ExceptionMessage.BLOCK_UNAVAILABLE.getMessage(info.getBlockId()));
        }
        workerPool = locations.stream().map(BlockLocation::getWorkerAddress).collect(toSet());
    }
    // Workers to read the block, after considering failed workers.
    Set<WorkerNetAddress> workers = handleFailedWorkers(workerPool, failedWorkers);
    // TODO(calvin, jianjian): Consider containing these two variables in one object
    BlockInStreamSource dataSourceType = null;
    WorkerNetAddress dataSource = null;
    locations = locations.stream().filter(location -> workers.contains(location.getWorkerAddress())).collect(toList());
    // First try to read data from Alluxio
    if (!locations.isEmpty()) {
        // TODO(calvin): Get location via a policy
        List<WorkerNetAddress> tieredLocations = locations.stream().map(location -> location.getWorkerAddress()).collect(toList());
        Collections.shuffle(tieredLocations);
        Optional<Pair<WorkerNetAddress, Boolean>> nearest = BlockLocationUtils.nearest(mTieredIdentity, tieredLocations, mContext.getClusterConf());
        if (nearest.isPresent()) {
            dataSource = nearest.get().getFirst();
            dataSourceType = nearest.get().getSecond() ? mContext.hasProcessLocalWorker() ? BlockInStreamSource.PROCESS_LOCAL : BlockInStreamSource.NODE_LOCAL : BlockInStreamSource.REMOTE;
        }
    }
    // Can't get data from Alluxio, get it from the UFS instead
    if (dataSource == null) {
        dataSourceType = BlockInStreamSource.UFS;
        Preconditions.checkNotNull(policy, PreconditionMessage.UFS_READ_LOCATION_POLICY_UNSPECIFIED);
        blockWorkerInfo = blockWorkerInfo.stream().filter(workerInfo -> workers.contains(workerInfo.getNetAddress())).collect(toList());
        GetWorkerOptions getWorkerOptions = GetWorkerOptions.defaults().setBlockInfo(new BlockInfo().setBlockId(info.getBlockId()).setLength(info.getLength()).setLocations(locations)).setBlockWorkerInfos(blockWorkerInfo);
        dataSource = policy.getWorker(getWorkerOptions);
        if (dataSource != null) {
            if (mContext.hasProcessLocalWorker() && dataSource.equals(mContext.getNodeLocalWorker())) {
                dataSourceType = BlockInStreamSource.PROCESS_LOCAL;
                LOG.debug("Create BlockInStream to read data from UFS through process local worker {}", dataSource);
            } else {
                LOG.debug("Create BlockInStream to read data from UFS through worker {} " + "(client embedded in local worker process: {}," + "client co-located with worker in different processes: {}, " + "local worker address: {})", dataSource, mContext.hasProcessLocalWorker(), mContext.hasNodeLocalWorker(), mContext.hasNodeLocalWorker() ? mContext.getNodeLocalWorker() : "N/A");
            }
        }
    }
    if (dataSource == null) {
        throw new UnavailableException(ExceptionMessage.NO_WORKER_AVAILABLE.getMessage());
    }
    return new Pair<>(dataSource, dataSourceType);
}

Example 99 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class DeterministicHashPolicy method getWorker.

@Override
public WorkerNetAddress getWorker(GetWorkerOptions options) {
    List<BlockWorkerInfo> workerInfos = Lists.newArrayList(options.getBlockWorkerInfos());
    workerInfos.sort((o1, o2) -> o1.getNetAddress().toString().compareToIgnoreCase(o2.getNetAddress().toString()));
    HashMap<WorkerNetAddress, BlockWorkerInfo> blockWorkerInfoMap = new HashMap<>();
    for (BlockWorkerInfo workerInfo : options.getBlockWorkerInfos()) {
        blockWorkerInfoMap.put(workerInfo.getNetAddress(), workerInfo);
    }
    List<WorkerNetAddress> workers = new ArrayList<>();
    // Try the next one if the worker mapped from the blockId doesn't work until all the workers
    // are examined.
    int hv = Math.abs(mHashFunc.newHasher().putLong(options.getBlockInfo().getBlockId()).hash().asInt());
    int index = hv % workerInfos.size();
    for (BlockWorkerInfo blockWorkerInfoUnused : workerInfos) {
        WorkerNetAddress candidate = workerInfos.get(index).getNetAddress();
        BlockWorkerInfo workerInfo = blockWorkerInfoMap.get(candidate);
        if (workerInfo != null && workerInfo.getCapacityBytes() >= options.getBlockInfo().getLength()) {
            workers.add(candidate);
            if (workers.size() >= mShards) {
                break;
            }
        }
        index = (index + 1) % workerInfos.size();
    }
    return workers.isEmpty() ? null : workers.get(mRandom.nextInt(workers.size()));
}

Example 100 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class RoundRobinPolicy method getWorker.

/**
 * The policy uses the first fetch of worker info list as the base, and visits each of them in a
 * round-robin manner in the subsequent calls. The policy doesn't assume the list of worker info
 * in the subsequent calls has the same order from the first, and it will skip the workers that
 * are no longer active.
 *
 * Returns null if no worker can be found.
 *
 * @param options options
 * @return the address of the worker to write to
 */
@Override
@Nullable
public WorkerNetAddress getWorker(GetWorkerOptions options) {
    Set<WorkerNetAddress> eligibleAddresses = new HashSet<>();
    for (BlockWorkerInfo info : options.getBlockWorkerInfos()) {
        eligibleAddresses.add(info.getNetAddress());
    }
    WorkerNetAddress address = mBlockLocationCache.get(options.getBlockInfo().getBlockId());
    if (address != null && eligibleAddresses.contains(address)) {
        return address;
    } else {
        address = null;
    }
    if (!mInitialized) {
        mWorkerInfoList = Lists.newArrayList(options.getBlockWorkerInfos());
        Collections.shuffle(mWorkerInfoList);
        mIndex = 0;
        mInitialized = true;
    }
    // at most try all the workers
    for (int i = 0; i < mWorkerInfoList.size(); i++) {
        WorkerNetAddress candidate = mWorkerInfoList.get(mIndex).getNetAddress();
        BlockWorkerInfo workerInfo = findBlockWorkerInfo(options.getBlockWorkerInfos(), candidate);
        mIndex = (mIndex + 1) % mWorkerInfoList.size();
        if (workerInfo != null && workerInfo.getCapacityBytes() >= options.getBlockInfo().getLength() && eligibleAddresses.contains(candidate)) {
            address = candidate;
            break;
        }
    }
    mBlockLocationCache.put(options.getBlockInfo().getBlockId(), address);
    return address;
}