Examples with WorkerInfo alluxio.wire.WorkerInfo used on opensource projects

Example 41 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class ReplicateDefinition method selectExecutors.

@Override
public Set<Pair<WorkerInfo, SerializableVoid>> selectExecutors(ReplicateConfig config, List<WorkerInfo> jobWorkerInfoList, SelectExecutorsContext context) throws Exception {
    Preconditions.checkArgument(!jobWorkerInfoList.isEmpty(), "No worker is available");
    long blockId = config.getBlockId();
    int numReplicas = config.getReplicas();
    Preconditions.checkArgument(numReplicas > 0);
    AlluxioBlockStore blockStore = AlluxioBlockStore.create(context.getFsContext());
    BlockInfo blockInfo = blockStore.getInfo(blockId);
    Set<String> hosts = new HashSet<>();
    for (BlockLocation blockLocation : blockInfo.getLocations()) {
        hosts.add(blockLocation.getWorkerAddress().getHost());
    }
    Set<Pair<WorkerInfo, SerializableVoid>> result = Sets.newHashSet();
    Collections.shuffle(jobWorkerInfoList);
    for (WorkerInfo workerInfo : jobWorkerInfoList) {
        // Select job workers that don't have this block locally to replicate
        if (!hosts.contains(workerInfo.getAddress().getHost())) {
            result.add(new Pair<>(workerInfo, null));
            if (result.size() >= numReplicas) {
                break;
            }
        }
    }
    return result;
}

Example 42 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class BatchedJobDefinition method selectExecutors.

@Override
public Set<Pair<WorkerInfo, BatchedJobTask>> selectExecutors(BatchedJobConfig config, List<WorkerInfo> jobWorkerInfoList, SelectExecutorsContext context) throws Exception {
    // get job type and config
    String jobType = config.getJobType();
    PlanDefinition plan = JobDefinitionFactory.create(jobType);
    // convert map to config
    final ObjectMapper mapper = new ObjectMapper();
    Class<?> jobConfigClass = plan.getJobConfigClass();
    Set<Pair<WorkerInfo, BatchedJobTask>> allTasks = Sets.newHashSet();
    for (Map<String, String> configMap : config.getJobConfigs()) {
        JobConfig jobConfig = (JobConfig) mapper.convertValue(configMap, jobConfigClass);
        Set<Pair<WorkerInfo, Serializable>> tasks = plan.selectExecutors(jobConfig, jobWorkerInfoList, context);
        for (Pair<WorkerInfo, Serializable> task : tasks) {
            BatchedJobTask batchedTask = new BatchedJobTask(jobConfig, task.getSecond());
            allTasks.add(new Pair<>(task.getFirst(), batchedTask));
        }
    }
    return allTasks;
}

Example 43 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class DefaultBlockMaster method generateBlockInfo.

/**
 * Generates block info, including worker locations, for a block id.
 * This requires no locks on the {@link MasterWorkerInfo} because it is only reading
 * final fields.
 *
 * @param blockId a block id
 * @return optional block info, empty if the block does not exist
 */
private Optional<BlockInfo> generateBlockInfo(long blockId) throws UnavailableException {
    if (mSafeModeManager.isInSafeMode()) {
        throw new UnavailableException(ExceptionMessage.MASTER_IN_SAFEMODE.getMessage());
    }
    BlockMeta block;
    List<BlockLocation> blockLocations;
    try (LockResource r = lockBlock(blockId)) {
        Optional<BlockMeta> blockOpt = mBlockStore.getBlock(blockId);
        if (!blockOpt.isPresent()) {
            return Optional.empty();
        }
        block = blockOpt.get();
        blockLocations = new ArrayList<>(mBlockStore.getLocations(blockId));
    }
    // Sort the block locations by their alias ordinal in the master storage tier mapping
    Collections.sort(blockLocations, Comparator.comparingInt(o -> mGlobalStorageTierAssoc.getOrdinal(o.getTier())));
    List<alluxio.wire.BlockLocation> locations = new ArrayList<>();
    for (BlockLocation location : blockLocations) {
        MasterWorkerInfo workerInfo = mWorkers.getFirstByField(ID_INDEX, location.getWorkerId());
        if (workerInfo != null) {
            // worker metadata is intentionally not locked here because:
            // - it would be an incorrect order (correct order is lock worker first, then block)
            // - only uses getters of final variables
            locations.add(new alluxio.wire.BlockLocation().setWorkerId(location.getWorkerId()).setWorkerAddress(workerInfo.getWorkerAddress()).setTierAlias(location.getTier()).setMediumType(location.getMediumType()));
        }
    }
    return Optional.of(new BlockInfo().setBlockId(blockId).setLength(block.getLength()).setLocations(locations));
}

Example 44 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class DefaultBlockMaster method getLostWorkersInfoList.

@Override
public List<WorkerInfo> getLostWorkersInfoList() throws UnavailableException {
    if (mSafeModeManager.isInSafeMode()) {
        throw new UnavailableException(ExceptionMessage.MASTER_IN_SAFEMODE.getMessage());
    }
    List<WorkerInfo> workerInfoList = new ArrayList<>(mLostWorkers.size());
    for (MasterWorkerInfo worker : mLostWorkers) {
        // extractWorkerInfo handles the locking internally
        workerInfoList.add(extractWorkerInfo(worker, null, false));
    }
    Collections.sort(workerInfoList, new WorkerInfo.LastContactSecComparator());
    return workerInfoList;
}

Example 45 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class DefaultBlockMaster method getWorkerReport.

@Override
public List<WorkerInfo> getWorkerReport(GetWorkerReportOptions options) throws UnavailableException, InvalidArgumentException {
    if (mSafeModeManager.isInSafeMode()) {
        throw new UnavailableException(ExceptionMessage.MASTER_IN_SAFEMODE.getMessage());
    }
    Set<MasterWorkerInfo> selectedLiveWorkers = new HashSet<>();
    Set<MasterWorkerInfo> selectedLostWorkers = new HashSet<>();
    WorkerRange workerRange = options.getWorkerRange();
    switch(workerRange) {
        case ALL:
            selectedLiveWorkers.addAll(mWorkers);
            selectedLostWorkers.addAll(mLostWorkers);
            break;
        case LIVE:
            selectedLiveWorkers.addAll(mWorkers);
            break;
        case LOST:
            selectedLostWorkers.addAll(mLostWorkers);
            break;
        case SPECIFIED:
            Set<String> addresses = options.getAddresses();
            Set<String> workerNames = new HashSet<>();
            selectedLiveWorkers = selectInfoByAddress(addresses, mWorkers, workerNames);
            selectedLostWorkers = selectInfoByAddress(addresses, mLostWorkers, workerNames);
            if (!addresses.isEmpty()) {
                String info = String.format("Unrecognized worker names: %s%n" + "Supported worker names: %s%n", addresses.toString(), workerNames.toString());
                throw new InvalidArgumentException(info);
            }
            break;
        default:
            throw new InvalidArgumentException("Unrecognized worker range: " + workerRange);
    }
    List<WorkerInfo> workerInfoList = new ArrayList<>(selectedLiveWorkers.size() + selectedLostWorkers.size());
    for (MasterWorkerInfo worker : selectedLiveWorkers) {
        // extractWorkerInfo handles the locking internally
        workerInfoList.add(extractWorkerInfo(worker, options.getFieldRange(), true));
    }
    for (MasterWorkerInfo worker : selectedLostWorkers) {
        // extractWorkerInfo handles the locking internally
        workerInfoList.add(extractWorkerInfo(worker, options.getFieldRange(), false));
    }
    return workerInfoList;
}