Examples with WorkerInfo alluxio.wire.WorkerInfo used on opensource projects

Example 56 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class PersistDefinitionTest method selectExecutorsMissingLocationTest.

@Test
public void selectExecutorsMissingLocationTest() throws Exception {
    AlluxioURI uri = new AlluxioURI("/test");
    PersistConfig config = new PersistConfig(uri.getPath(), -1, true, "");
    long blockId = 1;
    BlockInfo blockInfo = new BlockInfo().setBlockId(blockId);
    FileBlockInfo fileBlockInfo = new FileBlockInfo().setBlockInfo(blockInfo);
    FileInfo testFileInfo = new FileInfo();
    testFileInfo.setFileBlockInfos(Lists.newArrayList(fileBlockInfo));
    Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo));
    try {
        new PersistDefinition().selectExecutors(config, Lists.newArrayList(new WorkerInfo()), new SelectExecutorsContext(1, mJobServerContext));
    } catch (Exception e) {
        Assert.assertEquals("Block " + blockId + " does not exist", e.getMessage());
    }
}

Example 57 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class BatchedJobDefinitionTest method batchLoad.

@Test
public void batchLoad() throws Exception {
    int numBlocks = 2;
    int replication = 2;
    int batchSize = 2;
    HashSet<Map<String, String>> configs = Sets.newHashSet();
    for (int i = 0; i < batchSize; i++) {
        createFileWithNoLocations(TEST_URI + i, numBlocks);
        LoadConfig loadConfig = new LoadConfig(TEST_URI + i, replication, Collections.EMPTY_SET, Collections.EMPTY_SET, Collections.EMPTY_SET, Collections.EMPTY_SET, true);
        ObjectMapper oMapper = new ObjectMapper();
        Map<String, String> map = oMapper.convertValue(loadConfig, Map.class);
        configs.add(map);
    }
    BatchedJobConfig config = new BatchedJobConfig("Load", configs);
    Set<Pair<WorkerInfo, BatchedJobDefinition.BatchedJobTask>> assignments = new BatchedJobDefinition().selectExecutors(config, JOB_WORKERS, new SelectExecutorsContext(1, mJobServerContext));
    // Check that we are loading the right number of blocks.
    int totalBlockLoads = 0;
    for (Pair<WorkerInfo, BatchedJobDefinition.BatchedJobTask> assignment : assignments) {
        ArrayList<LoadTask> second = (ArrayList<LoadTask>) assignment.getSecond().getJobTaskArgs();
        totalBlockLoads += second.size();
    }
    Assert.assertEquals(numBlocks * replication * batchSize, totalBlockLoads);
}

Example 58 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class FileSystemContext method getWorkerAddresses.

/**
 * @return if there are any local workers, the returned list will ONLY contain the local workers,
 *         otherwise a list of all remote workers will be returned
 */
private List<WorkerNetAddress> getWorkerAddresses() throws IOException {
    List<WorkerInfo> infos;
    BlockMasterClient blockMasterClient = mBlockMasterClientPool.acquire();
    try {
        infos = blockMasterClient.getWorkerInfoList();
    } finally {
        mBlockMasterClientPool.release(blockMasterClient);
    }
    if (infos.isEmpty()) {
        throw new UnavailableException(ExceptionMessage.NO_WORKER_AVAILABLE.getMessage());
    }
    // Convert the worker infos into net addresses, if there are local addresses, only keep those
    List<WorkerNetAddress> workerNetAddresses = new ArrayList<>();
    List<WorkerNetAddress> localWorkerNetAddresses = new ArrayList<>();
    String localHostname = NetworkAddressUtils.getClientHostName(getClusterConf());
    for (WorkerInfo info : infos) {
        WorkerNetAddress netAddress = info.getAddress();
        if (netAddress.getHost().equals(localHostname)) {
            localWorkerNetAddresses.add(netAddress);
        }
        workerNetAddresses.add(netAddress);
    }
    return localWorkerNetAddresses.isEmpty() ? workerNetAddresses : localWorkerNetAddresses;
}

Example 59 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class DefaultAsyncPersistHandler method getWorkerStoringFile.

/**
 * Gets a worker where the given file is stored.
 *
 * @param path the path to the file
 * @return the id of the storing worker
 * @throws FileDoesNotExistException when the file does not exist on any worker
 * @throws AccessControlException if permission checking fails
 */
// TODO(calvin): Propagate the exceptions in certain cases
private long getWorkerStoringFile(AlluxioURI path) throws FileDoesNotExistException, AccessControlException, UnavailableException {
    long fileId = mFileSystemMasterView.getFileId(path);
    try {
        if (mFileSystemMasterView.getFileInfo(fileId).getLength() == 0) {
            // if file is empty, return any worker
            List<WorkerInfo> workerInfoList = mFileSystemMasterView.getWorkerInfoList();
            if (workerInfoList.isEmpty()) {
                LOG.error("No worker is available");
                return IdUtils.INVALID_WORKER_ID;
            }
            // randomly pick a worker
            int index = new Random().nextInt(workerInfoList.size());
            return workerInfoList.get(index).getId();
        }
    } catch (UnavailableException e) {
        return IdUtils.INVALID_WORKER_ID;
    }
    Map<Long, Integer> workerBlockCounts = new HashMap<>();
    List<FileBlockInfo> blockInfoList;
    try {
        blockInfoList = mFileSystemMasterView.getFileBlockInfoList(path);
        for (FileBlockInfo fileBlockInfo : blockInfoList) {
            for (BlockLocation blockLocation : fileBlockInfo.getBlockInfo().getLocations()) {
                if (workerBlockCounts.containsKey(blockLocation.getWorkerId())) {
                    workerBlockCounts.put(blockLocation.getWorkerId(), workerBlockCounts.get(blockLocation.getWorkerId()) + 1);
                } else {
                    workerBlockCounts.put(blockLocation.getWorkerId(), 1);
                }
                // all the blocks of a file must be stored on the same worker
                if (workerBlockCounts.get(blockLocation.getWorkerId()) == blockInfoList.size()) {
                    return blockLocation.getWorkerId();
                }
            }
        }
    } catch (FileDoesNotExistException e) {
        LOG.error("The file {} to persist does not exist", path);
        return IdUtils.INVALID_WORKER_ID;
    } catch (InvalidPathException e) {
        LOG.error("The file {} to persist is invalid", path);
        return IdUtils.INVALID_WORKER_ID;
    } catch (UnavailableException e) {
        return IdUtils.INVALID_WORKER_ID;
    }
    if (workerBlockCounts.size() == 0) {
        LOG.error("The file " + path + " does not exist on any worker");
        return IdUtils.INVALID_WORKER_ID;
    }
    LOG.error("Not all the blocks of file {} stored on the same worker", path);
    return IdUtils.INVALID_WORKER_ID;
}

Example 60 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class MasterWorkerInfo method generateWorkerInfo.

/**
 * Gets the selected field information for this worker.
 *
 * You should lock externally with {@link MasterWorkerInfo#lockWorkerMeta(EnumSet, boolean)}
 * with {@link WorkerMetaLockSection#USAGE} specified.
 * A shared lock is required.
 *
 * @param fieldRange the client selected fields
 * @param isLiveWorker the worker is live or not
 * @return generated worker information
 */
public WorkerInfo generateWorkerInfo(Set<WorkerInfoField> fieldRange, boolean isLiveWorker) {
    WorkerInfo info = new WorkerInfo();
    Set<WorkerInfoField> checkedFieldRange = fieldRange != null ? fieldRange : new HashSet<>(Arrays.asList(WorkerInfoField.values()));
    for (WorkerInfoField field : checkedFieldRange) {
        switch(field) {
            case ADDRESS:
                info.setAddress(mMeta.mWorkerAddress);
                break;
            case BLOCK_COUNT:
                info.setBlockCount(getBlockCount());
                break;
            case WORKER_CAPACITY_BYTES:
                info.setCapacityBytes(mUsage.mCapacityBytes);
                break;
            case WORKER_CAPACITY_BYTES_ON_TIERS:
                info.setCapacityBytesOnTiers(mUsage.mTotalBytesOnTiers);
                break;
            case ID:
                info.setId(mMeta.mId);
                break;
            case LAST_CONTACT_SEC:
                info.setLastContactSec((int) ((CommonUtils.getCurrentMs() - mLastUpdatedTimeMs.get()) / Constants.SECOND_MS));
                break;
            case START_TIME_MS:
                info.setStartTimeMs(mMeta.mStartTimeMs);
                break;
            case STATE:
                if (isLiveWorker) {
                    info.setState(LIVE_WORKER_STATE);
                } else {
                    info.setState(LOST_WORKER_STATE);
                }
                break;
            case WORKER_USED_BYTES:
                info.setUsedBytes(mUsage.mUsedBytes);
                break;
            case WORKER_USED_BYTES_ON_TIERS:
                info.setUsedBytesOnTiers(mUsage.mUsedBytesOnTiers);
                break;
            default:
                LOG.warn("Unrecognized worker info field: " + field);
        }
    }
    return info;
}