Examples with BlockLocation alluxio.wire.BlockLocation used on opensource projects

Example 41 with BlockLocation

use of alluxio.wire.BlockLocation in project alluxio by Alluxio.

the class AlluxioFileInStream method updateStream.

/**
 * Initializes the underlying block stream if necessary. This method must be called before
 * reading from mBlockInStream.
 */
private void updateStream() throws IOException {
    if (mBlockInStream != null && mBlockInStream.remaining() > 0) {
        // can still read from stream
        return;
    }
    if (mBlockInStream != null && mBlockInStream.remaining() == 0) {
        // current stream is done
        closeBlockInStream(mBlockInStream);
    }
    /* Create a new stream to read from mPosition. */
    // Calculate block id.
    long blockId = mStatus.getBlockIds().get(Math.toIntExact(mPosition / mBlockSize));
    BlockInfo blockInfo = mStatus.getBlockInfo(blockId);
    if (blockInfo == null) {
        throw new IOException("No BlockInfo for block(id=" + blockId + ") of file" + "(id=" + mStatus.getFileId() + ", path=" + mStatus.getPath() + ")");
    }
    // Create stream
    boolean isBlockInfoOutdated = true;
    // if there is at least one location that is not a failed worker, then it's not outdated.
    if (mFailedWorkers.isEmpty() || mFailedWorkers.size() < blockInfo.getLocations().size()) {
        isBlockInfoOutdated = false;
    } else {
        for (BlockLocation location : blockInfo.getLocations()) {
            if (!mFailedWorkers.containsKey(location.getWorkerAddress())) {
                isBlockInfoOutdated = false;
                break;
            }
        }
    }
    if (isBlockInfoOutdated) {
        mBlockInStream = mBlockStore.getInStream(blockId, mOptions, mFailedWorkers);
    } else {
        mBlockInStream = mBlockStore.getInStream(blockInfo, mOptions, mFailedWorkers);
    }
    // Set the stream to the correct position.
    long offset = mPosition % mBlockSize;
    mBlockInStream.seek(offset);
}

Example 42 with BlockLocation

use of alluxio.wire.BlockLocation in project alluxio by Alluxio.

the class DefaultAsyncPersistHandler method getWorkerStoringFile.

/**
 * Gets a worker where the given file is stored.
 *
 * @param path the path to the file
 * @return the id of the storing worker
 * @throws FileDoesNotExistException when the file does not exist on any worker
 * @throws AccessControlException if permission checking fails
 */
// TODO(calvin): Propagate the exceptions in certain cases
private long getWorkerStoringFile(AlluxioURI path) throws FileDoesNotExistException, AccessControlException, UnavailableException {
    long fileId = mFileSystemMasterView.getFileId(path);
    try {
        if (mFileSystemMasterView.getFileInfo(fileId).getLength() == 0) {
            // if file is empty, return any worker
            List<WorkerInfo> workerInfoList = mFileSystemMasterView.getWorkerInfoList();
            if (workerInfoList.isEmpty()) {
                LOG.error("No worker is available");
                return IdUtils.INVALID_WORKER_ID;
            }
            // randomly pick a worker
            int index = new Random().nextInt(workerInfoList.size());
            return workerInfoList.get(index).getId();
        }
    } catch (UnavailableException e) {
        return IdUtils.INVALID_WORKER_ID;
    }
    Map<Long, Integer> workerBlockCounts = new HashMap<>();
    List<FileBlockInfo> blockInfoList;
    try {
        blockInfoList = mFileSystemMasterView.getFileBlockInfoList(path);
        for (FileBlockInfo fileBlockInfo : blockInfoList) {
            for (BlockLocation blockLocation : fileBlockInfo.getBlockInfo().getLocations()) {
                if (workerBlockCounts.containsKey(blockLocation.getWorkerId())) {
                    workerBlockCounts.put(blockLocation.getWorkerId(), workerBlockCounts.get(blockLocation.getWorkerId()) + 1);
                } else {
                    workerBlockCounts.put(blockLocation.getWorkerId(), 1);
                }
                // all the blocks of a file must be stored on the same worker
                if (workerBlockCounts.get(blockLocation.getWorkerId()) == blockInfoList.size()) {
                    return blockLocation.getWorkerId();
                }
            }
        }
    } catch (FileDoesNotExistException e) {
        LOG.error("The file {} to persist does not exist", path);
        return IdUtils.INVALID_WORKER_ID;
    } catch (InvalidPathException e) {
        LOG.error("The file {} to persist is invalid", path);
        return IdUtils.INVALID_WORKER_ID;
    } catch (UnavailableException e) {
        return IdUtils.INVALID_WORKER_ID;
    }
    if (workerBlockCounts.size() == 0) {
        LOG.error("The file " + path + " does not exist on any worker");
        return IdUtils.INVALID_WORKER_ID;
    }
    LOG.error("Not all the blocks of file {} stored on the same worker", path);
    return IdUtils.INVALID_WORKER_ID;
}