Examples with FileBlockInfo alluxio.wire.FileBlockInfo used on opensource projects

Example 6 with FileBlockInfo

use of alluxio.wire.FileBlockInfo in project alluxio by Alluxio.

the class DefaultAsyncPersistHandlerTest method scheduleAsyncPersist.

@Test
public void scheduleAsyncPersist() throws Exception {
    DefaultAsyncPersistHandler handler = new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster));
    AlluxioURI path = new AlluxioURI("/test");
    long blockId = 0;
    long workerId = 1;
    long fileId = 2;
    List<FileBlockInfo> blockInfoList = new ArrayList<>();
    BlockLocation location = new BlockLocation().setWorkerId(workerId);
    blockInfoList.add(new FileBlockInfo().setBlockInfo(new BlockInfo().setBlockId(blockId).setLocations(Lists.newArrayList(location))));
    Mockito.when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList);
    Mockito.when(mFileSystemMaster.getFileId(path)).thenReturn(fileId);
    Mockito.when(mFileSystemMaster.getPath(fileId)).thenReturn(path);
    Mockito.when(mFileSystemMaster.getFileInfo(fileId)).thenReturn(new FileInfo().setLength(1).setCompleted(true));
    handler.scheduleAsyncPersistence(path);
    List<PersistFile> persistFiles = handler.pollFilesToPersist(workerId);
    Assert.assertEquals(1, persistFiles.size());
    Assert.assertEquals(Lists.newArrayList(blockId), persistFiles.get(0).getBlockIds());
}

Example 7 with FileBlockInfo

use of alluxio.wire.FileBlockInfo in project alluxio by Alluxio.

the class FileSystemMaster method generateFileBlockInfo.

/**
   * Generates a {@link FileBlockInfo} object from internal metadata. This adds file information to
   * the block, such as the file offset, and additional UFS locations for the block.
   *
   * @param inodePath the file the block is a part of
   * @param blockInfo the {@link BlockInfo} to generate the {@link FileBlockInfo} from
   * @return a new {@link FileBlockInfo} for the block
   * @throws InvalidPathException if the mount table is not able to resolve the file
   */
private FileBlockInfo generateFileBlockInfo(LockedInodePath inodePath, BlockInfo blockInfo) throws InvalidPathException, FileDoesNotExistException {
    InodeFile file = inodePath.getInodeFile();
    FileBlockInfo fileBlockInfo = new FileBlockInfo();
    fileBlockInfo.setBlockInfo(blockInfo);
    fileBlockInfo.setUfsLocations(new ArrayList<String>());
    // The sequence number part of the block id is the block index.
    long offset = file.getBlockSizeBytes() * BlockId.getSequenceNumber(blockInfo.getBlockId());
    fileBlockInfo.setOffset(offset);
    if (fileBlockInfo.getBlockInfo().getLocations().isEmpty() && file.isPersisted()) {
        // No alluxio locations, but there is a checkpoint in the under storage system. Add the
        // locations from the under storage system.
        MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
        String ufsUri = resolution.getUri().toString();
        UnderFileSystem ufs = resolution.getUfs();
        List<String> locs;
        try {
            locs = ufs.getFileLocations(ufsUri, FileLocationOptions.defaults().setOffset(fileBlockInfo.getOffset()));
        } catch (IOException e) {
            return fileBlockInfo;
        }
        if (locs != null) {
            for (String loc : locs) {
                fileBlockInfo.getUfsLocations().add(loc);
            }
        }
    }
    return fileBlockInfo;
}

Example 8 with FileBlockInfo

use of alluxio.wire.FileBlockInfo in project alluxio by Alluxio.

the class FileSystemMaster method getFileBlockInfoList.

/**
   * Gets the {@link FileBlockInfo} for all blocks of a file. If path is a directory, an exception
   * is thrown.
   * <p>
   * This operation requires the client user to have {@link Mode.Bits#READ} permission on the
   * the path.
   *
   * @param path the path to get the info for
   * @return a list of {@link FileBlockInfo} for all the blocks of the given path
   * @throws FileDoesNotExistException if the file does not exist or path is a directory
   * @throws InvalidPathException if the path of the given file is invalid
   * @throws AccessControlException if permission checking fails
   */
public List<FileBlockInfo> getFileBlockInfoList(AlluxioURI path) throws FileDoesNotExistException, InvalidPathException, AccessControlException {
    Metrics.GET_FILE_BLOCK_INFO_OPS.inc();
    try (LockedInodePath inodePath = mInodeTree.lockFullInodePath(path, InodeTree.LockMode.READ)) {
        mPermissionChecker.checkPermission(Mode.Bits.READ, inodePath);
        List<FileBlockInfo> ret = getFileBlockInfoListInternal(inodePath);
        Metrics.FILE_BLOCK_INFOS_GOT.inc();
        return ret;
    }
}

Example 9 with FileBlockInfo

use of alluxio.wire.FileBlockInfo in project alluxio by Alluxio.

the class FileSystemMaster method reportLostFile.

/**
   * Reports a file as lost.
   *
   * @param fileId the id of the file
   * @throws FileDoesNotExistException if the file does not exist
   */
// Currently used by Lineage Master
// TODO(binfan): Add permission checking for internal APIs
public void reportLostFile(long fileId) throws FileDoesNotExistException {
    try (LockedInodePath inodePath = mInodeTree.lockFullInodePath(fileId, InodeTree.LockMode.READ)) {
        Inode<?> inode = inodePath.getInode();
        if (inode.isDirectory()) {
            LOG.warn("Reported file is a directory {}", inode);
            return;
        }
        List<Long> blockIds = new ArrayList<>();
        try {
            for (FileBlockInfo fileBlockInfo : getFileBlockInfoListInternal(inodePath)) {
                blockIds.add(fileBlockInfo.getBlockInfo().getBlockId());
            }
        } catch (InvalidPathException e) {
            LOG.info("Failed to get file info {}", fileId, e);
        }
        mBlockMaster.reportLostBlocks(blockIds);
        LOG.info("Reported file loss of blocks {}. Alluxio will recompute it: {}", blockIds, fileId);
    }
}

Example 10 with FileBlockInfo

use of alluxio.wire.FileBlockInfo in project alluxio by Alluxio.

the class DefaultAsyncPersistHandler method getWorkerStoringFile.

/**
   * Gets a worker where the given file is stored.
   *
   * @param path the path to the file
   * @return the id of the storing worker
   * @throws FileDoesNotExistException when the file does not exist on any worker
   * @throws AccessControlException if permission checking fails
   */
// TODO(calvin): Propagate the exceptions in certain cases
private long getWorkerStoringFile(AlluxioURI path) throws FileDoesNotExistException, AccessControlException {
    long fileId = mFileSystemMasterView.getFileId(path);
    if (mFileSystemMasterView.getFileInfo(fileId).getLength() == 0) {
        // if file is empty, return any worker
        List<WorkerInfo> workerInfoList = mFileSystemMasterView.getWorkerInfoList();
        if (workerInfoList.isEmpty()) {
            LOG.error("No worker is available");
            return IdUtils.INVALID_WORKER_ID;
        }
        // randomly pick a worker
        int index = new Random().nextInt(workerInfoList.size());
        return workerInfoList.get(index).getId();
    }
    Map<Long, Integer> workerBlockCounts = new HashMap<>();
    List<FileBlockInfo> blockInfoList;
    try {
        blockInfoList = mFileSystemMasterView.getFileBlockInfoList(path);
        for (FileBlockInfo fileBlockInfo : blockInfoList) {
            for (BlockLocation blockLocation : fileBlockInfo.getBlockInfo().getLocations()) {
                if (workerBlockCounts.containsKey(blockLocation.getWorkerId())) {
                    workerBlockCounts.put(blockLocation.getWorkerId(), workerBlockCounts.get(blockLocation.getWorkerId()) + 1);
                } else {
                    workerBlockCounts.put(blockLocation.getWorkerId(), 1);
                }
                // all the blocks of a file must be stored on the same worker
                if (workerBlockCounts.get(blockLocation.getWorkerId()) == blockInfoList.size()) {
                    return blockLocation.getWorkerId();
                }
            }
        }
    } catch (FileDoesNotExistException e) {
        LOG.error("The file {} to persist does not exist", path);
        return IdUtils.INVALID_WORKER_ID;
    } catch (InvalidPathException e) {
        LOG.error("The file {} to persist is invalid", path);
        return IdUtils.INVALID_WORKER_ID;
    }
    if (workerBlockCounts.size() == 0) {
        LOG.error("The file " + path + " does not exist on any worker");
        return IdUtils.INVALID_WORKER_ID;
    }
    LOG.error("Not all the blocks of file {} stored on the same worker", path);
    return IdUtils.INVALID_WORKER_ID;
}