Examples with FileBlockInfo alluxio.wire.FileBlockInfo used on opensource projects

Example 21 with FileBlockInfo

use of alluxio.wire.FileBlockInfo in project alluxio by Alluxio.

the class LoadDefinition method selectExecutors.

@Override
public Set<Pair<WorkerInfo, ArrayList<LoadTask>>> selectExecutors(LoadConfig config, List<WorkerInfo> jobWorkerInfoList, SelectExecutorsContext context) throws Exception {
    Map<String, WorkerInfo> jobWorkersByAddress = jobWorkerInfoList.stream().collect(Collectors.toMap(info -> info.getAddress().getHost(), info -> info));
    // Filter out workers which have no local job worker available.
    List<String> missingJobWorkerHosts = new ArrayList<>();
    List<BlockWorkerInfo> workers = new ArrayList<>();
    for (BlockWorkerInfo worker : context.getFsContext().getCachedWorkers()) {
        if (jobWorkersByAddress.containsKey(worker.getNetAddress().getHost())) {
            String workerHost = worker.getNetAddress().getHost().toUpperCase();
            if (!isEmptySet(config.getExcludedWorkerSet()) && config.getExcludedWorkerSet().contains(workerHost)) {
                continue;
            }
            // If specified the locality id, the candidate worker must match one at least
            boolean match = false;
            if (worker.getNetAddress().getTieredIdentity().getTiers() != null) {
                if (!(isEmptySet(config.getLocalityIds()) && isEmptySet(config.getExcludedLocalityIds()))) {
                    boolean exclude = false;
                    for (LocalityTier tier : worker.getNetAddress().getTieredIdentity().getTiers()) {
                        if (!isEmptySet(config.getExcludedLocalityIds()) && config.getExcludedLocalityIds().contains(tier.getValue().toUpperCase())) {
                            exclude = true;
                            break;
                        }
                        if (!isEmptySet(config.getLocalityIds()) && config.getLocalityIds().contains(tier.getValue().toUpperCase())) {
                            match = true;
                            break;
                        }
                    }
                    if (exclude) {
                        continue;
                    }
                }
            }
            // Or user specified neither worker-set nor locality id
            if ((isEmptySet(config.getWorkerSet()) && isEmptySet(config.getLocalityIds())) || match || (!isEmptySet(config.getWorkerSet()) && config.getWorkerSet().contains(workerHost))) {
                workers.add(worker);
            }
        } else {
            LOG.warn("Worker on host {} has no local job worker", worker.getNetAddress().getHost());
            missingJobWorkerHosts.add(worker.getNetAddress().getHost());
        }
    }
    // Mapping from worker to block ids which that worker is supposed to load.
    Multimap<WorkerInfo, LoadTask> assignments = LinkedListMultimap.create();
    AlluxioURI uri = new AlluxioURI(config.getFilePath());
    for (FileBlockInfo blockInfo : context.getFileSystem().getStatus(uri).getFileBlockInfos()) {
        List<BlockWorkerInfo> workersWithoutBlock = getWorkersWithoutBlock(workers, blockInfo);
        int neededReplicas = config.getReplication() - blockInfo.getBlockInfo().getLocations().size();
        if (workersWithoutBlock.size() < neededReplicas) {
            String missingJobWorkersMessage = "";
            if (!missingJobWorkerHosts.isEmpty()) {
                missingJobWorkersMessage = ". The following workers could not be used because they have " + "no local job workers: " + missingJobWorkerHosts;
            }
            throw new FailedPreconditionException(String.format("Failed to find enough block workers to replicate to. Needed %s but only found %s. " + "Available workers without the block: %s" + missingJobWorkersMessage, neededReplicas, workersWithoutBlock.size(), workersWithoutBlock));
        }
        Collections.shuffle(workersWithoutBlock);
        for (int i = 0; i < neededReplicas; i++) {
            String address = workersWithoutBlock.get(i).getNetAddress().getHost();
            WorkerInfo jobWorker = jobWorkersByAddress.get(address);
            assignments.put(jobWorker, new LoadTask(blockInfo.getBlockInfo().getBlockId(), workersWithoutBlock.get(i).getNetAddress()));
        }
    }
    Set<Pair<WorkerInfo, ArrayList<LoadTask>>> result = Sets.newHashSet();
    for (Map.Entry<WorkerInfo, Collection<LoadTask>> assignment : assignments.asMap().entrySet()) {
        Collection<LoadTask> loadTasks = assignment.getValue();
        List<List<LoadTask>> partitionedTasks = CommonUtils.partition(Lists.newArrayList(loadTasks), JOBS_PER_WORKER);
        for (List<LoadTask> tasks : partitionedTasks) {
            if (!tasks.isEmpty()) {
                result.add(new Pair<>(assignment.getKey(), Lists.newArrayList(tasks)));
            }
        }
    }
    return result;
}

Example 22 with FileBlockInfo

use of alluxio.wire.FileBlockInfo in project alluxio by Alluxio.

the class WebInterfaceBrowseServlet method displayFile.

/**
   * This function displays 5KB of a file from a specific offset if it is in ASCII format.
   *
   * @param path the path of the file to display
   * @param request the {@link HttpServletRequest} object
   * @param offset where the file starts to display
   * @throws FileDoesNotExistException if the file does not exist
   * @throws IOException if an I/O error occurs
   * @throws InvalidPathException if an invalid path is encountered
   * @throws AlluxioException if an unexpected Alluxio exception is thrown
   */
private void displayFile(AlluxioURI path, HttpServletRequest request, long offset) throws FileDoesNotExistException, InvalidPathException, IOException, AlluxioException {
    FileSystem fs = FileSystem.Factory.get();
    String fileData;
    URIStatus status = fs.getStatus(path);
    if (status.isCompleted()) {
        OpenFileOptions options = OpenFileOptions.defaults().setReadType(ReadType.NO_CACHE);
        try (FileInStream is = fs.openFile(path, options)) {
            int len = (int) Math.min(5 * Constants.KB, status.getLength() - offset);
            byte[] data = new byte[len];
            long skipped = is.skip(offset);
            if (skipped < 0) {
                // nothing was skipped
                fileData = "Unable to traverse to offset; is file empty?";
            } else if (skipped < offset) {
                // couldn't skip all the way to offset
                fileData = "Unable to traverse to offset; is offset larger than the file?";
            } else {
                // read may not read up to len, so only convert what was read
                int read = is.read(data, 0, len);
                if (read < 0) {
                    // stream couldn't read anything, skip went to EOF?
                    fileData = "Unable to read file";
                } else {
                    fileData = WebUtils.convertByteArrayToStringWithoutEscape(data, 0, read);
                }
            }
        }
    } else {
        fileData = "The requested file is not complete yet.";
    }
    List<UIFileBlockInfo> uiBlockInfo = new ArrayList<>();
    for (FileBlockInfo fileBlockInfo : mMaster.getFileSystemMaster().getFileBlockInfoList(path)) {
        uiBlockInfo.add(new UIFileBlockInfo(fileBlockInfo));
    }
    request.setAttribute("fileBlocks", uiBlockInfo);
    request.setAttribute("fileData", fileData);
    request.setAttribute("highestTierAlias", mMaster.getBlockMaster().getGlobalStorageTierAssoc().getAlias(0));
}

Example 23 with FileBlockInfo

use of alluxio.wire.FileBlockInfo in project alluxio by Alluxio.

the class FileSystemMaster method generateFileBlockInfo.

/**
   * Generates a {@link FileBlockInfo} object from internal metadata. This adds file information to
   * the block, such as the file offset, and additional UFS locations for the block.
   *
   * @param inodePath the file the block is a part of
   * @param blockInfo the {@link BlockInfo} to generate the {@link FileBlockInfo} from
   * @return a new {@link FileBlockInfo} for the block
   * @throws InvalidPathException if the mount table is not able to resolve the file
   */
private FileBlockInfo generateFileBlockInfo(LockedInodePath inodePath, BlockInfo blockInfo) throws InvalidPathException, FileDoesNotExistException {
    InodeFile file = inodePath.getInodeFile();
    FileBlockInfo fileBlockInfo = new FileBlockInfo();
    fileBlockInfo.setBlockInfo(blockInfo);
    fileBlockInfo.setUfsLocations(new ArrayList<String>());
    // The sequence number part of the block id is the block index.
    long offset = file.getBlockSizeBytes() * BlockId.getSequenceNumber(blockInfo.getBlockId());
    fileBlockInfo.setOffset(offset);
    if (fileBlockInfo.getBlockInfo().getLocations().isEmpty() && file.isPersisted()) {
        // No alluxio locations, but there is a checkpoint in the under storage system. Add the
        // locations from the under storage system.
        MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
        String ufsUri = resolution.getUri().toString();
        UnderFileSystem ufs = resolution.getUfs();
        List<String> locs;
        try {
            locs = ufs.getFileLocations(ufsUri, FileLocationOptions.defaults().setOffset(fileBlockInfo.getOffset()));
        } catch (IOException e) {
            return fileBlockInfo;
        }
        if (locs != null) {
            for (String loc : locs) {
                fileBlockInfo.getUfsLocations().add(loc);
            }
        }
    }
    return fileBlockInfo;
}

Example 24 with FileBlockInfo

use of alluxio.wire.FileBlockInfo in project alluxio by Alluxio.

the class FileSystemMaster method getFileBlockInfoList.

/**
   * Gets the {@link FileBlockInfo} for all blocks of a file. If path is a directory, an exception
   * is thrown.
   * <p>
   * This operation requires the client user to have {@link Mode.Bits#READ} permission on the
   * the path.
   *
   * @param path the path to get the info for
   * @return a list of {@link FileBlockInfo} for all the blocks of the given path
   * @throws FileDoesNotExistException if the file does not exist or path is a directory
   * @throws InvalidPathException if the path of the given file is invalid
   * @throws AccessControlException if permission checking fails
   */
public List<FileBlockInfo> getFileBlockInfoList(AlluxioURI path) throws FileDoesNotExistException, InvalidPathException, AccessControlException {
    Metrics.GET_FILE_BLOCK_INFO_OPS.inc();
    try (LockedInodePath inodePath = mInodeTree.lockFullInodePath(path, InodeTree.LockMode.READ)) {
        mPermissionChecker.checkPermission(Mode.Bits.READ, inodePath);
        List<FileBlockInfo> ret = getFileBlockInfoListInternal(inodePath);
        Metrics.FILE_BLOCK_INFOS_GOT.inc();
        return ret;
    }
}

Example 25 with FileBlockInfo

use of alluxio.wire.FileBlockInfo in project alluxio by Alluxio.

the class FileSystemMaster method reportLostFile.

/**
   * Reports a file as lost.
   *
   * @param fileId the id of the file
   * @throws FileDoesNotExistException if the file does not exist
   */
// Currently used by Lineage Master
// TODO(binfan): Add permission checking for internal APIs
public void reportLostFile(long fileId) throws FileDoesNotExistException {
    try (LockedInodePath inodePath = mInodeTree.lockFullInodePath(fileId, InodeTree.LockMode.READ)) {
        Inode<?> inode = inodePath.getInode();
        if (inode.isDirectory()) {
            LOG.warn("Reported file is a directory {}", inode);
            return;
        }
        List<Long> blockIds = new ArrayList<>();
        try {
            for (FileBlockInfo fileBlockInfo : getFileBlockInfoListInternal(inodePath)) {
                blockIds.add(fileBlockInfo.getBlockInfo().getBlockId());
            }
        } catch (InvalidPathException e) {
            LOG.info("Failed to get file info {}", fileId, e);
        }
        mBlockMaster.reportLostBlocks(blockIds);
        LOG.info("Reported file loss of blocks {}. Alluxio will recompute it: {}", blockIds, fileId);
    }
}