Examples with BlockStoreMeta alluxio.worker.block.BlockStoreMeta used on opensource projects

Example 1 with BlockStoreMeta

use of alluxio.worker.block.BlockStoreMeta in project alluxio by Alluxio.

the class AlluxioWorkerRestServiceHandler method getWebUIBlockInfo.

/**
 * Gets web ui block info page data.
 *
 * @param requestPath the request path
 * @param requestOffset the request offset
 * @param requestLimit the request limit
 * @return the response object
 */
@GET
@Path(WEBUI_BLOCKINFO)
public Response getWebUIBlockInfo(@QueryParam("path") String requestPath, @DefaultValue("0") @QueryParam("offset") String requestOffset, @DefaultValue("20") @QueryParam("limit") String requestLimit) {
    return RestUtils.call(() -> {
        WorkerWebUIBlockInfo response = new WorkerWebUIBlockInfo();
        if (!ServerConfiguration.getBoolean(PropertyKey.WEB_FILE_INFO_ENABLED)) {
            return response;
        }
        response.setFatalError("").setInvalidPathError("");
        if (!(requestPath == null || requestPath.isEmpty())) {
            // Display file block info
            try {
                URIStatus status = mFsClient.getStatus(new AlluxioURI(requestPath));
                UIFileInfo uiFileInfo = new UIFileInfo(status, ServerConfiguration.global(), new WorkerStorageTierAssoc().getOrderedStorageAliases());
                for (long blockId : status.getBlockIds()) {
                    if (mBlockWorker.hasBlockMeta(blockId)) {
                        BlockMeta blockMeta = mBlockWorker.getVolatileBlockMeta(blockId);
                        long blockSize = blockMeta.getBlockSize();
                        // The block last access time is not available. Use -1 for now.
                        // It's not necessary to show location information here since
                        // we are viewing at the context of this worker.
                        uiFileInfo.addBlock(blockMeta.getBlockLocation().tierAlias(), blockId, blockSize, -1);
                    }
                }
                List<ImmutablePair<String, List<UIFileBlockInfo>>> fileBlocksOnTier = new ArrayList<>();
                for (Map.Entry<String, List<UIFileBlockInfo>> e : uiFileInfo.getBlocksOnTier().entrySet()) {
                    fileBlocksOnTier.add(new ImmutablePair<>(e.getKey(), e.getValue()));
                }
                response.setFileBlocksOnTier(fileBlocksOnTier).setBlockSizeBytes(uiFileInfo.getBlockSizeBytes()).setPath(requestPath);
            } catch (FileDoesNotExistException e) {
                response.setFatalError("Error: Invalid Path " + e.getMessage());
            } catch (IOException e) {
                response.setInvalidPathError("Error: File " + requestPath + " is not available " + e.getMessage());
            } catch (BlockDoesNotExistException e) {
                response.setFatalError("Error: block not found. " + e.getMessage());
            } catch (AlluxioException e) {
                response.setFatalError("Error: alluxio exception. " + e.getMessage());
            }
        }
        Set<Long> unsortedFileIds = new HashSet<>();
        BlockStoreMeta storeMeta = mBlockWorker.getStoreMetaFull();
        for (List<Long> blockIds : storeMeta.getBlockList().values()) {
            for (long blockId : blockIds) {
                unsortedFileIds.add(BlockId.getFileId(blockId));
            }
        }
        List<Long> fileIds = new ArrayList<>(unsortedFileIds);
        Collections.sort(fileIds);
        response.setNTotalFile(unsortedFileIds.size()).setOrderedTierAliases(new WorkerStorageTierAssoc().getOrderedStorageAliases());
        try {
            int offset = Integer.parseInt(requestOffset);
            int limit = Integer.parseInt(requestLimit);
            // make the limit the total number of files if request limit is > than what is available
            limit = offset == 0 && limit > fileIds.size() ? fileIds.size() : limit;
            // offset+limit can't be greater than the size of the list
            limit = offset + limit > fileIds.size() ? fileIds.size() - offset : limit;
            int sum = Math.addExact(offset, limit);
            List<Long> subFileIds = fileIds.subList(offset, sum);
            List<UIFileInfo> uiFileInfos = new ArrayList<>(subFileIds.size());
            for (long fileId : subFileIds) {
                try {
                    URIStatus status = new URIStatus(mBlockWorker.getFileInfo(fileId));
                    UIFileInfo uiFileInfo = new UIFileInfo(status, ServerConfiguration.global(), new WorkerStorageTierAssoc().getOrderedStorageAliases());
                    for (long blockId : status.getBlockIds()) {
                        if (mBlockWorker.hasBlockMeta(blockId)) {
                            BlockMeta blockMeta = mBlockWorker.getVolatileBlockMeta(blockId);
                            long blockSize = blockMeta.getBlockSize();
                            // The block last access time is not available. Use -1 for now.
                            // It's not necessary to show location information here since
                            // we are viewing at the context of this worker.
                            uiFileInfo.addBlock(blockMeta.getBlockLocation().tierAlias(), blockId, blockSize, -1);
                        }
                    }
                    if (!uiFileInfo.getBlockIds().isEmpty()) {
                        uiFileInfos.add(uiFileInfo);
                    }
                } catch (Exception e) {
                    // The file might have been deleted, log a warning and ignore this file.
                    LOG.warn("Unable to get file info for fileId {}. {}", fileId, e.toString());
                }
            }
            response.setFileInfos(uiFileInfos);
        } catch (NumberFormatException e) {
            response.setFatalError("Error: offset or limit parse error, " + e.getLocalizedMessage());
        } catch (ArithmeticException e) {
            response.setFatalError("Error: offset or offset + limit is out ofbound, " + e.getLocalizedMessage());
        } catch (Exception e) {
            response.setFatalError(e.getLocalizedMessage());
        }
        return response;
    }, ServerConfiguration.global());
}

Example 2 with BlockStoreMeta

use of alluxio.worker.block.BlockStoreMeta in project alluxio by Alluxio.

the class WebInterfaceWorkerBlockInfoServlet method getSortedFileIds.

/***
   * Gets sorted file ids of the files cached in the worker.
   *
   * @return a sorted file id list
   */
private List<Long> getSortedFileIds() {
    Set<Long> fileIds = new HashSet<>();
    BlockStoreMeta storeMeta = mBlockWorker.getStoreMetaFull();
    for (List<Long> blockIds : storeMeta.getBlockList().values()) {
        for (long blockId : blockIds) {
            long fileId = BlockId.createBlockId(BlockId.getContainerId(blockId), BlockId.getMaxSequenceNumber());
            fileIds.add(fileId);
        }
    }
    List<Long> sortedFileIds = new ArrayList<>(fileIds);
    Collections.sort(sortedFileIds);
    return sortedFileIds;
}

Example 3 with BlockStoreMeta

use of alluxio.worker.block.BlockStoreMeta in project alluxio by Alluxio.

the class WebInterfaceWorkerGeneralServlet method populateValues.

/**
   * Populates key, value pairs for UI display.
   *
   * @param request the {@link HttpServletRequest} object
   * @throws IOException if an I/O error occurs
   */
private void populateValues(HttpServletRequest request) throws IOException {
    request.setAttribute("workerInfo", mUiWorkerInfo);
    BlockStoreMeta storeMeta = mBlockWorker.getStoreMeta();
    long capacityBytes = 0L;
    long usedBytes = 0L;
    Map<String, Long> capacityBytesOnTiers = storeMeta.getCapacityBytesOnTiers();
    Map<String, Long> usedBytesOnTiers = storeMeta.getUsedBytesOnTiers();
    List<UIUsageOnTier> usageOnTiers = new ArrayList<>();
    for (Entry<String, Long> entry : capacityBytesOnTiers.entrySet()) {
        String tier = entry.getKey();
        long capacity = entry.getValue();
        Long nullableUsed = usedBytesOnTiers.get(tier);
        long used = nullableUsed == null ? 0 : nullableUsed;
        capacityBytes += capacity;
        usedBytes += used;
        usageOnTiers.add(new UIUsageOnTier(tier, capacity, used));
    }
    request.setAttribute("capacityBytes", FormatUtils.getSizeFromBytes(capacityBytes));
    request.setAttribute("usedBytes", FormatUtils.getSizeFromBytes(usedBytes));
    request.setAttribute("usageOnTiers", usageOnTiers);
    request.setAttribute("version", RuntimeConstants.VERSION);
    List<UIStorageDir> storageDirs = new ArrayList<>(storeMeta.getCapacityBytesOnDirs().size());
    for (Pair<String, String> tierAndDirPath : storeMeta.getCapacityBytesOnDirs().keySet()) {
        storageDirs.add(new UIStorageDir(tierAndDirPath.getFirst(), tierAndDirPath.getSecond(), storeMeta.getCapacityBytesOnDirs().get(tierAndDirPath), storeMeta.getUsedBytesOnDirs().get(tierAndDirPath)));
    }
    request.setAttribute("storageDirs", storageDirs);
}

Example 4 with BlockStoreMeta

use of alluxio.worker.block.BlockStoreMeta in project alluxio by Alluxio.

the class AlluxioWorkerRestServiceHandler method getWebUIOverview.

/**
 * Gets web ui overview page data.
 *
 * @return the response object
 */
@GET
@Path(WEBUI_OVERVIEW)
public Response getWebUIOverview() {
    return RestUtils.call(() -> {
        WorkerWebUIOverview response = new WorkerWebUIOverview();
        response.setWorkerInfo(new UIWorkerInfo(mWorkerProcess.getRpcAddress().toString(), mWorkerProcess.getStartTimeMs(), ServerConfiguration.getString(PropertyKey.USER_DATE_FORMAT_PATTERN)));
        BlockStoreMeta storeMeta = mBlockWorker.getStoreMetaFull();
        long capacityBytes = 0L;
        long usedBytes = 0L;
        Map<String, Long> capacityBytesOnTiers = storeMeta.getCapacityBytesOnTiers();
        Map<String, Long> usedBytesOnTiers = storeMeta.getUsedBytesOnTiers();
        List<UIUsageOnTier> usageOnTiers = new ArrayList<>();
        for (Map.Entry<String, Long> entry : capacityBytesOnTiers.entrySet()) {
            String tier = entry.getKey();
            long capacity = entry.getValue();
            Long nullableUsed = usedBytesOnTiers.get(tier);
            long used = nullableUsed == null ? 0 : nullableUsed;
            capacityBytes += capacity;
            usedBytes += used;
            usageOnTiers.add(new UIUsageOnTier(tier, capacity, used));
        }
        response.setCapacityBytes(FormatUtils.getSizeFromBytes(capacityBytes)).setUsedBytes(FormatUtils.getSizeFromBytes(usedBytes)).setUsageOnTiers(usageOnTiers).setBlockCount(Long.toString(storeMeta.getNumberOfBlocks())).setVersion(RuntimeConstants.VERSION);
        List<UIStorageDir> storageDirs = new ArrayList<>(storeMeta.getCapacityBytesOnDirs().size());
        for (Pair<String, String> tierAndDirPath : storeMeta.getCapacityBytesOnDirs().keySet()) {
            storageDirs.add(new UIStorageDir(tierAndDirPath.getFirst(), tierAndDirPath.getSecond(), storeMeta.getCapacityBytesOnDirs().get(tierAndDirPath), storeMeta.getUsedBytesOnDirs().get(tierAndDirPath)));
        }
        response.setStorageDirs(storageDirs);
        return response;
    }, ServerConfiguration.global());
}