Search in sources :

Example 1 with UIFileBlockInfo

use of alluxio.util.webui.UIFileBlockInfo in project alluxio by Alluxio.

the class AlluxioMasterRestServiceHandler method getWebUIBrowse.

/**
 * Gets Web UI browse page data.
 *
 * @param requestPath the request path
 * @param requestOffset the request offset
 * @param requestEnd the request end
 * @param requestLimit the request limit
 * @return the response object
 */
@GET
@Path(WEBUI_BROWSE)
public Response getWebUIBrowse(@DefaultValue("/") @QueryParam("path") String requestPath, @DefaultValue("0") @QueryParam("offset") String requestOffset, @DefaultValue("") @QueryParam("end") String requestEnd, @DefaultValue("20") @QueryParam("limit") String requestLimit) {
    return RestUtils.call(() -> {
        MasterWebUIBrowse response = new MasterWebUIBrowse();
        if (!ServerConfiguration.getBoolean(PropertyKey.WEB_FILE_INFO_ENABLED)) {
            return response;
        }
        if (SecurityUtils.isSecurityEnabled(ServerConfiguration.global()) && AuthenticatedClientUser.get(ServerConfiguration.global()) == null) {
            AuthenticatedClientUser.set(ServerUserState.global().getUser().getName());
        }
        response.setDebug(ServerConfiguration.getBoolean(PropertyKey.DEBUG)).setShowPermissions(ServerConfiguration.getBoolean(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_ENABLED)).setMasterNodeAddress(mMasterProcess.getRpcAddress().toString()).setInvalidPathError("");
        List<FileInfo> filesInfo;
        String path = URLDecoder.decode(requestPath, "UTF-8");
        if (path.isEmpty()) {
            path = AlluxioURI.SEPARATOR;
        }
        AlluxioURI currentPath = new AlluxioURI(path);
        response.setCurrentPath(currentPath.toString()).setViewingOffset(0);
        try {
            long fileId = mFileSystemMaster.getFileId(currentPath);
            FileInfo fileInfo = mFileSystemMaster.getFileInfo(fileId);
            UIFileInfo currentFileInfo = new UIFileInfo(fileInfo, ServerConfiguration.global(), new MasterStorageTierAssoc().getOrderedStorageAliases());
            if (currentFileInfo.getAbsolutePath() == null) {
                throw new FileDoesNotExistException(currentPath.toString());
            }
            response.setCurrentDirectory(currentFileInfo).setBlockSizeBytes(currentFileInfo.getBlockSizeBytes());
            if (!currentFileInfo.getIsDirectory()) {
                long relativeOffset = 0;
                long offset;
                try {
                    if (requestOffset != null) {
                        relativeOffset = Long.parseLong(requestOffset);
                    }
                } catch (NumberFormatException e) {
                // ignore the exception
                }
                // relative to the end of the file.
                if (requestEnd.equals("")) {
                    offset = relativeOffset;
                } else {
                    offset = fileInfo.getLength() - relativeOffset;
                }
                if (offset < 0) {
                    offset = 0;
                } else if (offset > fileInfo.getLength()) {
                    offset = fileInfo.getLength();
                }
                try {
                    AlluxioURI absolutePath = new AlluxioURI(currentFileInfo.getAbsolutePath());
                    FileSystem fs = mFsClient;
                    String fileData;
                    URIStatus status = fs.getStatus(absolutePath);
                    if (status.isCompleted()) {
                        OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).build();
                        try (FileInStream is = fs.openFile(absolutePath, options)) {
                            int len = (int) Math.min(5L * Constants.KB, status.getLength() - offset);
                            byte[] data = new byte[len];
                            long skipped = is.skip(offset);
                            if (skipped < 0) {
                                // nothing was skipped
                                fileData = "Unable to traverse to offset; is file empty?";
                            } else if (skipped < offset) {
                                // couldn't skip all the way to offset
                                fileData = "Unable to traverse to offset; is offset larger than the file?";
                            } else {
                                // read may not read up to len, so only convert what was read
                                int read = is.read(data, 0, len);
                                if (read < 0) {
                                    // stream couldn't read anything, skip went to EOF?
                                    fileData = "Unable to read file";
                                } else {
                                    fileData = WebUtils.convertByteArrayToStringWithoutEscape(data, 0, read);
                                }
                            }
                        }
                    } else {
                        fileData = "The requested file is not complete yet.";
                    }
                    List<UIFileBlockInfo> uiBlockInfo = new ArrayList<>();
                    for (FileBlockInfo fileBlockInfo : mFileSystemMaster.getFileBlockInfoList(absolutePath)) {
                        uiBlockInfo.add(new UIFileBlockInfo(fileBlockInfo, ServerConfiguration.global()));
                    }
                    response.setFileBlocks(uiBlockInfo).setFileData(fileData).setHighestTierAlias(mBlockMaster.getGlobalStorageTierAssoc().getAlias(0));
                } catch (AlluxioException e) {
                    throw new IOException(e);
                }
                response.setViewingOffset(offset);
                return response;
            }
            if (currentPath.isRoot()) {
                response.setPathInfos(new UIFileInfo[0]);
            } else {
                String[] splitPath = PathUtils.getPathComponents(currentPath.toString());
                UIFileInfo[] pathInfos = new UIFileInfo[splitPath.length - 1];
                fileId = mFileSystemMaster.getFileId(currentPath);
                pathInfos[0] = new UIFileInfo(mFileSystemMaster.getFileInfo(fileId), ServerConfiguration.global(), new MasterStorageTierAssoc().getOrderedStorageAliases());
                AlluxioURI breadcrumb = new AlluxioURI(AlluxioURI.SEPARATOR);
                for (int i = 1; i < splitPath.length - 1; i++) {
                    breadcrumb = breadcrumb.join(splitPath[i]);
                    fileId = mFileSystemMaster.getFileId(breadcrumb);
                    pathInfos[i] = new UIFileInfo(mFileSystemMaster.getFileInfo(fileId), ServerConfiguration.global(), new MasterStorageTierAssoc().getOrderedStorageAliases());
                }
                response.setPathInfos(pathInfos);
            }
            filesInfo = mFileSystemMaster.listStatus(currentPath, ListStatusContext.defaults());
        } catch (FileDoesNotExistException e) {
            response.setInvalidPathError("Error: Invalid Path " + e.getMessage());
            return response;
        } catch (InvalidPathException e) {
            response.setInvalidPathError("Error: Invalid Path " + e.getLocalizedMessage());
            return response;
        } catch (UnavailableException e) {
            response.setInvalidPathError("The service is temporarily unavailable. " + e.getMessage());
            return response;
        } catch (IOException e) {
            response.setInvalidPathError("Error: File " + currentPath + " is not available " + e.getMessage());
            return response;
        } catch (AccessControlException e) {
            response.setInvalidPathError("Error: File " + currentPath + " cannot be accessed " + e.getMessage());
            return response;
        }
        List<UIFileInfo> fileInfos = new ArrayList<>(filesInfo.size());
        for (FileInfo fileInfo : filesInfo) {
            UIFileInfo toAdd = new UIFileInfo(fileInfo, ServerConfiguration.global(), new MasterStorageTierAssoc().getOrderedStorageAliases());
            try {
                if (!toAdd.getIsDirectory() && fileInfo.getLength() > 0) {
                    FileBlockInfo blockInfo = mFileSystemMaster.getFileBlockInfoList(new AlluxioURI(toAdd.getAbsolutePath())).get(0);
                    List<String> locations = new ArrayList<>();
                    // add the in-Alluxio block locations
                    for (BlockLocation location : blockInfo.getBlockInfo().getLocations()) {
                        WorkerNetAddress address = location.getWorkerAddress();
                        locations.add(address.getHost() + ":" + address.getDataPort());
                    }
                    // add underFS locations
                    locations.addAll(blockInfo.getUfsLocations());
                    toAdd.setFileLocations(locations);
                }
            } catch (FileDoesNotExistException e) {
                response.setFileDoesNotExistException("Error: non-existing file " + e.getMessage());
                return response;
            } catch (InvalidPathException e) {
                response.setInvalidPathException("Error: invalid path " + e.getMessage());
                return response;
            } catch (AccessControlException e) {
                response.setAccessControlException("Error: File " + currentPath + " cannot be accessed " + e.getMessage());
                return response;
            }
            fileInfos.add(toAdd);
        }
        fileInfos.sort(UIFileInfo.PATH_STRING_COMPARE);
        response.setNTotalFile(fileInfos.size());
        try {
            int offset = Integer.parseInt(requestOffset);
            int limit = Integer.parseInt(requestLimit);
            limit = offset == 0 && limit > fileInfos.size() ? fileInfos.size() : limit;
            limit = offset + limit > fileInfos.size() ? fileInfos.size() - offset : limit;
            int sum = Math.addExact(offset, limit);
            fileInfos = fileInfos.subList(offset, sum);
            response.setFileInfos(fileInfos);
        } catch (NumberFormatException e) {
            response.setFatalError("Error: offset or limit parse error, " + e.getLocalizedMessage());
            return response;
        } catch (ArithmeticException e) {
            response.setFatalError("Error: offset or offset + limit is out of bound, " + e.getLocalizedMessage());
            return response;
        } catch (IllegalArgumentException e) {
            response.setFatalError(e.getLocalizedMessage());
            return response;
        }
        return response;
    }, ServerConfiguration.global());
}
Also used : FileDoesNotExistException(alluxio.exception.FileDoesNotExistException) ArrayList(java.util.ArrayList) UnavailableException(alluxio.exception.status.UnavailableException) UIFileBlockInfo(alluxio.util.webui.UIFileBlockInfo) FileBlockInfo(alluxio.wire.FileBlockInfo) BlockLocation(alluxio.wire.BlockLocation) InvalidPathException(alluxio.exception.InvalidPathException) MasterWebUIBrowse(alluxio.wire.MasterWebUIBrowse) UIFileInfo(alluxio.util.webui.UIFileInfo) FileInfo(alluxio.wire.FileInfo) FileSystem(alluxio.client.file.FileSystem) UIFileInfo(alluxio.util.webui.UIFileInfo) AlluxioException(alluxio.exception.AlluxioException) UIFileBlockInfo(alluxio.util.webui.UIFileBlockInfo) AccessControlException(alluxio.exception.AccessControlException) IOException(java.io.IOException) URIStatus(alluxio.client.file.URIStatus) MasterStorageTierAssoc(alluxio.MasterStorageTierAssoc) WorkerNetAddress(alluxio.wire.WorkerNetAddress) FileInStream(alluxio.client.file.FileInStream) OpenFilePOptions(alluxio.grpc.OpenFilePOptions) AlluxioURI(alluxio.AlluxioURI) Path(javax.ws.rs.Path) GET(javax.ws.rs.GET)

Example 2 with UIFileBlockInfo

use of alluxio.util.webui.UIFileBlockInfo in project alluxio by Alluxio.

the class AlluxioWorkerRestServiceHandler method getWebUIBlockInfo.

/**
 * Gets web ui block info page data.
 *
 * @param requestPath the request path
 * @param requestOffset the request offset
 * @param requestLimit the request limit
 * @return the response object
 */
@GET
@Path(WEBUI_BLOCKINFO)
public Response getWebUIBlockInfo(@QueryParam("path") String requestPath, @DefaultValue("0") @QueryParam("offset") String requestOffset, @DefaultValue("20") @QueryParam("limit") String requestLimit) {
    return RestUtils.call(() -> {
        WorkerWebUIBlockInfo response = new WorkerWebUIBlockInfo();
        if (!ServerConfiguration.getBoolean(PropertyKey.WEB_FILE_INFO_ENABLED)) {
            return response;
        }
        response.setFatalError("").setInvalidPathError("");
        if (!(requestPath == null || requestPath.isEmpty())) {
            // Display file block info
            try {
                URIStatus status = mFsClient.getStatus(new AlluxioURI(requestPath));
                UIFileInfo uiFileInfo = new UIFileInfo(status, ServerConfiguration.global(), new WorkerStorageTierAssoc().getOrderedStorageAliases());
                for (long blockId : status.getBlockIds()) {
                    if (mBlockWorker.hasBlockMeta(blockId)) {
                        BlockMeta blockMeta = mBlockWorker.getVolatileBlockMeta(blockId);
                        long blockSize = blockMeta.getBlockSize();
                        // The block last access time is not available. Use -1 for now.
                        // It's not necessary to show location information here since
                        // we are viewing at the context of this worker.
                        uiFileInfo.addBlock(blockMeta.getBlockLocation().tierAlias(), blockId, blockSize, -1);
                    }
                }
                List<ImmutablePair<String, List<UIFileBlockInfo>>> fileBlocksOnTier = new ArrayList<>();
                for (Map.Entry<String, List<UIFileBlockInfo>> e : uiFileInfo.getBlocksOnTier().entrySet()) {
                    fileBlocksOnTier.add(new ImmutablePair<>(e.getKey(), e.getValue()));
                }
                response.setFileBlocksOnTier(fileBlocksOnTier).setBlockSizeBytes(uiFileInfo.getBlockSizeBytes()).setPath(requestPath);
            } catch (FileDoesNotExistException e) {
                response.setFatalError("Error: Invalid Path " + e.getMessage());
            } catch (IOException e) {
                response.setInvalidPathError("Error: File " + requestPath + " is not available " + e.getMessage());
            } catch (BlockDoesNotExistException e) {
                response.setFatalError("Error: block not found. " + e.getMessage());
            } catch (AlluxioException e) {
                response.setFatalError("Error: alluxio exception. " + e.getMessage());
            }
        }
        Set<Long> unsortedFileIds = new HashSet<>();
        BlockStoreMeta storeMeta = mBlockWorker.getStoreMetaFull();
        for (List<Long> blockIds : storeMeta.getBlockList().values()) {
            for (long blockId : blockIds) {
                unsortedFileIds.add(BlockId.getFileId(blockId));
            }
        }
        List<Long> fileIds = new ArrayList<>(unsortedFileIds);
        Collections.sort(fileIds);
        response.setNTotalFile(unsortedFileIds.size()).setOrderedTierAliases(new WorkerStorageTierAssoc().getOrderedStorageAliases());
        try {
            int offset = Integer.parseInt(requestOffset);
            int limit = Integer.parseInt(requestLimit);
            // make the limit the total number of files if request limit is > than what is available
            limit = offset == 0 && limit > fileIds.size() ? fileIds.size() : limit;
            // offset+limit can't be greater than the size of the list
            limit = offset + limit > fileIds.size() ? fileIds.size() - offset : limit;
            int sum = Math.addExact(offset, limit);
            List<Long> subFileIds = fileIds.subList(offset, sum);
            List<UIFileInfo> uiFileInfos = new ArrayList<>(subFileIds.size());
            for (long fileId : subFileIds) {
                try {
                    URIStatus status = new URIStatus(mBlockWorker.getFileInfo(fileId));
                    UIFileInfo uiFileInfo = new UIFileInfo(status, ServerConfiguration.global(), new WorkerStorageTierAssoc().getOrderedStorageAliases());
                    for (long blockId : status.getBlockIds()) {
                        if (mBlockWorker.hasBlockMeta(blockId)) {
                            BlockMeta blockMeta = mBlockWorker.getVolatileBlockMeta(blockId);
                            long blockSize = blockMeta.getBlockSize();
                            // The block last access time is not available. Use -1 for now.
                            // It's not necessary to show location information here since
                            // we are viewing at the context of this worker.
                            uiFileInfo.addBlock(blockMeta.getBlockLocation().tierAlias(), blockId, blockSize, -1);
                        }
                    }
                    if (!uiFileInfo.getBlockIds().isEmpty()) {
                        uiFileInfos.add(uiFileInfo);
                    }
                } catch (Exception e) {
                    // The file might have been deleted, log a warning and ignore this file.
                    LOG.warn("Unable to get file info for fileId {}. {}", fileId, e.toString());
                }
            }
            response.setFileInfos(uiFileInfos);
        } catch (NumberFormatException e) {
            response.setFatalError("Error: offset or limit parse error, " + e.getLocalizedMessage());
        } catch (ArithmeticException e) {
            response.setFatalError("Error: offset or offset + limit is out ofbound, " + e.getLocalizedMessage());
        } catch (Exception e) {
            response.setFatalError(e.getLocalizedMessage());
        }
        return response;
    }, ServerConfiguration.global());
}
Also used : FileDoesNotExistException(alluxio.exception.FileDoesNotExistException) ArrayList(java.util.ArrayList) UIFileInfo(alluxio.util.webui.UIFileInfo) List(java.util.List) ArrayList(java.util.ArrayList) BlockDoesNotExistException(alluxio.exception.BlockDoesNotExistException) AlluxioException(alluxio.exception.AlluxioException) HashSet(java.util.HashSet) UIFileBlockInfo(alluxio.util.webui.UIFileBlockInfo) WorkerWebUIBlockInfo(alluxio.wire.WorkerWebUIBlockInfo) WorkerStorageTierAssoc(alluxio.WorkerStorageTierAssoc) IOException(java.io.IOException) URIStatus(alluxio.client.file.URIStatus) AlluxioException(alluxio.exception.AlluxioException) FileDoesNotExistException(alluxio.exception.FileDoesNotExistException) IOException(java.io.IOException) BlockDoesNotExistException(alluxio.exception.BlockDoesNotExistException) ImmutablePair(org.apache.commons.lang3.tuple.ImmutablePair) BlockMeta(alluxio.worker.block.meta.BlockMeta) Map(java.util.Map) SortedMap(java.util.SortedMap) TreeMap(java.util.TreeMap) BlockStoreMeta(alluxio.worker.block.BlockStoreMeta) AlluxioURI(alluxio.AlluxioURI) Path(javax.ws.rs.Path) GET(javax.ws.rs.GET)

Aggregations

AlluxioURI (alluxio.AlluxioURI)2 URIStatus (alluxio.client.file.URIStatus)2 AlluxioException (alluxio.exception.AlluxioException)2 FileDoesNotExistException (alluxio.exception.FileDoesNotExistException)2 UIFileBlockInfo (alluxio.util.webui.UIFileBlockInfo)2 UIFileInfo (alluxio.util.webui.UIFileInfo)2 IOException (java.io.IOException)2 ArrayList (java.util.ArrayList)2 GET (javax.ws.rs.GET)2 Path (javax.ws.rs.Path)2 MasterStorageTierAssoc (alluxio.MasterStorageTierAssoc)1 WorkerStorageTierAssoc (alluxio.WorkerStorageTierAssoc)1 FileInStream (alluxio.client.file.FileInStream)1 FileSystem (alluxio.client.file.FileSystem)1 AccessControlException (alluxio.exception.AccessControlException)1 BlockDoesNotExistException (alluxio.exception.BlockDoesNotExistException)1 InvalidPathException (alluxio.exception.InvalidPathException)1 UnavailableException (alluxio.exception.status.UnavailableException)1 OpenFilePOptions (alluxio.grpc.OpenFilePOptions)1 BlockLocation (alluxio.wire.BlockLocation)1