Examples with OpenFilePOptions alluxio.grpc.OpenFilePOptions used on opensource projects

Example 6 with OpenFilePOptions

use of alluxio.grpc.OpenFilePOptions in project alluxio by Alluxio.

the class AlluxioMasterRestServiceHandler method getWebUIBrowse.

/**
 * Gets Web UI browse page data.
 *
 * @param requestPath the request path
 * @param requestOffset the request offset
 * @param requestEnd the request end
 * @param requestLimit the request limit
 * @return the response object
 */
@GET
@Path(WEBUI_BROWSE)
public Response getWebUIBrowse(@DefaultValue("/") @QueryParam("path") String requestPath, @DefaultValue("0") @QueryParam("offset") String requestOffset, @DefaultValue("") @QueryParam("end") String requestEnd, @DefaultValue("20") @QueryParam("limit") String requestLimit) {
    return RestUtils.call(() -> {
        MasterWebUIBrowse response = new MasterWebUIBrowse();
        if (!ServerConfiguration.getBoolean(PropertyKey.WEB_FILE_INFO_ENABLED)) {
            return response;
        }
        if (SecurityUtils.isSecurityEnabled(ServerConfiguration.global()) && AuthenticatedClientUser.get(ServerConfiguration.global()) == null) {
            AuthenticatedClientUser.set(ServerUserState.global().getUser().getName());
        }
        response.setDebug(ServerConfiguration.getBoolean(PropertyKey.DEBUG)).setShowPermissions(ServerConfiguration.getBoolean(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_ENABLED)).setMasterNodeAddress(mMasterProcess.getRpcAddress().toString()).setInvalidPathError("");
        List<FileInfo> filesInfo;
        String path = URLDecoder.decode(requestPath, "UTF-8");
        if (path.isEmpty()) {
            path = AlluxioURI.SEPARATOR;
        }
        AlluxioURI currentPath = new AlluxioURI(path);
        response.setCurrentPath(currentPath.toString()).setViewingOffset(0);
        try {
            long fileId = mFileSystemMaster.getFileId(currentPath);
            FileInfo fileInfo = mFileSystemMaster.getFileInfo(fileId);
            UIFileInfo currentFileInfo = new UIFileInfo(fileInfo, ServerConfiguration.global(), new MasterStorageTierAssoc().getOrderedStorageAliases());
            if (currentFileInfo.getAbsolutePath() == null) {
                throw new FileDoesNotExistException(currentPath.toString());
            }
            response.setCurrentDirectory(currentFileInfo).setBlockSizeBytes(currentFileInfo.getBlockSizeBytes());
            if (!currentFileInfo.getIsDirectory()) {
                long relativeOffset = 0;
                long offset;
                try {
                    if (requestOffset != null) {
                        relativeOffset = Long.parseLong(requestOffset);
                    }
                } catch (NumberFormatException e) {
                // ignore the exception
                }
                // relative to the end of the file.
                if (requestEnd.equals("")) {
                    offset = relativeOffset;
                } else {
                    offset = fileInfo.getLength() - relativeOffset;
                }
                if (offset < 0) {
                    offset = 0;
                } else if (offset > fileInfo.getLength()) {
                    offset = fileInfo.getLength();
                }
                try {
                    AlluxioURI absolutePath = new AlluxioURI(currentFileInfo.getAbsolutePath());
                    FileSystem fs = mFsClient;
                    String fileData;
                    URIStatus status = fs.getStatus(absolutePath);
                    if (status.isCompleted()) {
                        OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).build();
                        try (FileInStream is = fs.openFile(absolutePath, options)) {
                            int len = (int) Math.min(5L * Constants.KB, status.getLength() - offset);
                            byte[] data = new byte[len];
                            long skipped = is.skip(offset);
                            if (skipped < 0) {
                                // nothing was skipped
                                fileData = "Unable to traverse to offset; is file empty?";
                            } else if (skipped < offset) {
                                // couldn't skip all the way to offset
                                fileData = "Unable to traverse to offset; is offset larger than the file?";
                            } else {
                                // read may not read up to len, so only convert what was read
                                int read = is.read(data, 0, len);
                                if (read < 0) {
                                    // stream couldn't read anything, skip went to EOF?
                                    fileData = "Unable to read file";
                                } else {
                                    fileData = WebUtils.convertByteArrayToStringWithoutEscape(data, 0, read);
                                }
                            }
                        }
                    } else {
                        fileData = "The requested file is not complete yet.";
                    }
                    List<UIFileBlockInfo> uiBlockInfo = new ArrayList<>();
                    for (FileBlockInfo fileBlockInfo : mFileSystemMaster.getFileBlockInfoList(absolutePath)) {
                        uiBlockInfo.add(new UIFileBlockInfo(fileBlockInfo, ServerConfiguration.global()));
                    }
                    response.setFileBlocks(uiBlockInfo).setFileData(fileData).setHighestTierAlias(mBlockMaster.getGlobalStorageTierAssoc().getAlias(0));
                } catch (AlluxioException e) {
                    throw new IOException(e);
                }
                response.setViewingOffset(offset);
                return response;
            }
            if (currentPath.isRoot()) {
                response.setPathInfos(new UIFileInfo[0]);
            } else {
                String[] splitPath = PathUtils.getPathComponents(currentPath.toString());
                UIFileInfo[] pathInfos = new UIFileInfo[splitPath.length - 1];
                fileId = mFileSystemMaster.getFileId(currentPath);
                pathInfos[0] = new UIFileInfo(mFileSystemMaster.getFileInfo(fileId), ServerConfiguration.global(), new MasterStorageTierAssoc().getOrderedStorageAliases());
                AlluxioURI breadcrumb = new AlluxioURI(AlluxioURI.SEPARATOR);
                for (int i = 1; i < splitPath.length - 1; i++) {
                    breadcrumb = breadcrumb.join(splitPath[i]);
                    fileId = mFileSystemMaster.getFileId(breadcrumb);
                    pathInfos[i] = new UIFileInfo(mFileSystemMaster.getFileInfo(fileId), ServerConfiguration.global(), new MasterStorageTierAssoc().getOrderedStorageAliases());
                }
                response.setPathInfos(pathInfos);
            }
            filesInfo = mFileSystemMaster.listStatus(currentPath, ListStatusContext.defaults());
        } catch (FileDoesNotExistException e) {
            response.setInvalidPathError("Error: Invalid Path " + e.getMessage());
            return response;
        } catch (InvalidPathException e) {
            response.setInvalidPathError("Error: Invalid Path " + e.getLocalizedMessage());
            return response;
        } catch (UnavailableException e) {
            response.setInvalidPathError("The service is temporarily unavailable. " + e.getMessage());
            return response;
        } catch (IOException e) {
            response.setInvalidPathError("Error: File " + currentPath + " is not available " + e.getMessage());
            return response;
        } catch (AccessControlException e) {
            response.setInvalidPathError("Error: File " + currentPath + " cannot be accessed " + e.getMessage());
            return response;
        }
        List<UIFileInfo> fileInfos = new ArrayList<>(filesInfo.size());
        for (FileInfo fileInfo : filesInfo) {
            UIFileInfo toAdd = new UIFileInfo(fileInfo, ServerConfiguration.global(), new MasterStorageTierAssoc().getOrderedStorageAliases());
            try {
                if (!toAdd.getIsDirectory() && fileInfo.getLength() > 0) {
                    FileBlockInfo blockInfo = mFileSystemMaster.getFileBlockInfoList(new AlluxioURI(toAdd.getAbsolutePath())).get(0);
                    List<String> locations = new ArrayList<>();
                    // add the in-Alluxio block locations
                    for (BlockLocation location : blockInfo.getBlockInfo().getLocations()) {
                        WorkerNetAddress address = location.getWorkerAddress();
                        locations.add(address.getHost() + ":" + address.getDataPort());
                    }
                    // add underFS locations
                    locations.addAll(blockInfo.getUfsLocations());
                    toAdd.setFileLocations(locations);
                }
            } catch (FileDoesNotExistException e) {
                response.setFileDoesNotExistException("Error: non-existing file " + e.getMessage());
                return response;
            } catch (InvalidPathException e) {
                response.setInvalidPathException("Error: invalid path " + e.getMessage());
                return response;
            } catch (AccessControlException e) {
                response.setAccessControlException("Error: File " + currentPath + " cannot be accessed " + e.getMessage());
                return response;
            }
            fileInfos.add(toAdd);
        }
        fileInfos.sort(UIFileInfo.PATH_STRING_COMPARE);
        response.setNTotalFile(fileInfos.size());
        try {
            int offset = Integer.parseInt(requestOffset);
            int limit = Integer.parseInt(requestLimit);
            limit = offset == 0 && limit > fileInfos.size() ? fileInfos.size() : limit;
            limit = offset + limit > fileInfos.size() ? fileInfos.size() - offset : limit;
            int sum = Math.addExact(offset, limit);
            fileInfos = fileInfos.subList(offset, sum);
            response.setFileInfos(fileInfos);
        } catch (NumberFormatException e) {
            response.setFatalError("Error: offset or limit parse error, " + e.getLocalizedMessage());
            return response;
        } catch (ArithmeticException e) {
            response.setFatalError("Error: offset or offset + limit is out of bound, " + e.getLocalizedMessage());
            return response;
        } catch (IllegalArgumentException e) {
            response.setFatalError(e.getLocalizedMessage());
            return response;
        }
        return response;
    }, ServerConfiguration.global());
}

Example 7 with OpenFilePOptions

use of alluxio.grpc.OpenFilePOptions in project alluxio by Alluxio.

the class CpCommandIntegrationTest method equals.

private boolean equals(AlluxioURI file1, AlluxioURI file2) throws Exception {
    try (Closer closer = Closer.create()) {
        OpenFilePOptions openFileOptions = OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).build();
        FileInStream is1 = closer.register(sFileSystem.openFile(file1, openFileOptions));
        FileInStream is2 = closer.register(sFileSystem.openFile(file2, openFileOptions));
        return IOUtils.contentEquals(is1, is2);
    }
}

Example 8 with OpenFilePOptions

use of alluxio.grpc.OpenFilePOptions in project alluxio by Alluxio.

the class BlockWorkerDataReaderTest method readChunkUfs.

@Test
public void readChunkUfs() throws Exception {
    // Write an actual file to UFS
    String testFilePath = File.createTempFile("temp", null, new File(mRootUfs)).getAbsolutePath();
    byte[] buffer = BufferUtils.getIncreasingByteArray(CHUNK_SIZE * 5);
    BufferUtils.writeBufferToFile(testFilePath, buffer);
    BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLength(CHUNK_SIZE * 5);
    URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true).setUfsPath(testFilePath).setBlockIds(Collections.singletonList(BLOCK_ID)).setLength(CHUNK_SIZE * 5).setBlockSizeBytes(CHUNK_SIZE).setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info))));
    OpenFilePOptions readOptions = OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).build();
    InStreamOptions options = new InStreamOptions(dummyStatus, readOptions, mConf);
    BlockWorkerDataReader.Factory factory = new BlockWorkerDataReader.Factory(mBlockWorker, BLOCK_ID, CHUNK_SIZE, options);
    int len = CHUNK_SIZE * 3 / 2;
    try (DataReader dataReader = factory.create(0, len)) {
        validateBuffer(dataReader.readChunk(), 0, CHUNK_SIZE);
        assertEquals(CHUNK_SIZE, dataReader.pos());
        validateBuffer(dataReader.readChunk(), CHUNK_SIZE, len - CHUNK_SIZE);
        assertEquals(len, dataReader.pos());
    }
}

Example 9 with OpenFilePOptions

use of alluxio.grpc.OpenFilePOptions in project alluxio by Alluxio.

the class JobUtils method loadThroughRead.

private static void loadThroughRead(URIStatus status, FileSystemContext context, long blockId, AlluxioConfiguration conf) throws IOException {
    // This does not work for remote worker unless we have passive cache on.
    AlluxioProperties prop = context.getClusterConf().copyProperties();
    prop.set(PropertyKey.USER_FILE_PASSIVE_CACHE_ENABLED, true);
    AlluxioConfiguration config = new InstancedConfiguration(prop);
    FileSystemContext loadContext = FileSystemContext.create(config);
    AlluxioBlockStore blockStore = AlluxioBlockStore.create(loadContext);
    OpenFilePOptions openOptions = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE).build();
    InStreamOptions inOptions = new InStreamOptions(status, openOptions, conf);
    inOptions.setUfsReadLocationPolicy(BlockLocationPolicy.Factory.create(LocalFirstPolicy.class.getCanonicalName(), conf));
    BlockInfo info = Preconditions.checkNotNull(status.getBlockInfo(blockId));
    try (InputStream inputStream = blockStore.getInStream(info, inOptions, ImmutableMap.of())) {
        while (inputStream.read(sIgnoredReadBuf) != -1) {
        }
    }
}

Example 10 with OpenFilePOptions

use of alluxio.grpc.OpenFilePOptions in project alluxio by Alluxio.

the class JobUtils method loadBlock.

/**
 * Loads a block into the local worker. If the block doesn't exist in Alluxio, it will be read
 * from the UFS.
 * @param status the uriStatus
 * @param context filesystem context
 * @param blockId the id of the block to load
 * @param address specify a worker to load into
 * @param directCache Use passive-cache or direct cache request
 */
public static void loadBlock(URIStatus status, FileSystemContext context, long blockId, WorkerNetAddress address, boolean directCache) throws AlluxioException, IOException {
    AlluxioConfiguration conf = ServerConfiguration.global();
    // Explicitly specified a worker to load
    WorkerNetAddress localNetAddress = address;
    String localHostName = NetworkAddressUtils.getConnectHost(ServiceType.WORKER_RPC, conf);
    List<WorkerNetAddress> netAddress = context.getCachedWorkers().stream().map(BlockWorkerInfo::getNetAddress).filter(x -> Objects.equals(x.getHost(), localHostName)).collect(Collectors.toList());
    if (localNetAddress == null && !netAddress.isEmpty()) {
        localNetAddress = netAddress.get(0);
    }
    if (localNetAddress == null) {
        throw new NotFoundException(ExceptionMessage.NO_LOCAL_BLOCK_WORKER_LOAD_TASK.getMessage(blockId));
    }
    Set<String> pinnedLocation = status.getPinnedMediumTypes();
    if (pinnedLocation.size() > 1) {
        throw new AlluxioException(ExceptionMessage.PINNED_TO_MULTIPLE_MEDIUMTYPES.getMessage(status.getPath()));
    }
    // Only use this read local first method to load if nearest worker is clear
    if (netAddress.size() <= 1 && pinnedLocation.isEmpty() && status.isPersisted()) {
        if (directCache) {
            loadThroughCacheRequest(status, context, blockId, conf, localNetAddress);
        } else {
            loadThroughRead(status, context, blockId, conf);
        }
        return;
    }
    // TODO(bin): remove the following case when we consolidate tier and medium
    // since there is only one element in the set, we take the first element in the set
    String medium = pinnedLocation.isEmpty() ? "" : pinnedLocation.iterator().next();
    OpenFilePOptions openOptions = OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).build();
    InStreamOptions inOptions = new InStreamOptions(status, openOptions, conf);
    // Set read location policy always to local first for loading blocks for job tasks
    inOptions.setUfsReadLocationPolicy(BlockLocationPolicy.Factory.create(LocalFirstPolicy.class.getCanonicalName(), conf));
    OutStreamOptions outOptions = OutStreamOptions.defaults(context.getClientContext());
    outOptions.setMediumType(medium);
    // Set write location policy always to local first for loading blocks for job tasks
    outOptions.setLocationPolicy(BlockLocationPolicy.Factory.create(LocalFirstPolicy.class.getCanonicalName(), conf));
    BlockInfo blockInfo = status.getBlockInfo(blockId);
    Preconditions.checkNotNull(blockInfo, "Can not find block %s in status %s", blockId, status);
    long blockSize = blockInfo.getLength();
    AlluxioBlockStore blockStore = AlluxioBlockStore.create(context);
    try (OutputStream outputStream = blockStore.getOutStream(blockId, blockSize, localNetAddress, outOptions)) {
        try (InputStream inputStream = blockStore.getInStream(blockId, inOptions)) {
            ByteStreams.copy(inputStream, outputStream);
        } catch (Throwable t) {
            try {
                ((Cancelable) outputStream).cancel();
            } catch (Throwable t2) {
                t.addSuppressed(t2);
            }
            throw t;
        }
    }
}