Examples with OpenFilePOptions alluxio.grpc.OpenFilePOptions used on opensource projects

Example 11 with OpenFilePOptions

use of alluxio.grpc.OpenFilePOptions in project alluxio by Alluxio.

the class MigrateDefinition method migrate.

/**
 * @param command the migrate command to execute
 * @param writeType the write type to use for the moved file
 * @param fileSystem the Alluxio file system
 * @param overwrite whether to overwrite destination
 */
private static void migrate(MigrateCommand command, WritePType writeType, FileSystem fileSystem, boolean overwrite) throws Exception {
    String source = command.getSource();
    String destination = command.getDestination();
    LOG.debug("Migrating {} to {}", source, destination);
    CreateFilePOptions createOptions = CreateFilePOptions.newBuilder().setWriteType(writeType).build();
    OpenFilePOptions openFileOptions = OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).build();
    final AlluxioURI destinationURI = new AlluxioURI(destination);
    boolean retry;
    do {
        retry = false;
        try (FileInStream in = fileSystem.openFile(new AlluxioURI(source), openFileOptions);
            FileOutStream out = fileSystem.createFile(destinationURI, createOptions)) {
            try {
                IOUtils.copyLarge(in, out, new byte[8 * Constants.MB]);
            } catch (Throwable t) {
                try {
                    out.cancel();
                } catch (Throwable t2) {
                    t.addSuppressed(t2);
                }
                throw t;
            }
        } catch (FileAlreadyExistsException e) {
            if (overwrite) {
                fileSystem.delete(destinationURI);
                retry = true;
            } else {
                throw e;
            }
        }
    } while (retry);
}

Example 12 with OpenFilePOptions

use of alluxio.grpc.OpenFilePOptions in project alluxio by Alluxio.

the class BasicNonByteBufferOperations method read.

private boolean read(FileSystem alluxioClient) throws IOException, AlluxioException {
    OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(mReadType.toProto()).build();
    boolean pass = true;
    long startTimeMs = CommonUtils.getCurrentMs();
    try (DataInputStream input = new DataInputStream(alluxioClient.openFile(mFilePath, options))) {
        int length = input.readInt();
        for (int i = 0; i < length; i++) {
            if (input.readInt() != i) {
                pass = false;
                break;
            }
        }
    }
    LOG.info(FormatUtils.formatTimeTakenMs(startTimeMs, "readFile file " + mFilePath));
    return pass;
}

Example 13 with OpenFilePOptions

use of alluxio.grpc.OpenFilePOptions in project alluxio by Alluxio.

the class FileInStreamIntegrationTest method remoteReadLargeFile.

/**
 * Read large file remotely. Make sure the test does not timeout.
 */
@Test(timeout = 30000)
@LocalAlluxioClusterResource.Config(confParams = { PropertyKey.Name.USER_SHORT_CIRCUIT_ENABLED, "false", PropertyKey.Name.USER_BLOCK_SIZE_BYTES_DEFAULT, "16MB", PropertyKey.Name.USER_STREAMING_READER_CHUNK_SIZE_BYTES, "64KB", PropertyKey.Name.WORKER_RAMDISK_SIZE, "1GB" })
public void remoteReadLargeFile() throws Exception {
    // write a file outside of Alluxio
    AlluxioURI filePath = new AlluxioURI(mTestPath + "/test");
    try (FileOutStream os = mFileSystem.createFile(filePath, CreateFilePOptions.newBuilder().setBlockSizeBytes(16 * Constants.MB).setWriteType(WritePType.THROUGH).build())) {
        // Write a smaller byte array 10 times to avoid demanding 500mb of contiguous memory.
        byte[] bytes = BufferUtils.getIncreasingByteArray(50 * Constants.MB);
        for (int i = 0; i < 10; i++) {
            os.write(bytes);
        }
    }
    OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build();
    try (FileInStream in = mFileSystem.openFile(filePath, options)) {
        byte[] buf = new byte[8 * Constants.MB];
        while (in.read(buf) != -1) {
        }
    }
}

Example 14 with OpenFilePOptions

use of alluxio.grpc.OpenFilePOptions in project alluxio by Alluxio.

the class PersistDefinition method runTask.

@Override
public SerializableVoid runTask(PersistConfig config, SerializableVoid args, RunTaskContext context) throws Exception {
    AlluxioURI uri = new AlluxioURI(config.getFilePath());
    String ufsPath = config.getUfsPath();
    // check if the file is persisted in UFS and delete it, if we are overwriting it
    UfsManager.UfsClient ufsClient = context.getUfsManager().get(config.getMountId());
    try (CloseableResource<UnderFileSystem> ufsResource = ufsClient.acquireUfsResource()) {
        UnderFileSystem ufs = ufsResource.get();
        if (ufs == null) {
            throw new IOException("Failed to create UFS instance for " + ufsPath);
        }
        if (ufs.exists(ufsPath)) {
            if (config.isOverwrite()) {
                LOG.info("File {} is already persisted in UFS. Removing it.", config.getFilePath());
                ufs.deleteExistingFile(ufsPath);
            } else {
                throw new IOException("File " + config.getFilePath() + " is already persisted in UFS, to overwrite the file, please set the overwrite flag" + " in the config.");
            }
        }
        URIStatus uriStatus = context.getFileSystem().getStatus(uri);
        if (!uriStatus.isCompleted()) {
            throw new IOException("Cannot persist an incomplete Alluxio file: " + uri);
        }
        long bytesWritten;
        try (Closer closer = Closer.create()) {
            OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).setUpdateLastAccessTime(false).build();
            FileInStream in = closer.register(context.getFileSystem().openFile(uri, options));
            AlluxioURI dstPath = new AlluxioURI(ufsPath);
            // Create ancestor directories from top to the bottom. We cannot use recursive create
            // parents here because the permission for the ancestors can be different.
            Stack<Pair<String, String>> ancestorUfsAndAlluxioPaths = new Stack<>();
            AlluxioURI curAlluxioPath = uri.getParent();
            AlluxioURI curUfsPath = dstPath.getParent();
            // exist.
            while (!ufs.isDirectory(curUfsPath.toString()) && curAlluxioPath != null) {
                ancestorUfsAndAlluxioPaths.push(new Pair<>(curUfsPath.toString(), curAlluxioPath.toString()));
                curAlluxioPath = curAlluxioPath.getParent();
                curUfsPath = curUfsPath.getParent();
            }
            while (!ancestorUfsAndAlluxioPaths.empty()) {
                Pair<String, String> ancestorUfsAndAlluxioPath = ancestorUfsAndAlluxioPaths.pop();
                String ancestorUfsPath = ancestorUfsAndAlluxioPath.getFirst();
                String ancestorAlluxioPath = ancestorUfsAndAlluxioPath.getSecond();
                URIStatus status = context.getFileSystem().getStatus(new AlluxioURI(ancestorAlluxioPath));
                MkdirsOptions mkdirOptions = MkdirsOptions.defaults(ServerConfiguration.global()).setCreateParent(false).setOwner(status.getOwner()).setGroup(status.getGroup()).setMode(new Mode((short) status.getMode()));
                // and assume the directory is already prepared, regardless of permission matching.
                if (ufs.mkdirs(ancestorUfsPath, mkdirOptions)) {
                    List<AclEntry> allAcls = Stream.concat(status.getDefaultAcl().getEntries().stream(), status.getAcl().getEntries().stream()).collect(Collectors.toList());
                    ufs.setAclEntries(ancestorUfsPath, allAcls);
                } else if (!ufs.isDirectory(ancestorUfsPath)) {
                    throw new IOException("Failed to create " + ufsPath + " with permission " + options.toString() + " because its ancestor " + ancestorUfsPath + " is not a directory");
                }
            }
            OutputStream out = closer.register(ufs.createNonexistingFile(dstPath.toString(), CreateOptions.defaults(ServerConfiguration.global()).setOwner(uriStatus.getOwner()).setGroup(uriStatus.getGroup()).setMode(new Mode((short) uriStatus.getMode()))));
            URIStatus status = context.getFileSystem().getStatus(uri);
            List<AclEntry> allAcls = Stream.concat(status.getDefaultAcl().getEntries().stream(), status.getAcl().getEntries().stream()).collect(Collectors.toList());
            ufs.setAclEntries(dstPath.toString(), allAcls);
            bytesWritten = IOUtils.copyLarge(in, out, new byte[8 * Constants.MB]);
            incrementPersistedMetric(ufsClient.getUfsMountPointUri(), bytesWritten);
        }
        LOG.info("Persisted file {} with size {}", ufsPath, bytesWritten);
    }
    return null;
}

Example 15 with OpenFilePOptions

use of alluxio.grpc.OpenFilePOptions in project alluxio by Alluxio.

the class AlluxioFileInStreamTest method shortSeekBackwardCachingPartiallyReadBlocks.

/**
 * Tests seeking with incomplete block caching enabled. It seeks backward within 1 block.
 */
@Test
public void shortSeekBackwardCachingPartiallyReadBlocks() throws IOException {
    OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build();
    mTestStream = new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, options, mConf), mContext);
    int seekAmount = (int) (BLOCK_LENGTH / 4);
    int readAmount = (int) (BLOCK_LENGTH * 2 - BLOCK_LENGTH / 2);
    byte[] buffer = new byte[readAmount];
    mTestStream.read(buffer);
    // Seek backward.
    mTestStream.seek(readAmount - seekAmount);
    // Block 1 is cached though it is not fully read.
    validatePartialCaching(1, (int) BLOCK_LENGTH / 2);
    // Seek many times. It will cache block 1 only once.
    for (int i = 0; i <= seekAmount; i++) {
        mTestStream.seek(readAmount - seekAmount - i);
    }
    validatePartialCaching(1, (int) BLOCK_LENGTH / 2);
}