Examples with CreateFilePOptions alluxio.grpc.CreateFilePOptions used on opensource projects

Example 26 with CreateFilePOptions

use of alluxio.grpc.CreateFilePOptions in project alluxio by Alluxio.

the class UfsJournalIntegrationTest method addBlock.

/**
 * Tests adding a block.
 */
@Test
public void addBlock() throws Exception {
    AlluxioURI uri = new AlluxioURI("/xyz");
    CreateFilePOptions options = CreateFilePOptions.newBuilder().setBlockSizeBytes(64).setRecursive(true).build();
    FileOutStream os = mFileSystem.createFile(uri, options);
    for (int k = 0; k < 1000; k++) {
        os.write(k);
    }
    os.close();
    URIStatus status = mFileSystem.getStatus(uri);
    mLocalAlluxioCluster.stopFS();
    addBlockTestUtil(status);
}

Example 27 with CreateFilePOptions

use of alluxio.grpc.CreateFilePOptions in project alluxio by Alluxio.

the class UfsJournalIntegrationTest method pin.

/**
 * Tests journalling of inodes being pinned.
 */
@Test
public void pin() throws Exception {
    SetAttributePOptions setPinned = SetAttributePOptions.newBuilder().setPinned(true).build();
    SetAttributePOptions setUnpinned = SetAttributePOptions.newBuilder().setPinned(false).build();
    AlluxioURI dirUri = new AlluxioURI("/myFolder");
    mFileSystem.createDirectory(dirUri);
    mFileSystem.setAttribute(dirUri, setPinned);
    AlluxioURI file0Path = new AlluxioURI("/myFolder/file0");
    CreateFilePOptions op = CreateFilePOptions.newBuilder().setBlockSizeBytes(64).build();
    mFileSystem.createFile(file0Path, op).close();
    mFileSystem.setAttribute(file0Path, setUnpinned);
    AlluxioURI file1Path = new AlluxioURI("/myFolder/file1");
    mFileSystem.createFile(file1Path, op).close();
    URIStatus directoryStatus = mFileSystem.getStatus(dirUri);
    URIStatus file0Status = mFileSystem.getStatus(file0Path);
    URIStatus file1Status = mFileSystem.getStatus(file1Path);
    mLocalAlluxioCluster.stopFS();
    pinTestUtil(directoryStatus, file0Status, file1Status);
    deleteFsMasterJournalLogs();
    pinTestUtil(directoryStatus, file0Status, file1Status);
}

Example 28 with CreateFilePOptions

use of alluxio.grpc.CreateFilePOptions in project alluxio by Alluxio.

the class MultiMount method main.

/**
 * Entry point for the {@link MultiMount} program.
 *
 * @param args command-line arguments
 */
public static void main(String[] args) {
    if (args.length != 1) {
        System.err.println("Usage: ./bin/alluxio runClass alluxio.examples.MultiMount <HDFS_URL>");
        System.exit(-1);
    }
    AlluxioConfiguration alluxioConf = new InstancedConfiguration(ConfigurationUtils.defaults());
    AlluxioURI mntPath = new AlluxioURI("/mnt");
    AlluxioURI s3Mount = new AlluxioURI("/mnt/s3");
    AlluxioURI inputPath = new AlluxioURI("/mnt/s3/hello.txt");
    AlluxioURI s3Path = new AlluxioURI("s3a://alluxio-demo/");
    AlluxioURI hdfsMount = new AlluxioURI("/mnt/hdfs");
    AlluxioURI outputPath = new AlluxioURI("/mnt/hdfs/hello.txt");
    AlluxioURI hdfsPath = new AlluxioURI(args[0]);
    FileSystem fileSystem = FileSystem.Factory.create(alluxioConf);
    try {
        // Make sure mount directory exists.
        if (!fileSystem.exists(mntPath)) {
            System.out.print("creating " + mntPath + " ... ");
            fileSystem.createDirectory(mntPath);
            System.out.println("done");
        }
        // Make sure the S3 mount point does not exist.
        if (fileSystem.exists(s3Mount)) {
            System.out.print("unmounting " + s3Mount + " ... ");
            fileSystem.unmount(s3Mount);
            System.out.println("done");
        }
        // Make sure the HDFS mount point does not exist.
        if (fileSystem.exists(hdfsMount)) {
            System.out.print("unmounting " + hdfsMount + " ... ");
            fileSystem.unmount(hdfsMount);
            System.out.println("done");
        }
        // Mount S3.
        System.out.print("mounting " + s3Path + " to " + s3Mount + " ... ");
        fileSystem.mount(s3Mount, s3Path);
        System.out.println("done");
        // Mount HDFS.
        System.out.print("mounting " + hdfsPath + " to " + hdfsMount + " ... ");
        fileSystem.mount(hdfsMount, hdfsPath);
        System.out.println("done");
        // Make sure output file does not exist.
        if (fileSystem.exists(outputPath)) {
            System.out.print("deleting " + outputPath + " ... ");
            fileSystem.delete(outputPath);
            System.out.println("done");
        }
        // Open the input stream.
        System.out.print("opening " + inputPath + " ... ");
        FileInStream is = fileSystem.openFile(inputPath);
        System.out.println("done");
        // Open the output stream, setting the write type to make sure result is persisted.
        System.out.print("opening " + outputPath + " ... ");
        CreateFilePOptions options = CreateFilePOptions.newBuilder().setWriteType(WritePType.CACHE_THROUGH).setRecursive(true).build();
        FileOutStream os = fileSystem.createFile(outputPath, options);
        System.out.println("done");
        // Copy the data
        System.out.print("transferring data from " + inputPath + " to " + outputPath + " ... ");
        IOUtils.copy(is, os);
        System.out.println("done");
        // Close the input stream.
        System.out.print("closing " + inputPath + " ... ");
        is.close();
        System.out.println("done");
        // Close the output stream.
        System.out.print("closing " + outputPath + " ... ");
        os.close();
        System.out.println("done");
    } catch (Exception e) {
        System.out.println("fail");
        e.printStackTrace();
    } finally {
        // Make sure the S3 mount point is removed.
        try {
            if (fileSystem.exists(s3Mount)) {
                System.out.print("unmounting " + s3Mount + " ... ");
                fileSystem.unmount(s3Mount);
                System.out.println("done");
            }
        } catch (Exception e) {
            System.out.println("fail");
            e.printStackTrace();
        }
        // Make sure the HDFS mount point is removed.
        try {
            if (fileSystem.exists(hdfsMount)) {
                System.out.print("unmounting " + hdfsMount + " ... ");
                fileSystem.unmount(hdfsMount);
                System.out.println("done");
            }
        } catch (Exception e) {
            System.out.println("fail");
            e.printStackTrace();
        }
    }
}

Example 29 with CreateFilePOptions

use of alluxio.grpc.CreateFilePOptions in project alluxio by Alluxio.

the class MigrateDefinition method migrate.

/**
 * @param command the migrate command to execute
 * @param writeType the write type to use for the moved file
 * @param fileSystem the Alluxio file system
 * @param overwrite whether to overwrite destination
 */
private static void migrate(MigrateCommand command, WritePType writeType, FileSystem fileSystem, boolean overwrite) throws Exception {
    String source = command.getSource();
    String destination = command.getDestination();
    LOG.debug("Migrating {} to {}", source, destination);
    CreateFilePOptions createOptions = CreateFilePOptions.newBuilder().setWriteType(writeType).build();
    OpenFilePOptions openFileOptions = OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).build();
    final AlluxioURI destinationURI = new AlluxioURI(destination);
    boolean retry;
    do {
        retry = false;
        try (FileInStream in = fileSystem.openFile(new AlluxioURI(source), openFileOptions);
            FileOutStream out = fileSystem.createFile(destinationURI, createOptions)) {
            try {
                IOUtils.copyLarge(in, out, new byte[8 * Constants.MB]);
            } catch (Throwable t) {
                try {
                    out.cancel();
                } catch (Throwable t2) {
                    t.addSuppressed(t2);
                }
                throw t;
            }
        } catch (FileAlreadyExistsException e) {
            if (overwrite) {
                fileSystem.delete(destinationURI);
                retry = true;
            } else {
                throw e;
            }
        }
    } while (retry);
}

Example 30 with CreateFilePOptions

use of alluxio.grpc.CreateFilePOptions in project alluxio by Alluxio.

the class PinIntegrationTest method createEmptyFile.

private void createEmptyFile(AlluxioURI fileURI) throws IOException, AlluxioException {
    CreateFilePOptions options = CreateFilePOptions.newBuilder().setWriteType(WritePType.MUST_CACHE).build();
    FileOutStream os = mFileSystem.createFile(fileURI, options);
    os.close();
}