Examples with AlluxioConfiguration alluxio.conf.AlluxioConfiguration used on opensource projects

Example 21 with AlluxioConfiguration

use of alluxio.conf.AlluxioConfiguration in project alluxio by Alluxio.

the class AlluxioMasterMonitor method main.

/**
 * Starts the Alluxio master monitor.
 *
 * @param args command line arguments, should be empty
 */
public static void main(String[] args) {
    if (args.length != 0) {
        LOG.warn("java -cp {} {}", RuntimeConstants.ALLUXIO_JAR, AlluxioMasterMonitor.class.getCanonicalName());
        LOG.warn("ignoring arguments");
    }
    AlluxioConfiguration alluxioConf = new InstancedConfiguration(ConfigurationUtils.defaults());
    MasterHealthCheckClient.Builder builder = new MasterHealthCheckClient.Builder(alluxioConf);
    if (ConfigurationUtils.isHaMode(alluxioConf)) {
        builder.withProcessCheck(true);
    } else {
        builder.withProcessCheck(false);
    }
    HealthCheckClient client = builder.build();
    if (!client.isServing()) {
        System.exit(1);
    }
    System.exit(0);
}

Example 22 with AlluxioConfiguration

use of alluxio.conf.AlluxioConfiguration in project alluxio by Alluxio.

the class AlluxioJobMasterMonitor method main.

/**
 * Starts the Alluxio job_master monitor.
 *
 * @param args command line arguments, should be empty
 */
public static void main(String[] args) {
    if (args.length != 0) {
        LOG.info("java -cp {} {}", RuntimeConstants.ALLUXIO_JAR, AlluxioJobMasterMonitor.class.getCanonicalName());
        LOG.warn("ignoring arguments");
    }
    AlluxioConfiguration conf = new InstancedConfiguration(ConfigurationUtils.defaults());
    HealthCheckClient client;
    // checking for the running process.
    if (ConfigurationUtils.isHaMode(conf)) {
        client = new MasterHealthCheckClient.Builder(conf).withAlluxioMasterType(MasterHealthCheckClient.MasterType.JOB_MASTER).build();
    } else {
        client = new JobMasterRpcHealthCheckClient(NetworkAddressUtils.getConnectAddress(NetworkAddressUtils.ServiceType.JOB_MASTER_RPC, conf), AlluxioMasterMonitor.TWO_MIN_EXP_BACKOFF, conf);
    }
    if (!client.isServing()) {
        System.exit(1);
    }
    System.exit(0);
}

Example 23 with AlluxioConfiguration

use of alluxio.conf.AlluxioConfiguration in project alluxio by Alluxio.

the class LocalUnderFileSystemFactoryTest method factory.

/**
 * This test ensures the local UFS module correctly accepts paths that begin with / or file://.
 */
@Test
public void factory() {
    AlluxioConfiguration conf = new InstancedConfiguration(ConfigurationUtils.defaults());
    UnderFileSystemFactory factory = UnderFileSystemFactoryRegistry.find("/local/test/path", conf);
    UnderFileSystemFactory factory2 = UnderFileSystemFactoryRegistry.find("file://local/test/path", conf);
    UnderFileSystemFactory factory3 = UnderFileSystemFactoryRegistry.find("hdfs://test-bucket/path", conf);
    UnderFileSystemFactory factory4 = UnderFileSystemFactoryRegistry.find("R:\\ramfs\\", conf);
    UnderFileSystemFactory factory5 = UnderFileSystemFactoryRegistry.find("file://R:/famfs", conf);
    UnderFileSystemFactory factory6 = UnderFileSystemFactoryRegistry.find("R:/ramfs/", conf);
    Assert.assertNotNull("A UnderFileSystemFactory should exist for local paths when using this module", factory);
    Assert.assertNotNull("A UnderFileSystemFactory should exist for local paths when using this module", factory2);
    Assert.assertNull("A UnderFileSystemFactory should not exist for non local paths when using this module", factory3);
    Assert.assertNotNull("A UnderFileSystemFactory should exist for local paths when using this module", factory4);
    Assert.assertNotNull("A UnderFileSystemFactory should exist for local paths when using this module", factory5);
    Assert.assertNotNull("A UnderFileSystemFactory should exist for local paths when using this module", factory6);
}

Example 24 with AlluxioConfiguration

use of alluxio.conf.AlluxioConfiguration in project alluxio by Alluxio.

the class MiniBenchmark method main.

/**
 * @param args there are no arguments needed
 * @throws Exception if error occurs during tests
 */
public static void main(String[] args) throws Exception {
    if (!parseInputArgs(args)) {
        usage();
        System.exit(-1);
    }
    if (sHelp) {
        usage();
        System.exit(0);
    }
    CommonUtils.warmUpLoop();
    AlluxioConfiguration alluxioConf = new InstancedConfiguration(ConfigurationUtils.defaults());
    for (int i = 0; i < sIterations; ++i) {
        final AtomicInteger count = new AtomicInteger(0);
        final CyclicBarrier barrier = new CyclicBarrier(sConcurrency);
        ExecutorService executorService = Executors.newFixedThreadPool(sConcurrency);
        final AtomicLong runtime = new AtomicLong(0);
        for (int j = 0; j < sConcurrency; ++j) {
            switch(sType) {
                case READ:
                    executorService.submit(() -> {
                        try {
                            readFile(barrier, runtime, count.addAndGet(1), alluxioConf);
                        } catch (Exception e) {
                            LOG.error("Failed to read file.", e);
                            System.exit(-1);
                        }
                    });
                    break;
                case WRITE:
                    executorService.submit(() -> {
                        try {
                            writeFile(barrier, runtime, count.addAndGet(1), alluxioConf);
                        } catch (Exception e) {
                            LOG.error("Failed to write file.", e);
                            System.exit(-1);
                        }
                    });
                    break;
                default:
                    throw new RuntimeException("Unsupported type.");
            }
        }
        executorService.shutdown();
        Preconditions.checkState(executorService.awaitTermination(1, TimeUnit.HOURS));
        double time = runtime.get() * 1.0 / sConcurrency / Constants.SECOND_NANO;
        System.out.printf("Iteration: %d; Duration: %f seconds; Aggregated throughput: %f GB/second.%n", i, time, sConcurrency * 1.0 * sFileSize / time / Constants.GB);
    }
}

Example 25 with AlluxioConfiguration

use of alluxio.conf.AlluxioConfiguration in project alluxio by Alluxio.

the class MultiMount method main.

/**
 * Entry point for the {@link MultiMount} program.
 *
 * @param args command-line arguments
 */
public static void main(String[] args) {
    if (args.length != 1) {
        System.err.println("Usage: ./bin/alluxio runClass alluxio.examples.MultiMount <HDFS_URL>");
        System.exit(-1);
    }
    AlluxioConfiguration alluxioConf = new InstancedConfiguration(ConfigurationUtils.defaults());
    AlluxioURI mntPath = new AlluxioURI("/mnt");
    AlluxioURI s3Mount = new AlluxioURI("/mnt/s3");
    AlluxioURI inputPath = new AlluxioURI("/mnt/s3/hello.txt");
    AlluxioURI s3Path = new AlluxioURI("s3a://alluxio-demo/");
    AlluxioURI hdfsMount = new AlluxioURI("/mnt/hdfs");
    AlluxioURI outputPath = new AlluxioURI("/mnt/hdfs/hello.txt");
    AlluxioURI hdfsPath = new AlluxioURI(args[0]);
    FileSystem fileSystem = FileSystem.Factory.create(alluxioConf);
    try {
        // Make sure mount directory exists.
        if (!fileSystem.exists(mntPath)) {
            System.out.print("creating " + mntPath + " ... ");
            fileSystem.createDirectory(mntPath);
            System.out.println("done");
        }
        // Make sure the S3 mount point does not exist.
        if (fileSystem.exists(s3Mount)) {
            System.out.print("unmounting " + s3Mount + " ... ");
            fileSystem.unmount(s3Mount);
            System.out.println("done");
        }
        // Make sure the HDFS mount point does not exist.
        if (fileSystem.exists(hdfsMount)) {
            System.out.print("unmounting " + hdfsMount + " ... ");
            fileSystem.unmount(hdfsMount);
            System.out.println("done");
        }
        // Mount S3.
        System.out.print("mounting " + s3Path + " to " + s3Mount + " ... ");
        fileSystem.mount(s3Mount, s3Path);
        System.out.println("done");
        // Mount HDFS.
        System.out.print("mounting " + hdfsPath + " to " + hdfsMount + " ... ");
        fileSystem.mount(hdfsMount, hdfsPath);
        System.out.println("done");
        // Make sure output file does not exist.
        if (fileSystem.exists(outputPath)) {
            System.out.print("deleting " + outputPath + " ... ");
            fileSystem.delete(outputPath);
            System.out.println("done");
        }
        // Open the input stream.
        System.out.print("opening " + inputPath + " ... ");
        FileInStream is = fileSystem.openFile(inputPath);
        System.out.println("done");
        // Open the output stream, setting the write type to make sure result is persisted.
        System.out.print("opening " + outputPath + " ... ");
        CreateFilePOptions options = CreateFilePOptions.newBuilder().setWriteType(WritePType.CACHE_THROUGH).setRecursive(true).build();
        FileOutStream os = fileSystem.createFile(outputPath, options);
        System.out.println("done");
        // Copy the data
        System.out.print("transferring data from " + inputPath + " to " + outputPath + " ... ");
        IOUtils.copy(is, os);
        System.out.println("done");
        // Close the input stream.
        System.out.print("closing " + inputPath + " ... ");
        is.close();
        System.out.println("done");
        // Close the output stream.
        System.out.print("closing " + outputPath + " ... ");
        os.close();
        System.out.println("done");
    } catch (Exception e) {
        System.out.println("fail");
        e.printStackTrace();
    } finally {
        // Make sure the S3 mount point is removed.
        try {
            if (fileSystem.exists(s3Mount)) {
                System.out.print("unmounting " + s3Mount + " ... ");
                fileSystem.unmount(s3Mount);
                System.out.println("done");
            }
        } catch (Exception e) {
            System.out.println("fail");
            e.printStackTrace();
        }
        // Make sure the HDFS mount point is removed.
        try {
            if (fileSystem.exists(hdfsMount)) {
                System.out.print("unmounting " + hdfsMount + " ... ");
                fileSystem.unmount(hdfsMount);
                System.out.println("done");
            }
        } catch (Exception e) {
            System.out.println("fail");
            e.printStackTrace();
        }
    }
}