Examples with InstancedConfiguration alluxio.conf.InstancedConfiguration used on opensource projects

Example 36 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class OSSUnderFileSystemFactoryTest method factory.

/**
 * Tests that the OSS UFS module correctly accepts paths that begin with oss://.
 */
@Test
public void factory() {
    UnderFileSystemFactory factory = UnderFileSystemFactoryRegistry.find("oss://test-bucket/path", new InstancedConfiguration(ConfigurationUtils.defaults()));
    Assert.assertNotNull("A UnderFileSystemFactory should exist for oss paths when using this module", factory);
}

Example 37 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class LocalUnderFileSystemFactoryTest method factory.

/**
 * This test ensures the local UFS module correctly accepts paths that begin with / or file://.
 */
@Test
public void factory() {
    AlluxioConfiguration conf = new InstancedConfiguration(ConfigurationUtils.defaults());
    UnderFileSystemFactory factory = UnderFileSystemFactoryRegistry.find("/local/test/path", conf);
    UnderFileSystemFactory factory2 = UnderFileSystemFactoryRegistry.find("file://local/test/path", conf);
    UnderFileSystemFactory factory3 = UnderFileSystemFactoryRegistry.find("hdfs://test-bucket/path", conf);
    UnderFileSystemFactory factory4 = UnderFileSystemFactoryRegistry.find("R:\\ramfs\\", conf);
    UnderFileSystemFactory factory5 = UnderFileSystemFactoryRegistry.find("file://R:/famfs", conf);
    UnderFileSystemFactory factory6 = UnderFileSystemFactoryRegistry.find("R:/ramfs/", conf);
    Assert.assertNotNull("A UnderFileSystemFactory should exist for local paths when using this module", factory);
    Assert.assertNotNull("A UnderFileSystemFactory should exist for local paths when using this module", factory2);
    Assert.assertNull("A UnderFileSystemFactory should not exist for non local paths when using this module", factory3);
    Assert.assertNotNull("A UnderFileSystemFactory should exist for local paths when using this module", factory4);
    Assert.assertNotNull("A UnderFileSystemFactory should exist for local paths when using this module", factory5);
    Assert.assertNotNull("A UnderFileSystemFactory should exist for local paths when using this module", factory6);
}

Example 38 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class RpcClient method createChannel.

/**
 * Creates a gRPC channel that uses NOSASL for security authentication type.
 *
 * @param addr the address to connect to
 * @param conf the configuration used to make the connection
 * @return new instance of {@link GrpcChannel}
 * @throws AlluxioStatusException
 */
public static GrpcChannel createChannel(InetSocketAddress addr, AlluxioConfiguration conf) throws AlluxioStatusException {
    InstancedConfiguration modifiedConfig = InstancedConfiguration.defaults();
    Map<String, Object> properties = new HashMap<>(conf.toMap());
    properties.put(PropertyKey.SECURITY_AUTHENTICATION_TYPE.getName(), AuthType.NOSASL.toString());
    modifiedConfig.merge(properties, Source.RUNTIME);
    LOG.info("Auth type = {}", modifiedConfig.get(PropertyKey.SECURITY_AUTHENTICATION_TYPE));
    GrpcChannelBuilder builder = GrpcChannelBuilder.newBuilder(GrpcServerAddress.create(addr), modifiedConfig);
    return builder.setSubject(ServerUserState.global().getSubject()).build();
}

Example 39 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class MiniBenchmark method main.

/**
 * @param args there are no arguments needed
 * @throws Exception if error occurs during tests
 */
public static void main(String[] args) throws Exception {
    if (!parseInputArgs(args)) {
        usage();
        System.exit(-1);
    }
    if (sHelp) {
        usage();
        System.exit(0);
    }
    CommonUtils.warmUpLoop();
    AlluxioConfiguration alluxioConf = new InstancedConfiguration(ConfigurationUtils.defaults());
    for (int i = 0; i < sIterations; ++i) {
        final AtomicInteger count = new AtomicInteger(0);
        final CyclicBarrier barrier = new CyclicBarrier(sConcurrency);
        ExecutorService executorService = Executors.newFixedThreadPool(sConcurrency);
        final AtomicLong runtime = new AtomicLong(0);
        for (int j = 0; j < sConcurrency; ++j) {
            switch(sType) {
                case READ:
                    executorService.submit(() -> {
                        try {
                            readFile(barrier, runtime, count.addAndGet(1), alluxioConf);
                        } catch (Exception e) {
                            LOG.error("Failed to read file.", e);
                            System.exit(-1);
                        }
                    });
                    break;
                case WRITE:
                    executorService.submit(() -> {
                        try {
                            writeFile(barrier, runtime, count.addAndGet(1), alluxioConf);
                        } catch (Exception e) {
                            LOG.error("Failed to write file.", e);
                            System.exit(-1);
                        }
                    });
                    break;
                default:
                    throw new RuntimeException("Unsupported type.");
            }
        }
        executorService.shutdown();
        Preconditions.checkState(executorService.awaitTermination(1, TimeUnit.HOURS));
        double time = runtime.get() * 1.0 / sConcurrency / Constants.SECOND_NANO;
        System.out.printf("Iteration: %d; Duration: %f seconds; Aggregated throughput: %f GB/second.%n", i, time, sConcurrency * 1.0 * sFileSize / time / Constants.GB);
    }
}

Example 40 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class Performance method main.

/**
 * Runs the performance test.
 *
 * Usage:
 * {@code java -cp <ALLUXIO-VERSION> alluxio.examples.Performance <MasterIp> <FileNamePrefix>
 * <WriteBlockSizeInBytes> <BlocksPerFile> <DebugMode:true/false> <Threads> <FilesPerThread>
 * <TestCaseNumber> <BaseFileNumber>}
 *
 * @param args the arguments for this example
 */
public static void main(String[] args) throws Exception {
    if (args.length < 9) {
        System.out.println("java -cp " + RuntimeConstants.ALLUXIO_JAR + " alluxio.examples.Performance " + "<MasterIp> <FileNamePrefix> <WriteBlockSizeInBytes> <BlocksPerFile> " + "<DebugMode:true/false> <Threads> <FilesPerThread> <TestCaseNumber> " + "<BaseFileNumber> [folderDir] \n" + "1: Files Write Test\n" + "2: Files Read Test\n" + "3: RamFile Write Test \n" + "4: RamFile Read Test \n" + "5: ByteBuffer Write Test \n" + "6: ByteBuffer Read Test \n");
        System.exit(-1);
    }
    InstancedConfiguration conf = new InstancedConfiguration(ConfigurationUtils.defaults());
    HostAndPort masterAddress = HostAndPort.fromString(args[0]);
    sFileName = args[1];
    sBlockSizeBytes = Integer.parseInt(args[2]);
    sBlocksPerFile = Long.parseLong(args[3]);
    sDebugMode = ("true".equals(args[4]));
    sThreads = Integer.parseInt(args[5]);
    sFiles = Integer.parseInt(args[6]) * sThreads;
    final int testCase = Integer.parseInt(args[7]);
    sBaseFileNumber = Integer.parseInt(args[8]);
    if (args.length > 9) {
        sFolder = args[9];
    }
    sFileBytes = sBlocksPerFile * sBlockSizeBytes;
    sFilesBytes = sFileBytes * sFiles;
    long fileBufferBytes = conf.getBytes(PropertyKey.USER_FILE_BUFFER_BYTES);
    sResultPrefix = String.format("Threads %d FilesPerThread %d TotalFiles %d " + "BLOCK_SIZE_KB %d BLOCKS_PER_FILE %d FILE_SIZE_MB %d " + "Alluxio_WRITE_BUFFER_SIZE_KB %d BaseFileNumber %d : ", sThreads, sFiles / sThreads, sFiles, sBlockSizeBytes / 1024, sBlocksPerFile, sFileBytes / Constants.MB, fileBufferBytes / 1024, sBaseFileNumber);
    CommonUtils.warmUpLoop();
    conf.set(PropertyKey.MASTER_HOSTNAME, masterAddress.getHost());
    conf.set(PropertyKey.MASTER_RPC_PORT, Integer.toString(masterAddress.getPort()));
    FileSystemContext fsContext = FileSystemContext.create(conf);
    if (testCase == 1) {
        sResultPrefix = "AlluxioFilesWriteTest " + sResultPrefix;
        LOG.info(sResultPrefix);
        AlluxioTest(true, /* write */
        FileSystem.Factory.create(fsContext));
    } else if (testCase == 2 || testCase == 9) {
        sResultPrefix = "AlluxioFilesReadTest " + sResultPrefix;
        LOG.info(sResultPrefix);
        sAlluxioStreamingRead = (9 == testCase);
        AlluxioTest(false, /* read */
        FileSystem.Factory.create(fsContext));
    } else if (testCase == 3) {
        sResultPrefix = "RamFile Write " + sResultPrefix;
        LOG.info(sResultPrefix);
        memoryCopyTest(true, false);
    } else if (testCase == 4) {
        sResultPrefix = "RamFile Read " + sResultPrefix;
        LOG.info(sResultPrefix);
        memoryCopyTest(false, false);
    } else if (testCase == 5) {
        sResultPrefix = "ByteBuffer Write Test " + sResultPrefix;
        LOG.info(sResultPrefix);
        memoryCopyTest(true, true);
    } else if (testCase == 6) {
        sResultPrefix = "ByteBuffer Read Test " + sResultPrefix;
        LOG.info(sResultPrefix);
        memoryCopyTest(false, true);
    } else if (testCase == 7) {
        sResultPrefix = "HdfsFilesWriteTest " + sResultPrefix;
        LOG.info(sResultPrefix);
        HdfsTest(true);
    } else if (testCase == 8) {
        sResultPrefix = "HdfsFilesReadTest " + sResultPrefix;
        LOG.info(sResultPrefix);
        HdfsTest(false);
    } else {
        throw new RuntimeException("No Test Case " + testCase);
    }
    for (int k = 0; k < RESULT_ARRAY_SIZE; k++) {
        System.out.print(sResults[k] + " ");
    }
    System.out.println();
    System.exit(0);
}