Examples with InstancedConfiguration alluxio.conf.InstancedConfiguration used on opensource projects

Example 61 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class HmsTests method main.

/**
 * @param args the input arguments
 */
public static void main(String[] args) throws Exception {
    CommandLineParser parser = new DefaultParser();
    CommandLine cmd = null;
    try {
        cmd = parser.parse(OPTIONS, args, true);
    } catch (ParseException e) {
        printUsage();
        System.exit(1);
    }
    if (cmd.hasOption(HELP_OPTION_NAME)) {
        printUsage();
        return;
    }
    String metastoreUri = cmd.getOptionValue(ValidationConfig.METASTORE_URI_OPTION_NAME);
    String database = cmd.getOptionValue(ValidationConfig.DATABASE_OPTION_NAME, "");
    String tables = cmd.getOptionValue(ValidationConfig.TABLES_OPTION_NAME, "");
    int socketTimeout = Integer.parseInt(cmd.getOptionValue(ValidationConfig.SOCKET_TIMEOUT_OPTION_NAME, "-1"));
    ValidationToolRegistry registry = new ValidationToolRegistry(new InstancedConfiguration(ConfigurationUtils.defaults()));
    // Load hms validation tool from alluxio lib directory
    registry.refresh();
    Map<Object, Object> configMap = new HashMap<>();
    configMap.put(ValidationConfig.METASTORE_URI_CONFIG_NAME, metastoreUri);
    configMap.put(ValidationConfig.DATABASE_CONFIG_NAME, database);
    configMap.put(ValidationConfig.TABLES_CONFIG_NAME, tables);
    configMap.put(ValidationConfig.SOCKET_TIMEOUT_CONFIG_NAME, socketTimeout);
    ValidationTool tests = registry.create(ValidationConfig.HMS_TOOL_TYPE, configMap);
    String result = ValidationTool.toJson(ValidationTool.convertResults(tests.runAllTests()));
    System.out.println(result);
    if (result.contains(ValidationUtils.State.FAILED.toString())) {
        System.exit(-1);
    }
    System.exit(0);
}

Example 62 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class TestRunner method runTests.

/**
 * Runs combinations of tests of operation, read and write type.
 *
 * @return the number of failed tests
 */
private int runTests() throws Exception {
    mDirectory = PathUtils.concatPath(mDirectory, TEST_DIRECTORY_NAME);
    AlluxioURI testDir = new AlluxioURI(mDirectory);
    FileSystemContext fsContext = FileSystemContext.create(new InstancedConfiguration(ConfigurationUtils.defaults()));
    FileSystem fs = FileSystem.Factory.create(fsContext);
    if (fs.exists(testDir)) {
        fs.delete(testDir, DeletePOptions.newBuilder().setRecursive(true).setUnchecked(true).build());
    }
    int failed = 0;
    List<ReadType> readTypes = mReadType == null ? READ_TYPES : Lists.newArrayList(ReadType.valueOf(mReadType));
    List<WriteType> writeTypes = mWriteType == null ? WRITE_TYPES : Lists.newArrayList(WriteType.valueOf(mWriteType));
    List<OperationType> operations = mOperation == null ? Lists.newArrayList(OperationType.values()) : Lists.newArrayList(OperationType.valueOf(mOperation));
    for (ReadType readType : readTypes) {
        for (WriteType writeType : writeTypes) {
            for (OperationType opType : operations) {
                System.out.println(String.format("runTest --operation %s --readType %s --writeType %s", opType, readType, writeType));
                failed += runTest(opType, readType, writeType, fsContext);
            }
        }
    }
    if (failed > 0) {
        System.out.println("Number of failed tests: " + failed);
    }
    return failed;
}

Example 63 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class AbstractClientTest method confAddress.

@Test
public void confAddress() throws Exception {
    ClientContext context = Mockito.mock(ClientContext.class);
    Mockito.when(context.getClusterConf()).thenReturn(new InstancedConfiguration(ConfigurationUtils.defaults()));
    InetSocketAddress baseAddress = new InetSocketAddress("0.0.0.0", 2000);
    InetSocketAddress confAddress = new InetSocketAddress("0.0.0.0", 2000);
    final alluxio.Client client = new BaseTestClient(context) {

        @Override
        public synchronized InetSocketAddress getAddress() {
            return baseAddress;
        }

        @Override
        public synchronized InetSocketAddress getConfAddress() {
            return confAddress;
        }
    };
    ArgumentCaptor<InetSocketAddress> argument = ArgumentCaptor.forClass(InetSocketAddress.class);
    Mockito.doThrow(new RuntimeException("test")).when(context).loadConfIfNotLoaded(argument.capture());
    Mockito.doReturn(Mockito.mock(BaseUserState.class)).when(context).getUserState();
    try {
        client.connect();
        Assert.fail();
    } catch (Exception e) {
    // ignore any exceptions. It's expected.
    }
    Assert.assertEquals(confAddress, argument.getValue());
}

Example 64 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class StressMasterBench method prepare.

@Override
@SuppressFBWarnings("BC_UNCONFIRMED_CAST")
public void prepare() throws Exception {
    if (mParameters.mFixedCount <= 0) {
        throw new IllegalStateException("fixed count must be > 0. fixedCount: " + mParameters.mFixedCount);
    }
    if (!mBaseParameters.mDistributed) {
        // set hdfs conf for preparation client
        Configuration hdfsConf = new Configuration();
        // force delete, create dirs through to UFS
        hdfsConf.set(PropertyKey.Name.USER_FILE_DELETE_UNCHECKED, "true");
        hdfsConf.set(PropertyKey.Name.USER_FILE_WRITE_TYPE_DEFAULT, mParameters.mWriteType);
        // more threads for parallel deletes for cleanup
        hdfsConf.set(PropertyKey.Name.USER_FILE_MASTER_CLIENT_POOL_SIZE_MAX, "256");
        FileSystem prepareFs = FileSystem.get(new URI(mParameters.mBasePath), hdfsConf);
        // initialize the base, for only the non-distributed task (the cluster launching task)
        Path path = new Path(mParameters.mBasePath);
        // the base path depends on the operation
        Path basePath;
        if (mParameters.mOperation == Operation.CREATE_DIR) {
            basePath = new Path(path, "dirs");
        } else {
            basePath = new Path(path, "files");
        }
        if (mParameters.mOperation == Operation.CREATE_FILE || mParameters.mOperation == Operation.CREATE_DIR) {
            LOG.info("Cleaning base path: {}", basePath);
            long start = CommonUtils.getCurrentMs();
            deletePaths(prepareFs, basePath);
            long end = CommonUtils.getCurrentMs();
            LOG.info("Cleanup took: {} s", (end - start) / 1000.0);
            prepareFs.mkdirs(basePath);
        } else {
            // these are read operations. the directory must exist
            if (!prepareFs.exists(basePath)) {
                throw new IllegalStateException(String.format("base path (%s) must exist for operation (%s)", basePath, mParameters.mOperation));
            }
        }
        if (!prepareFs.isDirectory(basePath)) {
            throw new IllegalStateException(String.format("base path (%s) must be a directory for operation (%s)", basePath, mParameters.mOperation));
        }
    }
    // set hdfs conf for all test clients
    Configuration hdfsConf = new Configuration();
    // do not cache these clients
    hdfsConf.set(String.format("fs.%s.impl.disable.cache", (new URI(mParameters.mBasePath)).getScheme()), "true");
    for (Map.Entry<String, String> entry : mParameters.mConf.entrySet()) {
        hdfsConf.set(entry.getKey(), entry.getValue());
    }
    hdfsConf.set(PropertyKey.Name.USER_FILE_WRITE_TYPE_DEFAULT, mParameters.mWriteType);
    if (mParameters.mClientType == FileSystemClientType.ALLUXIO_HDFS) {
        LOG.info("Using ALLUXIO HDFS Compatible API to perform the test.");
        mCachedFs = new FileSystem[mParameters.mClients];
        for (int i = 0; i < mCachedFs.length; i++) {
            mCachedFs[i] = FileSystem.get(new URI(mParameters.mBasePath), hdfsConf);
        }
    } else {
        LOG.info("Using ALLUXIO Native API to perform the test.");
        alluxio.conf.AlluxioProperties alluxioProperties = ConfigurationUtils.defaults();
        alluxioProperties.merge(HadoopConfigurationUtils.getConfigurationFromHadoop(hdfsConf), Source.RUNTIME);
        mCachedNativeFs = new alluxio.client.file.FileSystem[mParameters.mClients];
        for (int i = 0; i < mCachedNativeFs.length; i++) {
            mCachedNativeFs[i] = alluxio.client.file.FileSystem.Factory.create(new InstancedConfiguration(alluxioProperties));
        }
    }
}

Example 65 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class StressClientIOBench method prepare.

@Override
@SuppressFBWarnings("BC_UNCONFIRMED_CAST")
public void prepare() throws Exception {
    if (mBaseParameters.mCluster && mBaseParameters.mClusterLimit != 1) {
        throw new IllegalArgumentException(String.format("%s is a single-node client IO stress test, so it cannot be run in cluster mode without" + " flag '%s 1'.", this.getClass().getName(), BaseParameters.CLUSTER_LIMIT_FLAG));
    }
    if (FormatUtils.parseSpaceSize(mParameters.mFileSize) < FormatUtils.parseSpaceSize(mParameters.mBufferSize)) {
        throw new IllegalArgumentException(String.format("File size (%s) must be larger than buffer size (%s)", mParameters.mFileSize, mParameters.mBufferSize));
    }
    if (mParameters.mOperation == ClientIOOperation.WRITE) {
        LOG.warn("Cannot write repeatedly, so warmup is not possible. Setting warmup to 0s.");
        mParameters.mWarmup = "0s";
    }
    if (!mBaseParameters.mDistributed) {
        // set hdfs conf for preparation client
        Configuration hdfsConf = new Configuration();
        hdfsConf.set(PropertyKey.Name.USER_FILE_DELETE_UNCHECKED, "true");
        hdfsConf.set(PropertyKey.Name.USER_FILE_WRITE_TYPE_DEFAULT, mParameters.mWriteType);
        FileSystem prepareFs = FileSystem.get(new URI(mParameters.mBasePath), hdfsConf);
        // initialize the base, for only the non-distributed task (the cluster launching task)
        Path path = new Path(mParameters.mBasePath);
        if (!ClientIOOperation.isRead(mParameters.mOperation)) {
            prepareFs.delete(path, true);
            prepareFs.mkdirs(path);
        }
    }
    ClientIOWritePolicy.setMaxWorkers(mParameters.mWriteNumWorkers);
    // set hdfs conf for all test clients
    Configuration hdfsConf = new Configuration();
    // do not cache these clients
    hdfsConf.set(String.format("fs.%s.impl.disable.cache", (new URI(mParameters.mBasePath)).getScheme()), "true");
    hdfsConf.set(PropertyKey.Name.USER_BLOCK_WRITE_LOCATION_POLICY, ClientIOWritePolicy.class.getName());
    for (Map.Entry<String, String> entry : mParameters.mConf.entrySet()) {
        hdfsConf.set(entry.getKey(), entry.getValue());
    }
    hdfsConf.set(PropertyKey.Name.USER_FILE_WRITE_TYPE_DEFAULT, mParameters.mWriteType);
    if (mParameters.mClientType == FileSystemClientType.ALLUXIO_HDFS) {
        LOG.info("Using ALLUXIO HDFS Compatible API to perform the test.");
        mCachedFs = new FileSystem[mParameters.mClients];
        for (int i = 0; i < mCachedFs.length; i++) {
            mCachedFs[i] = FileSystem.get(new URI(mParameters.mBasePath), hdfsConf);
        }
    } else {
        LOG.info("Using ALLUXIO Native API to perform the test.");
        alluxio.conf.AlluxioProperties alluxioProperties = ConfigurationUtils.defaults();
        alluxioProperties.merge(HadoopConfigurationUtils.getConfigurationFromHadoop(hdfsConf), Source.RUNTIME);
        mCachedNativeFs = new alluxio.client.file.FileSystem[mParameters.mClients];
        for (int i = 0; i < mCachedNativeFs.length; i++) {
            mCachedNativeFs[i] = alluxio.client.file.FileSystem.Factory.create(new InstancedConfiguration(alluxioProperties));
        }
    }
}