Examples with InstancedConfiguration alluxio.conf.InstancedConfiguration used on opensource projects

Example 66 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class Benchmark method runSingleTask.

protected String runSingleTask(String[] args) throws Exception {
    // prepare the benchmark.
    prepare();
    if (!mBaseParameters.mProfileAgent.isEmpty()) {
        mBaseParameters.mJavaOpts.add("-javaagent:" + mBaseParameters.mProfileAgent + "=" + BaseParameters.AGENT_OUTPUT_PATH);
    }
    AlluxioConfiguration conf = new InstancedConfiguration(ConfigurationUtils.defaults());
    String className = this.getClass().getCanonicalName();
    if (mBaseParameters.mCluster) {
        // run on job service
        long jobId = JobGrpcClientUtils.run(generateJobConfig(args), 0, conf);
        JobInfo jobInfo = JobGrpcClientUtils.getJobStatus(jobId, conf, true);
        return jobInfo.getResult().toString();
    }
    // run locally
    if (mBaseParameters.mInProcess) {
        // run in process
        T result = runLocal();
        if (mBaseParameters.mDistributed) {
            return result.toJson();
        }
        // aggregate the results
        final String s = result.aggregator().aggregate(Collections.singletonList(result)).toJson();
        return s;
    } else {
        // Spawn a new process
        List<String> command = new ArrayList<>();
        command.add(conf.get(PropertyKey.HOME) + "/bin/alluxio");
        command.add("runClass");
        command.add(className);
        command.addAll(Arrays.asList(args));
        command.add(BaseParameters.IN_PROCESS_FLAG);
        command.addAll(mBaseParameters.mJavaOpts.stream().map(String::trim).collect(Collectors.toList()));
        LOG.info("running command: " + String.join(" ", command));
        return ShellUtils.execCommand(command.toArray(new String[0]));
    }
}

Example 67 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class StressJobServiceBench method prepare.

@Override
public void prepare() throws Exception {
    mFsContext = FileSystemContext.create(new InstancedConfiguration(ConfigurationUtils.defaults()));
    final ClientContext clientContext = mFsContext.getClientContext();
    mJobMasterClient = JobMasterClient.Factory.create(JobMasterClientContext.newBuilder(clientContext).build());
}

Example 68 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class FileSystemFactoryTest method zkFileSystemCacheTest.

@Test
public void zkFileSystemCacheTest() {
    Map<String, String> sysProps = new HashMap<>();
    sysProps.put(PropertyKey.ZOOKEEPER_ENABLED.getName(), Boolean.toString(true));
    sysProps.put(PropertyKey.ZOOKEEPER_ADDRESS.getName(), "zk@192.168.0.5");
    sysProps.put(PropertyKey.ZOOKEEPER_ELECTION_PATH.getName(), "/alluxio/leader");
    try (Closeable p = new SystemPropertyRule(sysProps).toResource()) {
        ConfigurationUtils.reloadProperties();
        InstancedConfiguration conf = new InstancedConfiguration(ConfigurationUtils.defaults());
        MasterInquireClient.ConnectDetails connectDetails = MasterInquireClient.Factory.getConnectDetails(conf);
        // Make sure we have a Zookeeper authority
        assertTrue(connectDetails.toAuthority() instanceof ZookeeperAuthority);
        fileSystemCacheTest();
    } catch (IOException e) {
        fail("Unable to set system properties");
    }
}

Example 69 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class MetricsDocGenerator method generate.

/**
 * Writes the supported files for metrics system docs.
 */
public static void generate() throws IOException {
    // Gets and sorts the metric keys
    List<MetricKey> defaultKeys = new ArrayList<>(MetricKey.allMetricKeys());
    Collections.sort(defaultKeys);
    String homeDir = new InstancedConfiguration(ConfigurationUtils.defaults()).getString(PropertyKey.HOME);
    // Map from metric key prefix to metric category
    Map<String, String> metricTypeMap = new HashMap<>();
    for (MetricsSystem.InstanceType type : MetricsSystem.InstanceType.values()) {
        String typeStr = type.toString();
        String category = typeStr.toLowerCase();
        metricTypeMap.put(typeStr, category);
    }
    try (Closer closer = Closer.create()) {
        Map<FileWriterKey, FileWriter> fileWriterMap = new HashMap<>();
        String csvFolder = PathUtils.concatPath(homeDir, CSV_FILE_DIR);
        String ymlFolder = PathUtils.concatPath(homeDir, YML_FILE_DIR);
        FileWriter csvFileWriter;
        FileWriter ymlFileWriter;
        for (String category : CATEGORIES) {
            csvFileWriter = new FileWriter(PathUtils.concatPath(csvFolder, category + "-metrics." + CSV_SUFFIX));
            csvFileWriter.append(CSV_FILE_HEADER + "\n");
            ymlFileWriter = new FileWriter(PathUtils.concatPath(ymlFolder, category + "-metrics." + YML_SUFFIX));
            fileWriterMap.put(new FileWriterKey(category, CSV_SUFFIX), csvFileWriter);
            fileWriterMap.put(new FileWriterKey(category, YML_SUFFIX), ymlFileWriter);
            // register file writer
            closer.register(csvFileWriter);
            closer.register(ymlFileWriter);
        }
        for (MetricKey metricKey : defaultKeys) {
            String key = metricKey.toString();
            String[] components = key.split("\\.");
            if (components.length < 2) {
                throw new IOException(String.format("The given metric key %s doesn't have two or more components", key));
            }
            if (!metricTypeMap.containsKey(components[0])) {
                throw new IOException(String.format("The metric key %s starts with invalid instance type %s", key, components[0]));
            }
            csvFileWriter = fileWriterMap.get(new FileWriterKey(metricTypeMap.get(components[0]), CSV_SUFFIX));
            ymlFileWriter = fileWriterMap.get(new FileWriterKey(metricTypeMap.get(components[0]), YML_SUFFIX));
            csvFileWriter.append(String.format("%s,%s%n", key, metricKey.getMetricType().toString()));
            ymlFileWriter.append(String.format("%s:%n  '%s'%n", key, StringEscapeUtils.escapeHtml4(metricKey.getDescription().replace("'", "''"))));
        }
    }
    LOG.info("Metrics CSV/YML files were created successfully.");
}

Example 70 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class ListCommandIntegrationTest method setPathConfigurations.

/**
 * Sets path level configurations through meta master client, and update client configurations
 * from meta master afterwards.
 *
 * @return the configuration after updating from meta master
 */
private InstancedConfiguration setPathConfigurations() throws Exception {
    FileSystemContext metaCtx = FileSystemContext.create(ServerConfiguration.global());
    MetaMasterConfigClient client = new RetryHandlingMetaMasterConfigClient(MasterClientContext.newBuilder(metaCtx.getClientContext()).build());
    client.setPathConfiguration(new AlluxioURI(DIR1), PROPERTY_KEY1, PROPERTY_VALUE1);
    client.setPathConfiguration(new AlluxioURI(DIR2), PROPERTY_KEY2, PROPERTY_VALUE2);
    InetSocketAddress address = sLocalAlluxioClusterResource.get().getLocalAlluxioMaster().getAddress();
    FileSystemContext fsCtx = FileSystemContext.create(ServerConfiguration.global());
    fsCtx.getClientContext().loadConf(address, true, true);
    return (InstancedConfiguration) fsCtx.getClusterConf();
}