Examples with InstancedConfiguration alluxio.conf.InstancedConfiguration used on opensource projects

Example 46 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class BasicCheckpoint method main.

/**
 * Example program for using checkpoints.
 * Usage: {@code java -cp <ALLUXIO-VERSION> alluxio.cli.BasicCheckpoint <FileFolder> <Files>}
 *
 * @param args the folder for the files and the files to use
 */
public static void main(String[] args) throws IOException {
    if (args.length != 2) {
        System.out.println("java -cp " + RuntimeConstants.ALLUXIO_JAR + " alluxio.cli.BasicCheckpoint <FileFolder> <Files>");
        System.exit(-1);
    }
    FileSystemContext fsContext = FileSystemContext.create(new InstancedConfiguration(ConfigurationUtils.defaults()));
    boolean result = RunTestUtils.runExample(new BasicCheckpoint(args[0], Integer.parseInt(args[1]), fsContext));
    System.exit(result ? 0 : 1);
}

Example 47 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class CollectInfo method main.

/**
 * Main method, starts a new CollectInfo shell.
 * CollectInfo will SSH to all hosts and invoke {@link CollectInfo} with --local option.
 * Then collect the tarballs generated on each of the hosts to the localhost.
 * And tarball all results into one final tarball.
 *
 * @param argv array of arguments given by the user's input from the terminal
 */
public static void main(String[] argv) throws IOException {
    // Parse cmdline args
    CommandLineParser parser = new DefaultParser();
    CommandLine cmd;
    try {
        cmd = parser.parse(OPTIONS, argv, true);
    } catch (ParseException e) {
        return;
    }
    String[] args = cmd.getArgs();
    // Print help message
    if (cmd.hasOption(HELP_OPTION_NAME)) {
        printHelp("");
        System.exit(0);
    }
    // Create the shell instance
    InstancedConfiguration conf = new InstancedConfiguration(ConfigurationUtils.defaults());
    // Reduce the RPC retry max duration to fail earlier for CLIs
    conf.set(PropertyKey.USER_RPC_RETRY_MAX_DURATION, "5s", Source.DEFAULT);
    CollectInfo shell = new CollectInfo(conf);
    // Validate command args
    if (args.length < 2) {
        printHelp(String.format("Command requires at least %s arguments (%s provided)%n", 2, argv.length));
        System.exit(-1);
    }
    // Choose mode based on option
    int ret;
    if (cmd.hasOption(LOCAL_OPTION_NAME)) {
        System.out.println("Executing collectInfo locally");
        ret = shell.collectInfoLocal(cmd);
    } else {
        System.out.println("Executing collectInfo on all nodes in the cluster");
        ret = shell.collectInfoRemote(cmd);
    }
    // Clean up before exiting
    shell.close();
    System.exit(ret);
}

Example 48 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class LogLevel method main.

/**
 * Sets or gets log level of master and worker through their REST API.
 *
 * @param args same arguments as {@link LogLevel}
 */
public static void main(String[] args) {
    int exitCode = 1;
    try {
        logLevel(args, new InstancedConfiguration(ConfigurationUtils.defaults()));
        exitCode = 0;
    } catch (ParseException e) {
        printHelp("Unable to parse input args: " + e.getMessage());
    } catch (IOException e) {
        System.err.println("Failed to set log level:");
        e.printStackTrace();
    }
    System.exit(exitCode);
}

Example 49 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class FuseIOBench method prepare.

@Override
public void prepare() throws Exception {
    if (mBaseParameters.mCluster) {
        // Create the designated test directory before job submitted to job service.
        Files.createDirectories(Paths.get(mParameters.mLocalPath, TEST_DIR));
        return;
    }
    if (mParameters.mThreads > mParameters.mNumDirs && mParameters.mOperation != FuseIOOperation.LIST_FILE) {
        throw new IllegalArgumentException(String.format("Some of the threads are not being used. Please set the number of directories to " + "be at least the number of threads, preferably a multiple of it."));
    }
    // Update mLocalPath to always include the designated test directory.
    mParameters.mLocalPath = Paths.get(mParameters.mLocalPath, TEST_DIR).toString();
    File localPath = new File(mParameters.mLocalPath);
    if (mParameters.mOperation == FuseIOOperation.WRITE) {
        for (int i = 0; i < mParameters.mNumDirs; i++) {
            Files.createDirectories(Paths.get(String.format(TEST_DIR_STRING_FORMAT, mParameters.mLocalPath, mBaseParameters.mId, i)));
        }
        return;
    }
    if ((mParameters.mOperation == FuseIOOperation.REMOTE_READ || mParameters.mOperation == FuseIOOperation.CLUSTER_READ) && !mBaseParameters.mDistributed) {
        throw new IllegalArgumentException(String.format("Single-node Fuse IO stress bench doesn't support RemoteRead or ClusterRead."));
    }
    File[] jobWorkerDirs = localPath.listFiles();
    if (jobWorkerDirs == null) {
        throw new IOException(String.format("--local-path %s is not a valid path for this bench. Make sure using the correct path", mParameters.mLocalPath));
    }
    if (!mBaseParameters.mDistributed) {
        // single-node case only has one directory
        mJobWorkerDirNames = Arrays.asList(mBaseParameters.mId);
        return;
    }
    // for cluster mode, find 0-based id, and make sure directories and job workers are 1-to-1
    int numJobWorkers;
    try (JobMasterClient client = JobMasterClient.Factory.create(JobMasterClientContext.newBuilder(ClientContext.create(new InstancedConfiguration(ConfigurationUtils.defaults()))).build())) {
        numJobWorkers = client.getAllWorkerHealth().size();
    }
    if (numJobWorkers != jobWorkerDirs.length) {
        throw new IllegalStateException("Some job worker crashed or joined after data are written. " + "The test is stopped.");
    }
    mJobWorkerDirNames = Arrays.asList(jobWorkerDirs).stream().map(file -> file.getName()).collect(Collectors.toList());
    mJobWorkerZeroBasedId = mJobWorkerDirNames.indexOf(mBaseParameters.mId);
    if (mJobWorkerZeroBasedId == -1) {
        throw new IllegalStateException(String.format("Directory %s is not found. Please use this bench to generate test files, and make sure " + "no job worker crashes or joins after data is written. The test is stopped.", mBaseParameters.mId));
    }
}

Example 50 with InstancedConfiguration

use of alluxio.conf.InstancedConfiguration in project alluxio by Alluxio.

the class MaxThroughput method runSuite.

@Override
public MaxThroughputSummary runSuite(String[] args) throws Exception {
    try (JobMasterClient client = JobMasterClient.Factory.create(JobMasterClientContext.newBuilder(ClientContext.create(new InstancedConfiguration(ConfigurationUtils.defaults()))).build())) {
        mNumWorkers = client.getAllWorkerHealth().size();
    }
    if (mNumWorkers <= 0) {
        throw new IllegalStateException("No workers available for testing!");
    }
    MaxThroughputSummary summary = new MaxThroughputSummary();
    summary.setParameters(mParameters);
    List<String> baseArgs = new ArrayList<>(Arrays.asList(args));
    if (!mParameters.mSkipPrepare) {
        prepareBeforeAllTests(baseArgs);
    }
    int lower = 0;
    int upper = Integer.MAX_VALUE;
    // use the input target throughput as the starting point
    int next = mParameters.mTargetThroughput;
    int best = 0;
    while (true) {
        int perWorkerThroughput = next / mNumWorkers;
        int requestedThroughput = perWorkerThroughput * mNumWorkers;
        if (perWorkerThroughput == 0) {
            // Cannot run with a target of 0
            break;
        }
        List<String> newArgs = new ArrayList<>(baseArgs);
        updateArgValue(newArgs, "--target-throughput", Integer.toString(perWorkerThroughput));
        long runSec = (FormatUtils.parseTimeSize(mParameters.mDuration) + FormatUtils.parseTimeSize(mParameters.mWarmup)) / 1000;
        // the expected number of paths required for the test to complete successfully
        long requiredCount = next * runSec;
        MasterBenchSummary mbr = runSingleTest(requiredCount, newArgs);
        int current = next;
        final float actualThroughput = mbr.getThroughput();
        if ((actualThroughput > requestedThroughput) || ((requestedThroughput - actualThroughput) / (float) requestedThroughput) < 0.02) {
            // the throughput was achieved. increase.
            summary.addPassedRun(current, mbr);
            best = current;
            // update the lower bound.
            lower = current;
            if (upper == Integer.MAX_VALUE) {
                next *= 2;
            } else {
                next = (next + upper) / 2;
            }
        } else {
            // Failed to achieve the target throughput. update the upper bound.
            summary.addFailedRun(current, mbr);
            upper = current;
            // throughput was not achieved. decrease.
            next = (lower + next) / 2;
        }
        LOG.info("target: " + requestedThroughput + " actual: " + actualThroughput + " [" + lower + " " + next + " " + upper + "]");
        for (String error : mbr.collectErrorsFromAllNodes()) {
            LOG.error("{}", error);
        }
        if (Math.abs(current - next) / (float) current <= 0.02) {
            break;
        }
    }
    LOG.info("max throughput: " + best);
    summary.setEndTimeMs(CommonUtils.getCurrentMs());
    summary.setMaxThroughput(best);
    return summary;
}