Examples with JobMasterClient alluxio.client.job.JobMasterClient used on opensource projects

Example 1 with JobMasterClient

use of alluxio.client.job.JobMasterClient in project alluxio by Alluxio.

the class LogLevelTest method parseJobWorkerTargets.

@Test
public void parseJobWorkerTargets() throws Exception {
    CommandLine mockCommandLine = mock(CommandLine.class);
    String[] mockArgs = new String[] { "--target", "job_workers" };
    when(mockCommandLine.getArgs()).thenReturn(mockArgs);
    when(mockCommandLine.hasOption(LogLevel.TARGET_OPTION_NAME)).thenReturn(true);
    when(mockCommandLine.getOptionValue(LogLevel.TARGET_OPTION_NAME)).thenReturn(mockArgs[1]);
    // Prepare a list of job workers
    List<JobWorkerHealth> jobWorkers = new ArrayList<>();
    jobWorkers.add(new JobWorkerHealth(0, new ArrayList<>(), 10, 0, 0, "workers-1"));
    jobWorkers.add(new JobWorkerHealth(1, new ArrayList<>(), 10, 0, 0, "workers-2"));
    PowerMockito.mockStatic(JobMasterClient.Factory.class);
    JobMasterClient mockJobClient = mock(JobMasterClient.class);
    when(mockJobClient.getAllWorkerHealth()).thenReturn(jobWorkers);
    when(JobMasterClient.Factory.create(any())).thenReturn(mockJobClient);
    List<LogLevel.TargetInfo> targets = LogLevel.parseOptTarget(mockCommandLine, mConf);
    assertEquals(2, targets.size());
    assertEquals(new LogLevel.TargetInfo("workers-1", JOB_WORKER_WEB_PORT, "job_worker"), targets.get(0));
    assertEquals(new LogLevel.TargetInfo("workers-2", JOB_WORKER_WEB_PORT, "job_worker"), targets.get(1));
}

Example 2 with JobMasterClient

use of alluxio.client.job.JobMasterClient in project alluxio by Alluxio.

the class JobServiceMaxThroughput method runSuite.

@Override
public JobServiceMaxThroughputSummary runSuite(String[] args) throws Exception {
    try (JobMasterClient client = JobMasterClient.Factory.create(JobMasterClientContext.newBuilder(ClientContext.create(new InstancedConfiguration(ConfigurationUtils.defaults()))).build())) {
        mNumWorkers = client.getAllWorkerHealth().size();
    }
    if (mNumWorkers <= 0) {
        throw new IllegalStateException("No workers available for testing!");
    }
    JobServiceMaxThroughputSummary summary = new JobServiceMaxThroughputSummary();
    summary.setParameters(mParameters);
    List<String> baseArgs = new ArrayList<>(Arrays.asList(args));
    int best = getBestThroughput(mParameters.mTargetThroughput, summary, baseArgs, mNumWorkers);
    LOG.info("max throughput: " + best);
    summary.setEndTimeMs(CommonUtils.getCurrentMs());
    summary.setMaxThroughput(best);
    return summary;
}

Example 3 with JobMasterClient

use of alluxio.client.job.JobMasterClient in project alluxio by Alluxio.

the class LogLevel method getTargetInfos.

private static List<TargetInfo> getTargetInfos(String[] targets, AlluxioConfiguration conf) throws IOException {
    // Trim the elements
    Set<String> targetSet = Arrays.stream(targets).map(String::trim).collect(Collectors.toSet());
    List<TargetInfo> targetInfoList = new ArrayList<>();
    // Allow plural form for the master/job_master and print a notice
    if (targetSet.contains(ROLE_MASTERS)) {
        System.out.println("The logLevel command will only take effect on the primary master, " + "instead of on all the masters. ");
        targetSet.remove(ROLE_MASTERS);
        targetSet.add(ROLE_MASTER);
        System.out.println("Target `masters` is replaced with `master`.");
    }
    if (targetSet.contains(ROLE_JOB_MASTERS)) {
        System.out.println("The logLevel command will only take effect on the primary job master, " + "instead of on all the masters. ");
        targetSet.remove(ROLE_JOB_MASTERS);
        targetSet.add(ROLE_JOB_MASTER);
        System.out.println("Target `job_masters` is replaced with `job_master`.");
    }
    ClientContext clientContext = ClientContext.create(conf);
    // Created only when needed by master and workers
    FileSystemContext fsContext = null;
    // Created only when needed by the job master and job workers
    JobMasterClient jobClient = null;
    // Process each target
    for (String target : targetSet) {
        if (target.isEmpty()) {
            continue;
        } else if (target.equals(ROLE_MASTER)) {
            if (fsContext == null) {
                fsContext = FileSystemContext.create(clientContext);
            }
            String masterHost = fsContext.getMasterAddress().getHostName();
            int masterPort = NetworkAddressUtils.getPort(ServiceType.MASTER_WEB, conf);
            TargetInfo master = new TargetInfo(masterHost, masterPort, ROLE_MASTER);
            targetInfoList.add(master);
        } else if (target.equals(ROLE_JOB_MASTER)) {
            if (jobClient == null) {
                jobClient = JobMasterClient.Factory.create(JobMasterClientContext.newBuilder(clientContext).build());
            }
            String jobMasterHost = jobClient.getAddress().getHostName();
            int jobMasterPort = NetworkAddressUtils.getPort(ServiceType.JOB_MASTER_WEB, conf);
            TargetInfo jobMaster = new TargetInfo(jobMasterHost, jobMasterPort, ROLE_JOB_MASTER);
            targetInfoList.add(jobMaster);
        } else if (target.equals(ROLE_WORKERS)) {
            if (fsContext == null) {
                fsContext = FileSystemContext.create(ClientContext.create(conf));
            }
            List<BlockWorkerInfo> workerInfoList = fsContext.getCachedWorkers();
            if (workerInfoList.size() == 0) {
                System.out.println("No workers found");
                System.exit(1);
            }
            for (BlockWorkerInfo workerInfo : workerInfoList) {
                WorkerNetAddress netAddress = workerInfo.getNetAddress();
                TargetInfo worker = new TargetInfo(netAddress.getHost(), netAddress.getWebPort(), ROLE_WORKER);
                targetInfoList.add(worker);
            }
        } else if (target.equals(ROLE_JOB_WORKERS)) {
            if (jobClient == null) {
                jobClient = JobMasterClient.Factory.create(JobMasterClientContext.newBuilder(clientContext).build());
            }
            List<JobWorkerHealth> jobWorkerInfoList = jobClient.getAllWorkerHealth();
            if (jobWorkerInfoList.size() == 0) {
                System.out.println("No job workers found");
                System.exit(1);
            }
            int jobWorkerPort = conf.getInt(PropertyKey.JOB_WORKER_WEB_PORT);
            for (JobWorkerHealth jobWorkerInfo : jobWorkerInfoList) {
                String jobWorkerHost = jobWorkerInfo.getHostname();
                TargetInfo jobWorker = new TargetInfo(jobWorkerHost, jobWorkerPort, ROLE_JOB_WORKER);
                targetInfoList.add(jobWorker);
            }
        } else if (target.contains(":")) {
            String[] hostPortPair = target.split(":");
            int port = Integer.parseInt(hostPortPair[1]);
            String role = inferRoleFromPort(port, conf);
            LOG.debug("Port {} maps to role {}", port, role);
            TargetInfo unspecifiedTarget = new TargetInfo(hostPortPair[0], port, role);
            System.out.format("Role inferred from port: %s%n", unspecifiedTarget);
            targetInfoList.add(unspecifiedTarget);
        } else {
            throw new IOException(String.format("Unrecognized target argument: %s. " + "Please pass the targets in the form of <host>:<port>, " + "with comma as the separator.", target));
        }
    }
    return targetInfoList;
}

Example 4 with JobMasterClient

use of alluxio.client.job.JobMasterClient in project alluxio by Alluxio.

the class FuseIOBench method prepare.

@Override
public void prepare() throws Exception {
    if (mBaseParameters.mCluster) {
        // Create the designated test directory before job submitted to job service.
        Files.createDirectories(Paths.get(mParameters.mLocalPath, TEST_DIR));
        return;
    }
    if (mParameters.mThreads > mParameters.mNumDirs && mParameters.mOperation != FuseIOOperation.LIST_FILE) {
        throw new IllegalArgumentException(String.format("Some of the threads are not being used. Please set the number of directories to " + "be at least the number of threads, preferably a multiple of it."));
    }
    // Update mLocalPath to always include the designated test directory.
    mParameters.mLocalPath = Paths.get(mParameters.mLocalPath, TEST_DIR).toString();
    File localPath = new File(mParameters.mLocalPath);
    if (mParameters.mOperation == FuseIOOperation.WRITE) {
        for (int i = 0; i < mParameters.mNumDirs; i++) {
            Files.createDirectories(Paths.get(String.format(TEST_DIR_STRING_FORMAT, mParameters.mLocalPath, mBaseParameters.mId, i)));
        }
        return;
    }
    if ((mParameters.mOperation == FuseIOOperation.REMOTE_READ || mParameters.mOperation == FuseIOOperation.CLUSTER_READ) && !mBaseParameters.mDistributed) {
        throw new IllegalArgumentException(String.format("Single-node Fuse IO stress bench doesn't support RemoteRead or ClusterRead."));
    }
    File[] jobWorkerDirs = localPath.listFiles();
    if (jobWorkerDirs == null) {
        throw new IOException(String.format("--local-path %s is not a valid path for this bench. Make sure using the correct path", mParameters.mLocalPath));
    }
    if (!mBaseParameters.mDistributed) {
        // single-node case only has one directory
        mJobWorkerDirNames = Arrays.asList(mBaseParameters.mId);
        return;
    }
    // for cluster mode, find 0-based id, and make sure directories and job workers are 1-to-1
    int numJobWorkers;
    try (JobMasterClient client = JobMasterClient.Factory.create(JobMasterClientContext.newBuilder(ClientContext.create(new InstancedConfiguration(ConfigurationUtils.defaults()))).build())) {
        numJobWorkers = client.getAllWorkerHealth().size();
    }
    if (numJobWorkers != jobWorkerDirs.length) {
        throw new IllegalStateException("Some job worker crashed or joined after data are written. " + "The test is stopped.");
    }
    mJobWorkerDirNames = Arrays.asList(jobWorkerDirs).stream().map(file -> file.getName()).collect(Collectors.toList());
    mJobWorkerZeroBasedId = mJobWorkerDirNames.indexOf(mBaseParameters.mId);
    if (mJobWorkerZeroBasedId == -1) {
        throw new IllegalStateException(String.format("Directory %s is not found. Please use this bench to generate test files, and make sure " + "no job worker crashes or joins after data is written. The test is stopped.", mBaseParameters.mId));
    }
}

Example 5 with JobMasterClient

use of alluxio.client.job.JobMasterClient in project alluxio by Alluxio.

the class MaxThroughput method runSuite.

@Override
public MaxThroughputSummary runSuite(String[] args) throws Exception {
    try (JobMasterClient client = JobMasterClient.Factory.create(JobMasterClientContext.newBuilder(ClientContext.create(new InstancedConfiguration(ConfigurationUtils.defaults()))).build())) {
        mNumWorkers = client.getAllWorkerHealth().size();
    }
    if (mNumWorkers <= 0) {
        throw new IllegalStateException("No workers available for testing!");
    }
    MaxThroughputSummary summary = new MaxThroughputSummary();
    summary.setParameters(mParameters);
    List<String> baseArgs = new ArrayList<>(Arrays.asList(args));
    if (!mParameters.mSkipPrepare) {
        prepareBeforeAllTests(baseArgs);
    }
    int lower = 0;
    int upper = Integer.MAX_VALUE;
    // use the input target throughput as the starting point
    int next = mParameters.mTargetThroughput;
    int best = 0;
    while (true) {
        int perWorkerThroughput = next / mNumWorkers;
        int requestedThroughput = perWorkerThroughput * mNumWorkers;
        if (perWorkerThroughput == 0) {
            // Cannot run with a target of 0
            break;
        }
        List<String> newArgs = new ArrayList<>(baseArgs);
        updateArgValue(newArgs, "--target-throughput", Integer.toString(perWorkerThroughput));
        long runSec = (FormatUtils.parseTimeSize(mParameters.mDuration) + FormatUtils.parseTimeSize(mParameters.mWarmup)) / 1000;
        // the expected number of paths required for the test to complete successfully
        long requiredCount = next * runSec;
        MasterBenchSummary mbr = runSingleTest(requiredCount, newArgs);
        int current = next;
        final float actualThroughput = mbr.getThroughput();
        if ((actualThroughput > requestedThroughput) || ((requestedThroughput - actualThroughput) / (float) requestedThroughput) < 0.02) {
            // the throughput was achieved. increase.
            summary.addPassedRun(current, mbr);
            best = current;
            // update the lower bound.
            lower = current;
            if (upper == Integer.MAX_VALUE) {
                next *= 2;
            } else {
                next = (next + upper) / 2;
            }
        } else {
            // Failed to achieve the target throughput. update the upper bound.
            summary.addFailedRun(current, mbr);
            upper = current;
            // throughput was not achieved. decrease.
            next = (lower + next) / 2;
        }
        LOG.info("target: " + requestedThroughput + " actual: " + actualThroughput + " [" + lower + " " + next + " " + upper + "]");
        for (String error : mbr.collectErrorsFromAllNodes()) {
            LOG.error("{}", error);
        }
        if (Math.abs(current - next) / (float) current <= 0.02) {
            break;
        }
    }
    LOG.info("max throughput: " + best);
    summary.setEndTimeMs(CommonUtils.getCurrentMs());
    summary.setMaxThroughput(best);
    return summary;
}