Examples with BatchedJobConfig alluxio.job.plan.BatchedJobConfig used on opensource projects

Example 1 with BatchedJobConfig

use of alluxio.job.plan.BatchedJobConfig in project alluxio by Alluxio.

the class PlanCoordinator method start.

private synchronized void start() throws JobDoesNotExistException {
    // get the job definition
    LOG.info("Starting job Id={} Config={}", mPlanInfo.getId(), mPlanInfo.getJobConfig());
    PlanDefinition<JobConfig, ?, ?> definition;
    try {
        definition = PlanDefinitionRegistry.INSTANCE.getJobDefinition(mPlanInfo.getJobConfig());
    } catch (JobDoesNotExistException e) {
        LOG.info("Exception when getting jobDefinition from jobConfig: ", e);
        mPlanInfo.setErrorType(ErrorUtils.getErrorType(e));
        mPlanInfo.setErrorMessage(e.getMessage());
        DistributedCmdMetrics.incrementForAllConfigsFailStatus(mPlanInfo.getJobConfig());
        mPlanInfo.setStatus(Status.FAILED);
        throw e;
    }
    SelectExecutorsContext context = new SelectExecutorsContext(mPlanInfo.getId(), mJobServerContext);
    Set<? extends Pair<WorkerInfo, ?>> taskAddressToArgs;
    ArrayList<WorkerInfo> workersInfoListCopy = Lists.newArrayList(mWorkersInfoList);
    Collections.shuffle(workersInfoListCopy);
    try {
        taskAddressToArgs = definition.selectExecutors(mPlanInfo.getJobConfig(), workersInfoListCopy, context);
    } catch (Exception e) {
        LOG.warn("Failed to select executor. {})", e.toString());
        LOG.info("Exception: ", e);
        setJobAsFailed(ErrorUtils.getErrorType(e), e.getMessage());
        return;
    }
    if (taskAddressToArgs.isEmpty()) {
        LOG.warn("No executor was selected.");
        updateStatus();
    }
    for (Pair<WorkerInfo, ?> pair : taskAddressToArgs) {
        LOG.debug("Selected executor {} with parameters {}.", pair.getFirst(), pair.getSecond());
        int taskId = mTaskIdToWorkerInfo.size();
        // create task
        mPlanInfo.addTask(taskId, pair.getFirst(), pair.getSecond());
        // submit commands
        JobConfig config;
        if (mPlanInfo.getJobConfig() instanceof BatchedJobConfig) {
            BatchedJobConfig planConfig = (BatchedJobConfig) mPlanInfo.getJobConfig();
            config = new BatchedJobConfig(planConfig.getJobType(), new HashSet<>());
        } else {
            config = mPlanInfo.getJobConfig();
        }
        mCommandManager.submitRunTaskCommand(mPlanInfo.getId(), taskId, config, pair.getSecond(), pair.getFirst().getId());
        mTaskIdToWorkerInfo.put((long) taskId, pair.getFirst());
        mWorkerIdToTaskIds.putIfAbsent(pair.getFirst().getId(), Lists.newArrayList());
        mWorkerIdToTaskIds.get(pair.getFirst().getId()).add((long) taskId);
    }
}

Example 2 with BatchedJobConfig

use of alluxio.job.plan.BatchedJobConfig in project alluxio by Alluxio.

the class DistributedCpCommand method create.

private JobAttempt create(List<Pair<String, String>> filePath, boolean overwrite) {
    int poolSize = filePath.size();
    JobAttempt jobAttempt;
    if (poolSize == 1) {
        Pair<String, String> pair = filePath.iterator().next();
        System.out.println("Copying " + pair.getFirst() + " to " + pair.getSecond());
        jobAttempt = new CopyJobAttempt(mClient, new MigrateConfig(pair.getFirst(), pair.getSecond(), mWriteType, overwrite), new CountingRetry(3));
    } else {
        HashSet<Map<String, String>> configs = Sets.newHashSet();
        ObjectMapper oMapper = new ObjectMapper();
        for (Pair<String, String> pair : filePath) {
            MigrateConfig config = new MigrateConfig(pair.getFirst(), pair.getSecond(), mWriteType, overwrite);
            System.out.println("Copying " + pair.getFirst() + " to " + pair.getSecond());
            Map<String, String> map = oMapper.convertValue(config, Map.class);
            configs.add(map);
        }
        BatchedJobConfig config = new BatchedJobConfig(MigrateConfig.NAME, configs);
        jobAttempt = new BatchedCopyJobAttempt(mClient, config, new CountingRetry(3));
    }
    return jobAttempt;
}

Example 3 with BatchedJobConfig

use of alluxio.job.plan.BatchedJobConfig in project alluxio by Alluxio.

the class BatchedJobDefinition method selectExecutors.

@Override
public Set<Pair<WorkerInfo, BatchedJobTask>> selectExecutors(BatchedJobConfig config, List<WorkerInfo> jobWorkerInfoList, SelectExecutorsContext context) throws Exception {
    // get job type and config
    String jobType = config.getJobType();
    PlanDefinition plan = JobDefinitionFactory.create(jobType);
    // convert map to config
    final ObjectMapper mapper = new ObjectMapper();
    Class<?> jobConfigClass = plan.getJobConfigClass();
    Set<Pair<WorkerInfo, BatchedJobTask>> allTasks = Sets.newHashSet();
    for (Map<String, String> configMap : config.getJobConfigs()) {
        JobConfig jobConfig = (JobConfig) mapper.convertValue(configMap, jobConfigClass);
        Set<Pair<WorkerInfo, Serializable>> tasks = plan.selectExecutors(jobConfig, jobWorkerInfoList, context);
        for (Pair<WorkerInfo, Serializable> task : tasks) {
            BatchedJobTask batchedTask = new BatchedJobTask(jobConfig, task.getSecond());
            allTasks.add(new Pair<>(task.getFirst(), batchedTask));
        }
    }
    return allTasks;
}