Examples with JobConfig alluxio.job.JobConfig used on opensource projects

Example 21 with JobConfig

use of alluxio.job.JobConfig in project alluxio by Alluxio.

the class TransformPlan method computeJobConfigs.

private ArrayList<JobConfig> computeJobConfigs(TransformDefinition definition) {
    ArrayList<JobConfig> actions = new ArrayList<>();
    Layout baseLayout = mBaseLayout;
    boolean deleteSrc = false;
    for (TransformAction action : definition.getActions()) {
        actions.add(action.generateJobConfig(baseLayout, mTransformedLayout, deleteSrc));
        baseLayout = mTransformedLayout;
        deleteSrc = true;
    }
    if (actions.isEmpty()) {
        throw new IllegalArgumentException("At least one action should be defined for the transformation");
    }
    return actions;
}

Example 22 with JobConfig

use of alluxio.job.JobConfig in project alluxio by Alluxio.

the class TransformManager method execute.

/**
 * Executes the plans for the table transformation.
 *
 * This method executes a transformation job with type{@link CompositeConfig},
 * the transformation job concurrently executes the plans,
 * each plan has a list of jobs to be executed sequentially.
 *
 * This method triggers the execution of the transformation job asynchronously without waiting
 * for it to finish. The returned job ID can be used to poll the job service for the status of
 * this transformation.
 *
 * @param dbName the database name
 * @param tableName the table name
 * @param definition the parsed transformation definition
 * @return the job ID for the transformation job
 * @throws IOException when there is an ongoing transformation on the table, or the transformation
 *    job fails to be started, or all partitions of the table have been transformed with the same
 *    definition
 */
public long execute(String dbName, String tableName, TransformDefinition definition) throws IOException {
    List<TransformPlan> plans = mCatalog.getTransformPlan(dbName, tableName, definition);
    if (plans.isEmpty()) {
        throw new IOException(ExceptionMessage.TABLE_ALREADY_TRANSFORMED.getMessage(dbName, tableName, definition.getDefinition()));
    }
    Pair<String, String> dbTable = new Pair<>(dbName, tableName);
    // Atomically try to acquire the permit to execute the transformation job.
    // This PUT does not need to be journaled, because if this PUT succeeds and master crashes,
    // when master restarts, this temporary placeholder entry will not exist, which is correct
    // behavior.
    Long existingJobId = mState.acquireJobPermit(dbTable);
    if (existingJobId != null) {
        if (existingJobId == INVALID_JOB_ID) {
            throw new IOException("A concurrent transformation request is going to be executed");
        } else {
            throw new IOException(ExceptionMessage.TABLE_BEING_TRANSFORMED.getMessage(existingJobId.toString(), tableName, dbName));
        }
    }
    ArrayList<JobConfig> concurrentJobs = new ArrayList<>(plans.size());
    for (TransformPlan plan : plans) {
        concurrentJobs.add(new CompositeConfig(plan.getJobConfigs(), true));
    }
    CompositeConfig transformJob = new CompositeConfig(concurrentJobs, false);
    long jobId;
    try {
        jobId = mJobMasterClient.run(transformJob);
    } catch (IOException e) {
        // The job fails to start, clear the acquired permit for execution.
        // No need to journal this REMOVE, if master crashes, when it restarts, the permit placeholder
        // entry will not exist any more, which is correct behavior.
        mState.releaseJobPermit(dbTable);
        String error = String.format("Fails to start job to transform table %s in database %s", tableName, dbName);
        LOG.error(error, e);
        throw new IOException(error, e);
    }
    Map<String, Layout> transformedLayouts = new HashMap<>(plans.size());
    for (TransformPlan plan : plans) {
        transformedLayouts.put(plan.getBaseLayout().getSpec(), plan.getTransformedLayout());
    }
    AddTransformJobInfoEntry journalEntry = AddTransformJobInfoEntry.newBuilder().setDbName(dbName).setTableName(tableName).setDefinition(definition.getDefinition()).setJobId(jobId).putAllTransformedLayouts(Maps.transformValues(transformedLayouts, Layout::toProto)).build();
    try (JournalContext journalContext = mCreateJournalContext.apply()) {
        applyAndJournal(journalContext, Journal.JournalEntry.newBuilder().setAddTransformJobInfo(journalEntry).build());
    }
    return jobId;
}

Example 23 with JobConfig

use of alluxio.job.JobConfig in project alluxio by Alluxio.

the class WorkflowTracker method next.

private synchronized void next(long jobId) {
    WorkflowExecution workflowExecution = mWorkflows.get(jobId);
    mChildren.putIfAbsent(jobId, new ConcurrentHashSet<>());
    Set<JobConfig> childJobConfigs = workflowExecution.next();
    if (childJobConfigs.isEmpty()) {
        done(jobId);
        return;
    }
    ConcurrentHashSet<Long> childJobIds = new ConcurrentHashSet<>();
    for (int i = 0; i < childJobConfigs.size(); i++) {
        childJobIds.add(mJobMaster.getNewJobId());
    }
    mWaitingOn.put(jobId, childJobIds);
    mChildren.get(jobId).addAll(childJobIds);
    for (Long childJobId : childJobIds) {
        mParentWorkflow.put(childJobId, jobId);
    }
    Iterator<Long> childJobIdsIter = childJobIds.iterator();
    Iterator<JobConfig> childJobConfigsIter = childJobConfigs.iterator();
    while (childJobIdsIter.hasNext() && childJobConfigsIter.hasNext()) {
        Long childJobId = childJobIdsIter.next();
        JobConfig childJobConfig = childJobConfigsIter.next();
        try {
            mJobMaster.run(childJobConfig, childJobId);
        } catch (JobDoesNotExistException | ResourceExhaustedException e) {
            LOG.warn(e.getMessage());
            final String errorType = ErrorUtils.getErrorType(e);
            workflowExecution.stop(Status.FAILED, errorType, e.getMessage());
            stop(jobId, Status.FAILED, errorType, e.getMessage());
        }
    }
}

Example 24 with JobConfig

use of alluxio.job.JobConfig in project alluxio by Alluxio.

the class CommandHandlingExecutorTest method heartbeat.

@Test
public void heartbeat() throws Exception {
    JobCommand.Builder command = JobCommand.newBuilder();
    RunTaskCommand.Builder runTaskCommand = RunTaskCommand.newBuilder();
    long jobId = 1;
    runTaskCommand.setJobId(jobId);
    long taskId = 2;
    runTaskCommand.setTaskId(taskId);
    JobConfig jobConfig = new TestPlanConfig("/test");
    runTaskCommand.setJobConfig(ByteString.copyFrom(SerializationUtils.serialize(jobConfig)));
    Serializable taskArgs = Lists.newArrayList(1);
    runTaskCommand.setTaskArgs(ByteString.copyFrom(SerializationUtils.serialize(taskArgs)));
    command.setRunTaskCommand(runTaskCommand);
    Mockito.when(mJobMasterClient.heartbeat(any(JobWorkerHealth.class), eq(Lists.newArrayList()))).thenReturn(Lists.newArrayList(command.build()));
    mCommandHandlingExecutor.heartbeat();
    ExecutorService executorService = Whitebox.getInternalState(mCommandHandlingExecutor, "mCommandHandlingService");
    executorService.shutdown();
    Assert.assertTrue(executorService.awaitTermination(5000, TimeUnit.MILLISECONDS));
    Mockito.verify(mTaskExecutorManager).getAndClearTaskUpdates();
    Mockito.verify(mTaskExecutorManager).executeTask(Mockito.eq(jobId), Mockito.eq(taskId), Mockito.eq(runTaskCommand.build()), Mockito.any(RunTaskContext.class));
}

Example 25 with JobConfig

use of alluxio.job.JobConfig in project alluxio by Alluxio.

the class PlanCoordinator method updateStatus.

/**
 * Updates the status of the job. When all the tasks are completed, run the join method in the
 * definition.
 */
private synchronized void updateStatus() {
    int completed = 0;
    List<TaskInfo> taskInfoList = mPlanInfo.getTaskInfoList();
    JobConfig config = mPlanInfo.getJobConfig();
    Preconditions.checkNotNull(config);
    FileSystem fileSystem = mJobServerContext.getFileSystem();
    for (TaskInfo info : taskInfoList) {
        Status status = info.getStatus();
        switch(status) {
            case FAILED:
                setJobAsFailed(info.getErrorType(), "Task execution failed: " + info.getErrorMessage());
                return;
            case CANCELED:
                if (mPlanInfo.getStatus() != Status.FAILED) {
                    mPlanInfo.setStatus(Status.CANCELED);
                    DistributedCmdMetrics.incrementForAllConfigsCancelStatus(config);
                }
                return;
            case RUNNING:
                if (mPlanInfo.getStatus() != Status.FAILED && mPlanInfo.getStatus() != Status.CANCELED) {
                    mPlanInfo.setStatus(Status.RUNNING);
                }
                break;
            case COMPLETED:
                completed++;
                break;
            case CREATED:
                // do nothing
                break;
            default:
                throw new IllegalArgumentException("Unsupported status " + info.getStatus());
        }
    }
    if (completed == taskInfoList.size()) {
        if (mPlanInfo.getStatus() == Status.COMPLETED) {
            return;
        }
        // all the tasks completed, run join
        try {
            // Try to join first, so that in case of failure we don't move to a completed state yet
            mPlanInfo.setResult(join(taskInfoList));
            mPlanInfo.setStatus(Status.COMPLETED);
            // Increment the counter for Complete status when all the tasks in a job are completed.
            DistributedCmdMetrics.incrementForAllConfigsCompleteStatus(config, fileSystem, new CountingRetry(5));
        } catch (Exception e) {
            LOG.warn("Job error when joining tasks Job Id={} Config={}", mPlanInfo.getId(), mPlanInfo.getJobConfig(), e);
            setJobAsFailed(ErrorUtils.getErrorType(e), e.getMessage());
        }
    }
}