Examples with StageInfo org.apache.spark.scheduler.StageInfo used on opensource projects

Example 1 with StageInfo

use of org.apache.spark.scheduler.StageInfo in project kylo by Teradata.

the class Job method onStageProgress.

/**
 * Sets the job progress for the specified stage.
 *
 * @param stage          the current stage
 * @param tasksCompleted the number of completed tasks
 */
public void onStageProgress(@Nonnull final StageInfo stage, final int tasksCompleted) {
    int sum = 0;
    for (final StageInfo previousStage : stages) {
        if (previousStage.stageId() < stage.stageId()) {
            sum += previousStage.numTasks();
        }
    }
    this.tasksCompleted.set(sum + tasksCompleted);
}

Example 2 with StageInfo

use of org.apache.spark.scheduler.StageInfo in project kylo by Teradata.

the class Job method setStages.

/**
 * Sets the Spark stages.
 */
public void setStages(@Nonnull final List<StageInfo> stages) {
    this.stages = stages;
    tasksCompleted.set(0);
    int sum = 0;
    for (final StageInfo stage : stages) {
        sum += stage.numTasks();
    }
    tasksTotal.set(sum);
}

Example 3 with StageInfo

use of org.apache.spark.scheduler.StageInfo in project kylo by Teradata.

the class JobTrackerService method createCache.

/**
 * Creates a job cache.
 */
@Nonnull
private Cache<String, Job<?>> createCache() {
    final Cache<String, Job<?>> cache = CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.HOURS).removalListener(new RemovalListener<String, Job<?>>() {

        @Override
        public void onRemoval(@Nonnull final RemovalNotification<String, Job<?>> notification) {
            // noinspection ConstantConditions
            final Optional<Integer> jobId = notification.getValue().getJobId();
            if (jobId.isPresent()) {
                jobs.remove(jobId.get());
            }
            for (final StageInfo stage : notification.getValue().getStages()) {
                stages.remove(stage.stageId());
            }
        // TODO remove temp table
        }
    }).build();
    // Schedule clean-up of groups
    executor.scheduleAtFixedRate(new Runnable() {

        @Override
        public void run() {
            groups.cleanUp();
        }
    }, 1, 1, TimeUnit.HOURS);
    return cache;
}

Example 4 with StageInfo

use of org.apache.spark.scheduler.StageInfo in project kylo by Teradata.

the class JobTrackerService method onJobEnd.

@Override
public void onJobEnd(@Nonnull final SparkListenerJobEnd event) {
    log.trace("Job {} ended", event.jobId());
    final Job<?> job = jobs.remove(event.jobId());
    if (job != null) {
        job.onJobEnd();
        for (final StageInfo stage : job.getStages()) {
            stages.remove(stage.stageId());
        }
    }
}