Examples with ActivityClusterPlan org.apache.hyracks.control.cc.job.ActivityClusterPlan used on opensource projects

Example 1 with ActivityClusterPlan

use of org.apache.hyracks.control.cc.job.ActivityClusterPlan in project asterixdb by apache.

the class ActivityClusterPlanner method getTaskCluster.

private TaskCluster getTaskCluster(TaskId tid) {
    JobRun run = executor.getJobRun();
    ActivityCluster ac = run.getActivityClusterGraph().getActivityMap().get(tid.getActivityId());
    ActivityClusterPlan acp = run.getActivityClusterPlanMap().get(ac.getId());
    Task[] tasks = acp.getActivityPlanMap().get(tid.getActivityId()).getTasks();
    Task task = tasks[tid.getPartition()];
    assert task.getTaskId().equals(tid);
    return task.getTaskCluster();
}

Example 2 with ActivityClusterPlan

use of org.apache.hyracks.control.cc.job.ActivityClusterPlan in project asterixdb by apache.

the class ActivityClusterPlanner method buildActivityPlanMap.

private Map<ActivityId, ActivityPlan> buildActivityPlanMap(ActivityCluster ac, JobRun jobRun, Map<ActivityId, ActivityPartitionDetails> pcMap) {
    Map<ActivityId, ActivityPlan> activityPlanMap = new HashMap<>();
    Set<ActivityId> depAnIds = new HashSet<>();
    for (ActivityId anId : ac.getActivityMap().keySet()) {
        depAnIds.clear();
        getDependencyActivityIds(depAnIds, anId, ac);
        ActivityPartitionDetails apd = pcMap.get(anId);
        Task[] tasks = new Task[apd.getPartitionCount()];
        ActivityPlan activityPlan = new ActivityPlan(apd);
        for (int i = 0; i < tasks.length; ++i) {
            TaskId tid = new TaskId(anId, i);
            tasks[i] = new Task(tid, activityPlan);
            for (ActivityId danId : depAnIds) {
                ActivityCluster dAC = ac.getActivityClusterGraph().getActivityMap().get(danId);
                ActivityClusterPlan dACP = jobRun.getActivityClusterPlanMap().get(dAC.getId());
                assert dACP != null : "IllegalStateEncountered: Dependent AC is being planned without a plan for " + "dependency AC: Encountered no plan for ActivityID " + danId;
                Task[] dATasks = dACP.getActivityPlanMap().get(danId).getTasks();
                assert dATasks != null : "IllegalStateEncountered: Dependent AC is being planned without a plan for" + " dependency AC: Encountered no plan for ActivityID " + danId;
                assert dATasks.length == tasks.length : "Dependency activity partitioned differently from " + "dependent: " + dATasks.length + " != " + tasks.length;
                Task dTask = dATasks[i];
                TaskId dTaskId = dTask.getTaskId();
                tasks[i].getDependencies().add(dTaskId);
                dTask.getDependents().add(tid);
            }
        }
        activityPlan.setTasks(tasks);
        activityPlanMap.put(anId, activityPlan);
    }
    return activityPlanMap;
}

Example 3 with ActivityClusterPlan

use of org.apache.hyracks.control.cc.job.ActivityClusterPlan in project asterixdb by apache.

the class JobExecutor method findRunnableTaskClusterRoots.

private void findRunnableTaskClusterRoots(Set<TaskCluster> frontier, ActivityCluster candidate) throws HyracksException {
    boolean depsComplete = true;
    for (ActivityCluster depAC : candidate.getDependencies()) {
        if (!isPlanned(depAC)) {
            depsComplete = false;
            findRunnableTaskClusterRoots(frontier, depAC);
        } else {
            boolean tcRootsComplete = true;
            for (TaskCluster tc : getActivityClusterPlan(depAC).getTaskClusters()) {
                if (!tc.getProducedPartitions().isEmpty()) {
                    continue;
                }
                TaskClusterAttempt tca = findLastTaskClusterAttempt(tc);
                if (tca == null || tca.getStatus() != TaskClusterAttempt.TaskClusterStatus.COMPLETED) {
                    tcRootsComplete = false;
                    break;
                }
            }
            if (!tcRootsComplete) {
                depsComplete = false;
                findRunnableTaskClusterRoots(frontier, depAC);
            }
        }
    }
    if (!depsComplete) {
        return;
    }
    if (!isPlanned(candidate)) {
        ActivityClusterPlanner acp = new ActivityClusterPlanner(this);
        ActivityClusterPlan acPlan = acp.planActivityCluster(candidate);
        jobRun.getActivityClusterPlanMap().put(candidate.getId(), acPlan);
        partitionProducingTaskClusterMap.putAll(acp.getPartitionProducingTaskClusterMap());
    }
    for (TaskCluster tc : getActivityClusterPlan(candidate).getTaskClusters()) {
        if (!tc.getProducedPartitions().isEmpty()) {
            continue;
        }
        TaskClusterAttempt tca = findLastTaskClusterAttempt(tc);
        if (tca == null || tca.getStatus() != TaskClusterAttempt.TaskClusterStatus.COMPLETED) {
            frontier.add(tc);
        }
    }
}

Example 4 with ActivityClusterPlan

use of org.apache.hyracks.control.cc.job.ActivityClusterPlan in project asterixdb by apache.

the class ActivityClusterPlanner method planActivityCluster.

ActivityClusterPlan planActivityCluster(ActivityCluster ac) throws HyracksException {
    JobRun jobRun = executor.getJobRun();
    Map<ActivityId, ActivityPartitionDetails> pcMap = computePartitionCounts(ac);
    Map<ActivityId, ActivityPlan> activityPlanMap = buildActivityPlanMap(ac, jobRun, pcMap);
    assignConnectorPolicy(ac, activityPlanMap);
    TaskCluster[] taskClusters = computeTaskClusters(ac, jobRun, activityPlanMap);
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Plan for " + ac);
        LOGGER.info("Built " + taskClusters.length + " Task Clusters");
        for (TaskCluster tc : taskClusters) {
            LOGGER.info("Tasks: " + Arrays.toString(tc.getTasks()));
        }
    }
    return new ActivityClusterPlan(taskClusters, activityPlanMap);
}