Examples with ActivityPlan org.apache.hyracks.control.cc.job.ActivityPlan used on opensource projects

Example 1 with ActivityPlan

use of org.apache.hyracks.control.cc.job.ActivityPlan in project asterixdb by apache.

the class ActivityClusterPlanner method buildActivityPlanMap.

private Map<ActivityId, ActivityPlan> buildActivityPlanMap(ActivityCluster ac, JobRun jobRun, Map<ActivityId, ActivityPartitionDetails> pcMap) {
    Map<ActivityId, ActivityPlan> activityPlanMap = new HashMap<>();
    Set<ActivityId> depAnIds = new HashSet<>();
    for (ActivityId anId : ac.getActivityMap().keySet()) {
        depAnIds.clear();
        getDependencyActivityIds(depAnIds, anId, ac);
        ActivityPartitionDetails apd = pcMap.get(anId);
        Task[] tasks = new Task[apd.getPartitionCount()];
        ActivityPlan activityPlan = new ActivityPlan(apd);
        for (int i = 0; i < tasks.length; ++i) {
            TaskId tid = new TaskId(anId, i);
            tasks[i] = new Task(tid, activityPlan);
            for (ActivityId danId : depAnIds) {
                ActivityCluster dAC = ac.getActivityClusterGraph().getActivityMap().get(danId);
                ActivityClusterPlan dACP = jobRun.getActivityClusterPlanMap().get(dAC.getId());
                assert dACP != null : "IllegalStateEncountered: Dependent AC is being planned without a plan for " + "dependency AC: Encountered no plan for ActivityID " + danId;
                Task[] dATasks = dACP.getActivityPlanMap().get(danId).getTasks();
                assert dATasks != null : "IllegalStateEncountered: Dependent AC is being planned without a plan for" + " dependency AC: Encountered no plan for ActivityID " + danId;
                assert dATasks.length == tasks.length : "Dependency activity partitioned differently from " + "dependent: " + dATasks.length + " != " + tasks.length;
                Task dTask = dATasks[i];
                TaskId dTaskId = dTask.getTaskId();
                tasks[i].getDependencies().add(dTaskId);
                dTask.getDependents().add(tid);
            }
        }
        activityPlan.setTasks(tasks);
        activityPlanMap.put(anId, activityPlan);
    }
    return activityPlanMap;
}

Example 2 with ActivityPlan

use of org.apache.hyracks.control.cc.job.ActivityPlan in project asterixdb by apache.

the class ActivityClusterPlanner method buildConnectorPolicyUnawareTaskClusters.

private TaskCluster[] buildConnectorPolicyUnawareTaskClusters(ActivityCluster ac, Map<ActivityId, ActivityPlan> activityPlanMap) {
    List<Task> taskStates = new ArrayList<>();
    for (ActivityId anId : ac.getActivityMap().keySet()) {
        ActivityPlan ap = activityPlanMap.get(anId);
        Task[] tasks = ap.getTasks();
        taskStates.addAll(Arrays.asList(tasks));
    }
    TaskCluster tc = new TaskCluster(new TaskClusterId(ac.getId(), 0), ac, taskStates.toArray(new Task[taskStates.size()]));
    for (Task t : tc.getTasks()) {
        t.setTaskCluster(tc);
    }
    return new TaskCluster[] { tc };
}

Example 3 with ActivityPlan

use of org.apache.hyracks.control.cc.job.ActivityPlan in project asterixdb by apache.

the class ActivityClusterPlanner method planActivityCluster.

ActivityClusterPlan planActivityCluster(ActivityCluster ac) throws HyracksException {
    JobRun jobRun = executor.getJobRun();
    Map<ActivityId, ActivityPartitionDetails> pcMap = computePartitionCounts(ac);
    Map<ActivityId, ActivityPlan> activityPlanMap = buildActivityPlanMap(ac, jobRun, pcMap);
    assignConnectorPolicy(ac, activityPlanMap);
    TaskCluster[] taskClusters = computeTaskClusters(ac, jobRun, activityPlanMap);
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Plan for " + ac);
        LOGGER.info("Built " + taskClusters.length + " Task Clusters");
        for (TaskCluster tc : taskClusters) {
            LOGGER.info("Tasks: " + Arrays.toString(tc.getTasks()));
        }
    }
    return new ActivityClusterPlan(taskClusters, activityPlanMap);
}

Example 4 with ActivityPlan

use of org.apache.hyracks.control.cc.job.ActivityPlan in project asterixdb by apache.

the class ActivityClusterPlanner method buildConnectorPolicyAwareTaskClusters.

private TaskCluster[] buildConnectorPolicyAwareTaskClusters(ActivityCluster ac, Map<ActivityId, ActivityPlan> activityPlanMap, Map<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> taskConnectivity) {
    Map<TaskId, Set<TaskId>> taskClusterMap = new HashMap<>();
    for (ActivityId anId : ac.getActivityMap().keySet()) {
        ActivityPlan ap = activityPlanMap.get(anId);
        Task[] tasks = ap.getTasks();
        for (Task t : tasks) {
            Set<TaskId> cluster = new HashSet<>();
            TaskId tid = t.getTaskId();
            cluster.add(tid);
            taskClusterMap.put(tid, cluster);
        }
    }
    JobRun jobRun = executor.getJobRun();
    Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies = jobRun.getConnectorPolicyMap();
    for (Map.Entry<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> e : taskConnectivity.entrySet()) {
        Set<TaskId> cluster = taskClusterMap.get(e.getKey());
        for (Pair<TaskId, ConnectorDescriptorId> p : e.getValue()) {
            IConnectorPolicy cPolicy = connectorPolicies.get(p.getRight());
            if (cPolicy.requiresProducerConsumerCoscheduling()) {
                cluster.add(p.getLeft());
            }
        }
    }
    /*
         * We compute the transitive closure of this (producer-consumer) relation to find the largest set of
         * tasks that need to be co-scheduled.
         */
    int counter = 0;
    TaskId[] ordinalList = new TaskId[taskClusterMap.size()];
    Map<TaskId, Integer> ordinalMap = new HashMap<>();
    for (TaskId tid : taskClusterMap.keySet()) {
        ordinalList[counter] = tid;
        ordinalMap.put(tid, counter);
        ++counter;
    }
    int n = ordinalList.length;
    BitSet[] paths = new BitSet[n];
    for (Map.Entry<TaskId, Set<TaskId>> e : taskClusterMap.entrySet()) {
        int i = ordinalMap.get(e.getKey());
        BitSet bsi = paths[i];
        if (bsi == null) {
            bsi = new BitSet(n);
            paths[i] = bsi;
        }
        for (TaskId ttid : e.getValue()) {
            int j = ordinalMap.get(ttid);
            paths[i].set(j);
            BitSet bsj = paths[j];
            if (bsj == null) {
                bsj = new BitSet(n);
                paths[j] = bsj;
            }
            bsj.set(i);
        }
    }
    for (int k = 0; k < n; ++k) {
        for (int i = paths[k].nextSetBit(0); i >= 0; i = paths[k].nextSetBit(i + 1)) {
            for (int j = paths[i].nextClearBit(0); j < n && j >= 0; j = paths[i].nextClearBit(j + 1)) {
                paths[i].set(j, paths[k].get(j));
                paths[j].set(i, paths[i].get(j));
            }
        }
    }
    BitSet pending = new BitSet(n);
    pending.set(0, n);
    List<List<TaskId>> clusters = new ArrayList<>();
    for (int i = pending.nextSetBit(0); i >= 0; i = pending.nextSetBit(i)) {
        List<TaskId> cluster = new ArrayList<>();
        for (int j = paths[i].nextSetBit(0); j >= 0; j = paths[i].nextSetBit(j + 1)) {
            cluster.add(ordinalList[j]);
            pending.clear(j);
        }
        clusters.add(cluster);
    }
    List<TaskCluster> tcSet = new ArrayList<>();
    counter = 0;
    for (List<TaskId> cluster : clusters) {
        List<Task> taskStates = new ArrayList<>();
        for (TaskId tid : cluster) {
            taskStates.add(activityPlanMap.get(tid.getActivityId()).getTasks()[tid.getPartition()]);
        }
        TaskCluster tc = new TaskCluster(new TaskClusterId(ac.getId(), counter++), ac, taskStates.toArray(new Task[taskStates.size()]));
        tcSet.add(tc);
        for (TaskId tid : cluster) {
            activityPlanMap.get(tid.getActivityId()).getTasks()[tid.getPartition()].setTaskCluster(tc);
        }
    }
    return tcSet.toArray(new TaskCluster[tcSet.size()]);
}

Example 5 with ActivityPlan

use of org.apache.hyracks.control.cc.job.ActivityPlan in project asterixdb by apache.

the class AbstractTaskLifecycleWork method runWork.

@Override
public final void runWork() {
    IJobManager jobManager = ccs.getJobManager();
    JobRun run = jobManager.get(jobId);
    if (run != null) {
        TaskId tid = taId.getTaskId();
        Map<ActivityId, ActivityCluster> activityClusterMap = run.getActivityClusterGraph().getActivityMap();
        ActivityCluster ac = activityClusterMap.get(tid.getActivityId());
        if (ac != null) {
            Map<ActivityId, ActivityPlan> taskStateMap = run.getActivityClusterPlanMap().get(ac.getId()).getActivityPlanMap();
            Task[] taskStates = taskStateMap.get(tid.getActivityId()).getTasks();
            if (taskStates != null && taskStates.length > tid.getPartition()) {
                Task ts = taskStates[tid.getPartition()];
                TaskCluster tc = ts.getTaskCluster();
                List<TaskClusterAttempt> taskClusterAttempts = tc.getAttempts();
                if (taskClusterAttempts != null && taskClusterAttempts.size() > taId.getAttempt()) {
                    TaskClusterAttempt tca = taskClusterAttempts.get(taId.getAttempt());
                    TaskAttempt ta = tca.getTaskAttempts().get(tid);
                    if (ta != null) {
                        performEvent(ta);
                    }
                }
            }
        }
    }
}