Examples with ActivityClusterId org.apache.hyracks.api.job.ActivityClusterId used on opensource projects

Example 1 with ActivityClusterId

use of org.apache.hyracks.api.job.ActivityClusterId in project asterixdb by apache.

the class ActivityClusterGraphBuilder method inferActivityClusters.

public ActivityClusterGraph inferActivityClusters(JobId jobId, JobActivityGraph jag) {
    /*
         * Build initial equivalence sets map. We create a map such that for each IOperatorTask, t -> { t }
         */
    Map<ActivityId, Set<ActivityId>> stageMap = new HashMap<ActivityId, Set<ActivityId>>();
    Set<Set<ActivityId>> stages = new HashSet<Set<ActivityId>>();
    for (ActivityId taskId : jag.getActivityMap().keySet()) {
        Set<ActivityId> eqSet = new HashSet<ActivityId>();
        eqSet.add(taskId);
        stageMap.put(taskId, eqSet);
        stages.add(eqSet);
    }
    boolean changed = true;
    while (changed) {
        changed = false;
        Pair<ActivityId, ActivityId> pair = findMergePair(jag, stages);
        if (pair != null) {
            merge(stageMap, stages, pair.getLeft(), pair.getRight());
            changed = true;
        }
    }
    ActivityClusterGraph acg = new ActivityClusterGraph();
    Map<ActivityId, ActivityCluster> acMap = new HashMap<ActivityId, ActivityCluster>();
    int acCounter = 0;
    Map<ActivityId, IActivity> activityNodeMap = jag.getActivityMap();
    List<ActivityCluster> acList = new ArrayList<ActivityCluster>();
    for (Set<ActivityId> stage : stages) {
        ActivityCluster ac = new ActivityCluster(acg, new ActivityClusterId(jobId, acCounter++));
        acList.add(ac);
        for (ActivityId aid : stage) {
            IActivity activity = activityNodeMap.get(aid);
            ac.addActivity(activity);
            acMap.put(aid, ac);
        }
    }
    for (Set<ActivityId> stage : stages) {
        for (ActivityId aid : stage) {
            IActivity activity = activityNodeMap.get(aid);
            ActivityCluster ac = acMap.get(aid);
            List<IConnectorDescriptor> aOutputs = jag.getActivityOutputMap().get(aid);
            if (aOutputs == null || aOutputs.isEmpty()) {
                ac.addRoot(activity);
            } else {
                int nActivityOutputs = aOutputs.size();
                for (int i = 0; i < nActivityOutputs; ++i) {
                    IConnectorDescriptor conn = aOutputs.get(i);
                    ac.addConnector(conn);
                    Pair<Pair<IActivity, Integer>, Pair<IActivity, Integer>> pcPair = jag.getConnectorActivityMap().get(conn.getConnectorId());
                    ac.connect(conn, activity, i, pcPair.getRight().getLeft(), pcPair.getRight().getRight(), jag.getConnectorRecordDescriptorMap().get(conn.getConnectorId()));
                }
            }
        }
    }
    Map<ActivityId, Set<ActivityId>> blocked2BlockerMap = jag.getBlocked2BlockerMap();
    for (ActivityCluster s : acList) {
        Map<ActivityId, Set<ActivityId>> acBlocked2BlockerMap = s.getBlocked2BlockerMap();
        Set<ActivityCluster> blockerStages = new HashSet<ActivityCluster>();
        for (ActivityId t : s.getActivityMap().keySet()) {
            Set<ActivityId> blockerTasks = blocked2BlockerMap.get(t);
            acBlocked2BlockerMap.put(t, blockerTasks);
            if (blockerTasks != null) {
                for (ActivityId bt : blockerTasks) {
                    blockerStages.add(acMap.get(bt));
                }
            }
        }
        for (ActivityCluster bs : blockerStages) {
            s.getDependencies().add(bs);
        }
    }
    acg.addActivityClusters(acList);
    if (LOGGER.isLoggable(Level.FINE)) {
        LOGGER.fine(acg.toJSON().asText());
    }
    return acg;
}

Example 2 with ActivityClusterId

use of org.apache.hyracks.api.job.ActivityClusterId in project asterixdb by apache.

the class SuperActivity method createPushRuntime.

@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, final IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions) throws HyracksDataException {
    final Map<ActivityId, IActivity> startActivities = new HashMap<ActivityId, IActivity>();
    Map<ActivityId, IActivity> activities = getActivityMap();
    for (Entry<ActivityId, IActivity> entry : activities.entrySet()) {
        /**
             * extract start activities
             */
        List<IConnectorDescriptor> conns = getActivityInputMap().get(entry.getKey());
        if (conns == null || conns.isEmpty()) {
            startActivities.put(entry.getKey(), entry.getValue());
        }
    }
    /**
         * wrap a RecordDescriptorProvider for the super activity
         */
    IRecordDescriptorProvider wrappedRecDescProvider = new IRecordDescriptorProvider() {

        @Override
        public RecordDescriptor getInputRecordDescriptor(ActivityId aid, int inputIndex) {
            if (startActivities.get(aid) != null) {
                /**
                     * if the activity is a start (input boundary) activity
                     */
                int superActivityInputChannel = SuperActivity.this.getClusterInputIndex(Pair.of(aid, inputIndex));
                if (superActivityInputChannel >= 0) {
                    return recordDescProvider.getInputRecordDescriptor(activityId, superActivityInputChannel);
                }
            }
            if (SuperActivity.this.getActivityMap().get(aid) != null) {
                /**
                     * if the activity is an internal activity of the super activity
                     */
                IConnectorDescriptor conn = getActivityInputMap().get(aid).get(inputIndex);
                return getConnectorRecordDescriptorMap().get(conn.getConnectorId());
            }
            /**
                 * the following is for the case where the activity is in other SuperActivities
                 */
            ActivityClusterGraph acg = SuperActivity.this.getActivityClusterGraph();
            for (Entry<ActivityClusterId, ActivityCluster> entry : acg.getActivityClusterMap().entrySet()) {
                ActivityCluster ac = entry.getValue();
                for (Entry<ActivityId, IActivity> saEntry : ac.getActivityMap().entrySet()) {
                    SuperActivity sa = (SuperActivity) saEntry.getValue();
                    if (sa.getActivityMap().get(aid) != null) {
                        List<IConnectorDescriptor> conns = sa.getActivityInputMap().get(aid);
                        if (conns != null && conns.size() >= inputIndex) {
                            IConnectorDescriptor conn = conns.get(inputIndex);
                            return sa.getConnectorRecordDescriptorMap().get(conn.getConnectorId());
                        } else {
                            int superActivityInputChannel = sa.getClusterInputIndex(Pair.of(aid, inputIndex));
                            if (superActivityInputChannel >= 0) {
                                return recordDescProvider.getInputRecordDescriptor(sa.getActivityId(), superActivityInputChannel);
                            }
                        }
                    }
                }
            }
            return null;
        }

        @Override
        public RecordDescriptor getOutputRecordDescriptor(ActivityId aid, int outputIndex) {
            /**
                 * if the activity is an output-boundary activity
                 */
            int superActivityOutputChannel = SuperActivity.this.getClusterOutputIndex(Pair.of(aid, outputIndex));
            if (superActivityOutputChannel >= 0) {
                return recordDescProvider.getOutputRecordDescriptor(activityId, superActivityOutputChannel);
            }
            if (SuperActivity.this.getActivityMap().get(aid) != null) {
                /**
                     * if the activity is an internal activity of the super activity
                     */
                IConnectorDescriptor conn = getActivityOutputMap().get(aid).get(outputIndex);
                return getConnectorRecordDescriptorMap().get(conn.getConnectorId());
            }
            /**
                 * the following is for the case where the activity is in other SuperActivities
                 */
            ActivityClusterGraph acg = SuperActivity.this.getActivityClusterGraph();
            for (Entry<ActivityClusterId, ActivityCluster> entry : acg.getActivityClusterMap().entrySet()) {
                ActivityCluster ac = entry.getValue();
                for (Entry<ActivityId, IActivity> saEntry : ac.getActivityMap().entrySet()) {
                    SuperActivity sa = (SuperActivity) saEntry.getValue();
                    if (sa.getActivityMap().get(aid) != null) {
                        List<IConnectorDescriptor> conns = sa.getActivityOutputMap().get(aid);
                        if (conns != null && conns.size() >= outputIndex) {
                            IConnectorDescriptor conn = conns.get(outputIndex);
                            return sa.getConnectorRecordDescriptorMap().get(conn.getConnectorId());
                        } else {
                            superActivityOutputChannel = sa.getClusterOutputIndex(Pair.of(aid, outputIndex));
                            if (superActivityOutputChannel >= 0) {
                                return recordDescProvider.getOutputRecordDescriptor(sa.getActivityId(), superActivityOutputChannel);
                            }
                        }
                    }
                }
            }
            return null;
        }
    };
    return new SuperActivityOperatorNodePushable(this, startActivities, ctx, wrappedRecDescProvider, partition, nPartitions);
}