Examples with ActivityId org.apache.hyracks.api.dataflow.ActivityId used on opensource projects

Example 6 with ActivityId

use of org.apache.hyracks.api.dataflow.ActivityId in project asterixdb by apache.

the class JobActivityGraphBuilder method addActivity.

@Override
public void addActivity(IOperatorDescriptor op, IActivity task) {
    activityOperatorMap.put(task.getActivityId(), op);
    ActivityId activityId = task.getActivityId();
    jag.getActivityMap().put(activityId, task);
}

Example 7 with ActivityId

use of org.apache.hyracks.api.dataflow.ActivityId in project asterixdb by apache.

the class ActivityClusterGraphBuilder method findMergePair.

private static Pair<ActivityId, ActivityId> findMergePair(JobActivityGraph jag, Set<Set<ActivityId>> eqSets) {
    for (Set<ActivityId> eqSet : eqSets) {
        for (ActivityId t : eqSet) {
            List<IConnectorDescriptor> inputList = jag.getActivityInputMap().get(t);
            if (inputList != null) {
                for (IConnectorDescriptor conn : inputList) {
                    ActivityId inTask = jag.getProducerActivity(conn.getConnectorId());
                    if (!eqSet.contains(inTask)) {
                        return Pair.<ActivityId, ActivityId>of(t, inTask);
                    }
                }
            }
            List<IConnectorDescriptor> outputList = jag.getActivityOutputMap().get(t);
            if (outputList != null) {
                for (IConnectorDescriptor conn : outputList) {
                    ActivityId outTask = jag.getConsumerActivity(conn.getConnectorId());
                    if (!eqSet.contains(outTask)) {
                        return Pair.<ActivityId, ActivityId>of(t, outTask);
                    }
                }
            }
        }
    }
    return null;
}

Example 8 with ActivityId

use of org.apache.hyracks.api.dataflow.ActivityId in project asterixdb by apache.

the class ActivityClusterGraphBuilder method inferActivityClusters.

public ActivityClusterGraph inferActivityClusters(JobId jobId, JobActivityGraph jag) {
    /*
         * Build initial equivalence sets map. We create a map such that for each IOperatorTask, t -> { t }
         */
    Map<ActivityId, Set<ActivityId>> stageMap = new HashMap<ActivityId, Set<ActivityId>>();
    Set<Set<ActivityId>> stages = new HashSet<Set<ActivityId>>();
    for (ActivityId taskId : jag.getActivityMap().keySet()) {
        Set<ActivityId> eqSet = new HashSet<ActivityId>();
        eqSet.add(taskId);
        stageMap.put(taskId, eqSet);
        stages.add(eqSet);
    }
    boolean changed = true;
    while (changed) {
        changed = false;
        Pair<ActivityId, ActivityId> pair = findMergePair(jag, stages);
        if (pair != null) {
            merge(stageMap, stages, pair.getLeft(), pair.getRight());
            changed = true;
        }
    }
    ActivityClusterGraph acg = new ActivityClusterGraph();
    Map<ActivityId, ActivityCluster> acMap = new HashMap<ActivityId, ActivityCluster>();
    int acCounter = 0;
    Map<ActivityId, IActivity> activityNodeMap = jag.getActivityMap();
    List<ActivityCluster> acList = new ArrayList<ActivityCluster>();
    for (Set<ActivityId> stage : stages) {
        ActivityCluster ac = new ActivityCluster(acg, new ActivityClusterId(jobId, acCounter++));
        acList.add(ac);
        for (ActivityId aid : stage) {
            IActivity activity = activityNodeMap.get(aid);
            ac.addActivity(activity);
            acMap.put(aid, ac);
        }
    }
    for (Set<ActivityId> stage : stages) {
        for (ActivityId aid : stage) {
            IActivity activity = activityNodeMap.get(aid);
            ActivityCluster ac = acMap.get(aid);
            List<IConnectorDescriptor> aOutputs = jag.getActivityOutputMap().get(aid);
            if (aOutputs == null || aOutputs.isEmpty()) {
                ac.addRoot(activity);
            } else {
                int nActivityOutputs = aOutputs.size();
                for (int i = 0; i < nActivityOutputs; ++i) {
                    IConnectorDescriptor conn = aOutputs.get(i);
                    ac.addConnector(conn);
                    Pair<Pair<IActivity, Integer>, Pair<IActivity, Integer>> pcPair = jag.getConnectorActivityMap().get(conn.getConnectorId());
                    ac.connect(conn, activity, i, pcPair.getRight().getLeft(), pcPair.getRight().getRight(), jag.getConnectorRecordDescriptorMap().get(conn.getConnectorId()));
                }
            }
        }
    }
    Map<ActivityId, Set<ActivityId>> blocked2BlockerMap = jag.getBlocked2BlockerMap();
    for (ActivityCluster s : acList) {
        Map<ActivityId, Set<ActivityId>> acBlocked2BlockerMap = s.getBlocked2BlockerMap();
        Set<ActivityCluster> blockerStages = new HashSet<ActivityCluster>();
        for (ActivityId t : s.getActivityMap().keySet()) {
            Set<ActivityId> blockerTasks = blocked2BlockerMap.get(t);
            acBlocked2BlockerMap.put(t, blockerTasks);
            if (blockerTasks != null) {
                for (ActivityId bt : blockerTasks) {
                    blockerStages.add(acMap.get(bt));
                }
            }
        }
        for (ActivityCluster bs : blockerStages) {
            s.getDependencies().add(bs);
        }
    }
    acg.addActivityClusters(acList);
    if (LOGGER.isLoggable(Level.FINE)) {
        LOGGER.fine(acg.toJSON().asText());
    }
    return acg;
}

Example 9 with ActivityId

use of org.apache.hyracks.api.dataflow.ActivityId in project asterixdb by apache.

the class SuperActivityOperatorNodePushable method init.

private void init() throws HyracksDataException {
    Queue<Pair<Pair<IActivity, Integer>, Pair<IActivity, Integer>>> childQueue = new LinkedList<>();
    List<IConnectorDescriptor> outputConnectors;
    /*
         * Set up the source operators
         */
    for (Entry<ActivityId, IActivity> entry : startActivities.entrySet()) {
        IOperatorNodePushable opPushable = entry.getValue().createPushRuntime(ctx, recordDescProvider, partition, nPartitions);
        operatorNodePushablesBFSOrder.add(opPushable);
        operatorNodePushables.put(entry.getKey(), opPushable);
        inputArity += opPushable.getInputArity();
        outputConnectors = MapUtils.getObject(parent.getActivityOutputMap(), entry.getKey(), Collections.emptyList());
        for (IConnectorDescriptor conn : outputConnectors) {
            childQueue.add(parent.getConnectorActivityMap().get(conn.getConnectorId()));
        }
    }
    /*
         * Using BFS (breadth-first search) to construct to runtime execution DAG...
         */
    while (!childQueue.isEmpty()) {
        /*
             * construct the source to destination information
             */
        Pair<Pair<IActivity, Integer>, Pair<IActivity, Integer>> channel = childQueue.poll();
        ActivityId sourceId = channel.getLeft().getLeft().getActivityId();
        int outputChannel = channel.getLeft().getRight();
        ActivityId destId = channel.getRight().getLeft().getActivityId();
        int inputChannel = channel.getRight().getRight();
        IOperatorNodePushable sourceOp = operatorNodePushables.get(sourceId);
        IOperatorNodePushable destOp = operatorNodePushables.get(destId);
        if (destOp == null) {
            destOp = channel.getRight().getLeft().createPushRuntime(ctx, recordDescProvider, partition, nPartitions);
            operatorNodePushablesBFSOrder.add(destOp);
            operatorNodePushables.put(destId, destOp);
        }
        /*
             * construct the dataflow connection from a producer to a consumer
             */
        sourceOp.setOutputFrameWriter(outputChannel, destOp.getInputFrameWriter(inputChannel), recordDescProvider.getInputRecordDescriptor(destId, inputChannel));
        /*
             * traverse to the child of the current activity
             */
        outputConnectors = MapUtils.getObject(parent.getActivityOutputMap(), destId, Collections.emptyList());
        /*
             * expend the executing activities further to the downstream
             */
        for (IConnectorDescriptor conn : outputConnectors) {
            if (conn != null) {
                childQueue.add(parent.getConnectorActivityMap().get(conn.getConnectorId()));
            }
        }
    }
}

Example 10 with ActivityId

use of org.apache.hyracks.api.dataflow.ActivityId in project asterixdb by apache.

the class SuperActivityOperatorNodePushable method getInputFrameWriter.

@Override
public IFrameWriter getInputFrameWriter(final int index) {
    /*
         * get the right IFrameWriter from the cluster input index
         */
    Pair<ActivityId, Integer> activityIdInputIndex = parent.getActivityIdInputIndex(index);
    IOperatorNodePushable operatorNodePushable = operatorNodePushables.get(activityIdInputIndex.getLeft());
    return operatorNodePushable.getInputFrameWriter(activityIdInputIndex.getRight());
}