Examples with IConnectorDescriptor org.apache.hyracks.api.dataflow.IConnectorDescriptor used on opensource projects

Example 11 with IConnectorDescriptor

use of org.apache.hyracks.api.dataflow.IConnectorDescriptor in project asterixdb by apache.

the class JobSpecificationActivityClusterGraphGeneratorFactory method createActivityClusterGraphGenerator.

@Override
public IActivityClusterGraphGenerator createActivityClusterGraphGenerator(JobId jobId, final ICCServiceContext ccServiceCtx, Set<JobFlag> jobFlags) throws HyracksException {
    final JobActivityGraphBuilder builder = new JobActivityGraphBuilder(spec, jobFlags);
    PlanUtils.visit(spec, new IConnectorDescriptorVisitor() {

        @Override
        public void visit(IConnectorDescriptor conn) throws HyracksException {
            builder.addConnector(conn);
        }
    });
    PlanUtils.visit(spec, new IOperatorDescriptorVisitor() {

        @Override
        public void visit(IOperatorDescriptor op) {
            op.contributeActivities(builder);
        }
    });
    builder.finish();
    final JobActivityGraph jag = builder.getActivityGraph();
    ActivityClusterGraphBuilder acgb = new ActivityClusterGraphBuilder();
    final ActivityClusterGraph acg = acgb.inferActivityClusters(jobId, jag);
    acg.setFrameSize(spec.getFrameSize());
    acg.setMaxReattempts(spec.getMaxReattempts());
    acg.setJobletEventListenerFactory(spec.getJobletEventListenerFactory());
    acg.setGlobalJobDataFactory(spec.getGlobalJobDataFactory());
    acg.setConnectorPolicyAssignmentPolicy(spec.getConnectorPolicyAssignmentPolicy());
    acg.setUseConnectorPolicyForScheduling(spec.isUseConnectorPolicyForScheduling());
    final Set<Constraint> constraints = new HashSet<>();
    final IConstraintAcceptor acceptor = new IConstraintAcceptor() {

        @Override
        public void addConstraint(Constraint constraint) {
            constraints.add(constraint);
        }
    };
    PlanUtils.visit(spec, new IOperatorDescriptorVisitor() {

        @Override
        public void visit(IOperatorDescriptor op) {
            op.contributeSchedulingConstraints(acceptor, ccServiceCtx);
        }
    });
    PlanUtils.visit(spec, new IConnectorDescriptorVisitor() {

        @Override
        public void visit(IConnectorDescriptor conn) {
            conn.contributeSchedulingConstraints(acceptor, acg.getConnectorMap().get(conn.getConnectorId()), ccServiceCtx);
        }
    });
    constraints.addAll(spec.getUserConstraints());
    return new IActivityClusterGraphGenerator() {

        @Override
        public ActivityClusterGraph initialize() {
            ActivityClusterGraphRewriter rewriter = new ActivityClusterGraphRewriter();
            rewriter.rewrite(acg);
            return acg;
        }

        @Override
        public Set<Constraint> getConstraints() {
            return constraints;
        }
    };
}

Example 12 with IConnectorDescriptor

use of org.apache.hyracks.api.dataflow.IConnectorDescriptor in project asterixdb by apache.

the class ActivityClusterGraphBuilder method findMergePair.

private static Pair<ActivityId, ActivityId> findMergePair(JobActivityGraph jag, Set<Set<ActivityId>> eqSets) {
    for (Set<ActivityId> eqSet : eqSets) {
        for (ActivityId t : eqSet) {
            List<IConnectorDescriptor> inputList = jag.getActivityInputMap().get(t);
            if (inputList != null) {
                for (IConnectorDescriptor conn : inputList) {
                    ActivityId inTask = jag.getProducerActivity(conn.getConnectorId());
                    if (!eqSet.contains(inTask)) {
                        return Pair.<ActivityId, ActivityId>of(t, inTask);
                    }
                }
            }
            List<IConnectorDescriptor> outputList = jag.getActivityOutputMap().get(t);
            if (outputList != null) {
                for (IConnectorDescriptor conn : outputList) {
                    ActivityId outTask = jag.getConsumerActivity(conn.getConnectorId());
                    if (!eqSet.contains(outTask)) {
                        return Pair.<ActivityId, ActivityId>of(t, outTask);
                    }
                }
            }
        }
    }
    return null;
}

Example 13 with IConnectorDescriptor

use of org.apache.hyracks.api.dataflow.IConnectorDescriptor in project asterixdb by apache.

the class ActivityClusterGraphBuilder method inferActivityClusters.

public ActivityClusterGraph inferActivityClusters(JobId jobId, JobActivityGraph jag) {
    /*
         * Build initial equivalence sets map. We create a map such that for each IOperatorTask, t -> { t }
         */
    Map<ActivityId, Set<ActivityId>> stageMap = new HashMap<ActivityId, Set<ActivityId>>();
    Set<Set<ActivityId>> stages = new HashSet<Set<ActivityId>>();
    for (ActivityId taskId : jag.getActivityMap().keySet()) {
        Set<ActivityId> eqSet = new HashSet<ActivityId>();
        eqSet.add(taskId);
        stageMap.put(taskId, eqSet);
        stages.add(eqSet);
    }
    boolean changed = true;
    while (changed) {
        changed = false;
        Pair<ActivityId, ActivityId> pair = findMergePair(jag, stages);
        if (pair != null) {
            merge(stageMap, stages, pair.getLeft(), pair.getRight());
            changed = true;
        }
    }
    ActivityClusterGraph acg = new ActivityClusterGraph();
    Map<ActivityId, ActivityCluster> acMap = new HashMap<ActivityId, ActivityCluster>();
    int acCounter = 0;
    Map<ActivityId, IActivity> activityNodeMap = jag.getActivityMap();
    List<ActivityCluster> acList = new ArrayList<ActivityCluster>();
    for (Set<ActivityId> stage : stages) {
        ActivityCluster ac = new ActivityCluster(acg, new ActivityClusterId(jobId, acCounter++));
        acList.add(ac);
        for (ActivityId aid : stage) {
            IActivity activity = activityNodeMap.get(aid);
            ac.addActivity(activity);
            acMap.put(aid, ac);
        }
    }
    for (Set<ActivityId> stage : stages) {
        for (ActivityId aid : stage) {
            IActivity activity = activityNodeMap.get(aid);
            ActivityCluster ac = acMap.get(aid);
            List<IConnectorDescriptor> aOutputs = jag.getActivityOutputMap().get(aid);
            if (aOutputs == null || aOutputs.isEmpty()) {
                ac.addRoot(activity);
            } else {
                int nActivityOutputs = aOutputs.size();
                for (int i = 0; i < nActivityOutputs; ++i) {
                    IConnectorDescriptor conn = aOutputs.get(i);
                    ac.addConnector(conn);
                    Pair<Pair<IActivity, Integer>, Pair<IActivity, Integer>> pcPair = jag.getConnectorActivityMap().get(conn.getConnectorId());
                    ac.connect(conn, activity, i, pcPair.getRight().getLeft(), pcPair.getRight().getRight(), jag.getConnectorRecordDescriptorMap().get(conn.getConnectorId()));
                }
            }
        }
    }
    Map<ActivityId, Set<ActivityId>> blocked2BlockerMap = jag.getBlocked2BlockerMap();
    for (ActivityCluster s : acList) {
        Map<ActivityId, Set<ActivityId>> acBlocked2BlockerMap = s.getBlocked2BlockerMap();
        Set<ActivityCluster> blockerStages = new HashSet<ActivityCluster>();
        for (ActivityId t : s.getActivityMap().keySet()) {
            Set<ActivityId> blockerTasks = blocked2BlockerMap.get(t);
            acBlocked2BlockerMap.put(t, blockerTasks);
            if (blockerTasks != null) {
                for (ActivityId bt : blockerTasks) {
                    blockerStages.add(acMap.get(bt));
                }
            }
        }
        for (ActivityCluster bs : blockerStages) {
            s.getDependencies().add(bs);
        }
    }
    acg.addActivityClusters(acList);
    if (LOGGER.isLoggable(Level.FINE)) {
        LOGGER.fine(acg.toJSON().asText());
    }
    return acg;
}

Example 14 with IConnectorDescriptor

use of org.apache.hyracks.api.dataflow.IConnectorDescriptor in project asterixdb by apache.

the class SuperActivityOperatorNodePushable method init.

private void init() throws HyracksDataException {
    Queue<Pair<Pair<IActivity, Integer>, Pair<IActivity, Integer>>> childQueue = new LinkedList<>();
    List<IConnectorDescriptor> outputConnectors;
    /*
         * Set up the source operators
         */
    for (Entry<ActivityId, IActivity> entry : startActivities.entrySet()) {
        IOperatorNodePushable opPushable = entry.getValue().createPushRuntime(ctx, recordDescProvider, partition, nPartitions);
        operatorNodePushablesBFSOrder.add(opPushable);
        operatorNodePushables.put(entry.getKey(), opPushable);
        inputArity += opPushable.getInputArity();
        outputConnectors = MapUtils.getObject(parent.getActivityOutputMap(), entry.getKey(), Collections.emptyList());
        for (IConnectorDescriptor conn : outputConnectors) {
            childQueue.add(parent.getConnectorActivityMap().get(conn.getConnectorId()));
        }
    }
    /*
         * Using BFS (breadth-first search) to construct to runtime execution DAG...
         */
    while (!childQueue.isEmpty()) {
        /*
             * construct the source to destination information
             */
        Pair<Pair<IActivity, Integer>, Pair<IActivity, Integer>> channel = childQueue.poll();
        ActivityId sourceId = channel.getLeft().getLeft().getActivityId();
        int outputChannel = channel.getLeft().getRight();
        ActivityId destId = channel.getRight().getLeft().getActivityId();
        int inputChannel = channel.getRight().getRight();
        IOperatorNodePushable sourceOp = operatorNodePushables.get(sourceId);
        IOperatorNodePushable destOp = operatorNodePushables.get(destId);
        if (destOp == null) {
            destOp = channel.getRight().getLeft().createPushRuntime(ctx, recordDescProvider, partition, nPartitions);
            operatorNodePushablesBFSOrder.add(destOp);
            operatorNodePushables.put(destId, destOp);
        }
        /*
             * construct the dataflow connection from a producer to a consumer
             */
        sourceOp.setOutputFrameWriter(outputChannel, destOp.getInputFrameWriter(inputChannel), recordDescProvider.getInputRecordDescriptor(destId, inputChannel));
        /*
             * traverse to the child of the current activity
             */
        outputConnectors = MapUtils.getObject(parent.getActivityOutputMap(), destId, Collections.emptyList());
        /*
             * expend the executing activities further to the downstream
             */
        for (IConnectorDescriptor conn : outputConnectors) {
            if (conn != null) {
                childQueue.add(parent.getConnectorActivityMap().get(conn.getConnectorId()));
            }
        }
    }
}

Example 15 with IConnectorDescriptor

use of org.apache.hyracks.api.dataflow.IConnectorDescriptor in project asterixdb by apache.

the class RangePartitionExchangePOperator method createConnectorDescriptor.

@Override
public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec, ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
    int n = partitioningFields.size();
    int[] sortFields = new int[n];
    IBinaryComparatorFactory[] comps = new IBinaryComparatorFactory[n];
    INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider();
    INormalizedKeyComputerFactory nkcf = null;
    IVariableTypeEnvironment env = context.getTypeEnvironment(op);
    int i = 0;
    for (OrderColumn oc : partitioningFields) {
        LogicalVariable var = oc.getColumn();
        sortFields[i] = opSchema.findVariable(var);
        Object type = env.getVarType(var);
        OrderKind order = oc.getOrder();
        if (i == 0 && nkcfProvider != null && type != null) {
            nkcf = nkcfProvider.getNormalizedKeyComputerFactory(type, order == OrderKind.ASC);
        }
        IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
        comps[i] = bcfp.getBinaryComparatorFactory(type, oc.getOrder() == OrderKind.ASC);
        i++;
    }
    ITuplePartitionComputerFactory tpcf = new FieldRangePartitionComputerFactory(sortFields, comps, rangeMap);
    IConnectorDescriptor conn = new MToNPartitioningConnectorDescriptor(spec, tpcf);
    return new Pair<IConnectorDescriptor, TargetConstraint>(conn, null);
}