Examples with PTInput com.datatorrent.stram.plan.physical.PTOperator.PTInput used on opensource projects

Example 6 with PTInput

use of com.datatorrent.stram.plan.physical.PTOperator.PTInput in project apex-core by apache.

the class StreamMapping method redoMapping.

/**
   * rebuild the tree, which may cause more changes to execution layer than need be
   * TODO: investigate incremental logic
   */
private void redoMapping() {
    Set<Pair<PTOperator, InputPortMeta>> downstreamOpers = Sets.newHashSet();
    // figure out the downstream consumers
    for (InputPortMeta ipm : streamMeta.getSinks()) {
        // skipped for parallel partitions - those are handled elsewhere
        if (!ipm.getValue(PortContext.PARTITION_PARALLEL) && plan.hasMapping(ipm.getOperatorMeta())) {
            List<PTOperator> partitions = plan.getOperators(ipm.getOperatorMeta());
            for (PTOperator doper : partitions) {
                downstreamOpers.add(new Pair<>(doper, ipm));
            }
        }
    }
    if (!downstreamOpers.isEmpty()) {
        // unifiers are required
        for (PTOperator unifier : this.cascadingUnifiers) {
            detachUnifier(unifier);
        }
        if (this.finalUnifier != null) {
            detachUnifier(finalUnifier);
        }
        List<PTOperator> currentUnifiers = Lists.newArrayList(this.cascadingUnifiers);
        this.cascadingUnifiers.clear();
        plan.undeployOpers.addAll(currentUnifiers);
        addSlidingUnifiers();
        int limit = streamMeta.getSource().getValue(PortContext.UNIFIER_LIMIT);
        boolean separateUnifiers = false;
        Integer lastId = null;
        for (InputPortMeta ipm : streamMeta.getSinks()) {
            Integer id = plan.getStreamCodecIdentifier(ipm.getStreamCodec());
            if (lastId == null) {
                lastId = id;
            } else if (!id.equals(lastId)) {
                separateUnifiers = true;
                break;
            }
        }
        List<PTOutput> unifierSources = this.upstream;
        Map<StreamCodec<?>, List<PTOutput>> cascadeUnifierSourcesMap = Maps.newHashMap();
        if (limit > 1 && this.upstream.size() > limit) {
            // cascading unifier
            if (!separateUnifiers) {
                unifierSources = setupCascadingUnifiers(this.upstream, currentUnifiers, limit, 0);
            } else {
                for (InputPortMeta ipm : streamMeta.getSinks()) {
                    StreamCodec<?> streamCodec = ipm.getStreamCodec();
                    if (!cascadeUnifierSourcesMap.containsKey(streamCodec)) {
                        unifierSources = setupCascadingUnifiers(this.upstream, currentUnifiers, limit, 0);
                        cascadeUnifierSourcesMap.put(streamCodec, unifierSources);
                    }
                }
            }
        }
        // remove remaining unifiers
        for (PTOperator oper : currentUnifiers) {
            plan.removePTOperator(oper);
        }
        // Directly getting attribute from map to know if it is set or not as it can be overriden by the input
        Boolean sourceSingleFinal = streamMeta.getSource().getAttributes().get(PortContext.UNIFIER_SINGLE_FINAL);
        // link the downstream operators with the unifiers
        for (Pair<PTOperator, InputPortMeta> doperEntry : downstreamOpers) {
            Map<LogicalPlan.InputPortMeta, PartitionKeys> partKeys = doperEntry.first.partitionKeys;
            PartitionKeys pks = partKeys != null ? partKeys.get(doperEntry.second) : null;
            Boolean sinkSingleFinal = doperEntry.second.getAttributes().get(PortContext.UNIFIER_SINGLE_FINAL);
            boolean lastSingle = (sinkSingleFinal != null) ? sinkSingleFinal : (sourceSingleFinal != null ? sourceSingleFinal.booleanValue() : PortContext.UNIFIER_SINGLE_FINAL.defaultValue);
            if (upstream.size() > 1) {
                // detach downstream from upstream operator for the case where no unifier existed previously
                for (PTOutput source : upstream) {
                    Iterator<PTInput> sinks = source.sinks.iterator();
                    while (sinks.hasNext()) {
                        PTInput sink = sinks.next();
                        if (sink.target == doperEntry.first) {
                            doperEntry.first.inputs.remove(sink);
                            sinks.remove();
                        }
                    }
                }
                if (!separateUnifiers && lastSingle) {
                    if (finalUnifier == null) {
                        finalUnifier = createUnifier(streamMeta, plan);
                    }
                    setInput(doperEntry.first, doperEntry.second, finalUnifier, (pks == null) || (pks.mask == 0) ? null : pks);
                    if (finalUnifier.inputs.isEmpty()) {
                        // set unifier inputs once, regardless how many downstream operators there are
                        for (PTOutput out : unifierSources) {
                            addInput(this.finalUnifier, out, null);
                        }
                    }
                } else {
                    // MxN partitioning: unifier per downstream partition
                    LOG.debug("MxN unifier for {} {} {}", new Object[] { doperEntry.first, doperEntry.second.getPortName(), pks });
                    PTOperator unifier = doperEntry.first.upstreamMerge.get(doperEntry.second);
                    if (unifier == null) {
                        unifier = createUnifier(streamMeta, plan);
                        doperEntry.first.upstreamMerge.put(doperEntry.second, unifier);
                        setInput(doperEntry.first, doperEntry.second, unifier, null);
                    }
                    // sources may change dynamically, rebuild inputs (as for cascading unifiers)
                    for (PTInput in : unifier.inputs) {
                        in.source.sinks.remove(in);
                    }
                    unifier.inputs.clear();
                    List<PTOutput> doperUnifierSources = unifierSources;
                    if (separateUnifiers) {
                        List<PTOutput> cascadeSources = cascadeUnifierSourcesMap.get(doperEntry.second.getStreamCodec());
                        if (cascadeSources != null) {
                            doperUnifierSources = cascadeSources;
                        }
                    }
                    // add new inputs
                    for (PTOutput out : doperUnifierSources) {
                        addInput(unifier, out, (pks == null) || (pks.mask == 0) ? null : pks);
                    }
                }
            } else {
                // no partitioning
                PTOperator unifier = doperEntry.first.upstreamMerge.remove(doperEntry.second);
                if (unifier != null) {
                    plan.removePTOperator(unifier);
                }
                setInput(doperEntry.first, doperEntry.second, upstream.get(0).source, pks);
            }
        }
        // 2) Downstream operators partitions are scaled up from one to multiple. (replaced by merged unifier)
        if (finalUnifier != null && finalUnifier.inputs.isEmpty()) {
            plan.removePTOperator(finalUnifier);
            finalUnifier = null;
        }
    }
}

Example 7 with PTInput

use of com.datatorrent.stram.plan.physical.PTOperator.PTInput in project apex-core by apache.

the class PhysicalPlan method getActivationCheckpoint.

/**
   * Initialize the activation checkpoint for the given operator.
   * Recursively traverses inputs until existing checkpoint or root operator is found.
   * NoOp when already initialized.
   * @param oper
   */
private Checkpoint getActivationCheckpoint(PTOperator oper) {
    if (oper.recoveryCheckpoint == null && oper.checkpoints.isEmpty()) {
        Checkpoint activationCheckpoint = Checkpoint.INITIAL_CHECKPOINT;
        for (PTInput input : oper.inputs) {
            PTOperator sourceOper = input.source.source;
            Checkpoint checkpoint = sourceOper.recoveryCheckpoint;
            if (sourceOper.checkpoints.isEmpty()) {
                checkpoint = getActivationCheckpoint(sourceOper);
            }
            activationCheckpoint = Checkpoint.max(activationCheckpoint, checkpoint);
        }
        return activationCheckpoint;
    }
    return oper.recoveryCheckpoint;
}

Example 8 with PTInput

use of com.datatorrent.stram.plan.physical.PTOperator.PTInput in project apex-core by apache.

the class PhysicalPlan method removePTOperator.

void removePTOperator(PTOperator oper) {
    LOG.debug("Removing operator {}", oper);
    // per partition merge operators
    if (!oper.upstreamMerge.isEmpty()) {
        for (PTOperator unifier : oper.upstreamMerge.values()) {
            removePTOperator(unifier);
        }
    }
    // remove inputs from downstream operators
    for (PTOutput out : oper.outputs) {
        for (PTInput sinkIn : out.sinks) {
            if (sinkIn.source.source == oper) {
                ArrayList<PTInput> cowInputs = Lists.newArrayList(sinkIn.target.inputs);
                cowInputs.remove(sinkIn);
                sinkIn.target.inputs = cowInputs;
            }
        }
    }
    // remove from upstream operators
    for (PTInput in : oper.inputs) {
        in.source.sinks.remove(in);
    }
    for (HostOperatorSet s : oper.groupings.values()) {
        s.getOperatorSet().remove(oper);
    }
    // remove checkpoint states
    try {
        synchronized (oper.checkpoints) {
            for (Checkpoint checkpoint : oper.checkpoints) {
                oper.operatorMeta.getValue(OperatorContext.STORAGE_AGENT).delete(oper.id, checkpoint.windowId);
            }
        }
    } catch (IOException e) {
        LOG.warn("Failed to remove state for " + oper, e);
    }
    List<PTOperator> cowList = Lists.newArrayList(oper.container.operators);
    cowList.remove(oper);
    oper.container.operators = cowList;
    this.deployOpers.remove(oper);
    this.undeployOpers.add(oper);
    this.allOperators.remove(oper.id);
    this.ctx.recordEventAsync(new StramEvent.RemoveOperatorEvent(oper.getName(), oper.getId()));
}

Example 9 with PTInput

use of com.datatorrent.stram.plan.physical.PTOperator.PTInput in project apex-core by apache.

the class PhysicalPlan method removePartition.

/**
   * Remove the given partition with any associated parallel partitions and
   * per-partition outputStreams.
   *
   * @param oper
   * @return
   */
private void removePartition(PTOperator oper, PMapping operatorMapping) {
    // remove any parallel partition
    for (PTOutput out : oper.outputs) {
        // copy list as it is modified by recursive remove
        for (PTInput in : Lists.newArrayList(out.sinks)) {
            for (LogicalPlan.InputPortMeta im : in.logicalStream.getSinks()) {
                PMapping m = this.logicalToPTOperator.get(im.getOperatorMeta());
                if (m.parallelPartitions == operatorMapping.parallelPartitions) {
                    // associated operator parallel partitioned
                    removePartition(in.target, operatorMapping);
                    m.partitions.remove(in.target);
                }
            }
        }
    }
    // remove the operator
    removePTOperator(oper);
}

Example 10 with PTInput

use of com.datatorrent.stram.plan.physical.PTOperator.PTInput in project apex-core by apache.

the class PhysicalPlan method removeLogicalStream.

/**
   * Remove physical representation of given stream. Operators that are affected
   * in the execution layer will be added to the set. This method does not
   * automatically remove operators from the plan.
   *
   * @param sm
   */
public void removeLogicalStream(StreamMeta sm) {
    // remove incoming connections for logical stream
    for (InputPortMeta ipm : sm.getSinks()) {
        OperatorMeta om = ipm.getOperatorMeta();
        PMapping m = this.logicalToPTOperator.get(om);
        if (m == null) {
            throw new AssertionError("Unknown operator " + om);
        }
        for (PTOperator oper : m.partitions) {
            List<PTInput> inputsCopy = Lists.newArrayList(oper.inputs);
            for (PTInput input : oper.inputs) {
                if (input.logicalStream == sm) {
                    input.source.sinks.remove(input);
                    inputsCopy.remove(input);
                    undeployOpers.add(oper);
                    deployOpers.add(oper);
                }
            }
            oper.inputs = inputsCopy;
        }
    }
    // remove outgoing connections for logical stream
    PMapping m = this.logicalToPTOperator.get(sm.getSource().getOperatorMeta());
    for (PTOperator oper : m.partitions) {
        List<PTOutput> outputsCopy = Lists.newArrayList(oper.outputs);
        for (PTOutput out : oper.outputs) {
            if (out.logicalStream == sm) {
                for (PTInput input : out.sinks) {
                    PTOperator downstreamOper = input.source.source;
                    downstreamOper.inputs.remove(input);
                    Set<PTOperator> deps = this.getDependents(Collections.singletonList(downstreamOper));
                    undeployOpers.addAll(deps);
                    deployOpers.addAll(deps);
                }
                outputsCopy.remove(out);
                undeployOpers.add(oper);
                deployOpers.add(oper);
            }
        }
        oper.outputs = outputsCopy;
    }
}