Examples with PTOutput com.datatorrent.stram.plan.physical.PTOperator.PTOutput used on opensource projects

Example 11 with PTOutput

use of com.datatorrent.stram.plan.physical.PTOperator.PTOutput in project apex-core by apache.

the class PhysicalPlan method removeTerminatedPartition.

/**
   * Remove a partition that was reported as terminated by the execution layer.
   * Recursively removes all downstream operators with no remaining input.
   * @param p
   */
public void removeTerminatedPartition(PTOperator p) {
    // keep track of downstream operators for cascading remove
    Set<PTOperator> downstreamOpers = new HashSet<>(p.outputs.size());
    for (PTOutput out : p.outputs) {
        for (PTInput sinkIn : out.sinks) {
            downstreamOpers.add(sinkIn.target);
        }
    }
    PMapping currentMapping = this.logicalToPTOperator.get(p.operatorMeta);
    if (currentMapping != null) {
        List<PTOperator> copyPartitions = Lists.newArrayList(currentMapping.partitions);
        copyPartitions.remove(p);
        removePartition(p, currentMapping);
        currentMapping.partitions = copyPartitions;
    } else {
        // remove the operator
        removePTOperator(p);
    }
    // remove orphaned downstream operators
    for (PTOperator dop : downstreamOpers) {
        if (dop.inputs.isEmpty()) {
            removeTerminatedPartition(dop);
        }
    }
    deployChanges();
}

Example 12 with PTOutput

use of com.datatorrent.stram.plan.physical.PTOperator.PTOutput in project apex-core by apache.

the class StreamMapping method addSlidingUnifiers.

private void addSlidingUnifiers() {
    OperatorMeta sourceOM = streamMeta.getSource().getOperatorMeta();
    if (sourceOM.getAttributes().contains(Context.OperatorContext.SLIDE_BY_WINDOW_COUNT)) {
        if (sourceOM.getValue(Context.OperatorContext.SLIDE_BY_WINDOW_COUNT) < sourceOM.getValue(Context.OperatorContext.APPLICATION_WINDOW_COUNT)) {
            plan.undeployOpers.addAll(slidingUnifiers);
            slidingUnifiers.clear();
            List<PTOutput> newUpstream = Lists.newArrayList();
            PTOperator slidingUnifier;
            for (PTOutput source : upstream) {
                slidingUnifier = StreamMapping.createSlidingUnifier(streamMeta, plan, sourceOM.getValue(Context.OperatorContext.APPLICATION_WINDOW_COUNT), sourceOM.getValue(Context.OperatorContext.SLIDE_BY_WINDOW_COUNT));
                addInput(slidingUnifier, source, null);
                this.slidingUnifiers.add(slidingUnifier);
                newUpstream.add(slidingUnifier.outputs.get(0));
            }
            upstream.clear();
            upstream.addAll(newUpstream);
        } else {
            LOG.warn("Sliding Window Count {} should be less than APPLICATION WINDOW COUNT {}", sourceOM.getValue(Context.OperatorContext.SLIDE_BY_WINDOW_COUNT), sourceOM.getValue(Context.OperatorContext.APPLICATION_WINDOW_COUNT));
        }
    }
}

Example 13 with PTOutput

use of com.datatorrent.stram.plan.physical.PTOperator.PTOutput in project apex-core by apache.

the class StreamMapping method createUnifier.

public static PTOperator createUnifier(StreamMeta streamMeta, PhysicalPlan plan) {
    OperatorMeta um = streamMeta.getSource().getUnifierMeta();
    PTOperator pu = plan.newOperator(um, um.getName());
    Operator unifier = um.getOperator();
    PortMappingDescriptor mergeDesc = new PortMappingDescriptor();
    Operators.describe(unifier, mergeDesc);
    if (mergeDesc.outputPorts.size() != 1) {
        throw new AssertionError("Unifier must have a single output port, instead found : " + mergeDesc.outputPorts);
    }
    pu.unifiedOperatorMeta = streamMeta.getSource().getOperatorMeta();
    pu.outputs.add(new PTOutput(mergeDesc.outputPorts.keySet().iterator().next(), streamMeta, pu));
    plan.newOpers.put(pu, unifier);
    return pu;
}

Example 14 with PTOutput

use of com.datatorrent.stram.plan.physical.PTOperator.PTOutput in project apex-core by apache.

the class StreamMapping method setupCascadingUnifiers.

@SuppressWarnings("AssignmentToForLoopParameter")
private List<PTOutput> setupCascadingUnifiers(List<PTOutput> upstream, List<PTOperator> pooledUnifiers, int limit, int level) {
    List<PTOutput> nextLevel = Lists.newArrayList();
    PTOperator pu = null;
    for (int i = 0; i < upstream.size(); i++) {
        if (i % limit == 0) {
            if (upstream.size() - i < limit) {
                while (i < upstream.size()) {
                    nextLevel.add(upstream.get(i));
                    i++;
                }
                continue;
            }
            if (!pooledUnifiers.isEmpty()) {
                pu = pooledUnifiers.remove(0);
            } else {
                pu = createUnifier(streamMeta, plan);
            }
            assert (pu.outputs.size() == 1) : "unifier has single output";
            nextLevel.addAll(pu.outputs);
            this.cascadingUnifiers.add(pu);
        }
        PTOutput source = upstream.get(i);
        addInput(pu, source, null);
    }
    if (nextLevel.size() > limit) {
        return setupCascadingUnifiers(nextLevel, pooledUnifiers, limit, level);
    } else {
        return nextLevel;
    }
}

Example 15 with PTOutput

use of com.datatorrent.stram.plan.physical.PTOperator.PTOutput in project apex-core by apache.

the class StreamMapping method setInput.

private void setInput(PTOperator oper, InputPortMeta ipm, PTOperator sourceOper, PartitionKeys pks) {
    // TODO: see if this can be handled more efficiently
    for (PTInput in : oper.inputs) {
        if (in.source.source == sourceOper && in.logicalStream == streamMeta && ipm.getPortName().equals(in.portName)) {
            return;
        }
    }
    // link to upstream output(s) for this stream
    for (PTOutput upstreamOut : sourceOper.outputs) {
        if (upstreamOut.logicalStream == streamMeta) {
            PTInput input = new PTInput(ipm.getPortName(), streamMeta, oper, pks, upstreamOut, ipm.getValue(LogicalPlan.IS_CONNECTED_TO_DELAY_OPERATOR));
            oper.inputs.add(input);
        }
    }
}