Examples with PTOutput com.datatorrent.stram.plan.physical.PTOperator.PTOutput used on opensource projects

Example 16 with PTOutput

use of com.datatorrent.stram.plan.physical.PTOperator.PTOutput in project apex-core by apache.

the class StreamMapping method addSlidingUnifiers.

private void addSlidingUnifiers() {
    OperatorMeta sourceOM = streamMeta.getSource().getOperatorMeta();
    if (sourceOM.getAttributes().contains(Context.OperatorContext.SLIDE_BY_WINDOW_COUNT)) {
        if (sourceOM.getValue(Context.OperatorContext.SLIDE_BY_WINDOW_COUNT) < sourceOM.getValue(Context.OperatorContext.APPLICATION_WINDOW_COUNT)) {
            plan.undeployOpers.addAll(slidingUnifiers);
            slidingUnifiers.clear();
            List<PTOutput> newUpstream = Lists.newArrayList();
            PTOperator slidingUnifier;
            for (PTOutput source : upstream) {
                slidingUnifier = StreamMapping.createSlidingUnifier(streamMeta, plan, sourceOM.getValue(Context.OperatorContext.APPLICATION_WINDOW_COUNT), sourceOM.getValue(Context.OperatorContext.SLIDE_BY_WINDOW_COUNT));
                addInput(slidingUnifier, source, null);
                this.slidingUnifiers.add(slidingUnifier);
                newUpstream.add(slidingUnifier.outputs.get(0));
            }
            upstream.clear();
            upstream.addAll(newUpstream);
        } else {
            LOG.warn("Sliding Window Count {} should be less than APPLICATION WINDOW COUNT {}", sourceOM.getValue(Context.OperatorContext.SLIDE_BY_WINDOW_COUNT), sourceOM.getValue(Context.OperatorContext.APPLICATION_WINDOW_COUNT));
        }
    }
}

Example 17 with PTOutput

use of com.datatorrent.stram.plan.physical.PTOperator.PTOutput in project apex-core by apache.

the class StreamMapping method createUnifier.

public static PTOperator createUnifier(StreamMeta streamMeta, PhysicalPlan plan) {
    OperatorMeta um = streamMeta.getSource().getUnifierMeta();
    PTOperator pu = plan.newOperator(um, um.getName());
    Operator unifier = um.getOperator();
    PortMappingDescriptor mergeDesc = new PortMappingDescriptor();
    Operators.describe(unifier, mergeDesc);
    if (mergeDesc.outputPorts.size() != 1) {
        throw new AssertionError("Unifier must have a single output port, instead found : " + mergeDesc.outputPorts);
    }
    pu.unifiedOperatorMeta = streamMeta.getSource().getOperatorMeta();
    pu.outputs.add(new PTOutput(mergeDesc.outputPorts.keySet().iterator().next(), streamMeta, pu));
    plan.newOpers.put(pu, unifier);
    return pu;
}

Example 18 with PTOutput

use of com.datatorrent.stram.plan.physical.PTOperator.PTOutput in project apex-core by apache.

the class StreamMapping method setupCascadingUnifiers.

@SuppressWarnings("AssignmentToForLoopParameter")
private List<PTOutput> setupCascadingUnifiers(List<PTOutput> upstream, List<PTOperator> pooledUnifiers, int limit, int level) {
    List<PTOutput> nextLevel = Lists.newArrayList();
    PTOperator pu = null;
    for (int i = 0; i < upstream.size(); i++) {
        if (i % limit == 0) {
            if (upstream.size() - i < limit) {
                while (i < upstream.size()) {
                    nextLevel.add(upstream.get(i));
                    i++;
                }
                continue;
            }
            if (!pooledUnifiers.isEmpty()) {
                pu = pooledUnifiers.remove(0);
            } else {
                pu = createUnifier(streamMeta, plan);
            }
            assert (pu.outputs.size() == 1) : "unifier has single output";
            nextLevel.addAll(pu.outputs);
            this.cascadingUnifiers.add(pu);
        }
        PTOutput source = upstream.get(i);
        addInput(pu, source, null);
    }
    if (nextLevel.size() > limit) {
        return setupCascadingUnifiers(nextLevel, pooledUnifiers, limit, level);
    } else {
        return nextLevel;
    }
}

Example 19 with PTOutput

use of com.datatorrent.stram.plan.physical.PTOperator.PTOutput in project apex-core by apache.

the class StreamMapping method setInput.

private void setInput(PTOperator oper, InputPortMeta ipm, PTOperator sourceOper, PartitionKeys pks) {
    // TODO: see if this can be handled more efficiently
    for (PTInput in : oper.inputs) {
        if (in.source.source == sourceOper && in.logicalStream == streamMeta && ipm.getPortName().equals(in.portName)) {
            return;
        }
    }
    // link to upstream output(s) for this stream
    for (PTOutput upstreamOut : sourceOper.outputs) {
        if (upstreamOut.logicalStream == streamMeta) {
            PTInput input = new PTInput(ipm.getPortName(), streamMeta, oper, pks, upstreamOut, ipm.getValue(LogicalPlan.IS_CONNECTED_TO_DELAY_OPERATOR));
            oper.inputs.add(input);
        }
    }
}

Example 20 with PTOutput

use of com.datatorrent.stram.plan.physical.PTOperator.PTOutput in project apex-core by apache.

the class PhysicalPlan method setupOutput.

/**
   * Create output port mapping for given operator and port.
   * Occurs when adding new partition or new logical stream.
   * Does nothing if source was already setup (on add sink to existing stream).
   * @param mapping
   * @param oper
   * @param outputEntry
   */
private void setupOutput(PMapping mapping, PTOperator oper, Map.Entry<LogicalPlan.OutputPortMeta, StreamMeta> outputEntry) {
    for (PTOutput out : oper.outputs) {
        if (out.logicalStream == outputEntry.getValue()) {
            // already processed
            return;
        }
    }
    PTOutput out = new PTOutput(outputEntry.getKey().getPortName(), outputEntry.getValue(), oper);
    oper.outputs.add(out);
}