Examples with PTInput com.datatorrent.stram.plan.physical.PTOperator.PTInput used on opensource projects

Example 11 with PTInput

use of com.datatorrent.stram.plan.physical.PTOperator.PTInput in project apex-core by apache.

the class PhysicalPlan method removeTerminatedPartition.

/**
   * Remove a partition that was reported as terminated by the execution layer.
   * Recursively removes all downstream operators with no remaining input.
   * @param p
   */
public void removeTerminatedPartition(PTOperator p) {
    // keep track of downstream operators for cascading remove
    Set<PTOperator> downstreamOpers = new HashSet<>(p.outputs.size());
    for (PTOutput out : p.outputs) {
        for (PTInput sinkIn : out.sinks) {
            downstreamOpers.add(sinkIn.target);
        }
    }
    PMapping currentMapping = this.logicalToPTOperator.get(p.operatorMeta);
    if (currentMapping != null) {
        List<PTOperator> copyPartitions = Lists.newArrayList(currentMapping.partitions);
        copyPartitions.remove(p);
        removePartition(p, currentMapping);
        currentMapping.partitions = copyPartitions;
    } else {
        // remove the operator
        removePTOperator(p);
    }
    // remove orphaned downstream operators
    for (PTOperator dop : downstreamOpers) {
        if (dop.inputs.isEmpty()) {
            removeTerminatedPartition(dop);
        }
    }
    deployChanges();
}

Example 12 with PTInput

use of com.datatorrent.stram.plan.physical.PTOperator.PTInput in project apex-core by apache.

the class StreamMapping method addInput.

public static void addInput(PTOperator target, PTOutput upstreamOut, PartitionKeys pks) {
    StreamMeta lStreamMeta = upstreamOut.logicalStream;
    PTInput input = new PTInput("<merge#" + lStreamMeta.getSource().getPortName() + ">", lStreamMeta, target, pks, upstreamOut, false);
    target.inputs.add(input);
}

Example 13 with PTInput

use of com.datatorrent.stram.plan.physical.PTOperator.PTInput in project apex-core by apache.

the class StreamMapping method setInput.

private void setInput(PTOperator oper, InputPortMeta ipm, PTOperator sourceOper, PartitionKeys pks) {
    // TODO: see if this can be handled more efficiently
    for (PTInput in : oper.inputs) {
        if (in.source.source == sourceOper && in.logicalStream == streamMeta && ipm.getPortName().equals(in.portName)) {
            return;
        }
    }
    // link to upstream output(s) for this stream
    for (PTOutput upstreamOut : sourceOper.outputs) {
        if (upstreamOut.logicalStream == streamMeta) {
            PTInput input = new PTInput(ipm.getPortName(), streamMeta, oper, pks, upstreamOut, ipm.getValue(LogicalPlan.IS_CONNECTED_TO_DELAY_OPERATOR));
            oper.inputs.add(input);
        }
    }
}

Example 14 with PTInput

use of com.datatorrent.stram.plan.physical.PTOperator.PTInput in project apex-core by apache.

the class StreamingContainerManager method updateOperatorLatency.

public long updateOperatorLatency(PTOperator oper, UpdateOperatorLatencyContext ctx) {
    if (!oper.getInputs().isEmpty() && oper.stats.currentWindowId.get() > 0) {
        OperatorStatus status = oper.stats;
        long latency = Long.MAX_VALUE;
        PTOperator slowestUpstream = null;
        int windowWidthMillis = plan.getLogicalPlan().getValue(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS);
        int heartbeatTimeoutMillis = plan.getLogicalPlan().getValue(LogicalPlan.HEARTBEAT_TIMEOUT_MILLIS);
        long currentWindowId = status.currentWindowId.get();
        if (!ctx.endWindowStatsExists(currentWindowId)) {
            // the end window stats for the current window id is not available, estimate latency by looking at upstream window id
            for (PTInput input : oper.getInputs()) {
                PTOperator upstreamOp = input.source.source;
                if (upstreamOp.getOperatorMeta().getOperator() instanceof Operator.DelayOperator) {
                    continue;
                }
                if (upstreamOp.stats.currentWindowId.get() >= oper.stats.currentWindowId.get()) {
                    long portLatency = WindowGenerator.compareWindowId(upstreamOp.stats.currentWindowId.get(), oper.stats.currentWindowId.get(), windowWidthMillis) * windowWidthMillis;
                    if (latency > portLatency) {
                        latency = portLatency;
                        slowestUpstream = upstreamOp;
                    }
                }
            }
        } else {
            long endWindowEmitTime = ctx.getEndWindowEmitTimestamp(currentWindowId, oper);
            long adjustedEndWindowEmitTimestamp = endWindowEmitTime + ctx.getRPCLatency(oper);
            for (PTInput input : oper.getInputs()) {
                PTOperator upstreamOp = input.source.source;
                if (upstreamOp.getOperatorMeta().getOperator() instanceof Operator.DelayOperator) {
                    continue;
                }
                long upstreamEndWindowEmitTime = ctx.getEndWindowEmitTimestamp(currentWindowId, upstreamOp);
                if (upstreamEndWindowEmitTime < 0) {
                    continue;
                }
                long portLatency = adjustedEndWindowEmitTimestamp - (upstreamEndWindowEmitTime + ctx.getRPCLatency(upstreamOp));
                if (portLatency < 0) {
                    portLatency = 0;
                }
                long latencyFromWindowsBehind = WindowGenerator.compareWindowId(upstreamOp.stats.currentWindowId.get(), oper.stats.currentWindowId.get(), windowWidthMillis) * windowWidthMillis;
                if (latencyFromWindowsBehind > portLatency && latencyFromWindowsBehind > heartbeatTimeoutMillis) {
                    portLatency = latencyFromWindowsBehind;
                }
                if (latency > portLatency) {
                    latency = portLatency;
                    slowestUpstream = upstreamOp;
                }
            }
        }
        if (slowestUpstream != null) {
            status.latencyMA.add(latency);
            slowestUpstreamOp.put(oper, slowestUpstream);
            return latency;
        }
    }
    return -1;
}

Example 15 with PTInput

use of com.datatorrent.stram.plan.physical.PTOperator.PTInput in project apex-core by apache.

the class StreamingContainerManager method calculateLatency.

private long calculateLatency(PTOperator operator) {
    long latency = operator.stats.latencyMA.getAvg();
    long maxUnifierLatency = 0;
    for (PTOutput output : operator.getOutputs()) {
        for (PTInput input : output.sinks) {
            if (input.target.isUnifier()) {
                long thisUnifierLatency = calculateLatency(input.target);
                if (maxUnifierLatency < thisUnifierLatency) {
                    maxUnifierLatency = thisUnifierLatency;
                }
            }
        }
    }
    return latency + maxUnifierLatency;
}