Examples with OperatorStatus com.datatorrent.stram.plan.physical.OperatorStatus used on opensource projects

Example 1 with OperatorStatus

use of com.datatorrent.stram.plan.physical.OperatorStatus in project apex-core by apache.

the class StreamingContainerManager method fillOperatorAggregationInfo.

private OperatorAggregationInfo fillOperatorAggregationInfo(OperatorMeta operator) {
    OperatorAggregationInfo oai = new OperatorAggregationInfo();
    Collection<PTOperator> physicalOperators = getPhysicalPlan().getAllOperators(operator);
    if (physicalOperators.isEmpty()) {
        return null;
    }
    oai.name = operator.getName();
    for (PTOperator physicalOperator : physicalOperators) {
        if (!physicalOperator.isUnifier()) {
            OperatorStatus os = physicalOperator.stats;
            oai.latencyMA.addNumber(os.latencyMA.getAvg());
            oai.cpuPercentageMA.addNumber(os.cpuNanosPMSMA.getAvg() / 10000);
            oai.tuplesEmittedPSMA.addNumber(os.tuplesEmittedPSMA.get());
            oai.tuplesProcessedPSMA.addNumber(os.tuplesProcessedPSMA.get());
            oai.currentWindowId.addNumber(os.currentWindowId.get());
            oai.recoveryWindowId.addNumber(toWsWindowId(physicalOperator.getRecoveryCheckpoint().windowId));
            if (os.lastHeartbeat != null) {
                oai.lastHeartbeat.addNumber(os.lastHeartbeat.getGeneratedTms());
            }
            oai.checkpointTime.addNumber(os.checkpointTimeMA.getAvg());
        }
    }
    return oai;
}

Example 2 with OperatorStatus

use of com.datatorrent.stram.plan.physical.OperatorStatus in project apex-core by apache.

the class StreamingContainerManager method fillPhysicalOperatorInfo.

private OperatorInfo fillPhysicalOperatorInfo(PTOperator operator) {
    OperatorInfo oi = new OperatorInfo();
    oi.container = operator.getContainer().getExternalId();
    oi.host = operator.getContainer().host;
    oi.id = Integer.toString(operator.getId());
    oi.name = operator.getName();
    oi.className = operator.getOperatorMeta().getOperator().getClass().getName();
    oi.status = operator.getState().toString();
    if (operator.isUnifier()) {
        oi.unifierClass = operator.getUnifierClass().getName();
    }
    oi.logicalName = operator.getOperatorMeta().getName();
    OperatorStatus os = operator.stats;
    oi.recordingId = os.recordingId;
    oi.totalTuplesProcessed = os.totalTuplesProcessed.get();
    oi.totalTuplesEmitted = os.totalTuplesEmitted.get();
    oi.tuplesProcessedPSMA = os.tuplesProcessedPSMA.get();
    oi.tuplesEmittedPSMA = os.tuplesEmittedPSMA.get();
    oi.cpuPercentageMA = os.cpuNanosPMSMA.getAvg() / 10000;
    oi.latencyMA = os.latencyMA.getAvg();
    oi.failureCount = operator.failureCount;
    oi.recoveryWindowId = toWsWindowId(operator.getRecoveryCheckpoint().windowId);
    oi.currentWindowId = toWsWindowId(os.currentWindowId.get());
    if (os.lastHeartbeat != null) {
        oi.lastHeartbeat = os.lastHeartbeat.getGeneratedTms();
    }
    if (os.checkpointStats != null) {
        oi.checkpointTime = os.checkpointStats.checkpointTime;
        oi.checkpointStartTime = os.checkpointStats.checkpointStartTime;
    }
    oi.checkpointTimeMA = os.checkpointTimeMA.getAvg();
    for (PortStatus ps : os.inputPortStatusList.values()) {
        PortInfo pinfo = new PortInfo();
        pinfo.name = ps.portName;
        pinfo.type = "input";
        pinfo.totalTuples = ps.totalTuples;
        pinfo.tuplesPSMA = Math.round(ps.tuplesPMSMA.getAvg() * 1000);
        pinfo.bufferServerBytesPSMA = Math.round(ps.bufferServerBytesPMSMA.getAvg() * 1000);
        pinfo.queueSizeMA = ps.queueSizeMA.getAvg();
        pinfo.recordingId = ps.recordingId;
        oi.addPort(pinfo);
    }
    for (PortStatus ps : os.outputPortStatusList.values()) {
        PortInfo pinfo = new PortInfo();
        pinfo.name = ps.portName;
        pinfo.type = "output";
        pinfo.totalTuples = ps.totalTuples;
        pinfo.tuplesPSMA = Math.round(ps.tuplesPMSMA.getAvg() * 1000);
        pinfo.bufferServerBytesPSMA = Math.round(ps.bufferServerBytesPMSMA.getAvg() * 1000);
        pinfo.recordingId = ps.recordingId;
        oi.addPort(pinfo);
    }
    oi.counters = os.getLastWindowedStats().size() > 0 ? os.getLastWindowedStats().get(os.getLastWindowedStats().size() - 1).counters : null;
    oi.metrics = os.getLastWindowedStats().size() > 0 ? os.getLastWindowedStats().get(os.getLastWindowedStats().size() - 1).metrics : null;
    return oi;
}

Example 3 with OperatorStatus

use of com.datatorrent.stram.plan.physical.OperatorStatus in project apex-core by apache.

the class StreamingContainerManager method updateOperatorLatency.

public long updateOperatorLatency(PTOperator oper, UpdateOperatorLatencyContext ctx) {
    if (!oper.getInputs().isEmpty() && oper.stats.currentWindowId.get() > 0) {
        OperatorStatus status = oper.stats;
        long latency = Long.MAX_VALUE;
        PTOperator slowestUpstream = null;
        int windowWidthMillis = plan.getLogicalPlan().getValue(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS);
        int heartbeatTimeoutMillis = plan.getLogicalPlan().getValue(LogicalPlan.HEARTBEAT_TIMEOUT_MILLIS);
        long currentWindowId = status.currentWindowId.get();
        if (!ctx.endWindowStatsExists(currentWindowId)) {
            // the end window stats for the current window id is not available, estimate latency by looking at upstream window id
            for (PTInput input : oper.getInputs()) {
                PTOperator upstreamOp = input.source.source;
                if (upstreamOp.getOperatorMeta().getOperator() instanceof Operator.DelayOperator) {
                    continue;
                }
                if (upstreamOp.stats.currentWindowId.get() >= oper.stats.currentWindowId.get()) {
                    long portLatency = WindowGenerator.compareWindowId(upstreamOp.stats.currentWindowId.get(), oper.stats.currentWindowId.get(), windowWidthMillis) * windowWidthMillis;
                    if (latency > portLatency) {
                        latency = portLatency;
                        slowestUpstream = upstreamOp;
                    }
                }
            }
        } else {
            long endWindowEmitTime = ctx.getEndWindowEmitTimestamp(currentWindowId, oper);
            long adjustedEndWindowEmitTimestamp = endWindowEmitTime + ctx.getRPCLatency(oper);
            for (PTInput input : oper.getInputs()) {
                PTOperator upstreamOp = input.source.source;
                if (upstreamOp.getOperatorMeta().getOperator() instanceof Operator.DelayOperator) {
                    continue;
                }
                long upstreamEndWindowEmitTime = ctx.getEndWindowEmitTimestamp(currentWindowId, upstreamOp);
                if (upstreamEndWindowEmitTime < 0) {
                    continue;
                }
                long portLatency = adjustedEndWindowEmitTimestamp - (upstreamEndWindowEmitTime + ctx.getRPCLatency(upstreamOp));
                if (portLatency < 0) {
                    portLatency = 0;
                }
                long latencyFromWindowsBehind = WindowGenerator.compareWindowId(upstreamOp.stats.currentWindowId.get(), oper.stats.currentWindowId.get(), windowWidthMillis) * windowWidthMillis;
                if (latencyFromWindowsBehind > portLatency && latencyFromWindowsBehind > heartbeatTimeoutMillis) {
                    portLatency = latencyFromWindowsBehind;
                }
                if (latency > portLatency) {
                    latency = portLatency;
                    slowestUpstream = upstreamOp;
                }
            }
        }
        if (slowestUpstream != null) {
            status.latencyMA.add(latency);
            slowestUpstreamOp.put(oper, slowestUpstream);
            return latency;
        }
    }
    return -1;
}

Example 4 with OperatorStatus

use of com.datatorrent.stram.plan.physical.OperatorStatus in project apex-core by apache.

the class StreamingContainerManager method fillLogicalOperatorInfo.

private LogicalOperatorInfo fillLogicalOperatorInfo(OperatorMeta operator) {
    LogicalOperatorInfo loi = new LogicalOperatorInfo();
    loi.name = operator.getName();
    loi.className = operator.getOperator().getClass().getName();
    loi.totalTuplesEmitted = operator.getStatus().totalTuplesEmitted;
    loi.totalTuplesProcessed = operator.getStatus().totalTuplesProcessed;
    loi.failureCount = operator.getStatus().failureCount;
    loi.status = new HashMap<>();
    loi.partitions = new TreeSet<>();
    loi.unifiers = new TreeSet<>();
    loi.containerIds = new TreeSet<>();
    loi.hosts = new TreeSet<>();
    Collection<PTOperator> physicalOperators = getPhysicalPlan().getAllOperators(operator);
    NumberAggregate.LongAggregate checkpointTimeAggregate = new NumberAggregate.LongAggregate();
    for (PTOperator physicalOperator : physicalOperators) {
        OperatorStatus os = physicalOperator.stats;
        if (physicalOperator.isUnifier()) {
            loi.unifiers.add(physicalOperator.getId());
        } else {
            loi.partitions.add(physicalOperator.getId());
            // exclude unifier, not sure if we should include it in the future
            loi.tuplesEmittedPSMA += os.tuplesEmittedPSMA.get();
            loi.tuplesProcessedPSMA += os.tuplesProcessedPSMA.get();
            // calculate maximum latency for all partitions
            long latency = calculateLatency(physicalOperator);
            if (latency > loi.latencyMA) {
                loi.latencyMA = latency;
            }
            checkpointTimeAggregate.addNumber(os.checkpointTimeMA.getAvg());
        }
        loi.cpuPercentageMA += os.cpuNanosPMSMA.getAvg() / 10000;
        if (os.lastHeartbeat != null && (loi.lastHeartbeat == 0 || loi.lastHeartbeat > os.lastHeartbeat.getGeneratedTms())) {
            loi.lastHeartbeat = os.lastHeartbeat.getGeneratedTms();
        }
        long currentWindowId = toWsWindowId(os.currentWindowId.get());
        if (loi.currentWindowId == 0 || loi.currentWindowId > currentWindowId) {
            loi.currentWindowId = currentWindowId;
        }
        MutableInt count = loi.status.get(physicalOperator.getState().toString());
        if (count == null) {
            count = new MutableInt();
            loi.status.put(physicalOperator.getState().toString(), count);
        }
        count.increment();
        if (physicalOperator.getRecoveryCheckpoint() != null) {
            long recoveryWindowId = toWsWindowId(physicalOperator.getRecoveryCheckpoint().windowId);
            if (loi.recoveryWindowId == 0 || loi.recoveryWindowId > recoveryWindowId) {
                loi.recoveryWindowId = recoveryWindowId;
            }
        }
        PTContainer container = physicalOperator.getContainer();
        if (container != null) {
            String externalId = container.getExternalId();
            if (externalId != null) {
                loi.containerIds.add(externalId);
                loi.hosts.add(container.host);
            }
        }
    }
    if (physicalOperators.size() > 0 && checkpointTimeAggregate.getAvg() != null) {
        loi.checkpointTimeMA = checkpointTimeAggregate.getAvg().longValue();
        loi.counters = latestLogicalCounters.get(operator.getName());
        loi.autoMetrics = latestLogicalMetrics.get(operator.getName());
    }
    return loi;
}

Example 5 with OperatorStatus

use of com.datatorrent.stram.plan.physical.OperatorStatus in project apex-core by apache.

the class StreamingContainerManager method processHeartbeat.

/**
 * process the heartbeat from each container.
 * called by the RPC thread for each container. (i.e. called by multiple threads)
 *
 * @param heartbeat
 * @return heartbeat response
 */
@SuppressWarnings("StatementWithEmptyBody")
public ContainerHeartbeatResponse processHeartbeat(ContainerHeartbeat heartbeat) {
    long currentTimeMillis = clock.getTime();
    final StreamingContainerAgent sca = this.containers.get(heartbeat.getContainerId());
    if (sca == null || sca.container.getState() == PTContainer.State.KILLED) {
        // could be orphaned container that was replaced and needs to terminate
        LOG.error("Unknown container {}", heartbeat.getContainerId());
        ContainerHeartbeatResponse response = new ContainerHeartbeatResponse();
        response.shutdown = ShutdownType.ABORT;
        return response;
    }
    // LOG.debug("{} {} {}", new Object[]{sca.container.containerId, sca.container.bufferServerAddress, sca.container.getState()});
    if (sca.container.getState() == PTContainer.State.ALLOCATED) {
        // capture dynamically assigned address from container
        if (sca.container.bufferServerAddress == null && heartbeat.bufferServerHost != null) {
            sca.container.bufferServerAddress = InetSocketAddress.createUnresolved(heartbeat.bufferServerHost, heartbeat.bufferServerPort);
            LOG.info("Container {} buffer server: {}", sca.container.getExternalId(), sca.container.bufferServerAddress);
        }
        final long containerStartTime = System.currentTimeMillis();
        sca.container.setState(PTContainer.State.ACTIVE);
        sca.container.setStartedTime(containerStartTime);
        sca.container.setFinishedTime(-1);
        sca.jvmName = heartbeat.jvmName;
        poolExecutor.submit(new Runnable() {

            @Override
            public void run() {
                try {
                    containerFile.append(sca.getContainerInfo());
                } catch (IOException ex) {
                    LOG.warn("Cannot write to container file");
                }
                for (PTOperator ptOp : sca.container.getOperators()) {
                    try {
                        JSONObject operatorInfo = new JSONObject();
                        operatorInfo.put("name", ptOp.getName());
                        operatorInfo.put("id", ptOp.getId());
                        operatorInfo.put("container", sca.container.getExternalId());
                        operatorInfo.put("startTime", containerStartTime);
                        operatorFile.append(operatorInfo);
                    } catch (IOException | JSONException ex) {
                        LOG.warn("Cannot write to operator file: ", ex);
                    }
                }
            }
        });
    }
    sca.containerStackTrace = heartbeat.stackTrace;
    if (heartbeat.restartRequested) {
        LOG.error("Container {} restart request", sca.container.getExternalId());
        containerStopRequests.put(sca.container.getExternalId(), sca.container.getExternalId());
    }
    sca.memoryMBFree = heartbeat.memoryMBFree;
    sca.gcCollectionCount = heartbeat.gcCollectionCount;
    sca.gcCollectionTime = heartbeat.gcCollectionTime;
    sca.undeployOpers.clear();
    sca.deployOpers.clear();
    if (!this.deployChangeInProgress.get()) {
        sca.deployCnt = this.deployChangeCnt;
    }
    Set<Integer> reportedOperators = Sets.newHashSetWithExpectedSize(sca.container.getOperators().size());
    for (OperatorHeartbeat shb : heartbeat.getContainerStats().operators) {
        long maxEndWindowTimestamp = 0;
        reportedOperators.add(shb.nodeId);
        PTOperator oper = this.plan.getAllOperators().get(shb.getNodeId());
        if (oper == null) {
            LOG.info("Heartbeat for unknown operator {} (container {})", shb.getNodeId(), heartbeat.getContainerId());
            sca.undeployOpers.add(shb.nodeId);
            continue;
        }
        if (shb.requestResponse != null) {
            for (StatsListener.OperatorResponse obj : shb.requestResponse) {
                if (obj instanceof OperatorResponse) {
                    // This is to identify platform requests
                    commandResponse.put((Long) obj.getResponseId(), obj.getResponse());
                    LOG.debug(" Got back the response {} for the request {}", obj, obj.getResponseId());
                } else {
                    // This is to identify user requests
                    oper.stats.responses.add(obj);
                }
            }
        }
        // LOG.debug("heartbeat {} {}/{} {}", oper, oper.getState(), shb.getState(), oper.getContainer().getExternalId());
        if (!(oper.getState() == PTOperator.State.ACTIVE && shb.getState() == OperatorHeartbeat.DeployState.ACTIVE)) {
            // deploy state may require synchronization
            processOperatorDeployStatus(oper, shb, sca);
        }
        oper.stats.lastHeartbeat = shb;
        List<ContainerStats.OperatorStats> statsList = shb.getOperatorStatsContainer();
        if (!statsList.isEmpty()) {
            long tuplesProcessed = 0;
            long tuplesEmitted = 0;
            long totalCpuTimeUsed = 0;
            int statCount = 0;
            long maxDequeueTimestamp = -1;
            oper.stats.recordingId = null;
            final OperatorStatus status = oper.stats;
            status.statsRevs.checkout();
            for (Map.Entry<String, PortStatus> entry : status.inputPortStatusList.entrySet()) {
                entry.getValue().recordingId = null;
            }
            for (Map.Entry<String, PortStatus> entry : status.outputPortStatusList.entrySet()) {
                entry.getValue().recordingId = null;
            }
            for (ContainerStats.OperatorStats stats : statsList) {
                if (stats == null) {
                    LOG.warn("Operator {} statistics list contains null element", shb.getNodeId());
                    continue;
                }
                /* report checkpoint-ed WindowId status of the operator */
                if (stats.checkpoint instanceof Checkpoint) {
                    if (oper.getRecentCheckpoint() == null || oper.getRecentCheckpoint().windowId < stats.checkpoint.getWindowId()) {
                        addCheckpoint(oper, (Checkpoint) stats.checkpoint);
                        if (stats.checkpointStats != null) {
                            status.checkpointStats = stats.checkpointStats;
                            status.checkpointTimeMA.add(stats.checkpointStats.checkpointTime);
                        }
                        oper.failureCount = 0;
                    }
                }
                oper.stats.recordingId = stats.recordingId;
                /* report all the other stuff */
                // calculate the stats related to end window
                // end window stats for a particular window id for a particular node
                EndWindowStats endWindowStats = new EndWindowStats();
                Collection<ContainerStats.OperatorStats.PortStats> ports = stats.inputPorts;
                if (ports != null) {
                    Set<String> currentInputPortSet = Sets.newHashSetWithExpectedSize(ports.size());
                    for (ContainerStats.OperatorStats.PortStats s : ports) {
                        currentInputPortSet.add(s.id);
                        PortStatus ps = status.inputPortStatusList.get(s.id);
                        if (ps == null) {
                            ps = status.new PortStatus();
                            ps.portName = s.id;
                            status.inputPortStatusList.put(s.id, ps);
                        }
                        ps.totalTuples += s.tupleCount;
                        ps.recordingId = s.recordingId;
                        tuplesProcessed += s.tupleCount;
                        endWindowStats.dequeueTimestamps.put(s.id, s.endWindowTimestamp);
                        Pair<Integer, String> operatorPortName = new Pair<>(oper.getId(), s.id);
                        Long lastEndWindowTimestamp = operatorPortLastEndWindowTimestamps.get(operatorPortName);
                        if (lastEndWindowTimestamp == null) {
                            lastEndWindowTimestamp = lastStatsTimestamp;
                        }
                        long portElapsedMillis = Math.max(s.endWindowTimestamp - lastEndWindowTimestamp, 0);
                        // LOG.debug("=== PROCESSED TUPLE COUNT for {}: {}, {}, {}, {}", operatorPortName, s.tupleCount, portElapsedMillis, operatorPortLastEndWindowTimestamps.get(operatorPortName), lastStatsTimestamp);
                        ps.tuplesPMSMA.add(s.tupleCount, portElapsedMillis);
                        ps.bufferServerBytesPMSMA.add(s.bufferServerBytes, portElapsedMillis);
                        ps.queueSizeMA.add(s.queueSize);
                        operatorPortLastEndWindowTimestamps.put(operatorPortName, s.endWindowTimestamp);
                        if (maxEndWindowTimestamp < s.endWindowTimestamp) {
                            maxEndWindowTimestamp = s.endWindowTimestamp;
                        }
                        if (s.endWindowTimestamp > maxDequeueTimestamp) {
                            maxDequeueTimestamp = s.endWindowTimestamp;
                        }
                    }
                    // need to remove dead ports, for unifiers
                    Iterator<Map.Entry<String, PortStatus>> it = status.inputPortStatusList.entrySet().iterator();
                    while (it.hasNext()) {
                        Map.Entry<String, PortStatus> entry = it.next();
                        if (!currentInputPortSet.contains(entry.getKey())) {
                            it.remove();
                        }
                    }
                }
                ports = stats.outputPorts;
                if (ports != null) {
                    Set<String> currentOutputPortSet = Sets.newHashSetWithExpectedSize(ports.size());
                    for (ContainerStats.OperatorStats.PortStats s : ports) {
                        currentOutputPortSet.add(s.id);
                        PortStatus ps = status.outputPortStatusList.get(s.id);
                        if (ps == null) {
                            ps = status.new PortStatus();
                            ps.portName = s.id;
                            status.outputPortStatusList.put(s.id, ps);
                        }
                        ps.totalTuples += s.tupleCount;
                        ps.recordingId = s.recordingId;
                        tuplesEmitted += s.tupleCount;
                        Pair<Integer, String> operatorPortName = new Pair<>(oper.getId(), s.id);
                        Long lastEndWindowTimestamp = operatorPortLastEndWindowTimestamps.get(operatorPortName);
                        if (lastEndWindowTimestamp == null) {
                            lastEndWindowTimestamp = lastStatsTimestamp;
                        }
                        long portElapsedMillis = Math.max(s.endWindowTimestamp - lastEndWindowTimestamp, 0);
                        // LOG.debug("=== EMITTED TUPLE COUNT for {}: {}, {}, {}, {}", operatorPortName, s.tupleCount, portElapsedMillis, operatorPortLastEndWindowTimestamps.get(operatorPortName), lastStatsTimestamp);
                        ps.tuplesPMSMA.add(s.tupleCount, portElapsedMillis);
                        ps.bufferServerBytesPMSMA.add(s.bufferServerBytes, portElapsedMillis);
                        operatorPortLastEndWindowTimestamps.put(operatorPortName, s.endWindowTimestamp);
                        if (maxEndWindowTimestamp < s.endWindowTimestamp) {
                            maxEndWindowTimestamp = s.endWindowTimestamp;
                        }
                    }
                    if (ports.size() > 0) {
                        endWindowStats.emitTimestamp = ports.iterator().next().endWindowTimestamp;
                    }
                    // need to remove dead ports, for unifiers
                    Iterator<Map.Entry<String, PortStatus>> it = status.outputPortStatusList.entrySet().iterator();
                    while (it.hasNext()) {
                        Map.Entry<String, PortStatus> entry = it.next();
                        if (!currentOutputPortSet.contains(entry.getKey())) {
                            it.remove();
                        }
                    }
                }
                // (we don't know the latency for output operators because they don't emit tuples)
                if (endWindowStats.emitTimestamp < 0) {
                    endWindowStats.emitTimestamp = maxDequeueTimestamp;
                }
                if (status.currentWindowId.get() != stats.windowId) {
                    status.lastWindowIdChangeTms = currentTimeMillis;
                    status.currentWindowId.set(stats.windowId);
                }
                totalCpuTimeUsed += stats.cpuTimeUsed;
                statCount++;
                if (oper.getOperatorMeta().getValue(OperatorContext.COUNTERS_AGGREGATOR) != null) {
                    endWindowStats.counters = stats.counters;
                }
                if (oper.getOperatorMeta().getMetricAggregatorMeta() != null && oper.getOperatorMeta().getMetricAggregatorMeta().getAggregator() != null) {
                    endWindowStats.metrics = stats.metrics;
                }
                if (stats.windowId > currentEndWindowStatsWindowId) {
                    Map<Integer, EndWindowStats> endWindowStatsMap = endWindowStatsOperatorMap.get(stats.windowId);
                    if (endWindowStatsMap == null) {
                        endWindowStatsMap = new ConcurrentSkipListMap<>();
                        Map<Integer, EndWindowStats> endWindowStatsMapPrevious = endWindowStatsOperatorMap.putIfAbsent(stats.windowId, endWindowStatsMap);
                        if (endWindowStatsMapPrevious != null) {
                            endWindowStatsMap = endWindowStatsMapPrevious;
                        }
                    }
                    endWindowStatsMap.put(shb.getNodeId(), endWindowStats);
                    Set<Integer> allCurrentOperators = plan.getAllOperators().keySet();
                    int numOperators = plan.getAllOperators().size();
                    if (allCurrentOperators.containsAll(endWindowStatsMap.keySet()) && endWindowStatsMap.size() == numOperators) {
                        completeEndWindowStatsWindowId = stats.windowId;
                    }
                }
            }
            status.totalTuplesProcessed.add(tuplesProcessed);
            status.totalTuplesEmitted.add(tuplesEmitted);
            OperatorMeta logicalOperator = oper.getOperatorMeta();
            LogicalOperatorStatus logicalStatus = logicalOperator.getStatus();
            if (!oper.isUnifier()) {
                logicalStatus.totalTuplesProcessed += tuplesProcessed;
                logicalStatus.totalTuplesEmitted += tuplesEmitted;
            }
            long lastMaxEndWindowTimestamp = operatorLastEndWindowTimestamps.containsKey(oper.getId()) ? operatorLastEndWindowTimestamps.get(oper.getId()) : lastStatsTimestamp;
            if (maxEndWindowTimestamp >= lastMaxEndWindowTimestamp) {
                double tuplesProcessedPMSMA = 0.0;
                double tuplesEmittedPMSMA = 0.0;
                if (statCount != 0) {
                    // LOG.debug("CPU for {}: {} / {} - {}", oper.getId(), totalCpuTimeUsed, maxEndWindowTimestamp, lastMaxEndWindowTimestamp);
                    status.cpuNanosPMSMA.add(totalCpuTimeUsed, maxEndWindowTimestamp - lastMaxEndWindowTimestamp);
                }
                for (PortStatus ps : status.inputPortStatusList.values()) {
                    tuplesProcessedPMSMA += ps.tuplesPMSMA.getAvg();
                }
                for (PortStatus ps : status.outputPortStatusList.values()) {
                    tuplesEmittedPMSMA += ps.tuplesPMSMA.getAvg();
                }
                status.tuplesProcessedPSMA.set(Math.round(tuplesProcessedPMSMA * 1000));
                status.tuplesEmittedPSMA.set(Math.round(tuplesEmittedPMSMA * 1000));
            } else {
            // LOG.warn("This timestamp for {} is lower than the previous!! {} < {}", oper.getId(),
            // maxEndWindowTimestamp, lastMaxEndWindowTimestamp);
            }
            operatorLastEndWindowTimestamps.put(oper.getId(), maxEndWindowTimestamp);
            status.listenerStats.add(statsList);
            this.reportStats.put(oper, oper);
            status.statsRevs.commit();
        }
        if (lastStatsTimestamp < maxEndWindowTimestamp) {
            lastStatsTimestamp = maxEndWindowTimestamp;
        }
    }
    sca.lastHeartbeatMillis = currentTimeMillis;
    for (PTOperator oper : sca.container.getOperators()) {
        if (!reportedOperators.contains(oper.getId())) {
            processOperatorDeployStatus(oper, null, sca);
        }
    }
    ContainerHeartbeatResponse rsp = getHeartbeatResponse(sca);
    if (heartbeat.getContainerStats().operators.isEmpty() && isApplicationIdle()) {
        LOG.info("requesting idle shutdown for container {}", heartbeat.getContainerId());
        rsp.shutdown = ShutdownType.ABORT;
    } else {
        if (sca.isShutdownRequested()) {
            LOG.info("requesting shutdown for container {}", heartbeat.getContainerId());
            rsp.shutdown = sca.shutdownRequest;
        }
    }
    List<StramToNodeRequest> requests = rsp.nodeRequests != null ? rsp.nodeRequests : new ArrayList<StramToNodeRequest>();
    ConcurrentLinkedQueue<StramToNodeRequest> operatorRequests = sca.getOperatorRequests();
    while (true) {
        StramToNodeRequest r = operatorRequests.poll();
        if (r == null) {
            break;
        }
        requests.add(r);
    }
    rsp.nodeRequests = requests;
    rsp.committedWindowId = committedWindowId;
    rsp.stackTraceRequired = sca.stackTraceRequested;
    sca.stackTraceRequested = false;
    apexPluginDispatcher.dispatch(new DAGExecutionEvent.HeartbeatExecutionEvent(heartbeat));
    return rsp;
}