Examples with Checkpoint com.datatorrent.stram.api.Checkpoint used on opensource projects

Example 21 with Checkpoint

use of com.datatorrent.stram.api.Checkpoint in project apex-core by apache.

the class StreamingContainerManager method processEvents.

public int processEvents() {
    for (PTOperator o : reportStats.keySet()) {
        List<OperatorStats> stats = o.stats.listenerStats.poll();
        if (stats != null) {
            // append into single list
            List<OperatorStats> moreStats;
            while ((moreStats = o.stats.listenerStats.poll()) != null) {
                stats.addAll(moreStats);
            }
        }
        o.stats.lastWindowedStats = stats;
        o.stats.operatorResponses = null;
        if (!o.stats.responses.isEmpty()) {
            o.stats.operatorResponses = new ArrayList<>();
            StatsListener.OperatorResponse operatorResponse;
            while ((operatorResponse = o.stats.responses.poll()) != null) {
                o.stats.operatorResponses.add(operatorResponse);
            }
        }
        if (o.stats.lastWindowedStats != null) {
            // call listeners only with non empty window list
            if (o.statsListeners != null) {
                plan.onStatusUpdate(o);
            }
        }
        reportStats.remove(o);
    }
    if (!this.shutdownOperators.isEmpty()) {
        synchronized (this.shutdownOperators) {
            Iterator<Map.Entry<Long, Set<PTOperator>>> it = shutdownOperators.entrySet().iterator();
            while (it.hasNext()) {
                Map.Entry<Long, Set<PTOperator>> windowAndOpers = it.next();
                if (windowAndOpers.getKey().longValue() <= this.committedWindowId || checkDownStreamOperators(windowAndOpers)) {
                    LOG.info("Removing inactive operators at window {} {}", Codec.getStringWindowId(windowAndOpers.getKey()), windowAndOpers.getValue());
                    for (PTOperator oper : windowAndOpers.getValue()) {
                        plan.removeTerminatedPartition(oper);
                    }
                    it.remove();
                }
            }
        }
    }
    if (!eventQueue.isEmpty()) {
        for (PTOperator oper : plan.getAllOperators().values()) {
            if (oper.getState() != PTOperator.State.ACTIVE) {
                LOG.debug("Skipping plan updates due to inactive operator {} {}", oper, oper.getState());
                return 0;
            }
        }
    }
    int count = 0;
    Runnable command;
    while ((command = this.eventQueue.poll()) != null) {
        eventQueueProcessing.set(true);
        try {
            command.run();
            count++;
        } catch (Exception e) {
            // TODO: handle error
            LOG.error("Failed to execute {}", command, e);
        }
        eventQueueProcessing.set(false);
    }
    if (count > 0) {
        try {
            checkpoint();
        } catch (Exception e) {
            throw new RuntimeException("Failed to checkpoint state.", e);
        }
    }
    return count;
}

Example 22 with Checkpoint

use of com.datatorrent.stram.api.Checkpoint in project apex-core by apache.

the class StreamingContainerManager method processHeartbeat.

/**
   * process the heartbeat from each container.
   * called by the RPC thread for each container. (i.e. called by multiple threads)
   *
   * @param heartbeat
   * @return heartbeat response
   */
@SuppressWarnings("StatementWithEmptyBody")
public ContainerHeartbeatResponse processHeartbeat(ContainerHeartbeat heartbeat) {
    long currentTimeMillis = clock.getTime();
    final StreamingContainerAgent sca = this.containers.get(heartbeat.getContainerId());
    if (sca == null || sca.container.getState() == PTContainer.State.KILLED) {
        // could be orphaned container that was replaced and needs to terminate
        LOG.error("Unknown container {}", heartbeat.getContainerId());
        ContainerHeartbeatResponse response = new ContainerHeartbeatResponse();
        response.shutdown = ShutdownType.ABORT;
        return response;
    }
    //LOG.debug("{} {} {}", new Object[]{sca.container.containerId, sca.container.bufferServerAddress, sca.container.getState()});
    if (sca.container.getState() == PTContainer.State.ALLOCATED) {
        // capture dynamically assigned address from container
        if (sca.container.bufferServerAddress == null && heartbeat.bufferServerHost != null) {
            sca.container.bufferServerAddress = InetSocketAddress.createUnresolved(heartbeat.bufferServerHost, heartbeat.bufferServerPort);
            LOG.info("Container {} buffer server: {}", sca.container.getExternalId(), sca.container.bufferServerAddress);
        }
        final long containerStartTime = System.currentTimeMillis();
        sca.container.setState(PTContainer.State.ACTIVE);
        sca.container.setStartedTime(containerStartTime);
        sca.container.setFinishedTime(-1);
        sca.jvmName = heartbeat.jvmName;
        poolExecutor.submit(new Runnable() {

            @Override
            public void run() {
                try {
                    containerFile.append(sca.getContainerInfo());
                } catch (IOException ex) {
                    LOG.warn("Cannot write to container file");
                }
                for (PTOperator ptOp : sca.container.getOperators()) {
                    try {
                        JSONObject operatorInfo = new JSONObject();
                        operatorInfo.put("name", ptOp.getName());
                        operatorInfo.put("id", ptOp.getId());
                        operatorInfo.put("container", sca.container.getExternalId());
                        operatorInfo.put("startTime", containerStartTime);
                        operatorFile.append(operatorInfo);
                    } catch (IOException | JSONException ex) {
                        LOG.warn("Cannot write to operator file: ", ex);
                    }
                }
            }
        });
    }
    sca.containerStackTrace = heartbeat.stackTrace;
    if (heartbeat.restartRequested) {
        LOG.error("Container {} restart request", sca.container.getExternalId());
        containerStopRequests.put(sca.container.getExternalId(), sca.container.getExternalId());
    }
    sca.memoryMBFree = heartbeat.memoryMBFree;
    sca.gcCollectionCount = heartbeat.gcCollectionCount;
    sca.gcCollectionTime = heartbeat.gcCollectionTime;
    sca.undeployOpers.clear();
    sca.deployOpers.clear();
    if (!this.deployChangeInProgress.get()) {
        sca.deployCnt = this.deployChangeCnt;
    }
    Set<Integer> reportedOperators = Sets.newHashSetWithExpectedSize(sca.container.getOperators().size());
    for (OperatorHeartbeat shb : heartbeat.getContainerStats().operators) {
        long maxEndWindowTimestamp = 0;
        reportedOperators.add(shb.nodeId);
        PTOperator oper = this.plan.getAllOperators().get(shb.getNodeId());
        if (oper == null) {
            LOG.info("Heartbeat for unknown operator {} (container {})", shb.getNodeId(), heartbeat.getContainerId());
            sca.undeployOpers.add(shb.nodeId);
            continue;
        }
        if (shb.requestResponse != null) {
            for (StatsListener.OperatorResponse obj : shb.requestResponse) {
                if (obj instanceof OperatorResponse) {
                    // This is to identify platform requests
                    commandResponse.put((Long) obj.getResponseId(), obj.getResponse());
                    LOG.debug(" Got back the response {} for the request {}", obj, obj.getResponseId());
                } else {
                    // This is to identify user requests
                    oper.stats.responses.add(obj);
                }
            }
        }
        //LOG.debug("heartbeat {} {}/{} {}", oper, oper.getState(), shb.getState(), oper.getContainer().getExternalId());
        if (!(oper.getState() == PTOperator.State.ACTIVE && shb.getState() == OperatorHeartbeat.DeployState.ACTIVE)) {
            // deploy state may require synchronization
            processOperatorDeployStatus(oper, shb, sca);
        }
        oper.stats.lastHeartbeat = shb;
        List<ContainerStats.OperatorStats> statsList = shb.getOperatorStatsContainer();
        if (!statsList.isEmpty()) {
            long tuplesProcessed = 0;
            long tuplesEmitted = 0;
            long totalCpuTimeUsed = 0;
            int statCount = 0;
            long maxDequeueTimestamp = -1;
            oper.stats.recordingId = null;
            final OperatorStatus status = oper.stats;
            status.statsRevs.checkout();
            for (Map.Entry<String, PortStatus> entry : status.inputPortStatusList.entrySet()) {
                entry.getValue().recordingId = null;
            }
            for (Map.Entry<String, PortStatus> entry : status.outputPortStatusList.entrySet()) {
                entry.getValue().recordingId = null;
            }
            for (ContainerStats.OperatorStats stats : statsList) {
                if (stats == null) {
                    LOG.warn("Operator {} statistics list contains null element", shb.getNodeId());
                    continue;
                }
                /* report checkpoint-ed WindowId status of the operator */
                if (stats.checkpoint instanceof Checkpoint) {
                    if (oper.getRecentCheckpoint() == null || oper.getRecentCheckpoint().windowId < stats.checkpoint.getWindowId()) {
                        addCheckpoint(oper, (Checkpoint) stats.checkpoint);
                        if (stats.checkpointStats != null) {
                            status.checkpointStats = stats.checkpointStats;
                            status.checkpointTimeMA.add(stats.checkpointStats.checkpointTime);
                        }
                        oper.failureCount = 0;
                    }
                }
                oper.stats.recordingId = stats.recordingId;
                /* report all the other stuff */
                // calculate the stats related to end window
                // end window stats for a particular window id for a particular node
                EndWindowStats endWindowStats = new EndWindowStats();
                Collection<ContainerStats.OperatorStats.PortStats> ports = stats.inputPorts;
                if (ports != null) {
                    Set<String> currentInputPortSet = Sets.newHashSetWithExpectedSize(ports.size());
                    for (ContainerStats.OperatorStats.PortStats s : ports) {
                        currentInputPortSet.add(s.id);
                        PortStatus ps = status.inputPortStatusList.get(s.id);
                        if (ps == null) {
                            ps = status.new PortStatus();
                            ps.portName = s.id;
                            status.inputPortStatusList.put(s.id, ps);
                        }
                        ps.totalTuples += s.tupleCount;
                        ps.recordingId = s.recordingId;
                        tuplesProcessed += s.tupleCount;
                        endWindowStats.dequeueTimestamps.put(s.id, s.endWindowTimestamp);
                        Pair<Integer, String> operatorPortName = new Pair<>(oper.getId(), s.id);
                        Long lastEndWindowTimestamp = operatorPortLastEndWindowTimestamps.get(operatorPortName);
                        if (lastEndWindowTimestamp == null) {
                            lastEndWindowTimestamp = lastStatsTimestamp;
                        }
                        long portElapsedMillis = Math.max(s.endWindowTimestamp - lastEndWindowTimestamp, 0);
                        //LOG.debug("=== PROCESSED TUPLE COUNT for {}: {}, {}, {}, {}", operatorPortName, s.tupleCount, portElapsedMillis, operatorPortLastEndWindowTimestamps.get(operatorPortName), lastStatsTimestamp);
                        ps.tuplesPMSMA.add(s.tupleCount, portElapsedMillis);
                        ps.bufferServerBytesPMSMA.add(s.bufferServerBytes, portElapsedMillis);
                        ps.queueSizeMA.add(s.queueSize);
                        operatorPortLastEndWindowTimestamps.put(operatorPortName, s.endWindowTimestamp);
                        if (maxEndWindowTimestamp < s.endWindowTimestamp) {
                            maxEndWindowTimestamp = s.endWindowTimestamp;
                        }
                        if (s.endWindowTimestamp > maxDequeueTimestamp) {
                            maxDequeueTimestamp = s.endWindowTimestamp;
                        }
                    }
                    // need to remove dead ports, for unifiers
                    Iterator<Map.Entry<String, PortStatus>> it = status.inputPortStatusList.entrySet().iterator();
                    while (it.hasNext()) {
                        Map.Entry<String, PortStatus> entry = it.next();
                        if (!currentInputPortSet.contains(entry.getKey())) {
                            it.remove();
                        }
                    }
                }
                ports = stats.outputPorts;
                if (ports != null) {
                    Set<String> currentOutputPortSet = Sets.newHashSetWithExpectedSize(ports.size());
                    for (ContainerStats.OperatorStats.PortStats s : ports) {
                        currentOutputPortSet.add(s.id);
                        PortStatus ps = status.outputPortStatusList.get(s.id);
                        if (ps == null) {
                            ps = status.new PortStatus();
                            ps.portName = s.id;
                            status.outputPortStatusList.put(s.id, ps);
                        }
                        ps.totalTuples += s.tupleCount;
                        ps.recordingId = s.recordingId;
                        tuplesEmitted += s.tupleCount;
                        Pair<Integer, String> operatorPortName = new Pair<>(oper.getId(), s.id);
                        Long lastEndWindowTimestamp = operatorPortLastEndWindowTimestamps.get(operatorPortName);
                        if (lastEndWindowTimestamp == null) {
                            lastEndWindowTimestamp = lastStatsTimestamp;
                        }
                        long portElapsedMillis = Math.max(s.endWindowTimestamp - lastEndWindowTimestamp, 0);
                        //LOG.debug("=== EMITTED TUPLE COUNT for {}: {}, {}, {}, {}", operatorPortName, s.tupleCount, portElapsedMillis, operatorPortLastEndWindowTimestamps.get(operatorPortName), lastStatsTimestamp);
                        ps.tuplesPMSMA.add(s.tupleCount, portElapsedMillis);
                        ps.bufferServerBytesPMSMA.add(s.bufferServerBytes, portElapsedMillis);
                        operatorPortLastEndWindowTimestamps.put(operatorPortName, s.endWindowTimestamp);
                        if (maxEndWindowTimestamp < s.endWindowTimestamp) {
                            maxEndWindowTimestamp = s.endWindowTimestamp;
                        }
                    }
                    if (ports.size() > 0) {
                        endWindowStats.emitTimestamp = ports.iterator().next().endWindowTimestamp;
                    }
                    // need to remove dead ports, for unifiers
                    Iterator<Map.Entry<String, PortStatus>> it = status.outputPortStatusList.entrySet().iterator();
                    while (it.hasNext()) {
                        Map.Entry<String, PortStatus> entry = it.next();
                        if (!currentOutputPortSet.contains(entry.getKey())) {
                            it.remove();
                        }
                    }
                }
                // (we don't know the latency for output operators because they don't emit tuples)
                if (endWindowStats.emitTimestamp < 0) {
                    endWindowStats.emitTimestamp = maxDequeueTimestamp;
                }
                if (status.currentWindowId.get() != stats.windowId) {
                    status.lastWindowIdChangeTms = currentTimeMillis;
                    status.currentWindowId.set(stats.windowId);
                }
                totalCpuTimeUsed += stats.cpuTimeUsed;
                statCount++;
                if (oper.getOperatorMeta().getValue(OperatorContext.COUNTERS_AGGREGATOR) != null) {
                    endWindowStats.counters = stats.counters;
                }
                if (oper.getOperatorMeta().getMetricAggregatorMeta() != null && oper.getOperatorMeta().getMetricAggregatorMeta().getAggregator() != null) {
                    endWindowStats.metrics = stats.metrics;
                }
                if (stats.windowId > currentEndWindowStatsWindowId) {
                    Map<Integer, EndWindowStats> endWindowStatsMap = endWindowStatsOperatorMap.get(stats.windowId);
                    if (endWindowStatsMap == null) {
                        endWindowStatsMap = new ConcurrentSkipListMap<>();
                        Map<Integer, EndWindowStats> endWindowStatsMapPrevious = endWindowStatsOperatorMap.putIfAbsent(stats.windowId, endWindowStatsMap);
                        if (endWindowStatsMapPrevious != null) {
                            endWindowStatsMap = endWindowStatsMapPrevious;
                        }
                    }
                    endWindowStatsMap.put(shb.getNodeId(), endWindowStats);
                    Set<Integer> allCurrentOperators = plan.getAllOperators().keySet();
                    int numOperators = plan.getAllOperators().size();
                    if (allCurrentOperators.containsAll(endWindowStatsMap.keySet()) && endWindowStatsMap.size() == numOperators) {
                        completeEndWindowStatsWindowId = stats.windowId;
                    }
                }
            }
            status.totalTuplesProcessed.add(tuplesProcessed);
            status.totalTuplesEmitted.add(tuplesEmitted);
            OperatorMeta logicalOperator = oper.getOperatorMeta();
            LogicalOperatorStatus logicalStatus = logicalOperator.getStatus();
            if (!oper.isUnifier()) {
                logicalStatus.totalTuplesProcessed += tuplesProcessed;
                logicalStatus.totalTuplesEmitted += tuplesEmitted;
            }
            long lastMaxEndWindowTimestamp = operatorLastEndWindowTimestamps.containsKey(oper.getId()) ? operatorLastEndWindowTimestamps.get(oper.getId()) : lastStatsTimestamp;
            if (maxEndWindowTimestamp >= lastMaxEndWindowTimestamp) {
                double tuplesProcessedPMSMA = 0.0;
                double tuplesEmittedPMSMA = 0.0;
                if (statCount != 0) {
                    //LOG.debug("CPU for {}: {} / {} - {}", oper.getId(), totalCpuTimeUsed, maxEndWindowTimestamp, lastMaxEndWindowTimestamp);
                    status.cpuNanosPMSMA.add(totalCpuTimeUsed, maxEndWindowTimestamp - lastMaxEndWindowTimestamp);
                }
                for (PortStatus ps : status.inputPortStatusList.values()) {
                    tuplesProcessedPMSMA += ps.tuplesPMSMA.getAvg();
                }
                for (PortStatus ps : status.outputPortStatusList.values()) {
                    tuplesEmittedPMSMA += ps.tuplesPMSMA.getAvg();
                }
                status.tuplesProcessedPSMA.set(Math.round(tuplesProcessedPMSMA * 1000));
                status.tuplesEmittedPSMA.set(Math.round(tuplesEmittedPMSMA * 1000));
            } else {
            //LOG.warn("This timestamp for {} is lower than the previous!! {} < {}", oper.getId(),
            // maxEndWindowTimestamp, lastMaxEndWindowTimestamp);
            }
            operatorLastEndWindowTimestamps.put(oper.getId(), maxEndWindowTimestamp);
            status.listenerStats.add(statsList);
            this.reportStats.put(oper, oper);
            status.statsRevs.commit();
        }
        if (lastStatsTimestamp < maxEndWindowTimestamp) {
            lastStatsTimestamp = maxEndWindowTimestamp;
        }
    }
    sca.lastHeartbeatMillis = currentTimeMillis;
    for (PTOperator oper : sca.container.getOperators()) {
        if (!reportedOperators.contains(oper.getId())) {
            processOperatorDeployStatus(oper, null, sca);
        }
    }
    ContainerHeartbeatResponse rsp = getHeartbeatResponse(sca);
    if (heartbeat.getContainerStats().operators.isEmpty() && isApplicationIdle()) {
        LOG.info("requesting idle shutdown for container {}", heartbeat.getContainerId());
        rsp.shutdown = ShutdownType.ABORT;
    } else {
        if (sca.isShutdownRequested()) {
            LOG.info("requesting shutdown for container {}", heartbeat.getContainerId());
            rsp.shutdown = sca.shutdownRequest;
        }
    }
    List<StramToNodeRequest> requests = rsp.nodeRequests != null ? rsp.nodeRequests : new ArrayList<StramToNodeRequest>();
    ConcurrentLinkedQueue<StramToNodeRequest> operatorRequests = sca.getOperatorRequests();
    while (true) {
        StramToNodeRequest r = operatorRequests.poll();
        if (r == null) {
            break;
        }
        requests.add(r);
    }
    rsp.nodeRequests = requests;
    rsp.committedWindowId = committedWindowId;
    rsp.stackTraceRequired = sca.stackTraceRequested;
    sca.stackTraceRequested = false;
    apexPluginDispatcher.dispatch(new DAGExecutionEvent.HeartbeatExecutionEvent(heartbeat));
    return rsp;
}

Example 23 with Checkpoint

use of com.datatorrent.stram.api.Checkpoint in project apex-core by apache.

the class StreamingContainerManager method updateCheckpoints.

/**
   * Visit all operators to update current checkpoint based on updated downstream state.
   * Purge older checkpoints that are no longer needed.
   */
private long updateCheckpoints(boolean recovery) {
    int operatorCount = 0;
    UpdateCheckpointsContext ctx = new UpdateCheckpointsContext(clock, recovery, getCheckpointGroups());
    for (OperatorMeta logicalOperator : plan.getLogicalPlan().getRootOperators()) {
        List<PTOperator> operators = plan.getOperators(logicalOperator);
        if (operators != null) {
            for (PTOperator operator : operators) {
                operatorCount++;
                updateRecoveryCheckpoints(operator, ctx, recovery);
            }
        }
    }
    // if no physical operators are available, then don't update committedWindowId
    if (operatorCount == 0) {
        return committedWindowId;
    }
    purgeCheckpoints();
    for (PTOperator oper : ctx.blocked) {
        String containerId = oper.getContainer().getExternalId();
        if (containerId != null) {
            LOG.info("Blocked operator {} container {} time {}ms", oper, oper.getContainer().toIdStateString(), ctx.currentTms - oper.stats.lastWindowIdChangeTms);
            this.containerStopRequests.put(containerId, containerId);
        }
    }
    return ctx.committedWindowId.longValue();
}

Example 24 with Checkpoint

use of com.datatorrent.stram.api.Checkpoint in project apex-core by apache.

the class PhysicalPlan method removePTOperator.

void removePTOperator(PTOperator oper) {
    LOG.debug("Removing operator {}", oper);
    // per partition merge operators
    if (!oper.upstreamMerge.isEmpty()) {
        for (PTOperator unifier : oper.upstreamMerge.values()) {
            removePTOperator(unifier);
        }
    }
    // remove inputs from downstream operators
    for (PTOutput out : oper.outputs) {
        for (PTInput sinkIn : out.sinks) {
            if (sinkIn.source.source == oper) {
                ArrayList<PTInput> cowInputs = Lists.newArrayList(sinkIn.target.inputs);
                cowInputs.remove(sinkIn);
                sinkIn.target.inputs = cowInputs;
            }
        }
    }
    // remove from upstream operators
    for (PTInput in : oper.inputs) {
        in.source.sinks.remove(in);
    }
    for (HostOperatorSet s : oper.groupings.values()) {
        s.getOperatorSet().remove(oper);
    }
    // remove checkpoint states
    try {
        synchronized (oper.checkpoints) {
            for (Checkpoint checkpoint : oper.checkpoints) {
                oper.operatorMeta.getValue(OperatorContext.STORAGE_AGENT).delete(oper.id, checkpoint.windowId);
            }
        }
    } catch (IOException e) {
        LOG.warn("Failed to remove state for " + oper, e);
    }
    List<PTOperator> cowList = Lists.newArrayList(oper.container.operators);
    cowList.remove(oper);
    oper.container.operators = cowList;
    this.deployOpers.remove(oper);
    this.undeployOpers.add(oper);
    this.allOperators.remove(oper.id);
    this.ctx.recordEventAsync(new StramEvent.RemoveOperatorEvent(oper.getName(), oper.getId()));
}

Example 25 with Checkpoint

use of com.datatorrent.stram.api.Checkpoint in project apex-core by apache.

the class StreamingContainer method getFinishedCheckpoint.

protected Checkpoint getFinishedCheckpoint(OperatorDeployInfo ndi) {
    Checkpoint checkpoint;
    if (ndi.contextAttributes != null && ndi.contextAttributes.get(OperatorContext.PROCESSING_MODE) == ProcessingMode.AT_MOST_ONCE) {
        long now = System.currentTimeMillis();
        long windowCount = WindowGenerator.getWindowCount(now, firstWindowMillis, firstWindowMillis);
        Integer temp = ndi.contextAttributes.get(OperatorContext.APPLICATION_WINDOW_COUNT);
        if (temp == null) {
            temp = containerContext.getValue(OperatorContext.APPLICATION_WINDOW_COUNT);
        }
        int appWindowCount = (int) (windowCount % temp);
        temp = ndi.contextAttributes.get(OperatorContext.CHECKPOINT_WINDOW_COUNT);
        if (temp == null) {
            temp = containerContext.getValue(OperatorContext.CHECKPOINT_WINDOW_COUNT);
        }
        int lCheckpointWindowCount = (int) (windowCount % temp);
        checkpoint = new Checkpoint(WindowGenerator.getWindowId(now, firstWindowMillis, windowWidthMillis), appWindowCount, lCheckpointWindowCount);
        logger.debug("using {} on {} at {}", ProcessingMode.AT_MOST_ONCE, ndi.name, checkpoint);
    } else {
        checkpoint = ndi.checkpoint;
        logger.debug("using {} on {} at {}", ndi.contextAttributes == null ? ProcessingMode.AT_LEAST_ONCE : (ndi.contextAttributes.get(OperatorContext.PROCESSING_MODE) == null ? ProcessingMode.AT_LEAST_ONCE : ndi.contextAttributes.get(OperatorContext.PROCESSING_MODE)), ndi.name, checkpoint);
    }
    return checkpoint;
}