Examples with CheckpointConfig org.apache.flink.streaming.api.environment.CheckpointConfig used on opensource projects

Example 1 with CheckpointConfig

use of org.apache.flink.streaming.api.environment.CheckpointConfig in project flink by apache.

the class StreamingJobGraphGenerator method setVertexConfig.

@SuppressWarnings("unchecked")
private void setVertexConfig(Integer vertexID, StreamConfig config, List<StreamEdge> chainableOutputs, List<StreamEdge> nonChainableOutputs) {
    StreamNode vertex = streamGraph.getStreamNode(vertexID);
    config.setVertexID(vertexID);
    config.setBufferTimeout(vertex.getBufferTimeout());
    config.setTypeSerializerIn1(vertex.getTypeSerializerIn1());
    config.setTypeSerializerIn2(vertex.getTypeSerializerIn2());
    config.setTypeSerializerOut(vertex.getTypeSerializerOut());
    // iterate edges, find sideOutput edges create and save serializers for each outputTag type
    for (StreamEdge edge : chainableOutputs) {
        if (edge.getOutputTag() != null) {
            config.setTypeSerializerSideOut(edge.getOutputTag(), edge.getOutputTag().getTypeInfo().createSerializer(streamGraph.getExecutionConfig()));
        }
    }
    for (StreamEdge edge : nonChainableOutputs) {
        if (edge.getOutputTag() != null) {
            config.setTypeSerializerSideOut(edge.getOutputTag(), edge.getOutputTag().getTypeInfo().createSerializer(streamGraph.getExecutionConfig()));
        }
    }
    config.setStreamOperator(vertex.getOperator());
    config.setOutputSelectors(vertex.getOutputSelectors());
    config.setNumberOfOutputs(nonChainableOutputs.size());
    config.setNonChainedOutputs(nonChainableOutputs);
    config.setChainedOutputs(chainableOutputs);
    config.setTimeCharacteristic(streamGraph.getEnvironment().getStreamTimeCharacteristic());
    final CheckpointConfig ceckpointCfg = streamGraph.getCheckpointConfig();
    config.setStateBackend(streamGraph.getStateBackend());
    config.setCheckpointingEnabled(ceckpointCfg.isCheckpointingEnabled());
    if (ceckpointCfg.isCheckpointingEnabled()) {
        config.setCheckpointMode(ceckpointCfg.getCheckpointingMode());
    } else {
        // the "at-least-once" input handler is slightly cheaper (in the absence of checkpoints),
        // so we use that one if checkpointing is not enabled
        config.setCheckpointMode(CheckpointingMode.AT_LEAST_ONCE);
    }
    config.setStatePartitioner(0, vertex.getStatePartitioner1());
    config.setStatePartitioner(1, vertex.getStatePartitioner2());
    config.setStateKeySerializer(vertex.getStateKeySerializer());
    Class<? extends AbstractInvokable> vertexClass = vertex.getJobVertexClass();
    if (vertexClass.equals(StreamIterationHead.class) || vertexClass.equals(StreamIterationTail.class)) {
        config.setIterationId(streamGraph.getBrokerID(vertexID));
        config.setIterationWaitTime(streamGraph.getLoopTimeout(vertexID));
    }
    List<StreamEdge> allOutputs = new ArrayList<StreamEdge>(chainableOutputs);
    allOutputs.addAll(nonChainableOutputs);
    vertexConfigs.put(vertexID, config);
}

Example 2 with CheckpointConfig

use of org.apache.flink.streaming.api.environment.CheckpointConfig in project flink by apache.

the class StreamingJobGraphGenerator method configureCheckpointing.

private void configureCheckpointing() {
    CheckpointConfig cfg = streamGraph.getCheckpointConfig();
    long interval = cfg.getCheckpointInterval();
    if (interval > 0) {
        // check if a restart strategy has been set, if not then set the FixedDelayRestartStrategy
        if (streamGraph.getExecutionConfig().getRestartStrategy() == null) {
            // if the user enabled checkpointing, the default number of exec retries is infinite.
            streamGraph.getExecutionConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, DEFAULT_RESTART_DELAY));
        }
    } else {
        // interval of max value means disable periodic checkpoint
        interval = Long.MAX_VALUE;
    }
    // collect the vertices that receive "trigger checkpoint" messages.
    // currently, these are all the sources
    List<JobVertexID> triggerVertices = new ArrayList<>();
    // collect the vertices that need to acknowledge the checkpoint
    // currently, these are all vertices
    List<JobVertexID> ackVertices = new ArrayList<>(jobVertices.size());
    // collect the vertices that receive "commit checkpoint" messages
    // currently, these are all vertices
    List<JobVertexID> commitVertices = new ArrayList<>();
    for (JobVertex vertex : jobVertices.values()) {
        if (vertex.isInputVertex()) {
            triggerVertices.add(vertex.getID());
        }
        commitVertices.add(vertex.getID());
        ackVertices.add(vertex.getID());
    }
    ExternalizedCheckpointSettings externalizedCheckpointSettings;
    if (cfg.isExternalizedCheckpointsEnabled()) {
        CheckpointConfig.ExternalizedCheckpointCleanup cleanup = cfg.getExternalizedCheckpointCleanup();
        // Sanity check
        if (cleanup == null) {
            throw new IllegalStateException("Externalized checkpoints enabled, but no cleanup mode configured.");
        }
        externalizedCheckpointSettings = ExternalizedCheckpointSettings.externalizeCheckpoints(cleanup.deleteOnCancellation());
    } else {
        externalizedCheckpointSettings = ExternalizedCheckpointSettings.none();
    }
    CheckpointingMode mode = cfg.getCheckpointingMode();
    boolean isExactlyOnce;
    if (mode == CheckpointingMode.EXACTLY_ONCE) {
        isExactlyOnce = true;
    } else if (mode == CheckpointingMode.AT_LEAST_ONCE) {
        isExactlyOnce = false;
    } else {
        throw new IllegalStateException("Unexpected checkpointing mode. " + "Did not expect there to be another checkpointing mode besides " + "exactly-once or at-least-once.");
    }
    JobSnapshottingSettings settings = new JobSnapshottingSettings(triggerVertices, ackVertices, commitVertices, interval, cfg.getCheckpointTimeout(), cfg.getMinPauseBetweenCheckpoints(), cfg.getMaxConcurrentCheckpoints(), externalizedCheckpointSettings, streamGraph.getStateBackend(), isExactlyOnce);
    jobGraph.setSnapshotSettings(settings);
}