Examples with FileCopyFunction org.apache.flink.state.api.output.FileCopyFunction used on opensource projects

Example 1 with FileCopyFunction

use of org.apache.flink.state.api.output.FileCopyFunction in project flink by apache.

the class SavepointWriter method write.

/**
 * Write out a new or updated savepoint.
 *
 * @param path The path to where the savepoint should be written.
 */
public final void write(String path) {
    final Path savepointPath = new Path(path);
    List<StateBootstrapTransformationWithID<?>> newOperatorTransformations = metadata.getNewOperators();
    DataStream<OperatorState> newOperatorStates = writeOperatorStates(newOperatorTransformations, configuration, savepointPath);
    List<OperatorState> existingOperators = metadata.getExistingOperators();
    DataStream<OperatorState> finalOperatorStates;
    if (existingOperators.isEmpty()) {
        finalOperatorStates = newOperatorStates;
    } else {
        DataStream<OperatorState> existingOperatorStates = newOperatorStates.getExecutionEnvironment().fromCollection(existingOperators).name("existingOperatorStates");
        existingOperatorStates.flatMap(new StatePathExtractor()).setParallelism(1).addSink(new OutputFormatSinkFunction<>(new FileCopyFunction(path)));
        finalOperatorStates = newOperatorStates.union(existingOperatorStates);
    }
    finalOperatorStates.transform("reduce(OperatorState)", TypeInformation.of(CheckpointMetadata.class), new GroupReduceOperator<>(new MergeOperatorStates(metadata.getMasterStates()))).forceNonParallel().addSink(new OutputFormatSinkFunction<>(new SavepointOutputFormat(savepointPath))).setParallelism(1).name(path);
}

Example 2 with FileCopyFunction

use of org.apache.flink.state.api.output.FileCopyFunction in project flink by apache.

the class WritableSavepoint method write.

/**
 * Write out a new or updated savepoint.
 *
 * @param path The path to where the savepoint should be written.
 */
public final void write(String path) {
    final Path savepointPath = new Path(path);
    List<BootstrapTransformationWithID<?>> newOperatorTransformations = metadata.getNewOperators();
    DataSet<OperatorState> newOperatorStates = writeOperatorStates(newOperatorTransformations, configuration, savepointPath);
    List<OperatorState> existingOperators = metadata.getExistingOperators();
    DataSet<OperatorState> finalOperatorStates;
    if (existingOperators.isEmpty()) {
        finalOperatorStates = newOperatorStates;
    } else {
        DataSet<OperatorState> existingOperatorStates = newOperatorStates.getExecutionEnvironment().fromCollection(existingOperators).name("existingOperatorStates");
        existingOperatorStates.flatMap(new StatePathExtractor()).setParallelism(1).output(new FileCopyFunction(path));
        finalOperatorStates = newOperatorStates.union(existingOperatorStates);
    }
    finalOperatorStates.reduceGroup(new MergeOperatorStates(metadata.getMasterStates())).name("reduce(OperatorState)").output(new SavepointOutputFormat(savepointPath)).name(path);
}