Examples with BroadcastStateBootstrapOperator org.apache.flink.state.api.output.operators.BroadcastStateBootstrapOperator used on opensource projects

Example 1 with BroadcastStateBootstrapOperator

use of org.apache.flink.state.api.output.operators.BroadcastStateBootstrapOperator in project flink by apache.

the class BootstrapTransformation method writeOperatorSubtaskStates.

private MapPartitionOperator<T, TaggedOperatorSubtaskState> writeOperatorSubtaskStates(OperatorID operatorID, @Nullable StateBackend stateBackend, Configuration additionalConfig, Path savepointPath, int localMaxParallelism) {
    DataSet<T> input = dataSet;
    if (originalKeySelector != null) {
        input = dataSet.partitionCustom(new KeyGroupRangePartitioner(localMaxParallelism), hashKeySelector);
    }
    StreamOperator<TaggedOperatorSubtaskState> operator = factory.createOperator(System.currentTimeMillis(), savepointPath);
    operator = dataSet.clean(operator);
    final StreamConfig config = getConfig(operatorID, stateBackend, additionalConfig, operator);
    BoundedOneInputStreamTaskRunner<T> operatorRunner = new BoundedOneInputStreamTaskRunner<>(config, localMaxParallelism, timestamper);
    MapPartitionOperator<T, TaggedOperatorSubtaskState> subtaskStates = input.mapPartition(operatorRunner).name(operatorID.toHexString());
    if (operator instanceof BroadcastStateBootstrapOperator) {
        subtaskStates = subtaskStates.setParallelism(1);
    } else {
        int currentParallelism = getParallelism(subtaskStates);
        if (currentParallelism > localMaxParallelism) {
            subtaskStates.setParallelism(localMaxParallelism);
        }
    }
    return subtaskStates;
}

Example 2 with BroadcastStateBootstrapOperator

use of org.apache.flink.state.api.output.operators.BroadcastStateBootstrapOperator in project flink by apache.

the class StateBootstrapTransformation method writeOperatorSubtaskStates.

private SingleOutputStreamOperator<TaggedOperatorSubtaskState> writeOperatorSubtaskStates(OperatorID operatorID, StateBackend stateBackend, Configuration additionalConfig, Path savepointPath, int localMaxParallelism) {
    StreamOperator<TaggedOperatorSubtaskState> operator = factory.createOperator(System.currentTimeMillis(), savepointPath);
    operator = stream.getExecutionEnvironment().clean(operator);
    final StreamConfig config = getConfig(operatorID, stateBackend, additionalConfig, operator);
    BootstrapStreamTaskRunner<T> operatorRunner = new BootstrapStreamTaskRunner<>(config, localMaxParallelism);
    DataStream<T> input = stream;
    if (keySelector != null) {
        input = stream.keyBy(this.keySelector);
    }
    SingleOutputStreamOperator<TaggedOperatorSubtaskState> subtaskStates = input.transform(operatorID.toHexString(), TypeInformation.of(TaggedOperatorSubtaskState.class), operatorRunner).setMaxParallelism(localMaxParallelism);
    if (operator instanceof BroadcastStateBootstrapOperator) {
        subtaskStates = subtaskStates.setParallelism(1);
    } else {
        int currentParallelism = getParallelism(subtaskStates);
        if (currentParallelism > localMaxParallelism) {
            subtaskStates.setParallelism(localMaxParallelism);
        }
    }
    return subtaskStates;
}