Examples with TaggedOperatorSubtaskState org.apache.flink.state.api.output.TaggedOperatorSubtaskState used on opensource projects

Example 6 with TaggedOperatorSubtaskState

use of org.apache.flink.state.api.output.TaggedOperatorSubtaskState in project flink by apache.

the class BootstrapTransformationTest method testBroadcastStateTransformationParallelism.

@Test
public void testBroadcastStateTransformationParallelism() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(10);
    DataSet<Integer> input = env.fromElements(0);
    BootstrapTransformation<Integer> transformation = OperatorTransformation.bootstrapWith(input).transform(new ExampleBroadcastStateBootstrapFunction());
    int maxParallelism = transformation.getMaxParallelism(4);
    DataSet<TaggedOperatorSubtaskState> result = transformation.writeOperatorSubtaskStates(OperatorIDGenerator.fromUid("uid"), new MemoryStateBackend(), new Path(), maxParallelism);
    Assert.assertEquals("Broadcast transformations should always be run at parallelism 1", 1, getParallelism(result));
}

Example 7 with TaggedOperatorSubtaskState

use of org.apache.flink.state.api.output.TaggedOperatorSubtaskState in project flink by apache.

the class BootstrapTransformationTest method testDefaultParallelismRespectedWhenLessThanMaxParallelism.

@Test
public void testDefaultParallelismRespectedWhenLessThanMaxParallelism() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(4);
    DataSource<Integer> input = env.fromElements(0);
    BootstrapTransformation<Integer> transformation = OperatorTransformation.bootstrapWith(input).transform(new ExampleStateBootstrapFunction());
    int maxParallelism = transformation.getMaxParallelism(10);
    DataSet<TaggedOperatorSubtaskState> result = transformation.writeOperatorSubtaskStates(OperatorIDGenerator.fromUid("uid"), new MemoryStateBackend(), new Path(), maxParallelism);
    Assert.assertEquals("The parallelism of a data set should not change when less than the max parallelism of the savepoint", ExecutionConfig.PARALLELISM_DEFAULT, getParallelism(result));
}

Example 8 with TaggedOperatorSubtaskState

use of org.apache.flink.state.api.output.TaggedOperatorSubtaskState in project flink by apache.

the class BootstrapTransformationTest method testMaxParallelismRespected.

@Test
public void testMaxParallelismRespected() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(10);
    DataSource<Integer> input = env.fromElements(0);
    BootstrapTransformation<Integer> transformation = OperatorTransformation.bootstrapWith(input).transform(new ExampleStateBootstrapFunction());
    int maxParallelism = transformation.getMaxParallelism(4);
    DataSet<TaggedOperatorSubtaskState> result = transformation.writeOperatorSubtaskStates(OperatorIDGenerator.fromUid("uid"), new MemoryStateBackend(), new Path(), maxParallelism);
    Assert.assertEquals("The parallelism of a data set should be constrained my the savepoint max parallelism", 4, getParallelism(result));
}

Example 9 with TaggedOperatorSubtaskState

use of org.apache.flink.state.api.output.TaggedOperatorSubtaskState in project flink by apache.

the class BootstrapTransformationTest method testOperatorSpecificMaxParallelismRespected.

@Test
public void testOperatorSpecificMaxParallelismRespected() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(4);
    DataSource<Integer> input = env.fromElements(0);
    BootstrapTransformation<Integer> transformation = OperatorTransformation.bootstrapWith(input).setMaxParallelism(1).transform(new ExampleStateBootstrapFunction());
    int maxParallelism = transformation.getMaxParallelism(4);
    DataSet<TaggedOperatorSubtaskState> result = transformation.writeOperatorSubtaskStates(OperatorIDGenerator.fromUid("uid"), new MemoryStateBackend(), new Path(), maxParallelism);
    Assert.assertEquals("The parallelism of a data set should be constrained my the savepoint max parallelism", 1, getParallelism(result));
}

Example 10 with TaggedOperatorSubtaskState

use of org.apache.flink.state.api.output.TaggedOperatorSubtaskState in project flink by apache.

the class BootstrapTransformation method writeOperatorSubtaskStates.

private MapPartitionOperator<T, TaggedOperatorSubtaskState> writeOperatorSubtaskStates(OperatorID operatorID, @Nullable StateBackend stateBackend, Configuration additionalConfig, Path savepointPath, int localMaxParallelism) {
    DataSet<T> input = dataSet;
    if (originalKeySelector != null) {
        input = dataSet.partitionCustom(new KeyGroupRangePartitioner(localMaxParallelism), hashKeySelector);
    }
    StreamOperator<TaggedOperatorSubtaskState> operator = factory.createOperator(System.currentTimeMillis(), savepointPath);
    operator = dataSet.clean(operator);
    final StreamConfig config = getConfig(operatorID, stateBackend, additionalConfig, operator);
    BoundedOneInputStreamTaskRunner<T> operatorRunner = new BoundedOneInputStreamTaskRunner<>(config, localMaxParallelism, timestamper);
    MapPartitionOperator<T, TaggedOperatorSubtaskState> subtaskStates = input.mapPartition(operatorRunner).name(operatorID.toHexString());
    if (operator instanceof BroadcastStateBootstrapOperator) {
        subtaskStates = subtaskStates.setParallelism(1);
    } else {
        int currentParallelism = getParallelism(subtaskStates);
        if (currentParallelism > localMaxParallelism) {
            subtaskStates.setParallelism(localMaxParallelism);
        }
    }
    return subtaskStates;
}