Examples with SlotSharingGroup org.apache.flink.api.common.operators.SlotSharingGroup used on opensource projects

Example 6 with SlotSharingGroup

use of org.apache.flink.api.common.operators.SlotSharingGroup in project flink by apache.

the class StreamGraphGeneratorTest method testConfigureSlotSharingGroupResource.

@Test
public void testConfigureSlotSharingGroupResource() {
    final SlotSharingGroup ssg1 = SlotSharingGroup.newBuilder("ssg1").setCpuCores(1).setTaskHeapMemoryMB(100).build();
    final SlotSharingGroup ssg2 = SlotSharingGroup.newBuilder("ssg2").setCpuCores(2).setTaskHeapMemoryMB(200).build();
    final SlotSharingGroup ssg3 = SlotSharingGroup.newBuilder(StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP).setCpuCores(3).setTaskHeapMemoryMB(300).build();
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    final DataStream<Integer> source = env.fromElements(1).slotSharingGroup("ssg1");
    source.map(value -> value).slotSharingGroup(ssg2).map(value -> value * 2).map(value -> value * 3).slotSharingGroup(SlotSharingGroup.newBuilder("ssg4").build()).map(value -> value * 4).slotSharingGroup(ssg3).addSink(new DiscardingSink<>()).slotSharingGroup(ssg1);
    final StreamGraph streamGraph = env.getStreamGraph();
    assertThat(streamGraph.getSlotSharingGroupResource("ssg1").get(), is(ResourceProfile.fromResources(1, 100)));
    assertThat(streamGraph.getSlotSharingGroupResource("ssg2").get(), is(ResourceProfile.fromResources(2, 200)));
    assertThat(streamGraph.getSlotSharingGroupResource(StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP).get(), is(ResourceProfile.fromResources(3, 300)));
}

Example 7 with SlotSharingGroup

use of org.apache.flink.api.common.operators.SlotSharingGroup in project flink by apache.

the class AdaptiveBatchSchedulerITCase method executeJob.

private void executeJob(Boolean isFineGrained) throws Exception {
    final Configuration configuration = new Configuration();
    configuration.setString(RestOptions.BIND_PORT, "0");
    configuration.setLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT, 5000L);
    configuration.set(JobManagerOptions.SCHEDULER, JobManagerOptions.SchedulerType.AdaptiveBatch);
    configuration.setInteger(JobManagerOptions.ADAPTIVE_BATCH_SCHEDULER_MAX_PARALLELISM, DEFAULT_MAX_PARALLELISM);
    configuration.set(TaskManagerOptions.MEMORY_SEGMENT_SIZE, MemorySize.parse("4kb"));
    configuration.set(TaskManagerOptions.NUM_TASK_SLOTS, 1);
    if (isFineGrained) {
        configuration.set(ClusterOptions.ENABLE_FINE_GRAINED_RESOURCE_MANAGEMENT, true);
        configuration.set(ClusterOptions.FINE_GRAINED_SHUFFLE_MODE_ALL_BLOCKING, true);
    }
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(configuration);
    env.setParallelism(-1);
    env.setRuntimeMode(RuntimeExecutionMode.BATCH);
    List<SlotSharingGroup> slotSharingGroups = new ArrayList<>();
    for (int i = 0; i < 3; ++i) {
        SlotSharingGroup group;
        if (isFineGrained) {
            group = SlotSharingGroup.newBuilder("group" + i).setCpuCores(1.0).setTaskHeapMemory(MemorySize.parse("100m")).build();
        } else {
            group = SlotSharingGroup.newBuilder("group" + i).build();
        }
        slotSharingGroups.add(group);
    }
    final DataStream<Long> source1 = env.fromSequence(0, NUMBERS_TO_PRODUCE - 1).setParallelism(SOURCE_PARALLELISM_1).name("source1").slotSharingGroup(slotSharingGroups.get(0));
    final DataStream<Long> source2 = env.fromSequence(0, NUMBERS_TO_PRODUCE - 1).setParallelism(SOURCE_PARALLELISM_2).name("source2").slotSharingGroup(slotSharingGroups.get(1));
    source1.union(source2).rescale().map(new NumberCounter()).name("map").slotSharingGroup(slotSharingGroups.get(2));
    env.execute();
}