Examples with SlotSharingGroup org.apache.flink.api.common.operators.SlotSharingGroup used on opensource projects

Example 1 with SlotSharingGroup

use of org.apache.flink.api.common.operators.SlotSharingGroup in project flink by apache.

the class StreamGraphGeneratorTest method testSetSlotSharingResource.

@Test
public void testSetSlotSharingResource() {
    final String slotSharingGroup1 = "a";
    final String slotSharingGroup2 = "b";
    final ResourceProfile resourceProfile1 = ResourceProfile.fromResources(1, 10);
    final ResourceProfile resourceProfile2 = ResourceProfile.fromResources(2, 20);
    final ResourceProfile resourceProfile3 = ResourceProfile.fromResources(3, 30);
    final Map<String, ResourceProfile> slotSharingGroupResource = new HashMap<>();
    slotSharingGroupResource.put(slotSharingGroup1, resourceProfile1);
    slotSharingGroupResource.put(slotSharingGroup2, resourceProfile2);
    slotSharingGroupResource.put(StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP, resourceProfile3);
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    final DataStream<Integer> sourceDataStream = env.fromElements(1, 2, 3).slotSharingGroup(slotSharingGroup1);
    final DataStream<Integer> mapDataStream1 = sourceDataStream.map(x -> x + 1).slotSharingGroup(slotSharingGroup2);
    final DataStream<Integer> mapDataStream2 = mapDataStream1.map(x -> x * 2);
    final List<Transformation<?>> transformations = new ArrayList<>();
    transformations.add(sourceDataStream.getTransformation());
    transformations.add(mapDataStream1.getTransformation());
    transformations.add(mapDataStream2.getTransformation());
    // all stream nodes share default group by default
    final StreamGraph streamGraph = new StreamGraphGenerator(transformations, env.getConfig(), env.getCheckpointConfig()).setSlotSharingGroupResource(slotSharingGroupResource).generate();
    assertThat(streamGraph.getSlotSharingGroupResource(slotSharingGroup1).get(), equalTo(resourceProfile1));
    assertThat(streamGraph.getSlotSharingGroupResource(slotSharingGroup2).get(), equalTo(resourceProfile2));
    assertThat(streamGraph.getSlotSharingGroupResource(StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP).get(), equalTo(resourceProfile3));
}

Example 2 with SlotSharingGroup

use of org.apache.flink.api.common.operators.SlotSharingGroup in project flink by apache.

the class StreamGraphGeneratorTest method testConflictSlotSharingGroup.

@Test(expected = IllegalArgumentException.class)
public void testConflictSlotSharingGroup() {
    final SlotSharingGroup ssg = SlotSharingGroup.newBuilder("ssg").setCpuCores(1).setTaskHeapMemoryMB(100).build();
    final SlotSharingGroup ssgConflict = SlotSharingGroup.newBuilder("ssg").setCpuCores(2).setTaskHeapMemoryMB(200).build();
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    final DataStream<Integer> source = env.fromElements(1).slotSharingGroup(ssg);
    source.map(value -> value).slotSharingGroup(ssgConflict).addSink(new DiscardingSink<>()).slotSharingGroup(ssgConflict);
    env.getStreamGraph();
}

Example 3 with SlotSharingGroup

use of org.apache.flink.api.common.operators.SlotSharingGroup in project flink by apache.

the class SlotSharingGroupUtilsTest method testCovertToResourceSpec.

@Test
public void testCovertToResourceSpec() {
    final ExternalResource gpu = new ExternalResource("gpu", 1);
    final ResourceSpec resourceSpec = ResourceSpec.newBuilder(1.0, 100).setManagedMemoryMB(200).setTaskOffHeapMemoryMB(300).setExtendedResource(gpu).build();
    final SlotSharingGroup slotSharingGroup1 = SlotSharingGroup.newBuilder("ssg").setCpuCores(resourceSpec.getCpuCores().getValue().doubleValue()).setTaskHeapMemory(resourceSpec.getTaskHeapMemory()).setTaskOffHeapMemory(resourceSpec.getTaskOffHeapMemory()).setManagedMemory(resourceSpec.getManagedMemory()).setExternalResource(gpu.getName(), gpu.getValue().doubleValue()).build();
    final SlotSharingGroup slotSharingGroup2 = SlotSharingGroup.newBuilder("ssg").build();
    assertThat(SlotSharingGroupUtils.extractResourceSpec(slotSharingGroup1), is(resourceSpec));
    assertThat(SlotSharingGroupUtils.extractResourceSpec(slotSharingGroup2), is(ResourceSpec.UNKNOWN));
}

Example 4 with SlotSharingGroup

use of org.apache.flink.api.common.operators.SlotSharingGroup in project flink by apache.

the class StreamExecutionEnvironmentTest method testRegisterSlotSharingGroupConflict.

@Test(expected = IllegalArgumentException.class)
public void testRegisterSlotSharingGroupConflict() {
    final SlotSharingGroup ssg = SlotSharingGroup.newBuilder("ssg1").setCpuCores(1).setTaskHeapMemoryMB(100).build();
    final SlotSharingGroup ssgConflict = SlotSharingGroup.newBuilder("ssg1").setCpuCores(2).setTaskHeapMemoryMB(200).build();
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.registerSlotSharingGroup(ssg);
    final DataStream<Integer> source = env.fromElements(1).slotSharingGroup("ssg1");
    source.map(value -> value).slotSharingGroup(ssgConflict).addSink(new DiscardingSink<>());
    env.getStreamGraph();
}

Example 5 with SlotSharingGroup

use of org.apache.flink.api.common.operators.SlotSharingGroup in project flink by apache.

the class StreamExecutionEnvironmentTest method testRegisterSlotSharingGroup.

@Test
public void testRegisterSlotSharingGroup() {
    final SlotSharingGroup ssg1 = SlotSharingGroup.newBuilder("ssg1").setCpuCores(1).setTaskHeapMemoryMB(100).build();
    final SlotSharingGroup ssg2 = SlotSharingGroup.newBuilder("ssg2").setCpuCores(2).setTaskHeapMemoryMB(200).build();
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.registerSlotSharingGroup(ssg1);
    env.registerSlotSharingGroup(ssg2);
    env.registerSlotSharingGroup(SlotSharingGroup.newBuilder("ssg3").build());
    final DataStream<Integer> source = env.fromElements(1).slotSharingGroup("ssg1");
    source.map(value -> value).slotSharingGroup(ssg2).addSink(new DiscardingSink<>());
    final StreamGraph streamGraph = env.getStreamGraph();
    assertThat(streamGraph.getSlotSharingGroupResource("ssg1").get(), is(ResourceProfile.fromResources(1, 100)));
    assertThat(streamGraph.getSlotSharingGroupResource("ssg2").get(), is(ResourceProfile.fromResources(2, 200)));
    assertFalse(streamGraph.getSlotSharingGroupResource("ssg3").isPresent());
}