Examples with SlotSharingGroup org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup used on opensource projects

Example 21 with SlotSharingGroup

use of org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup in project flink by apache.

the class StreamingJobGraphGeneratorTest method testIteration.

/**
 * Test iteration job, check slot sharing group and co-location group.
 */
@Test
public void testIteration() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Integer> source = env.fromElements(1, 2, 3).name("source");
    IterativeStream<Integer> iteration = source.iterate(3000);
    iteration.name("iteration").setParallelism(2);
    DataStream<Integer> map = iteration.map(x -> x + 1).name("map").setParallelism(2);
    DataStream<Integer> filter = map.filter((x) -> false).name("filter").setParallelism(2);
    iteration.closeWith(filter).print();
    JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
    SlotSharingGroup slotSharingGroup = jobGraph.getVerticesAsArray()[0].getSlotSharingGroup();
    assertNotNull(slotSharingGroup);
    CoLocationGroup iterationSourceCoLocationGroup = null;
    CoLocationGroup iterationSinkCoLocationGroup = null;
    for (JobVertex jobVertex : jobGraph.getVertices()) {
        // all vertices have same slot sharing group by default
        assertEquals(slotSharingGroup, jobVertex.getSlotSharingGroup());
        // others have no co-location group by default
        if (jobVertex.getName().startsWith(StreamGraph.ITERATION_SOURCE_NAME_PREFIX)) {
            iterationSourceCoLocationGroup = jobVertex.getCoLocationGroup();
            assertTrue(iterationSourceCoLocationGroup.getVertexIds().contains(jobVertex.getID()));
        } else if (jobVertex.getName().startsWith(StreamGraph.ITERATION_SINK_NAME_PREFIX)) {
            iterationSinkCoLocationGroup = jobVertex.getCoLocationGroup();
            assertTrue(iterationSinkCoLocationGroup.getVertexIds().contains(jobVertex.getID()));
        } else {
            assertNull(jobVertex.getCoLocationGroup());
        }
    }
    assertNotNull(iterationSourceCoLocationGroup);
    assertNotNull(iterationSinkCoLocationGroup);
    assertEquals(iterationSourceCoLocationGroup, iterationSinkCoLocationGroup);
}

Example 22 with SlotSharingGroup

use of org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup in project flink by apache.

the class StreamingJobGraphGeneratorTest method testSlotSharingResourceConfiguration.

@Test
public void testSlotSharingResourceConfiguration() {
    final String slotSharingGroup1 = "slot-a";
    final String slotSharingGroup2 = "slot-b";
    final ResourceProfile resourceProfile1 = ResourceProfile.fromResources(1, 10);
    final ResourceProfile resourceProfile2 = ResourceProfile.fromResources(2, 20);
    final ResourceProfile resourceProfile3 = ResourceProfile.fromResources(3, 30);
    final Map<String, ResourceProfile> slotSharingGroupResource = new HashMap<>();
    slotSharingGroupResource.put(slotSharingGroup1, resourceProfile1);
    slotSharingGroupResource.put(slotSharingGroup2, resourceProfile2);
    slotSharingGroupResource.put(StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP, resourceProfile3);
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.fromElements(1, 2, 3).name(slotSharingGroup1).slotSharingGroup(slotSharingGroup1).map(x -> x + 1).name(slotSharingGroup2).slotSharingGroup(slotSharingGroup2).map(x -> x * x).name(StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP).slotSharingGroup(StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP);
    final StreamGraph streamGraph = env.getStreamGraph();
    streamGraph.setSlotSharingGroupResource(slotSharingGroupResource);
    final JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(streamGraph);
    int numVertex = 0;
    for (JobVertex jobVertex : jobGraph.getVertices()) {
        numVertex += 1;
        if (jobVertex.getName().contains(slotSharingGroup1)) {
            assertEquals(jobVertex.getSlotSharingGroup().getResourceProfile(), resourceProfile1);
        } else if (jobVertex.getName().contains(slotSharingGroup2)) {
            assertEquals(jobVertex.getSlotSharingGroup().getResourceProfile(), resourceProfile2);
        } else if (jobVertex.getName().contains(StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP)) {
            assertEquals(jobVertex.getSlotSharingGroup().getResourceProfile(), resourceProfile3);
        } else {
            fail();
        }
    }
    assertThat(numVertex, is(3));
}

Example 23 with SlotSharingGroup

use of org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup in project flink by apache.

the class StreamingJobGraphGeneratorTest method createJobGraphForManagedMemoryFractionTest.

private JobGraph createJobGraphForManagedMemoryFractionTest(final List<ResourceSpec> resourceSpecs, final List<Map<ManagedMemoryUseCase, Integer>> operatorScopeUseCaseWeights, final List<Set<ManagedMemoryUseCase>> slotScopeUseCases) throws Exception {
    final Method opMethod = getSetResourcesMethodAndSetAccessible(SingleOutputStreamOperator.class);
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    final DataStream<Integer> source = env.addSource(new ParallelSourceFunction<Integer>() {

        @Override
        public void run(SourceContext<Integer> ctx) {
        }

        @Override
        public void cancel() {
        }
    });
    opMethod.invoke(source, resourceSpecs.get(0));
    // CHAIN(source -> map1) in default slot sharing group
    final DataStream<Integer> map1 = source.map((MapFunction<Integer, Integer>) value -> value);
    opMethod.invoke(map1, resourceSpecs.get(1));
    // CHAIN(map2) in default slot sharing group
    final DataStream<Integer> map2 = map1.rebalance().map((MapFunction<Integer, Integer>) value -> value);
    opMethod.invoke(map2, resourceSpecs.get(2));
    // CHAIN(map3) in test slot sharing group
    final DataStream<Integer> map3 = map2.rebalance().map(value -> value).slotSharingGroup("test");
    opMethod.invoke(map3, resourceSpecs.get(3));
    declareManagedMemoryUseCaseForTranformation(source.getTransformation(), operatorScopeUseCaseWeights.get(0), slotScopeUseCases.get(0));
    declareManagedMemoryUseCaseForTranformation(map1.getTransformation(), operatorScopeUseCaseWeights.get(1), slotScopeUseCases.get(1));
    declareManagedMemoryUseCaseForTranformation(map2.getTransformation(), operatorScopeUseCaseWeights.get(2), slotScopeUseCases.get(2));
    declareManagedMemoryUseCaseForTranformation(map3.getTransformation(), operatorScopeUseCaseWeights.get(3), slotScopeUseCases.get(3));
    return StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
}

Example 24 with SlotSharingGroup

use of org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup in project flink by apache.

the class JobGraphTest method testGetCoLocationGroups.

@Test
public void testGetCoLocationGroups() {
    final JobVertex v1 = new JobVertex("1");
    final JobVertex v2 = new JobVertex("2");
    final JobVertex v3 = new JobVertex("3");
    final JobVertex v4 = new JobVertex("4");
    final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
    v1.setSlotSharingGroup(slotSharingGroup);
    v2.setSlotSharingGroup(slotSharingGroup);
    v1.setStrictlyCoLocatedWith(v2);
    final JobGraph jobGraph = JobGraphBuilder.newStreamingJobGraphBuilder().addJobVertices(Arrays.asList(v1, v2, v3, v4)).build();
    assertThat(jobGraph.getCoLocationGroups(), hasSize(1));
    final CoLocationGroup onlyCoLocationGroup = jobGraph.getCoLocationGroups().iterator().next();
    assertThat(onlyCoLocationGroup.getVertexIds(), containsInAnyOrder(v1.getID(), v2.getID()));
}

Example 25 with SlotSharingGroup

use of org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup in project flink by apache.

the class JobGraphTest method testGetSlotSharingGroups.

@Test
public void testGetSlotSharingGroups() {
    final JobVertex v1 = new JobVertex("1");
    final JobVertex v2 = new JobVertex("2");
    final JobVertex v3 = new JobVertex("3");
    final JobVertex v4 = new JobVertex("4");
    final SlotSharingGroup group1 = new SlotSharingGroup();
    v1.setSlotSharingGroup(group1);
    v2.setSlotSharingGroup(group1);
    final SlotSharingGroup group2 = new SlotSharingGroup();
    v3.setSlotSharingGroup(group2);
    v4.setSlotSharingGroup(group2);
    final JobGraph jobGraph = JobGraphBuilder.newStreamingJobGraphBuilder().addJobVertices(Arrays.asList(v1, v2, v3, v4)).build();
    assertThat(jobGraph.getSlotSharingGroups(), containsInAnyOrder(group1, group2));
}