Examples with SlotSharingGroup org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup used on opensource projects

Example 11 with SlotSharingGroup

use of org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup in project flink by apache.

the class PipelinedRegionSchedulingITCase method createJobGraphWithThreeStages.

private JobGraph createJobGraphWithThreeStages(final int parallelism) {
    final SlotSharingGroup group1 = new SlotSharingGroup();
    final JobVertex source = new JobVertex("source");
    source.setInvokableClass(NoOpInvokable.class);
    source.setParallelism(parallelism);
    source.setSlotSharingGroup(group1);
    final SlotSharingGroup group2 = new SlotSharingGroup();
    final JobVertex map = new JobVertex("map");
    map.setInvokableClass(NoOpInvokable.class);
    map.setParallelism(parallelism);
    map.setSlotSharingGroup(group2);
    final SlotSharingGroup group3 = new SlotSharingGroup();
    final JobVertex sink = new JobVertex("sink");
    sink.setInvokableClass(OneTimeFailingReceiverWithPartitionException.class);
    sink.setParallelism(parallelism);
    sink.setSlotSharingGroup(group3);
    map.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE, ResultPartitionType.BLOCKING);
    sink.connectNewDataSetAsInput(map, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
    return JobGraphTestUtils.batchJobGraph(source, map, sink);
}

Example 12 with SlotSharingGroup

use of org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup in project flink by apache.

the class MiniClusterITCase method testJobWithSomeVerticesFailingDuringInstantiation.

@Test
public void testJobWithSomeVerticesFailingDuringInstantiation() throws Exception {
    final int parallelism = 11;
    final MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder().setNumTaskManagers(1).setNumSlotsPerTaskManager(parallelism).setConfiguration(getDefaultConfiguration()).build();
    try (final MiniCluster miniCluster = new MiniCluster(cfg)) {
        miniCluster.start();
        // putting sender and receiver vertex in the same slot sharing group is required
        // to ensure all senders can be deployed. Otherwise this case can fail if the
        // expected failing sender is not deployed.
        final SlotSharingGroup group = new SlotSharingGroup();
        final JobVertex sender = new JobVertex("Sender");
        sender.setInvokableClass(SometimesInstantiationErrorSender.class);
        sender.setParallelism(parallelism);
        sender.setSlotSharingGroup(group);
        // set failing senders
        SometimesInstantiationErrorSender.configFailingSenders(parallelism);
        final JobVertex receiver = new JobVertex("Receiver");
        receiver.setInvokableClass(Receiver.class);
        receiver.setParallelism(parallelism);
        receiver.setSlotSharingGroup(group);
        receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
        final JobGraph jobGraph = JobGraphTestUtils.streamingJobGraph(sender, receiver);
        try {
            miniCluster.executeJobBlocking(jobGraph);
            fail("Job should fail.");
        } catch (JobExecutionException e) {
            assertTrue(findThrowable(e, Exception.class).isPresent());
            assertTrue(findThrowableWithMessage(e, "Test exception in constructor").isPresent());
        }
    }
}

Example 13 with SlotSharingGroup

use of org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup in project flink by apache.

the class MiniClusterITCase method testSchedulingAllAtOnce.

@Test
public void testSchedulingAllAtOnce() throws Exception {
    final int parallelism = 11;
    final MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder().setNumTaskManagers(1).setNumSlotsPerTaskManager(parallelism).setConfiguration(getDefaultConfiguration()).build();
    try (final MiniCluster miniCluster = new MiniCluster(cfg)) {
        miniCluster.start();
        final JobVertex sender = new JobVertex("Sender");
        sender.setInvokableClass(Sender.class);
        sender.setParallelism(parallelism);
        final JobVertex forwarder = new JobVertex("Forwarder");
        forwarder.setInvokableClass(Forwarder.class);
        forwarder.setParallelism(parallelism);
        final JobVertex receiver = new JobVertex("Receiver");
        receiver.setInvokableClass(AgnosticReceiver.class);
        receiver.setParallelism(parallelism);
        final SlotSharingGroup sharingGroup = new SlotSharingGroup();
        sender.setSlotSharingGroup(sharingGroup);
        forwarder.setSlotSharingGroup(sharingGroup);
        receiver.setSlotSharingGroup(sharingGroup);
        forwarder.connectNewDataSetAsInput(sender, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
        receiver.connectNewDataSetAsInput(forwarder, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
        final JobGraph jobGraph = JobGraphTestUtils.streamingJobGraph(sender, forwarder, receiver);
        miniCluster.executeJobBlocking(jobGraph);
    }
}

Example 14 with SlotSharingGroup

use of org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup in project flink by apache.

the class MiniClusterITCase method testJobWithAnOccasionallyFailingSenderVertex.

@Test
public void testJobWithAnOccasionallyFailingSenderVertex() throws Exception {
    final int parallelism = 11;
    final MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder().setNumTaskManagers(1).setNumSlotsPerTaskManager(parallelism).setConfiguration(getDefaultConfiguration()).build();
    try (final MiniCluster miniCluster = new MiniCluster(cfg)) {
        miniCluster.start();
        // putting sender and receiver vertex in the same slot sharing group is required
        // to ensure all senders can be deployed. Otherwise this case can fail if the
        // expected failing sender is not deployed.
        final SlotSharingGroup group = new SlotSharingGroup();
        final JobVertex sender = new JobVertex("Sender");
        sender.setInvokableClass(SometimesExceptionSender.class);
        sender.setParallelism(parallelism);
        sender.setSlotSharingGroup(group);
        // set failing senders
        SometimesExceptionSender.configFailingSenders(parallelism);
        final JobVertex receiver = new JobVertex("Receiver");
        receiver.setInvokableClass(Receiver.class);
        receiver.setParallelism(parallelism);
        receiver.setSlotSharingGroup(group);
        receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
        final JobGraph jobGraph = JobGraphTestUtils.streamingJobGraph(sender, receiver);
        try {
            miniCluster.executeJobBlocking(jobGraph);
            fail("Job should fail.");
        } catch (JobExecutionException e) {
            assertTrue(findThrowable(e, Exception.class).isPresent());
            assertTrue(findThrowableWithMessage(e, "Test exception").isPresent());
        }
    }
}

Example 15 with SlotSharingGroup

use of org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup in project flink by apache.

the class LocalInputPreferredSlotSharingStrategyTest method testSetSlotSharingGroupResource.

@Test
public void testSetSlotSharingGroupResource() {
    final SlotSharingGroup slotSharingGroup1 = new SlotSharingGroup();
    final ResourceProfile resourceProfile1 = ResourceProfile.fromResources(1, 10);
    slotSharingGroup1.addVertexToGroup(JOB_VERTEX_ID_1);
    slotSharingGroup1.setResourceProfile(resourceProfile1);
    final SlotSharingGroup slotSharingGroup2 = new SlotSharingGroup();
    final ResourceProfile resourceProfile2 = ResourceProfile.fromResources(2, 20);
    slotSharingGroup2.addVertexToGroup(JOB_VERTEX_ID_2);
    slotSharingGroup2.setResourceProfile(resourceProfile2);
    final Set<SlotSharingGroup> slotSharingGroups = new HashSet<>();
    slotSharingGroups.add(slotSharingGroup1);
    slotSharingGroups.add(slotSharingGroup2);
    final SlotSharingStrategy strategy = new LocalInputPreferredSlotSharingStrategy(topology, slotSharingGroups, Collections.emptySet());
    assertThat(strategy.getExecutionSlotSharingGroups(), hasSize(4));
    assertThat(strategy.getExecutionSlotSharingGroup(ev11.getId()).getResourceProfile(), equalTo(resourceProfile1));
    assertThat(strategy.getExecutionSlotSharingGroup(ev12.getId()).getResourceProfile(), equalTo(resourceProfile1));
    assertThat(strategy.getExecutionSlotSharingGroup(ev21.getId()).getResourceProfile(), equalTo(resourceProfile2));
    assertThat(strategy.getExecutionSlotSharingGroup(ev22.getId()).getResourceProfile(), equalTo(resourceProfile2));
}