use of org.apache.flink.runtime.minicluster.TestingMiniCluster in project flink by apache.
the class JobExecutionITCase method testCoLocationConstraintJobExecution.
/**
* Tests that tasks with a co-location constraint are scheduled in the same slots. In fact it
* also tests that consumers are scheduled wrt their input location if the co-location
* constraint is deactivated.
*/
@Test
public void testCoLocationConstraintJobExecution() throws Exception {
final int numSlotsPerTaskExecutor = 1;
final int numTaskExecutors = 3;
final int parallelism = numTaskExecutors * numSlotsPerTaskExecutor;
final JobGraph jobGraph = createJobGraph(parallelism);
final TestingMiniClusterConfiguration miniClusterConfiguration = TestingMiniClusterConfiguration.newBuilder().setNumSlotsPerTaskManager(numSlotsPerTaskExecutor).setNumTaskManagers(numTaskExecutors).setLocalCommunication(true).build();
try (TestingMiniCluster miniCluster = TestingMiniCluster.newBuilder(miniClusterConfiguration).build()) {
miniCluster.start();
miniCluster.submitJob(jobGraph).get();
final CompletableFuture<JobResult> jobResultFuture = miniCluster.requestJobResult(jobGraph.getJobID());
assertThat(jobResultFuture.get().isSuccess(), is(true));
}
}
use of org.apache.flink.runtime.minicluster.TestingMiniCluster in project flink by apache.
the class ZooKeeperLeaderElectionITCase method testJobExecutionOnClusterWithLeaderChange.
/**
* Tests that a job can be executed after a new leader has been elected. For all except for the
* last leader, the job is blocking. The JobManager will be terminated while executing the
* blocking job. Once only one JobManager is left, it is checked that a non-blocking can be
* successfully executed.
*/
@Test
@Ignore("FLINK-25235")
public void testJobExecutionOnClusterWithLeaderChange() throws Exception {
final int numDispatchers = 3;
final int numTMs = 2;
final int numSlotsPerTM = 2;
final Configuration configuration = ZooKeeperTestUtils.createZooKeeperHAConfig(zkServer.getConnectString(), tempFolder.newFolder().getAbsolutePath());
// speed up refused registration retries
configuration.setLong(ClusterOptions.REFUSED_REGISTRATION_DELAY, 50L);
final TestingMiniClusterConfiguration miniClusterConfiguration = TestingMiniClusterConfiguration.newBuilder().setConfiguration(configuration).setNumberDispatcherResourceManagerComponents(numDispatchers).setNumTaskManagers(numTMs).setNumSlotsPerTaskManager(numSlotsPerTM).build();
final Deadline timeout = Deadline.fromNow(TEST_TIMEOUT);
try (TestingMiniCluster miniCluster = TestingMiniCluster.newBuilder(miniClusterConfiguration).build();
final CuratorFrameworkWithUnhandledErrorListener curatorFramework = ZooKeeperUtils.startCuratorFramework(configuration, exception -> fail("Fatal error in curator framework."))) {
// We need to watch for resource manager leader changes to avoid race conditions.
final DefaultLeaderRetrievalService resourceManagerLeaderRetrieval = ZooKeeperUtils.createLeaderRetrievalService(curatorFramework.asCuratorFramework(), ZooKeeperUtils.getLeaderPathForResourceManager(), configuration);
@SuppressWarnings("unchecked") final CompletableFuture<String>[] resourceManagerLeaderFutures = (CompletableFuture<String>[]) new CompletableFuture[numDispatchers];
for (int i = 0; i < numDispatchers; i++) {
resourceManagerLeaderFutures[i] = new CompletableFuture<>();
}
resourceManagerLeaderRetrieval.start(new TestLeaderRetrievalListener(resourceManagerLeaderFutures));
miniCluster.start();
final int parallelism = numTMs * numSlotsPerTM;
JobGraph jobGraph = createJobGraph(parallelism);
miniCluster.submitJob(jobGraph).get();
String previousLeaderAddress = null;
for (int i = 0; i < numDispatchers - 1; i++) {
final DispatcherGateway leaderDispatcherGateway = getNextLeadingDispatcherGateway(miniCluster, previousLeaderAddress, timeout);
// Make sure resource manager has also changed leadership.
resourceManagerLeaderFutures[i].get();
previousLeaderAddress = leaderDispatcherGateway.getAddress();
awaitRunningStatus(leaderDispatcherGateway, jobGraph, timeout);
leaderDispatcherGateway.shutDownCluster();
}
final DispatcherGateway leaderDispatcherGateway = getNextLeadingDispatcherGateway(miniCluster, previousLeaderAddress, timeout);
// Make sure resource manager has also changed leadership.
resourceManagerLeaderFutures[numDispatchers - 1].get();
awaitRunningStatus(leaderDispatcherGateway, jobGraph, timeout);
CompletableFuture<JobResult> jobResultFuture = leaderDispatcherGateway.requestJobResult(jobGraph.getJobID(), RPC_TIMEOUT);
BlockingOperator.unblock();
assertThat(jobResultFuture.get().isSuccess(), is(true));
resourceManagerLeaderRetrieval.stop();
}
}
Aggregations