Examples with MiniCluster org.apache.flink.runtime.minicluster.MiniCluster used on opensource projects

Example 1 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class JobManagerWatermarkTrackerTest method setUp.

@BeforeClass
public static void setUp() throws Exception {
    final Configuration config = new Configuration();
    config.setInteger(RestOptions.PORT, 0);
    final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder().setConfiguration(config).setNumTaskManagers(1).setNumSlotsPerTaskManager(1).build();
    flink = new MiniCluster(miniClusterConfiguration);
    flink.start();
}

Example 2 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class JobDispatcherITCase method testRecoverFromCheckpointAfterLosingAndRegainingLeadership.

@Test
public void testRecoverFromCheckpointAfterLosingAndRegainingLeadership(@TempDir Path tmpPath) throws Exception {
    final Deadline deadline = Deadline.fromNow(TIMEOUT);
    final Configuration configuration = new Configuration();
    configuration.set(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.name());
    final TestingMiniClusterConfiguration clusterConfiguration = TestingMiniClusterConfiguration.newBuilder().setConfiguration(configuration).build();
    final EmbeddedHaServicesWithLeadershipControl haServices = new EmbeddedHaServicesWithLeadershipControl(TestingUtils.defaultExecutor());
    final Configuration newConfiguration = new Configuration(clusterConfiguration.getConfiguration());
    final long checkpointInterval = 100;
    final JobID jobID = generateAndPersistJobGraph(newConfiguration, checkpointInterval, tmpPath);
    final TestingMiniCluster.Builder clusterBuilder = TestingMiniCluster.newBuilder(clusterConfiguration).setHighAvailabilityServicesSupplier(() -> haServices).setDispatcherResourceManagerComponentFactorySupplier(createJobModeDispatcherResourceManagerComponentFactorySupplier(newConfiguration));
    AtLeastOneCheckpointInvokable.reset();
    try (final MiniCluster cluster = clusterBuilder.build()) {
        // start mini cluster and submit the job
        cluster.start();
        AtLeastOneCheckpointInvokable.atLeastOneCheckpointCompleted.await();
        final CompletableFuture<JobResult> firstJobResult = cluster.requestJobResult(jobID);
        haServices.revokeDispatcherLeadership();
        // make sure the leadership is revoked to avoid race conditions
        Assertions.assertEquals(ApplicationStatus.UNKNOWN, firstJobResult.get().getApplicationStatus());
        haServices.grantDispatcherLeadership();
        // job is suspended, wait until it's running
        awaitJobStatus(cluster, jobID, JobStatus.RUNNING, deadline);
        CommonTestUtils.waitUntilCondition(() -> cluster.getArchivedExecutionGraph(jobID).get().getCheckpointStatsSnapshot().getLatestRestoredCheckpoint() != null, deadline);
    }
}

Example 3 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class FileSinkITBase method testFileSink.

@Test
public void testFileSink() throws Exception {
    String path = TEMPORARY_FOLDER.newFolder().getAbsolutePath();
    JobGraph jobGraph = createJobGraph(path);
    final Configuration config = new Configuration();
    config.setString(RestOptions.BIND_PORT, "18081-19000");
    final MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder().setNumTaskManagers(1).setNumSlotsPerTaskManager(4).setConfiguration(config).build();
    try (MiniCluster miniCluster = new MiniCluster(cfg)) {
        miniCluster.start();
        miniCluster.executeJobBlocking(jobGraph);
    }
    IntegerFileSinkTestDataUtils.checkIntegerSequenceSinkOutput(path, NUM_RECORDS, NUM_BUCKETS, NUM_SOURCES);
}

Example 4 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class ApplicationDispatcherBootstrapITCase method testDispatcherRecoversAfterLosingAndRegainingLeadership.

@Test
public void testDispatcherRecoversAfterLosingAndRegainingLeadership() throws Exception {
    final String blockId = UUID.randomUUID().toString();
    final Deadline deadline = Deadline.fromNow(TIMEOUT);
    final Configuration configuration = new Configuration();
    configuration.set(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.name());
    configuration.set(DeploymentOptions.TARGET, EmbeddedExecutor.NAME);
    configuration.set(ClientOptions.CLIENT_RETRY_PERIOD, Duration.ofMillis(100));
    final TestingMiniClusterConfiguration clusterConfiguration = TestingMiniClusterConfiguration.newBuilder().setConfiguration(configuration).build();
    final EmbeddedHaServicesWithLeadershipControl haServices = new EmbeddedHaServicesWithLeadershipControl(TestingUtils.defaultExecutor());
    final TestingMiniCluster.Builder clusterBuilder = TestingMiniCluster.newBuilder(clusterConfiguration).setHighAvailabilityServicesSupplier(() -> haServices).setDispatcherResourceManagerComponentFactorySupplier(createApplicationModeDispatcherResourceManagerComponentFactorySupplier(clusterConfiguration.getConfiguration(), BlockingJob.getProgram(blockId)));
    try (final MiniCluster cluster = clusterBuilder.build()) {
        // start mini cluster and submit the job
        cluster.start();
        // wait until job is running
        awaitJobStatus(cluster, ApplicationDispatcherBootstrap.ZERO_JOB_ID, JobStatus.RUNNING, deadline);
        // make sure the operator is actually running
        BlockingJob.awaitRunning(blockId);
        final CompletableFuture<JobResult> firstJobResult = cluster.requestJobResult(ApplicationDispatcherBootstrap.ZERO_JOB_ID);
        haServices.revokeDispatcherLeadership();
        // make sure the leadership is revoked to avoid race conditions
        assertThat(firstJobResult.get()).extracting(JobResult::getApplicationStatus).isEqualTo(ApplicationStatus.UNKNOWN);
        haServices.grantDispatcherLeadership();
        // job is suspended, wait until it's running
        awaitJobStatus(cluster, ApplicationDispatcherBootstrap.ZERO_JOB_ID, JobStatus.RUNNING, deadline);
        // unblock processing so the job can finish
        BlockingJob.unblock(blockId);
        // and wait for it to actually finish
        final JobResult secondJobResult = cluster.requestJobResult(ApplicationDispatcherBootstrap.ZERO_JOB_ID).get();
        assertThat(secondJobResult.isSuccess()).isTrue();
        assertThat(secondJobResult.getApplicationStatus()).isEqualTo(ApplicationStatus.SUCCEEDED);
        // the cluster should shut down automatically once the application completes
        awaitClusterStopped(cluster, deadline);
    } finally {
        BlockingJob.cleanUp(blockId);
    }
}

Example 5 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class FileSinkMigrationITCase method loadSavepointAndExecute.

private void loadSavepointAndExecute(MiniClusterConfiguration cfg, JobGraph jobGraph, String savepointPath) throws Exception {
    jobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath, false));
    try (MiniCluster miniCluster = new MiniCluster(cfg)) {
        miniCluster.start();
        miniCluster.executeJobBlocking(jobGraph);
    }
}