Examples with MiniCluster org.apache.flink.runtime.minicluster.MiniCluster used on opensource projects

Example 26 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class ApplicationDispatcherBootstrapITCase method testDirtyJobResultRecoveryInApplicationMode.

@Test
public void testDirtyJobResultRecoveryInApplicationMode() throws Exception {
    final Deadline deadline = Deadline.fromNow(TIMEOUT);
    final Configuration configuration = new Configuration();
    configuration.set(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.name());
    configuration.set(DeploymentOptions.TARGET, EmbeddedExecutor.NAME);
    configuration.set(ClientOptions.CLIENT_RETRY_PERIOD, Duration.ofMillis(100));
    final TestingMiniClusterConfiguration clusterConfiguration = TestingMiniClusterConfiguration.newBuilder().setConfiguration(configuration).build();
    // having a dirty entry in the JobResultStore should make the ApplicationDispatcherBootstrap
    // implementation fail to submit the job
    final JobResultStore jobResultStore = new EmbeddedJobResultStore();
    jobResultStore.createDirtyResult(new JobResultEntry(TestingJobResultStore.createSuccessfulJobResult(ApplicationDispatcherBootstrap.ZERO_JOB_ID)));
    final EmbeddedHaServicesWithLeadershipControl haServices = new EmbeddedHaServicesWithLeadershipControl(TestingUtils.defaultExecutor()) {

        @Override
        public JobResultStore getJobResultStore() {
            return jobResultStore;
        }
    };
    final TestingMiniCluster.Builder clusterBuilder = TestingMiniCluster.newBuilder(clusterConfiguration).setHighAvailabilityServicesSupplier(() -> haServices).setDispatcherResourceManagerComponentFactorySupplier(createApplicationModeDispatcherResourceManagerComponentFactorySupplier(clusterConfiguration.getConfiguration(), ErrorHandlingSubmissionJob.createPackagedProgram()));
    try (final MiniCluster cluster = clusterBuilder.build()) {
        // start mini cluster and submit the job
        cluster.start();
        // the cluster should shut down automatically once the application completes
        awaitClusterStopped(cluster, deadline);
    }
    FlinkAssertions.assertThatChainOfCauses(ErrorHandlingSubmissionJob.getSubmissionException()).as("The job's main method shouldn't have been succeeded due to a DuplicateJobSubmissionException.").hasAtLeastOneElementOfType(DuplicateJobSubmissionException.class);
    assertThat(jobResultStore.hasDirtyJobResultEntry(ApplicationDispatcherBootstrap.ZERO_JOB_ID)).isFalse();
    assertThat(jobResultStore.hasCleanJobResultEntry(ApplicationDispatcherBootstrap.ZERO_JOB_ID)).isTrue();
}

Example 27 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class ApplicationDispatcherBootstrapITCase method testSubmitFailedJobOnApplicationError.

@Test
public void testSubmitFailedJobOnApplicationError() throws Exception {
    final Deadline deadline = Deadline.fromNow(TIMEOUT);
    final JobID jobId = new JobID();
    final Configuration configuration = new Configuration();
    configuration.set(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.name());
    configuration.set(DeploymentOptions.TARGET, EmbeddedExecutor.NAME);
    configuration.set(ClientOptions.CLIENT_RETRY_PERIOD, Duration.ofMillis(100));
    configuration.set(DeploymentOptions.SHUTDOWN_ON_APPLICATION_FINISH, false);
    configuration.set(DeploymentOptions.SUBMIT_FAILED_JOB_ON_APPLICATION_ERROR, true);
    configuration.set(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, jobId.toHexString());
    final TestingMiniClusterConfiguration clusterConfiguration = TestingMiniClusterConfiguration.newBuilder().setConfiguration(configuration).build();
    final EmbeddedHaServicesWithLeadershipControl haServices = new EmbeddedHaServicesWithLeadershipControl(TestingUtils.defaultExecutor());
    final TestingMiniCluster.Builder clusterBuilder = TestingMiniCluster.newBuilder(clusterConfiguration).setHighAvailabilityServicesSupplier(() -> haServices).setDispatcherResourceManagerComponentFactorySupplier(createApplicationModeDispatcherResourceManagerComponentFactorySupplier(clusterConfiguration.getConfiguration(), FailingJob.getProgram()));
    try (final MiniCluster cluster = clusterBuilder.build()) {
        // start mini cluster and submit the job
        cluster.start();
        // wait until the failed job has been submitted
        awaitJobStatus(cluster, jobId, JobStatus.FAILED, deadline);
        final ArchivedExecutionGraph graph = cluster.getArchivedExecutionGraph(jobId).get();
        assertThat(graph.getJobID()).isEqualTo(jobId);
        assertThat(graph.getJobName()).isEqualTo(ApplicationDispatcherBootstrap.FAILED_JOB_NAME);
        assertThat(graph.getFailureInfo()).isNotNull().extracting(ErrorInfo::getException).extracting(e -> e.deserializeError(Thread.currentThread().getContextClassLoader())).satisfies(e -> assertThat(e).isInstanceOf(ProgramInvocationException.class).hasRootCauseInstanceOf(RuntimeException.class).hasRootCauseMessage(FailingJob.EXCEPTION_MESSAGE));
    }
}

Example 28 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class PerJobMiniClusterFactory method submitJob.

/**
 * Starts a {@link MiniCluster} and submits a job.
 */
public CompletableFuture<JobClient> submitJob(JobGraph jobGraph, ClassLoader userCodeClassloader) throws Exception {
    MiniClusterConfiguration miniClusterConfig = getMiniClusterConfig(jobGraph.getMaximumParallelism());
    MiniCluster miniCluster = miniClusterFactory.apply(miniClusterConfig);
    miniCluster.start();
    return miniCluster.submitJob(jobGraph).thenApplyAsync(FunctionUtils.uncheckedFunction(submissionResult -> {
        org.apache.flink.client.ClientUtils.waitUntilJobInitializationFinished(() -> miniCluster.getJobStatus(submissionResult.getJobID()).get(), () -> miniCluster.requestJobResult(submissionResult.getJobID()).get(), userCodeClassloader);
        return submissionResult;
    })).thenApply(result -> new MiniClusterJobClient(result.getJobID(), miniCluster, userCodeClassloader, MiniClusterJobClient.JobFinalizationBehavior.SHUTDOWN_CLUSTER)).whenComplete((ignored, throwable) -> {
        if (throwable != null) {
            // We failed to create the JobClient and must shutdown to ensure
            // cleanup.
            shutDownCluster(miniCluster);
        }
    }).thenApply(Function.identity());
}

Example 29 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project beam by apache.

the class FlinkMiniClusterEntryPoint method main.

public static void main(String[] args) throws Exception {
    MiniClusterArgs miniClusterArgs = parseArgs(args);
    Configuration flinkConfig = new Configuration();
    flinkConfig.setInteger(RestOptions.PORT, miniClusterArgs.restPort);
    if (!miniClusterArgs.restBindAddress.isEmpty()) {
        flinkConfig.setString(RestOptions.BIND_ADDRESS, miniClusterArgs.restBindAddress);
    }
    MiniClusterConfiguration clusterConfig = new MiniClusterConfiguration.Builder().setConfiguration(flinkConfig).setNumTaskManagers(miniClusterArgs.numTaskManagers).setNumSlotsPerTaskManager(miniClusterArgs.numSlotsPerTaskManager).build();
    try (MiniCluster miniCluster = new MiniCluster(clusterConfig)) {
        miniCluster.start();
        System.out.println(String.format("Started Flink mini cluster (%s TaskManagers with %s task slots) with Rest API at %s", miniClusterArgs.numTaskManagers, miniClusterArgs.numSlotsPerTaskManager, miniCluster.getRestAddress()));
        Thread.sleep(Long.MAX_VALUE);
    }
}

Example 30 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project beam by apache.

the class FlinkSavepointTest method beforeClass.

@BeforeClass
public static void beforeClass() throws Exception {
    flinkJobExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    Configuration config = new Configuration();
    // Avoid port collision in parallel tests
    config.setInteger(RestOptions.PORT, 0);
    config.setString(CheckpointingOptions.STATE_BACKEND, "filesystem");
    String savepointPath = "file://" + tempFolder.getRoot().getAbsolutePath();
    LOG.info("Savepoints will be written to {}", savepointPath);
    // It is necessary to configure the checkpoint directory for the state backend,
    // even though we only create savepoints in this test.
    config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, savepointPath);
    // Checkpoints will go into a subdirectory of this directory
    config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointPath);
    MiniClusterConfiguration clusterConfig = new MiniClusterConfiguration.Builder().setConfiguration(config).setNumTaskManagers(2).setNumSlotsPerTaskManager(2).build();
    flinkCluster = new MiniCluster(clusterConfig);
    flinkCluster.start();
}