Examples with MiniCluster org.apache.flink.runtime.minicluster.MiniCluster used on opensource projects

Example 31 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project beam by apache.

the class FlinkRequiresStableInputTest method beforeClass.

@BeforeClass
public static void beforeClass() throws Exception {
    final int parallelism = 1;
    Configuration config = new Configuration();
    // Avoid port collision in parallel tests
    config.setInteger(RestOptions.PORT, 0);
    config.setString(CheckpointingOptions.STATE_BACKEND, "filesystem");
    // It is necessary to configure the checkpoint directory for the state backend,
    // even though we only create savepoints in this test.
    config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, "file://" + tempFolder.getRoot().getAbsolutePath());
    // Checkpoints will go into a subdirectory of this directory
    config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, "file://" + tempFolder.getRoot().getAbsolutePath());
    MiniClusterConfiguration clusterConfig = new MiniClusterConfiguration.Builder().setConfiguration(config).setNumTaskManagers(1).setNumSlotsPerTaskManager(1).build();
    flinkCluster = new MiniCluster(clusterConfig);
    flinkCluster.start();
    TestStreamEnvironment.setAsContext(flinkCluster, parallelism);
}

Example 32 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class AdaptiveSchedulerSlotSharingITCase method runJob.

private void runJob() throws Exception {
    final MiniCluster miniCluster = MINI_CLUSTER_RESOURCE.getMiniCluster();
    final JobGraph jobGraph = createJobGraphWithSlotSharingGroup();
    miniCluster.submitJob(jobGraph).join();
    final JobResult jobResult = miniCluster.requestJobResult(jobGraph.getJobID()).join();
    // this throws an exception if the job failed
    jobResult.toJobExecutionResult(getClass().getClassLoader());
    assertTrue(jobResult.isSuccess());
}

Example 33 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class AdaptiveSchedulerClusterITCase method testAutomaticScaleUp.

@Test
public void testAutomaticScaleUp() throws Exception {
    final MiniCluster miniCluster = miniClusterResource.getMiniCluster();
    int initialInstanceCount = NUMBER_SLOTS_PER_TASK_MANAGER * NUMBER_TASK_MANAGERS;
    int targetInstanceCount = initialInstanceCount + NUMBER_SLOTS_PER_TASK_MANAGER;
    final JobGraph jobGraph = createBlockingJobGraph(targetInstanceCount);
    log.info("Submitting job with parallelism of " + targetInstanceCount + ", to a cluster with only one TM.");
    miniCluster.submitJob(jobGraph).join();
    final CompletableFuture<JobResult> jobResultFuture = miniCluster.requestJobResult(jobGraph.getJobID());
    waitUntilParallelismForVertexReached(jobGraph.getJobID(), JOB_VERTEX_ID, initialInstanceCount);
    log.info("Start additional TaskManager to scale up to the full parallelism.");
    miniCluster.startTaskManager();
    log.info("Waiting until Invokable is running with higher parallelism");
    waitUntilParallelismForVertexReached(jobGraph.getJobID(), JOB_VERTEX_ID, targetInstanceCount);
    OnceBlockingNoOpInvokable.unblock();
    assertTrue(jobResultFuture.join().isSuccess());
}

Example 34 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class AdaptiveSchedulerSimpleITCase method testJobCancellationWhileRestartingSucceeds.

@Test
public void testJobCancellationWhileRestartingSucceeds() throws Exception {
    final long timeInRestartingState = 10000L;
    final MiniCluster miniCluster = MINI_CLUSTER_RESOURCE.getMiniCluster();
    final JobVertex alwaysFailingOperator = new JobVertex("Always failing operator");
    alwaysFailingOperator.setInvokableClass(AlwaysFailingInvokable.class);
    alwaysFailingOperator.setParallelism(1);
    final JobGraph jobGraph = JobGraphTestUtils.streamingJobGraph(alwaysFailingOperator);
    ExecutionConfig executionConfig = new ExecutionConfig();
    // configure a high delay between attempts: We'll stay in RESTARTING for 10 seconds.
    executionConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, timeInRestartingState));
    jobGraph.setExecutionConfig(executionConfig);
    miniCluster.submitJob(jobGraph).join();
    // wait until we are in RESTARTING state
    CommonTestUtils.waitUntilCondition(() -> miniCluster.getJobStatus(jobGraph.getJobID()).get() == JobStatus.RESTARTING, Deadline.fromNow(Duration.of(timeInRestartingState, ChronoUnit.MILLIS)), 5);
    // now cancel while in RESTARTING state
    miniCluster.cancelJob(jobGraph.getJobID()).get();
}

Example 35 with MiniCluster

use of org.apache.flink.runtime.minicluster.MiniCluster in project flink by apache.

the class MiniClusterResource method startMiniCluster.

private void startMiniCluster() throws Exception {
    final Configuration configuration = new Configuration(miniClusterResourceConfiguration.getConfiguration());
    configuration.setString(CoreOptions.TMP_DIRS, temporaryFolder.newFolder().getAbsolutePath());
    // enabled this by default
    if (!configuration.contains(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE)) {
        configuration.setBoolean(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE, true);
    }
    if (!configuration.contains(TaskManagerOptions.MANAGED_MEMORY_SIZE)) {
        configuration.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, DEFAULT_MANAGED_MEMORY_SIZE);
    }
    // set rest and rpc port to 0 to avoid clashes with concurrent MiniClusters
    configuration.setInteger(JobManagerOptions.PORT, 0);
    configuration.setString(RestOptions.BIND_PORT, "0");
    randomizeConfiguration(configuration);
    final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder().setConfiguration(configuration).setNumTaskManagers(miniClusterResourceConfiguration.getNumberTaskManagers()).setNumSlotsPerTaskManager(miniClusterResourceConfiguration.getNumberSlotsPerTaskManager()).setRpcServiceSharing(miniClusterResourceConfiguration.getRpcServiceSharing()).setHaServices(miniClusterResourceConfiguration.getHaServices()).build();
    miniCluster = new MiniCluster(miniClusterConfiguration, () -> Reference.borrowed(rpcSystem));
    miniCluster.start();
    final URI restAddress = miniCluster.getRestAddress().get();
    createClientConfiguration(restAddress);
}