Examples with MiniClusterClient org.apache.flink.client.program.MiniClusterClient used on opensource projects

Example 1 with MiniClusterClient

use of org.apache.flink.client.program.MiniClusterClient in project flink by apache.

the class DefaultSchedulerLocalRecoveryITCase method executeSchedulingTest.

private ArchivedExecutionGraph executeSchedulingTest(Configuration configuration, int parallelism) throws Exception {
    configuration.setString(RestOptions.BIND_PORT, "0");
    final long slotIdleTimeout = TIMEOUT;
    configuration.setLong(JobManagerOptions.SLOT_IDLE_TIMEOUT, slotIdleTimeout);
    configuration.set(TaskManagerOptions.TOTAL_FLINK_MEMORY, MemorySize.parse("64mb"));
    configuration.set(TaskManagerOptions.FRAMEWORK_HEAP_MEMORY, MemorySize.parse("16mb"));
    configuration.set(TaskManagerOptions.FRAMEWORK_OFF_HEAP_MEMORY, MemorySize.parse("16mb"));
    final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder().setConfiguration(configuration).setNumTaskManagers(parallelism).setNumSlotsPerTaskManager(1).build();
    try (MiniCluster miniCluster = new MiniCluster(miniClusterConfiguration)) {
        miniCluster.start();
        MiniClusterClient miniClusterClient = new MiniClusterClient(configuration, miniCluster);
        JobGraph jobGraph = createJobGraph(parallelism);
        // wait for the submission to succeed
        JobID jobId = miniClusterClient.submitJob(jobGraph).get(TIMEOUT, TimeUnit.SECONDS);
        // wait until all tasks running before triggering task failures
        waitUntilAllVerticesRunning(jobId, miniCluster);
        // kill one TM to trigger task failure and remove one existing slot
        CompletableFuture<Void> terminationFuture = miniCluster.terminateTaskManager(0);
        terminationFuture.get();
        // restart a taskmanager as a replacement for the killed one
        miniCluster.startTaskManager();
        // wait until all tasks running again
        waitUntilAllVerticesRunning(jobId, miniCluster);
        ArchivedExecutionGraph graph = miniCluster.getArchivedExecutionGraph(jobGraph.getJobID()).get();
        miniCluster.cancelJob(jobId).get();
        return graph;
    }
}

Example 2 with MiniClusterClient

use of org.apache.flink.client.program.MiniClusterClient in project flink by apache.

the class JobMasterStopWithSavepointITCase method setUpJobGraph.

private void setUpJobGraph(final Class<? extends TaskInvokable> invokable, final RestartStrategies.RestartStrategyConfiguration restartStrategy) throws Exception {
    finishingLatch = new OneShotLatch();
    invokeLatch = new CountDownLatch(PARALLELISM);
    numberOfRestarts = new CountDownLatch(2);
    checkpointsToWaitFor = new CountDownLatch(10);
    syncSavepointId.set(-1);
    savepointDirectory = temporaryFolder.newFolder().toPath();
    Assume.assumeTrue("ClusterClient is not an instance of MiniClusterClient", MINI_CLUSTER_RESOURCE.getClusterClient() instanceof MiniClusterClient);
    clusterClient = (MiniClusterClient) MINI_CLUSTER_RESOURCE.getClusterClient();
    final ExecutionConfig config = new ExecutionConfig();
    config.setRestartStrategy(restartStrategy);
    final JobVertex vertex = new JobVertex("testVertex");
    vertex.setInvokableClass(invokable);
    vertex.setParallelism(PARALLELISM);
    final JobCheckpointingSettings jobCheckpointingSettings = new JobCheckpointingSettings(new CheckpointCoordinatorConfiguration(CHECKPOINT_INTERVAL, 60_000, 10, 1, CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION, true, false, 0, 0), null);
    jobGraph = JobGraphBuilder.newStreamingJobGraphBuilder().setExecutionConfig(config).addJobVertex(vertex).setJobCheckpointingSettings(jobCheckpointingSettings).build();
    clusterClient.submitJob(jobGraph).get();
    assertTrue(invokeLatch.await(60, TimeUnit.SECONDS));
    waitForJob();
}

Example 3 with MiniClusterClient

use of org.apache.flink.client.program.MiniClusterClient in project flink by apache.

the class JobMasterTriggerSavepointITCase method setUpWithCheckpointInterval.

private void setUpWithCheckpointInterval(long checkpointInterval) throws Exception {
    invokeLatch = new CountDownLatch(1);
    triggerCheckpointLatch = new CountDownLatch(1);
    savepointDirectory = temporaryFolder.newFolder().toPath();
    Assume.assumeTrue("ClusterClient is not an instance of MiniClusterClient", MINI_CLUSTER_RESOURCE.getClusterClient() instanceof MiniClusterClient);
    clusterClient = (MiniClusterClient) MINI_CLUSTER_RESOURCE.getClusterClient();
    final JobVertex vertex = new JobVertex("testVertex");
    vertex.setInvokableClass(NoOpBlockingInvokable.class);
    vertex.setParallelism(1);
    final JobCheckpointingSettings jobCheckpointingSettings = new JobCheckpointingSettings(new CheckpointCoordinatorConfiguration(checkpointInterval, 60_000, 10, 1, CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION, true, false, 0, 0), null);
    jobGraph = JobGraphBuilder.newStreamingJobGraphBuilder().addJobVertex(vertex).setJobCheckpointingSettings(jobCheckpointingSettings).build();
    clusterClient.submitJob(jobGraph).get();
    assertTrue(invokeLatch.await(60, TimeUnit.SECONDS));
    waitForJob();
}

Example 4 with MiniClusterClient

use of org.apache.flink.client.program.MiniClusterClient in project flink by apache.

the class JobGraphRunningUtil method execute.

public static void execute(JobGraph jobGraph, Configuration configuration, int numTaskManagers, int numSlotsPerTaskManager) throws Exception {
    configuration.set(TaskManagerOptions.TOTAL_FLINK_MEMORY, MemorySize.parse("1g"));
    configuration.setString(RestOptions.BIND_PORT, "0");
    final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder().setConfiguration(configuration).setNumTaskManagers(numTaskManagers).setNumSlotsPerTaskManager(numSlotsPerTaskManager).build();
    try (MiniCluster miniCluster = new MiniCluster(miniClusterConfiguration)) {
        miniCluster.start();
        MiniClusterClient miniClusterClient = new MiniClusterClient(configuration, miniCluster);
        // wait for the submission to succeed
        JobID jobID = miniClusterClient.submitJob(jobGraph).get();
        JobResult jobResult = miniClusterClient.requestJobResult(jobID).get();
        if (jobResult.getSerializedThrowable().isPresent()) {
            throw new AssertionError(jobResult.getSerializedThrowable().get());
        }
    }
}

Example 5 with MiniClusterClient

use of org.apache.flink.client.program.MiniClusterClient in project flink by apache.

the class PipelinedRegionSchedulingITCase method executeSchedulingTest.

private JobResult executeSchedulingTest(JobGraph jobGraph, int numSlots, Configuration configuration) throws Exception {
    configuration.setString(RestOptions.BIND_PORT, "0");
    configuration.setLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT, 5000L);
    final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder().setConfiguration(configuration).setNumTaskManagers(1).setNumSlotsPerTaskManager(numSlots).build();
    try (MiniCluster miniCluster = new MiniCluster(miniClusterConfiguration)) {
        miniCluster.start();
        final MiniClusterClient miniClusterClient = new MiniClusterClient(configuration, miniCluster);
        // wait for the submission to succeed
        final JobID jobID = miniClusterClient.submitJob(jobGraph).get();
        final CompletableFuture<JobResult> resultFuture = miniClusterClient.requestJobResult(jobID);
        final JobResult jobResult = resultFuture.get();
        return jobResult;
    }
}