Examples with JobClient org.apache.flink.core.execution.JobClient used on opensource projects

Example 36 with JobClient

use of org.apache.flink.core.execution.JobClient in project flink by apache.

the class KafkaTableITCase method testPerPartitionWatermarkWithIdleSource.

@Test
public void testPerPartitionWatermarkWithIdleSource() throws Exception {
    // we always use a different topic name for each parameterized topic,
    // in order to make sure the topic can be created.
    final String topic = "idle_partition_watermark_topic_" + format;
    createTestTopic(topic, 4, 1);
    // ---------- Produce an event time stream into Kafka -------------------
    String groupId = getStandardProps().getProperty("group.id");
    String bootstraps = getBootstrapServers();
    tEnv.getConfig().getConfiguration().set(TABLE_EXEC_SOURCE_IDLE_TIMEOUT, Duration.ofMillis(100));
    final String createTable = String.format("CREATE TABLE kafka (\n" + "  `partition_id` INT,\n" + "  `value` INT,\n" + "  `timestamp` TIMESTAMP(3),\n" + "  WATERMARK FOR `timestamp` AS `timestamp`\n" + ") WITH (\n" + "  'connector' = 'kafka',\n" + "  'topic' = '%s',\n" + "  'properties.bootstrap.servers' = '%s',\n" + "  'properties.group.id' = '%s',\n" + "  'scan.startup.mode' = 'earliest-offset',\n" + "  'sink.partitioner' = '%s',\n" + "  'format' = '%s'\n" + ")", topic, bootstraps, groupId, TestPartitioner.class.getName(), format);
    tEnv.executeSql(createTable);
    // Only two partitions have elements and others are idle.
    // When idle timer triggers, the WatermarkOutputMultiplexer will use the minimum watermark
    // among active partitions as the output watermark.
    // Therefore, we need to make sure the watermark in the each partition is large enough to
    // trigger the window.
    String initialValues = "INSERT INTO kafka\n" + "VALUES\n" + " (0, 0, TIMESTAMP '2020-03-08 13:12:11.123'),\n" + " (0, 1, TIMESTAMP '2020-03-08 13:15:12.223'),\n" + " (0, 2, TIMESTAMP '2020-03-08 16:12:13.323'),\n" + " (1, 3, TIMESTAMP '2020-03-08 13:13:11.123'),\n" + " (1, 4, TIMESTAMP '2020-03-08 13:19:11.133'),\n" + " (1, 5, TIMESTAMP '2020-03-08 16:13:11.143')\n";
    tEnv.executeSql(initialValues).await();
    // ---------- Consume stream from Kafka -------------------
    env.setParallelism(1);
    String createSink = "CREATE TABLE MySink(\n" + "  `id` INT,\n" + "  `cnt` BIGINT\n" + ") WITH (\n" + "  'connector' = 'values'\n" + ")";
    tEnv.executeSql(createSink);
    TableResult tableResult = tEnv.executeSql("INSERT INTO MySink\n" + "SELECT `partition_id` as `id`, COUNT(`value`) as `cnt`\n" + "FROM kafka\n" + "GROUP BY `partition_id`, TUMBLE(`timestamp`, INTERVAL '1' HOUR) ");
    final List<String> expected = Arrays.asList("+I[0, 2]", "+I[1, 2]");
    KafkaTableTestUtils.waitingExpectedResults("MySink", expected, Duration.ofSeconds(5));
    // ------------- cleanup -------------------
    tableResult.getJobClient().ifPresent(JobClient::cancel);
    deleteTestTopic(topic);
}

Example 37 with JobClient

use of org.apache.flink.core.execution.JobClient in project flink by apache.

the class PerJobMiniClusterFactory method submitJob.

/**
 * Starts a {@link MiniCluster} and submits a job.
 */
public CompletableFuture<JobClient> submitJob(JobGraph jobGraph, ClassLoader userCodeClassloader) throws Exception {
    MiniClusterConfiguration miniClusterConfig = getMiniClusterConfig(jobGraph.getMaximumParallelism());
    MiniCluster miniCluster = miniClusterFactory.apply(miniClusterConfig);
    miniCluster.start();
    return miniCluster.submitJob(jobGraph).thenApplyAsync(FunctionUtils.uncheckedFunction(submissionResult -> {
        org.apache.flink.client.ClientUtils.waitUntilJobInitializationFinished(() -> miniCluster.getJobStatus(submissionResult.getJobID()).get(), () -> miniCluster.requestJobResult(submissionResult.getJobID()).get(), userCodeClassloader);
        return submissionResult;
    })).thenApply(result -> new MiniClusterJobClient(result.getJobID(), miniCluster, userCodeClassloader, MiniClusterJobClient.JobFinalizationBehavior.SHUTDOWN_CLUSTER)).whenComplete((ignored, throwable) -> {
        if (throwable != null) {
            // We failed to create the JobClient and must shutdown to ensure
            // cleanup.
            shutDownCluster(miniCluster);
        }
    }).thenApply(Function.identity());
}

Example 38 with JobClient

use of org.apache.flink.core.execution.JobClient in project flink by apache.

the class PerJobMiniClusterFactoryTest method testJobClient.

@Test
public void testJobClient() throws Exception {
    PerJobMiniClusterFactory perJobMiniClusterFactory = initializeMiniCluster();
    JobGraph cancellableJobGraph = getCancellableJobGraph();
    JobClient jobClient = perJobMiniClusterFactory.submitJob(cancellableJobGraph, ClassLoader.getSystemClassLoader()).get();
    assertThat(jobClient.getJobID(), is(cancellableJobGraph.getJobID()));
    assertThat(jobClient.getJobStatus().get(), is(JobStatus.RUNNING));
    jobClient.cancel().get();
    assertThrows("Job was cancelled.", ExecutionException.class, () -> jobClient.getJobExecutionResult().get());
    assertThatMiniClusterIsShutdown();
}

Example 39 with JobClient

use of org.apache.flink.core.execution.JobClient in project flink by apache.

the class PerJobMiniClusterFactoryTest method testJobClientSavepoint.

@Test
public void testJobClientSavepoint() throws Exception {
    PerJobMiniClusterFactory perJobMiniClusterFactory = initializeMiniCluster();
    JobClient jobClient = perJobMiniClusterFactory.submitJob(getCancellableJobGraph(), ClassLoader.getSystemClassLoader()).get();
    assertThrows("is not a streaming job.", ExecutionException.class, () -> jobClient.triggerSavepoint(null, SavepointFormatType.DEFAULT).get());
    assertThrows("is not a streaming job.", ExecutionException.class, () -> jobClient.stopWithSavepoint(true, null, SavepointFormatType.DEFAULT).get());
}

Example 40 with JobClient

use of org.apache.flink.core.execution.JobClient in project flink by apache.

the class PerJobMiniClusterFactoryTest method testJobClientInteractionAfterShutdown.

@Test
public void testJobClientInteractionAfterShutdown() throws Exception {
    PerJobMiniClusterFactory perJobMiniClusterFactory = initializeMiniCluster();
    JobClient jobClient = perJobMiniClusterFactory.submitJob(getNoopJobGraph(), ClassLoader.getSystemClassLoader()).get();
    jobClient.getJobExecutionResult().get();
    assertThatMiniClusterIsShutdown();
    assertThrows("MiniCluster is not yet running or has already been shut down.", IllegalStateException.class, jobClient::cancel);
}