Examples with JobClient org.apache.flink.core.execution.JobClient used on opensource projects

Example 6 with JobClient

use of org.apache.flink.core.execution.JobClient in project flink by apache.

the class PerJobMiniClusterFactoryTest method testJobExecution.

@Test
public void testJobExecution() throws Exception {
    PerJobMiniClusterFactory perJobMiniClusterFactory = initializeMiniCluster();
    JobClient jobClient = perJobMiniClusterFactory.submitJob(getNoopJobGraph(), ClassLoader.getSystemClassLoader()).get();
    JobExecutionResult jobExecutionResult = jobClient.getJobExecutionResult().get();
    assertThat(jobExecutionResult, is(notNullValue()));
    Map<String, Object> actual = jobClient.getAccumulators().get();
    assertThat(actual, is(notNullValue()));
    assertThatMiniClusterIsShutdown();
}

Example 7 with JobClient

use of org.apache.flink.core.execution.JobClient in project flink by apache.

the class SourceMetricsITCase method testMetrics.

private void testMetrics(WatermarkStrategy<Integer> strategy, boolean hasTimestamps) throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    int numSplits = Math.max(1, env.getParallelism() - 2);
    env.getConfig().setAutoWatermarkInterval(1L);
    int numRecordsPerSplit = 10;
    MockBaseSource source = new MockBaseSource(numSplits, numRecordsPerSplit, Boundedness.BOUNDED);
    // make sure all parallel instances have processed the same amount of records before
    // validating metrics
    SharedReference<CyclicBarrier> beforeBarrier = sharedObjects.add(new CyclicBarrier(numSplits + 1));
    SharedReference<CyclicBarrier> afterBarrier = sharedObjects.add(new CyclicBarrier(numSplits + 1));
    int stopAtRecord1 = 3;
    int stopAtRecord2 = numRecordsPerSplit - 1;
    DataStream<Integer> stream = env.fromSource(source, strategy, "MetricTestingSource").map(i -> {
        if (i % numRecordsPerSplit == stopAtRecord1 || i % numRecordsPerSplit == stopAtRecord2) {
            beforeBarrier.get().await();
            afterBarrier.get().await();
        }
        return i;
    });
    stream.addSink(new DiscardingSink<>());
    JobClient jobClient = env.executeAsync();
    final JobID jobId = jobClient.getJobID();
    beforeBarrier.get().await();
    assertSourceMetrics(jobId, reporter, stopAtRecord1 + 1, numRecordsPerSplit, env.getParallelism(), numSplits, hasTimestamps);
    afterBarrier.get().await();
    beforeBarrier.get().await();
    assertSourceMetrics(jobId, reporter, stopAtRecord2 + 1, numRecordsPerSplit, env.getParallelism(), numSplits, hasTimestamps);
    afterBarrier.get().await();
    jobClient.getJobExecutionResult().get();
}

Example 8 with JobClient

use of org.apache.flink.core.execution.JobClient in project flink by apache.

the class KafkaTableITCase method testPerPartitionWatermarkKafka.

@Test
public void testPerPartitionWatermarkKafka() throws Exception {
    // we always use a different topic name for each parameterized topic,
    // in order to make sure the topic can be created.
    final String topic = "per_partition_watermark_topic_" + format;
    createTestTopic(topic, 4, 1);
    // ---------- Produce an event time stream into Kafka -------------------
    String groupId = getStandardProps().getProperty("group.id");
    String bootstraps = getBootstrapServers();
    final String createTable = String.format("CREATE TABLE kafka (\n" + "  `partition_id` INT,\n" + "  `name` STRING,\n" + "  `timestamp` TIMESTAMP(3),\n" + "  WATERMARK FOR `timestamp` AS `timestamp`\n" + ") WITH (\n" + "  'connector' = 'kafka',\n" + "  'topic' = '%s',\n" + "  'properties.bootstrap.servers' = '%s',\n" + "  'properties.group.id' = '%s',\n" + "  'scan.startup.mode' = 'earliest-offset',\n" + "  'sink.partitioner' = '%s',\n" + "  'format' = '%s'\n" + ")", topic, bootstraps, groupId, TestPartitioner.class.getName(), format);
    tEnv.executeSql(createTable);
    // make every partition have more than one record
    String initialValues = "INSERT INTO kafka\n" + "VALUES\n" + " (0, 'partition-0-name-0', TIMESTAMP '2020-03-08 13:12:11.123'),\n" + " (0, 'partition-0-name-1', TIMESTAMP '2020-03-08 14:12:12.223'),\n" + " (0, 'partition-0-name-2', TIMESTAMP '2020-03-08 15:12:13.323'),\n" + " (1, 'partition-1-name-0', TIMESTAMP '2020-03-09 13:13:11.123'),\n" + " (1, 'partition-1-name-1', TIMESTAMP '2020-03-09 15:13:11.133'),\n" + " (1, 'partition-1-name-2', TIMESTAMP '2020-03-09 16:13:11.143'),\n" + " (2, 'partition-2-name-0', TIMESTAMP '2020-03-10 13:12:14.123'),\n" + " (2, 'partition-2-name-1', TIMESTAMP '2020-03-10 14:12:14.123'),\n" + " (2, 'partition-2-name-2', TIMESTAMP '2020-03-10 14:13:14.123'),\n" + " (2, 'partition-2-name-3', TIMESTAMP '2020-03-10 14:14:14.123'),\n" + " (2, 'partition-2-name-4', TIMESTAMP '2020-03-10 14:15:14.123'),\n" + " (2, 'partition-2-name-5', TIMESTAMP '2020-03-10 14:16:14.123'),\n" + " (3, 'partition-3-name-0', TIMESTAMP '2020-03-11 17:12:11.123'),\n" + " (3, 'partition-3-name-1', TIMESTAMP '2020-03-11 18:12:11.123')";
    tEnv.executeSql(initialValues).await();
    // ---------- Consume stream from Kafka -------------------
    env.setParallelism(1);
    String createSink = "CREATE TABLE MySink(\n" + "  id INT,\n" + "  name STRING,\n" + "  ts TIMESTAMP(3),\n" + "  WATERMARK FOR ts as ts\n" + ") WITH (\n" + "  'connector' = 'values',\n" + "  'sink.drop-late-event' = 'true'\n" + ")";
    tEnv.executeSql(createSink);
    TableResult tableResult = tEnv.executeSql("INSERT INTO MySink SELECT * FROM kafka");
    final List<String> expected = Arrays.asList("+I[0, partition-0-name-0, 2020-03-08T13:12:11.123]", "+I[0, partition-0-name-1, 2020-03-08T14:12:12.223]", "+I[0, partition-0-name-2, 2020-03-08T15:12:13.323]", "+I[1, partition-1-name-0, 2020-03-09T13:13:11.123]", "+I[1, partition-1-name-1, 2020-03-09T15:13:11.133]", "+I[1, partition-1-name-2, 2020-03-09T16:13:11.143]", "+I[2, partition-2-name-0, 2020-03-10T13:12:14.123]", "+I[2, partition-2-name-1, 2020-03-10T14:12:14.123]", "+I[2, partition-2-name-2, 2020-03-10T14:13:14.123]", "+I[2, partition-2-name-3, 2020-03-10T14:14:14.123]", "+I[2, partition-2-name-4, 2020-03-10T14:15:14.123]", "+I[2, partition-2-name-5, 2020-03-10T14:16:14.123]", "+I[3, partition-3-name-0, 2020-03-11T17:12:11.123]", "+I[3, partition-3-name-1, 2020-03-11T18:12:11.123]");
    KafkaTableTestUtils.waitingExpectedResults("MySink", expected, Duration.ofSeconds(5));
    // ------------- cleanup -------------------
    tableResult.getJobClient().ifPresent(JobClient::cancel);
    deleteTestTopic(topic);
}

Example 9 with JobClient

use of org.apache.flink.core.execution.JobClient in project flink by apache.

the class CollectResultIteratorTest method testEarlyClose.

@Test
public void testEarlyClose() throws Exception {
    List<Integer> expected = new ArrayList<>();
    for (int i = 0; i < 200; i++) {
        expected.add(i);
    }
    Tuple2<CollectResultIterator<Integer>, JobClient> tuple2 = createIteratorAndJobClient(new CheckpointedCollectResultBuffer<>(serializer), new TestCheckpointedCoordinationRequestHandler<>(expected, serializer, ACCUMULATOR_NAME));
    CollectResultIterator<Integer> iterator = tuple2.f0;
    JobClient jobClient = tuple2.f1;
    for (int i = 0; i < 100; i++) {
        Assert.assertTrue(iterator.hasNext());
        Assert.assertNotNull(iterator.next());
    }
    Assert.assertTrue(iterator.hasNext());
    iterator.close();
    Assert.assertEquals(JobStatus.CANCELED, jobClient.getJobStatus().get());
}

Example 10 with JobClient

use of org.apache.flink.core.execution.JobClient in project flink by apache.

the class BufferTimeoutITCase method testDisablingBufferTimeout.

/**
 * The test verifies that it is possible to disable explicit buffer flushing. It checks that
 * OutputFlasher thread would not be started when the task is running. But this doesn't
 * guarantee that the unfinished buffers can not be flushed by another events.
 */
@Test
public void testDisablingBufferTimeout() throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(1);
    env.setBufferTimeout(-1);
    final SharedReference<ArrayList<Integer>> results = sharedObjects.add(new ArrayList<>());
    env.addSource(new SourceFunction<Integer>() {

        @Override
        public void run(SourceContext<Integer> ctx) throws Exception {
            ctx.collect(1);
            // just sleep forever
            Thread.sleep(Long.MAX_VALUE);
        }

        @Override
        public void cancel() {
        }
    }).slotSharingGroup("source").addSink(new SinkFunction<Integer>() {

        @Override
        public void invoke(Integer value, Context context) {
            results.get().add(value);
        }
    }).slotSharingGroup("sink");
    final JobClient jobClient = env.executeAsync();
    CommonTestUtils.waitForAllTaskRunning(MINI_CLUSTER_RESOURCE.getMiniCluster(), jobClient.getJobID(), false);
    assertTrue(RecordWriter.DEFAULT_OUTPUT_FLUSH_THREAD_NAME + " thread is unexpectedly running", Thread.getAllStackTraces().keySet().stream().noneMatch(thread -> thread.getName().startsWith(RecordWriter.DEFAULT_OUTPUT_FLUSH_THREAD_NAME)));
}