Examples with IntegerSource org.apache.flink.streaming.connectors.kafka.testutils.IntegerSource used on opensource projects

Example 1 with IntegerSource

use of org.apache.flink.streaming.connectors.kafka.testutils.IntegerSource in project flink by apache.

the class KafkaProducerTestBase method testExactlyOnce.

/**
 * This test sets KafkaProducer so that it will automatically flush the data and and fails the
 * broker to check whether flushed records since last checkpoint were not duplicated.
 */
protected void testExactlyOnce(boolean regularSink, int sinksCount) throws Exception {
    final String topic = (regularSink ? "exactlyOnceTopicRegularSink" : "exactlyTopicCustomOperator") + sinksCount;
    final int partition = 0;
    final int numElements = 1000;
    final int failAfterElements = 333;
    for (int i = 0; i < sinksCount; i++) {
        createTestTopic(topic + i, 1, 1);
    }
    TypeInformationSerializationSchema<Integer> schema = new TypeInformationSerializationSchema<>(BasicTypeInfo.INT_TYPE_INFO, new ExecutionConfig());
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.enableCheckpointing(500);
    env.setParallelism(1);
    env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
    Properties properties = new Properties();
    properties.putAll(standardProps);
    properties.putAll(secureProps);
    // process exactly failAfterElements number of elements and then shutdown Kafka broker and
    // fail application
    List<Integer> expectedElements = getIntegersSequence(numElements);
    DataStream<Integer> inputStream = env.addSource(new IntegerSource(numElements)).map(new FailingIdentityMapper<Integer>(failAfterElements));
    for (int i = 0; i < sinksCount; i++) {
        FlinkKafkaPartitioner<Integer> partitioner = new FlinkKafkaPartitioner<Integer>() {

            @Override
            public int partition(Integer record, byte[] key, byte[] value, String targetTopic, int[] partitions) {
                return partition;
            }
        };
        if (regularSink) {
            StreamSink<Integer> kafkaSink = kafkaServer.getProducerSink(topic + i, schema, properties, partitioner);
            inputStream.addSink(kafkaSink.getUserFunction());
        } else {
            kafkaServer.produceIntoKafka(inputStream, topic + i, schema, properties, partitioner);
        }
    }
    FailingIdentityMapper.failedBefore = false;
    TestUtils.tryExecute(env, "Exactly once test");
    for (int i = 0; i < sinksCount; i++) {
        // assert that before failure we successfully snapshot/flushed all expected elements
        assertExactlyOnceForTopic(properties, topic + i, expectedElements);
        deleteTestTopic(topic + i);
    }
}