Examples with KafkaCommitCallback org.apache.flink.streaming.connectors.kafka.internals.KafkaCommitCallback used on opensource projects

Example 1 with KafkaCommitCallback

use of org.apache.flink.streaming.connectors.kafka.internals.KafkaCommitCallback in project flink by apache.

the class FlinkKafkaConsumerBase method run.

@Override
public void run(SourceContext<T> sourceContext) throws Exception {
    if (subscribedPartitionsToStartOffsets == null) {
        throw new Exception("The partitions were not set for the consumer");
    }
    // initialize commit metrics and default offset callback method
    this.successfulCommits = this.getRuntimeContext().getMetricGroup().counter(COMMITS_SUCCEEDED_METRICS_COUNTER);
    this.failedCommits = this.getRuntimeContext().getMetricGroup().counter(COMMITS_FAILED_METRICS_COUNTER);
    final int subtaskIndex = this.getRuntimeContext().getIndexOfThisSubtask();
    this.offsetCommitCallback = new KafkaCommitCallback() {

        @Override
        public void onSuccess() {
            successfulCommits.inc();
        }

        @Override
        public void onException(Throwable cause) {
            LOG.warn(String.format("Consumer subtask %d failed async Kafka commit.", subtaskIndex), cause);
            failedCommits.inc();
        }
    };
    // status will automatically be triggered back to be active.
    if (subscribedPartitionsToStartOffsets.isEmpty()) {
        sourceContext.markAsTemporarilyIdle();
    }
    LOG.info("Consumer subtask {} creating fetcher with offsets {}.", getRuntimeContext().getIndexOfThisSubtask(), subscribedPartitionsToStartOffsets);
    // from this point forward:
    // - 'snapshotState' will draw offsets from the fetcher,
    // instead of being built from `subscribedPartitionsToStartOffsets`
    // - 'notifyCheckpointComplete' will start to do work (i.e. commit offsets to
    // Kafka through the fetcher, if configured to do so)
    this.kafkaFetcher = createFetcher(sourceContext, subscribedPartitionsToStartOffsets, watermarkStrategy, (StreamingRuntimeContext) getRuntimeContext(), offsetCommitMode, getRuntimeContext().getMetricGroup().addGroup(KAFKA_CONSUMER_METRICS_GROUP), useMetrics);
    if (!running) {
        return;
    }
    // executed
    if (discoveryIntervalMillis == PARTITION_DISCOVERY_DISABLED) {
        kafkaFetcher.runFetchLoop();
    } else {
        runWithPartitionDiscovery();
    }
}