Examples with OffsetAndMetadata org.apache.kafka.clients.consumer.OffsetAndMetadata used on opensource projects

Example 21 with OffsetAndMetadata

use of org.apache.kafka.clients.consumer.OffsetAndMetadata in project flink by apache.

the class KafkaConsumerThread method run.

// ------------------------------------------------------------------------
@Override
public void run() {
    // early exit check
    if (!running) {
        return;
    }
    // this is the means to talk to FlinkKafkaConsumer's main thread
    final Handover handover = this.handover;
    // This method initializes the KafkaConsumer and guarantees it is torn down properly.
    // This is important, because the consumer has multi-threading issues,
    // including concurrent 'close()' calls.
    final KafkaConsumer<byte[], byte[]> consumer;
    try {
        consumer = new KafkaConsumer<>(kafkaProperties);
    } catch (Throwable t) {
        handover.reportError(t);
        return;
    }
    // from here on, the consumer is guaranteed to be closed properly
    try {
        // The callback invoked by Kafka once an offset commit is complete
        final OffsetCommitCallback offsetCommitCallback = new CommitCallback();
        // tell the consumer which partitions to work with
        consumerCallBridge.assignPartitions(consumer, convertKafkaPartitions(subscribedPartitionStates));
        // register Kafka's very own metrics in Flink's metric reporters
        if (useMetrics) {
            // register Kafka metrics to Flink
            Map<MetricName, ? extends Metric> metrics = consumer.metrics();
            if (metrics == null) {
                // MapR's Kafka implementation returns null here.
                log.info("Consumer implementation does not support metrics");
            } else {
                // we have Kafka metrics, register them
                for (Map.Entry<MetricName, ? extends Metric> metric : metrics.entrySet()) {
                    kafkaMetricGroup.gauge(metric.getKey().name(), new KafkaMetricWrapper(metric.getValue()));
                }
            }
        }
        // early exit check
        if (!running) {
            return;
        }
        // values yet; replace those with actual offsets, according to what the sentinel value represent.
        for (KafkaTopicPartitionState<TopicPartition> partition : subscribedPartitionStates) {
            if (partition.getOffset() == KafkaTopicPartitionStateSentinel.EARLIEST_OFFSET) {
                consumerCallBridge.seekPartitionToBeginning(consumer, partition.getKafkaPartitionHandle());
                partition.setOffset(consumer.position(partition.getKafkaPartitionHandle()) - 1);
            } else if (partition.getOffset() == KafkaTopicPartitionStateSentinel.LATEST_OFFSET) {
                consumerCallBridge.seekPartitionToEnd(consumer, partition.getKafkaPartitionHandle());
                partition.setOffset(consumer.position(partition.getKafkaPartitionHandle()) - 1);
            } else if (partition.getOffset() == KafkaTopicPartitionStateSentinel.GROUP_OFFSET) {
                // the KafkaConsumer by default will automatically seek the consumer position
                // to the committed group offset, so we do not need to do it.
                partition.setOffset(consumer.position(partition.getKafkaPartitionHandle()) - 1);
            } else {
                consumer.seek(partition.getKafkaPartitionHandle(), partition.getOffset() + 1);
            }
        }
        // from now on, external operations may call the consumer
        this.consumer = consumer;
        // the latest bulk of records. may carry across the loop if the thread is woken up
        // from blocking on the handover
        ConsumerRecords<byte[], byte[]> records = null;
        // main fetch loop
        while (running) {
            // check if there is something to commit
            if (!commitInProgress) {
                // get and reset the work-to-be committed, so we don't repeatedly commit the same
                final Map<TopicPartition, OffsetAndMetadata> toCommit = nextOffsetsToCommit.getAndSet(null);
                if (toCommit != null) {
                    log.debug("Sending async offset commit request to Kafka broker");
                    // also record that a commit is already in progress
                    // the order here matters! first set the flag, then send the commit command.
                    commitInProgress = true;
                    consumer.commitAsync(toCommit, offsetCommitCallback);
                }
            }
            // get the next batch of records, unless we did not manage to hand the old batch over
            if (records == null) {
                try {
                    records = consumer.poll(pollTimeout);
                } catch (WakeupException we) {
                    continue;
                }
            }
            try {
                handover.produce(records);
                records = null;
            } catch (Handover.WakeupException e) {
            // fall through the loop
            }
        }
    // end main fetch loop
    } catch (Throwable t) {
        // let the main thread know and exit
        // it may be that this exception comes because the main thread closed the handover, in
        // which case the below reporting is irrelevant, but does not hurt either
        handover.reportError(t);
    } finally {
        // make sure the handover is closed if it is not already closed or has an error
        handover.close();
        // make sure the KafkaConsumer is closed
        try {
            consumer.close();
        } catch (Throwable t) {
            log.warn("Error while closing Kafka consumer", t);
        }
    }
}

Example 22 with OffsetAndMetadata

use of org.apache.kafka.clients.consumer.OffsetAndMetadata in project flink by apache.

the class Kafka09Fetcher method commitInternalOffsetsToKafka.

@Override
public void commitInternalOffsetsToKafka(Map<KafkaTopicPartition, Long> offsets) throws Exception {
    KafkaTopicPartitionState<TopicPartition>[] partitions = subscribedPartitionStates();
    Map<TopicPartition, OffsetAndMetadata> offsetsToCommit = new HashMap<>(partitions.length);
    for (KafkaTopicPartitionState<TopicPartition> partition : partitions) {
        Long lastProcessedOffset = offsets.get(partition.getKafkaTopicPartition());
        if (lastProcessedOffset != null) {
            // committed offsets through the KafkaConsumer need to be 1 more than the last processed offset.
            // This does not affect Flink's checkpoints/saved state.
            long offsetToCommit = lastProcessedOffset + 1;
            offsetsToCommit.put(partition.getKafkaPartitionHandle(), new OffsetAndMetadata(offsetToCommit));
            partition.setCommittedOffset(offsetToCommit);
        }
    }
    // record the work to be committed by the main consumer thread and make sure the consumer notices that
    consumerThread.setOffsetsToCommit(offsetsToCommit);
}

Example 23 with OffsetAndMetadata

use of org.apache.kafka.clients.consumer.OffsetAndMetadata in project flink by apache.

the class Kafka09FetcherTest method ensureOffsetsGetCommitted.

@Test
public void ensureOffsetsGetCommitted() throws Exception {
    // test data
    final KafkaTopicPartition testPartition1 = new KafkaTopicPartition("test", 42);
    final KafkaTopicPartition testPartition2 = new KafkaTopicPartition("another", 99);
    final Map<KafkaTopicPartition, Long> testCommitData1 = new HashMap<>();
    testCommitData1.put(testPartition1, 11L);
    testCommitData1.put(testPartition2, 18L);
    final Map<KafkaTopicPartition, Long> testCommitData2 = new HashMap<>();
    testCommitData2.put(testPartition1, 19L);
    testCommitData2.put(testPartition2, 28L);
    final BlockingQueue<Map<TopicPartition, OffsetAndMetadata>> commitStore = new LinkedBlockingQueue<>();
    // ----- the mock consumer with poll(), wakeup(), and commit(A)sync calls ----
    final MultiShotLatch blockerLatch = new MultiShotLatch();
    KafkaConsumer<?, ?> mockConsumer = mock(KafkaConsumer.class);
    when(mockConsumer.poll(anyLong())).thenAnswer(new Answer<ConsumerRecords<?, ?>>() {

        @Override
        public ConsumerRecords<?, ?> answer(InvocationOnMock invocation) throws InterruptedException {
            blockerLatch.await();
            return ConsumerRecords.empty();
        }
    });
    doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) {
            blockerLatch.trigger();
            return null;
        }
    }).when(mockConsumer).wakeup();
    doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) {
            @SuppressWarnings("unchecked") Map<TopicPartition, OffsetAndMetadata> offsets = (Map<TopicPartition, OffsetAndMetadata>) invocation.getArguments()[0];
            OffsetCommitCallback callback = (OffsetCommitCallback) invocation.getArguments()[1];
            commitStore.add(offsets);
            callback.onComplete(offsets, null);
            return null;
        }
    }).when(mockConsumer).commitAsync(Mockito.<Map<TopicPartition, OffsetAndMetadata>>any(), any(OffsetCommitCallback.class));
    // make sure the fetcher creates the mock consumer
    whenNew(KafkaConsumer.class).withAnyArguments().thenReturn(mockConsumer);
    // ----- create the test fetcher -----
    @SuppressWarnings("unchecked") SourceContext<String> sourceContext = mock(SourceContext.class);
    Map<KafkaTopicPartition, Long> partitionsWithInitialOffsets = Collections.singletonMap(new KafkaTopicPartition("test", 42), KafkaTopicPartitionStateSentinel.GROUP_OFFSET);
    KeyedDeserializationSchema<String> schema = new KeyedDeserializationSchemaWrapper<>(new SimpleStringSchema());
    final Kafka09Fetcher<String> fetcher = new Kafka09Fetcher<>(sourceContext, partitionsWithInitialOffsets, null, /* periodic watermark extractor */
    null, /* punctuated watermark extractor */
    new TestProcessingTimeService(), 10, /* watermark interval */
    this.getClass().getClassLoader(), "task_name", new UnregisteredMetricsGroup(), schema, new Properties(), 0L, false);
    // ----- run the fetcher -----
    final AtomicReference<Throwable> error = new AtomicReference<>();
    final Thread fetcherRunner = new Thread("fetcher runner") {

        @Override
        public void run() {
            try {
                fetcher.runFetchLoop();
            } catch (Throwable t) {
                error.set(t);
            }
        }
    };
    fetcherRunner.start();
    // ----- trigger the first offset commit -----
    fetcher.commitInternalOffsetsToKafka(testCommitData1);
    Map<TopicPartition, OffsetAndMetadata> result1 = commitStore.take();
    for (Entry<TopicPartition, OffsetAndMetadata> entry : result1.entrySet()) {
        TopicPartition partition = entry.getKey();
        if (partition.topic().equals("test")) {
            assertEquals(42, partition.partition());
            assertEquals(12L, entry.getValue().offset());
        } else if (partition.topic().equals("another")) {
            assertEquals(99, partition.partition());
            assertEquals(17L, entry.getValue().offset());
        }
    }
    // ----- trigger the second offset commit -----
    fetcher.commitInternalOffsetsToKafka(testCommitData2);
    Map<TopicPartition, OffsetAndMetadata> result2 = commitStore.take();
    for (Entry<TopicPartition, OffsetAndMetadata> entry : result2.entrySet()) {
        TopicPartition partition = entry.getKey();
        if (partition.topic().equals("test")) {
            assertEquals(42, partition.partition());
            assertEquals(20L, entry.getValue().offset());
        } else if (partition.topic().equals("another")) {
            assertEquals(99, partition.partition());
            assertEquals(27L, entry.getValue().offset());
        }
    }
    // ----- test done, wait till the fetcher is done for a clean shutdown -----
    fetcher.cancel();
    fetcherRunner.join();
    // check that there were no errors in the fetcher
    final Throwable caughtError = error.get();
    if (caughtError != null && !(caughtError instanceof Handover.ClosedException)) {
        throw new Exception("Exception in the fetcher", caughtError);
    }
}

Example 24 with OffsetAndMetadata

use of org.apache.kafka.clients.consumer.OffsetAndMetadata in project kafka by apache.

the class SubscriptionStateTest method topicSubscription.

@Test
public void topicSubscription() {
    state.subscribe(singleton(topic), rebalanceListener);
    assertEquals(1, state.subscription().size());
    assertTrue(state.assignedPartitions().isEmpty());
    assertTrue(state.partitionsAutoAssigned());
    state.assignFromSubscribed(singleton(tp0));
    state.seek(tp0, 1);
    state.committed(tp0, new OffsetAndMetadata(1));
    assertAllPositions(tp0, 1L);
    state.assignFromSubscribed(singleton(tp1));
    assertTrue(state.isAssigned(tp1));
    assertFalse(state.isAssigned(tp0));
    assertFalse(state.isFetchable(tp1));
    assertEquals(singleton(tp1), state.assignedPartitions());
}

Example 25 with OffsetAndMetadata

use of org.apache.kafka.clients.consumer.OffsetAndMetadata in project kafka by apache.

the class ConsumerCoordinatorTest method testCommitAfterLeaveGroup.

@Test
public void testCommitAfterLeaveGroup() {
    // enable auto-assignment
    subscriptions.subscribe(singleton(topic1), rebalanceListener);
    client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
    coordinator.ensureCoordinatorReady();
    client.prepareResponse(joinGroupFollowerResponse(1, "consumer", "leader", Errors.NONE));
    client.prepareResponse(syncGroupResponse(singletonList(t1p), Errors.NONE));
    client.prepareMetadataUpdate(cluster, Collections.<String>emptySet());
    coordinator.joinGroupIfNeeded();
    // now switch to manual assignment
    client.prepareResponse(new LeaveGroupResponse(Errors.NONE));
    subscriptions.unsubscribe();
    coordinator.maybeLeaveGroup();
    subscriptions.assignFromUser(singleton(t1p));
    // the client should not reuse generation/memberId from auto-subscribed generation
    client.prepareResponse(new MockClient.RequestMatcher() {

        @Override
        public boolean matches(AbstractRequest body) {
            OffsetCommitRequest commitRequest = (OffsetCommitRequest) body;
            return commitRequest.memberId().equals(OffsetCommitRequest.DEFAULT_MEMBER_ID) && commitRequest.generationId() == OffsetCommitRequest.DEFAULT_GENERATION_ID;
        }
    }, offsetCommitResponse(Collections.singletonMap(t1p, Errors.NONE)));
    AtomicBoolean success = new AtomicBoolean(false);
    coordinator.commitOffsetsAsync(Collections.singletonMap(t1p, new OffsetAndMetadata(100L)), callback(success));
    coordinator.invokeCompletedOffsetCommitCallbacks();
    assertTrue(success.get());
}