Examples with OffsetCommitCallback org.apache.kafka.clients.consumer.OffsetCommitCallback used on opensource projects

Example 21 with OffsetCommitCallback

use of org.apache.kafka.clients.consumer.OffsetCommitCallback in project apache-kafka-on-k8s by banzaicloud.

the class ConsumerCoordinator method doCommitOffsetsAsync.

private void doCommitOffsetsAsync(final Map<TopicPartition, OffsetAndMetadata> offsets, final OffsetCommitCallback callback) {
    RequestFuture<Void> future = sendOffsetCommitRequest(offsets);
    final OffsetCommitCallback cb = callback == null ? defaultOffsetCommitCallback : callback;
    future.addListener(new RequestFutureListener<Void>() {

        @Override
        public void onSuccess(Void value) {
            if (interceptors != null)
                interceptors.onCommit(offsets);
            completedOffsetCommits.add(new OffsetCommitCompletion(cb, offsets, null));
        }

        @Override
        public void onFailure(RuntimeException e) {
            Exception commitException = e;
            if (e instanceof RetriableException)
                commitException = new RetriableCommitFailedException(e);
            completedOffsetCommits.add(new OffsetCommitCompletion(cb, offsets, commitException));
        }
    });
}

Example 22 with OffsetCommitCallback

use of org.apache.kafka.clients.consumer.OffsetCommitCallback in project apache-kafka-on-k8s by banzaicloud.

the class ConsumerCoordinatorTest method testAsyncCommitCallbacksInvokedPriorToSyncCommitCompletion.

@Test
public void testAsyncCommitCallbacksInvokedPriorToSyncCommitCompletion() throws Exception {
    client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
    coordinator.ensureCoordinatorReady();
    final List<OffsetAndMetadata> committedOffsets = Collections.synchronizedList(new ArrayList<OffsetAndMetadata>());
    final OffsetAndMetadata firstOffset = new OffsetAndMetadata(0L);
    final OffsetAndMetadata secondOffset = new OffsetAndMetadata(1L);
    coordinator.commitOffsetsAsync(singletonMap(t1p, firstOffset), new OffsetCommitCallback() {

        @Override
        public void onComplete(Map<TopicPartition, OffsetAndMetadata> offsets, Exception exception) {
            committedOffsets.add(firstOffset);
        }
    });
    // Do a synchronous commit in the background so that we can send both responses at the same time
    Thread thread = new Thread() {

        @Override
        public void run() {
            coordinator.commitOffsetsSync(singletonMap(t1p, secondOffset), 10000);
            committedOffsets.add(secondOffset);
        }
    };
    thread.start();
    client.waitForRequests(2, 5000);
    respondToOffsetCommitRequest(singletonMap(t1p, firstOffset.offset()), Errors.NONE);
    respondToOffsetCommitRequest(singletonMap(t1p, secondOffset.offset()), Errors.NONE);
    thread.join();
    assertEquals(Arrays.asList(firstOffset, secondOffset), committedOffsets);
}

Example 23 with OffsetCommitCallback

use of org.apache.kafka.clients.consumer.OffsetCommitCallback in project kafka by apache.

the class WorkerSinkTask method doCommitAsync.

private void doCommitAsync(Map<TopicPartition, OffsetAndMetadata> offsets, final int seqno) {
    log.debug("{} Committing offsets asynchronously using sequence number {}: {}", this, seqno, offsets);
    OffsetCommitCallback cb = (tpOffsets, error) -> onCommitCompleted(error, seqno, tpOffsets);
    consumer.commitAsync(offsets, cb);
}

Example 24 with OffsetCommitCallback

use of org.apache.kafka.clients.consumer.OffsetCommitCallback in project kafka by apache.

the class ConsumerCoordinator method doCommitOffsetsAsync.

private RequestFuture<Void> doCommitOffsetsAsync(final Map<TopicPartition, OffsetAndMetadata> offsets, final OffsetCommitCallback callback) {
    RequestFuture<Void> future = sendOffsetCommitRequest(offsets);
    final OffsetCommitCallback cb = callback == null ? defaultOffsetCommitCallback : callback;
    future.addListener(new RequestFutureListener<Void>() {

        @Override
        public void onSuccess(Void value) {
            if (interceptors != null)
                interceptors.onCommit(offsets);
            completedOffsetCommits.add(new OffsetCommitCompletion(cb, offsets, null));
        }

        @Override
        public void onFailure(RuntimeException e) {
            Exception commitException = e;
            if (e instanceof RetriableException) {
                commitException = new RetriableCommitFailedException(e);
            }
            completedOffsetCommits.add(new OffsetCommitCompletion(cb, offsets, commitException));
            if (commitException instanceof FencedInstanceIdException) {
                asyncCommitFenced.set(true);
            }
        }
    });
    return future;
}

Example 25 with OffsetCommitCallback

use of org.apache.kafka.clients.consumer.OffsetCommitCallback in project kafka by apache.

the class WorkerSinkTaskTest method testCommitWithOutOfOrderCallback.

// Verify that when commitAsync is called but the supplied callback is not called by the consumer before a
// rebalance occurs, the async callback does not reset the last committed offset from the rebalance.
// See KAFKA-5731 for more information.
@Test
public void testCommitWithOutOfOrderCallback() throws Exception {
    createTask(initialState);
    expectInitializeTask();
    expectTaskGetTopic(true);
    // iter 1
    expectPollInitialAssignment();
    // iter 2
    expectConsumerPoll(1);
    expectConversionAndTransformation(4);
    sinkTask.put(EasyMock.anyObject());
    EasyMock.expectLastCall();
    final Map<TopicPartition, OffsetAndMetadata> workerStartingOffsets = new HashMap<>();
    workerStartingOffsets.put(TOPIC_PARTITION, new OffsetAndMetadata(FIRST_OFFSET));
    workerStartingOffsets.put(TOPIC_PARTITION2, new OffsetAndMetadata(FIRST_OFFSET));
    final Map<TopicPartition, OffsetAndMetadata> workerCurrentOffsets = new HashMap<>();
    workerCurrentOffsets.put(TOPIC_PARTITION, new OffsetAndMetadata(FIRST_OFFSET + 1));
    workerCurrentOffsets.put(TOPIC_PARTITION2, new OffsetAndMetadata(FIRST_OFFSET));
    final List<TopicPartition> originalPartitions = new ArrayList<>(INITIAL_ASSIGNMENT);
    final List<TopicPartition> rebalancedPartitions = asList(TOPIC_PARTITION, TOPIC_PARTITION2, TOPIC_PARTITION3);
    final Map<TopicPartition, OffsetAndMetadata> rebalanceOffsets = new HashMap<>();
    rebalanceOffsets.put(TOPIC_PARTITION, workerCurrentOffsets.get(TOPIC_PARTITION));
    rebalanceOffsets.put(TOPIC_PARTITION2, workerCurrentOffsets.get(TOPIC_PARTITION2));
    rebalanceOffsets.put(TOPIC_PARTITION3, new OffsetAndMetadata(FIRST_OFFSET));
    final Map<TopicPartition, OffsetAndMetadata> postRebalanceCurrentOffsets = new HashMap<>();
    postRebalanceCurrentOffsets.put(TOPIC_PARTITION, new OffsetAndMetadata(FIRST_OFFSET + 3));
    postRebalanceCurrentOffsets.put(TOPIC_PARTITION2, new OffsetAndMetadata(FIRST_OFFSET));
    postRebalanceCurrentOffsets.put(TOPIC_PARTITION3, new OffsetAndMetadata(FIRST_OFFSET + 2));
    EasyMock.expect(consumer.assignment()).andReturn(new HashSet<>(originalPartitions)).times(2);
    // iter 3 - note that we return the current offset to indicate they should be committed
    sinkTask.preCommit(workerCurrentOffsets);
    EasyMock.expectLastCall().andReturn(workerCurrentOffsets);
    // We need to delay the result of trying to commit offsets to Kafka via the consumer.commitAsync
    // method. We do this so that we can test that the callback is not called until after the rebalance
    // changes the lastCommittedOffsets. To fake this for tests we have the commitAsync build a function
    // that will call the callback with the appropriate parameters, and we'll run that function later.
    final AtomicReference<Runnable> asyncCallbackRunner = new AtomicReference<>();
    final AtomicBoolean asyncCallbackRan = new AtomicBoolean();
    consumer.commitAsync(EasyMock.eq(workerCurrentOffsets), EasyMock.anyObject());
    EasyMock.expectLastCall().andAnswer(() -> {
        // Grab the arguments passed to the consumer.commitAsync method
        final Object[] args = EasyMock.getCurrentArguments();
        @SuppressWarnings("unchecked") final Map<TopicPartition, OffsetAndMetadata> offsets = (Map<TopicPartition, OffsetAndMetadata>) args[0];
        final OffsetCommitCallback callback = (OffsetCommitCallback) args[1];
        asyncCallbackRunner.set(() -> {
            callback.onComplete(offsets, null);
            asyncCallbackRan.set(true);
        });
        return null;
    });
    // Expect the next poll to discover and perform the rebalance, THEN complete the previous callback handler,
    // and then return one record for TP1 and one for TP3.
    final AtomicBoolean rebalanced = new AtomicBoolean();
    EasyMock.expect(consumer.poll(Duration.ofMillis(EasyMock.anyLong()))).andAnswer(() -> {
        // Rebalance always begins with revoking current partitions ...
        rebalanceListener.getValue().onPartitionsRevoked(originalPartitions);
        // Respond to the rebalance
        Map<TopicPartition, Long> offsets = new HashMap<>();
        offsets.put(TOPIC_PARTITION, rebalanceOffsets.get(TOPIC_PARTITION).offset());
        offsets.put(TOPIC_PARTITION2, rebalanceOffsets.get(TOPIC_PARTITION2).offset());
        offsets.put(TOPIC_PARTITION3, rebalanceOffsets.get(TOPIC_PARTITION3).offset());
        sinkTaskContext.getValue().offset(offsets);
        rebalanceListener.getValue().onPartitionsAssigned(rebalancedPartitions);
        rebalanced.set(true);
        // Run the previous async commit handler
        asyncCallbackRunner.get().run();
        // And prep the two records to return
        long timestamp = RecordBatch.NO_TIMESTAMP;
        TimestampType timestampType = TimestampType.NO_TIMESTAMP_TYPE;
        List<ConsumerRecord<byte[], byte[]>> records = new ArrayList<>();
        records.add(new ConsumerRecord<>(TOPIC, PARTITION, FIRST_OFFSET + recordsReturnedTp1 + 1, timestamp, timestampType, 0, 0, RAW_KEY, RAW_VALUE, new RecordHeaders(), Optional.empty()));
        records.add(new ConsumerRecord<>(TOPIC, PARTITION3, FIRST_OFFSET + recordsReturnedTp3 + 1, timestamp, timestampType, 0, 0, RAW_KEY, RAW_VALUE, new RecordHeaders(), Optional.empty()));
        recordsReturnedTp1 += 1;
        recordsReturnedTp3 += 1;
        return new ConsumerRecords<>(Collections.singletonMap(new TopicPartition(TOPIC, PARTITION), records));
    });
    // onPartitionsRevoked
    sinkTask.preCommit(workerCurrentOffsets);
    EasyMock.expectLastCall().andReturn(workerCurrentOffsets);
    sinkTask.put(EasyMock.anyObject());
    EasyMock.expectLastCall();
    sinkTask.close(new ArrayList<>(workerCurrentOffsets.keySet()));
    EasyMock.expectLastCall();
    consumer.commitSync(workerCurrentOffsets);
    EasyMock.expectLastCall();
    // onPartitionsAssigned - step 1
    final long offsetTp1 = rebalanceOffsets.get(TOPIC_PARTITION).offset();
    final long offsetTp2 = rebalanceOffsets.get(TOPIC_PARTITION2).offset();
    final long offsetTp3 = rebalanceOffsets.get(TOPIC_PARTITION3).offset();
    EasyMock.expect(consumer.position(TOPIC_PARTITION)).andReturn(offsetTp1);
    EasyMock.expect(consumer.position(TOPIC_PARTITION2)).andReturn(offsetTp2);
    EasyMock.expect(consumer.position(TOPIC_PARTITION3)).andReturn(offsetTp3);
    EasyMock.expect(consumer.assignment()).andReturn(new HashSet<>(rebalancedPartitions)).times(6);
    // onPartitionsAssigned - step 2
    sinkTask.open(EasyMock.eq(rebalancedPartitions));
    EasyMock.expectLastCall();
    // onPartitionsAssigned - step 3 rewind
    consumer.seek(TOPIC_PARTITION, offsetTp1);
    EasyMock.expectLastCall();
    consumer.seek(TOPIC_PARTITION2, offsetTp2);
    EasyMock.expectLastCall();
    consumer.seek(TOPIC_PARTITION3, offsetTp3);
    EasyMock.expectLastCall();
    // iter 4 - note that we return the current offset to indicate they should be committed
    sinkTask.preCommit(postRebalanceCurrentOffsets);
    EasyMock.expectLastCall().andReturn(postRebalanceCurrentOffsets);
    final Capture<OffsetCommitCallback> callback = EasyMock.newCapture();
    consumer.commitAsync(EasyMock.eq(postRebalanceCurrentOffsets), EasyMock.capture(callback));
    EasyMock.expectLastCall().andAnswer(() -> {
        callback.getValue().onComplete(postRebalanceCurrentOffsets, null);
        return null;
    });
    // no actual consumer.commit() triggered
    expectConsumerPoll(1);
    sinkTask.put(EasyMock.anyObject());
    EasyMock.expectLastCall();
    PowerMock.replayAll();
    workerTask.initialize(TASK_CONFIG);
    workerTask.initializeAndStart();
    // iter 1 -- initial assignment
    workerTask.iteration();
    assertEquals(workerStartingOffsets, Whitebox.getInternalState(workerTask, "currentOffsets"));
    assertEquals(workerStartingOffsets, Whitebox.getInternalState(workerTask, "lastCommittedOffsets"));
    time.sleep(WorkerConfig.OFFSET_COMMIT_TIMEOUT_MS_DEFAULT);
    // iter 2 -- deliver 2 records
    workerTask.iteration();
    sinkTaskContext.getValue().requestCommit();
    // iter 3 -- commit in progress
    workerTask.iteration();
    assertSinkMetricValue("partition-count", 3);
    assertSinkMetricValue("sink-record-read-total", 3.0);
    assertSinkMetricValue("sink-record-send-total", 3.0);
    assertSinkMetricValue("sink-record-active-count", 4.0);
    assertSinkMetricValue("sink-record-active-count-max", 4.0);
    assertSinkMetricValue("sink-record-active-count-avg", 0.71429);
    assertSinkMetricValue("offset-commit-seq-no", 2.0);
    assertSinkMetricValue("offset-commit-completion-total", 1.0);
    assertSinkMetricValue("offset-commit-skip-total", 1.0);
    assertTaskMetricValue("status", "running");
    assertTaskMetricValue("running-ratio", 1.0);
    assertTaskMetricValue("pause-ratio", 0.0);
    assertTaskMetricValue("batch-size-max", 2.0);
    assertTaskMetricValue("batch-size-avg", 1.0);
    assertTaskMetricValue("offset-commit-max-time-ms", 0.0);
    assertTaskMetricValue("offset-commit-avg-time-ms", 0.0);
    assertTaskMetricValue("offset-commit-failure-percentage", 0.0);
    assertTaskMetricValue("offset-commit-success-percentage", 1.0);
    assertTrue(asyncCallbackRan.get());
    assertTrue(rebalanced.get());
    // Check that the offsets were not reset by the out-of-order async commit callback
    assertEquals(postRebalanceCurrentOffsets, Whitebox.getInternalState(workerTask, "currentOffsets"));
    assertEquals(rebalanceOffsets, Whitebox.getInternalState(workerTask, "lastCommittedOffsets"));
    time.sleep(WorkerConfig.OFFSET_COMMIT_TIMEOUT_MS_DEFAULT);
    sinkTaskContext.getValue().requestCommit();
    // iter 4 -- commit in progress
    workerTask.iteration();
    // Check that the offsets were not reset by the out-of-order async commit callback
    assertEquals(postRebalanceCurrentOffsets, Whitebox.getInternalState(workerTask, "currentOffsets"));
    assertEquals(postRebalanceCurrentOffsets, Whitebox.getInternalState(workerTask, "lastCommittedOffsets"));
    assertSinkMetricValue("partition-count", 3);
    assertSinkMetricValue("sink-record-read-total", 4.0);
    assertSinkMetricValue("sink-record-send-total", 4.0);
    assertSinkMetricValue("sink-record-active-count", 0.0);
    assertSinkMetricValue("sink-record-active-count-max", 4.0);
    assertSinkMetricValue("sink-record-active-count-avg", 0.5555555);
    assertSinkMetricValue("offset-commit-seq-no", 3.0);
    assertSinkMetricValue("offset-commit-completion-total", 2.0);
    assertSinkMetricValue("offset-commit-skip-total", 1.0);
    assertTaskMetricValue("status", "running");
    assertTaskMetricValue("running-ratio", 1.0);
    assertTaskMetricValue("pause-ratio", 0.0);
    assertTaskMetricValue("batch-size-max", 2.0);
    assertTaskMetricValue("batch-size-avg", 1.0);
    assertTaskMetricValue("offset-commit-max-time-ms", 0.0);
    assertTaskMetricValue("offset-commit-avg-time-ms", 0.0);
    assertTaskMetricValue("offset-commit-failure-percentage", 0.0);
    assertTaskMetricValue("offset-commit-success-percentage", 1.0);
    PowerMock.verifyAll();
}