Examples with ListOffsetsOptions org.apache.kafka.clients.admin.ListOffsetsOptions used on opensource projects

Example 1 with ListOffsetsOptions

use of org.apache.kafka.clients.admin.ListOffsetsOptions in project kafka by apache.

the class StoreChangelogReaderTest method shouldThrowIfEndOffsetsFail.

@Test
public void shouldThrowIfEndOffsetsFail() {
    EasyMock.expect(storeMetadata.offset()).andReturn(10L).anyTimes();
    EasyMock.replay(activeStateManager, storeMetadata, store);
    final MockAdminClient adminClient = new MockAdminClient() {

        @Override
        public ListOffsetsResult listOffsets(final Map<TopicPartition, OffsetSpec> topicPartitionOffsets, final ListOffsetsOptions options) {
            throw kaboom;
        }
    };
    adminClient.updateEndOffsets(Collections.singletonMap(tp, 0L));
    final StoreChangelogReader changelogReader = new StoreChangelogReader(time, config, logContext, adminClient, consumer, callback);
    changelogReader.register(tp, activeStateManager);
    final StreamsException thrown = assertThrows(StreamsException.class, () -> changelogReader.restore(Collections.emptyMap()));
    assertEquals(kaboom, thrown.getCause());
}

Example 2 with ListOffsetsOptions

use of org.apache.kafka.clients.admin.ListOffsetsOptions in project kafka by apache.

the class GetListOffsetsCallsBenchmark method setup.

@Setup(Level.Trial)
public void setup() {
    MetadataResponseData data = new MetadataResponseData();
    List<MetadataResponseTopic> mrTopicList = new ArrayList<>();
    Set<String> topics = new HashSet<>();
    for (int topicIndex = 0; topicIndex < topicCount; topicIndex++) {
        Uuid topicId = Uuid.randomUuid();
        String topicName = "topic-" + topicIndex;
        MetadataResponseTopic mrTopic = new MetadataResponseTopic().setTopicId(topicId).setName(topicName).setErrorCode((short) 0).setIsInternal(false);
        List<MetadataResponsePartition> mrPartitionList = new ArrayList<>();
        for (int partition = 0; partition < partitionCount; partition++) {
            TopicPartition tp = new TopicPartition(topicName, partition);
            topics.add(tp.topic());
            futures.put(tp, new KafkaFutureImpl<>());
            topicPartitionOffsets.put(tp, OffsetSpec.latest());
            MetadataResponsePartition mrPartition = new MetadataResponsePartition().setLeaderId(partition % numNodes).setPartitionIndex(partition).setIsrNodes(Arrays.asList(0, 1, 2)).setReplicaNodes(Arrays.asList(0, 1, 2)).setOfflineReplicas(Collections.emptyList()).setErrorCode((short) 0);
            mrPartitionList.add(mrPartition);
        }
        mrTopic.setPartitions(mrPartitionList);
        mrTopicList.add(mrTopic);
    }
    data.setTopics(new MetadataResponseData.MetadataResponseTopicCollection(mrTopicList.listIterator()));
    long deadline = 0L;
    short version = 0;
    context = new MetadataOperationContext<>(topics, new ListOffsetsOptions(), deadline, futures);
    context.setResponse(Optional.of(new MetadataResponse(data, version)));
    AdminClientUnitTestEnv adminEnv = new AdminClientUnitTestEnv(mockCluster());
    admin = (KafkaAdminClient) adminEnv.adminClient();
}

Example 3 with ListOffsetsOptions

use of org.apache.kafka.clients.admin.ListOffsetsOptions in project kafka by apache.

the class StoreChangelogReader method endOffsetForChangelogs.

private Map<TopicPartition, Long> endOffsetForChangelogs(final Map<TaskId, Task> tasks, final Set<TopicPartition> partitions) {
    if (partitions.isEmpty()) {
        return Collections.emptyMap();
    }
    try {
        final ListOffsetsResult result = adminClient.listOffsets(partitions.stream().collect(Collectors.toMap(Function.identity(), tp -> OffsetSpec.latest())), new ListOffsetsOptions(IsolationLevel.READ_UNCOMMITTED));
        final Map<TopicPartition, ListOffsetsResult.ListOffsetsResultInfo> resultPerPartition = result.all().get();
        clearTaskTimeout(getTasksFromPartitions(tasks, partitions));
        return resultPerPartition.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().offset()));
    } catch (final TimeoutException | InterruptedException | ExecutionException retriableException) {
        log.debug("Could not fetch all end offsets for {}, will retry in the next run loop", partitions);
        maybeInitTaskTimeoutOrThrow(getTasksFromPartitions(tasks, partitions), retriableException);
        return Collections.emptyMap();
    } catch (final KafkaException e) {
        throw new StreamsException(String.format("Failed to retrieve end offsets for %s", partitions), e);
    }
}

Example 4 with ListOffsetsOptions

use of org.apache.kafka.clients.admin.ListOffsetsOptions in project kafka by apache.

the class StoreChangelogReaderTest method shouldRequestEndOffsetsAndHandleTimeoutException.

@Test
public void shouldRequestEndOffsetsAndHandleTimeoutException() {
    final TaskId taskId = new TaskId(0, 0);
    final Task mockTask = mock(Task.class);
    mockTask.maybeInitTaskTimeoutOrThrow(anyLong(), anyObject());
    EasyMock.expectLastCall();
    EasyMock.expect(storeMetadata.offset()).andReturn(5L).anyTimes();
    EasyMock.expect(activeStateManager.changelogOffsets()).andReturn(singletonMap(tp, 5L));
    EasyMock.expect(activeStateManager.taskId()).andReturn(taskId).anyTimes();
    EasyMock.replay(mockTask, activeStateManager, storeMetadata, store);
    final AtomicBoolean functionCalled = new AtomicBoolean(false);
    final MockAdminClient adminClient = new MockAdminClient() {

        @Override
        public ListOffsetsResult listOffsets(final Map<TopicPartition, OffsetSpec> topicPartitionOffsets, final ListOffsetsOptions options) {
            if (functionCalled.get()) {
                return super.listOffsets(topicPartitionOffsets, options);
            } else {
                functionCalled.set(true);
                throw new TimeoutException("KABOOM!");
            }
        }
    };
    adminClient.updateEndOffsets(Collections.singletonMap(tp, 10L));
    final MockConsumer<byte[], byte[]> consumer = new MockConsumer<byte[], byte[]>(OffsetResetStrategy.EARLIEST) {

        @Override
        public Map<TopicPartition, OffsetAndMetadata> committed(final Set<TopicPartition> partitions) {
            throw new AssertionError("Should not trigger this function");
        }
    };
    final StoreChangelogReader changelogReader = new StoreChangelogReader(time, config, logContext, adminClient, consumer, callback);
    changelogReader.register(tp, activeStateManager);
    changelogReader.restore(Collections.singletonMap(taskId, mockTask));
    assertEquals(StoreChangelogReader.ChangelogState.REGISTERED, changelogReader.changelogMetadata(tp).state());
    assertNull(changelogReader.changelogMetadata(tp).endOffset());
    assertTrue(functionCalled.get());
    verify(mockTask);
    EasyMock.resetToDefault(mockTask);
    mockTask.clearTaskTimeout();
    EasyMock.replay(mockTask);
    changelogReader.restore(Collections.singletonMap(taskId, mockTask));
    assertEquals(StoreChangelogReader.ChangelogState.RESTORING, changelogReader.changelogMetadata(tp).state());
    assertEquals(10L, (long) changelogReader.changelogMetadata(tp).endOffset());
    assertEquals(6L, consumer.position(tp));
    verify(mockTask);
}