Examples with ListOffsetsResult org.apache.kafka.clients.admin.ListOffsetsResult used on opensource projects

Example 6 with ListOffsetsResult

use of org.apache.kafka.clients.admin.ListOffsetsResult in project kafka by apache.

the class ClientUtilsTest method fetchEndOffsetsShouldRethrowExecutionExceptionAsStreamsException.

@Test
public void fetchEndOffsetsShouldRethrowExecutionExceptionAsStreamsException() throws Exception {
    final Admin adminClient = EasyMock.createMock(AdminClient.class);
    final ListOffsetsResult result = EasyMock.createNiceMock(ListOffsetsResult.class);
    final KafkaFuture<Map<TopicPartition, ListOffsetsResultInfo>> allFuture = EasyMock.createMock(KafkaFuture.class);
    EasyMock.expect(adminClient.listOffsets(EasyMock.anyObject())).andStubReturn(result);
    EasyMock.expect(result.all()).andStubReturn(allFuture);
    EasyMock.expect(allFuture.get()).andThrow(new ExecutionException(new RuntimeException()));
    replay(adminClient, result, allFuture);
    assertThrows(StreamsException.class, () -> fetchEndOffsets(PARTITIONS, adminClient));
    verify(adminClient);
}

Example 7 with ListOffsetsResult

use of org.apache.kafka.clients.admin.ListOffsetsResult in project ksql by confluentinc.

the class KsqlEngine method getEndOffsetsForStreamPullQuery.

private ImmutableMap<TopicPartition, Long> getEndOffsetsForStreamPullQuery(final Admin admin, final TopicDescription topicDescription) {
    final Map<TopicPartition, OffsetSpec> topicPartitions = topicDescription.partitions().stream().map(td -> new TopicPartition(topicDescription.name(), td.partition())).collect(toMap(identity(), tp -> OffsetSpec.latest()));
    final ListOffsetsResult listOffsetsResult = admin.listOffsets(topicPartitions, // so we should do the same when checking end offsets.
    new ListOffsetsOptions(IsolationLevel.READ_UNCOMMITTED));
    final ImmutableMap<TopicPartition, Long> startOffsetsForStreamPullQuery = getStartOffsetsForStreamPullQuery(admin, topicDescription);
    try {
        final Map<TopicPartition, ListOffsetsResult.ListOffsetsResultInfo> partitionResultMap = listOffsetsResult.all().get(10, TimeUnit.SECONDS);
        final Map<TopicPartition, Long> result = partitionResultMap.entrySet().stream().filter(e -> e.getValue().offset() > 0L && e.getValue().offset() > startOffsetsForStreamPullQuery.get(e.getKey())).collect(toMap(Entry::getKey, e -> e.getValue().offset()));
        return ImmutableMap.copyOf(result);
    } catch (final InterruptedException e) {
        log.error("Admin#listOffsets(" + topicDescription.name() + ") interrupted", e);
        throw new KsqlServerException("Interrupted");
    } catch (final ExecutionException e) {
        log.error("Error executing Admin#listOffsets(" + topicDescription.name() + ")", e);
        throw new KsqlServerException("Internal Server Error");
    } catch (final TimeoutException e) {
        log.error("Admin#listOffsets(" + topicDescription.name() + ") timed out", e);
        throw new KsqlServerException("Backend timed out");
    }
}

Example 8 with ListOffsetsResult

use of org.apache.kafka.clients.admin.ListOffsetsResult in project kafka by apache.

the class HighAvailabilityStreamsPartitionAssignorTest method createMockAdminClient.

// If you don't care about setting the end offsets for each specific topic partition, the helper method
// getTopicPartitionOffsetMap is useful for building this input map for all partitions
private void createMockAdminClient(final Map<TopicPartition, Long> changelogEndOffsets) {
    adminClient = EasyMock.createMock(AdminClient.class);
    final ListOffsetsResult result = EasyMock.createNiceMock(ListOffsetsResult.class);
    final KafkaFutureImpl<Map<TopicPartition, ListOffsetsResultInfo>> allFuture = new KafkaFutureImpl<>();
    allFuture.complete(changelogEndOffsets.entrySet().stream().collect(Collectors.toMap(Entry::getKey, t -> {
        final ListOffsetsResultInfo info = EasyMock.createNiceMock(ListOffsetsResultInfo.class);
        expect(info.offset()).andStubReturn(t.getValue());
        EasyMock.replay(info);
        return info;
    })));
    expect(adminClient.listOffsets(anyObject())).andStubReturn(result);
    expect(result.all()).andReturn(allFuture);
    EasyMock.replay(result);
}

Example 9 with ListOffsetsResult

use of org.apache.kafka.clients.admin.ListOffsetsResult in project kafka by apache.

the class StoreChangelogReader method endOffsetForChangelogs.

private Map<TopicPartition, Long> endOffsetForChangelogs(final Map<TaskId, Task> tasks, final Set<TopicPartition> partitions) {
    if (partitions.isEmpty()) {
        return Collections.emptyMap();
    }
    try {
        final ListOffsetsResult result = adminClient.listOffsets(partitions.stream().collect(Collectors.toMap(Function.identity(), tp -> OffsetSpec.latest())), new ListOffsetsOptions(IsolationLevel.READ_UNCOMMITTED));
        final Map<TopicPartition, ListOffsetsResult.ListOffsetsResultInfo> resultPerPartition = result.all().get();
        clearTaskTimeout(getTasksFromPartitions(tasks, partitions));
        return resultPerPartition.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().offset()));
    } catch (final TimeoutException | InterruptedException | ExecutionException retriableException) {
        log.debug("Could not fetch all end offsets for {}, will retry in the next run loop", partitions);
        maybeInitTaskTimeoutOrThrow(getTasksFromPartitions(tasks, partitions), retriableException);
        return Collections.emptyMap();
    } catch (final KafkaException e) {
        throw new StreamsException(String.format("Failed to retrieve end offsets for %s", partitions), e);
    }
}

Example 10 with ListOffsetsResult

use of org.apache.kafka.clients.admin.ListOffsetsResult in project kafka by apache.

the class StreamsPartitionAssignorTest method shouldSkipListOffsetsRequestForNewlyCreatedChangelogTopics.

@Test
public void shouldSkipListOffsetsRequestForNewlyCreatedChangelogTopics() {
    adminClient = EasyMock.createMock(AdminClient.class);
    final ListOffsetsResult result = EasyMock.createNiceMock(ListOffsetsResult.class);
    final KafkaFutureImpl<Map<TopicPartition, ListOffsetsResultInfo>> allFuture = new KafkaFutureImpl<>();
    allFuture.complete(emptyMap());
    expect(adminClient.listOffsets(emptyMap())).andStubReturn(result);
    expect(result.all()).andReturn(allFuture);
    builder.addSource(null, "source1", null, null, null, "topic1");
    builder.addProcessor("processor1", new MockApiProcessorSupplier<>(), "source1");
    builder.addStateStore(new MockKeyValueStoreBuilder("store1", false), "processor1");
    subscriptions.put("consumer10", new Subscription(singletonList("topic1"), defaultSubscriptionInfo.encode()));
    EasyMock.replay(result);
    configureDefault();
    overwriteInternalTopicManagerWithMock(true);
    partitionAssignor.assign(metadata, new GroupSubscription(subscriptions));
    EasyMock.verify(adminClient);
}