Examples with OffsetAndMetadata io.streamnative.pulsar.handlers.kop.offset.OffsetAndMetadata used on opensource projects

Example 1 with OffsetAndMetadata

use of io.streamnative.pulsar.handlers.kop.offset.OffsetAndMetadata in project starlight-for-kafka by datastax.

the class GroupMetadataManagerTest method testGroupLoadWithConsumerAndTransactionalOffsetCommitsTransactionWins.

@Test
public void testGroupLoadWithConsumerAndTransactionalOffsetCommitsTransactionWins() throws Exception {
    long producerId = 1000L;
    short producerEpoch = 2;
    Map<TopicPartition, Long> transactionalOffsetCommits = new HashMap<>();
    transactionalOffsetCommits.put(new TopicPartition("foo", 0), 23L);
    Map<TopicPartition, Long> consumerOffsetCommits = new HashMap<>();
    consumerOffsetCommits.put(new TopicPartition("foo", 0), 24L);
    ByteBuffer buffer = ByteBuffer.allocate(1024);
    int nextOffset = 0;
    nextOffset += appendConsumerOffsetCommit(buffer, nextOffset, consumerOffsetCommits);
    nextOffset += appendTransactionalOffsetCommits(buffer, producerId, producerEpoch, nextOffset, transactionalOffsetCommits, NAMESPACE_PREFIX);
    nextOffset += completeTransactionalOffsetCommit(buffer, producerId, producerEpoch, nextOffset, true);
    buffer.flip();
    byte[] key = groupMetadataKey(groupId);
    Producer<ByteBuffer> producer = groupMetadataManager.getOffsetsTopicProducer(groupPartitionId).get();
    producer.newMessage().keyBytes(key).value(buffer).eventTime(Time.SYSTEM.milliseconds()).send();
    CompletableFuture<GroupMetadata> onLoadedFuture = new CompletableFuture<>();
    groupMetadataManager.scheduleLoadGroupAndOffsets(groupPartitionId, groupMetadata -> onLoadedFuture.complete(groupMetadata)).get();
    GroupMetadata group = onLoadedFuture.get();
    GroupMetadata groupInCache = groupMetadataManager.getGroup(groupId).orElseGet(() -> {
        fail("Group was not loaded into the cache");
        return null;
    });
    assertSame(group, groupInCache);
    assertEquals(groupId, group.groupId());
    assertEquals(Empty, group.currentState());
    // The group should be loaded with pending offsets.
    assertEquals(1, group.allOffsets().size());
    assertTrue(group.hasOffsets());
    assertFalse(group.hasPendingOffsetCommitsFromProducer(producerId));
    assertEquals(consumerOffsetCommits.size(), group.allOffsets().size());
    transactionalOffsetCommits.forEach((tp, offset) -> {
        assertEquals(Optional.of(offset), group.offset(tp, NAMESPACE_PREFIX).map(OffsetAndMetadata::offset));
    });
}

Example 2 with OffsetAndMetadata

use of io.streamnative.pulsar.handlers.kop.offset.OffsetAndMetadata in project starlight-for-kafka by datastax.

the class GroupMetadataManagerTest method testLoadTransactionalOffsetCommitsFromMultipleProducers.

@Test
public void testLoadTransactionalOffsetCommitsFromMultipleProducers() throws Exception {
    long firstProducerId = 1000L;
    short firstProducerEpoch = 2;
    long secondProducerId = 1001L;
    short secondProducerEpoch = 3;
    Map<TopicPartition, Long> committedOffsetsFirstProducer = new HashMap<>();
    committedOffsetsFirstProducer.put(new TopicPartition("foo", 0), 23L);
    committedOffsetsFirstProducer.put(new TopicPartition("foo", 1), 455L);
    committedOffsetsFirstProducer.put(new TopicPartition("bar", 0), 8992L);
    Map<TopicPartition, Long> committedOffsetsSecondProducer = new HashMap<>();
    committedOffsetsSecondProducer.put(new TopicPartition("foo", 2), 231L);
    committedOffsetsSecondProducer.put(new TopicPartition("foo", 3), 4551L);
    committedOffsetsSecondProducer.put(new TopicPartition("bar", 1), 89921L);
    ByteBuffer buffer = ByteBuffer.allocate(1024);
    int nextOffset = 0;
    int firstProduceRecordOffset = nextOffset;
    nextOffset += appendTransactionalOffsetCommits(buffer, firstProducerId, firstProducerEpoch, nextOffset, committedOffsetsFirstProducer, NAMESPACE_PREFIX);
    nextOffset += completeTransactionalOffsetCommit(buffer, firstProducerId, firstProducerEpoch, nextOffset, true);
    int secondProduceRecordOffset = nextOffset;
    nextOffset += appendTransactionalOffsetCommits(buffer, secondProducerId, secondProducerEpoch, nextOffset, committedOffsetsSecondProducer, NAMESPACE_PREFIX);
    nextOffset += completeTransactionalOffsetCommit(buffer, secondProducerId, secondProducerEpoch, nextOffset, true);
    buffer.flip();
    byte[] key = groupMetadataKey(groupId);
    Producer<ByteBuffer> producer = groupMetadataManager.getOffsetsTopicProducer(groupPartitionId).get();
    producer.newMessage().keyBytes(key).value(buffer).eventTime(Time.SYSTEM.milliseconds()).send();
    CompletableFuture<GroupMetadata> onLoadedFuture = new CompletableFuture<>();
    groupMetadataManager.scheduleLoadGroupAndOffsets(groupPartitionId, groupMetadata -> onLoadedFuture.complete(groupMetadata)).get();
    GroupMetadata group = onLoadedFuture.get();
    GroupMetadata groupInCache = groupMetadataManager.getGroup(groupId).orElseGet(() -> {
        fail("Group was not loaded into the cache");
        return null;
    });
    assertSame(group, groupInCache);
    assertEquals(groupId, group.groupId());
    assertEquals(Empty, group.currentState());
    // Ensure that only the committed offsets are materialized, and that there are no pending commits
    // for the producer. This allows us to be certain that the aborted offset commits are truly discarded.
    assertEquals(committedOffsetsFirstProducer.size() + committedOffsetsSecondProducer.size(), group.allOffsets().size());
    committedOffsetsFirstProducer.forEach((tp, offset) -> {
        assertEquals(Optional.of(offset), group.offset(tp, NAMESPACE_PREFIX).map(OffsetAndMetadata::offset));
        assertEquals(Optional.of((long) firstProduceRecordOffset), group.offsetWithRecordMetadata(tp).flatMap(CommitRecordMetadataAndOffset::appendedBatchOffset));
    });
    committedOffsetsSecondProducer.forEach((tp, offset) -> {
        assertEquals(Optional.of(offset), group.offset(tp, NAMESPACE_PREFIX).map(OffsetAndMetadata::offset));
        assertEquals(Optional.of((long) secondProduceRecordOffset), group.offsetWithRecordMetadata(tp).flatMap(CommitRecordMetadataAndOffset::appendedBatchOffset));
    });
}

Example 3 with OffsetAndMetadata

use of io.streamnative.pulsar.handlers.kop.offset.OffsetAndMetadata in project starlight-for-kafka by datastax.

the class GroupMetadataManagerTest method testLoadOffsetsWithTombstones.

@Test
public void testLoadOffsetsWithTombstones() throws Exception {
    TopicPartition tombstonePartition = new TopicPartition("foo", 1);
    Map<TopicPartition, Long> committedOffsets = new HashMap<>();
    committedOffsets.put(new TopicPartition("foo", 0), 23L);
    committedOffsets.put(tombstonePartition, 455L);
    committedOffsets.put(new TopicPartition("bar", 0), 8992L);
    List<SimpleRecord> offsetCommitRecords = createCommittedOffsetRecords(committedOffsets, groupId, NAMESPACE_PREFIX);
    SimpleRecord tombstone = new SimpleRecord(offsetCommitKey(groupId, tombstonePartition, NAMESPACE_PREFIX), null);
    offsetCommitRecords.add(tombstone);
    ByteBuffer buffer = newMemoryRecordsBuffer(offsetCommitRecords);
    byte[] key = groupMetadataKey(groupId);
    Producer<ByteBuffer> producer = groupMetadataManager.getOffsetsTopicProducer(groupPartitionId).get();
    producer.newMessage().keyBytes(key).value(buffer).eventTime(Time.SYSTEM.milliseconds()).send();
    CompletableFuture<GroupMetadata> onLoadedFuture = new CompletableFuture<>();
    groupMetadataManager.scheduleLoadGroupAndOffsets(groupPartitionId, groupMetadata -> onLoadedFuture.complete(groupMetadata)).get();
    GroupMetadata group = onLoadedFuture.get();
    GroupMetadata groupInCache = groupMetadataManager.getGroup(groupId).orElseGet(() -> {
        fail("Group was not loaded into the cache");
        return null;
    });
    assertSame(group, groupInCache);
    assertEquals(groupId, group.groupId());
    assertEquals(Empty, group.currentState());
    // The group should be loaded with pending offsets.
    assertEquals(committedOffsets.size() - 1, group.allOffsets().size());
    committedOffsets.forEach((tp, offset) -> {
        if (tp == tombstonePartition) {
            assertEquals(Optional.empty(), group.offset(tp, NAMESPACE_PREFIX));
        } else {
            assertEquals(Optional.of(offset), group.offset(tp, NAMESPACE_PREFIX).map(OffsetAndMetadata::offset));
        }
    });
}

Example 4 with OffsetAndMetadata

use of io.streamnative.pulsar.handlers.kop.offset.OffsetAndMetadata in project starlight-for-kafka by datastax.

the class GroupMetadataManagerTest method testCommitOffset.

@Test
public void testCommitOffset() throws Exception {
    @Cleanup Consumer<ByteBuffer> consumer = pulsarClient.newConsumer(Schema.BYTEBUFFER).topic(groupMetadataManager.getTopicPartitionName()).subscriptionName("test-sub").subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe();
    String memberId = "fakeMemberId";
    TopicPartition topicPartition = new TopicPartition("foo", 0);
    groupMetadataManager.addPartitionOwnership(groupPartitionId);
    long offset = 37L;
    GroupMetadata group = new GroupMetadata(groupId, Empty);
    groupMetadataManager.addGroup(group);
    Map<TopicPartition, OffsetAndMetadata> offsets = ImmutableMap.<TopicPartition, OffsetAndMetadata>builder().put(topicPartition, OffsetAndMetadata.apply(offset)).build();
    Map<TopicPartition, Errors> commitErrors = groupMetadataManager.storeOffsets(group, memberId, offsets).get();
    assertTrue(group.hasOffsets());
    assertFalse(commitErrors.isEmpty());
    Errors maybeError = commitErrors.get(topicPartition);
    assertEquals(Errors.NONE, maybeError);
    assertTrue(group.hasOffsets());
    Map<TopicPartition, PartitionData> cachedOffsets = groupMetadataManager.getOffsets(groupId, Optional.of(Lists.newArrayList(topicPartition)));
    PartitionData maybePartitionResponse = cachedOffsets.get(topicPartition);
    assertNotNull(maybePartitionResponse);
    assertEquals(Errors.NONE, maybePartitionResponse.error);
    assertEquals(offset, maybePartitionResponse.offset);
    Message<ByteBuffer> message = consumer.receive();
    while (message.getValue().array().length == 0) {
        // bypass above place holder message.
        message = consumer.receive();
    }
    assertTrue(message.getEventTime() > 0L);
    assertTrue(message.hasKey());
    byte[] key = message.getKeyBytes();
    BaseKey groupKey = GroupMetadataConstants.readMessageKey(ByteBuffer.wrap(key));
    assertTrue(groupKey instanceof OffsetKey);
    ByteBuffer value = message.getValue();
    MemoryRecords memRecords = MemoryRecords.readableRecords(value);
    AtomicBoolean verified = new AtomicBoolean(false);
    memRecords.batches().forEach(batch -> {
        for (Record record : batch) {
            assertFalse(verified.get());
            BaseKey bk = GroupMetadataConstants.readMessageKey(record.key());
            assertTrue(bk instanceof OffsetKey);
            OffsetKey ok = (OffsetKey) bk;
            GroupTopicPartition gtp = ok.key();
            assertEquals(groupId, gtp.group());
            assertEquals(new TopicPartition(new KopTopic(topicPartition.topic(), NAMESPACE_PREFIX).getFullName(), topicPartition.partition()), gtp.topicPartition());
            OffsetAndMetadata gm = GroupMetadataConstants.readOffsetMessageValue(record.value());
            assertEquals(offset, gm.offset());
            verified.set(true);
        }
    });
    assertTrue(verified.get());
}

Example 5 with OffsetAndMetadata

use of io.streamnative.pulsar.handlers.kop.offset.OffsetAndMetadata in project starlight-for-kafka by datastax.

the class GroupMetadataManagerTest method testOffsetWriteAfterGroupRemoved.

@Test
public void testOffsetWriteAfterGroupRemoved() throws Exception {
    // this test case checks the following scenario:
    // 1. the group exists at some point in time, but is later removed (because all members left)
    // 2. a "simple" consumer (i.e. not a consumer group) then uses the same groupId to commit some offsets
    int generation = 293;
    String memberId = "98098230493";
    String protocolType = "consumer";
    String protocol = "range";
    Map<TopicPartition, Long> committedOffsets = new HashMap<>();
    committedOffsets.put(new TopicPartition("foo", 0), 23L);
    committedOffsets.put(new TopicPartition("foo", 1), 455L);
    committedOffsets.put(new TopicPartition("bar", 0), 8992L);
    List<SimpleRecord> offsetCommitRecords = createCommittedOffsetRecords(committedOffsets, groupId, NAMESPACE_PREFIX);
    SimpleRecord groupMetadataRecord = buildStableGroupRecordWithMember(generation, protocolType, protocol, memberId);
    SimpleRecord groupMetadataTombstone = new SimpleRecord(groupMetadataKey(groupId), null);
    List<SimpleRecord> newOffsetCommitRecords = new ArrayList<>();
    newOffsetCommitRecords.add(groupMetadataRecord);
    newOffsetCommitRecords.add(groupMetadataTombstone);
    newOffsetCommitRecords.addAll(offsetCommitRecords);
    ByteBuffer buffer = newMemoryRecordsBuffer(newOffsetCommitRecords);
    byte[] key = groupMetadataKey(groupId);
    int consumerGroupPartitionId = GroupMetadataManager.getPartitionId(groupId, conf.getOffsetsTopicNumPartitions());
    Producer<ByteBuffer> producer = groupMetadataManager.getOffsetsTopicProducer(consumerGroupPartitionId).get();
    producer.newMessage().keyBytes(key).value(buffer).eventTime(Time.SYSTEM.milliseconds()).send();
    groupMetadataManager.removeLoadingPartition(consumerGroupPartitionId);
    CompletableFuture<GroupMetadata> onLoadedFuture = new CompletableFuture<>();
    groupMetadataManager.scheduleLoadGroupAndOffsets(consumerGroupPartitionId, groupMetadata -> onLoadedFuture.complete(groupMetadata)).get();
    GroupMetadata group = onLoadedFuture.get();
    GroupMetadata groupInCache = groupMetadataManager.getGroup(groupId).orElseGet(() -> {
        fail("Group was not loaded into the cache");
        return null;
    });
    assertSame(group, groupInCache);
    assertEquals(groupId, group.groupId());
    assertEquals(Empty, group.currentState());
    assertEquals(committedOffsets.size(), group.allOffsets().size());
    committedOffsets.forEach((tp, offset) -> {
        assertEquals(Optional.of(offset), group.offset(tp, NAMESPACE_PREFIX).map(OffsetAndMetadata::offset));
    });
}