Examples with GroupMetadataKey io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.GroupMetadataKey used on opensource projects

Example 1 with GroupMetadataKey

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.GroupMetadataKey in project starlight-for-kafka by datastax.

the class GroupMetadataManagerTest method testGroupLoadWithConsumerAndTransactionalOffsetCommitsTransactionWins.

@Test
public void testGroupLoadWithConsumerAndTransactionalOffsetCommitsTransactionWins() throws Exception {
    long producerId = 1000L;
    short producerEpoch = 2;
    Map<TopicPartition, Long> transactionalOffsetCommits = new HashMap<>();
    transactionalOffsetCommits.put(new TopicPartition("foo", 0), 23L);
    Map<TopicPartition, Long> consumerOffsetCommits = new HashMap<>();
    consumerOffsetCommits.put(new TopicPartition("foo", 0), 24L);
    ByteBuffer buffer = ByteBuffer.allocate(1024);
    int nextOffset = 0;
    nextOffset += appendConsumerOffsetCommit(buffer, nextOffset, consumerOffsetCommits);
    nextOffset += appendTransactionalOffsetCommits(buffer, producerId, producerEpoch, nextOffset, transactionalOffsetCommits, NAMESPACE_PREFIX);
    nextOffset += completeTransactionalOffsetCommit(buffer, producerId, producerEpoch, nextOffset, true);
    buffer.flip();
    byte[] key = groupMetadataKey(groupId);
    Producer<ByteBuffer> producer = groupMetadataManager.getOffsetsTopicProducer(groupPartitionId).get();
    producer.newMessage().keyBytes(key).value(buffer).eventTime(Time.SYSTEM.milliseconds()).send();
    CompletableFuture<GroupMetadata> onLoadedFuture = new CompletableFuture<>();
    groupMetadataManager.scheduleLoadGroupAndOffsets(groupPartitionId, groupMetadata -> onLoadedFuture.complete(groupMetadata)).get();
    GroupMetadata group = onLoadedFuture.get();
    GroupMetadata groupInCache = groupMetadataManager.getGroup(groupId).orElseGet(() -> {
        fail("Group was not loaded into the cache");
        return null;
    });
    assertSame(group, groupInCache);
    assertEquals(groupId, group.groupId());
    assertEquals(Empty, group.currentState());
    // The group should be loaded with pending offsets.
    assertEquals(1, group.allOffsets().size());
    assertTrue(group.hasOffsets());
    assertFalse(group.hasPendingOffsetCommitsFromProducer(producerId));
    assertEquals(consumerOffsetCommits.size(), group.allOffsets().size());
    transactionalOffsetCommits.forEach((tp, offset) -> {
        assertEquals(Optional.of(offset), group.offset(tp, NAMESPACE_PREFIX).map(OffsetAndMetadata::offset));
    });
}

Example 2 with GroupMetadataKey

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.GroupMetadataKey in project starlight-for-kafka by datastax.

the class GroupMetadataManagerTest method testStoreNoneEmptyGroup.

@Test
public void testStoreNoneEmptyGroup() throws Exception {
    @Cleanup Consumer<ByteBuffer> consumer = pulsarClient.newConsumer(Schema.BYTEBUFFER).topic(groupMetadataManager.getTopicPartitionName()).subscriptionName("test-sub").subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe();
    String memberId = "memberId";
    String clientId = "clientId";
    String clientHost = "localhost";
    GroupMetadata group = new GroupMetadata(groupId, Empty);
    groupMetadataManager.addGroup(group);
    Map<String, byte[]> protocols = new HashMap<>();
    protocols.put("protocol", new byte[0]);
    MemberMetadata member = new MemberMetadata(memberId, groupId, clientId, clientHost, rebalanceTimeout, sessionTimeout, protocolType, protocols);
    CompletableFuture<JoinGroupResult> joinFuture = new CompletableFuture<>();
    member.awaitingJoinCallback(joinFuture);
    group.add(member);
    group.transitionTo(GroupState.PreparingRebalance);
    group.initNextGeneration();
    Map<String, byte[]> assignments = new HashMap<>();
    assignments.put(memberId, new byte[0]);
    Errors errors = groupMetadataManager.storeGroup(group, assignments).get();
    assertEquals(Errors.NONE, errors);
    Message<ByteBuffer> message = consumer.receive();
    while (message.getValue().array().length == 0) {
        // bypass above place holder message.
        message = consumer.receive();
    }
    assertTrue(message.getEventTime() > 0L);
    assertTrue(message.hasKey());
    byte[] key = message.getKeyBytes();
    BaseKey groupKey = GroupMetadataConstants.readMessageKey(ByteBuffer.wrap(key));
    assertTrue(groupKey instanceof GroupMetadataKey);
    GroupMetadataKey groupMetadataKey = (GroupMetadataKey) groupKey;
    assertEquals(groupId, groupMetadataKey.key());
    ByteBuffer value = message.getValue();
    MemoryRecords memRecords = MemoryRecords.readableRecords(value);
    AtomicBoolean verified = new AtomicBoolean(false);
    memRecords.batches().forEach(batch -> {
        for (Record record : batch) {
            assertFalse(verified.get());
            BaseKey bk = GroupMetadataConstants.readMessageKey(record.key());
            assertTrue(bk instanceof GroupMetadataKey);
            GroupMetadataKey gmk = (GroupMetadataKey) bk;
            assertEquals(groupId, gmk.key());
            GroupMetadata gm = GroupMetadataConstants.readGroupMessageValue(groupId, record.value());
            assertEquals(Stable, gm.currentState());
            assertEquals(1, gm.generationId());
            assertEquals(Optional.of(protocolType), gm.protocolType());
            assertEquals("protocol", gm.protocolOrNull());
            assertTrue(gm.has(memberId));
            verified.set(true);
        }
    });
    assertTrue(verified.get());
}

Example 3 with GroupMetadataKey

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.GroupMetadataKey in project starlight-for-kafka by datastax.

the class GroupMetadataManagerTest method testLoadTransactionalOffsetCommitsFromMultipleProducers.

@Test
public void testLoadTransactionalOffsetCommitsFromMultipleProducers() throws Exception {
    long firstProducerId = 1000L;
    short firstProducerEpoch = 2;
    long secondProducerId = 1001L;
    short secondProducerEpoch = 3;
    Map<TopicPartition, Long> committedOffsetsFirstProducer = new HashMap<>();
    committedOffsetsFirstProducer.put(new TopicPartition("foo", 0), 23L);
    committedOffsetsFirstProducer.put(new TopicPartition("foo", 1), 455L);
    committedOffsetsFirstProducer.put(new TopicPartition("bar", 0), 8992L);
    Map<TopicPartition, Long> committedOffsetsSecondProducer = new HashMap<>();
    committedOffsetsSecondProducer.put(new TopicPartition("foo", 2), 231L);
    committedOffsetsSecondProducer.put(new TopicPartition("foo", 3), 4551L);
    committedOffsetsSecondProducer.put(new TopicPartition("bar", 1), 89921L);
    ByteBuffer buffer = ByteBuffer.allocate(1024);
    int nextOffset = 0;
    int firstProduceRecordOffset = nextOffset;
    nextOffset += appendTransactionalOffsetCommits(buffer, firstProducerId, firstProducerEpoch, nextOffset, committedOffsetsFirstProducer, NAMESPACE_PREFIX);
    nextOffset += completeTransactionalOffsetCommit(buffer, firstProducerId, firstProducerEpoch, nextOffset, true);
    int secondProduceRecordOffset = nextOffset;
    nextOffset += appendTransactionalOffsetCommits(buffer, secondProducerId, secondProducerEpoch, nextOffset, committedOffsetsSecondProducer, NAMESPACE_PREFIX);
    nextOffset += completeTransactionalOffsetCommit(buffer, secondProducerId, secondProducerEpoch, nextOffset, true);
    buffer.flip();
    byte[] key = groupMetadataKey(groupId);
    Producer<ByteBuffer> producer = groupMetadataManager.getOffsetsTopicProducer(groupPartitionId).get();
    producer.newMessage().keyBytes(key).value(buffer).eventTime(Time.SYSTEM.milliseconds()).send();
    CompletableFuture<GroupMetadata> onLoadedFuture = new CompletableFuture<>();
    groupMetadataManager.scheduleLoadGroupAndOffsets(groupPartitionId, groupMetadata -> onLoadedFuture.complete(groupMetadata)).get();
    GroupMetadata group = onLoadedFuture.get();
    GroupMetadata groupInCache = groupMetadataManager.getGroup(groupId).orElseGet(() -> {
        fail("Group was not loaded into the cache");
        return null;
    });
    assertSame(group, groupInCache);
    assertEquals(groupId, group.groupId());
    assertEquals(Empty, group.currentState());
    // Ensure that only the committed offsets are materialized, and that there are no pending commits
    // for the producer. This allows us to be certain that the aborted offset commits are truly discarded.
    assertEquals(committedOffsetsFirstProducer.size() + committedOffsetsSecondProducer.size(), group.allOffsets().size());
    committedOffsetsFirstProducer.forEach((tp, offset) -> {
        assertEquals(Optional.of(offset), group.offset(tp, NAMESPACE_PREFIX).map(OffsetAndMetadata::offset));
        assertEquals(Optional.of((long) firstProduceRecordOffset), group.offsetWithRecordMetadata(tp).flatMap(CommitRecordMetadataAndOffset::appendedBatchOffset));
    });
    committedOffsetsSecondProducer.forEach((tp, offset) -> {
        assertEquals(Optional.of(offset), group.offset(tp, NAMESPACE_PREFIX).map(OffsetAndMetadata::offset));
        assertEquals(Optional.of((long) secondProduceRecordOffset), group.offsetWithRecordMetadata(tp).flatMap(CommitRecordMetadataAndOffset::appendedBatchOffset));
    });
}

Example 4 with GroupMetadataKey

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.GroupMetadataKey in project starlight-for-kafka by datastax.

the class GroupMetadataManagerTest method testGroupMetadataRemoval.

@Test
public void testGroupMetadataRemoval() throws Exception {
    @Cleanup Consumer<ByteBuffer> consumer = pulsarClient.newConsumer(Schema.BYTEBUFFER).topic(groupMetadataManager.getTopicPartitionName()).subscriptionName("test-sub").subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe();
    TopicPartition topicPartition1 = new TopicPartition("foo", 0);
    TopicPartition topicPartition2 = new TopicPartition("foo", 1);
    groupMetadataManager.addPartitionOwnership(groupPartitionId);
    GroupMetadata group = new GroupMetadata(groupId, Empty);
    groupMetadataManager.addGroup(group);
    group.generationId(5);
    groupMetadataManager.cleanupGroupMetadata().get();
    Message<ByteBuffer> message = consumer.receive();
    while (message.getValue().array().length == 0) {
        // bypass above place holder message.
        message = consumer.receive();
    }
    assertTrue(message.getEventTime() > 0L);
    assertTrue(message.hasKey());
    byte[] key = message.getKeyBytes();
    BaseKey groupKey = GroupMetadataConstants.readMessageKey(ByteBuffer.wrap(key));
    assertTrue(groupKey instanceof GroupMetadataKey);
    GroupMetadataKey groupMetadataKey = (GroupMetadataKey) groupKey;
    assertEquals(groupId, groupMetadataKey.key());
    ByteBuffer value = message.getValue();
    MemoryRecords memRecords = MemoryRecords.readableRecords(value);
    AtomicBoolean verified = new AtomicBoolean(false);
    memRecords.batches().forEach(batch -> {
        assertEquals(RecordBatch.CURRENT_MAGIC_VALUE, batch.magic());
        assertEquals(TimestampType.CREATE_TIME, batch.timestampType());
        for (Record record : batch) {
            assertFalse(verified.get());
            assertTrue(record.hasKey());
            assertFalse(record.hasValue());
            assertTrue(record.timestamp() > 0);
            BaseKey bk = GroupMetadataConstants.readMessageKey(record.key());
            assertTrue(bk instanceof GroupMetadataKey);
            GroupMetadataKey gmk = (GroupMetadataKey) bk;
            assertEquals(groupId, gmk.key());
            verified.set(true);
        }
    });
    assertTrue(verified.get());
    assertEquals(Optional.empty(), groupMetadataManager.getGroup(groupId));
    Map<TopicPartition, PartitionData> cachedOffsets = groupMetadataManager.getOffsets(groupId, Optional.of(Lists.newArrayList(topicPartition1, topicPartition2)));
    assertEquals(OffsetFetchResponse.INVALID_OFFSET, cachedOffsets.get(topicPartition1).offset);
    assertEquals(OffsetFetchResponse.INVALID_OFFSET, cachedOffsets.get(topicPartition2).offset);
}

Example 5 with GroupMetadataKey

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.GroupMetadataKey in project starlight-for-kafka by datastax.

the class GroupMetadataManagerTest method testLoadOffsetsWithTombstones.

@Test
public void testLoadOffsetsWithTombstones() throws Exception {
    TopicPartition tombstonePartition = new TopicPartition("foo", 1);
    Map<TopicPartition, Long> committedOffsets = new HashMap<>();
    committedOffsets.put(new TopicPartition("foo", 0), 23L);
    committedOffsets.put(tombstonePartition, 455L);
    committedOffsets.put(new TopicPartition("bar", 0), 8992L);
    List<SimpleRecord> offsetCommitRecords = createCommittedOffsetRecords(committedOffsets, groupId, NAMESPACE_PREFIX);
    SimpleRecord tombstone = new SimpleRecord(offsetCommitKey(groupId, tombstonePartition, NAMESPACE_PREFIX), null);
    offsetCommitRecords.add(tombstone);
    ByteBuffer buffer = newMemoryRecordsBuffer(offsetCommitRecords);
    byte[] key = groupMetadataKey(groupId);
    Producer<ByteBuffer> producer = groupMetadataManager.getOffsetsTopicProducer(groupPartitionId).get();
    producer.newMessage().keyBytes(key).value(buffer).eventTime(Time.SYSTEM.milliseconds()).send();
    CompletableFuture<GroupMetadata> onLoadedFuture = new CompletableFuture<>();
    groupMetadataManager.scheduleLoadGroupAndOffsets(groupPartitionId, groupMetadata -> onLoadedFuture.complete(groupMetadata)).get();
    GroupMetadata group = onLoadedFuture.get();
    GroupMetadata groupInCache = groupMetadataManager.getGroup(groupId).orElseGet(() -> {
        fail("Group was not loaded into the cache");
        return null;
    });
    assertSame(group, groupInCache);
    assertEquals(groupId, group.groupId());
    assertEquals(Empty, group.currentState());
    // The group should be loaded with pending offsets.
    assertEquals(committedOffsets.size() - 1, group.allOffsets().size());
    committedOffsets.forEach((tp, offset) -> {
        if (tp == tombstonePartition) {
            assertEquals(Optional.empty(), group.offset(tp, NAMESPACE_PREFIX));
        } else {
            assertEquals(Optional.of(offset), group.offset(tp, NAMESPACE_PREFIX).map(OffsetAndMetadata::offset));
        }
    });
}