Examples with BaseKey io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.BaseKey used on opensource projects

Example 1 with BaseKey

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.BaseKey in project starlight-for-kafka by datastax.

the class GroupMetadataManagerTest method testStoreNoneEmptyGroup.

@Test
public void testStoreNoneEmptyGroup() throws Exception {
    @Cleanup Consumer<ByteBuffer> consumer = pulsarClient.newConsumer(Schema.BYTEBUFFER).topic(groupMetadataManager.getTopicPartitionName()).subscriptionName("test-sub").subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe();
    String memberId = "memberId";
    String clientId = "clientId";
    String clientHost = "localhost";
    GroupMetadata group = new GroupMetadata(groupId, Empty);
    groupMetadataManager.addGroup(group);
    Map<String, byte[]> protocols = new HashMap<>();
    protocols.put("protocol", new byte[0]);
    MemberMetadata member = new MemberMetadata(memberId, groupId, clientId, clientHost, rebalanceTimeout, sessionTimeout, protocolType, protocols);
    CompletableFuture<JoinGroupResult> joinFuture = new CompletableFuture<>();
    member.awaitingJoinCallback(joinFuture);
    group.add(member);
    group.transitionTo(GroupState.PreparingRebalance);
    group.initNextGeneration();
    Map<String, byte[]> assignments = new HashMap<>();
    assignments.put(memberId, new byte[0]);
    Errors errors = groupMetadataManager.storeGroup(group, assignments).get();
    assertEquals(Errors.NONE, errors);
    Message<ByteBuffer> message = consumer.receive();
    while (message.getValue().array().length == 0) {
        // bypass above place holder message.
        message = consumer.receive();
    }
    assertTrue(message.getEventTime() > 0L);
    assertTrue(message.hasKey());
    byte[] key = message.getKeyBytes();
    BaseKey groupKey = GroupMetadataConstants.readMessageKey(ByteBuffer.wrap(key));
    assertTrue(groupKey instanceof GroupMetadataKey);
    GroupMetadataKey groupMetadataKey = (GroupMetadataKey) groupKey;
    assertEquals(groupId, groupMetadataKey.key());
    ByteBuffer value = message.getValue();
    MemoryRecords memRecords = MemoryRecords.readableRecords(value);
    AtomicBoolean verified = new AtomicBoolean(false);
    memRecords.batches().forEach(batch -> {
        for (Record record : batch) {
            assertFalse(verified.get());
            BaseKey bk = GroupMetadataConstants.readMessageKey(record.key());
            assertTrue(bk instanceof GroupMetadataKey);
            GroupMetadataKey gmk = (GroupMetadataKey) bk;
            assertEquals(groupId, gmk.key());
            GroupMetadata gm = GroupMetadataConstants.readGroupMessageValue(groupId, record.value());
            assertEquals(Stable, gm.currentState());
            assertEquals(1, gm.generationId());
            assertEquals(Optional.of(protocolType), gm.protocolType());
            assertEquals("protocol", gm.protocolOrNull());
            assertTrue(gm.has(memberId));
            verified.set(true);
        }
    });
    assertTrue(verified.get());
}

Example 2 with BaseKey

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.BaseKey in project starlight-for-kafka by datastax.

the class GroupMetadataManagerTest method testGroupMetadataRemoval.

@Test
public void testGroupMetadataRemoval() throws Exception {
    @Cleanup Consumer<ByteBuffer> consumer = pulsarClient.newConsumer(Schema.BYTEBUFFER).topic(groupMetadataManager.getTopicPartitionName()).subscriptionName("test-sub").subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe();
    TopicPartition topicPartition1 = new TopicPartition("foo", 0);
    TopicPartition topicPartition2 = new TopicPartition("foo", 1);
    groupMetadataManager.addPartitionOwnership(groupPartitionId);
    GroupMetadata group = new GroupMetadata(groupId, Empty);
    groupMetadataManager.addGroup(group);
    group.generationId(5);
    groupMetadataManager.cleanupGroupMetadata().get();
    Message<ByteBuffer> message = consumer.receive();
    while (message.getValue().array().length == 0) {
        // bypass above place holder message.
        message = consumer.receive();
    }
    assertTrue(message.getEventTime() > 0L);
    assertTrue(message.hasKey());
    byte[] key = message.getKeyBytes();
    BaseKey groupKey = GroupMetadataConstants.readMessageKey(ByteBuffer.wrap(key));
    assertTrue(groupKey instanceof GroupMetadataKey);
    GroupMetadataKey groupMetadataKey = (GroupMetadataKey) groupKey;
    assertEquals(groupId, groupMetadataKey.key());
    ByteBuffer value = message.getValue();
    MemoryRecords memRecords = MemoryRecords.readableRecords(value);
    AtomicBoolean verified = new AtomicBoolean(false);
    memRecords.batches().forEach(batch -> {
        assertEquals(RecordBatch.CURRENT_MAGIC_VALUE, batch.magic());
        assertEquals(TimestampType.CREATE_TIME, batch.timestampType());
        for (Record record : batch) {
            assertFalse(verified.get());
            assertTrue(record.hasKey());
            assertFalse(record.hasValue());
            assertTrue(record.timestamp() > 0);
            BaseKey bk = GroupMetadataConstants.readMessageKey(record.key());
            assertTrue(bk instanceof GroupMetadataKey);
            GroupMetadataKey gmk = (GroupMetadataKey) bk;
            assertEquals(groupId, gmk.key());
            verified.set(true);
        }
    });
    assertTrue(verified.get());
    assertEquals(Optional.empty(), groupMetadataManager.getGroup(groupId));
    Map<TopicPartition, PartitionData> cachedOffsets = groupMetadataManager.getOffsets(groupId, Optional.of(Lists.newArrayList(topicPartition1, topicPartition2)));
    assertEquals(OffsetFetchResponse.INVALID_OFFSET, cachedOffsets.get(topicPartition1).offset);
    assertEquals(OffsetFetchResponse.INVALID_OFFSET, cachedOffsets.get(topicPartition2).offset);
}

Example 3 with BaseKey

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.BaseKey in project starlight-for-kafka by datastax.

the class GroupMetadataManagerTest method testCommitOffset.

@Test
public void testCommitOffset() throws Exception {
    @Cleanup Consumer<ByteBuffer> consumer = pulsarClient.newConsumer(Schema.BYTEBUFFER).topic(groupMetadataManager.getTopicPartitionName()).subscriptionName("test-sub").subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe();
    String memberId = "fakeMemberId";
    TopicPartition topicPartition = new TopicPartition("foo", 0);
    groupMetadataManager.addPartitionOwnership(groupPartitionId);
    long offset = 37L;
    GroupMetadata group = new GroupMetadata(groupId, Empty);
    groupMetadataManager.addGroup(group);
    Map<TopicPartition, OffsetAndMetadata> offsets = ImmutableMap.<TopicPartition, OffsetAndMetadata>builder().put(topicPartition, OffsetAndMetadata.apply(offset)).build();
    Map<TopicPartition, Errors> commitErrors = groupMetadataManager.storeOffsets(group, memberId, offsets).get();
    assertTrue(group.hasOffsets());
    assertFalse(commitErrors.isEmpty());
    Errors maybeError = commitErrors.get(topicPartition);
    assertEquals(Errors.NONE, maybeError);
    assertTrue(group.hasOffsets());
    Map<TopicPartition, PartitionData> cachedOffsets = groupMetadataManager.getOffsets(groupId, Optional.of(Lists.newArrayList(topicPartition)));
    PartitionData maybePartitionResponse = cachedOffsets.get(topicPartition);
    assertNotNull(maybePartitionResponse);
    assertEquals(Errors.NONE, maybePartitionResponse.error);
    assertEquals(offset, maybePartitionResponse.offset);
    Message<ByteBuffer> message = consumer.receive();
    while (message.getValue().array().length == 0) {
        // bypass above place holder message.
        message = consumer.receive();
    }
    assertTrue(message.getEventTime() > 0L);
    assertTrue(message.hasKey());
    byte[] key = message.getKeyBytes();
    BaseKey groupKey = GroupMetadataConstants.readMessageKey(ByteBuffer.wrap(key));
    assertTrue(groupKey instanceof OffsetKey);
    ByteBuffer value = message.getValue();
    MemoryRecords memRecords = MemoryRecords.readableRecords(value);
    AtomicBoolean verified = new AtomicBoolean(false);
    memRecords.batches().forEach(batch -> {
        for (Record record : batch) {
            assertFalse(verified.get());
            BaseKey bk = GroupMetadataConstants.readMessageKey(record.key());
            assertTrue(bk instanceof OffsetKey);
            OffsetKey ok = (OffsetKey) bk;
            GroupTopicPartition gtp = ok.key();
            assertEquals(groupId, gtp.group());
            assertEquals(new TopicPartition(new KopTopic(topicPartition.topic(), NAMESPACE_PREFIX).getFullName(), topicPartition.partition()), gtp.topicPartition());
            OffsetAndMetadata gm = GroupMetadataConstants.readOffsetMessageValue(record.value());
            assertEquals(offset, gm.offset());
            verified.set(true);
        }
    });
    assertTrue(verified.get());
}

Example 4 with BaseKey

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.BaseKey in project starlight-for-kafka by datastax.

the class GroupMetadataManagerTest method testStoreEmptySimpleGroup.

@Test
public void testStoreEmptySimpleGroup() throws Exception {
    @Cleanup Consumer<ByteBuffer> consumer = pulsarClient.newConsumer(Schema.BYTEBUFFER).topic(groupMetadataManager.getTopicPartitionName()).subscriptionName("test-sub").subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe();
    GroupMetadata group = new GroupMetadata(groupId, Empty);
    groupMetadataManager.addGroup(group);
    Errors errors = groupMetadataManager.storeGroup(group, Collections.emptyMap()).get();
    assertEquals(Errors.NONE, errors);
    Message<ByteBuffer> message = consumer.receive();
    while (message.getValue().array().length == 0) {
        // bypass above place holder message.
        message = consumer.receive();
    }
    assertTrue(message.getEventTime() > 0L);
    assertTrue(message.hasKey());
    byte[] key = message.getKeyBytes();
    BaseKey groupKey = GroupMetadataConstants.readMessageKey(ByteBuffer.wrap(key));
    assertTrue(groupKey instanceof GroupMetadataKey);
    GroupMetadataKey groupMetadataKey = (GroupMetadataKey) groupKey;
    assertEquals(groupId, groupMetadataKey.key());
    ByteBuffer value = message.getValue();
    MemoryRecords memRecords = MemoryRecords.readableRecords(value);
    AtomicBoolean verified = new AtomicBoolean(false);
    memRecords.batches().forEach(batch -> {
        for (Record record : batch) {
            assertFalse(verified.get());
            BaseKey bk = GroupMetadataConstants.readMessageKey(record.key());
            assertTrue(bk instanceof GroupMetadataKey);
            GroupMetadataKey gmk = (GroupMetadataKey) bk;
            assertEquals(groupId, gmk.key());
            GroupMetadata gm = GroupMetadataConstants.readGroupMessageValue(groupId, record.value());
            assertTrue(gm.is(Empty));
            assertEquals(0, gm.generationId());
            assertEquals(Optional.empty(), gm.protocolType());
            verified.set(true);
        }
    });
    assertTrue(verified.get());
}

Example 5 with BaseKey

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.BaseKey in project starlight-for-kafka by datastax.

the class OffsetResetTest method readFromOffsetMessagePulsar.

private void readFromOffsetMessagePulsar() throws Exception {
    Reader<ByteBuffer> reader = pulsarClient.newReader(Schema.BYTEBUFFER).startMessageId(MessageId.earliest).topic("persistent://public/default/__consumer_offsets" + PARTITIONED_TOPIC_SUFFIX + "0").readCompacted(true).create();
    log.info("start reading __consumer_offsets:{}", reader.getTopic());
    while (true) {
        Message<ByteBuffer> offsetMetadata = reader.readNext(1, TimeUnit.SECONDS);
        if (offsetMetadata == null) {
            break;
        }
        MemoryRecords memRecords = MemoryRecords.readableRecords(offsetMetadata.getValue());
        log.info("__consumer_offsets MessageId:{}", offsetMetadata.getMessageId());
        for (MutableRecordBatch batch : memRecords.batches()) {
            for (Record record : batch) {
                BaseKey baseKey = GroupMetadataConstants.readMessageKey(record.key());
                if (baseKey != null && (baseKey instanceof OffsetKey)) {
                    OffsetKey offsetKey = (OffsetKey) baseKey;
                    String formattedValue = String.valueOf(GroupMetadataConstants.readOffsetMessageValue(record.value()));
                    log.info("__consumer_offsets - key:{}, value:{}", offsetKey, formattedValue);
                }
            }
        }
    }
    reader.close();
}