Search in sources :

Example 1 with GroupTopicPartition

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.GroupTopicPartition in project starlight-for-kafka by datastax.

the class GroupMetadataManagerTest method testCommitOffset.

@Test
public void testCommitOffset() throws Exception {
    @Cleanup Consumer<ByteBuffer> consumer = pulsarClient.newConsumer(Schema.BYTEBUFFER).topic(groupMetadataManager.getTopicPartitionName()).subscriptionName("test-sub").subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe();
    String memberId = "fakeMemberId";
    TopicPartition topicPartition = new TopicPartition("foo", 0);
    groupMetadataManager.addPartitionOwnership(groupPartitionId);
    long offset = 37L;
    GroupMetadata group = new GroupMetadata(groupId, Empty);
    groupMetadataManager.addGroup(group);
    Map<TopicPartition, OffsetAndMetadata> offsets = ImmutableMap.<TopicPartition, OffsetAndMetadata>builder().put(topicPartition, OffsetAndMetadata.apply(offset)).build();
    Map<TopicPartition, Errors> commitErrors = groupMetadataManager.storeOffsets(group, memberId, offsets).get();
    assertTrue(group.hasOffsets());
    assertFalse(commitErrors.isEmpty());
    Errors maybeError = commitErrors.get(topicPartition);
    assertEquals(Errors.NONE, maybeError);
    assertTrue(group.hasOffsets());
    Map<TopicPartition, PartitionData> cachedOffsets = groupMetadataManager.getOffsets(groupId, Optional.of(Lists.newArrayList(topicPartition)));
    PartitionData maybePartitionResponse = cachedOffsets.get(topicPartition);
    assertNotNull(maybePartitionResponse);
    assertEquals(Errors.NONE, maybePartitionResponse.error);
    assertEquals(offset, maybePartitionResponse.offset);
    Message<ByteBuffer> message = consumer.receive();
    while (message.getValue().array().length == 0) {
        // bypass above place holder message.
        message = consumer.receive();
    }
    assertTrue(message.getEventTime() > 0L);
    assertTrue(message.hasKey());
    byte[] key = message.getKeyBytes();
    BaseKey groupKey = GroupMetadataConstants.readMessageKey(ByteBuffer.wrap(key));
    assertTrue(groupKey instanceof OffsetKey);
    ByteBuffer value = message.getValue();
    MemoryRecords memRecords = MemoryRecords.readableRecords(value);
    AtomicBoolean verified = new AtomicBoolean(false);
    memRecords.batches().forEach(batch -> {
        for (Record record : batch) {
            assertFalse(verified.get());
            BaseKey bk = GroupMetadataConstants.readMessageKey(record.key());
            assertTrue(bk instanceof OffsetKey);
            OffsetKey ok = (OffsetKey) bk;
            GroupTopicPartition gtp = ok.key();
            assertEquals(groupId, gtp.group());
            assertEquals(new TopicPartition(new KopTopic(topicPartition.topic(), NAMESPACE_PREFIX).getFullName(), topicPartition.partition()), gtp.topicPartition());
            OffsetAndMetadata gm = GroupMetadataConstants.readOffsetMessageValue(record.value());
            assertEquals(offset, gm.offset());
            verified.set(true);
        }
    });
    assertTrue(verified.get());
}
Also used : BaseKey(io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.BaseKey) Cleanup(lombok.Cleanup) ByteBuffer(java.nio.ByteBuffer) GroupTopicPartition(io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.GroupTopicPartition) Errors(org.apache.kafka.common.protocol.Errors) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) PartitionData(org.apache.kafka.common.requests.OffsetFetchResponse.PartitionData) TopicPartition(org.apache.kafka.common.TopicPartition) GroupTopicPartition(io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.GroupTopicPartition) OffsetAndMetadata(io.streamnative.pulsar.handlers.kop.offset.OffsetAndMetadata) OffsetKey(io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.OffsetKey) Record(org.apache.kafka.common.record.Record) SimpleRecord(org.apache.kafka.common.record.SimpleRecord) KopTopic(io.streamnative.pulsar.handlers.kop.utils.KopTopic) MemoryRecords(org.apache.kafka.common.record.MemoryRecords) Test(org.testng.annotations.Test)

Example 2 with GroupTopicPartition

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.GroupTopicPartition in project starlight-for-kafka by datastax.

the class GroupMetadataConstants method readMessageKey.

/**
 * Decodes the offset messages' key.
 */
public static BaseKey readMessageKey(ByteBuffer buffer) {
    short version = buffer.getShort();
    Schema keySchema = schemaForKey(version);
    Struct key = keySchema.read(buffer);
    if (version <= CURRENT_OFFSET_KEY_SCHEMA_VERSION) {
        // version 0 and 1 refer to offset
        String group = key.getString(OFFSET_KEY_GROUP_FIELD);
        String topic = key.getString(OFFSET_KEY_TOPIC_FIELD);
        int partition = key.getInt(OFFSET_KEY_PARTITION_FIELD);
        return new OffsetKey(version, new GroupTopicPartition(group, topic, partition));
    } else if (version == CURRENT_GROUP_KEY_SCHEMA_VERSION) {
        // version 2 refers to group
        String group = key.getString(GROUP_KEY_GROUP_FIELD);
        return new GroupMetadataKey(version, group);
    } else {
        throw new IllegalStateException("Unknown version " + version + " for group metadata message");
    }
}
Also used : Schema(org.apache.kafka.common.protocol.types.Schema) OffsetKey(io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.OffsetKey) GroupMetadataKey(io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.GroupMetadataKey) Struct(org.apache.kafka.common.protocol.types.Struct) GroupTopicPartition(io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.GroupTopicPartition)

Example 3 with GroupTopicPartition

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.GroupTopicPartition in project kop by streamnative.

the class GroupMetadataManagerTest method testCommitOffset.

@Test
public void testCommitOffset() throws Exception {
    String memberId = "fakeMemberId";
    TopicPartition topicPartition = new TopicPartition("foo", 0);
    groupMetadataManager.addPartitionOwnership(groupPartitionId);
    long offset = 37L;
    GroupMetadata group = new GroupMetadata(groupId, Empty);
    groupMetadataManager.addGroup(group);
    Map<TopicPartition, OffsetAndMetadata> offsets = ImmutableMap.<TopicPartition, OffsetAndMetadata>builder().put(topicPartition, OffsetAndMetadata.apply(offset)).build();
    Map<TopicPartition, Errors> commitErrors = groupMetadataManager.storeOffsets(group, memberId, offsets).get();
    assertTrue(group.hasOffsets());
    assertFalse(commitErrors.isEmpty());
    Errors maybeError = commitErrors.get(topicPartition);
    assertEquals(Errors.NONE, maybeError);
    assertTrue(group.hasOffsets());
    Map<TopicPartition, PartitionData> cachedOffsets = groupMetadataManager.getOffsets(groupId, Optional.of(Lists.newArrayList(topicPartition)));
    PartitionData maybePartitionResponse = cachedOffsets.get(topicPartition);
    assertNotNull(maybePartitionResponse);
    assertEquals(Errors.NONE, maybePartitionResponse.error);
    assertEquals(offset, maybePartitionResponse.offset);
    Message<ByteBuffer> message = consumer.receive();
    while (message.getValue().array().length == 0) {
        // bypass above place holder message.
        message = consumer.receive();
    }
    assertTrue(message.getEventTime() > 0L);
    assertTrue(message.hasKey());
    byte[] key = message.getKeyBytes();
    BaseKey groupKey = GroupMetadataConstants.readMessageKey(ByteBuffer.wrap(key));
    assertTrue(groupKey instanceof OffsetKey);
    ByteBuffer value = message.getValue();
    MemoryRecords memRecords = MemoryRecords.readableRecords(value);
    AtomicBoolean verified = new AtomicBoolean(false);
    memRecords.batches().forEach(batch -> {
        for (Record record : batch) {
            assertFalse(verified.get());
            BaseKey bk = GroupMetadataConstants.readMessageKey(record.key());
            assertTrue(bk instanceof OffsetKey);
            OffsetKey ok = (OffsetKey) bk;
            GroupTopicPartition gtp = ok.key();
            assertEquals(groupId, gtp.group());
            assertEquals(new TopicPartition(new KopTopic(topicPartition.topic(), NAMESPACE_PREFIX).getFullName(), topicPartition.partition()), gtp.topicPartition());
            OffsetAndMetadata gm = GroupMetadataConstants.readOffsetMessageValue(record.value());
            assertEquals(offset, gm.offset());
            verified.set(true);
        }
    });
    assertTrue(verified.get());
}
Also used : BaseKey(io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.BaseKey) ByteBuffer(java.nio.ByteBuffer) GroupTopicPartition(io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.GroupTopicPartition) Errors(org.apache.kafka.common.protocol.Errors) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) PartitionData(org.apache.kafka.common.requests.OffsetFetchResponse.PartitionData) TopicPartition(org.apache.kafka.common.TopicPartition) GroupTopicPartition(io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.GroupTopicPartition) OffsetAndMetadata(io.streamnative.pulsar.handlers.kop.offset.OffsetAndMetadata) OffsetKey(io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.OffsetKey) Record(org.apache.kafka.common.record.Record) SimpleRecord(org.apache.kafka.common.record.SimpleRecord) KopTopic(io.streamnative.pulsar.handlers.kop.utils.KopTopic) MemoryRecords(org.apache.kafka.common.record.MemoryRecords) Test(org.testng.annotations.Test)

Example 4 with GroupTopicPartition

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.GroupTopicPartition in project kop by streamnative.

the class GroupMetadataConstants method readMessageKey.

/**
 * Decodes the offset messages' key.
 */
public static BaseKey readMessageKey(ByteBuffer buffer) {
    short version = buffer.getShort();
    Schema keySchema = schemaForKey(version);
    Struct key = keySchema.read(buffer);
    if (version <= CURRENT_OFFSET_KEY_SCHEMA_VERSION) {
        // version 0 and 1 refer to offset
        String group = key.getString(OFFSET_KEY_GROUP_FIELD);
        String topic = key.getString(OFFSET_KEY_TOPIC_FIELD);
        int partition = key.getInt(OFFSET_KEY_PARTITION_FIELD);
        return new OffsetKey(version, new GroupTopicPartition(group, topic, partition));
    } else if (version == CURRENT_GROUP_KEY_SCHEMA_VERSION) {
        // version 2 refers to group
        String group = key.getString(GROUP_KEY_GROUP_FIELD);
        return new GroupMetadataKey(version, group);
    } else {
        throw new IllegalStateException("Unknown version " + version + " for group metadata message");
    }
}
Also used : Schema(org.apache.kafka.common.protocol.types.Schema) OffsetKey(io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.OffsetKey) GroupMetadataKey(io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.GroupMetadataKey) Struct(org.apache.kafka.common.protocol.types.Struct) GroupTopicPartition(io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.GroupTopicPartition)

Aggregations

GroupTopicPartition (io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.GroupTopicPartition)4 OffsetKey (io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.OffsetKey)4 BaseKey (io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.BaseKey)2 GroupMetadataKey (io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.GroupMetadataKey)2 OffsetAndMetadata (io.streamnative.pulsar.handlers.kop.offset.OffsetAndMetadata)2 KopTopic (io.streamnative.pulsar.handlers.kop.utils.KopTopic)2 ByteBuffer (java.nio.ByteBuffer)2 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)2 TopicPartition (org.apache.kafka.common.TopicPartition)2 Errors (org.apache.kafka.common.protocol.Errors)2 Schema (org.apache.kafka.common.protocol.types.Schema)2 Struct (org.apache.kafka.common.protocol.types.Struct)2 MemoryRecords (org.apache.kafka.common.record.MemoryRecords)2 Record (org.apache.kafka.common.record.Record)2 SimpleRecord (org.apache.kafka.common.record.SimpleRecord)2 PartitionData (org.apache.kafka.common.requests.OffsetFetchResponse.PartitionData)2 Test (org.testng.annotations.Test)2 Cleanup (lombok.Cleanup)1