Examples with OffsetKey io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.OffsetKey used on opensource projects

Example 1 with OffsetKey

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.OffsetKey in project starlight-for-kafka by datastax.

the class GroupMetadataManagerTest method testCommitOffset.

@Test
public void testCommitOffset() throws Exception {
    @Cleanup Consumer<ByteBuffer> consumer = pulsarClient.newConsumer(Schema.BYTEBUFFER).topic(groupMetadataManager.getTopicPartitionName()).subscriptionName("test-sub").subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe();
    String memberId = "fakeMemberId";
    TopicPartition topicPartition = new TopicPartition("foo", 0);
    groupMetadataManager.addPartitionOwnership(groupPartitionId);
    long offset = 37L;
    GroupMetadata group = new GroupMetadata(groupId, Empty);
    groupMetadataManager.addGroup(group);
    Map<TopicPartition, OffsetAndMetadata> offsets = ImmutableMap.<TopicPartition, OffsetAndMetadata>builder().put(topicPartition, OffsetAndMetadata.apply(offset)).build();
    Map<TopicPartition, Errors> commitErrors = groupMetadataManager.storeOffsets(group, memberId, offsets).get();
    assertTrue(group.hasOffsets());
    assertFalse(commitErrors.isEmpty());
    Errors maybeError = commitErrors.get(topicPartition);
    assertEquals(Errors.NONE, maybeError);
    assertTrue(group.hasOffsets());
    Map<TopicPartition, PartitionData> cachedOffsets = groupMetadataManager.getOffsets(groupId, Optional.of(Lists.newArrayList(topicPartition)));
    PartitionData maybePartitionResponse = cachedOffsets.get(topicPartition);
    assertNotNull(maybePartitionResponse);
    assertEquals(Errors.NONE, maybePartitionResponse.error);
    assertEquals(offset, maybePartitionResponse.offset);
    Message<ByteBuffer> message = consumer.receive();
    while (message.getValue().array().length == 0) {
        // bypass above place holder message.
        message = consumer.receive();
    }
    assertTrue(message.getEventTime() > 0L);
    assertTrue(message.hasKey());
    byte[] key = message.getKeyBytes();
    BaseKey groupKey = GroupMetadataConstants.readMessageKey(ByteBuffer.wrap(key));
    assertTrue(groupKey instanceof OffsetKey);
    ByteBuffer value = message.getValue();
    MemoryRecords memRecords = MemoryRecords.readableRecords(value);
    AtomicBoolean verified = new AtomicBoolean(false);
    memRecords.batches().forEach(batch -> {
        for (Record record : batch) {
            assertFalse(verified.get());
            BaseKey bk = GroupMetadataConstants.readMessageKey(record.key());
            assertTrue(bk instanceof OffsetKey);
            OffsetKey ok = (OffsetKey) bk;
            GroupTopicPartition gtp = ok.key();
            assertEquals(groupId, gtp.group());
            assertEquals(new TopicPartition(new KopTopic(topicPartition.topic(), NAMESPACE_PREFIX).getFullName(), topicPartition.partition()), gtp.topicPartition());
            OffsetAndMetadata gm = GroupMetadataConstants.readOffsetMessageValue(record.value());
            assertEquals(offset, gm.offset());
            verified.set(true);
        }
    });
    assertTrue(verified.get());
}

Example 2 with OffsetKey

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.OffsetKey in project starlight-for-kafka by datastax.

the class OffsetResetTest method readFromOffsetMessagePulsar.

private void readFromOffsetMessagePulsar() throws Exception {
    Reader<ByteBuffer> reader = pulsarClient.newReader(Schema.BYTEBUFFER).startMessageId(MessageId.earliest).topic("persistent://public/default/__consumer_offsets" + PARTITIONED_TOPIC_SUFFIX + "0").readCompacted(true).create();
    log.info("start reading __consumer_offsets:{}", reader.getTopic());
    while (true) {
        Message<ByteBuffer> offsetMetadata = reader.readNext(1, TimeUnit.SECONDS);
        if (offsetMetadata == null) {
            break;
        }
        MemoryRecords memRecords = MemoryRecords.readableRecords(offsetMetadata.getValue());
        log.info("__consumer_offsets MessageId:{}", offsetMetadata.getMessageId());
        for (MutableRecordBatch batch : memRecords.batches()) {
            for (Record record : batch) {
                BaseKey baseKey = GroupMetadataConstants.readMessageKey(record.key());
                if (baseKey != null && (baseKey instanceof OffsetKey)) {
                    OffsetKey offsetKey = (OffsetKey) baseKey;
                    String formattedValue = String.valueOf(GroupMetadataConstants.readOffsetMessageValue(record.value()));
                    log.info("__consumer_offsets - key:{}, value:{}", offsetKey, formattedValue);
                }
            }
        }
    }
    reader.close();
}

Example 3 with OffsetKey

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.OffsetKey in project starlight-for-kafka by datastax.

the class GroupMetadataConstants method readMessageKey.

/**
 * Decodes the offset messages' key.
 */
public static BaseKey readMessageKey(ByteBuffer buffer) {
    short version = buffer.getShort();
    Schema keySchema = schemaForKey(version);
    Struct key = keySchema.read(buffer);
    if (version <= CURRENT_OFFSET_KEY_SCHEMA_VERSION) {
        // version 0 and 1 refer to offset
        String group = key.getString(OFFSET_KEY_GROUP_FIELD);
        String topic = key.getString(OFFSET_KEY_TOPIC_FIELD);
        int partition = key.getInt(OFFSET_KEY_PARTITION_FIELD);
        return new OffsetKey(version, new GroupTopicPartition(group, topic, partition));
    } else if (version == CURRENT_GROUP_KEY_SCHEMA_VERSION) {
        // version 2 refers to group
        String group = key.getString(GROUP_KEY_GROUP_FIELD);
        return new GroupMetadataKey(version, group);
    } else {
        throw new IllegalStateException("Unknown version " + version + " for group metadata message");
    }
}

Example 4 with OffsetKey

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.OffsetKey in project kop by streamnative.

the class GroupMetadataManagerTest method testCommitOffset.

@Test
public void testCommitOffset() throws Exception {
    String memberId = "fakeMemberId";
    TopicPartition topicPartition = new TopicPartition("foo", 0);
    groupMetadataManager.addPartitionOwnership(groupPartitionId);
    long offset = 37L;
    GroupMetadata group = new GroupMetadata(groupId, Empty);
    groupMetadataManager.addGroup(group);
    Map<TopicPartition, OffsetAndMetadata> offsets = ImmutableMap.<TopicPartition, OffsetAndMetadata>builder().put(topicPartition, OffsetAndMetadata.apply(offset)).build();
    Map<TopicPartition, Errors> commitErrors = groupMetadataManager.storeOffsets(group, memberId, offsets).get();
    assertTrue(group.hasOffsets());
    assertFalse(commitErrors.isEmpty());
    Errors maybeError = commitErrors.get(topicPartition);
    assertEquals(Errors.NONE, maybeError);
    assertTrue(group.hasOffsets());
    Map<TopicPartition, PartitionData> cachedOffsets = groupMetadataManager.getOffsets(groupId, Optional.of(Lists.newArrayList(topicPartition)));
    PartitionData maybePartitionResponse = cachedOffsets.get(topicPartition);
    assertNotNull(maybePartitionResponse);
    assertEquals(Errors.NONE, maybePartitionResponse.error);
    assertEquals(offset, maybePartitionResponse.offset);
    Message<ByteBuffer> message = consumer.receive();
    while (message.getValue().array().length == 0) {
        // bypass above place holder message.
        message = consumer.receive();
    }
    assertTrue(message.getEventTime() > 0L);
    assertTrue(message.hasKey());
    byte[] key = message.getKeyBytes();
    BaseKey groupKey = GroupMetadataConstants.readMessageKey(ByteBuffer.wrap(key));
    assertTrue(groupKey instanceof OffsetKey);
    ByteBuffer value = message.getValue();
    MemoryRecords memRecords = MemoryRecords.readableRecords(value);
    AtomicBoolean verified = new AtomicBoolean(false);
    memRecords.batches().forEach(batch -> {
        for (Record record : batch) {
            assertFalse(verified.get());
            BaseKey bk = GroupMetadataConstants.readMessageKey(record.key());
            assertTrue(bk instanceof OffsetKey);
            OffsetKey ok = (OffsetKey) bk;
            GroupTopicPartition gtp = ok.key();
            assertEquals(groupId, gtp.group());
            assertEquals(new TopicPartition(new KopTopic(topicPartition.topic(), NAMESPACE_PREFIX).getFullName(), topicPartition.partition()), gtp.topicPartition());
            OffsetAndMetadata gm = GroupMetadataConstants.readOffsetMessageValue(record.value());
            assertEquals(offset, gm.offset());
            verified.set(true);
        }
    });
    assertTrue(verified.get());
}

Example 5 with OffsetKey

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.OffsetKey in project kop by streamnative.

the class OffsetResetTest method readFromOffsetMessagePulsar.

private void readFromOffsetMessagePulsar() throws Exception {
    Reader<ByteBuffer> reader = pulsarClient.newReader(Schema.BYTEBUFFER).startMessageId(MessageId.earliest).topic("persistent://public/default/__consumer_offsets" + PARTITIONED_TOPIC_SUFFIX + "0").readCompacted(true).create();
    log.info("start reading __consumer_offsets:{}", reader.getTopic());
    while (true) {
        Message<ByteBuffer> offsetMetadata = reader.readNext(1, TimeUnit.SECONDS);
        if (offsetMetadata == null) {
            break;
        }
        MemoryRecords memRecords = MemoryRecords.readableRecords(offsetMetadata.getValue());
        log.info("__consumer_offsets MessageId:{}", offsetMetadata.getMessageId());
        for (MutableRecordBatch batch : memRecords.batches()) {
            for (Record record : batch) {
                BaseKey baseKey = GroupMetadataConstants.readMessageKey(record.key());
                if (baseKey != null && (baseKey instanceof OffsetKey)) {
                    OffsetKey offsetKey = (OffsetKey) baseKey;
                    String formattedValue = String.valueOf(GroupMetadataConstants.readOffsetMessageValue(record.value()));
                    log.info("__consumer_offsets - key:{}, value:{}", offsetKey, formattedValue);
                }
            }
        }
    }
    reader.close();
}