use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.GroupTopicPartition in project starlight-for-kafka by datastax.
the class GroupMetadataManagerTest method testCommitOffset.
@Test
public void testCommitOffset() throws Exception {
@Cleanup Consumer<ByteBuffer> consumer = pulsarClient.newConsumer(Schema.BYTEBUFFER).topic(groupMetadataManager.getTopicPartitionName()).subscriptionName("test-sub").subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe();
String memberId = "fakeMemberId";
TopicPartition topicPartition = new TopicPartition("foo", 0);
groupMetadataManager.addPartitionOwnership(groupPartitionId);
long offset = 37L;
GroupMetadata group = new GroupMetadata(groupId, Empty);
groupMetadataManager.addGroup(group);
Map<TopicPartition, OffsetAndMetadata> offsets = ImmutableMap.<TopicPartition, OffsetAndMetadata>builder().put(topicPartition, OffsetAndMetadata.apply(offset)).build();
Map<TopicPartition, Errors> commitErrors = groupMetadataManager.storeOffsets(group, memberId, offsets).get();
assertTrue(group.hasOffsets());
assertFalse(commitErrors.isEmpty());
Errors maybeError = commitErrors.get(topicPartition);
assertEquals(Errors.NONE, maybeError);
assertTrue(group.hasOffsets());
Map<TopicPartition, PartitionData> cachedOffsets = groupMetadataManager.getOffsets(groupId, Optional.of(Lists.newArrayList(topicPartition)));
PartitionData maybePartitionResponse = cachedOffsets.get(topicPartition);
assertNotNull(maybePartitionResponse);
assertEquals(Errors.NONE, maybePartitionResponse.error);
assertEquals(offset, maybePartitionResponse.offset);
Message<ByteBuffer> message = consumer.receive();
while (message.getValue().array().length == 0) {
// bypass above place holder message.
message = consumer.receive();
}
assertTrue(message.getEventTime() > 0L);
assertTrue(message.hasKey());
byte[] key = message.getKeyBytes();
BaseKey groupKey = GroupMetadataConstants.readMessageKey(ByteBuffer.wrap(key));
assertTrue(groupKey instanceof OffsetKey);
ByteBuffer value = message.getValue();
MemoryRecords memRecords = MemoryRecords.readableRecords(value);
AtomicBoolean verified = new AtomicBoolean(false);
memRecords.batches().forEach(batch -> {
for (Record record : batch) {
assertFalse(verified.get());
BaseKey bk = GroupMetadataConstants.readMessageKey(record.key());
assertTrue(bk instanceof OffsetKey);
OffsetKey ok = (OffsetKey) bk;
GroupTopicPartition gtp = ok.key();
assertEquals(groupId, gtp.group());
assertEquals(new TopicPartition(new KopTopic(topicPartition.topic(), NAMESPACE_PREFIX).getFullName(), topicPartition.partition()), gtp.topicPartition());
OffsetAndMetadata gm = GroupMetadataConstants.readOffsetMessageValue(record.value());
assertEquals(offset, gm.offset());
verified.set(true);
}
});
assertTrue(verified.get());
}
use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.GroupTopicPartition in project starlight-for-kafka by datastax.
the class GroupMetadataConstants method readMessageKey.
/**
* Decodes the offset messages' key.
*/
public static BaseKey readMessageKey(ByteBuffer buffer) {
short version = buffer.getShort();
Schema keySchema = schemaForKey(version);
Struct key = keySchema.read(buffer);
if (version <= CURRENT_OFFSET_KEY_SCHEMA_VERSION) {
// version 0 and 1 refer to offset
String group = key.getString(OFFSET_KEY_GROUP_FIELD);
String topic = key.getString(OFFSET_KEY_TOPIC_FIELD);
int partition = key.getInt(OFFSET_KEY_PARTITION_FIELD);
return new OffsetKey(version, new GroupTopicPartition(group, topic, partition));
} else if (version == CURRENT_GROUP_KEY_SCHEMA_VERSION) {
// version 2 refers to group
String group = key.getString(GROUP_KEY_GROUP_FIELD);
return new GroupMetadataKey(version, group);
} else {
throw new IllegalStateException("Unknown version " + version + " for group metadata message");
}
}
use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.GroupTopicPartition in project kop by streamnative.
the class GroupMetadataManagerTest method testCommitOffset.
@Test
public void testCommitOffset() throws Exception {
String memberId = "fakeMemberId";
TopicPartition topicPartition = new TopicPartition("foo", 0);
groupMetadataManager.addPartitionOwnership(groupPartitionId);
long offset = 37L;
GroupMetadata group = new GroupMetadata(groupId, Empty);
groupMetadataManager.addGroup(group);
Map<TopicPartition, OffsetAndMetadata> offsets = ImmutableMap.<TopicPartition, OffsetAndMetadata>builder().put(topicPartition, OffsetAndMetadata.apply(offset)).build();
Map<TopicPartition, Errors> commitErrors = groupMetadataManager.storeOffsets(group, memberId, offsets).get();
assertTrue(group.hasOffsets());
assertFalse(commitErrors.isEmpty());
Errors maybeError = commitErrors.get(topicPartition);
assertEquals(Errors.NONE, maybeError);
assertTrue(group.hasOffsets());
Map<TopicPartition, PartitionData> cachedOffsets = groupMetadataManager.getOffsets(groupId, Optional.of(Lists.newArrayList(topicPartition)));
PartitionData maybePartitionResponse = cachedOffsets.get(topicPartition);
assertNotNull(maybePartitionResponse);
assertEquals(Errors.NONE, maybePartitionResponse.error);
assertEquals(offset, maybePartitionResponse.offset);
Message<ByteBuffer> message = consumer.receive();
while (message.getValue().array().length == 0) {
// bypass above place holder message.
message = consumer.receive();
}
assertTrue(message.getEventTime() > 0L);
assertTrue(message.hasKey());
byte[] key = message.getKeyBytes();
BaseKey groupKey = GroupMetadataConstants.readMessageKey(ByteBuffer.wrap(key));
assertTrue(groupKey instanceof OffsetKey);
ByteBuffer value = message.getValue();
MemoryRecords memRecords = MemoryRecords.readableRecords(value);
AtomicBoolean verified = new AtomicBoolean(false);
memRecords.batches().forEach(batch -> {
for (Record record : batch) {
assertFalse(verified.get());
BaseKey bk = GroupMetadataConstants.readMessageKey(record.key());
assertTrue(bk instanceof OffsetKey);
OffsetKey ok = (OffsetKey) bk;
GroupTopicPartition gtp = ok.key();
assertEquals(groupId, gtp.group());
assertEquals(new TopicPartition(new KopTopic(topicPartition.topic(), NAMESPACE_PREFIX).getFullName(), topicPartition.partition()), gtp.topicPartition());
OffsetAndMetadata gm = GroupMetadataConstants.readOffsetMessageValue(record.value());
assertEquals(offset, gm.offset());
verified.set(true);
}
});
assertTrue(verified.get());
}
use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadataManager.GroupTopicPartition in project kop by streamnative.
the class GroupMetadataConstants method readMessageKey.
/**
* Decodes the offset messages' key.
*/
public static BaseKey readMessageKey(ByteBuffer buffer) {
short version = buffer.getShort();
Schema keySchema = schemaForKey(version);
Struct key = keySchema.read(buffer);
if (version <= CURRENT_OFFSET_KEY_SCHEMA_VERSION) {
// version 0 and 1 refer to offset
String group = key.getString(OFFSET_KEY_GROUP_FIELD);
String topic = key.getString(OFFSET_KEY_TOPIC_FIELD);
int partition = key.getInt(OFFSET_KEY_PARTITION_FIELD);
return new OffsetKey(version, new GroupTopicPartition(group, topic, partition));
} else if (version == CURRENT_GROUP_KEY_SCHEMA_VERSION) {
// version 2 refers to group
String group = key.getString(GROUP_KEY_GROUP_FIELD);
return new GroupMetadataKey(version, group);
} else {
throw new IllegalStateException("Unknown version " + version + " for group metadata message");
}
}
Aggregations