Examples with Stable io.streamnative.pulsar.handlers.kop.coordinator.group.GroupState.Stable used on opensource projects

Example 1 with Stable

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupState.Stable in project kop by streamnative.

the class GroupMetadataManagerTest method testLoadGroupAndOffsetsFromDifferentSegments.

@Test
public void testLoadGroupAndOffsetsFromDifferentSegments() throws Exception {
    int generation = 293;
    String protocolType = "consumer";
    String protocol = "range";
    TopicPartition tp0 = new TopicPartition("foo", 0);
    TopicPartition tp1 = new TopicPartition("foo", 1);
    TopicPartition tp2 = new TopicPartition("bar", 0);
    TopicPartition tp3 = new TopicPartition("xxx", 0);
    String segment1MemberId = "a";
    Map<TopicPartition, Long> segment1Offsets = new HashMap<>();
    segment1Offsets.put(tp0, 23L);
    segment1Offsets.put(tp1, 455L);
    segment1Offsets.put(tp3, 42L);
    List<SimpleRecord> segment1Records = createCommittedOffsetRecords(segment1Offsets, groupId, NAMESPACE_PREFIX);
    SimpleRecord segment1Group = buildStableGroupRecordWithMember(generation, protocolType, protocol, segment1MemberId);
    segment1Records.add(segment1Group);
    ByteBuffer segment1Buffer = newMemoryRecordsBuffer(segment1Records);
    String segment2MemberId = "a";
    Map<TopicPartition, Long> segment2Offsets = new HashMap<>();
    segment2Offsets.put(tp0, 33L);
    segment2Offsets.put(tp2, 8992L);
    segment2Offsets.put(tp3, 10L);
    List<SimpleRecord> segment2Records = createCommittedOffsetRecords(segment2Offsets, groupId, NAMESPACE_PREFIX);
    SimpleRecord segment2Group = buildStableGroupRecordWithMember(generation, protocolType, protocol, segment2MemberId);
    segment2Records.add(segment2Group);
    ByteBuffer segment2Buffer = newMemoryRecordsBuffer(segment2Records);
    byte[] key = groupMetadataKey(groupId);
    Producer<ByteBuffer> producer = groupMetadataManager.getOffsetsTopicProducer(groupPartitionId).get();
    producer.newMessage().keyBytes(key).value(segment1Buffer).eventTime(Time.SYSTEM.milliseconds()).send();
    producer.newMessage().keyBytes(key).value(segment2Buffer).eventTime(Time.SYSTEM.milliseconds()).send();
    CompletableFuture<GroupMetadata> onLoadedFuture = new CompletableFuture<>();
    groupMetadataManager.scheduleLoadGroupAndOffsets(groupPartitionId, groupMetadata -> onLoadedFuture.complete(groupMetadata)).get();
    GroupMetadata group = onLoadedFuture.get();
    GroupMetadata groupInCache = groupMetadataManager.getGroup(groupId).orElseGet(() -> {
        fail("Group was not loaded into the cache");
        return null;
    });
    assertSame(group, groupInCache);
    assertEquals(groupId, group.groupId());
    assertEquals(Stable, group.currentState());
    assertEquals(segment2MemberId, group.leaderOrNull(), "segment2 group record member should be elected");
    assertEquals(Lists.newArrayList(segment2MemberId), group.allMembers().stream().collect(Collectors.toList()), "segment2 group record member should be only member");
    // offsets of segment1 should be overridden by segment2 offsets of the same topic partitions
    Map<TopicPartition, Long> committedOffsets = new HashMap<>();
    committedOffsets.putAll(segment1Offsets);
    committedOffsets.putAll(segment2Offsets);
    assertEquals(committedOffsets.size(), group.allOffsets().size());
    committedOffsets.forEach((tp, offset) -> {
        assertEquals(Optional.of(offset), group.offset(tp, NAMESPACE_PREFIX).map(OffsetAndMetadata::offset));
    });
}

Example 2 with Stable

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupState.Stable in project starlight-for-kafka by datastax.

the class GroupMetadataManagerTest method testLoadOffsetsAndGroup.

@Test
public void testLoadOffsetsAndGroup() throws Exception {
    int generation = 935;
    String protocolType = "consumer";
    String protocol = "range";
    Map<TopicPartition, Long> committedOffsets = new HashMap<>();
    committedOffsets.put(new TopicPartition("foo", 0), 23L);
    committedOffsets.put(new TopicPartition("foo", 1), 455L);
    committedOffsets.put(new TopicPartition("bar", 0), 8992L);
    List<SimpleRecord> offsetCommitRecords = createCommittedOffsetRecords(committedOffsets, groupId, NAMESPACE_PREFIX);
    String memberId = "98098230493";
    SimpleRecord groupMetadataRecord = buildStableGroupRecordWithMember(generation, protocolType, protocol, memberId);
    offsetCommitRecords.add(groupMetadataRecord);
    ByteBuffer buffer = newMemoryRecordsBuffer(offsetCommitRecords);
    byte[] key = groupMetadataKey(groupId);
    Producer<ByteBuffer> producer = groupMetadataManager.getOffsetsTopicProducer(groupPartitionId).get();
    producer.newMessage().keyBytes(key).value(buffer).eventTime(Time.SYSTEM.milliseconds()).send();
    CompletableFuture<GroupMetadata> onLoadedFuture = new CompletableFuture<>();
    groupMetadataManager.scheduleLoadGroupAndOffsets(groupPartitionId, groupMetadata -> onLoadedFuture.complete(groupMetadata)).get();
    GroupMetadata group = onLoadedFuture.get();
    GroupMetadata groupInCache = groupMetadataManager.getGroup(groupId).orElseGet(() -> {
        fail("Group was not loaded into the cache");
        return null;
    });
    assertSame(group, groupInCache);
    assertEquals(groupId, group.groupId());
    assertEquals(Stable, group.currentState());
    assertEquals(memberId, group.leaderOrNull());
    assertEquals(generation, group.generationId());
    assertEquals(Optional.of(protocolType), group.protocolType());
    assertEquals(Lists.newArrayList(memberId), group.allMembers().stream().collect(Collectors.toList()));
    assertEquals(committedOffsets.size(), group.allOffsets().size());
    committedOffsets.forEach((tp, offset) -> {
        assertEquals(Optional.of(offset), group.offset(tp, NAMESPACE_PREFIX).map(OffsetAndMetadata::offset));
    });
}

Example 3 with Stable

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupState.Stable in project starlight-for-kafka by datastax.

the class GroupMetadataManagerTest method testLoadGroupAndOffsetsFromDifferentSegments.

@Test
public void testLoadGroupAndOffsetsFromDifferentSegments() throws Exception {
    int generation = 293;
    String protocolType = "consumer";
    String protocol = "range";
    TopicPartition tp0 = new TopicPartition("foo", 0);
    TopicPartition tp1 = new TopicPartition("foo", 1);
    TopicPartition tp2 = new TopicPartition("bar", 0);
    TopicPartition tp3 = new TopicPartition("xxx", 0);
    String segment1MemberId = "a";
    Map<TopicPartition, Long> segment1Offsets = new HashMap<>();
    segment1Offsets.put(tp0, 23L);
    segment1Offsets.put(tp1, 455L);
    segment1Offsets.put(tp3, 42L);
    List<SimpleRecord> segment1Records = createCommittedOffsetRecords(segment1Offsets, groupId, NAMESPACE_PREFIX);
    SimpleRecord segment1Group = buildStableGroupRecordWithMember(generation, protocolType, protocol, segment1MemberId);
    segment1Records.add(segment1Group);
    ByteBuffer segment1Buffer = newMemoryRecordsBuffer(segment1Records);
    String segment2MemberId = "a";
    Map<TopicPartition, Long> segment2Offsets = new HashMap<>();
    segment2Offsets.put(tp0, 33L);
    segment2Offsets.put(tp2, 8992L);
    segment2Offsets.put(tp3, 10L);
    List<SimpleRecord> segment2Records = createCommittedOffsetRecords(segment2Offsets, groupId, NAMESPACE_PREFIX);
    SimpleRecord segment2Group = buildStableGroupRecordWithMember(generation, protocolType, protocol, segment2MemberId);
    segment2Records.add(segment2Group);
    ByteBuffer segment2Buffer = newMemoryRecordsBuffer(segment2Records);
    byte[] key = groupMetadataKey(groupId);
    Producer<ByteBuffer> producer = groupMetadataManager.getOffsetsTopicProducer(groupPartitionId).get();
    producer.newMessage().keyBytes(key).value(segment1Buffer).eventTime(Time.SYSTEM.milliseconds()).send();
    producer.newMessage().keyBytes(key).value(segment2Buffer).eventTime(Time.SYSTEM.milliseconds()).send();
    CompletableFuture<GroupMetadata> onLoadedFuture = new CompletableFuture<>();
    groupMetadataManager.scheduleLoadGroupAndOffsets(groupPartitionId, groupMetadata -> onLoadedFuture.complete(groupMetadata)).get();
    GroupMetadata group = onLoadedFuture.get();
    GroupMetadata groupInCache = groupMetadataManager.getGroup(groupId).orElseGet(() -> {
        fail("Group was not loaded into the cache");
        return null;
    });
    assertSame(group, groupInCache);
    assertEquals(groupId, group.groupId());
    assertEquals(Stable, group.currentState());
    assertEquals(segment2MemberId, group.leaderOrNull(), "segment2 group record member should be elected");
    assertEquals(Lists.newArrayList(segment2MemberId), group.allMembers().stream().collect(Collectors.toList()), "segment2 group record member should be only member");
    // offsets of segment1 should be overridden by segment2 offsets of the same topic partitions
    Map<TopicPartition, Long> committedOffsets = new HashMap<>();
    committedOffsets.putAll(segment1Offsets);
    committedOffsets.putAll(segment2Offsets);
    assertEquals(committedOffsets.size(), group.allOffsets().size());
    committedOffsets.forEach((tp, offset) -> {
        assertEquals(Optional.of(offset), group.offset(tp, NAMESPACE_PREFIX).map(OffsetAndMetadata::offset));
    });
}

Example 4 with Stable

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupState.Stable in project kop by streamnative.

the class GroupMetadataManagerTest method testLoadOffsetsAndGroup.

@Test
public void testLoadOffsetsAndGroup() throws Exception {
    int generation = 935;
    String protocolType = "consumer";
    String protocol = "range";
    Map<TopicPartition, Long> committedOffsets = new HashMap<>();
    committedOffsets.put(new TopicPartition("foo", 0), 23L);
    committedOffsets.put(new TopicPartition("foo", 1), 455L);
    committedOffsets.put(new TopicPartition("bar", 0), 8992L);
    List<SimpleRecord> offsetCommitRecords = createCommittedOffsetRecords(committedOffsets, groupId, NAMESPACE_PREFIX);
    String memberId = "98098230493";
    SimpleRecord groupMetadataRecord = buildStableGroupRecordWithMember(generation, protocolType, protocol, memberId);
    offsetCommitRecords.add(groupMetadataRecord);
    ByteBuffer buffer = newMemoryRecordsBuffer(offsetCommitRecords);
    byte[] key = groupMetadataKey(groupId);
    Producer<ByteBuffer> producer = groupMetadataManager.getOffsetsTopicProducer(groupPartitionId).get();
    producer.newMessage().keyBytes(key).value(buffer).eventTime(Time.SYSTEM.milliseconds()).send();
    CompletableFuture<GroupMetadata> onLoadedFuture = new CompletableFuture<>();
    groupMetadataManager.scheduleLoadGroupAndOffsets(groupPartitionId, groupMetadata -> onLoadedFuture.complete(groupMetadata)).get();
    GroupMetadata group = onLoadedFuture.get();
    GroupMetadata groupInCache = groupMetadataManager.getGroup(groupId).orElseGet(() -> {
        fail("Group was not loaded into the cache");
        return null;
    });
    assertSame(group, groupInCache);
    assertEquals(groupId, group.groupId());
    assertEquals(Stable, group.currentState());
    assertEquals(memberId, group.leaderOrNull());
    assertEquals(generation, group.generationId());
    assertEquals(Optional.of(protocolType), group.protocolType());
    assertEquals(Lists.newArrayList(memberId), group.allMembers().stream().collect(Collectors.toList()));
    assertEquals(committedOffsets.size(), group.allOffsets().size());
    committedOffsets.forEach((tp, offset) -> {
        assertEquals(Optional.of(offset), group.offset(tp, NAMESPACE_PREFIX).map(OffsetAndMetadata::offset));
    });
}