Examples with Empty io.streamnative.pulsar.handlers.kop.coordinator.group.GroupState.Empty used on opensource projects

Example 1 with Empty

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupState.Empty in project starlight-for-kafka by datastax.

the class GroupMetadataManagerTest method testGroupLoadWithConsumerAndTransactionalOffsetCommitsTransactionWins.

@Test
public void testGroupLoadWithConsumerAndTransactionalOffsetCommitsTransactionWins() throws Exception {
    long producerId = 1000L;
    short producerEpoch = 2;
    Map<TopicPartition, Long> transactionalOffsetCommits = new HashMap<>();
    transactionalOffsetCommits.put(new TopicPartition("foo", 0), 23L);
    Map<TopicPartition, Long> consumerOffsetCommits = new HashMap<>();
    consumerOffsetCommits.put(new TopicPartition("foo", 0), 24L);
    ByteBuffer buffer = ByteBuffer.allocate(1024);
    int nextOffset = 0;
    nextOffset += appendConsumerOffsetCommit(buffer, nextOffset, consumerOffsetCommits);
    nextOffset += appendTransactionalOffsetCommits(buffer, producerId, producerEpoch, nextOffset, transactionalOffsetCommits, NAMESPACE_PREFIX);
    nextOffset += completeTransactionalOffsetCommit(buffer, producerId, producerEpoch, nextOffset, true);
    buffer.flip();
    byte[] key = groupMetadataKey(groupId);
    Producer<ByteBuffer> producer = groupMetadataManager.getOffsetsTopicProducer(groupPartitionId).get();
    producer.newMessage().keyBytes(key).value(buffer).eventTime(Time.SYSTEM.milliseconds()).send();
    CompletableFuture<GroupMetadata> onLoadedFuture = new CompletableFuture<>();
    groupMetadataManager.scheduleLoadGroupAndOffsets(groupPartitionId, groupMetadata -> onLoadedFuture.complete(groupMetadata)).get();
    GroupMetadata group = onLoadedFuture.get();
    GroupMetadata groupInCache = groupMetadataManager.getGroup(groupId).orElseGet(() -> {
        fail("Group was not loaded into the cache");
        return null;
    });
    assertSame(group, groupInCache);
    assertEquals(groupId, group.groupId());
    assertEquals(Empty, group.currentState());
    // The group should be loaded with pending offsets.
    assertEquals(1, group.allOffsets().size());
    assertTrue(group.hasOffsets());
    assertFalse(group.hasPendingOffsetCommitsFromProducer(producerId));
    assertEquals(consumerOffsetCommits.size(), group.allOffsets().size());
    transactionalOffsetCommits.forEach((tp, offset) -> {
        assertEquals(Optional.of(offset), group.offset(tp, NAMESPACE_PREFIX).map(OffsetAndMetadata::offset));
    });
}

Example 2 with Empty

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupState.Empty in project starlight-for-kafka by datastax.

the class GroupMetadataManagerTest method testLoadTransactionalOffsetCommitsFromMultipleProducers.

@Test
public void testLoadTransactionalOffsetCommitsFromMultipleProducers() throws Exception {
    long firstProducerId = 1000L;
    short firstProducerEpoch = 2;
    long secondProducerId = 1001L;
    short secondProducerEpoch = 3;
    Map<TopicPartition, Long> committedOffsetsFirstProducer = new HashMap<>();
    committedOffsetsFirstProducer.put(new TopicPartition("foo", 0), 23L);
    committedOffsetsFirstProducer.put(new TopicPartition("foo", 1), 455L);
    committedOffsetsFirstProducer.put(new TopicPartition("bar", 0), 8992L);
    Map<TopicPartition, Long> committedOffsetsSecondProducer = new HashMap<>();
    committedOffsetsSecondProducer.put(new TopicPartition("foo", 2), 231L);
    committedOffsetsSecondProducer.put(new TopicPartition("foo", 3), 4551L);
    committedOffsetsSecondProducer.put(new TopicPartition("bar", 1), 89921L);
    ByteBuffer buffer = ByteBuffer.allocate(1024);
    int nextOffset = 0;
    int firstProduceRecordOffset = nextOffset;
    nextOffset += appendTransactionalOffsetCommits(buffer, firstProducerId, firstProducerEpoch, nextOffset, committedOffsetsFirstProducer, NAMESPACE_PREFIX);
    nextOffset += completeTransactionalOffsetCommit(buffer, firstProducerId, firstProducerEpoch, nextOffset, true);
    int secondProduceRecordOffset = nextOffset;
    nextOffset += appendTransactionalOffsetCommits(buffer, secondProducerId, secondProducerEpoch, nextOffset, committedOffsetsSecondProducer, NAMESPACE_PREFIX);
    nextOffset += completeTransactionalOffsetCommit(buffer, secondProducerId, secondProducerEpoch, nextOffset, true);
    buffer.flip();
    byte[] key = groupMetadataKey(groupId);
    Producer<ByteBuffer> producer = groupMetadataManager.getOffsetsTopicProducer(groupPartitionId).get();
    producer.newMessage().keyBytes(key).value(buffer).eventTime(Time.SYSTEM.milliseconds()).send();
    CompletableFuture<GroupMetadata> onLoadedFuture = new CompletableFuture<>();
    groupMetadataManager.scheduleLoadGroupAndOffsets(groupPartitionId, groupMetadata -> onLoadedFuture.complete(groupMetadata)).get();
    GroupMetadata group = onLoadedFuture.get();
    GroupMetadata groupInCache = groupMetadataManager.getGroup(groupId).orElseGet(() -> {
        fail("Group was not loaded into the cache");
        return null;
    });
    assertSame(group, groupInCache);
    assertEquals(groupId, group.groupId());
    assertEquals(Empty, group.currentState());
    // Ensure that only the committed offsets are materialized, and that there are no pending commits
    // for the producer. This allows us to be certain that the aborted offset commits are truly discarded.
    assertEquals(committedOffsetsFirstProducer.size() + committedOffsetsSecondProducer.size(), group.allOffsets().size());
    committedOffsetsFirstProducer.forEach((tp, offset) -> {
        assertEquals(Optional.of(offset), group.offset(tp, NAMESPACE_PREFIX).map(OffsetAndMetadata::offset));
        assertEquals(Optional.of((long) firstProduceRecordOffset), group.offsetWithRecordMetadata(tp).flatMap(CommitRecordMetadataAndOffset::appendedBatchOffset));
    });
    committedOffsetsSecondProducer.forEach((tp, offset) -> {
        assertEquals(Optional.of(offset), group.offset(tp, NAMESPACE_PREFIX).map(OffsetAndMetadata::offset));
        assertEquals(Optional.of((long) secondProduceRecordOffset), group.offsetWithRecordMetadata(tp).flatMap(CommitRecordMetadataAndOffset::appendedBatchOffset));
    });
}

Example 3 with Empty

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupState.Empty in project starlight-for-kafka by datastax.

the class GroupMetadataManagerTest method testLoadOffsetsWithTombstones.

@Test
public void testLoadOffsetsWithTombstones() throws Exception {
    TopicPartition tombstonePartition = new TopicPartition("foo", 1);
    Map<TopicPartition, Long> committedOffsets = new HashMap<>();
    committedOffsets.put(new TopicPartition("foo", 0), 23L);
    committedOffsets.put(tombstonePartition, 455L);
    committedOffsets.put(new TopicPartition("bar", 0), 8992L);
    List<SimpleRecord> offsetCommitRecords = createCommittedOffsetRecords(committedOffsets, groupId, NAMESPACE_PREFIX);
    SimpleRecord tombstone = new SimpleRecord(offsetCommitKey(groupId, tombstonePartition, NAMESPACE_PREFIX), null);
    offsetCommitRecords.add(tombstone);
    ByteBuffer buffer = newMemoryRecordsBuffer(offsetCommitRecords);
    byte[] key = groupMetadataKey(groupId);
    Producer<ByteBuffer> producer = groupMetadataManager.getOffsetsTopicProducer(groupPartitionId).get();
    producer.newMessage().keyBytes(key).value(buffer).eventTime(Time.SYSTEM.milliseconds()).send();
    CompletableFuture<GroupMetadata> onLoadedFuture = new CompletableFuture<>();
    groupMetadataManager.scheduleLoadGroupAndOffsets(groupPartitionId, groupMetadata -> onLoadedFuture.complete(groupMetadata)).get();
    GroupMetadata group = onLoadedFuture.get();
    GroupMetadata groupInCache = groupMetadataManager.getGroup(groupId).orElseGet(() -> {
        fail("Group was not loaded into the cache");
        return null;
    });
    assertSame(group, groupInCache);
    assertEquals(groupId, group.groupId());
    assertEquals(Empty, group.currentState());
    // The group should be loaded with pending offsets.
    assertEquals(committedOffsets.size() - 1, group.allOffsets().size());
    committedOffsets.forEach((tp, offset) -> {
        if (tp == tombstonePartition) {
            assertEquals(Optional.empty(), group.offset(tp, NAMESPACE_PREFIX));
        } else {
            assertEquals(Optional.of(offset), group.offset(tp, NAMESPACE_PREFIX).map(OffsetAndMetadata::offset));
        }
    });
}

Example 4 with Empty

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupState.Empty in project starlight-for-kafka by datastax.

the class GroupMetadataManagerTest method testOffsetWriteAfterGroupRemoved.

@Test
public void testOffsetWriteAfterGroupRemoved() throws Exception {
    // this test case checks the following scenario:
    // 1. the group exists at some point in time, but is later removed (because all members left)
    // 2. a "simple" consumer (i.e. not a consumer group) then uses the same groupId to commit some offsets
    int generation = 293;
    String memberId = "98098230493";
    String protocolType = "consumer";
    String protocol = "range";
    Map<TopicPartition, Long> committedOffsets = new HashMap<>();
    committedOffsets.put(new TopicPartition("foo", 0), 23L);
    committedOffsets.put(new TopicPartition("foo", 1), 455L);
    committedOffsets.put(new TopicPartition("bar", 0), 8992L);
    List<SimpleRecord> offsetCommitRecords = createCommittedOffsetRecords(committedOffsets, groupId, NAMESPACE_PREFIX);
    SimpleRecord groupMetadataRecord = buildStableGroupRecordWithMember(generation, protocolType, protocol, memberId);
    SimpleRecord groupMetadataTombstone = new SimpleRecord(groupMetadataKey(groupId), null);
    List<SimpleRecord> newOffsetCommitRecords = new ArrayList<>();
    newOffsetCommitRecords.add(groupMetadataRecord);
    newOffsetCommitRecords.add(groupMetadataTombstone);
    newOffsetCommitRecords.addAll(offsetCommitRecords);
    ByteBuffer buffer = newMemoryRecordsBuffer(newOffsetCommitRecords);
    byte[] key = groupMetadataKey(groupId);
    int consumerGroupPartitionId = GroupMetadataManager.getPartitionId(groupId, conf.getOffsetsTopicNumPartitions());
    Producer<ByteBuffer> producer = groupMetadataManager.getOffsetsTopicProducer(consumerGroupPartitionId).get();
    producer.newMessage().keyBytes(key).value(buffer).eventTime(Time.SYSTEM.milliseconds()).send();
    groupMetadataManager.removeLoadingPartition(consumerGroupPartitionId);
    CompletableFuture<GroupMetadata> onLoadedFuture = new CompletableFuture<>();
    groupMetadataManager.scheduleLoadGroupAndOffsets(consumerGroupPartitionId, groupMetadata -> onLoadedFuture.complete(groupMetadata)).get();
    GroupMetadata group = onLoadedFuture.get();
    GroupMetadata groupInCache = groupMetadataManager.getGroup(groupId).orElseGet(() -> {
        fail("Group was not loaded into the cache");
        return null;
    });
    assertSame(group, groupInCache);
    assertEquals(groupId, group.groupId());
    assertEquals(Empty, group.currentState());
    assertEquals(committedOffsets.size(), group.allOffsets().size());
    committedOffsets.forEach((tp, offset) -> {
        assertEquals(Optional.of(offset), group.offset(tp, NAMESPACE_PREFIX).map(OffsetAndMetadata::offset));
    });
}

Example 5 with Empty

use of io.streamnative.pulsar.handlers.kop.coordinator.group.GroupState.Empty in project starlight-for-kafka by datastax.

the class GroupMetadataManagerTest method testLoadWithCommitedAndAbortedAndPendingTransactionOffsetCommits.

@Test
public void testLoadWithCommitedAndAbortedAndPendingTransactionOffsetCommits() throws Exception {
    long producerId = 1000L;
    short producerEpoch = 2;
    Map<TopicPartition, Long> committedOffsets = new HashMap<>();
    committedOffsets.put(new TopicPartition("foo", 0), 23L);
    committedOffsets.put(new TopicPartition("foo", 1), 455L);
    committedOffsets.put(new TopicPartition("bar", 0), 8992L);
    Map<TopicPartition, Long> abortedOffsets = new HashMap<>();
    abortedOffsets.put(new TopicPartition("foo", 2), 231L);
    abortedOffsets.put(new TopicPartition("foo", 3), 4551L);
    abortedOffsets.put(new TopicPartition("bar", 1), 89921L);
    Map<TopicPartition, Long> pendingOffsets = new HashMap<>();
    pendingOffsets.put(new TopicPartition("foo", 3), 2312L);
    pendingOffsets.put(new TopicPartition("foo", 4), 45512L);
    pendingOffsets.put(new TopicPartition("bar", 2), 899212L);
    ByteBuffer buffer = ByteBuffer.allocate(2048);
    int nextOffset = 0;
    nextOffset += appendTransactionalOffsetCommits(buffer, producerId, producerEpoch, nextOffset, committedOffsets, NAMESPACE_PREFIX);
    nextOffset += completeTransactionalOffsetCommit(buffer, producerId, producerEpoch, nextOffset, true);
    nextOffset += appendTransactionalOffsetCommits(buffer, producerId, producerEpoch, nextOffset, abortedOffsets, NAMESPACE_PREFIX);
    nextOffset += completeTransactionalOffsetCommit(buffer, producerId, producerEpoch, nextOffset, false);
    nextOffset += appendTransactionalOffsetCommits(buffer, producerId, producerEpoch, nextOffset, pendingOffsets, NAMESPACE_PREFIX);
    buffer.flip();
    byte[] key = groupMetadataKey(groupId);
    Producer<ByteBuffer> producer = groupMetadataManager.getOffsetsTopicProducer(groupPartitionId).get();
    producer.newMessage().keyBytes(key).value(buffer).eventTime(Time.SYSTEM.milliseconds()).send();
    CompletableFuture<GroupMetadata> onLoadedFuture = new CompletableFuture<>();
    groupMetadataManager.scheduleLoadGroupAndOffsets(groupPartitionId, groupMetadata -> onLoadedFuture.complete(groupMetadata)).get();
    GroupMetadata groupInCache = groupMetadataManager.getGroup(groupId).orElseGet(() -> {
        fail("Group was not loaded into the cache");
        return null;
    });
    GroupMetadata group = onLoadedFuture.get();
    assertSame(group, groupInCache);
    assertEquals(groupId, group.groupId());
    assertEquals(Empty, group.currentState());
    // Ensure that only the committed offsets are materialized, and that there are no pending commits
    // for the producer. This allows us to be certain that the aborted offset commits are truly discarded.
    assertEquals(committedOffsets.size(), group.allOffsets().size());
    committedOffsets.forEach((tp, offset) -> assertEquals(Optional.of(offset), group.offset(tp, NAMESPACE_PREFIX).map(OffsetAndMetadata::offset)));
    // We should have pending commits.
    assertTrue(group.hasPendingOffsetCommitsFromProducer(producerId));
    // The loaded pending commits should materialize after a commit marker comes in.
    groupMetadataManager.handleTxnCompletion(producerId, Sets.newHashSet(groupPartitionId), true, new CompletableFuture<>());
    assertFalse(group.hasPendingOffsetCommitsFromProducer(producerId));
    pendingOffsets.forEach((tp, offset) -> assertEquals(Optional.of(offset), group.offset(tp, NAMESPACE_PREFIX).map(OffsetAndMetadata::offset)));
}