Examples with ConsumerGroupMetadata org.apache.kafka.clients.consumer.ConsumerGroupMetadata used on opensource projects

Example 6 with ConsumerGroupMetadata

use of org.apache.kafka.clients.consumer.ConsumerGroupMetadata in project kafka by apache.

the class MockProducerTest method shouldIgnoreEmptyOffsetsWhenSendOffsetsToTransactionByGroupMetadata.

@Test
public void shouldIgnoreEmptyOffsetsWhenSendOffsetsToTransactionByGroupMetadata() {
    buildMockProducer(true);
    producer.initTransactions();
    producer.beginTransaction();
    producer.sendOffsetsToTransaction(Collections.emptyMap(), new ConsumerGroupMetadata("groupId"));
    assertFalse(producer.sentOffsets());
}

Example 7 with ConsumerGroupMetadata

use of org.apache.kafka.clients.consumer.ConsumerGroupMetadata in project kafka by apache.

the class ConsumerCoordinator method onJoinComplete.

@Override
protected void onJoinComplete(int generation, String memberId, String assignmentStrategy, ByteBuffer assignmentBuffer) {
    log.debug("Executing onJoinComplete with generation {} and memberId {}", generation, memberId);
    // Only the leader is responsible for monitoring for metadata changes (i.e. partition changes)
    if (!isLeader)
        assignmentSnapshot = null;
    ConsumerPartitionAssignor assignor = lookupAssignor(assignmentStrategy);
    if (assignor == null)
        throw new IllegalStateException("Coordinator selected invalid assignment protocol: " + assignmentStrategy);
    // Give the assignor a chance to update internal state based on the received assignment
    groupMetadata = new ConsumerGroupMetadata(rebalanceConfig.groupId, generation, memberId, rebalanceConfig.groupInstanceId);
    SortedSet<TopicPartition> ownedPartitions = new TreeSet<>(COMPARATOR);
    ownedPartitions.addAll(subscriptions.assignedPartitions());
    // should at least encode the short version
    if (assignmentBuffer.remaining() < 2)
        throw new IllegalStateException("There are insufficient bytes available to read assignment from the sync-group response (" + "actual byte size " + assignmentBuffer.remaining() + ") , this is not expected; " + "it is possible that the leader's assign function is buggy and did not return any assignment for this member, " + "or because static member is configured and the protocol is buggy hence did not get the assignment for this member");
    Assignment assignment = ConsumerProtocol.deserializeAssignment(assignmentBuffer);
    SortedSet<TopicPartition> assignedPartitions = new TreeSet<>(COMPARATOR);
    assignedPartitions.addAll(assignment.partitions());
    if (!subscriptions.checkAssignmentMatchedSubscription(assignedPartitions)) {
        final String reason = String.format("received assignment %s does not match the current subscription %s; " + "it is likely that the subscription has changed since we joined the group, will re-join with current subscription", assignment.partitions(), subscriptions.prettyString());
        requestRejoin(reason);
        return;
    }
    final AtomicReference<Exception> firstException = new AtomicReference<>(null);
    SortedSet<TopicPartition> addedPartitions = new TreeSet<>(COMPARATOR);
    addedPartitions.addAll(assignedPartitions);
    addedPartitions.removeAll(ownedPartitions);
    if (protocol == RebalanceProtocol.COOPERATIVE) {
        SortedSet<TopicPartition> revokedPartitions = new TreeSet<>(COMPARATOR);
        revokedPartitions.addAll(ownedPartitions);
        revokedPartitions.removeAll(assignedPartitions);
        log.info("Updating assignment with\n" + "\tAssigned partitions:                       {}\n" + "\tCurrent owned partitions:                  {}\n" + "\tAdded partitions (assigned - owned):       {}\n" + "\tRevoked partitions (owned - assigned):     {}\n", assignedPartitions, ownedPartitions, addedPartitions, revokedPartitions);
        if (!revokedPartitions.isEmpty()) {
            // Revoke partitions that were previously owned but no longer assigned;
            // note that we should only change the assignment (or update the assignor's state)
            // AFTER we've triggered  the revoke callback
            firstException.compareAndSet(null, invokePartitionsRevoked(revokedPartitions));
            // If revoked any partitions, need to re-join the group afterwards
            final String reason = String.format("need to revoke partitions %s as indicated " + "by the current assignment and re-join", revokedPartitions);
            requestRejoin(reason);
        }
    }
    // The leader may have assigned partitions which match our subscription pattern, but which
    // were not explicitly requested, so we update the joined subscription here.
    maybeUpdateJoinedSubscription(assignedPartitions);
    // Catch any exception here to make sure we could complete the user callback.
    firstException.compareAndSet(null, invokeOnAssignment(assignor, assignment));
    // Reschedule the auto commit starting from now
    if (autoCommitEnabled)
        this.nextAutoCommitTimer.updateAndReset(autoCommitIntervalMs);
    subscriptions.assignFromSubscribed(assignedPartitions);
    // Add partitions that were not previously owned but are now assigned
    firstException.compareAndSet(null, invokePartitionsAssigned(addedPartitions));
    if (firstException.get() != null) {
        if (firstException.get() instanceof KafkaException) {
            throw (KafkaException) firstException.get();
        } else {
            throw new KafkaException("User rebalance callback throws an error", firstException.get());
        }
    }
}

Example 8 with ConsumerGroupMetadata

use of org.apache.kafka.clients.consumer.ConsumerGroupMetadata in project kafka by apache.

the class TransactionManagerTest method testUnsupportedForMessageFormatInTxnOffsetCommit.

@Test
public void testUnsupportedForMessageFormatInTxnOffsetCommit() {
    final TopicPartition tp = new TopicPartition("foo", 0);
    doInitTransactions();
    transactionManager.beginTransaction();
    TransactionalRequestResult sendOffsetsResult = transactionManager.sendOffsetsToTransaction(singletonMap(tp, new OffsetAndMetadata(39L)), new ConsumerGroupMetadata(consumerGroupId));
    prepareAddOffsetsToTxnResponse(Errors.NONE, consumerGroupId, producerId, epoch);
    prepareFindCoordinatorResponse(Errors.NONE, false, CoordinatorType.GROUP, consumerGroupId);
    prepareTxnOffsetCommitResponse(consumerGroupId, producerId, epoch, singletonMap(tp, Errors.UNSUPPORTED_FOR_MESSAGE_FORMAT));
    runUntil(transactionManager::hasError);
    assertTrue(transactionManager.lastError() instanceof UnsupportedForMessageFormatException);
    assertTrue(sendOffsetsResult.isCompleted());
    assertFalse(sendOffsetsResult.isSuccessful());
    assertTrue(sendOffsetsResult.error() instanceof UnsupportedForMessageFormatException);
    assertFatalError(UnsupportedForMessageFormatException.class);
}

Example 9 with ConsumerGroupMetadata

use of org.apache.kafka.clients.consumer.ConsumerGroupMetadata in project kafka by apache.

the class TransactionManagerTest method prepareGroupMetadataCommit.

private TransactionalRequestResult prepareGroupMetadataCommit(Runnable prepareTxnCommitResponse) {
    doInitTransactions();
    transactionManager.beginTransaction();
    Map<TopicPartition, OffsetAndMetadata> offsets = new HashMap<>();
    offsets.put(tp0, new OffsetAndMetadata(1));
    offsets.put(tp1, new OffsetAndMetadata(1));
    TransactionalRequestResult addOffsetsResult = transactionManager.sendOffsetsToTransaction(offsets, new ConsumerGroupMetadata(consumerGroupId, generationId, memberId, Optional.of(groupInstanceId)));
    prepareAddOffsetsToTxnResponse(Errors.NONE, consumerGroupId, producerId, epoch);
    // send AddOffsetsToTxnResult
    sender.runOnce();
    // The request should complete only after the TxnOffsetCommit completes
    assertFalse(addOffsetsResult.isCompleted());
    prepareFindCoordinatorResponse(Errors.NONE, false, CoordinatorType.GROUP, consumerGroupId);
    prepareTxnCommitResponse.run();
    assertNull(transactionManager.coordinator(CoordinatorType.GROUP));
    // try to send TxnOffsetCommitRequest, but find we don't have a group coordinator
    sender.runOnce();
    // send find coordinator for group request
    sender.runOnce();
    assertNotNull(transactionManager.coordinator(CoordinatorType.GROUP));
    assertTrue(transactionManager.hasPendingOffsetCommits());
    return addOffsetsResult;
}

Example 10 with ConsumerGroupMetadata

use of org.apache.kafka.clients.consumer.ConsumerGroupMetadata in project kafka by apache.

the class TransactionManagerTest method shouldNotSendAbortTxnRequestWhenOnlyAddOffsetsRequestFailed.

@Test
public void shouldNotSendAbortTxnRequestWhenOnlyAddOffsetsRequestFailed() {
    doInitTransactions();
    transactionManager.beginTransaction();
    Map<TopicPartition, OffsetAndMetadata> offsets = new HashMap<>();
    offsets.put(tp1, new OffsetAndMetadata(1));
    transactionManager.sendOffsetsToTransaction(offsets, new ConsumerGroupMetadata(consumerGroupId));
    TransactionalRequestResult abortResult = transactionManager.beginAbort();
    prepareAddOffsetsToTxnResponse(Errors.GROUP_AUTHORIZATION_FAILED, consumerGroupId, producerId, epoch);
    runUntil(abortResult::isCompleted);
    assertTrue(transactionManager.isReady());
    assertTrue(abortResult.isCompleted());
    assertTrue(abortResult.isSuccessful());
}