Examples with TaskAssignmentException org.apache.kafka.streams.errors.TaskAssignmentException used on opensource projects

Example 6 with TaskAssignmentException

use of org.apache.kafka.streams.errors.TaskAssignmentException in project kafka by apache.

the class StreamsRebalanceListenerTest method shouldThrowTaskAssignmentException.

@Test
public void shouldThrowTaskAssignmentException() {
    taskManager.handleRebalanceComplete();
    expectLastCall();
    replay(taskManager, streamThread);
    assignmentErrorCode.set(AssignorError.ASSIGNMENT_ERROR.code());
    final TaskAssignmentException exception = assertThrows(TaskAssignmentException.class, () -> streamsRebalanceListener.onPartitionsAssigned(Collections.emptyList()));
    assertThat(exception.getMessage(), is("Hit an unexpected exception during task assignment phase of rebalance"));
    verify(taskManager, streamThread);
}

Example 7 with TaskAssignmentException

use of org.apache.kafka.streams.errors.TaskAssignmentException in project kafka by apache.

the class RepartitionTopicsTest method shouldThrowTaskAssignmentExceptionIfSourceTopicHasNoPartitionCount.

@Test
public void shouldThrowTaskAssignmentExceptionIfSourceTopicHasNoPartitionCount() {
    final RepartitionTopicConfig repartitionTopicConfigWithoutPartitionCount = new RepartitionTopicConfig(REPARTITION_WITHOUT_PARTITION_COUNT, TOPIC_CONFIG5);
    final TopicsInfo topicsInfo = new TopicsInfo(mkSet(REPARTITION_WITHOUT_PARTITION_COUNT), mkSet(SOURCE_TOPIC_NAME1), mkMap(mkEntry(REPARTITION_WITHOUT_PARTITION_COUNT, repartitionTopicConfigWithoutPartitionCount)), Collections.emptyMap());
    expect(internalTopologyBuilder.subtopologyToTopicsInfo()).andReturn(mkMap(mkEntry(SUBTOPOLOGY_0, topicsInfo), mkEntry(SUBTOPOLOGY_1, setupTopicInfoWithRepartitionTopicWithoutPartitionCount(repartitionTopicConfigWithoutPartitionCount))));
    expect(internalTopologyBuilder.copartitionGroups()).andReturn(Collections.emptyList());
    copartitionedTopicsEnforcer.enforce(eq(Collections.emptySet()), anyObject(), eq(clusterMetadata));
    expect(internalTopicManager.makeReady(mkMap(mkEntry(REPARTITION_WITHOUT_PARTITION_COUNT, repartitionTopicConfigWithoutPartitionCount)))).andReturn(Collections.emptySet());
    setupClusterWithMissingPartitionCounts(mkSet(SOURCE_TOPIC_NAME1));
    replay(internalTopicManager, internalTopologyBuilder, clusterMetadata);
    final RepartitionTopics repartitionTopics = new RepartitionTopics(new TopologyMetadata(internalTopologyBuilder, config), internalTopicManager, copartitionedTopicsEnforcer, clusterMetadata, "[test] ");
    final TaskAssignmentException exception = assertThrows(TaskAssignmentException.class, repartitionTopics::setup);
    assertThat(exception.getMessage(), is("No partition count found for source topic " + SOURCE_TOPIC_NAME1 + ", but it should have been."));
    assertThat(repartitionTopics.topologiesWithMissingInputTopics().isEmpty(), is(true));
    assertThat(repartitionTopics.missingSourceTopicExceptions().isEmpty(), is(true));
}

Example 8 with TaskAssignmentException

use of org.apache.kafka.streams.errors.TaskAssignmentException in project kafka by apache.

the class RepartitionTopicsTest method shouldThrowTaskAssignmentExceptionIfPartitionCountCannotBeComputedForAllRepartitionTopics.

@Test
public void shouldThrowTaskAssignmentExceptionIfPartitionCountCannotBeComputedForAllRepartitionTopics() {
    final RepartitionTopicConfig repartitionTopicConfigWithoutPartitionCount = new RepartitionTopicConfig(REPARTITION_WITHOUT_PARTITION_COUNT, TOPIC_CONFIG5);
    expect(internalTopologyBuilder.subtopologyToTopicsInfo()).andReturn(mkMap(mkEntry(SUBTOPOLOGY_0, TOPICS_INFO1), mkEntry(SUBTOPOLOGY_1, setupTopicInfoWithRepartitionTopicWithoutPartitionCount(repartitionTopicConfigWithoutPartitionCount))));
    expect(internalTopologyBuilder.copartitionGroups()).andReturn(Collections.emptyList());
    copartitionedTopicsEnforcer.enforce(eq(Collections.emptySet()), anyObject(), eq(clusterMetadata));
    expect(internalTopicManager.makeReady(mkMap(mkEntry(REPARTITION_TOPIC_NAME1, REPARTITION_TOPIC_CONFIG1)))).andReturn(Collections.emptySet());
    setupCluster();
    replay(internalTopicManager, internalTopologyBuilder, clusterMetadata);
    final RepartitionTopics repartitionTopics = new RepartitionTopics(new TopologyMetadata(internalTopologyBuilder, config), internalTopicManager, copartitionedTopicsEnforcer, clusterMetadata, "[test] ");
    final TaskAssignmentException exception = assertThrows(TaskAssignmentException.class, repartitionTopics::setup);
    assertThat(exception.getMessage(), is("Failed to compute number of partitions for all repartition topics, make sure all user input topics are created and all Pattern subscriptions match at least one topic in the cluster"));
    assertThat(repartitionTopics.topologiesWithMissingInputTopics().isEmpty(), is(true));
    assertThat(repartitionTopics.missingSourceTopicExceptions().isEmpty(), is(true));
}

Example 9 with TaskAssignmentException

use of org.apache.kafka.streams.errors.TaskAssignmentException in project kafka by apache.

the class RepartitionTopics method computePartitionCount.

private Integer computePartitionCount(final Map<String, InternalTopicConfig> repartitionTopicMetadata, final Collection<TopicsInfo> topicGroups, final Cluster clusterMetadata, final String repartitionSourceTopic) {
    Integer partitionCount = null;
    // try set the number of partitions for this repartition topic if it is not set yet
    for (final TopicsInfo topicsInfo : topicGroups) {
        final Set<String> sinkTopics = topicsInfo.sinkTopics;
        if (sinkTopics.contains(repartitionSourceTopic)) {
            // use the maximum of all its source topic partitions as the number of partitions
            for (final String upstreamSourceTopic : topicsInfo.sourceTopics) {
                Integer numPartitionsCandidate = null;
                // map().join().join(map())
                if (repartitionTopicMetadata.containsKey(upstreamSourceTopic)) {
                    if (repartitionTopicMetadata.get(upstreamSourceTopic).numberOfPartitions().isPresent()) {
                        numPartitionsCandidate = repartitionTopicMetadata.get(upstreamSourceTopic).numberOfPartitions().get();
                    }
                } else {
                    final Integer count = clusterMetadata.partitionCountForTopic(upstreamSourceTopic);
                    if (count == null) {
                        throw new TaskAssignmentException("No partition count found for source topic " + upstreamSourceTopic + ", but it should have been.");
                    }
                    numPartitionsCandidate = count;
                }
                if (numPartitionsCandidate != null) {
                    if (partitionCount == null || numPartitionsCandidate > partitionCount) {
                        partitionCount = numPartitionsCandidate;
                    }
                }
            }
        }
    }
    return partitionCount;
}

Example 10 with TaskAssignmentException

use of org.apache.kafka.streams.errors.TaskAssignmentException in project kafka by apache.

the class StreamsPartitionAssignor method assign.

/*
     * This assigns tasks to consumer clients in the following steps.
     *
     * 0. decode the subscriptions to assemble the metadata for each client and check for version probing
     *
     * 1. check all repartition source topics and use internal topic manager to make sure
     *    they have been created with the right number of partitions. Also verify and/or create
     *    any changelog topics with the correct number of partitions.
     *
     * 2. use the partition grouper to generate tasks along with their assigned partitions, then use
     *    the configured TaskAssignor to construct the mapping of tasks to clients.
     *
     * 3. construct the global mapping of host to partitions to enable query routing.
     *
     * 4. within each client, assign tasks to consumer clients.
     */
@Override
public GroupAssignment assign(final Cluster metadata, final GroupSubscription groupSubscription) {
    final Map<String, Subscription> subscriptions = groupSubscription.groupSubscription();
    // ---------------- Step Zero ---------------- //
    // construct the client metadata from the decoded subscription info
    final Map<UUID, ClientMetadata> clientMetadataMap = new HashMap<>();
    final Set<TopicPartition> allOwnedPartitions = new HashSet<>();
    int minReceivedMetadataVersion = LATEST_SUPPORTED_VERSION;
    int minSupportedMetadataVersion = LATEST_SUPPORTED_VERSION;
    boolean shutdownRequested = false;
    boolean assignmentErrorFound = false;
    int futureMetadataVersion = UNKNOWN;
    for (final Map.Entry<String, Subscription> entry : subscriptions.entrySet()) {
        final String consumerId = entry.getKey();
        final Subscription subscription = entry.getValue();
        final SubscriptionInfo info = SubscriptionInfo.decode(subscription.userData());
        final int usedVersion = info.version();
        if (info.errorCode() == AssignorError.SHUTDOWN_REQUESTED.code()) {
            shutdownRequested = true;
        }
        minReceivedMetadataVersion = updateMinReceivedVersion(usedVersion, minReceivedMetadataVersion);
        minSupportedMetadataVersion = updateMinSupportedVersion(info.latestSupportedVersion(), minSupportedMetadataVersion);
        final UUID processId;
        if (usedVersion > LATEST_SUPPORTED_VERSION) {
            futureMetadataVersion = usedVersion;
            processId = FUTURE_ID;
            if (!clientMetadataMap.containsKey(FUTURE_ID)) {
                clientMetadataMap.put(FUTURE_ID, new ClientMetadata(null));
            }
        } else {
            processId = info.processId();
        }
        ClientMetadata clientMetadata = clientMetadataMap.get(processId);
        // create the new client metadata if necessary
        if (clientMetadata == null) {
            clientMetadata = new ClientMetadata(info.userEndPoint());
            clientMetadataMap.put(info.processId(), clientMetadata);
        }
        // add the consumer and any info in its subscription to the client
        clientMetadata.addConsumer(consumerId, subscription.ownedPartitions());
        final int prevSize = allOwnedPartitions.size();
        allOwnedPartitions.addAll(subscription.ownedPartitions());
        if (allOwnedPartitions.size() < prevSize + subscription.ownedPartitions().size()) {
            assignmentErrorFound = true;
        }
        clientMetadata.addPreviousTasksAndOffsetSums(consumerId, info.taskOffsetSums());
    }
    if (assignmentErrorFound) {
        log.warn("The previous assignment contains a partition more than once. " + "\t Mapping: {}", subscriptions);
    }
    try {
        final boolean versionProbing = checkMetadataVersions(minReceivedMetadataVersion, minSupportedMetadataVersion, futureMetadataVersion);
        log.debug("Constructed client metadata {} from the member subscriptions.", clientMetadataMap);
        if (shutdownRequested) {
            return new GroupAssignment(errorAssignment(clientMetadataMap, AssignorError.SHUTDOWN_REQUESTED.code()));
        }
        // parse the topology to determine the repartition source topics,
        // making sure they are created with the number of partitions as
        // the maximum of the depending sub-topologies source topics' number of partitions
        final RepartitionTopics repartitionTopics = prepareRepartitionTopics(metadata);
        final Map<TopicPartition, PartitionInfo> allRepartitionTopicPartitions = repartitionTopics.topicPartitionsInfo();
        final Cluster fullMetadata = metadata.withPartitions(allRepartitionTopicPartitions);
        log.debug("Created repartition topics {} from the parsed topology.", allRepartitionTopicPartitions.values());
        // ---------------- Step Two ---------------- //
        // construct the assignment of tasks to clients
        final Map<Subtopology, TopicsInfo> topicGroups = taskManager.topologyMetadata().subtopologyTopicsInfoMapExcluding(repartitionTopics.topologiesWithMissingInputTopics());
        final Set<String> allSourceTopics = new HashSet<>();
        final Map<Subtopology, Set<String>> sourceTopicsByGroup = new HashMap<>();
        for (final Map.Entry<Subtopology, TopicsInfo> entry : topicGroups.entrySet()) {
            allSourceTopics.addAll(entry.getValue().sourceTopics);
            sourceTopicsByGroup.put(entry.getKey(), entry.getValue().sourceTopics);
        }
        // get the tasks as partition groups from the partition grouper
        final Map<TaskId, Set<TopicPartition>> partitionsForTask = partitionGrouper.partitionGroups(sourceTopicsByGroup, fullMetadata);
        final Set<TaskId> statefulTasks = new HashSet<>();
        final boolean probingRebalanceNeeded = assignTasksToClients(fullMetadata, allSourceTopics, topicGroups, clientMetadataMap, partitionsForTask, statefulTasks);
        // ---------------- Step Three ---------------- //
        // construct the global partition assignment per host map
        final Map<HostInfo, Set<TopicPartition>> partitionsByHost = new HashMap<>();
        final Map<HostInfo, Set<TopicPartition>> standbyPartitionsByHost = new HashMap<>();
        if (minReceivedMetadataVersion >= 2) {
            populatePartitionsByHostMaps(partitionsByHost, standbyPartitionsByHost, partitionsForTask, clientMetadataMap);
        }
        streamsMetadataState.onChange(partitionsByHost, standbyPartitionsByHost, fullMetadata);
        // ---------------- Step Four ---------------- //
        // compute the assignment of tasks to threads within each client and build the final group assignment
        final Map<String, Assignment> assignment = computeNewAssignment(statefulTasks, clientMetadataMap, partitionsForTask, partitionsByHost, standbyPartitionsByHost, allOwnedPartitions, minReceivedMetadataVersion, minSupportedMetadataVersion, versionProbing, probingRebalanceNeeded);
        return new GroupAssignment(assignment);
    } catch (final MissingSourceTopicException e) {
        log.error("Caught an error in the task assignment. Returning an error assignment.", e);
        return new GroupAssignment(errorAssignment(clientMetadataMap, AssignorError.INCOMPLETE_SOURCE_TOPIC_METADATA.code()));
    } catch (final TaskAssignmentException e) {
        log.error("Caught an error in the task assignment. Returning an error assignment.", e);
        return new GroupAssignment(errorAssignment(clientMetadataMap, AssignorError.ASSIGNMENT_ERROR.code()));
    }
}