Examples with TaskAssignmentException org.apache.kafka.streams.errors.TaskAssignmentException used on opensource projects

Example 11 with TaskAssignmentException

use of org.apache.kafka.streams.errors.TaskAssignmentException in project kafka by apache.

the class StreamsPartitionAssignor method assignTasksToClients.

/**
 * Assigns a set of tasks to each client (Streams instance) using the configured task assignor, and also
 * populate the stateful tasks that have been assigned to the clients
 * @return true if a probing rebalance should be triggered
 */
private boolean assignTasksToClients(final Cluster fullMetadata, final Set<String> allSourceTopics, final Map<Subtopology, TopicsInfo> topicGroups, final Map<UUID, ClientMetadata> clientMetadataMap, final Map<TaskId, Set<TopicPartition>> partitionsForTask, final Set<TaskId> statefulTasks) {
    if (!statefulTasks.isEmpty()) {
        throw new TaskAssignmentException("The stateful tasks should not be populated before assigning tasks to clients");
    }
    final Map<TopicPartition, TaskId> taskForPartition = new HashMap<>();
    final Map<Subtopology, Set<TaskId>> tasksForTopicGroup = new HashMap<>();
    populateTasksForMaps(taskForPartition, tasksForTopicGroup, allSourceTopics, partitionsForTask, fullMetadata);
    final ChangelogTopics changelogTopics = new ChangelogTopics(internalTopicManager, topicGroups, tasksForTopicGroup, logPrefix);
    changelogTopics.setup();
    final Map<UUID, ClientState> clientStates = new HashMap<>();
    final boolean lagComputationSuccessful = populateClientStatesMap(clientStates, clientMetadataMap, taskForPartition, changelogTopics);
    log.info("All members participating in this rebalance: \n{}.", clientStates.entrySet().stream().map(entry -> entry.getKey() + ": " + entry.getValue().consumers()).collect(Collectors.joining(Utils.NL)));
    final Set<TaskId> allTasks = partitionsForTask.keySet();
    statefulTasks.addAll(changelogTopics.statefulTaskIds());
    log.debug("Assigning tasks {} including stateful {} to clients {} with number of replicas {}", allTasks, statefulTasks, clientStates, numStandbyReplicas());
    final TaskAssignor taskAssignor = createTaskAssignor(lagComputationSuccessful);
    final boolean probingRebalanceNeeded = taskAssignor.assign(clientStates, allTasks, statefulTasks, assignmentConfigs);
    log.info("Assigned tasks {} including stateful {} to clients as: \n{}.", allTasks, statefulTasks, clientStates.entrySet().stream().map(entry -> entry.getKey() + "=" + entry.getValue().currentAssignment()).collect(Collectors.joining(Utils.NL)));
    return probingRebalanceNeeded;
}

Example 12 with TaskAssignmentException

use of org.apache.kafka.streams.errors.TaskAssignmentException in project kafka by apache.

the class AssignmentInfo method encode.

/**
 * @throws TaskAssignmentException if method fails to encode the data, e.g., if there is an
 * IO exception during encoding
 */
public ByteBuffer encode() {
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (final DataOutputStream out = new DataOutputStream(baos)) {
        switch(usedVersion) {
            case 1:
                // version
                out.writeInt(usedVersion);
                encodeActiveAndStandbyTaskAssignment(out);
                break;
            case 2:
                // version
                out.writeInt(usedVersion);
                encodeActiveAndStandbyTaskAssignment(out);
                encodePartitionsByHost(out);
                break;
            case 3:
                out.writeInt(usedVersion);
                out.writeInt(commonlySupportedVersion);
                encodeActiveAndStandbyTaskAssignment(out);
                encodePartitionsByHost(out);
                break;
            case 4:
                out.writeInt(usedVersion);
                out.writeInt(commonlySupportedVersion);
                encodeActiveAndStandbyTaskAssignment(out);
                encodePartitionsByHost(out);
                out.writeInt(errCode);
                break;
            case 5:
                out.writeInt(usedVersion);
                out.writeInt(commonlySupportedVersion);
                encodeActiveAndStandbyTaskAssignment(out);
                encodePartitionsByHostAsDictionary(out);
                out.writeInt(errCode);
                break;
            case 6:
                out.writeInt(usedVersion);
                out.writeInt(commonlySupportedVersion);
                encodeActiveAndStandbyTaskAssignment(out);
                encodeActiveAndStandbyHostPartitions(out);
                out.writeInt(errCode);
                break;
            case 7:
            case 8:
            case 9:
            case 10:
                out.writeInt(usedVersion);
                out.writeInt(commonlySupportedVersion);
                encodeActiveAndStandbyTaskAssignment(out);
                encodeActiveAndStandbyHostPartitions(out);
                out.writeInt(errCode);
                out.writeLong(nextRebalanceMs);
                break;
            default:
                throw new IllegalStateException("Unknown metadata version: " + usedVersion + "; latest commonly supported version: " + commonlySupportedVersion);
        }
        out.flush();
        out.close();
        return ByteBuffer.wrap(baos.toByteArray());
    } catch (final IOException ex) {
        throw new TaskAssignmentException("Failed to encode AssignmentInfo", ex);
    }
}

Example 13 with TaskAssignmentException

use of org.apache.kafka.streams.errors.TaskAssignmentException in project kafka by apache.

the class AssignmentInfo method decode.

/**
 * @throws TaskAssignmentException if method fails to decode the data or if the data version is unknown
 */
public static AssignmentInfo decode(final ByteBuffer data) {
    // ensure we are at the beginning of the ByteBuffer
    data.rewind();
    try (final DataInputStream in = new DataInputStream(new ByteBufferInputStream(data))) {
        final AssignmentInfo assignmentInfo;
        final int usedVersion = in.readInt();
        final int commonlySupportedVersion;
        switch(usedVersion) {
            case 1:
                assignmentInfo = new AssignmentInfo(usedVersion, UNKNOWN);
                decodeActiveTasks(assignmentInfo, in);
                decodeStandbyTasks(assignmentInfo, in);
                assignmentInfo.partitionsByHost = new HashMap<>();
                break;
            case 2:
                assignmentInfo = new AssignmentInfo(usedVersion, UNKNOWN);
                decodeActiveTasks(assignmentInfo, in);
                decodeStandbyTasks(assignmentInfo, in);
                decodePartitionsByHost(assignmentInfo, in);
                break;
            case 3:
                commonlySupportedVersion = in.readInt();
                assignmentInfo = new AssignmentInfo(usedVersion, commonlySupportedVersion);
                decodeActiveTasks(assignmentInfo, in);
                decodeStandbyTasks(assignmentInfo, in);
                decodePartitionsByHost(assignmentInfo, in);
                break;
            case 4:
                commonlySupportedVersion = in.readInt();
                assignmentInfo = new AssignmentInfo(usedVersion, commonlySupportedVersion);
                decodeActiveTasks(assignmentInfo, in);
                decodeStandbyTasks(assignmentInfo, in);
                decodePartitionsByHost(assignmentInfo, in);
                assignmentInfo.errCode = in.readInt();
                break;
            case 5:
                commonlySupportedVersion = in.readInt();
                assignmentInfo = new AssignmentInfo(usedVersion, commonlySupportedVersion);
                decodeActiveTasks(assignmentInfo, in);
                decodeStandbyTasks(assignmentInfo, in);
                decodePartitionsByHostUsingDictionary(assignmentInfo, in);
                assignmentInfo.errCode = in.readInt();
                break;
            case 6:
                commonlySupportedVersion = in.readInt();
                assignmentInfo = new AssignmentInfo(usedVersion, commonlySupportedVersion);
                decodeActiveTasks(assignmentInfo, in);
                decodeStandbyTasks(assignmentInfo, in);
                decodeActiveAndStandbyHostPartitions(assignmentInfo, in);
                assignmentInfo.errCode = in.readInt();
                break;
            case 7:
            case 8:
            case 9:
            case 10:
                commonlySupportedVersion = in.readInt();
                assignmentInfo = new AssignmentInfo(usedVersion, commonlySupportedVersion);
                decodeActiveTasks(assignmentInfo, in);
                decodeStandbyTasks(assignmentInfo, in);
                decodeActiveAndStandbyHostPartitions(assignmentInfo, in);
                assignmentInfo.errCode = in.readInt();
                assignmentInfo.nextRebalanceMs = in.readLong();
                break;
            default:
                final TaskAssignmentException fatalException = new TaskAssignmentException("Unable to decode assignment data: " + "used version: " + usedVersion + "; latest supported version: " + LATEST_SUPPORTED_VERSION);
                log.error(fatalException.getMessage(), fatalException);
                throw fatalException;
        }
        return assignmentInfo;
    } catch (final IOException ex) {
        throw new TaskAssignmentException("Failed to decode AssignmentInfo", ex);
    }
}

Example 14 with TaskAssignmentException

use of org.apache.kafka.streams.errors.TaskAssignmentException in project kafka by apache.

the class ConsumerProtocolUtils method writeTaskIdTo.

public static void writeTaskIdTo(final TaskId taskId, final ByteBuffer buf, final int version) {
    buf.putInt(taskId.subtopology());
    buf.putInt(taskId.partition());
    if (version >= MIN_NAMED_TOPOLOGY_VERSION) {
        if (taskId.topologyName() != null) {
            buf.putInt(taskId.topologyName().length());
            for (final char c : taskId.topologyName().toCharArray()) {
                buf.putChar(c);
            }
        } else {
            buf.putInt(0);
        }
    } else if (taskId.topologyName() != null) {
        throw new TaskAssignmentException("Named topologies are not compatible with protocol version " + version);
    }
}

Example 15 with TaskAssignmentException

use of org.apache.kafka.streams.errors.TaskAssignmentException in project kafka by apache.

the class RepartitionTopics method setRepartitionSourceTopicPartitionCount.

/**
 * Computes the number of partitions and sets it for each repartition topic in repartitionTopicMetadata
 */
private void setRepartitionSourceTopicPartitionCount(final Map<String, InternalTopicConfig> repartitionTopicMetadata, final Collection<TopicsInfo> topicGroups, final Cluster clusterMetadata) {
    boolean partitionCountNeeded;
    do {
        partitionCountNeeded = false;
        // avoid infinitely looping without making any progress on unknown repartitions
        boolean progressMadeThisIteration = false;
        for (final TopicsInfo topicsInfo : topicGroups) {
            for (final String repartitionSourceTopic : topicsInfo.repartitionSourceTopics.keySet()) {
                final Optional<Integer> repartitionSourceTopicPartitionCount = repartitionTopicMetadata.get(repartitionSourceTopic).numberOfPartitions();
                if (!repartitionSourceTopicPartitionCount.isPresent()) {
                    final Integer numPartitions = computePartitionCount(repartitionTopicMetadata, topicGroups, clusterMetadata, repartitionSourceTopic);
                    if (numPartitions == null) {
                        partitionCountNeeded = true;
                        log.trace("Unable to determine number of partitions for {}, another iteration is needed", repartitionSourceTopic);
                    } else {
                        log.trace("Determined number of partitions for {} to be {}", repartitionSourceTopic, numPartitions);
                        repartitionTopicMetadata.get(repartitionSourceTopic).setNumberOfPartitions(numPartitions);
                        progressMadeThisIteration = true;
                    }
                }
            }
        }
        if (!progressMadeThisIteration && partitionCountNeeded) {
            log.error("Unable to determine the number of partitions of all repartition topics, most likely a source topic is missing or pattern doesn't match any topics\n" + "topic groups: {}\n" + "cluster topics: {}.", topicGroups, clusterMetadata.topics());
            throw new TaskAssignmentException("Failed to compute number of partitions for all repartition topics, " + "make sure all user input topics are created and all Pattern subscriptions match at least one topic in the cluster");
        }
    } while (partitionCountNeeded);
}