Examples with PartitionRegistration org.apache.kafka.metadata.PartitionRegistration used on opensource projects

Example 21 with PartitionRegistration

use of org.apache.kafka.metadata.PartitionRegistration in project kafka by apache.

the class ReplicationControlManager method createTopic.

private ApiError createTopic(CreatableTopic topic, List<ApiMessageAndVersion> records, Map<String, CreatableTopicResult> successes) {
    Map<Integer, PartitionRegistration> newParts = new HashMap<>();
    if (!topic.assignments().isEmpty()) {
        if (topic.replicationFactor() != -1) {
            return new ApiError(INVALID_REQUEST, "A manual partition assignment was specified, but replication " + "factor was not set to -1.");
        }
        if (topic.numPartitions() != -1) {
            return new ApiError(INVALID_REQUEST, "A manual partition assignment was specified, but numPartitions " + "was not set to -1.");
        }
        OptionalInt replicationFactor = OptionalInt.empty();
        for (CreatableReplicaAssignment assignment : topic.assignments()) {
            if (newParts.containsKey(assignment.partitionIndex())) {
                return new ApiError(Errors.INVALID_REPLICA_ASSIGNMENT, "Found multiple manual partition assignments for partition " + assignment.partitionIndex());
            }
            validateManualPartitionAssignment(assignment.brokerIds(), replicationFactor);
            replicationFactor = OptionalInt.of(assignment.brokerIds().size());
            List<Integer> isr = assignment.brokerIds().stream().filter(clusterControl::unfenced).collect(Collectors.toList());
            if (isr.isEmpty()) {
                return new ApiError(Errors.INVALID_REPLICA_ASSIGNMENT, "All brokers specified in the manual partition assignment for " + "partition " + assignment.partitionIndex() + " are fenced.");
            }
            newParts.put(assignment.partitionIndex(), new PartitionRegistration(Replicas.toArray(assignment.brokerIds()), Replicas.toArray(isr), Replicas.NONE, Replicas.NONE, isr.get(0), 0, 0));
        }
        ApiError error = maybeCheckCreateTopicPolicy(() -> {
            Map<Integer, List<Integer>> assignments = new HashMap<>();
            newParts.entrySet().forEach(e -> assignments.put(e.getKey(), Replicas.toList(e.getValue().replicas)));
            Map<String, String> configs = new HashMap<>();
            topic.configs().forEach(config -> configs.put(config.name(), config.value()));
            return new CreateTopicPolicy.RequestMetadata(topic.name(), null, null, assignments, configs);
        });
        if (error.isFailure())
            return error;
    } else if (topic.replicationFactor() < -1 || topic.replicationFactor() == 0) {
        return new ApiError(Errors.INVALID_REPLICATION_FACTOR, "Replication factor must be larger than 0, or -1 to use the default value.");
    } else if (topic.numPartitions() < -1 || topic.numPartitions() == 0) {
        return new ApiError(Errors.INVALID_PARTITIONS, "Number of partitions was set to an invalid non-positive value.");
    } else {
        int numPartitions = topic.numPartitions() == -1 ? defaultNumPartitions : topic.numPartitions();
        short replicationFactor = topic.replicationFactor() == -1 ? defaultReplicationFactor : topic.replicationFactor();
        try {
            List<List<Integer>> replicas = clusterControl.placeReplicas(0, numPartitions, replicationFactor);
            for (int partitionId = 0; partitionId < replicas.size(); partitionId++) {
                int[] r = Replicas.toArray(replicas.get(partitionId));
                newParts.put(partitionId, new PartitionRegistration(r, r, Replicas.NONE, Replicas.NONE, r[0], 0, 0));
            }
        } catch (InvalidReplicationFactorException e) {
            return new ApiError(Errors.INVALID_REPLICATION_FACTOR, "Unable to replicate the partition " + replicationFactor + " time(s): " + e.getMessage());
        }
        ApiError error = maybeCheckCreateTopicPolicy(() -> {
            Map<String, String> configs = new HashMap<>();
            topic.configs().forEach(config -> configs.put(config.name(), config.value()));
            return new CreateTopicPolicy.RequestMetadata(topic.name(), numPartitions, replicationFactor, null, configs);
        });
        if (error.isFailure())
            return error;
    }
    Uuid topicId = Uuid.randomUuid();
    successes.put(topic.name(), new CreatableTopicResult().setName(topic.name()).setTopicId(topicId).setErrorCode((short) 0).setErrorMessage(null).setNumPartitions(newParts.size()).setReplicationFactor((short) newParts.get(0).replicas.length));
    records.add(new ApiMessageAndVersion(new TopicRecord().setName(topic.name()).setTopicId(topicId), TOPIC_RECORD.highestSupportedVersion()));
    for (Entry<Integer, PartitionRegistration> partEntry : newParts.entrySet()) {
        int partitionIndex = partEntry.getKey();
        PartitionRegistration info = partEntry.getValue();
        records.add(info.toRecord(topicId, partitionIndex));
    }
    return ApiError.NONE;
}

Example 22 with PartitionRegistration

use of org.apache.kafka.metadata.PartitionRegistration in project kafka by apache.

the class ReplicationControlManager method createPartitions.

void createPartitions(CreatePartitionsTopic topic, List<ApiMessageAndVersion> records) {
    Uuid topicId = topicsByName.get(topic.name());
    if (topicId == null) {
        throw new UnknownTopicOrPartitionException();
    }
    TopicControlInfo topicInfo = topics.get(topicId);
    if (topicInfo == null) {
        throw new UnknownTopicOrPartitionException();
    }
    if (topic.count() == topicInfo.parts.size()) {
        throw new InvalidPartitionsException("Topic already has " + topicInfo.parts.size() + " partition(s).");
    } else if (topic.count() < topicInfo.parts.size()) {
        throw new InvalidPartitionsException("The topic " + topic.name() + " currently " + "has " + topicInfo.parts.size() + " partition(s); " + topic.count() + " would not be an increase.");
    }
    int additional = topic.count() - topicInfo.parts.size();
    if (topic.assignments() != null) {
        if (topic.assignments().size() != additional) {
            throw new InvalidReplicaAssignmentException("Attempted to add " + additional + " additional partition(s), but only " + topic.assignments().size() + " assignment(s) were specified.");
        }
    }
    Iterator<PartitionRegistration> iterator = topicInfo.parts.values().iterator();
    if (!iterator.hasNext()) {
        throw new UnknownServerException("Invalid state: topic " + topic.name() + " appears to have no partitions.");
    }
    PartitionRegistration partitionInfo = iterator.next();
    if (partitionInfo.replicas.length > Short.MAX_VALUE) {
        throw new UnknownServerException("Invalid replication factor " + partitionInfo.replicas.length + ": expected a number equal to less than " + Short.MAX_VALUE);
    }
    short replicationFactor = (short) partitionInfo.replicas.length;
    int startPartitionId = topicInfo.parts.size();
    List<List<Integer>> placements;
    List<List<Integer>> isrs;
    if (topic.assignments() != null) {
        placements = new ArrayList<>();
        isrs = new ArrayList<>();
        for (int i = 0; i < topic.assignments().size(); i++) {
            CreatePartitionsAssignment assignment = topic.assignments().get(i);
            validateManualPartitionAssignment(assignment.brokerIds(), OptionalInt.of(replicationFactor));
            placements.add(assignment.brokerIds());
            List<Integer> isr = assignment.brokerIds().stream().filter(clusterControl::unfenced).collect(Collectors.toList());
            if (isr.isEmpty()) {
                throw new InvalidReplicaAssignmentException("All brokers specified in the manual partition assignment for " + "partition " + (startPartitionId + i) + " are fenced.");
            }
            isrs.add(isr);
        }
    } else {
        placements = clusterControl.placeReplicas(startPartitionId, additional, replicationFactor);
        isrs = placements;
    }
    int partitionId = startPartitionId;
    for (int i = 0; i < placements.size(); i++) {
        List<Integer> placement = placements.get(i);
        List<Integer> isr = isrs.get(i);
        records.add(new ApiMessageAndVersion(new PartitionRecord().setPartitionId(partitionId).setTopicId(topicId).setReplicas(placement).setIsr(isr).setRemovingReplicas(Collections.emptyList()).setAddingReplicas(Collections.emptyList()).setLeader(isr.get(0)).setLeaderEpoch(0).setPartitionEpoch(0), PARTITION_RECORD.highestSupportedVersion()));
        partitionId++;
    }
}

Example 23 with PartitionRegistration

use of org.apache.kafka.metadata.PartitionRegistration in project kafka by apache.

the class ReplicationControlManager method replay.

public void replay(RemoveTopicRecord record) {
    // Remove this topic from the topics map and the topicsByName map.
    TopicControlInfo topic = topics.remove(record.topicId());
    if (topic == null) {
        throw new UnknownTopicIdException("Can't find topic with ID " + record.topicId() + " to remove.");
    }
    topicsByName.remove(topic.name);
    reassigningTopics.remove(record.topicId());
    // Delete the configurations associated with this topic.
    configurationControl.deleteTopicConfigs(topic.name);
    // Remove the entries for this topic in brokersToIsrs.
    for (PartitionRegistration partition : topic.parts.values()) {
        for (int i = 0; i < partition.isr.length; i++) {
            brokersToIsrs.removeTopicEntryForBroker(topic.id, partition.isr[i]);
        }
        if (partition.leader != partition.preferredReplica()) {
            preferredReplicaImbalanceCount.decrement();
        }
        globalPartitionCount.decrement();
    }
    brokersToIsrs.removeTopicEntryForBroker(topic.id, NO_LEADER);
    controllerMetrics.setGlobalTopicsCount(topics.size());
    controllerMetrics.setGlobalPartitionCount(globalPartitionCount.get());
    controllerMetrics.setOfflinePartitionCount(brokersToIsrs.offlinePartitionCount());
    controllerMetrics.setPreferredReplicaImbalanceCount(preferredReplicaImbalanceCount.get());
    log.info("Removed topic {} with ID {}.", topic.name, record.topicId());
}

Example 24 with PartitionRegistration

use of org.apache.kafka.metadata.PartitionRegistration in project kafka by apache.

the class ReplicationControlManager method generateLeaderAndIsrUpdates.

/**
 * Iterate over a sequence of partitions and generate ISR changes and/or leader
 * changes if necessary.
 *
 * @param context           A human-readable context string used in log4j logging.
 * @param brokerToRemove    NO_LEADER if no broker is being removed; the ID of the
 *                          broker to remove from the ISR and leadership, otherwise.
 * @param brokerToAdd       NO_LEADER if no broker is being added; the ID of the
 *                          broker which is now eligible to be a leader, otherwise.
 * @param records           A list of records which we will append to.
 * @param iterator          The iterator containing the partitions to examine.
 */
void generateLeaderAndIsrUpdates(String context, int brokerToRemove, int brokerToAdd, List<ApiMessageAndVersion> records, Iterator<TopicIdPartition> iterator) {
    int oldSize = records.size();
    // If the caller passed a valid broker ID for brokerToAdd, rather than passing
    // NO_LEADER, that node will be considered an acceptable leader even if it is
    // currently fenced. This is useful when handling unfencing. The reason is that
    // while we're generating the records to handle unfencing, the ClusterControlManager
    // still shows the node as fenced.
    // 
    // Similarly, if the caller passed a valid broker ID for brokerToRemove, rather
    // than passing NO_LEADER, that node will never be considered an acceptable leader.
    // This is useful when handling a newly fenced node. We also exclude brokerToRemove
    // from the target ISR, but we need to exclude it here too, to handle the case
    // where there is an unclean leader election which chooses a leader from outside
    // the ISR.
    Function<Integer, Boolean> isAcceptableLeader = r -> (r != brokerToRemove) && (r == brokerToAdd || clusterControl.unfenced(r));
    while (iterator.hasNext()) {
        TopicIdPartition topicIdPart = iterator.next();
        TopicControlInfo topic = topics.get(topicIdPart.topicId());
        if (topic == null) {
            throw new RuntimeException("Topic ID " + topicIdPart.topicId() + " existed in isrMembers, but not in the topics map.");
        }
        PartitionRegistration partition = topic.parts.get(topicIdPart.partitionId());
        if (partition == null) {
            throw new RuntimeException("Partition " + topicIdPart + " existed in isrMembers, but not in the partitions map.");
        }
        PartitionChangeBuilder builder = new PartitionChangeBuilder(partition, topicIdPart.topicId(), topicIdPart.partitionId(), isAcceptableLeader, () -> configurationControl.uncleanLeaderElectionEnabledForTopic(topic.name));
        // Note: if brokerToRemove was passed as NO_LEADER, this is a no-op (the new
        // target ISR will be the same as the old one).
        builder.setTargetIsr(Replicas.toList(Replicas.copyWithout(partition.isr, brokerToRemove)));
        builder.build().ifPresent(records::add);
    }
    if (records.size() != oldSize) {
        if (log.isDebugEnabled()) {
            StringBuilder bld = new StringBuilder();
            String prefix = "";
            for (ListIterator<ApiMessageAndVersion> iter = records.listIterator(oldSize); iter.hasNext(); ) {
                ApiMessageAndVersion apiMessageAndVersion = iter.next();
                PartitionChangeRecord record = (PartitionChangeRecord) apiMessageAndVersion.message();
                bld.append(prefix).append(topics.get(record.topicId()).name).append("-").append(record.partitionId());
                prefix = ", ";
            }
            log.debug("{}: changing partition(s): {}", context, bld.toString());
        } else if (log.isInfoEnabled()) {
            log.info("{}: changing {} partition(s)", context, records.size() - oldSize);
        }
    }
}

Example 25 with PartitionRegistration

use of org.apache.kafka.metadata.PartitionRegistration in project kafka by apache.

the class TopicDelta method localChanges.

/**
 * Find the partitions that have change based on the replica given.
 *
 * The changes identified are:
 *   1. partitions for which the broker is not a replica anymore
 *   2. partitions for which the broker is now the leader
 *   3. partitions for which the broker is now a follower
 *
 * @param brokerId the broker id
 * @return the list of partitions which the broker should remove, become leader or become follower.
 */
public LocalReplicaChanges localChanges(int brokerId) {
    Set<TopicPartition> deletes = new HashSet<>();
    Map<TopicPartition, LocalReplicaChanges.PartitionInfo> leaders = new HashMap<>();
    Map<TopicPartition, LocalReplicaChanges.PartitionInfo> followers = new HashMap<>();
    for (Entry<Integer, PartitionRegistration> entry : partitionChanges.entrySet()) {
        if (!Replicas.contains(entry.getValue().replicas, brokerId)) {
            PartitionRegistration prevPartition = image.partitions().get(entry.getKey());
            if (prevPartition != null && Replicas.contains(prevPartition.replicas, brokerId)) {
                deletes.add(new TopicPartition(name(), entry.getKey()));
            }
        } else if (entry.getValue().leader == brokerId) {
            PartitionRegistration prevPartition = image.partitions().get(entry.getKey());
            if (prevPartition == null || prevPartition.partitionEpoch != entry.getValue().partitionEpoch) {
                leaders.put(new TopicPartition(name(), entry.getKey()), new LocalReplicaChanges.PartitionInfo(id(), entry.getValue()));
            }
        } else if (entry.getValue().leader != brokerId && Replicas.contains(entry.getValue().replicas, brokerId)) {
            PartitionRegistration prevPartition = image.partitions().get(entry.getKey());
            if (prevPartition == null || prevPartition.partitionEpoch != entry.getValue().partitionEpoch) {
                followers.put(new TopicPartition(name(), entry.getKey()), new LocalReplicaChanges.PartitionInfo(id(), entry.getValue()));
            }
        }
    }
    return new LocalReplicaChanges(deletes, leaders, followers);
}