Examples with CreatePartitionsAssignment org.apache.kafka.common.message.CreatePartitionsRequestData.CreatePartitionsAssignment used on opensource projects

Example 1 with CreatePartitionsAssignment

use of org.apache.kafka.common.message.CreatePartitionsRequestData.CreatePartitionsAssignment in project kafka by apache.

the class ReplicationControlManagerTest method testCreatePartitions.

@Test
public void testCreatePartitions() throws Exception {
    ReplicationControlTestContext ctx = new ReplicationControlTestContext();
    ReplicationControlManager replicationControl = ctx.replicationControl;
    CreateTopicsRequestData request = new CreateTopicsRequestData();
    request.topics().add(new CreatableTopic().setName("foo").setNumPartitions(3).setReplicationFactor((short) 2));
    request.topics().add(new CreatableTopic().setName("bar").setNumPartitions(4).setReplicationFactor((short) 2));
    request.topics().add(new CreatableTopic().setName("quux").setNumPartitions(2).setReplicationFactor((short) 2));
    request.topics().add(new CreatableTopic().setName("foo2").setNumPartitions(2).setReplicationFactor((short) 2));
    ctx.registerBrokers(0, 1);
    ctx.unfenceBrokers(0, 1);
    ControllerResult<CreateTopicsResponseData> createTopicResult = replicationControl.createTopics(request);
    ctx.replay(createTopicResult.records());
    List<CreatePartitionsTopic> topics = new ArrayList<>();
    topics.add(new CreatePartitionsTopic().setName("foo").setCount(5).setAssignments(null));
    topics.add(new CreatePartitionsTopic().setName("bar").setCount(3).setAssignments(null));
    topics.add(new CreatePartitionsTopic().setName("baz").setCount(3).setAssignments(null));
    topics.add(new CreatePartitionsTopic().setName("quux").setCount(2).setAssignments(null));
    ControllerResult<List<CreatePartitionsTopicResult>> createPartitionsResult = replicationControl.createPartitions(topics);
    assertEquals(asList(new CreatePartitionsTopicResult().setName("foo").setErrorCode(NONE.code()).setErrorMessage(null), new CreatePartitionsTopicResult().setName("bar").setErrorCode(INVALID_PARTITIONS.code()).setErrorMessage("The topic bar currently has 4 partition(s); 3 would not be an increase."), new CreatePartitionsTopicResult().setName("baz").setErrorCode(UNKNOWN_TOPIC_OR_PARTITION.code()).setErrorMessage(null), new CreatePartitionsTopicResult().setName("quux").setErrorCode(INVALID_PARTITIONS.code()).setErrorMessage("Topic already has 2 partition(s).")), createPartitionsResult.response());
    ctx.replay(createPartitionsResult.records());
    List<CreatePartitionsTopic> topics2 = new ArrayList<>();
    topics2.add(new CreatePartitionsTopic().setName("foo").setCount(6).setAssignments(asList(new CreatePartitionsAssignment().setBrokerIds(asList(1, 0)))));
    topics2.add(new CreatePartitionsTopic().setName("bar").setCount(5).setAssignments(asList(new CreatePartitionsAssignment().setBrokerIds(asList(1)))));
    topics2.add(new CreatePartitionsTopic().setName("quux").setCount(4).setAssignments(asList(new CreatePartitionsAssignment().setBrokerIds(asList(1, 0)))));
    topics2.add(new CreatePartitionsTopic().setName("foo2").setCount(3).setAssignments(asList(new CreatePartitionsAssignment().setBrokerIds(asList(2, 0)))));
    ControllerResult<List<CreatePartitionsTopicResult>> createPartitionsResult2 = replicationControl.createPartitions(topics2);
    assertEquals(asList(new CreatePartitionsTopicResult().setName("foo").setErrorCode(NONE.code()).setErrorMessage(null), new CreatePartitionsTopicResult().setName("bar").setErrorCode(INVALID_REPLICA_ASSIGNMENT.code()).setErrorMessage("The manual partition assignment includes a partition " + "with 1 replica(s), but this is not consistent with previous " + "partitions, which have 2 replica(s)."), new CreatePartitionsTopicResult().setName("quux").setErrorCode(INVALID_REPLICA_ASSIGNMENT.code()).setErrorMessage("Attempted to add 2 additional partition(s), but only 1 assignment(s) were specified."), new CreatePartitionsTopicResult().setName("foo2").setErrorCode(INVALID_REPLICA_ASSIGNMENT.code()).setErrorMessage("The manual partition assignment includes broker 2, but " + "no such broker is registered.")), createPartitionsResult2.response());
    ctx.replay(createPartitionsResult2.records());
}

Example 2 with CreatePartitionsAssignment

use of org.apache.kafka.common.message.CreatePartitionsRequestData.CreatePartitionsAssignment in project kafka by apache.

the class RequestResponseTest method createCreatePartitionsRequestWithAssignments.

private CreatePartitionsRequest createCreatePartitionsRequestWithAssignments(short version) {
    CreatePartitionsTopicCollection topics = new CreatePartitionsTopicCollection();
    CreatePartitionsAssignment myTopicAssignment = new CreatePartitionsAssignment().setBrokerIds(singletonList(2));
    topics.add(new CreatePartitionsTopic().setName("my_topic").setCount(3).setAssignments(singletonList(myTopicAssignment)));
    topics.add(new CreatePartitionsTopic().setName("my_other_topic").setCount(3).setAssignments(asList(new CreatePartitionsAssignment().setBrokerIds(asList(2, 3)), new CreatePartitionsAssignment().setBrokerIds(asList(3, 1)))));
    CreatePartitionsRequestData data = new CreatePartitionsRequestData().setTimeoutMs(0).setValidateOnly(false).setTopics(topics);
    return new CreatePartitionsRequest(data, version);
}

Example 3 with CreatePartitionsAssignment

use of org.apache.kafka.common.message.CreatePartitionsRequestData.CreatePartitionsAssignment in project kafka by apache.

the class ReplicationControlManager method createPartitions.

void createPartitions(CreatePartitionsTopic topic, List<ApiMessageAndVersion> records) {
    Uuid topicId = topicsByName.get(topic.name());
    if (topicId == null) {
        throw new UnknownTopicOrPartitionException();
    }
    TopicControlInfo topicInfo = topics.get(topicId);
    if (topicInfo == null) {
        throw new UnknownTopicOrPartitionException();
    }
    if (topic.count() == topicInfo.parts.size()) {
        throw new InvalidPartitionsException("Topic already has " + topicInfo.parts.size() + " partition(s).");
    } else if (topic.count() < topicInfo.parts.size()) {
        throw new InvalidPartitionsException("The topic " + topic.name() + " currently " + "has " + topicInfo.parts.size() + " partition(s); " + topic.count() + " would not be an increase.");
    }
    int additional = topic.count() - topicInfo.parts.size();
    if (topic.assignments() != null) {
        if (topic.assignments().size() != additional) {
            throw new InvalidReplicaAssignmentException("Attempted to add " + additional + " additional partition(s), but only " + topic.assignments().size() + " assignment(s) were specified.");
        }
    }
    Iterator<PartitionRegistration> iterator = topicInfo.parts.values().iterator();
    if (!iterator.hasNext()) {
        throw new UnknownServerException("Invalid state: topic " + topic.name() + " appears to have no partitions.");
    }
    PartitionRegistration partitionInfo = iterator.next();
    if (partitionInfo.replicas.length > Short.MAX_VALUE) {
        throw new UnknownServerException("Invalid replication factor " + partitionInfo.replicas.length + ": expected a number equal to less than " + Short.MAX_VALUE);
    }
    short replicationFactor = (short) partitionInfo.replicas.length;
    int startPartitionId = topicInfo.parts.size();
    List<List<Integer>> placements;
    List<List<Integer>> isrs;
    if (topic.assignments() != null) {
        placements = new ArrayList<>();
        isrs = new ArrayList<>();
        for (int i = 0; i < topic.assignments().size(); i++) {
            CreatePartitionsAssignment assignment = topic.assignments().get(i);
            validateManualPartitionAssignment(assignment.brokerIds(), OptionalInt.of(replicationFactor));
            placements.add(assignment.brokerIds());
            List<Integer> isr = assignment.brokerIds().stream().filter(clusterControl::unfenced).collect(Collectors.toList());
            if (isr.isEmpty()) {
                throw new InvalidReplicaAssignmentException("All brokers specified in the manual partition assignment for " + "partition " + (startPartitionId + i) + " are fenced.");
            }
            isrs.add(isr);
        }
    } else {
        placements = clusterControl.placeReplicas(startPartitionId, additional, replicationFactor);
        isrs = placements;
    }
    int partitionId = startPartitionId;
    for (int i = 0; i < placements.size(); i++) {
        List<Integer> placement = placements.get(i);
        List<Integer> isr = isrs.get(i);
        records.add(new ApiMessageAndVersion(new PartitionRecord().setPartitionId(partitionId).setTopicId(topicId).setReplicas(placement).setIsr(isr).setRemovingReplicas(Collections.emptyList()).setAddingReplicas(Collections.emptyList()).setLeader(isr.get(0)).setLeaderEpoch(0).setPartitionEpoch(0), PARTITION_RECORD.highestSupportedVersion()));
        partitionId++;
    }
}

Example 4 with CreatePartitionsAssignment

use of org.apache.kafka.common.message.CreatePartitionsRequestData.CreatePartitionsAssignment in project kafka by apache.

the class KafkaAdminClient method createPartitions.

@Override
public CreatePartitionsResult createPartitions(final Map<String, NewPartitions> newPartitions, final CreatePartitionsOptions options) {
    final Map<String, KafkaFutureImpl<Void>> futures = new HashMap<>(newPartitions.size());
    final CreatePartitionsTopicCollection topics = new CreatePartitionsTopicCollection(newPartitions.size());
    for (Map.Entry<String, NewPartitions> entry : newPartitions.entrySet()) {
        final String topic = entry.getKey();
        final NewPartitions newPartition = entry.getValue();
        List<List<Integer>> newAssignments = newPartition.assignments();
        List<CreatePartitionsAssignment> assignments = newAssignments == null ? null : newAssignments.stream().map(brokerIds -> new CreatePartitionsAssignment().setBrokerIds(brokerIds)).collect(Collectors.toList());
        topics.add(new CreatePartitionsTopic().setName(topic).setCount(newPartition.totalCount()).setAssignments(assignments));
        futures.put(topic, new KafkaFutureImpl<>());
    }
    if (!topics.isEmpty()) {
        final long now = time.milliseconds();
        final long deadline = calcDeadlineMs(now, options.timeoutMs());
        final Call call = getCreatePartitionsCall(options, futures, topics, Collections.emptyMap(), now, deadline);
        runnable.call(call, now);
    }
    return new CreatePartitionsResult(new HashMap<>(futures));
}