use of org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData.ReassignablePartition in project kafka by apache.
the class ReplicationControlManagerTest method testReassignPartitions.
@Test
public void testReassignPartitions() throws Exception {
ReplicationControlTestContext ctx = new ReplicationControlTestContext();
ReplicationControlManager replication = ctx.replicationControl;
ctx.registerBrokers(0, 1, 2, 3);
ctx.unfenceBrokers(0, 1, 2, 3);
Uuid fooId = ctx.createTestTopic("foo", new int[][] { new int[] { 1, 2, 3 }, new int[] { 3, 2, 1 } }).topicId();
ctx.createTestTopic("bar", new int[][] { new int[] { 1, 2, 3 } }).topicId();
assertEquals(NONE_REASSIGNING, replication.listPartitionReassignments(null));
ControllerResult<AlterPartitionReassignmentsResponseData> alterResult = replication.alterPartitionReassignments(new AlterPartitionReassignmentsRequestData().setTopics(asList(new ReassignableTopic().setName("foo").setPartitions(asList(new ReassignablePartition().setPartitionIndex(0).setReplicas(asList(3, 2, 1)), new ReassignablePartition().setPartitionIndex(1).setReplicas(asList(0, 2, 1)), new ReassignablePartition().setPartitionIndex(2).setReplicas(asList(0, 2, 1)))), new ReassignableTopic().setName("bar"))));
assertEquals(new AlterPartitionReassignmentsResponseData().setErrorMessage(null).setResponses(asList(new ReassignableTopicResponse().setName("foo").setPartitions(asList(new ReassignablePartitionResponse().setPartitionIndex(0).setErrorMessage(null), new ReassignablePartitionResponse().setPartitionIndex(1).setErrorMessage(null), new ReassignablePartitionResponse().setPartitionIndex(2).setErrorCode(UNKNOWN_TOPIC_OR_PARTITION.code()).setErrorMessage("Unable to find partition foo:2."))), new ReassignableTopicResponse().setName("bar"))), alterResult.response());
ctx.replay(alterResult.records());
ListPartitionReassignmentsResponseData currentReassigning = new ListPartitionReassignmentsResponseData().setErrorMessage(null).setTopics(asList(new OngoingTopicReassignment().setName("foo").setPartitions(asList(new OngoingPartitionReassignment().setPartitionIndex(1).setRemovingReplicas(asList(3)).setAddingReplicas(asList(0)).setReplicas(asList(0, 2, 1, 3))))));
assertEquals(currentReassigning, replication.listPartitionReassignments(null));
assertEquals(NONE_REASSIGNING, replication.listPartitionReassignments(asList(new ListPartitionReassignmentsTopics().setName("bar").setPartitionIndexes(asList(0, 1, 2)))));
assertEquals(currentReassigning, replication.listPartitionReassignments(asList(new ListPartitionReassignmentsTopics().setName("foo").setPartitionIndexes(asList(0, 1, 2)))));
ControllerResult<AlterPartitionReassignmentsResponseData> cancelResult = replication.alterPartitionReassignments(new AlterPartitionReassignmentsRequestData().setTopics(asList(new ReassignableTopic().setName("foo").setPartitions(asList(new ReassignablePartition().setPartitionIndex(0).setReplicas(null), new ReassignablePartition().setPartitionIndex(1).setReplicas(null), new ReassignablePartition().setPartitionIndex(2).setReplicas(null))), new ReassignableTopic().setName("bar").setPartitions(asList(new ReassignablePartition().setPartitionIndex(0).setReplicas(null))))));
assertEquals(ControllerResult.atomicOf(Collections.singletonList(new ApiMessageAndVersion(new PartitionChangeRecord().setTopicId(fooId).setPartitionId(1).setReplicas(asList(2, 1, 3)).setLeader(3).setRemovingReplicas(Collections.emptyList()).setAddingReplicas(Collections.emptyList()), (short) 0)), new AlterPartitionReassignmentsResponseData().setErrorMessage(null).setResponses(asList(new ReassignableTopicResponse().setName("foo").setPartitions(asList(new ReassignablePartitionResponse().setPartitionIndex(0).setErrorCode(NO_REASSIGNMENT_IN_PROGRESS.code()).setErrorMessage(null), new ReassignablePartitionResponse().setPartitionIndex(1).setErrorCode(NONE.code()).setErrorMessage(null), new ReassignablePartitionResponse().setPartitionIndex(2).setErrorCode(UNKNOWN_TOPIC_OR_PARTITION.code()).setErrorMessage("Unable to find partition foo:2."))), new ReassignableTopicResponse().setName("bar").setPartitions(asList(new ReassignablePartitionResponse().setPartitionIndex(0).setErrorCode(NO_REASSIGNMENT_IN_PROGRESS.code()).setErrorMessage(null)))))), cancelResult);
log.info("running final alterIsr...");
ControllerResult<AlterIsrResponseData> alterIsrResult = replication.alterIsr(new AlterIsrRequestData().setBrokerId(3).setBrokerEpoch(103).setTopics(asList(new TopicData().setName("foo").setPartitions(asList(new PartitionData().setPartitionIndex(1).setCurrentIsrVersion(1).setLeaderEpoch(0).setNewIsr(asList(3, 0, 2, 1)))))));
assertEquals(new AlterIsrResponseData().setTopics(asList(new AlterIsrResponseData.TopicData().setName("foo").setPartitions(asList(new AlterIsrResponseData.PartitionData().setPartitionIndex(1).setErrorCode(FENCED_LEADER_EPOCH.code()))))), alterIsrResult.response());
ctx.replay(alterIsrResult.records());
assertEquals(NONE_REASSIGNING, replication.listPartitionReassignments(null));
}
use of org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData.ReassignablePartition in project kafka by apache.
the class ReplicationControlManagerTest method testCancelReassignPartitions.
@Test
public void testCancelReassignPartitions() throws Exception {
ReplicationControlTestContext ctx = new ReplicationControlTestContext();
ReplicationControlManager replication = ctx.replicationControl;
ctx.registerBrokers(0, 1, 2, 3, 4);
ctx.unfenceBrokers(0, 1, 2, 3, 4);
Uuid fooId = ctx.createTestTopic("foo", new int[][] { new int[] { 1, 2, 3, 4 }, new int[] { 0, 1, 2, 3 }, new int[] { 4, 3, 1, 0 }, new int[] { 2, 3, 4, 1 } }).topicId();
Uuid barId = ctx.createTestTopic("bar", new int[][] { new int[] { 4, 3, 2 } }).topicId();
assertEquals(NONE_REASSIGNING, replication.listPartitionReassignments(null));
List<ApiMessageAndVersion> fenceRecords = new ArrayList<>();
replication.handleBrokerFenced(3, fenceRecords);
ctx.replay(fenceRecords);
assertEquals(new PartitionRegistration(new int[] { 1, 2, 3, 4 }, new int[] { 1, 2, 4 }, new int[] {}, new int[] {}, 1, 1, 1), replication.getPartition(fooId, 0));
ControllerResult<AlterPartitionReassignmentsResponseData> alterResult = replication.alterPartitionReassignments(new AlterPartitionReassignmentsRequestData().setTopics(asList(new ReassignableTopic().setName("foo").setPartitions(asList(new ReassignablePartition().setPartitionIndex(0).setReplicas(asList(1, 2, 3)), new ReassignablePartition().setPartitionIndex(1).setReplicas(asList(1, 2, 3, 0)), new ReassignablePartition().setPartitionIndex(2).setReplicas(asList(5, 6, 7)), new ReassignablePartition().setPartitionIndex(3).setReplicas(asList()))), new ReassignableTopic().setName("bar").setPartitions(asList(new ReassignablePartition().setPartitionIndex(0).setReplicas(asList(1, 2, 3, 4, 0)))))));
assertEquals(new AlterPartitionReassignmentsResponseData().setErrorMessage(null).setResponses(asList(new ReassignableTopicResponse().setName("foo").setPartitions(asList(new ReassignablePartitionResponse().setPartitionIndex(0).setErrorMessage(null), new ReassignablePartitionResponse().setPartitionIndex(1).setErrorMessage(null), new ReassignablePartitionResponse().setPartitionIndex(2).setErrorCode(INVALID_REPLICA_ASSIGNMENT.code()).setErrorMessage("The manual partition assignment includes broker 5, " + "but no such broker is registered."), new ReassignablePartitionResponse().setPartitionIndex(3).setErrorCode(INVALID_REPLICA_ASSIGNMENT.code()).setErrorMessage("The manual partition assignment includes an empty " + "replica list."))), new ReassignableTopicResponse().setName("bar").setPartitions(asList(new ReassignablePartitionResponse().setPartitionIndex(0).setErrorMessage(null))))), alterResult.response());
ctx.replay(alterResult.records());
assertEquals(new PartitionRegistration(new int[] { 1, 2, 3 }, new int[] { 1, 2 }, new int[] {}, new int[] {}, 1, 2, 2), replication.getPartition(fooId, 0));
assertEquals(new PartitionRegistration(new int[] { 1, 2, 3, 0 }, new int[] { 0, 1, 2 }, new int[] {}, new int[] {}, 0, 1, 2), replication.getPartition(fooId, 1));
assertEquals(new PartitionRegistration(new int[] { 1, 2, 3, 4, 0 }, new int[] { 4, 2 }, new int[] {}, new int[] { 0, 1 }, 4, 1, 2), replication.getPartition(barId, 0));
ListPartitionReassignmentsResponseData currentReassigning = new ListPartitionReassignmentsResponseData().setErrorMessage(null).setTopics(asList(new OngoingTopicReassignment().setName("bar").setPartitions(asList(new OngoingPartitionReassignment().setPartitionIndex(0).setRemovingReplicas(Collections.emptyList()).setAddingReplicas(asList(0, 1)).setReplicas(asList(1, 2, 3, 4, 0))))));
assertEquals(currentReassigning, replication.listPartitionReassignments(null));
assertEquals(NONE_REASSIGNING, replication.listPartitionReassignments(asList(new ListPartitionReassignmentsTopics().setName("foo").setPartitionIndexes(asList(0, 1, 2)))));
assertEquals(currentReassigning, replication.listPartitionReassignments(asList(new ListPartitionReassignmentsTopics().setName("bar").setPartitionIndexes(asList(0, 1, 2)))));
ControllerResult<AlterIsrResponseData> alterIsrResult = replication.alterIsr(new AlterIsrRequestData().setBrokerId(4).setBrokerEpoch(104).setTopics(asList(new TopicData().setName("bar").setPartitions(asList(new PartitionData().setPartitionIndex(0).setCurrentIsrVersion(2).setLeaderEpoch(1).setNewIsr(asList(4, 1, 2, 3, 0)))))));
assertEquals(new AlterIsrResponseData().setTopics(asList(new AlterIsrResponseData.TopicData().setName("bar").setPartitions(asList(new AlterIsrResponseData.PartitionData().setPartitionIndex(0).setLeaderId(4).setLeaderEpoch(1).setIsr(asList(4, 1, 2, 3, 0)).setCurrentIsrVersion(3).setErrorCode(NONE.code()))))), alterIsrResult.response());
ControllerResult<AlterPartitionReassignmentsResponseData> cancelResult = replication.alterPartitionReassignments(new AlterPartitionReassignmentsRequestData().setTopics(asList(new ReassignableTopic().setName("foo").setPartitions(asList(new ReassignablePartition().setPartitionIndex(0).setReplicas(null))), new ReassignableTopic().setName("bar").setPartitions(asList(new ReassignablePartition().setPartitionIndex(0).setReplicas(null))))));
assertEquals(ControllerResult.atomicOf(Collections.singletonList(new ApiMessageAndVersion(new PartitionChangeRecord().setTopicId(barId).setPartitionId(0).setLeader(4).setReplicas(asList(2, 3, 4)).setRemovingReplicas(null).setAddingReplicas(Collections.emptyList()), (short) 0)), new AlterPartitionReassignmentsResponseData().setErrorMessage(null).setResponses(asList(new ReassignableTopicResponse().setName("foo").setPartitions(asList(new ReassignablePartitionResponse().setPartitionIndex(0).setErrorCode(NO_REASSIGNMENT_IN_PROGRESS.code()).setErrorMessage(null))), new ReassignableTopicResponse().setName("bar").setPartitions(asList(new ReassignablePartitionResponse().setPartitionIndex(0).setErrorMessage(null)))))), cancelResult);
ctx.replay(cancelResult.records());
assertEquals(NONE_REASSIGNING, replication.listPartitionReassignments(null));
assertEquals(new PartitionRegistration(new int[] { 2, 3, 4 }, new int[] { 4, 2 }, new int[] {}, new int[] {}, 4, 2, 3), replication.getPartition(barId, 0));
}
use of org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData.ReassignablePartition in project kafka by apache.
the class ReplicationControlManager method alterPartitionReassignments.
ControllerResult<AlterPartitionReassignmentsResponseData> alterPartitionReassignments(AlterPartitionReassignmentsRequestData request) {
List<ApiMessageAndVersion> records = new ArrayList<>();
AlterPartitionReassignmentsResponseData result = new AlterPartitionReassignmentsResponseData().setErrorMessage(null);
int successfulAlterations = 0, totalAlterations = 0;
for (ReassignableTopic topic : request.topics()) {
ReassignableTopicResponse topicResponse = new ReassignableTopicResponse().setName(topic.name());
for (ReassignablePartition partition : topic.partitions()) {
ApiError error = ApiError.NONE;
try {
alterPartitionReassignment(topic.name(), partition, records);
successfulAlterations++;
} catch (Throwable e) {
log.info("Unable to alter partition reassignment for " + topic.name() + ":" + partition.partitionIndex() + " because " + "of an " + e.getClass().getSimpleName() + " error: " + e.getMessage());
error = ApiError.fromThrowable(e);
}
totalAlterations++;
topicResponse.partitions().add(new ReassignablePartitionResponse().setPartitionIndex(partition.partitionIndex()).setErrorCode(error.error().code()).setErrorMessage(error.message()));
}
result.responses().add(topicResponse);
}
log.info("Successfully altered {} out of {} partition reassignment(s).", successfulAlterations, totalAlterations);
return ControllerResult.atomicOf(records, result);
}
use of org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData.ReassignablePartition in project kafka by apache.
the class KafkaAdminClient method alterPartitionReassignments.
@Override
public AlterPartitionReassignmentsResult alterPartitionReassignments(Map<TopicPartition, Optional<NewPartitionReassignment>> reassignments, AlterPartitionReassignmentsOptions options) {
final Map<TopicPartition, KafkaFutureImpl<Void>> futures = new HashMap<>();
final Map<String, Map<Integer, Optional<NewPartitionReassignment>>> topicsToReassignments = new TreeMap<>();
for (Map.Entry<TopicPartition, Optional<NewPartitionReassignment>> entry : reassignments.entrySet()) {
String topic = entry.getKey().topic();
int partition = entry.getKey().partition();
TopicPartition topicPartition = new TopicPartition(topic, partition);
Optional<NewPartitionReassignment> reassignment = entry.getValue();
KafkaFutureImpl<Void> future = new KafkaFutureImpl<>();
futures.put(topicPartition, future);
if (topicNameIsUnrepresentable(topic)) {
future.completeExceptionally(new InvalidTopicException("The given topic name '" + topic + "' cannot be represented in a request."));
} else if (topicPartition.partition() < 0) {
future.completeExceptionally(new InvalidTopicException("The given partition index " + topicPartition.partition() + " is not valid."));
} else {
Map<Integer, Optional<NewPartitionReassignment>> partitionReassignments = topicsToReassignments.get(topicPartition.topic());
if (partitionReassignments == null) {
partitionReassignments = new TreeMap<>();
topicsToReassignments.put(topic, partitionReassignments);
}
partitionReassignments.put(partition, reassignment);
}
}
final long now = time.milliseconds();
Call call = new Call("alterPartitionReassignments", calcDeadlineMs(now, options.timeoutMs()), new ControllerNodeProvider()) {
@Override
public AlterPartitionReassignmentsRequest.Builder createRequest(int timeoutMs) {
AlterPartitionReassignmentsRequestData data = new AlterPartitionReassignmentsRequestData();
for (Map.Entry<String, Map<Integer, Optional<NewPartitionReassignment>>> entry : topicsToReassignments.entrySet()) {
String topicName = entry.getKey();
Map<Integer, Optional<NewPartitionReassignment>> partitionsToReassignments = entry.getValue();
List<ReassignablePartition> reassignablePartitions = new ArrayList<>();
for (Map.Entry<Integer, Optional<NewPartitionReassignment>> partitionEntry : partitionsToReassignments.entrySet()) {
int partitionIndex = partitionEntry.getKey();
Optional<NewPartitionReassignment> reassignment = partitionEntry.getValue();
ReassignablePartition reassignablePartition = new ReassignablePartition().setPartitionIndex(partitionIndex).setReplicas(reassignment.map(NewPartitionReassignment::targetReplicas).orElse(null));
reassignablePartitions.add(reassignablePartition);
}
ReassignableTopic reassignableTopic = new ReassignableTopic().setName(topicName).setPartitions(reassignablePartitions);
data.topics().add(reassignableTopic);
}
data.setTimeoutMs(timeoutMs);
return new AlterPartitionReassignmentsRequest.Builder(data);
}
@Override
public void handleResponse(AbstractResponse abstractResponse) {
AlterPartitionReassignmentsResponse response = (AlterPartitionReassignmentsResponse) abstractResponse;
Map<TopicPartition, ApiException> errors = new HashMap<>();
int receivedResponsesCount = 0;
Errors topLevelError = Errors.forCode(response.data().errorCode());
switch(topLevelError) {
case NONE:
receivedResponsesCount += validateTopicResponses(response.data().responses(), errors);
break;
case NOT_CONTROLLER:
handleNotControllerError(topLevelError);
break;
default:
for (ReassignableTopicResponse topicResponse : response.data().responses()) {
String topicName = topicResponse.name();
for (ReassignablePartitionResponse partition : topicResponse.partitions()) {
errors.put(new TopicPartition(topicName, partition.partitionIndex()), new ApiError(topLevelError, response.data().errorMessage()).exception());
receivedResponsesCount += 1;
}
}
break;
}
assertResponseCountMatch(errors, receivedResponsesCount);
for (Map.Entry<TopicPartition, ApiException> entry : errors.entrySet()) {
ApiException exception = entry.getValue();
if (exception == null)
futures.get(entry.getKey()).complete(null);
else
futures.get(entry.getKey()).completeExceptionally(exception);
}
}
private void assertResponseCountMatch(Map<TopicPartition, ApiException> errors, int receivedResponsesCount) {
int expectedResponsesCount = topicsToReassignments.values().stream().mapToInt(Map::size).sum();
if (errors.values().stream().noneMatch(Objects::nonNull) && receivedResponsesCount != expectedResponsesCount) {
String quantifier = receivedResponsesCount > expectedResponsesCount ? "many" : "less";
throw new UnknownServerException("The server returned too " + quantifier + " results." + "Expected " + expectedResponsesCount + " but received " + receivedResponsesCount);
}
}
private int validateTopicResponses(List<ReassignableTopicResponse> topicResponses, Map<TopicPartition, ApiException> errors) {
int receivedResponsesCount = 0;
for (ReassignableTopicResponse topicResponse : topicResponses) {
String topicName = topicResponse.name();
for (ReassignablePartitionResponse partResponse : topicResponse.partitions()) {
Errors partitionError = Errors.forCode(partResponse.errorCode());
TopicPartition tp = new TopicPartition(topicName, partResponse.partitionIndex());
if (partitionError == Errors.NONE) {
errors.put(tp, null);
} else {
errors.put(tp, new ApiError(partitionError, partResponse.errorMessage()).exception());
}
receivedResponsesCount += 1;
}
}
return receivedResponsesCount;
}
@Override
void handleFailure(Throwable throwable) {
for (KafkaFutureImpl<Void> future : futures.values()) {
future.completeExceptionally(throwable);
}
}
};
if (!topicsToReassignments.isEmpty()) {
runnable.call(call, now);
}
return new AlterPartitionReassignmentsResult(new HashMap<>(futures));
}
Aggregations