use of org.apache.kafka.common.config.ConfigResource.Type.TOPIC in project kafka by apache.
the class QuorumControllerTest method testMissingInMemorySnapshot.
@Test
public void testMissingInMemorySnapshot() throws Exception {
int numBrokers = 3;
int numPartitions = 3;
String topicName = "topic-name";
try (LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv(1, Optional.empty());
QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv(logEnv, b -> b.setConfigDefs(CONFIGS))) {
QuorumController controller = controlEnv.activeController();
Map<Integer, Long> brokerEpochs = registerBrokers(controller, numBrokers);
// Create a lot of partitions
List<CreatableReplicaAssignment> partitions = IntStream.range(0, numPartitions).mapToObj(partitionIndex -> new CreatableReplicaAssignment().setPartitionIndex(partitionIndex).setBrokerIds(Arrays.asList(0, 1, 2))).collect(Collectors.toList());
Uuid topicId = controller.createTopics(new CreateTopicsRequestData().setTopics(new CreatableTopicCollection(Collections.singleton(new CreatableTopic().setName(topicName).setNumPartitions(-1).setReplicationFactor((short) -1).setAssignments(new CreatableReplicaAssignmentCollection(partitions.iterator()))).iterator()))).get().topics().find(topicName).topicId();
// Create a lot of alter isr
List<AlterIsrRequestData.PartitionData> alterIsrs = IntStream.range(0, numPartitions).mapToObj(partitionIndex -> {
PartitionRegistration partitionRegistration = controller.replicationControl().getPartition(topicId, partitionIndex);
return new AlterIsrRequestData.PartitionData().setPartitionIndex(partitionIndex).setLeaderEpoch(partitionRegistration.leaderEpoch).setCurrentIsrVersion(partitionRegistration.partitionEpoch).setNewIsr(Arrays.asList(0, 1));
}).collect(Collectors.toList());
AlterIsrRequestData.TopicData topicData = new AlterIsrRequestData.TopicData().setName(topicName);
topicData.partitions().addAll(alterIsrs);
int leaderId = 0;
AlterIsrRequestData alterIsrRequest = new AlterIsrRequestData().setBrokerId(leaderId).setBrokerEpoch(brokerEpochs.get(leaderId));
alterIsrRequest.topics().add(topicData);
logEnv.logManagers().get(0).resignAfterNonAtomicCommit();
int oldClaimEpoch = controller.curClaimEpoch();
assertThrows(ExecutionException.class, () -> controller.alterIsr(alterIsrRequest).get());
// Wait for the controller to become active again
assertSame(controller, controlEnv.activeController());
assertTrue(oldClaimEpoch < controller.curClaimEpoch(), String.format("oldClaimEpoch = %s, newClaimEpoch = %s", oldClaimEpoch, controller.curClaimEpoch()));
// Since the alterIsr partially failed we expect to see
// some partitions to still have 2 in the ISR.
int partitionsWithReplica2 = Utils.toList(controller.replicationControl().brokersToIsrs().partitionsWithBrokerInIsr(2)).size();
int partitionsWithReplica0 = Utils.toList(controller.replicationControl().brokersToIsrs().partitionsWithBrokerInIsr(0)).size();
assertEquals(numPartitions, partitionsWithReplica0);
assertNotEquals(0, partitionsWithReplica2);
assertTrue(partitionsWithReplica0 > partitionsWithReplica2, String.format("partitionsWithReplica0 = %s, partitionsWithReplica2 = %s", partitionsWithReplica0, partitionsWithReplica2));
}
}
use of org.apache.kafka.common.config.ConfigResource.Type.TOPIC in project kafka by apache.
the class ReplicationControlManager method createTopics.
ControllerResult<CreateTopicsResponseData> createTopics(CreateTopicsRequestData request) {
Map<String, ApiError> topicErrors = new HashMap<>();
List<ApiMessageAndVersion> records = new ArrayList<>();
// Check the topic names.
validateNewTopicNames(topicErrors, request.topics());
// Identify topics that already exist and mark them with the appropriate error
request.topics().stream().filter(creatableTopic -> topicsByName.containsKey(creatableTopic.name())).forEach(t -> topicErrors.put(t.name(), new ApiError(Errors.TOPIC_ALREADY_EXISTS, "Topic '" + t.name() + "' already exists.")));
// Verify that the configurations for the new topics are OK, and figure out what
// ConfigRecords should be created.
Map<ConfigResource, Map<String, Entry<OpType, String>>> configChanges = computeConfigChanges(topicErrors, request.topics());
ControllerResult<Map<ConfigResource, ApiError>> configResult = configurationControl.incrementalAlterConfigs(configChanges, NO_OP_EXISTENCE_CHECKER);
for (Entry<ConfigResource, ApiError> entry : configResult.response().entrySet()) {
if (entry.getValue().isFailure()) {
topicErrors.put(entry.getKey().name(), entry.getValue());
}
}
records.addAll(configResult.records());
// Try to create whatever topics are needed.
Map<String, CreatableTopicResult> successes = new HashMap<>();
for (CreatableTopic topic : request.topics()) {
if (topicErrors.containsKey(topic.name()))
continue;
ApiError error;
try {
error = createTopic(topic, records, successes);
} catch (ApiException e) {
error = ApiError.fromThrowable(e);
}
if (error.isFailure()) {
topicErrors.put(topic.name(), error);
}
}
// Create responses for all topics.
CreateTopicsResponseData data = new CreateTopicsResponseData();
StringBuilder resultsBuilder = new StringBuilder();
String resultsPrefix = "";
for (CreatableTopic topic : request.topics()) {
ApiError error = topicErrors.get(topic.name());
if (error != null) {
data.topics().add(new CreatableTopicResult().setName(topic.name()).setErrorCode(error.error().code()).setErrorMessage(error.message()));
resultsBuilder.append(resultsPrefix).append(topic).append(": ").append(error.error()).append(" (").append(error.message()).append(")");
resultsPrefix = ", ";
continue;
}
CreatableTopicResult result = successes.get(topic.name());
data.topics().add(result);
resultsBuilder.append(resultsPrefix).append(topic).append(": ").append("SUCCESS");
resultsPrefix = ", ";
}
if (request.validateOnly()) {
log.info("Validate-only CreateTopics result(s): {}", resultsBuilder.toString());
return ControllerResult.atomicOf(Collections.emptyList(), data);
} else {
log.info("CreateTopics result(s): {}", resultsBuilder.toString());
return ControllerResult.atomicOf(records, data);
}
}
use of org.apache.kafka.common.config.ConfigResource.Type.TOPIC in project kafka by apache.
the class ReplicationControlManager method generateLeaderAndIsrUpdates.
/**
* Iterate over a sequence of partitions and generate ISR changes and/or leader
* changes if necessary.
*
* @param context A human-readable context string used in log4j logging.
* @param brokerToRemove NO_LEADER if no broker is being removed; the ID of the
* broker to remove from the ISR and leadership, otherwise.
* @param brokerToAdd NO_LEADER if no broker is being added; the ID of the
* broker which is now eligible to be a leader, otherwise.
* @param records A list of records which we will append to.
* @param iterator The iterator containing the partitions to examine.
*/
void generateLeaderAndIsrUpdates(String context, int brokerToRemove, int brokerToAdd, List<ApiMessageAndVersion> records, Iterator<TopicIdPartition> iterator) {
int oldSize = records.size();
// If the caller passed a valid broker ID for brokerToAdd, rather than passing
// NO_LEADER, that node will be considered an acceptable leader even if it is
// currently fenced. This is useful when handling unfencing. The reason is that
// while we're generating the records to handle unfencing, the ClusterControlManager
// still shows the node as fenced.
//
// Similarly, if the caller passed a valid broker ID for brokerToRemove, rather
// than passing NO_LEADER, that node will never be considered an acceptable leader.
// This is useful when handling a newly fenced node. We also exclude brokerToRemove
// from the target ISR, but we need to exclude it here too, to handle the case
// where there is an unclean leader election which chooses a leader from outside
// the ISR.
Function<Integer, Boolean> isAcceptableLeader = r -> (r != brokerToRemove) && (r == brokerToAdd || clusterControl.unfenced(r));
while (iterator.hasNext()) {
TopicIdPartition topicIdPart = iterator.next();
TopicControlInfo topic = topics.get(topicIdPart.topicId());
if (topic == null) {
throw new RuntimeException("Topic ID " + topicIdPart.topicId() + " existed in isrMembers, but not in the topics map.");
}
PartitionRegistration partition = topic.parts.get(topicIdPart.partitionId());
if (partition == null) {
throw new RuntimeException("Partition " + topicIdPart + " existed in isrMembers, but not in the partitions map.");
}
PartitionChangeBuilder builder = new PartitionChangeBuilder(partition, topicIdPart.topicId(), topicIdPart.partitionId(), isAcceptableLeader, () -> configurationControl.uncleanLeaderElectionEnabledForTopic(topic.name));
// Note: if brokerToRemove was passed as NO_LEADER, this is a no-op (the new
// target ISR will be the same as the old one).
builder.setTargetIsr(Replicas.toList(Replicas.copyWithout(partition.isr, brokerToRemove)));
builder.build().ifPresent(records::add);
}
if (records.size() != oldSize) {
if (log.isDebugEnabled()) {
StringBuilder bld = new StringBuilder();
String prefix = "";
for (ListIterator<ApiMessageAndVersion> iter = records.listIterator(oldSize); iter.hasNext(); ) {
ApiMessageAndVersion apiMessageAndVersion = iter.next();
PartitionChangeRecord record = (PartitionChangeRecord) apiMessageAndVersion.message();
bld.append(prefix).append(topics.get(record.topicId()).name).append("-").append(record.partitionId());
prefix = ", ";
}
log.debug("{}: changing partition(s): {}", context, bld.toString());
} else if (log.isInfoEnabled()) {
log.info("{}: changing {} partition(s)", context, records.size() - oldSize);
}
}
}
Aggregations