Examples with Topic org.apache.kafka.common.internals.Topic used on opensource projects

Example 6 with Topic

use of org.apache.kafka.common.internals.Topic in project kafka by apache.

the class MetadataTest method testUpdateLastEpoch.

@Test
public void testUpdateLastEpoch() {
    TopicPartition tp = new TopicPartition("topic-1", 0);
    MetadataResponse metadataResponse = emptyMetadataResponse();
    metadata.updateWithCurrentRequestVersion(metadataResponse, false, 0L);
    // if we have no leader epoch, this call shouldn't do anything
    assertFalse(metadata.updateLastSeenEpochIfNewer(tp, 0));
    assertFalse(metadata.updateLastSeenEpochIfNewer(tp, 1));
    assertFalse(metadata.updateLastSeenEpochIfNewer(tp, 2));
    assertFalse(metadata.lastSeenLeaderEpoch(tp).isPresent());
    // Metadata with newer epoch is handled
    metadataResponse = RequestTestUtils.metadataUpdateWith("dummy", 1, Collections.emptyMap(), Collections.singletonMap("topic-1", 1), _tp -> 10);
    metadata.updateWithCurrentRequestVersion(metadataResponse, false, 1L);
    assertOptional(metadata.lastSeenLeaderEpoch(tp), leaderAndEpoch -> assertEquals(leaderAndEpoch.intValue(), 10));
    // Don't update to an older one
    assertFalse(metadata.updateLastSeenEpochIfNewer(tp, 1));
    assertOptional(metadata.lastSeenLeaderEpoch(tp), leaderAndEpoch -> assertEquals(leaderAndEpoch.intValue(), 10));
    // Don't cause update if it's the same one
    assertFalse(metadata.updateLastSeenEpochIfNewer(tp, 10));
    assertOptional(metadata.lastSeenLeaderEpoch(tp), leaderAndEpoch -> assertEquals(leaderAndEpoch.intValue(), 10));
    // Update if we see newer epoch
    assertTrue(metadata.updateLastSeenEpochIfNewer(tp, 12));
    assertOptional(metadata.lastSeenLeaderEpoch(tp), leaderAndEpoch -> assertEquals(leaderAndEpoch.intValue(), 12));
    metadataResponse = RequestTestUtils.metadataUpdateWith("dummy", 1, Collections.emptyMap(), Collections.singletonMap("topic-1", 1), _tp -> 12);
    metadata.updateWithCurrentRequestVersion(metadataResponse, false, 2L);
    assertOptional(metadata.lastSeenLeaderEpoch(tp), leaderAndEpoch -> assertEquals(leaderAndEpoch.intValue(), 12));
    // Don't overwrite metadata with older epoch
    metadataResponse = RequestTestUtils.metadataUpdateWith("dummy", 1, Collections.emptyMap(), Collections.singletonMap("topic-1", 1), _tp -> 11);
    metadata.updateWithCurrentRequestVersion(metadataResponse, false, 3L);
    assertOptional(metadata.lastSeenLeaderEpoch(tp), leaderAndEpoch -> assertEquals(leaderAndEpoch.intValue(), 12));
}

Example 7 with Topic

use of org.apache.kafka.common.internals.Topic in project kafka by apache.

the class MetadataTest method testMetadataMergeOnIdDowngrade.

@Test
public void testMetadataMergeOnIdDowngrade() {
    Time time = new MockTime();
    Map<String, Uuid> topicIds = new HashMap<>();
    final AtomicReference<Set<String>> retainTopics = new AtomicReference<>(new HashSet<>());
    metadata = new Metadata(refreshBackoffMs, metadataExpireMs, new LogContext(), new ClusterResourceListeners()) {

        @Override
        protected boolean retainTopic(String topic, boolean isInternal, long nowMs) {
            return retainTopics.get().contains(topic);
        }
    };
    // Initialize a metadata instance with two topics. Both will be retained.
    String clusterId = "clusterId";
    int nodes = 2;
    Map<String, Integer> topicPartitionCounts = new HashMap<>();
    topicPartitionCounts.put("validTopic1", 2);
    topicPartitionCounts.put("validTopic2", 3);
    retainTopics.set(Utils.mkSet("validTopic1", "validTopic2"));
    topicIds.put("validTopic1", Uuid.randomUuid());
    topicIds.put("validTopic2", Uuid.randomUuid());
    MetadataResponse metadataResponse = RequestTestUtils.metadataUpdateWithIds(clusterId, nodes, Collections.emptyMap(), topicPartitionCounts, _tp -> 100, topicIds);
    metadata.updateWithCurrentRequestVersion(metadataResponse, true, time.milliseconds());
    Map<String, Uuid> metadataTopicIds1 = metadata.topicIds();
    retainTopics.get().forEach(topic -> assertEquals(metadataTopicIds1.get(topic), topicIds.get(topic)));
    // Try removing the topic ID from keepValidTopic (simulating receiving a request from a controller with an older IBP)
    topicIds.remove("validTopic1");
    metadataResponse = RequestTestUtils.metadataUpdateWithIds(clusterId, nodes, Collections.emptyMap(), topicPartitionCounts, _tp -> 200, topicIds);
    metadata.updateWithCurrentRequestVersion(metadataResponse, true, time.milliseconds());
    Map<String, Uuid> metadataTopicIds2 = metadata.topicIds();
    retainTopics.get().forEach(topic -> assertEquals(metadataTopicIds2.get(topic), topicIds.get(topic)));
    Cluster cluster = metadata.fetch();
    // We still have the topic, but it just doesn't have an ID.
    assertEquals(Utils.mkSet("validTopic1", "validTopic2"), cluster.topics());
    assertEquals(2, cluster.partitionsForTopic("validTopic1").size());
    assertEquals(new HashSet<>(topicIds.values()), new HashSet<>(cluster.topicIds()));
    assertEquals(Uuid.ZERO_UUID, cluster.topicId("validTopic1"));
}

Example 8 with Topic

use of org.apache.kafka.common.internals.Topic in project kafka by apache.

the class ReplicationControlManager method createTopics.

ControllerResult<CreateTopicsResponseData> createTopics(CreateTopicsRequestData request) {
    Map<String, ApiError> topicErrors = new HashMap<>();
    List<ApiMessageAndVersion> records = new ArrayList<>();
    // Check the topic names.
    validateNewTopicNames(topicErrors, request.topics());
    // Identify topics that already exist and mark them with the appropriate error
    request.topics().stream().filter(creatableTopic -> topicsByName.containsKey(creatableTopic.name())).forEach(t -> topicErrors.put(t.name(), new ApiError(Errors.TOPIC_ALREADY_EXISTS, "Topic '" + t.name() + "' already exists.")));
    // Verify that the configurations for the new topics are OK, and figure out what
    // ConfigRecords should be created.
    Map<ConfigResource, Map<String, Entry<OpType, String>>> configChanges = computeConfigChanges(topicErrors, request.topics());
    ControllerResult<Map<ConfigResource, ApiError>> configResult = configurationControl.incrementalAlterConfigs(configChanges, NO_OP_EXISTENCE_CHECKER);
    for (Entry<ConfigResource, ApiError> entry : configResult.response().entrySet()) {
        if (entry.getValue().isFailure()) {
            topicErrors.put(entry.getKey().name(), entry.getValue());
        }
    }
    records.addAll(configResult.records());
    // Try to create whatever topics are needed.
    Map<String, CreatableTopicResult> successes = new HashMap<>();
    for (CreatableTopic topic : request.topics()) {
        if (topicErrors.containsKey(topic.name()))
            continue;
        ApiError error;
        try {
            error = createTopic(topic, records, successes);
        } catch (ApiException e) {
            error = ApiError.fromThrowable(e);
        }
        if (error.isFailure()) {
            topicErrors.put(topic.name(), error);
        }
    }
    // Create responses for all topics.
    CreateTopicsResponseData data = new CreateTopicsResponseData();
    StringBuilder resultsBuilder = new StringBuilder();
    String resultsPrefix = "";
    for (CreatableTopic topic : request.topics()) {
        ApiError error = topicErrors.get(topic.name());
        if (error != null) {
            data.topics().add(new CreatableTopicResult().setName(topic.name()).setErrorCode(error.error().code()).setErrorMessage(error.message()));
            resultsBuilder.append(resultsPrefix).append(topic).append(": ").append(error.error()).append(" (").append(error.message()).append(")");
            resultsPrefix = ", ";
            continue;
        }
        CreatableTopicResult result = successes.get(topic.name());
        data.topics().add(result);
        resultsBuilder.append(resultsPrefix).append(topic).append(": ").append("SUCCESS");
        resultsPrefix = ", ";
    }
    if (request.validateOnly()) {
        log.info("Validate-only CreateTopics result(s): {}", resultsBuilder.toString());
        return ControllerResult.atomicOf(Collections.emptyList(), data);
    } else {
        log.info("CreateTopics result(s): {}", resultsBuilder.toString());
        return ControllerResult.atomicOf(records, data);
    }
}

Example 9 with Topic

use of org.apache.kafka.common.internals.Topic in project kafka by apache.

the class ReplicationControlManager method generateLeaderAndIsrUpdates.

/**
 * Iterate over a sequence of partitions and generate ISR changes and/or leader
 * changes if necessary.
 *
 * @param context           A human-readable context string used in log4j logging.
 * @param brokerToRemove    NO_LEADER if no broker is being removed; the ID of the
 *                          broker to remove from the ISR and leadership, otherwise.
 * @param brokerToAdd       NO_LEADER if no broker is being added; the ID of the
 *                          broker which is now eligible to be a leader, otherwise.
 * @param records           A list of records which we will append to.
 * @param iterator          The iterator containing the partitions to examine.
 */
void generateLeaderAndIsrUpdates(String context, int brokerToRemove, int brokerToAdd, List<ApiMessageAndVersion> records, Iterator<TopicIdPartition> iterator) {
    int oldSize = records.size();
    // If the caller passed a valid broker ID for brokerToAdd, rather than passing
    // NO_LEADER, that node will be considered an acceptable leader even if it is
    // currently fenced. This is useful when handling unfencing. The reason is that
    // while we're generating the records to handle unfencing, the ClusterControlManager
    // still shows the node as fenced.
    // 
    // Similarly, if the caller passed a valid broker ID for brokerToRemove, rather
    // than passing NO_LEADER, that node will never be considered an acceptable leader.
    // This is useful when handling a newly fenced node. We also exclude brokerToRemove
    // from the target ISR, but we need to exclude it here too, to handle the case
    // where there is an unclean leader election which chooses a leader from outside
    // the ISR.
    Function<Integer, Boolean> isAcceptableLeader = r -> (r != brokerToRemove) && (r == brokerToAdd || clusterControl.unfenced(r));
    while (iterator.hasNext()) {
        TopicIdPartition topicIdPart = iterator.next();
        TopicControlInfo topic = topics.get(topicIdPart.topicId());
        if (topic == null) {
            throw new RuntimeException("Topic ID " + topicIdPart.topicId() + " existed in isrMembers, but not in the topics map.");
        }
        PartitionRegistration partition = topic.parts.get(topicIdPart.partitionId());
        if (partition == null) {
            throw new RuntimeException("Partition " + topicIdPart + " existed in isrMembers, but not in the partitions map.");
        }
        PartitionChangeBuilder builder = new PartitionChangeBuilder(partition, topicIdPart.topicId(), topicIdPart.partitionId(), isAcceptableLeader, () -> configurationControl.uncleanLeaderElectionEnabledForTopic(topic.name));
        // Note: if brokerToRemove was passed as NO_LEADER, this is a no-op (the new
        // target ISR will be the same as the old one).
        builder.setTargetIsr(Replicas.toList(Replicas.copyWithout(partition.isr, brokerToRemove)));
        builder.build().ifPresent(records::add);
    }
    if (records.size() != oldSize) {
        if (log.isDebugEnabled()) {
            StringBuilder bld = new StringBuilder();
            String prefix = "";
            for (ListIterator<ApiMessageAndVersion> iter = records.listIterator(oldSize); iter.hasNext(); ) {
                ApiMessageAndVersion apiMessageAndVersion = iter.next();
                PartitionChangeRecord record = (PartitionChangeRecord) apiMessageAndVersion.message();
                bld.append(prefix).append(topics.get(record.topicId()).name).append("-").append(record.partitionId());
                prefix = ", ";
            }
            log.debug("{}: changing partition(s): {}", context, bld.toString());
        } else if (log.isInfoEnabled()) {
            log.info("{}: changing {} partition(s)", context, records.size() - oldSize);
        }
    }
}