Search in sources :

Example 1 with OpType

use of org.apache.kafka.clients.admin.AlterConfigOp.OpType in project kafka by apache.

the class ConfigurationControlManager method incrementalAlterConfigResource.

private void incrementalAlterConfigResource(ConfigResource configResource, Map<String, Entry<OpType, String>> keysToOps, Consumer<ConfigResource> existenceChecker, List<ApiMessageAndVersion> outputRecords, Map<ConfigResource, ApiError> outputResults) {
    List<ApiMessageAndVersion> newRecords = new ArrayList<>();
    for (Entry<String, Entry<OpType, String>> keysToOpsEntry : keysToOps.entrySet()) {
        String key = keysToOpsEntry.getKey();
        String currentValue = null;
        TimelineHashMap<String, String> currentConfigs = configData.get(configResource);
        if (currentConfigs != null) {
            currentValue = currentConfigs.get(key);
        }
        String newValue = currentValue;
        Entry<OpType, String> opTypeAndNewValue = keysToOpsEntry.getValue();
        OpType opType = opTypeAndNewValue.getKey();
        String opValue = opTypeAndNewValue.getValue();
        switch(opType) {
            case SET:
                newValue = opValue;
                break;
            case DELETE:
                newValue = null;
                break;
            case APPEND:
            case SUBTRACT:
                if (!isSplittable(configResource.type(), key)) {
                    outputResults.put(configResource, new ApiError(INVALID_CONFIG, "Can't " + opType + " to " + "key " + key + " because its type is not LIST."));
                    return;
                }
                List<String> newValueParts = getParts(newValue, key, configResource);
                if (opType == APPEND) {
                    if (!newValueParts.contains(opValue)) {
                        newValueParts.add(opValue);
                    }
                    newValue = String.join(",", newValueParts);
                } else if (newValueParts.remove(opValue)) {
                    newValue = String.join(",", newValueParts);
                }
                break;
        }
        if (!Objects.equals(currentValue, newValue)) {
            newRecords.add(new ApiMessageAndVersion(new ConfigRecord().setResourceType(configResource.type().id()).setResourceName(configResource.name()).setName(key).setValue(newValue), CONFIG_RECORD.highestSupportedVersion()));
        }
    }
    ApiError error = validateAlterConfig(configResource, newRecords, existenceChecker);
    if (error.isFailure()) {
        outputResults.put(configResource, error);
        return;
    }
    outputRecords.addAll(newRecords);
    outputResults.put(configResource, ApiError.NONE);
}
Also used : ConfigRecord(org.apache.kafka.common.metadata.ConfigRecord) Entry(java.util.Map.Entry) ApiMessageAndVersion(org.apache.kafka.server.common.ApiMessageAndVersion) ArrayList(java.util.ArrayList) OpType(org.apache.kafka.clients.admin.AlterConfigOp.OpType) ApiError(org.apache.kafka.common.requests.ApiError)

Example 2 with OpType

use of org.apache.kafka.clients.admin.AlterConfigOp.OpType in project kafka by apache.

the class ReplicationControlManager method createTopics.

ControllerResult<CreateTopicsResponseData> createTopics(CreateTopicsRequestData request) {
    Map<String, ApiError> topicErrors = new HashMap<>();
    List<ApiMessageAndVersion> records = new ArrayList<>();
    // Check the topic names.
    validateNewTopicNames(topicErrors, request.topics());
    // Identify topics that already exist and mark them with the appropriate error
    request.topics().stream().filter(creatableTopic -> topicsByName.containsKey(creatableTopic.name())).forEach(t -> topicErrors.put(t.name(), new ApiError(Errors.TOPIC_ALREADY_EXISTS, "Topic '" + t.name() + "' already exists.")));
    // Verify that the configurations for the new topics are OK, and figure out what
    // ConfigRecords should be created.
    Map<ConfigResource, Map<String, Entry<OpType, String>>> configChanges = computeConfigChanges(topicErrors, request.topics());
    ControllerResult<Map<ConfigResource, ApiError>> configResult = configurationControl.incrementalAlterConfigs(configChanges, NO_OP_EXISTENCE_CHECKER);
    for (Entry<ConfigResource, ApiError> entry : configResult.response().entrySet()) {
        if (entry.getValue().isFailure()) {
            topicErrors.put(entry.getKey().name(), entry.getValue());
        }
    }
    records.addAll(configResult.records());
    // Try to create whatever topics are needed.
    Map<String, CreatableTopicResult> successes = new HashMap<>();
    for (CreatableTopic topic : request.topics()) {
        if (topicErrors.containsKey(topic.name()))
            continue;
        ApiError error;
        try {
            error = createTopic(topic, records, successes);
        } catch (ApiException e) {
            error = ApiError.fromThrowable(e);
        }
        if (error.isFailure()) {
            topicErrors.put(topic.name(), error);
        }
    }
    // Create responses for all topics.
    CreateTopicsResponseData data = new CreateTopicsResponseData();
    StringBuilder resultsBuilder = new StringBuilder();
    String resultsPrefix = "";
    for (CreatableTopic topic : request.topics()) {
        ApiError error = topicErrors.get(topic.name());
        if (error != null) {
            data.topics().add(new CreatableTopicResult().setName(topic.name()).setErrorCode(error.error().code()).setErrorMessage(error.message()));
            resultsBuilder.append(resultsPrefix).append(topic).append(": ").append(error.error()).append(" (").append(error.message()).append(")");
            resultsPrefix = ", ";
            continue;
        }
        CreatableTopicResult result = successes.get(topic.name());
        data.topics().add(result);
        resultsBuilder.append(resultsPrefix).append(topic).append(": ").append("SUCCESS");
        resultsPrefix = ", ";
    }
    if (request.validateOnly()) {
        log.info("Validate-only CreateTopics result(s): {}", resultsBuilder.toString());
        return ControllerResult.atomicOf(Collections.emptyList(), data);
    } else {
        log.info("CreateTopics result(s): {}", resultsBuilder.toString());
        return ControllerResult.atomicOf(records, data);
    }
}
Also used : ListPartitionReassignmentsTopics(org.apache.kafka.common.message.ListPartitionReassignmentsRequestData.ListPartitionReassignmentsTopics) OpType(org.apache.kafka.clients.admin.AlterConfigOp.OpType) InvalidReplicationFactorException(org.apache.kafka.common.errors.InvalidReplicationFactorException) ElectLeadersRequestData(org.apache.kafka.common.message.ElectLeadersRequestData) CreatePartitionsTopic(org.apache.kafka.common.message.CreatePartitionsRequestData.CreatePartitionsTopic) PartitionResult(org.apache.kafka.common.message.ElectLeadersResponseData.PartitionResult) LogContext(org.apache.kafka.common.utils.LogContext) Map(java.util.Map) TimelineInteger(org.apache.kafka.timeline.TimelineInteger) SET(org.apache.kafka.clients.admin.AlterConfigOp.OpType.SET) AlterIsrResponseData(org.apache.kafka.common.message.AlterIsrResponseData) NoReassignmentInProgressException(org.apache.kafka.common.errors.NoReassignmentInProgressException) INVALID_REQUEST(org.apache.kafka.common.protocol.Errors.INVALID_REQUEST) InvalidTopicException(org.apache.kafka.common.errors.InvalidTopicException) UNREGISTER_BROKER_RECORD(org.apache.kafka.common.metadata.MetadataRecordType.UNREGISTER_BROKER_RECORD) SnapshotRegistry(org.apache.kafka.timeline.SnapshotRegistry) NO_LEADER(org.apache.kafka.metadata.LeaderConstants.NO_LEADER) InvalidPartitionsException(org.apache.kafka.common.errors.InvalidPartitionsException) UNKNOWN_TOPIC_ID(org.apache.kafka.common.protocol.Errors.UNKNOWN_TOPIC_ID) PartitionChangeRecord(org.apache.kafka.common.metadata.PartitionChangeRecord) Errors(org.apache.kafka.common.protocol.Errors) UnknownTopicOrPartitionException(org.apache.kafka.common.errors.UnknownTopicOrPartitionException) CreatableTopicResult(org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicResult) RemoveTopicRecord(org.apache.kafka.common.metadata.RemoveTopicRecord) TimelineHashMap(org.apache.kafka.timeline.TimelineHashMap) NO_REASSIGNMENT_IN_PROGRESS(org.apache.kafka.common.protocol.Errors.NO_REASSIGNMENT_IN_PROGRESS) BrokerRegistration(org.apache.kafka.metadata.BrokerRegistration) Supplier(java.util.function.Supplier) TOPIC(org.apache.kafka.common.config.ConfigResource.Type.TOPIC) ArrayList(java.util.ArrayList) FENCED_LEADER_EPOCH(org.apache.kafka.common.protocol.Errors.FENCED_LEADER_EPOCH) UnfenceBrokerRecord(org.apache.kafka.common.metadata.UnfenceBrokerRecord) ElectionType(org.apache.kafka.common.ElectionType) BrokerHeartbeatReply(org.apache.kafka.metadata.BrokerHeartbeatReply) AlterPartitionReassignmentsRequestData(org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData) UnknownTopicIdException(org.apache.kafka.common.errors.UnknownTopicIdException) Topic(org.apache.kafka.common.internals.Topic) CreatableReplicaAssignment(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableReplicaAssignment) FENCE_BROKER_RECORD(org.apache.kafka.common.metadata.MetadataRecordType.FENCE_BROKER_RECORD) ElectLeadersResponseData(org.apache.kafka.common.message.ElectLeadersResponseData) ReassignablePartition(org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData.ReassignablePartition) BrokerHeartbeatRequestData(org.apache.kafka.common.message.BrokerHeartbeatRequestData) CreatableTopicCollection(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopicCollection) CreateTopicsRequestData(org.apache.kafka.common.message.CreateTopicsRequestData) ReassignableTopic(org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData.ReassignableTopic) PartitionRecord(org.apache.kafka.common.metadata.PartitionRecord) CreatePartitionsAssignment(org.apache.kafka.common.message.CreatePartitionsRequestData.CreatePartitionsAssignment) UNFENCE_BROKER_RECORD(org.apache.kafka.common.metadata.MetadataRecordType.UNFENCE_BROKER_RECORD) PARTITION_RECORD(org.apache.kafka.common.metadata.MetadataRecordType.PARTITION_RECORD) BrokerIdNotRegisteredException(org.apache.kafka.common.errors.BrokerIdNotRegisteredException) ListIterator(java.util.ListIterator) CreatePartitionsTopicResult(org.apache.kafka.common.message.CreatePartitionsResponseData.CreatePartitionsTopicResult) ReassignableTopicResponse(org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignableTopicResponse) ReplicaElectionResult(org.apache.kafka.common.message.ElectLeadersResponseData.ReplicaElectionResult) TOPIC_RECORD(org.apache.kafka.common.metadata.MetadataRecordType.TOPIC_RECORD) FenceBrokerRecord(org.apache.kafka.common.metadata.FenceBrokerRecord) TopicRecord(org.apache.kafka.common.metadata.TopicRecord) Collection(java.util.Collection) UnregisterBrokerRecord(org.apache.kafka.common.metadata.UnregisterBrokerRecord) TopicPartitions(org.apache.kafka.common.message.ElectLeadersRequestData.TopicPartitions) PartitionRegistration(org.apache.kafka.metadata.PartitionRegistration) Collectors(java.util.stream.Collectors) Replicas(org.apache.kafka.metadata.Replicas) INVALID_UPDATE_VERSION(org.apache.kafka.common.protocol.Errors.INVALID_UPDATE_VERSION) TopicIdPartition(org.apache.kafka.controller.BrokersToIsrs.TopicIdPartition) List(java.util.List) NO_LEADER_CHANGE(org.apache.kafka.metadata.LeaderConstants.NO_LEADER_CHANGE) Entry(java.util.Map.Entry) Optional(java.util.Optional) CreateTopicsResponseData(org.apache.kafka.common.message.CreateTopicsResponseData) Uuid(org.apache.kafka.common.Uuid) ReassignablePartitionResponse(org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignablePartitionResponse) ListPartitionReassignmentsResponseData(org.apache.kafka.common.message.ListPartitionReassignmentsResponseData) HashMap(java.util.HashMap) REMOVE_TOPIC_RECORD(org.apache.kafka.common.metadata.MetadataRecordType.REMOVE_TOPIC_RECORD) SimpleImmutableEntry(java.util.AbstractMap.SimpleImmutableEntry) CreatableTopic(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic) Function(java.util.function.Function) OptionalInt(java.util.OptionalInt) ApiError(org.apache.kafka.common.requests.ApiError) UnknownServerException(org.apache.kafka.common.errors.UnknownServerException) ConfigResource(org.apache.kafka.common.config.ConfigResource) ApiMessageAndVersion(org.apache.kafka.server.common.ApiMessageAndVersion) PolicyViolationException(org.apache.kafka.common.errors.PolicyViolationException) OngoingPartitionReassignment(org.apache.kafka.common.message.ListPartitionReassignmentsResponseData.OngoingPartitionReassignment) OngoingTopicReassignment(org.apache.kafka.common.message.ListPartitionReassignmentsResponseData.OngoingTopicReassignment) NoSuchElementException(java.util.NoSuchElementException) UNKNOWN_TOPIC_OR_PARTITION(org.apache.kafka.common.protocol.Errors.UNKNOWN_TOPIC_OR_PARTITION) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) InvalidReplicaAssignmentException(org.apache.kafka.common.errors.InvalidReplicaAssignmentException) NO_OP_EXISTENCE_CHECKER(org.apache.kafka.controller.ConfigurationControlManager.NO_OP_EXISTENCE_CHECKER) AlterPartitionReassignmentsResponseData(org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData) CreateTopicPolicy(org.apache.kafka.server.policy.CreateTopicPolicy) AlterIsrRequestData(org.apache.kafka.common.message.AlterIsrRequestData) InvalidRequestException(org.apache.kafka.common.errors.InvalidRequestException) Collections(java.util.Collections) ApiException(org.apache.kafka.common.errors.ApiException) TimelineHashMap(org.apache.kafka.timeline.TimelineHashMap) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) CreateTopicsResponseData(org.apache.kafka.common.message.CreateTopicsResponseData) ConfigResource(org.apache.kafka.common.config.ConfigResource) CreatableTopic(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic) ApiMessageAndVersion(org.apache.kafka.server.common.ApiMessageAndVersion) CreatableTopicResult(org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicResult) OpType(org.apache.kafka.clients.admin.AlterConfigOp.OpType) ApiError(org.apache.kafka.common.requests.ApiError) Map(java.util.Map) TimelineHashMap(org.apache.kafka.timeline.TimelineHashMap) HashMap(java.util.HashMap) ApiException(org.apache.kafka.common.errors.ApiException)

Aggregations

ArrayList (java.util.ArrayList)2 Entry (java.util.Map.Entry)2 OpType (org.apache.kafka.clients.admin.AlterConfigOp.OpType)2 SimpleImmutableEntry (java.util.AbstractMap.SimpleImmutableEntry)1 Collection (java.util.Collection)1 Collections (java.util.Collections)1 HashMap (java.util.HashMap)1 Iterator (java.util.Iterator)1 List (java.util.List)1 ListIterator (java.util.ListIterator)1 Map (java.util.Map)1 NoSuchElementException (java.util.NoSuchElementException)1 Optional (java.util.Optional)1 OptionalInt (java.util.OptionalInt)1 Function (java.util.function.Function)1 Supplier (java.util.function.Supplier)1 Collectors (java.util.stream.Collectors)1 SET (org.apache.kafka.clients.admin.AlterConfigOp.OpType.SET)1 ElectionType (org.apache.kafka.common.ElectionType)1 Uuid (org.apache.kafka.common.Uuid)1