Examples with AbstractResponse org.apache.kafka.common.requests.AbstractResponse used on opensource projects

Example 36 with AbstractResponse

use of org.apache.kafka.common.requests.AbstractResponse in project kafka by apache.

the class KafkaAdminClient method deleteRecords.

@Override
public DeleteRecordsResult deleteRecords(final Map<TopicPartition, RecordsToDelete> recordsToDelete, final DeleteRecordsOptions options) {
    // requests need to be sent to partitions leader nodes so ...
    // ... from the provided map it's needed to create more maps grouping topic/partition per leader
    final Map<TopicPartition, KafkaFutureImpl<DeletedRecords>> futures = new HashMap<>(recordsToDelete.size());
    for (TopicPartition topicPartition : recordsToDelete.keySet()) {
        futures.put(topicPartition, new KafkaFutureImpl<>());
    }
    // preparing topics list for asking metadata about them
    final Set<String> topics = new HashSet<>();
    for (TopicPartition topicPartition : recordsToDelete.keySet()) {
        topics.add(topicPartition.topic());
    }
    final long nowMetadata = time.milliseconds();
    final long deadline = calcDeadlineMs(nowMetadata, options.timeoutMs());
    // asking for topics metadata for getting partitions leaders
    runnable.call(new Call("topicsMetadata", deadline, new LeastLoadedNodeProvider()) {

        @Override
        MetadataRequest.Builder createRequest(int timeoutMs) {
            return new MetadataRequest.Builder(new MetadataRequestData().setTopics(convertToMetadataRequestTopic(topics)).setAllowAutoTopicCreation(false));
        }

        @Override
        void handleResponse(AbstractResponse abstractResponse) {
            MetadataResponse response = (MetadataResponse) abstractResponse;
            Map<String, Errors> errors = response.errors();
            Cluster cluster = response.buildCluster();
            // Group topic partitions by leader
            Map<Node, Map<String, DeleteRecordsTopic>> leaders = new HashMap<>();
            for (Map.Entry<TopicPartition, RecordsToDelete> entry : recordsToDelete.entrySet()) {
                TopicPartition topicPartition = entry.getKey();
                KafkaFutureImpl<DeletedRecords> future = futures.get(topicPartition);
                // Fail partitions with topic errors
                Errors topicError = errors.get(topicPartition.topic());
                if (errors.containsKey(topicPartition.topic())) {
                    future.completeExceptionally(topicError.exception());
                } else {
                    Node node = cluster.leaderFor(topicPartition);
                    if (node != null) {
                        Map<String, DeleteRecordsTopic> deletionsForLeader = leaders.computeIfAbsent(node, key -> new HashMap<>());
                        DeleteRecordsTopic deleteRecords = deletionsForLeader.get(topicPartition.topic());
                        if (deleteRecords == null) {
                            deleteRecords = new DeleteRecordsTopic().setName(topicPartition.topic());
                            deletionsForLeader.put(topicPartition.topic(), deleteRecords);
                        }
                        deleteRecords.partitions().add(new DeleteRecordsPartition().setPartitionIndex(topicPartition.partition()).setOffset(entry.getValue().beforeOffset()));
                    } else {
                        future.completeExceptionally(Errors.LEADER_NOT_AVAILABLE.exception());
                    }
                }
            }
            final long deleteRecordsCallTimeMs = time.milliseconds();
            for (final Map.Entry<Node, Map<String, DeleteRecordsTopic>> entry : leaders.entrySet()) {
                final Map<String, DeleteRecordsTopic> partitionDeleteOffsets = entry.getValue();
                final int brokerId = entry.getKey().id();
                runnable.call(new Call("deleteRecords", deadline, new ConstantNodeIdProvider(brokerId)) {

                    @Override
                    DeleteRecordsRequest.Builder createRequest(int timeoutMs) {
                        return new DeleteRecordsRequest.Builder(new DeleteRecordsRequestData().setTimeoutMs(timeoutMs).setTopics(new ArrayList<>(partitionDeleteOffsets.values())));
                    }

                    @Override
                    void handleResponse(AbstractResponse abstractResponse) {
                        DeleteRecordsResponse response = (DeleteRecordsResponse) abstractResponse;
                        for (DeleteRecordsTopicResult topicResult : response.data().topics()) {
                            for (DeleteRecordsResponseData.DeleteRecordsPartitionResult partitionResult : topicResult.partitions()) {
                                KafkaFutureImpl<DeletedRecords> future = futures.get(new TopicPartition(topicResult.name(), partitionResult.partitionIndex()));
                                if (partitionResult.errorCode() == Errors.NONE.code()) {
                                    future.complete(new DeletedRecords(partitionResult.lowWatermark()));
                                } else {
                                    future.completeExceptionally(Errors.forCode(partitionResult.errorCode()).exception());
                                }
                            }
                        }
                    }

                    @Override
                    void handleFailure(Throwable throwable) {
                        Stream<KafkaFutureImpl<DeletedRecords>> callFutures = partitionDeleteOffsets.values().stream().flatMap(recordsToDelete -> recordsToDelete.partitions().stream().map(partitionsToDelete -> new TopicPartition(recordsToDelete.name(), partitionsToDelete.partitionIndex()))).map(futures::get);
                        completeAllExceptionally(callFutures, throwable);
                    }
                }, deleteRecordsCallTimeMs);
            }
        }

        @Override
        void handleFailure(Throwable throwable) {
            completeAllExceptionally(futures.values(), throwable);
        }
    }, nowMetadata);
    return new DeleteRecordsResult(new HashMap<>(futures));
}

Example 37 with AbstractResponse

use of org.apache.kafka.common.requests.AbstractResponse in project kafka by apache.

the class KafkaAdminClient method listConsumerGroups.

@Override
public ListConsumerGroupsResult listConsumerGroups(ListConsumerGroupsOptions options) {
    final KafkaFutureImpl<Collection<Object>> all = new KafkaFutureImpl<>();
    final long nowMetadata = time.milliseconds();
    final long deadline = calcDeadlineMs(nowMetadata, options.timeoutMs());
    runnable.call(new Call("findAllBrokers", deadline, new LeastLoadedNodeProvider()) {

        @Override
        MetadataRequest.Builder createRequest(int timeoutMs) {
            return new MetadataRequest.Builder(new MetadataRequestData().setTopics(Collections.emptyList()).setAllowAutoTopicCreation(true));
        }

        @Override
        void handleResponse(AbstractResponse abstractResponse) {
            MetadataResponse metadataResponse = (MetadataResponse) abstractResponse;
            Collection<Node> nodes = metadataResponse.brokers();
            if (nodes.isEmpty())
                throw new StaleMetadataException("Metadata fetch failed due to missing broker list");
            HashSet<Node> allNodes = new HashSet<>(nodes);
            final ListConsumerGroupsResults results = new ListConsumerGroupsResults(allNodes, all);
            for (final Node node : allNodes) {
                final long nowList = time.milliseconds();
                runnable.call(new Call("listConsumerGroups", deadline, new ConstantNodeIdProvider(node.id())) {

                    @Override
                    ListGroupsRequest.Builder createRequest(int timeoutMs) {
                        List<String> states = options.states().stream().map(s -> s.toString()).collect(Collectors.toList());
                        return new ListGroupsRequest.Builder(new ListGroupsRequestData().setStatesFilter(states));
                    }

                    private void maybeAddConsumerGroup(ListGroupsResponseData.ListedGroup group) {
                        String protocolType = group.protocolType();
                        if (protocolType.equals(ConsumerProtocol.PROTOCOL_TYPE) || protocolType.isEmpty()) {
                            final String groupId = group.groupId();
                            final Optional<ConsumerGroupState> state = group.groupState().equals("") ? Optional.empty() : Optional.of(ConsumerGroupState.parse(group.groupState()));
                            final ConsumerGroupListing groupListing = new ConsumerGroupListing(groupId, protocolType.isEmpty(), state);
                            results.addListing(groupListing);
                        }
                    }

                    @Override
                    void handleResponse(AbstractResponse abstractResponse) {
                        final ListGroupsResponse response = (ListGroupsResponse) abstractResponse;
                        synchronized (results) {
                            Errors error = Errors.forCode(response.data().errorCode());
                            if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS || error == Errors.COORDINATOR_NOT_AVAILABLE) {
                                throw error.exception();
                            } else if (error != Errors.NONE) {
                                results.addError(error.exception(), node);
                            } else {
                                for (ListGroupsResponseData.ListedGroup group : response.data().groups()) {
                                    maybeAddConsumerGroup(group);
                                }
                            }
                            results.tryComplete(node);
                        }
                    }

                    @Override
                    void handleFailure(Throwable throwable) {
                        synchronized (results) {
                            results.addError(throwable, node);
                            results.tryComplete(node);
                        }
                    }
                }, nowList);
            }
        }

        @Override
        void handleFailure(Throwable throwable) {
            KafkaException exception = new KafkaException("Failed to find brokers to send ListGroups", throwable);
            all.complete(Collections.singletonList(exception));
        }
    }, nowMetadata);
    return new ListConsumerGroupsResult(all);
}

Example 38 with AbstractResponse

use of org.apache.kafka.common.requests.AbstractResponse in project kafka by apache.

the class KafkaAdminClient method describeAcls.

@Override
public DescribeAclsResult describeAcls(final AclBindingFilter filter, DescribeAclsOptions options) {
    if (filter.isUnknown()) {
        KafkaFutureImpl<Collection<AclBinding>> future = new KafkaFutureImpl<>();
        future.completeExceptionally(new InvalidRequestException("The AclBindingFilter " + "must not contain UNKNOWN elements."));
        return new DescribeAclsResult(future);
    }
    final long now = time.milliseconds();
    final KafkaFutureImpl<Collection<AclBinding>> future = new KafkaFutureImpl<>();
    runnable.call(new Call("describeAcls", calcDeadlineMs(now, options.timeoutMs()), new LeastLoadedNodeProvider()) {

        @Override
        DescribeAclsRequest.Builder createRequest(int timeoutMs) {
            return new DescribeAclsRequest.Builder(filter);
        }

        @Override
        void handleResponse(AbstractResponse abstractResponse) {
            DescribeAclsResponse response = (DescribeAclsResponse) abstractResponse;
            if (response.error().isFailure()) {
                future.completeExceptionally(response.error().exception());
            } else {
                future.complete(DescribeAclsResponse.aclBindings(response.acls()));
            }
        }

        @Override
        void handleFailure(Throwable throwable) {
            future.completeExceptionally(throwable);
        }
    }, now);
    return new DescribeAclsResult(future);
}

Example 39 with AbstractResponse

use of org.apache.kafka.common.requests.AbstractResponse in project kafka by apache.

the class KafkaAdminClient method alterClientQuotas.

@Override
public AlterClientQuotasResult alterClientQuotas(Collection<ClientQuotaAlteration> entries, AlterClientQuotasOptions options) {
    Map<ClientQuotaEntity, KafkaFutureImpl<Void>> futures = new HashMap<>(entries.size());
    for (ClientQuotaAlteration entry : entries) {
        futures.put(entry.entity(), new KafkaFutureImpl<>());
    }
    final long now = time.milliseconds();
    runnable.call(new Call("alterClientQuotas", calcDeadlineMs(now, options.timeoutMs()), new LeastLoadedNodeProvider()) {

        @Override
        AlterClientQuotasRequest.Builder createRequest(int timeoutMs) {
            return new AlterClientQuotasRequest.Builder(entries, options.validateOnly());
        }

        @Override
        void handleResponse(AbstractResponse abstractResponse) {
            AlterClientQuotasResponse response = (AlterClientQuotasResponse) abstractResponse;
            response.complete(futures);
        }

        @Override
        void handleFailure(Throwable throwable) {
            completeAllExceptionally(futures.values(), throwable);
        }
    }, now);
    return new AlterClientQuotasResult(Collections.unmodifiableMap(futures));
}

Example 40 with AbstractResponse

use of org.apache.kafka.common.requests.AbstractResponse in project kafka by apache.

the class KafkaAdminClient method alterPartitionReassignments.

@Override
public AlterPartitionReassignmentsResult alterPartitionReassignments(Map<TopicPartition, Optional<NewPartitionReassignment>> reassignments, AlterPartitionReassignmentsOptions options) {
    final Map<TopicPartition, KafkaFutureImpl<Void>> futures = new HashMap<>();
    final Map<String, Map<Integer, Optional<NewPartitionReassignment>>> topicsToReassignments = new TreeMap<>();
    for (Map.Entry<TopicPartition, Optional<NewPartitionReassignment>> entry : reassignments.entrySet()) {
        String topic = entry.getKey().topic();
        int partition = entry.getKey().partition();
        TopicPartition topicPartition = new TopicPartition(topic, partition);
        Optional<NewPartitionReassignment> reassignment = entry.getValue();
        KafkaFutureImpl<Void> future = new KafkaFutureImpl<>();
        futures.put(topicPartition, future);
        if (topicNameIsUnrepresentable(topic)) {
            future.completeExceptionally(new InvalidTopicException("The given topic name '" + topic + "' cannot be represented in a request."));
        } else if (topicPartition.partition() < 0) {
            future.completeExceptionally(new InvalidTopicException("The given partition index " + topicPartition.partition() + " is not valid."));
        } else {
            Map<Integer, Optional<NewPartitionReassignment>> partitionReassignments = topicsToReassignments.get(topicPartition.topic());
            if (partitionReassignments == null) {
                partitionReassignments = new TreeMap<>();
                topicsToReassignments.put(topic, partitionReassignments);
            }
            partitionReassignments.put(partition, reassignment);
        }
    }
    final long now = time.milliseconds();
    Call call = new Call("alterPartitionReassignments", calcDeadlineMs(now, options.timeoutMs()), new ControllerNodeProvider()) {

        @Override
        public AlterPartitionReassignmentsRequest.Builder createRequest(int timeoutMs) {
            AlterPartitionReassignmentsRequestData data = new AlterPartitionReassignmentsRequestData();
            for (Map.Entry<String, Map<Integer, Optional<NewPartitionReassignment>>> entry : topicsToReassignments.entrySet()) {
                String topicName = entry.getKey();
                Map<Integer, Optional<NewPartitionReassignment>> partitionsToReassignments = entry.getValue();
                List<ReassignablePartition> reassignablePartitions = new ArrayList<>();
                for (Map.Entry<Integer, Optional<NewPartitionReassignment>> partitionEntry : partitionsToReassignments.entrySet()) {
                    int partitionIndex = partitionEntry.getKey();
                    Optional<NewPartitionReassignment> reassignment = partitionEntry.getValue();
                    ReassignablePartition reassignablePartition = new ReassignablePartition().setPartitionIndex(partitionIndex).setReplicas(reassignment.map(NewPartitionReassignment::targetReplicas).orElse(null));
                    reassignablePartitions.add(reassignablePartition);
                }
                ReassignableTopic reassignableTopic = new ReassignableTopic().setName(topicName).setPartitions(reassignablePartitions);
                data.topics().add(reassignableTopic);
            }
            data.setTimeoutMs(timeoutMs);
            return new AlterPartitionReassignmentsRequest.Builder(data);
        }

        @Override
        public void handleResponse(AbstractResponse abstractResponse) {
            AlterPartitionReassignmentsResponse response = (AlterPartitionReassignmentsResponse) abstractResponse;
            Map<TopicPartition, ApiException> errors = new HashMap<>();
            int receivedResponsesCount = 0;
            Errors topLevelError = Errors.forCode(response.data().errorCode());
            switch(topLevelError) {
                case NONE:
                    receivedResponsesCount += validateTopicResponses(response.data().responses(), errors);
                    break;
                case NOT_CONTROLLER:
                    handleNotControllerError(topLevelError);
                    break;
                default:
                    for (ReassignableTopicResponse topicResponse : response.data().responses()) {
                        String topicName = topicResponse.name();
                        for (ReassignablePartitionResponse partition : topicResponse.partitions()) {
                            errors.put(new TopicPartition(topicName, partition.partitionIndex()), new ApiError(topLevelError, response.data().errorMessage()).exception());
                            receivedResponsesCount += 1;
                        }
                    }
                    break;
            }
            assertResponseCountMatch(errors, receivedResponsesCount);
            for (Map.Entry<TopicPartition, ApiException> entry : errors.entrySet()) {
                ApiException exception = entry.getValue();
                if (exception == null)
                    futures.get(entry.getKey()).complete(null);
                else
                    futures.get(entry.getKey()).completeExceptionally(exception);
            }
        }

        private void assertResponseCountMatch(Map<TopicPartition, ApiException> errors, int receivedResponsesCount) {
            int expectedResponsesCount = topicsToReassignments.values().stream().mapToInt(Map::size).sum();
            if (errors.values().stream().noneMatch(Objects::nonNull) && receivedResponsesCount != expectedResponsesCount) {
                String quantifier = receivedResponsesCount > expectedResponsesCount ? "many" : "less";
                throw new UnknownServerException("The server returned too " + quantifier + " results." + "Expected " + expectedResponsesCount + " but received " + receivedResponsesCount);
            }
        }

        private int validateTopicResponses(List<ReassignableTopicResponse> topicResponses, Map<TopicPartition, ApiException> errors) {
            int receivedResponsesCount = 0;
            for (ReassignableTopicResponse topicResponse : topicResponses) {
                String topicName = topicResponse.name();
                for (ReassignablePartitionResponse partResponse : topicResponse.partitions()) {
                    Errors partitionError = Errors.forCode(partResponse.errorCode());
                    TopicPartition tp = new TopicPartition(topicName, partResponse.partitionIndex());
                    if (partitionError == Errors.NONE) {
                        errors.put(tp, null);
                    } else {
                        errors.put(tp, new ApiError(partitionError, partResponse.errorMessage()).exception());
                    }
                    receivedResponsesCount += 1;
                }
            }
            return receivedResponsesCount;
        }

        @Override
        void handleFailure(Throwable throwable) {
            for (KafkaFutureImpl<Void> future : futures.values()) {
                future.completeExceptionally(throwable);
            }
        }
    };
    if (!topicsToReassignments.isEmpty()) {
        runnable.call(call, now);
    }
    return new AlterPartitionReassignmentsResult(new HashMap<>(futures));
}