Examples with MetadataRequestData org.apache.kafka.common.message.MetadataRequestData used on opensource projects

Example 6 with MetadataRequestData

use of org.apache.kafka.common.message.MetadataRequestData in project kafka by apache.

the class KafkaAdminClient method listConsumerGroups.

@Override
public ListConsumerGroupsResult listConsumerGroups(ListConsumerGroupsOptions options) {
    final KafkaFutureImpl<Collection<Object>> all = new KafkaFutureImpl<>();
    final long nowMetadata = time.milliseconds();
    final long deadline = calcDeadlineMs(nowMetadata, options.timeoutMs());
    runnable.call(new Call("findAllBrokers", deadline, new LeastLoadedNodeProvider()) {

        @Override
        MetadataRequest.Builder createRequest(int timeoutMs) {
            return new MetadataRequest.Builder(new MetadataRequestData().setTopics(Collections.emptyList()).setAllowAutoTopicCreation(true));
        }

        @Override
        void handleResponse(AbstractResponse abstractResponse) {
            MetadataResponse metadataResponse = (MetadataResponse) abstractResponse;
            Collection<Node> nodes = metadataResponse.brokers();
            if (nodes.isEmpty())
                throw new StaleMetadataException("Metadata fetch failed due to missing broker list");
            HashSet<Node> allNodes = new HashSet<>(nodes);
            final ListConsumerGroupsResults results = new ListConsumerGroupsResults(allNodes, all);
            for (final Node node : allNodes) {
                final long nowList = time.milliseconds();
                runnable.call(new Call("listConsumerGroups", deadline, new ConstantNodeIdProvider(node.id())) {

                    @Override
                    ListGroupsRequest.Builder createRequest(int timeoutMs) {
                        List<String> states = options.states().stream().map(s -> s.toString()).collect(Collectors.toList());
                        return new ListGroupsRequest.Builder(new ListGroupsRequestData().setStatesFilter(states));
                    }

                    private void maybeAddConsumerGroup(ListGroupsResponseData.ListedGroup group) {
                        String protocolType = group.protocolType();
                        if (protocolType.equals(ConsumerProtocol.PROTOCOL_TYPE) || protocolType.isEmpty()) {
                            final String groupId = group.groupId();
                            final Optional<ConsumerGroupState> state = group.groupState().equals("") ? Optional.empty() : Optional.of(ConsumerGroupState.parse(group.groupState()));
                            final ConsumerGroupListing groupListing = new ConsumerGroupListing(groupId, protocolType.isEmpty(), state);
                            results.addListing(groupListing);
                        }
                    }

                    @Override
                    void handleResponse(AbstractResponse abstractResponse) {
                        final ListGroupsResponse response = (ListGroupsResponse) abstractResponse;
                        synchronized (results) {
                            Errors error = Errors.forCode(response.data().errorCode());
                            if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS || error == Errors.COORDINATOR_NOT_AVAILABLE) {
                                throw error.exception();
                            } else if (error != Errors.NONE) {
                                results.addError(error.exception(), node);
                            } else {
                                for (ListGroupsResponseData.ListedGroup group : response.data().groups()) {
                                    maybeAddConsumerGroup(group);
                                }
                            }
                            results.tryComplete(node);
                        }
                    }

                    @Override
                    void handleFailure(Throwable throwable) {
                        synchronized (results) {
                            results.addError(throwable, node);
                            results.tryComplete(node);
                        }
                    }
                }, nowList);
            }
        }

        @Override
        void handleFailure(Throwable throwable) {
            KafkaException exception = new KafkaException("Failed to find brokers to send ListGroups", throwable);
            all.complete(Collections.singletonList(exception));
        }
    }, nowMetadata);
    return new ListConsumerGroupsResult(all);
}

Example 7 with MetadataRequestData

use of org.apache.kafka.common.message.MetadataRequestData in project kafka by apache.

the class KafkaAdminClient method describeCluster.

@Override
public DescribeClusterResult describeCluster(DescribeClusterOptions options) {
    final KafkaFutureImpl<Collection<Node>> describeClusterFuture = new KafkaFutureImpl<>();
    final KafkaFutureImpl<Node> controllerFuture = new KafkaFutureImpl<>();
    final KafkaFutureImpl<String> clusterIdFuture = new KafkaFutureImpl<>();
    final KafkaFutureImpl<Set<AclOperation>> authorizedOperationsFuture = new KafkaFutureImpl<>();
    final long now = time.milliseconds();
    runnable.call(new Call("listNodes", calcDeadlineMs(now, options.timeoutMs()), new LeastLoadedNodeProvider()) {

        private boolean useMetadataRequest = false;

        @Override
        AbstractRequest.Builder createRequest(int timeoutMs) {
            if (!useMetadataRequest) {
                return new DescribeClusterRequest.Builder(new DescribeClusterRequestData().setIncludeClusterAuthorizedOperations(options.includeAuthorizedOperations()));
            } else {
                // simplifies communication with older brokers)
                return new MetadataRequest.Builder(new MetadataRequestData().setTopics(Collections.emptyList()).setAllowAutoTopicCreation(true).setIncludeClusterAuthorizedOperations(options.includeAuthorizedOperations()));
            }
        }

        @Override
        void handleResponse(AbstractResponse abstractResponse) {
            if (!useMetadataRequest) {
                DescribeClusterResponse response = (DescribeClusterResponse) abstractResponse;
                Errors error = Errors.forCode(response.data().errorCode());
                if (error != Errors.NONE) {
                    ApiError apiError = new ApiError(error, response.data().errorMessage());
                    handleFailure(apiError.exception());
                    return;
                }
                Map<Integer, Node> nodes = response.nodes();
                describeClusterFuture.complete(nodes.values());
                // Controller is null if controller id is equal to NO_CONTROLLER_ID
                controllerFuture.complete(nodes.get(response.data().controllerId()));
                clusterIdFuture.complete(response.data().clusterId());
                authorizedOperationsFuture.complete(validAclOperations(response.data().clusterAuthorizedOperations()));
            } else {
                MetadataResponse response = (MetadataResponse) abstractResponse;
                describeClusterFuture.complete(response.brokers());
                controllerFuture.complete(controller(response));
                clusterIdFuture.complete(response.clusterId());
                authorizedOperationsFuture.complete(validAclOperations(response.clusterAuthorizedOperations()));
            }
        }

        private Node controller(MetadataResponse response) {
            if (response.controller() == null || response.controller().id() == MetadataResponse.NO_CONTROLLER_ID)
                return null;
            return response.controller();
        }

        @Override
        void handleFailure(Throwable throwable) {
            describeClusterFuture.completeExceptionally(throwable);
            controllerFuture.completeExceptionally(throwable);
            clusterIdFuture.completeExceptionally(throwable);
            authorizedOperationsFuture.completeExceptionally(throwable);
        }

        @Override
        boolean handleUnsupportedVersionException(final UnsupportedVersionException exception) {
            if (useMetadataRequest) {
                return false;
            }
            useMetadataRequest = true;
            return true;
        }
    }, now);
    return new DescribeClusterResult(describeClusterFuture, controllerFuture, clusterIdFuture, authorizedOperationsFuture);
}

Example 8 with MetadataRequestData

use of org.apache.kafka.common.message.MetadataRequestData in project kafka by apache.

the class KafkaAdminClient method getMetadataCall.

/**
 * Returns a {@code Call} object to fetch the cluster metadata. Takes a List of Calls
 * parameter to schedule actions that need to be taken using the metadata. The param is a Supplier
 * so that it can be lazily created, so that it can use the results of the metadata call in its
 * construction.
 *
 * @param <T> The type of return value of the KafkaFuture, like ListOffsetsResultInfo, etc.
 * @param <O> The type of configuration option, like ListOffsetsOptions, etc
 */
private <T, O extends AbstractOptions<O>> Call getMetadataCall(MetadataOperationContext<T, O> context, Supplier<List<Call>> nextCalls) {
    return new Call("metadata", context.deadline(), new LeastLoadedNodeProvider()) {

        @Override
        MetadataRequest.Builder createRequest(int timeoutMs) {
            return new MetadataRequest.Builder(new MetadataRequestData().setTopics(convertToMetadataRequestTopic(context.topics())).setAllowAutoTopicCreation(false));
        }

        @Override
        void handleResponse(AbstractResponse abstractResponse) {
            MetadataResponse response = (MetadataResponse) abstractResponse;
            MetadataOperationContext.handleMetadataErrors(response);
            context.setResponse(Optional.of(response));
            for (Call call : nextCalls.get()) {
                runnable.call(call, time.milliseconds());
            }
        }

        @Override
        void handleFailure(Throwable throwable) {
            for (KafkaFutureImpl<T> future : context.futures().values()) {
                future.completeExceptionally(throwable);
            }
        }
    };
}

Example 9 with MetadataRequestData

use of org.apache.kafka.common.message.MetadataRequestData in project kafka by apache.

the class PartitionLeaderStrategy method buildRequest.

@Override
public MetadataRequest.Builder buildRequest(Set<TopicPartition> partitions) {
    MetadataRequestData request = new MetadataRequestData();
    request.setAllowAutoTopicCreation(false);
    partitions.stream().map(TopicPartition::topic).distinct().forEach(topic -> request.topics().add(new MetadataRequestData.MetadataRequestTopic().setName(topic)));
    return new MetadataRequest.Builder(request);
}