Examples with FetchSessionHandler org.apache.kafka.clients.FetchSessionHandler used on opensource projects

Example 1 with FetchSessionHandler

use of org.apache.kafka.clients.FetchSessionHandler in project apache-kafka-on-k8s by banzaicloud.

the class Fetcher method prepareFetchRequests.

/**
 * Create fetch requests for all nodes for which we have assigned partitions
 * that have no existing requests in flight.
 */
private Map<Node, FetchSessionHandler.FetchRequestData> prepareFetchRequests() {
    Cluster cluster = metadata.fetch();
    Map<Node, FetchSessionHandler.Builder> fetchable = new LinkedHashMap<>();
    for (TopicPartition partition : fetchablePartitions()) {
        Node node = cluster.leaderFor(partition);
        if (node == null) {
            metadata.requestUpdate();
        } else if (client.isUnavailable(node)) {
            client.maybeThrowAuthFailure(node);
            // If we try to send during the reconnect blackout window, then the request is just
            // going to be failed anyway before being sent, so skip the send for now
            log.trace("Skipping fetch for partition {} because node {} is awaiting reconnect backoff", partition, node);
        } else if (client.hasPendingRequests(node)) {
            log.trace("Skipping fetch for partition {} because there is an in-flight request to {}", partition, node);
        } else {
            // if there is a leader and no in-flight requests, issue a new fetch
            FetchSessionHandler.Builder builder = fetchable.get(node);
            if (builder == null) {
                FetchSessionHandler handler = sessionHandlers.get(node.id());
                if (handler == null) {
                    handler = new FetchSessionHandler(logContext, node.id());
                    sessionHandlers.put(node.id(), handler);
                }
                builder = handler.newBuilder();
                fetchable.put(node, builder);
            }
            long position = this.subscriptions.position(partition);
            builder.add(partition, new FetchRequest.PartitionData(position, FetchRequest.INVALID_LOG_START_OFFSET, this.fetchSize));
            log.debug("Added {} fetch request for partition {} at offset {} to node {}", isolationLevel, partition, position, node);
        }
    }
    Map<Node, FetchSessionHandler.FetchRequestData> reqs = new LinkedHashMap<>();
    for (Map.Entry<Node, FetchSessionHandler.Builder> entry : fetchable.entrySet()) {
        reqs.put(entry.getKey(), entry.getValue().build());
    }
    return reqs;
}

Example 2 with FetchSessionHandler

use of org.apache.kafka.clients.FetchSessionHandler in project apache-kafka-on-k8s by banzaicloud.

the class Fetcher method sendFetches.

/**
 * Set-up a fetch request for any node that we have assigned partitions for which doesn't already have
 * an in-flight fetch or pending fetch data.
 * @return number of fetches sent
 */
public int sendFetches() {
    Map<Node, FetchSessionHandler.FetchRequestData> fetchRequestMap = prepareFetchRequests();
    for (Map.Entry<Node, FetchSessionHandler.FetchRequestData> entry : fetchRequestMap.entrySet()) {
        final Node fetchTarget = entry.getKey();
        final FetchSessionHandler.FetchRequestData data = entry.getValue();
        final FetchRequest.Builder request = FetchRequest.Builder.forConsumer(this.maxWaitMs, this.minBytes, data.toSend()).isolationLevel(isolationLevel).setMaxBytes(this.maxBytes).metadata(data.metadata()).toForget(data.toForget());
        if (log.isDebugEnabled()) {
            log.debug("Sending {} {} to broker {}", isolationLevel, data.toString(), fetchTarget);
        }
        client.send(fetchTarget, request).addListener(new RequestFutureListener<ClientResponse>() {

            @Override
            public void onSuccess(ClientResponse resp) {
                FetchResponse response = (FetchResponse) resp.responseBody();
                FetchSessionHandler handler = sessionHandlers.get(fetchTarget.id());
                if (handler == null) {
                    log.error("Unable to find FetchSessionHandler for node {}. Ignoring fetch response.", fetchTarget.id());
                    return;
                }
                if (!handler.handleResponse(response)) {
                    return;
                }
                Set<TopicPartition> partitions = new HashSet<>(response.responseData().keySet());
                FetchResponseMetricAggregator metricAggregator = new FetchResponseMetricAggregator(sensors, partitions);
                for (Map.Entry<TopicPartition, FetchResponse.PartitionData> entry : response.responseData().entrySet()) {
                    TopicPartition partition = entry.getKey();
                    long fetchOffset = data.sessionPartitions().get(partition).fetchOffset;
                    FetchResponse.PartitionData fetchData = entry.getValue();
                    log.debug("Fetch {} at offset {} for partition {} returned fetch data {}", isolationLevel, fetchOffset, partition, fetchData);
                    completedFetches.add(new CompletedFetch(partition, fetchOffset, fetchData, metricAggregator, resp.requestHeader().apiVersion()));
                }
                sensors.fetchLatency.record(resp.requestLatencyMs());
            }

            @Override
            public void onFailure(RuntimeException e) {
                FetchSessionHandler handler = sessionHandlers.get(fetchTarget.id());
                if (handler != null) {
                    handler.handleError(e);
                }
            }
        });
    }
    return fetchRequestMap.size();
}

Example 3 with FetchSessionHandler

use of org.apache.kafka.clients.FetchSessionHandler in project kafka by apache.

the class FetchSessionBenchmark method setUp.

@Setup(Level.Trial)
public void setUp() {
    fetches = new LinkedHashMap<>();
    handler = new FetchSessionHandler(LOG_CONTEXT, 1);
    topicIds = new HashMap<>();
    FetchSessionHandler.Builder builder = handler.newBuilder();
    Uuid id = Uuid.randomUuid();
    topicIds.put("foo", id);
    LinkedHashMap<TopicIdPartition, FetchResponseData.PartitionData> respMap = new LinkedHashMap<>();
    for (int i = 0; i < partitionCount; i++) {
        TopicPartition tp = new TopicPartition("foo", i);
        FetchRequest.PartitionData partitionData = new FetchRequest.PartitionData(id, 0, 0, 200, Optional.empty());
        fetches.put(tp, partitionData);
        builder.add(tp, partitionData);
        respMap.put(new TopicIdPartition(id, tp), new FetchResponseData.PartitionData().setPartitionIndex(tp.partition()).setLastStableOffset(0).setLogStartOffset(0));
    }
    builder.build();
    // build and handle an initial response so that the next fetch will be incremental
    handler.handleResponse(FetchResponse.of(Errors.NONE, 0, 1, respMap), ApiKeys.FETCH.latestVersion());
    int counter = 0;
    for (TopicPartition topicPartition : new ArrayList<>(fetches.keySet())) {
        if (updatedPercentage != 0 && counter % (100 / updatedPercentage) == 0) {
            // reorder in fetch session, and update log start offset
            fetches.remove(topicPartition);
            fetches.put(topicPartition, new FetchRequest.PartitionData(id, 50, 40, 200, Optional.empty()));
        }
        counter++;
    }
}

Example 4 with FetchSessionHandler

use of org.apache.kafka.clients.FetchSessionHandler in project kafka by apache.

the class Fetcher method prepareFetchRequests.

/**
 * Create fetch requests for all nodes for which we have assigned partitions
 * that have no existing requests in flight.
 */
private Map<Node, FetchSessionHandler.FetchRequestData> prepareFetchRequests() {
    Map<Node, FetchSessionHandler.Builder> fetchable = new LinkedHashMap<>();
    validatePositionsOnMetadataChange();
    long currentTimeMs = time.milliseconds();
    Map<String, Uuid> topicIds = metadata.topicIds();
    for (TopicPartition partition : fetchablePartitions()) {
        FetchPosition position = this.subscriptions.position(partition);
        if (position == null) {
            throw new IllegalStateException("Missing position for fetchable partition " + partition);
        }
        Optional<Node> leaderOpt = position.currentLeader.leader;
        if (!leaderOpt.isPresent()) {
            log.debug("Requesting metadata update for partition {} since the position {} is missing the current leader node", partition, position);
            metadata.requestUpdate();
            continue;
        }
        // Use the preferred read replica if set, otherwise the position's leader
        Node node = selectReadReplica(partition, leaderOpt.get(), currentTimeMs);
        if (client.isUnavailable(node)) {
            client.maybeThrowAuthFailure(node);
            // If we try to send during the reconnect backoff window, then the request is just
            // going to be failed anyway before being sent, so skip the send for now
            log.trace("Skipping fetch for partition {} because node {} is awaiting reconnect backoff", partition, node);
        } else if (this.nodesWithPendingFetchRequests.contains(node.id())) {
            log.trace("Skipping fetch for partition {} because previous request to {} has not been processed", partition, node);
        } else {
            // if there is a leader and no in-flight requests, issue a new fetch
            FetchSessionHandler.Builder builder = fetchable.get(node);
            if (builder == null) {
                int id = node.id();
                FetchSessionHandler handler = sessionHandler(id);
                if (handler == null) {
                    handler = new FetchSessionHandler(logContext, id);
                    sessionHandlers.put(id, handler);
                }
                builder = handler.newBuilder();
                fetchable.put(node, builder);
            }
            builder.add(partition, new FetchRequest.PartitionData(topicIds.getOrDefault(partition.topic(), Uuid.ZERO_UUID), position.offset, FetchRequest.INVALID_LOG_START_OFFSET, this.fetchSize, position.currentLeader.epoch, Optional.empty()));
            log.debug("Added {} fetch request for partition {} at position {} to node {}", isolationLevel, partition, position, node);
        }
    }
    Map<Node, FetchSessionHandler.FetchRequestData> reqs = new LinkedHashMap<>();
    for (Map.Entry<Node, FetchSessionHandler.Builder> entry : fetchable.entrySet()) {
        reqs.put(entry.getKey(), entry.getValue().build());
    }
    return reqs;
}

Example 5 with FetchSessionHandler

use of org.apache.kafka.clients.FetchSessionHandler in project kafka by apache.

the class Fetcher method sendFetches.

/**
 * Set-up a fetch request for any node that we have assigned partitions for which doesn't already have
 * an in-flight fetch or pending fetch data.
 * @return number of fetches sent
 */
public synchronized int sendFetches() {
    // Update metrics in case there was an assignment change
    sensors.maybeUpdateAssignment(subscriptions);
    Map<Node, FetchSessionHandler.FetchRequestData> fetchRequestMap = prepareFetchRequests();
    for (Map.Entry<Node, FetchSessionHandler.FetchRequestData> entry : fetchRequestMap.entrySet()) {
        final Node fetchTarget = entry.getKey();
        final FetchSessionHandler.FetchRequestData data = entry.getValue();
        final short maxVersion;
        if (!data.canUseTopicIds()) {
            maxVersion = (short) 12;
        } else {
            maxVersion = ApiKeys.FETCH.latestVersion();
        }
        final FetchRequest.Builder request = FetchRequest.Builder.forConsumer(maxVersion, this.maxWaitMs, this.minBytes, data.toSend()).isolationLevel(isolationLevel).setMaxBytes(this.maxBytes).metadata(data.metadata()).removed(data.toForget()).replaced(data.toReplace()).rackId(clientRackId);
        if (log.isDebugEnabled()) {
            log.debug("Sending {} {} to broker {}", isolationLevel, data.toString(), fetchTarget);
        }
        RequestFuture<ClientResponse> future = client.send(fetchTarget, request);
        // We add the node to the set of nodes with pending fetch requests before adding the
        // listener because the future may have been fulfilled on another thread (e.g. during a
        // disconnection being handled by the heartbeat thread) which will mean the listener
        // will be invoked synchronously.
        this.nodesWithPendingFetchRequests.add(entry.getKey().id());
        future.addListener(new RequestFutureListener<ClientResponse>() {

            @Override
            public void onSuccess(ClientResponse resp) {
                synchronized (Fetcher.this) {
                    try {
                        FetchResponse response = (FetchResponse) resp.responseBody();
                        FetchSessionHandler handler = sessionHandler(fetchTarget.id());
                        if (handler == null) {
                            log.error("Unable to find FetchSessionHandler for node {}. Ignoring fetch response.", fetchTarget.id());
                            return;
                        }
                        if (!handler.handleResponse(response, resp.requestHeader().apiVersion())) {
                            if (response.error() == Errors.FETCH_SESSION_TOPIC_ID_ERROR) {
                                metadata.requestUpdate();
                            }
                            return;
                        }
                        Map<TopicPartition, FetchResponseData.PartitionData> responseData = response.responseData(handler.sessionTopicNames(), resp.requestHeader().apiVersion());
                        Set<TopicPartition> partitions = new HashSet<>(responseData.keySet());
                        FetchResponseMetricAggregator metricAggregator = new FetchResponseMetricAggregator(sensors, partitions);
                        for (Map.Entry<TopicPartition, FetchResponseData.PartitionData> entry : responseData.entrySet()) {
                            TopicPartition partition = entry.getKey();
                            FetchRequest.PartitionData requestData = data.sessionPartitions().get(partition);
                            if (requestData == null) {
                                String message;
                                if (data.metadata().isFull()) {
                                    message = MessageFormatter.arrayFormat("Response for missing full request partition: partition={}; metadata={}", new Object[] { partition, data.metadata() }).getMessage();
                                } else {
                                    message = MessageFormatter.arrayFormat("Response for missing session request partition: partition={}; metadata={}; toSend={}; toForget={}; toReplace={}", new Object[] { partition, data.metadata(), data.toSend(), data.toForget(), data.toReplace() }).getMessage();
                                }
                                // Received fetch response for missing session partition
                                throw new IllegalStateException(message);
                            } else {
                                long fetchOffset = requestData.fetchOffset;
                                FetchResponseData.PartitionData partitionData = entry.getValue();
                                log.debug("Fetch {} at offset {} for partition {} returned fetch data {}", isolationLevel, fetchOffset, partition, partitionData);
                                Iterator<? extends RecordBatch> batches = FetchResponse.recordsOrFail(partitionData).batches().iterator();
                                short responseVersion = resp.requestHeader().apiVersion();
                                completedFetches.add(new CompletedFetch(partition, partitionData, metricAggregator, batches, fetchOffset, responseVersion));
                            }
                        }
                        sensors.fetchLatency.record(resp.requestLatencyMs());
                    } finally {
                        nodesWithPendingFetchRequests.remove(fetchTarget.id());
                    }
                }
            }

            @Override
            public void onFailure(RuntimeException e) {
                synchronized (Fetcher.this) {
                    try {
                        FetchSessionHandler handler = sessionHandler(fetchTarget.id());
                        if (handler != null) {
                            handler.handleError(e);
                        }
                    } finally {
                        nodesWithPendingFetchRequests.remove(fetchTarget.id());
                    }
                }
            }
        });
    }
    return fetchRequestMap.size();
}