Examples with DecodeResult io.streamnative.pulsar.handlers.kop.format.DecodeResult used on opensource projects

Example 1 with DecodeResult

use of io.streamnative.pulsar.handlers.kop.format.DecodeResult in project starlight-for-kafka by datastax.

the class MessageFetchContext method handleEntries.

private void handleEntries(final List<Entry> entries, final TopicPartition topicPartition, final FetchRequest.PartitionData partitionData, final KafkaTopicConsumerManager tcm, final ManagedCursor cursor, final AtomicLong cursorOffset, final boolean readCommitted) {
    final long highWatermark = MessageMetadataUtils.getHighWatermark(cursor.getManagedLedger());
    // Add new offset back to TCM after entries are read successfully
    tcm.add(cursorOffset.get(), Pair.of(cursor, cursorOffset.get()));
    PartitionLog partitionLog = requestHandler.getReplicaManager().getPartitionLog(topicPartition, namespacePrefix);
    final long lso = (readCommitted ? partitionLog.firstUndecidedOffset().orElse(highWatermark) : highWatermark);
    List<Entry> committedEntries = entries;
    if (readCommitted) {
        committedEntries = getCommittedEntries(entries, lso);
        if (log.isDebugEnabled()) {
            log.debug("Request {}: read {} entries but only {} entries are committed", header, entries.size(), committedEntries.size());
        }
    } else {
        if (log.isDebugEnabled()) {
            log.debug("Request {}: read {} entries", header, entries.size());
        }
    }
    if (committedEntries.isEmpty()) {
        addErrorPartitionResponse(topicPartition, Errors.NONE);
        return;
    }
    // use compatible magic value by apiVersion
    short apiVersion = header.apiVersion();
    final byte magic;
    if (apiVersion <= 1) {
        magic = RecordBatch.MAGIC_VALUE_V0;
    } else if (apiVersion <= 3) {
        magic = RecordBatch.MAGIC_VALUE_V1;
    } else {
        magic = RecordBatch.CURRENT_MAGIC_VALUE;
    }
    CompletableFuture<String> groupNameFuture = requestHandler.getCurrentConnectedGroup().computeIfAbsent(clientHost, clientHost -> {
        CompletableFuture<String> future = new CompletableFuture<>();
        String groupIdPath = GroupIdUtils.groupIdPathFormat(clientHost, header.clientId());
        requestHandler.getMetadataStore().get(requestHandler.getGroupIdStoredPath() + groupIdPath).thenAccept(getResultOpt -> {
            if (getResultOpt.isPresent()) {
                GetResult getResult = getResultOpt.get();
                future.complete(new String(getResult.getValue() == null ? new byte[0] : getResult.getValue(), StandardCharsets.UTF_8));
            } else {
                future.complete("");
            }
        }).exceptionally(ex -> {
            future.completeExceptionally(ex);
            return null;
        });
        return future;
    });
    // this part is heavyweight, and we should not execute in the ManagedLedger Ordered executor thread
    groupNameFuture.whenCompleteAsync((groupName, ex) -> {
        if (ex != null) {
            log.error("Get groupId failed.", ex);
            groupName = "";
        }
        final long startDecodingEntriesNanos = MathUtils.nowInNano();
        final DecodeResult decodeResult = requestHandler.getEntryFormatter().decode(entries, magic);
        requestHandler.requestStats.getFetchDecodeStats().registerSuccessfulEvent(MathUtils.elapsedNanos(startDecodingEntriesNanos), TimeUnit.NANOSECONDS);
        decodeResults.add(decodeResult);
        final MemoryRecords kafkaRecords = decodeResult.getRecords();
        // collect consumer metrics
        decodeResult.updateConsumerStats(topicPartition, entries.size(), groupName, statsLogger);
        List<FetchResponse.AbortedTransaction> abortedTransactions;
        if (readCommitted) {
            abortedTransactions = partitionLog.getAbortedIndexList(partitionData.fetchOffset);
        } else {
            abortedTransactions = null;
        }
        responseData.put(topicPartition, new PartitionData<>(Errors.NONE, highWatermark, lso, // TODO: should it be changed to the logStartOffset?
        highWatermark, abortedTransactions, kafkaRecords));
        bytesReadable.getAndAdd(kafkaRecords.sizeInBytes());
        tryComplete();
    }, requestHandler.getDecodeExecutor());
}

Example 2 with DecodeResult

use of io.streamnative.pulsar.handlers.kop.format.DecodeResult in project starlight-for-kafka by datastax.

the class MessageFetchContext method complete.

public void complete() {
    if (resultFuture == null) {
        // the context has been recycled
        return;
    }
    if (resultFuture.isCancelled()) {
        // The request was cancelled by KafkaCommandDecoder when channel is closed or this request is expired,
        // so the Netty buffers should be released.
        decodeResults.forEach(DecodeResult::recycle);
        return;
    }
    if (resultFuture.isDone()) {
        // It may be triggered again in DelayedProduceAndFetch
        return;
    }
    // Keep the order of TopicPartition
    final LinkedHashMap<TopicPartition, PartitionData<MemoryRecords>> orderedResponseData = new LinkedHashMap<>();
    // add the topicPartition with timeout error if it's not existed in responseData
    fetchRequest.fetchData().keySet().forEach(topicPartition -> {
        final PartitionData<MemoryRecords> partitionData = responseData.remove(topicPartition);
        if (partitionData != null) {
            orderedResponseData.put(topicPartition, partitionData);
        } else {
            orderedResponseData.put(topicPartition, new FetchResponse.PartitionData<>(Errors.REQUEST_TIMED_OUT, FetchResponse.INVALID_HIGHWATERMARK, FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, null, MemoryRecords.EMPTY));
        }
    });
    // Create a copy of this.decodeResults so the lambda expression will capture the current state
    // because this.decodeResults will cleared after resultFuture is completed.
    final List<DecodeResult> decodeResults = new ArrayList<>(this.decodeResults);
    resultFuture.complete(new ResponseCallbackWrapper(new FetchResponse<>(Errors.NONE, orderedResponseData, ((Integer) THROTTLE_TIME_MS.defaultValue), fetchRequest.metadata().sessionId()), () -> {
        // release the batched ByteBuf if necessary
        decodeResults.forEach(DecodeResult::recycle);
    }));
    recycle();
}

Example 3 with DecodeResult

use of io.streamnative.pulsar.handlers.kop.format.DecodeResult in project kop by streamnative.

the class MessageFetchContext method handleEntries.

private void handleEntries(final List<Entry> entries, final TopicPartition topicPartition, final FetchRequest.PartitionData partitionData, final KafkaTopicConsumerManager tcm, final ManagedCursor cursor, final AtomicLong cursorOffset, final boolean readCommitted) {
    final long highWatermark = MessageMetadataUtils.getHighWatermark(cursor.getManagedLedger());
    // Add new offset back to TCM after entries are read successfully
    tcm.add(cursorOffset.get(), Pair.of(cursor, cursorOffset.get()));
    PartitionLog partitionLog = requestHandler.getReplicaManager().getPartitionLog(topicPartition, namespacePrefix);
    final long lso = (readCommitted ? partitionLog.firstUndecidedOffset().orElse(highWatermark) : highWatermark);
    List<Entry> committedEntries = entries;
    if (readCommitted) {
        committedEntries = getCommittedEntries(entries, lso);
        if (log.isDebugEnabled()) {
            log.debug("Request {}: read {} entries but only {} entries are committed", header, entries.size(), committedEntries.size());
        }
    } else {
        if (log.isDebugEnabled()) {
            log.debug("Request {}: read {} entries", header, entries.size());
        }
    }
    if (committedEntries.isEmpty()) {
        addErrorPartitionResponse(topicPartition, Errors.NONE);
        return;
    }
    // use compatible magic value by apiVersion
    short apiVersion = header.apiVersion();
    final byte magic;
    if (apiVersion <= 1) {
        magic = RecordBatch.MAGIC_VALUE_V0;
    } else if (apiVersion <= 3) {
        magic = RecordBatch.MAGIC_VALUE_V1;
    } else {
        magic = RecordBatch.CURRENT_MAGIC_VALUE;
    }
    CompletableFuture<String> groupNameFuture = requestHandler.getCurrentConnectedGroup().computeIfAbsent(clientHost, clientHost -> {
        CompletableFuture<String> future = new CompletableFuture<>();
        String groupIdPath = GroupIdUtils.groupIdPathFormat(clientHost, header.clientId());
        requestHandler.getMetadataStore().get(requestHandler.getGroupIdStoredPath() + groupIdPath).thenAccept(getResultOpt -> {
            if (getResultOpt.isPresent()) {
                GetResult getResult = getResultOpt.get();
                future.complete(new String(getResult.getValue() == null ? new byte[0] : getResult.getValue(), StandardCharsets.UTF_8));
            } else {
                future.complete("");
            }
        }).exceptionally(ex -> {
            future.completeExceptionally(ex);
            return null;
        });
        return future;
    });
    // this part is heavyweight, and we should not execute in the ManagedLedger Ordered executor thread
    groupNameFuture.whenCompleteAsync((groupName, ex) -> {
        if (ex != null) {
            log.error("Get groupId failed.", ex);
            groupName = "";
        }
        final long startDecodingEntriesNanos = MathUtils.nowInNano();
        final DecodeResult decodeResult = requestHandler.getEntryFormatter().decode(entries, magic);
        requestHandler.requestStats.getFetchDecodeStats().registerSuccessfulEvent(MathUtils.elapsedNanos(startDecodingEntriesNanos), TimeUnit.NANOSECONDS);
        decodeResults.add(decodeResult);
        final MemoryRecords kafkaRecords = decodeResult.getRecords();
        // collect consumer metrics
        decodeResult.updateConsumerStats(topicPartition, entries.size(), groupName, statsLogger);
        List<FetchResponse.AbortedTransaction> abortedTransactions;
        if (readCommitted) {
            abortedTransactions = partitionLog.getAbortedIndexList(partitionData.fetchOffset);
        } else {
            abortedTransactions = null;
        }
        responseData.put(topicPartition, new PartitionData<>(Errors.NONE, highWatermark, lso, // TODO: should it be changed to the logStartOffset?
        highWatermark, abortedTransactions, kafkaRecords));
        bytesReadable.getAndAdd(kafkaRecords.sizeInBytes());
        tryComplete();
    }, requestHandler.getDecodeExecutor());
}

Example 4 with DecodeResult

use of io.streamnative.pulsar.handlers.kop.format.DecodeResult in project kop by streamnative.

the class MessageFetchContext method complete.

public void complete() {
    if (resultFuture == null) {
        // the context has been recycled
        return;
    }
    if (resultFuture.isCancelled()) {
        // The request was cancelled by KafkaCommandDecoder when channel is closed or this request is expired,
        // so the Netty buffers should be released.
        decodeResults.forEach(DecodeResult::recycle);
        return;
    }
    if (resultFuture.isDone()) {
        // It may be triggered again in DelayedProduceAndFetch
        return;
    }
    // Keep the order of TopicPartition
    final LinkedHashMap<TopicPartition, PartitionData<MemoryRecords>> orderedResponseData = new LinkedHashMap<>();
    // add the topicPartition with timeout error if it's not existed in responseData
    fetchRequest.fetchData().keySet().forEach(topicPartition -> {
        final PartitionData<MemoryRecords> partitionData = responseData.remove(topicPartition);
        if (partitionData != null) {
            orderedResponseData.put(topicPartition, partitionData);
        } else {
            orderedResponseData.put(topicPartition, new FetchResponse.PartitionData<>(Errors.REQUEST_TIMED_OUT, FetchResponse.INVALID_HIGHWATERMARK, FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, null, MemoryRecords.EMPTY));
        }
    });
    // Create a copy of this.decodeResults so the lambda expression will capture the current state
    // because this.decodeResults will cleared after resultFuture is completed.
    final List<DecodeResult> decodeResults = new ArrayList<>(this.decodeResults);
    resultFuture.complete(new ResponseCallbackWrapper(new FetchResponse<>(Errors.NONE, orderedResponseData, ((Integer) THROTTLE_TIME_MS.defaultValue), fetchRequest.metadata().sessionId()), () -> {
        // release the batched ByteBuf if necessary
        decodeResults.forEach(DecodeResult::recycle);
    }));
    recycle();
}