Examples with AppendRecordsContext io.streamnative.pulsar.handlers.kop.storage.AppendRecordsContext used on opensource projects

Example 1 with AppendRecordsContext

use of io.streamnative.pulsar.handlers.kop.storage.AppendRecordsContext in project starlight-for-kafka by datastax.

the class KafkaRequestHandler method handleWriteTxnMarkers.

@Override
protected void handleWriteTxnMarkers(KafkaHeaderAndRequest kafkaHeaderAndRequest, CompletableFuture<AbstractResponse> response) {
    WriteTxnMarkersRequest request = (WriteTxnMarkersRequest) kafkaHeaderAndRequest.getRequest();
    Map<Long, Map<TopicPartition, Errors>> errors = new ConcurrentHashMap<>();
    List<WriteTxnMarkersRequest.TxnMarkerEntry> markers = request.markers();
    AtomicInteger numAppends = new AtomicInteger(markers.size());
    if (numAppends.get() == 0) {
        response.complete(new WriteTxnMarkersResponse(errors));
        return;
    }
    BiConsumer<Long, Map<TopicPartition, Errors>> updateErrors = (producerId, currentErrors) -> {
        Map<TopicPartition, Errors> previousErrors = errors.putIfAbsent(producerId, currentErrors);
        if (previousErrors != null) {
            previousErrors.putAll(currentErrors);
        }
    };
    Runnable completeOne = () -> {
        if (numAppends.decrementAndGet() == 0) {
            response.complete(new WriteTxnMarkersResponse(errors));
        }
    };
    for (WriteTxnMarkersRequest.TxnMarkerEntry marker : markers) {
        long producerId = marker.producerId();
        TransactionResult transactionResult = marker.transactionResult();
        Map<TopicPartition, MemoryRecords> controlRecords = generateTxnMarkerRecords(marker);
        AppendRecordsContext appendRecordsContext = AppendRecordsContext.get(topicManager, requestStats, this::startSendOperationForThrottling, this::completeSendOperationForThrottling, this.pendingTopicFuturesMap);
        getReplicaManager().appendRecords(kafkaConfig.getRequestTimeoutMs(), true, currentNamespacePrefix(), controlRecords, PartitionLog.AppendOrigin.Coordinator, appendRecordsContext).whenComplete((result, ex) -> {
            appendRecordsContext.recycle();
            if (ex != null) {
                log.error("[{}] Append txn marker ({}) failed.", ctx.channel(), marker, ex);
                Map<TopicPartition, Errors> currentErrors = new HashMap<>();
                controlRecords.forEach(((topicPartition, partitionResponse) -> {
                    currentErrors.put(topicPartition, Errors.KAFKA_STORAGE_ERROR);
                }));
                updateErrors.accept(producerId, currentErrors);
                completeOne.run();
                return;
            }
            Map<TopicPartition, Errors> currentErrors = new HashMap<>();
            result.forEach(((topicPartition, partitionResponse) -> {
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Append txn marker to topic : [{}], response: [{}].", ctx.channel(), topicPartition, partitionResponse);
                }
                currentErrors.put(topicPartition, partitionResponse.error);
            }));
            updateErrors.accept(producerId, currentErrors);
            final String metadataNamespace = kafkaConfig.getKafkaMetadataNamespace();
            Set<TopicPartition> successfulOffsetsPartitions = result.keySet().stream().filter(topicPartition -> KopTopic.isGroupMetadataTopicName(topicPartition.topic(), metadataNamespace)).collect(Collectors.toSet());
            if (!successfulOffsetsPartitions.isEmpty()) {
                getGroupCoordinator().scheduleHandleTxnCompletion(producerId, successfulOffsetsPartitions.stream().map(TopicPartition::partition).collect(Collectors.toSet()), transactionResult).whenComplete((__, e) -> {
                    if (e != null) {
                        log.error("Received an exception while trying to update the offsets cache on " + "transaction marker append", e);
                        ConcurrentHashMap<TopicPartition, Errors> updatedErrors = new ConcurrentHashMap<>();
                        successfulOffsetsPartitions.forEach(partition -> updatedErrors.put(partition, Errors.forException(e.getCause())));
                        updateErrors.accept(producerId, updatedErrors);
                    }
                    completeOne.run();
                });
                return;
            }
            completeOne.run();
        });
    }
}

Example 2 with AppendRecordsContext

use of io.streamnative.pulsar.handlers.kop.storage.AppendRecordsContext in project kop by streamnative.

the class KafkaRequestHandler method handleProduceRequest.

@Override
protected void handleProduceRequest(KafkaHeaderAndRequest produceHar, CompletableFuture<AbstractResponse> resultFuture) {
    checkArgument(produceHar.getRequest() instanceof ProduceRequest);
    ProduceRequest produceRequest = (ProduceRequest) produceHar.getRequest();
    final int numPartitions = produceRequest.partitionRecordsOrFail().size();
    if (numPartitions == 0) {
        resultFuture.complete(new ProduceResponse(Collections.emptyMap()));
        return;
    }
    final Map<TopicPartition, PartitionResponse> unauthorizedTopicResponsesMap = new ConcurrentHashMap<>();
    final Map<TopicPartition, PartitionResponse> invalidRequestResponses = new HashMap<>();
    final Map<TopicPartition, MemoryRecords> authorizedRequestInfo = new ConcurrentHashMap<>();
    int timeoutMs = produceRequest.timeout();
    String namespacePrefix = currentNamespacePrefix();
    final AtomicInteger unfinishedAuthorizationCount = new AtomicInteger(numPartitions);
    Runnable completeOne = () -> {
        // When complete one authorization or failed, will do the action first.
        if (unfinishedAuthorizationCount.decrementAndGet() == 0) {
            if (authorizedRequestInfo.isEmpty()) {
                resultFuture.complete(new ProduceResponse(unauthorizedTopicResponsesMap));
                return;
            }
            AppendRecordsContext appendRecordsContext = AppendRecordsContext.get(topicManager, requestStats, this::startSendOperationForThrottling, this::completeSendOperationForThrottling, pendingTopicFuturesMap);
            getReplicaManager().appendRecords(timeoutMs, false, namespacePrefix, authorizedRequestInfo, PartitionLog.AppendOrigin.Client, appendRecordsContext).whenComplete((response, ex) -> {
                appendRecordsContext.recycle();
                if (ex != null) {
                    resultFuture.completeExceptionally(ex.getCause());
                    return;
                }
                Map<TopicPartition, PartitionResponse> mergedResponse = new HashMap<>();
                mergedResponse.putAll(response);
                mergedResponse.putAll(unauthorizedTopicResponsesMap);
                mergedResponse.putAll(invalidRequestResponses);
                resultFuture.complete(new ProduceResponse(mergedResponse));
            });
        }
    };
    produceRequest.partitionRecordsOrFail().forEach((topicPartition, records) -> {
        try {
            validateRecords(produceHar.getRequest().version(), records);
        } catch (ApiException ex) {
            invalidRequestResponses.put(topicPartition, new ProduceResponse.PartitionResponse(Errors.forException(ex)));
            completeOne.run();
            return;
        }
        final String fullPartitionName = KopTopic.toString(topicPartition, namespacePrefix);
        authorize(AclOperation.WRITE, Resource.of(ResourceType.TOPIC, fullPartitionName)).whenComplete((isAuthorized, ex) -> {
            if (ex != null) {
                log.error("Write topic authorize failed, topic - {}. {}", fullPartitionName, ex.getMessage());
                unauthorizedTopicResponsesMap.put(topicPartition, new ProduceResponse.PartitionResponse(Errors.TOPIC_AUTHORIZATION_FAILED));
                completeOne.run();
                return;
            }
            if (!isAuthorized) {
                unauthorizedTopicResponsesMap.put(topicPartition, new ProduceResponse.PartitionResponse(Errors.TOPIC_AUTHORIZATION_FAILED));
                completeOne.run();
                return;
            }
            authorizedRequestInfo.put(topicPartition, records);
            completeOne.run();
        });
    });
}

Example 3 with AppendRecordsContext

use of io.streamnative.pulsar.handlers.kop.storage.AppendRecordsContext in project starlight-for-kafka by datastax.

the class KafkaRequestHandler method handleProduceRequest.

@Override
protected void handleProduceRequest(KafkaHeaderAndRequest produceHar, CompletableFuture<AbstractResponse> resultFuture) {
    checkArgument(produceHar.getRequest() instanceof ProduceRequest);
    ProduceRequest produceRequest = (ProduceRequest) produceHar.getRequest();
    final int numPartitions = produceRequest.partitionRecordsOrFail().size();
    if (numPartitions == 0) {
        resultFuture.complete(new ProduceResponse(Collections.emptyMap()));
        return;
    }
    final Map<TopicPartition, PartitionResponse> unauthorizedTopicResponsesMap = new ConcurrentHashMap<>();
    final Map<TopicPartition, PartitionResponse> invalidRequestResponses = new HashMap<>();
    final Map<TopicPartition, MemoryRecords> authorizedRequestInfo = new ConcurrentHashMap<>();
    int timeoutMs = produceRequest.timeout();
    String namespacePrefix = currentNamespacePrefix();
    final AtomicInteger unfinishedAuthorizationCount = new AtomicInteger(numPartitions);
    Runnable completeOne = () -> {
        // When complete one authorization or failed, will do the action first.
        if (unfinishedAuthorizationCount.decrementAndGet() == 0) {
            if (authorizedRequestInfo.isEmpty()) {
                resultFuture.complete(new ProduceResponse(unauthorizedTopicResponsesMap));
                return;
            }
            AppendRecordsContext appendRecordsContext = AppendRecordsContext.get(topicManager, requestStats, this::startSendOperationForThrottling, this::completeSendOperationForThrottling, pendingTopicFuturesMap);
            getReplicaManager().appendRecords(timeoutMs, false, namespacePrefix, authorizedRequestInfo, PartitionLog.AppendOrigin.Client, appendRecordsContext).whenComplete((response, ex) -> {
                appendRecordsContext.recycle();
                if (ex != null) {
                    resultFuture.completeExceptionally(ex.getCause());
                    return;
                }
                Map<TopicPartition, PartitionResponse> mergedResponse = new HashMap<>();
                mergedResponse.putAll(response);
                mergedResponse.putAll(unauthorizedTopicResponsesMap);
                mergedResponse.putAll(invalidRequestResponses);
                resultFuture.complete(new ProduceResponse(mergedResponse));
            });
        }
    };
    produceRequest.partitionRecordsOrFail().forEach((topicPartition, records) -> {
        try {
            validateRecords(produceHar.getRequest().version(), records);
        } catch (ApiException ex) {
            invalidRequestResponses.put(topicPartition, new ProduceResponse.PartitionResponse(Errors.forException(ex)));
            completeOne.run();
            return;
        }
        final String fullPartitionName = KopTopic.toString(topicPartition, namespacePrefix);
        authorize(AclOperation.WRITE, Resource.of(ResourceType.TOPIC, fullPartitionName)).whenComplete((isAuthorized, ex) -> {
            if (ex != null) {
                log.error("Write topic authorize failed, topic - {}. {}", fullPartitionName, ex.getMessage());
                unauthorizedTopicResponsesMap.put(topicPartition, new ProduceResponse.PartitionResponse(Errors.TOPIC_AUTHORIZATION_FAILED));
                completeOne.run();
                return;
            }
            if (!isAuthorized) {
                unauthorizedTopicResponsesMap.put(topicPartition, new ProduceResponse.PartitionResponse(Errors.TOPIC_AUTHORIZATION_FAILED));
                completeOne.run();
                return;
            }
            authorizedRequestInfo.put(topicPartition, records);
            completeOne.run();
        });
    });
}

Example 4 with AppendRecordsContext

use of io.streamnative.pulsar.handlers.kop.storage.AppendRecordsContext in project kop by streamnative.

the class KafkaRequestHandler method handleWriteTxnMarkers.

@Override
protected void handleWriteTxnMarkers(KafkaHeaderAndRequest kafkaHeaderAndRequest, CompletableFuture<AbstractResponse> response) {
    WriteTxnMarkersRequest request = (WriteTxnMarkersRequest) kafkaHeaderAndRequest.getRequest();
    Map<Long, Map<TopicPartition, Errors>> errors = new ConcurrentHashMap<>();
    List<WriteTxnMarkersRequest.TxnMarkerEntry> markers = request.markers();
    AtomicInteger numAppends = new AtomicInteger(markers.size());
    if (numAppends.get() == 0) {
        response.complete(new WriteTxnMarkersResponse(errors));
        return;
    }
    BiConsumer<Long, Map<TopicPartition, Errors>> updateErrors = (producerId, currentErrors) -> {
        Map<TopicPartition, Errors> previousErrors = errors.putIfAbsent(producerId, currentErrors);
        if (previousErrors != null) {
            previousErrors.putAll(currentErrors);
        }
    };
    Runnable completeOne = () -> {
        if (numAppends.decrementAndGet() == 0) {
            response.complete(new WriteTxnMarkersResponse(errors));
        }
    };
    for (WriteTxnMarkersRequest.TxnMarkerEntry marker : markers) {
        long producerId = marker.producerId();
        TransactionResult transactionResult = marker.transactionResult();
        Map<TopicPartition, MemoryRecords> controlRecords = generateTxnMarkerRecords(marker);
        AppendRecordsContext appendRecordsContext = AppendRecordsContext.get(topicManager, requestStats, this::startSendOperationForThrottling, this::completeSendOperationForThrottling, this.pendingTopicFuturesMap);
        getReplicaManager().appendRecords(kafkaConfig.getRequestTimeoutMs(), true, currentNamespacePrefix(), controlRecords, PartitionLog.AppendOrigin.Coordinator, appendRecordsContext).whenComplete((result, ex) -> {
            appendRecordsContext.recycle();
            if (ex != null) {
                log.error("[{}] Append txn marker ({}) failed.", ctx.channel(), marker, ex);
                Map<TopicPartition, Errors> currentErrors = new HashMap<>();
                controlRecords.forEach(((topicPartition, partitionResponse) -> {
                    currentErrors.put(topicPartition, Errors.KAFKA_STORAGE_ERROR);
                }));
                updateErrors.accept(producerId, currentErrors);
                completeOne.run();
                return;
            }
            Map<TopicPartition, Errors> currentErrors = new HashMap<>();
            result.forEach(((topicPartition, partitionResponse) -> {
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Append txn marker to topic : [{}], response: [{}].", ctx.channel(), topicPartition, partitionResponse);
                }
                currentErrors.put(topicPartition, partitionResponse.error);
            }));
            updateErrors.accept(producerId, currentErrors);
            final String metadataNamespace = kafkaConfig.getKafkaMetadataNamespace();
            Set<TopicPartition> successfulOffsetsPartitions = result.keySet().stream().filter(topicPartition -> KopTopic.isGroupMetadataTopicName(topicPartition.topic(), metadataNamespace)).collect(Collectors.toSet());
            if (!successfulOffsetsPartitions.isEmpty()) {
                getGroupCoordinator().scheduleHandleTxnCompletion(producerId, successfulOffsetsPartitions.stream().map(TopicPartition::partition).collect(Collectors.toSet()), transactionResult).whenComplete((__, e) -> {
                    if (e != null) {
                        log.error("Received an exception while trying to update the offsets cache on " + "transaction marker append", e);
                        ConcurrentHashMap<TopicPartition, Errors> updatedErrors = new ConcurrentHashMap<>();
                        successfulOffsetsPartitions.forEach(partition -> updatedErrors.put(partition, Errors.forException(e.getCause())));
                        updateErrors.accept(producerId, updatedErrors);
                    }
                    completeOne.run();
                });
                return;
            }
            completeOne.run();
        });
    }
}