Search in sources :

Example 1 with CommandPartitionedTopicMetadata

use of org.apache.pulsar.common.api.proto.CommandPartitionedTopicMetadata in project pulsar by apache.

the class LookupProxyHandler method handlePartitionMetadataResponse.

/**
 *   Always get partition metadata from broker service.
 */
private void handlePartitionMetadataResponse(CommandPartitionedTopicMetadata partitionMetadata, long clientRequestId) {
    TopicName topicName = TopicName.get(partitionMetadata.getTopic());
    URI brokerURI;
    try {
        String availableBrokerServiceURL = getBrokerServiceUrl(clientRequestId);
        if (availableBrokerServiceURL == null) {
            log.warn("No available broker for {} to lookup partition metadata", topicName);
            return;
        }
        brokerURI = new URI(availableBrokerServiceURL);
    } catch (URISyntaxException e) {
        proxyConnection.ctx().writeAndFlush(Commands.newPartitionMetadataResponse(ServerError.MetadataError, e.getMessage(), clientRequestId));
        return;
    }
    InetSocketAddress addr = new InetSocketAddress(brokerURI.getHost(), brokerURI.getPort());
    if (log.isDebugEnabled()) {
        log.debug("Getting connections to '{}' for Looking up topic '{}' with clientReq Id '{}'", addr, topicName.getPartitionedTopicName(), clientRequestId);
    }
    proxyConnection.getConnectionPool().getConnection(addr).thenAccept(clientCnx -> {
        // Connected to backend broker
        long requestId = proxyConnection.newRequestId();
        ByteBuf command;
        command = Commands.newPartitionMetadataRequest(topicName.toString(), requestId);
        clientCnx.newLookup(command, requestId).whenComplete((r, t) -> {
            if (t != null) {
                log.warn("[{}] failed to get Partitioned metadata : {}", topicName.toString(), t.getMessage(), t);
                proxyConnection.ctx().writeAndFlush(Commands.newLookupErrorResponse(getServerError(t), t.getMessage(), clientRequestId));
            } else {
                proxyConnection.ctx().writeAndFlush(Commands.newPartitionMetadataResponse(r.partitions, clientRequestId));
            }
            proxyConnection.getConnectionPool().releaseConnection(clientCnx);
        });
    }).exceptionally(ex -> {
        // Failed to connect to backend broker
        proxyConnection.ctx().writeAndFlush(Commands.newPartitionMetadataResponse(ServerError.ServiceNotReady, ex.getMessage(), clientRequestId));
        return null;
    });
}
Also used : CommandLookupTopic(org.apache.pulsar.common.api.proto.CommandLookupTopic) TopicName(org.apache.pulsar.common.naming.TopicName) Logger(org.slf4j.Logger) SocketAddress(java.net.SocketAddress) Semaphore(java.util.concurrent.Semaphore) URISyntaxException(java.net.URISyntaxException) CommandGetTopicsOfNamespace(org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace) ServerError(org.apache.pulsar.common.api.proto.ServerError) LoggerFactory(org.slf4j.LoggerFactory) SchemaVersion(org.apache.pulsar.common.protocol.schema.SchemaVersion) ServiceLookupData(org.apache.pulsar.policies.data.loadbalancer.ServiceLookupData) Counter(io.prometheus.client.Counter) InetSocketAddress(java.net.InetSocketAddress) StringUtils(org.apache.commons.lang3.StringUtils) Commands(org.apache.pulsar.common.protocol.Commands) ByteBuf(io.netty.buffer.ByteBuf) CommandGetSchema(org.apache.pulsar.common.api.proto.CommandGetSchema) CommandPartitionedTopicMetadata(org.apache.pulsar.common.api.proto.CommandPartitionedTopicMetadata) BytesSchemaVersion(org.apache.pulsar.common.protocol.schema.BytesSchemaVersion) Optional(java.util.Optional) URI(java.net.URI) LookupType(org.apache.pulsar.common.api.proto.CommandLookupTopicResponse.LookupType) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) InetSocketAddress(java.net.InetSocketAddress) URISyntaxException(java.net.URISyntaxException) ByteBuf(io.netty.buffer.ByteBuf) URI(java.net.URI) TopicName(org.apache.pulsar.common.naming.TopicName)

Example 2 with CommandPartitionedTopicMetadata

use of org.apache.pulsar.common.api.proto.CommandPartitionedTopicMetadata in project pulsar by apache.

the class ServerCnx method handlePartitionMetadataRequest.

@Override
protected void handlePartitionMetadataRequest(CommandPartitionedTopicMetadata partitionMetadata) {
    final long requestId = partitionMetadata.getRequestId();
    if (log.isDebugEnabled()) {
        log.debug("[{}] Received PartitionMetadataLookup from {} for {}", partitionMetadata.getTopic(), remoteAddress, requestId);
    }
    TopicName topicName = validateTopicName(partitionMetadata.getTopic(), requestId, partitionMetadata);
    if (topicName == null) {
        return;
    }
    final Semaphore lookupSemaphore = service.getLookupRequestSemaphore();
    if (lookupSemaphore.tryAcquire()) {
        if (invalidOriginalPrincipal(originalPrincipal)) {
            final String msg = "Valid Proxy Client role should be provided for getPartitionMetadataRequest ";
            log.warn("[{}] {} with role {} and proxyClientAuthRole {} on topic {}", remoteAddress, msg, authRole, originalPrincipal, topicName);
            commandSender.sendPartitionMetadataResponse(ServerError.AuthorizationError, msg, requestId);
            lookupSemaphore.release();
            return;
        }
        isTopicOperationAllowed(topicName, TopicOperation.LOOKUP).thenApply(isAuthorized -> {
            if (isAuthorized) {
                unsafeGetPartitionedTopicMetadataAsync(getBrokerService().pulsar(), topicName).handle((metadata, ex) -> {
                    if (ex == null) {
                        int partitions = metadata.partitions;
                        commandSender.sendPartitionMetadataResponse(partitions, requestId);
                    } else {
                        if (ex instanceof PulsarClientException) {
                            log.warn("Failed to authorize {} at [{}] on topic {} : {}", getRole(), remoteAddress, topicName, ex.getMessage());
                            commandSender.sendPartitionMetadataResponse(ServerError.AuthorizationError, ex.getMessage(), requestId);
                        } else {
                            log.warn("Failed to get Partitioned Metadata [{}] {}: {}", remoteAddress, topicName, ex.getMessage(), ex);
                            ServerError error = (ex instanceof RestException) && ((RestException) ex).getResponse().getStatus() < 500 ? ServerError.MetadataError : ServerError.ServiceNotReady;
                            commandSender.sendPartitionMetadataResponse(error, ex.getMessage(), requestId);
                        }
                    }
                    lookupSemaphore.release();
                    return null;
                });
            } else {
                final String msg = "Proxy Client is not authorized to Get Partition Metadata";
                log.warn("[{}] {} with role {} on topic {}", remoteAddress, msg, getPrincipal(), topicName);
                ctx.writeAndFlush(Commands.newPartitionMetadataResponse(ServerError.AuthorizationError, msg, requestId));
                lookupSemaphore.release();
            }
            return null;
        }).exceptionally(ex -> {
            logAuthException(remoteAddress, "partition-metadata", getPrincipal(), Optional.of(topicName), ex);
            final String msg = "Exception occurred while trying to authorize get Partition Metadata";
            ctx.writeAndFlush(Commands.newPartitionMetadataResponse(ServerError.AuthorizationError, msg, requestId));
            lookupSemaphore.release();
            return null;
        });
    } else {
        if (log.isDebugEnabled()) {
            log.debug("[{}] Failed Partition-Metadata lookup due to too many lookup-requests {}", remoteAddress, topicName);
        }
        commandSender.sendPartitionMetadataResponse(ServerError.TooManyRequests, "Failed due to too many pending lookup requests", requestId);
    }
}
Also used : CommandAuthResponse(org.apache.pulsar.common.api.proto.CommandAuthResponse) CommandUnsubscribe(org.apache.pulsar.common.api.proto.CommandUnsubscribe) ServiceUnitNotReadyException(org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException) CommandProducer(org.apache.pulsar.common.api.proto.CommandProducer) MessageIdData(org.apache.pulsar.common.api.proto.MessageIdData) ByteBufPair(org.apache.pulsar.common.protocol.ByteBufPair) StringUtils(org.apache.commons.lang3.StringUtils) ProtocolVersion(org.apache.pulsar.common.api.proto.ProtocolVersion) TxnID(org.apache.pulsar.client.api.transaction.TxnID) MLTransactionMetadataStore(org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStore) TopicOperation(org.apache.pulsar.common.policies.data.TopicOperation) MutableLong(org.apache.commons.lang3.mutable.MutableLong) Map(java.util.Map) BrokerInterceptor(org.apache.pulsar.broker.intercept.BrokerInterceptor) RestException(org.apache.pulsar.broker.web.RestException) NamespaceOperation(org.apache.pulsar.common.policies.data.NamespaceOperation) BaseCommand(org.apache.pulsar.common.api.proto.BaseCommand) CommandAck(org.apache.pulsar.common.api.proto.CommandAck) PositionImpl(org.apache.bookkeeper.mledger.impl.PositionImpl) InterceptException(org.apache.pulsar.common.intercept.InterceptException) CommandFlow(org.apache.pulsar.common.api.proto.CommandFlow) CommandConsumerStats(org.apache.pulsar.common.api.proto.CommandConsumerStats) ServerError(org.apache.pulsar.common.api.proto.ServerError) Set(java.util.Set) CommandNewTxn(org.apache.pulsar.common.api.proto.CommandNewTxn) StringUtils.isNotBlank(org.apache.commons.lang3.StringUtils.isNotBlank) BatchMessageIdImpl(org.apache.pulsar.client.impl.BatchMessageIdImpl) MessageMetadata(org.apache.pulsar.common.api.proto.MessageMetadata) SafeRun(org.apache.bookkeeper.mledger.util.SafeRun) SslHandler(io.netty.handler.ssl.SslHandler) AsyncCallbacks(org.apache.bookkeeper.mledger.AsyncCallbacks) ExceptionUtils(org.apache.commons.lang3.exception.ExceptionUtils) ClientCnx(org.apache.pulsar.client.impl.ClientCnx) ChannelOption(io.netty.channel.ChannelOption) SchemaRegistryService(org.apache.pulsar.broker.service.schema.SchemaRegistryService) AuthenticationState(org.apache.pulsar.broker.authentication.AuthenticationState) CommandGetTopicsOfNamespace(org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace) TopicNotFoundException(org.apache.pulsar.broker.service.BrokerServiceException.TopicNotFoundException) SchemaType(org.apache.pulsar.common.schema.SchemaType) Commands(org.apache.pulsar.common.protocol.Commands) CommandCloseProducer(org.apache.pulsar.common.api.proto.CommandCloseProducer) Strings(com.google.common.base.Strings) SubType(org.apache.pulsar.common.api.proto.CommandSubscribe.SubType) ServerMetadataException(org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException) SSLSession(javax.net.ssl.SSLSession) CommandGetOrCreateSchema(org.apache.pulsar.common.api.proto.CommandGetOrCreateSchema) ProtocolVersion.v5(org.apache.pulsar.common.api.proto.ProtocolVersion.v5) CommandGetSchema(org.apache.pulsar.common.api.proto.CommandGetSchema) SchemaInfoUtil(org.apache.pulsar.client.impl.schema.SchemaInfoUtil) CommandRedeliverUnacknowledgedMessages(org.apache.pulsar.common.api.proto.CommandRedeliverUnacknowledgedMessages) Metadata(org.apache.pulsar.common.naming.Metadata) Promise(io.netty.util.concurrent.Promise) AuthenticationProvider(org.apache.pulsar.broker.authentication.AuthenticationProvider) InitialPosition(org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition) lombok.val(lombok.val) PulsarService(org.apache.pulsar.broker.PulsarService) KeySharedMode(org.apache.pulsar.common.api.proto.KeySharedMode) CommandEndTxnOnSubscription(org.apache.pulsar.common.api.proto.CommandEndTxnOnSubscription) CommandConnect(org.apache.pulsar.common.api.proto.CommandConnect) SchemaData(org.apache.pulsar.common.protocol.schema.SchemaData) ProducerAccessMode(org.apache.pulsar.common.api.proto.ProducerAccessMode) CommandLookupTopic(org.apache.pulsar.common.api.proto.CommandLookupTopic) MutableInt(org.apache.commons.lang3.mutable.MutableInt) SocketAddress(java.net.SocketAddress) CommandAddPartitionToTxn(org.apache.pulsar.common.api.proto.CommandAddPartitionToTxn) LoggerFactory(org.slf4j.LoggerFactory) AuthData(org.apache.pulsar.common.api.AuthData) Exceptions(org.apache.pulsar.functions.utils.Exceptions) AuthenticationException(javax.naming.AuthenticationException) KeyValue(org.apache.pulsar.common.api.proto.KeyValue) PersistentTopicsBase.unsafeGetPartitionedTopicMetadataAsync(org.apache.pulsar.broker.admin.impl.PersistentTopicsBase.unsafeGetPartitionedTopicMetadataAsync) CommandEndTxnOnPartition(org.apache.pulsar.common.api.proto.CommandEndTxnOnPartition) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) BacklogQuotaType(org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType) Gauge(io.prometheus.client.Gauge) TxnAction(org.apache.pulsar.common.api.proto.TxnAction) NamespaceName(org.apache.pulsar.common.naming.NamespaceName) Schema(org.apache.pulsar.common.api.proto.Schema) TransactionMetadataStoreService(org.apache.pulsar.broker.TransactionMetadataStoreService) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) IdentityHashMap(java.util.IdentityHashMap) SubscriptionNotFoundException(org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionNotFoundException) Position(org.apache.bookkeeper.mledger.Position) InetSocketAddress(java.net.InetSocketAddress) Collectors(java.util.stream.Collectors) CommandEndTxn(org.apache.pulsar.common.api.proto.CommandEndTxn) AuthenticationDataSource(org.apache.pulsar.broker.authentication.AuthenticationDataSource) Objects(java.util.Objects) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) PersistentTopic(org.apache.pulsar.broker.service.persistent.PersistentTopic) FutureUtil(org.apache.pulsar.common.util.FutureUtil) PersistentSubscription(org.apache.pulsar.broker.service.persistent.PersistentSubscription) Optional(java.util.Optional) IncompatibleSchemaException(org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException) PulsarHandler(org.apache.pulsar.common.protocol.PulsarHandler) CommandTcClientConnectRequest(org.apache.pulsar.common.api.proto.CommandTcClientConnectRequest) TopicName(org.apache.pulsar.common.naming.TopicName) CommandAddSubscriptionToTxn(org.apache.pulsar.common.api.proto.CommandAddSubscriptionToTxn) Entry(org.apache.bookkeeper.mledger.Entry) CommandSend(org.apache.pulsar.common.api.proto.CommandSend) BacklogQuota(org.apache.pulsar.common.policies.data.BacklogQuota) CompletableFuture(java.util.concurrent.CompletableFuture) TopicLookupBase.lookupTopicAsync(org.apache.pulsar.broker.lookup.TopicLookupBase.lookupTopicAsync) KeySharedMeta(org.apache.pulsar.common.api.proto.KeySharedMeta) CommandSubscribe(org.apache.pulsar.common.api.proto.CommandSubscribe) ChannelHandlerContext(io.netty.channel.ChannelHandlerContext) ByteBuf(io.netty.buffer.ByteBuf) CoordinatorException(org.apache.pulsar.transaction.coordinator.exceptions.CoordinatorException) FastThreadLocal(io.netty.util.concurrent.FastThreadLocal) CommandPartitionedTopicMetadata(org.apache.pulsar.common.api.proto.CommandPartitionedTopicMetadata) CommandSeek(org.apache.pulsar.common.api.proto.CommandSeek) ConsumerStatsImpl(org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl) NoSuchElementException(java.util.NoSuchElementException) CommandGetLastMessageId(org.apache.pulsar.common.api.proto.CommandGetLastMessageId) CommandUtils(org.apache.pulsar.common.protocol.CommandUtils) ManagedLedgerImpl(org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl) AuthenticationDataCommand(org.apache.pulsar.broker.authentication.AuthenticationDataCommand) DEFAULT_CONSUMER_EPOCH(org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH) Logger(org.slf4j.Logger) Semaphore(java.util.concurrent.Semaphore) ServiceConfiguration(org.apache.pulsar.broker.ServiceConfiguration) FeatureFlags(org.apache.pulsar.common.api.proto.FeatureFlags) CommandCloseConsumer(org.apache.pulsar.common.api.proto.CommandCloseConsumer) SchemaVersion(org.apache.pulsar.common.protocol.schema.SchemaVersion) Commands.newLookupErrorResponse(org.apache.pulsar.common.protocol.Commands.newLookupErrorResponse) TimeUnit(java.util.concurrent.TimeUnit) HAProxyMessage(io.netty.handler.codec.haproxy.HAProxyMessage) ConcurrentLongHashMap(org.apache.pulsar.common.util.collections.ConcurrentLongHashMap) ConsumerBusyException(org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException) ChannelHandler(io.netty.channel.ChannelHandler) VisibleForTesting(com.google.common.annotations.VisibleForTesting) TransactionCoordinatorID(org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID) Collections(java.util.Collections) ServerError(org.apache.pulsar.common.api.proto.ServerError) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) RestException(org.apache.pulsar.broker.web.RestException) Semaphore(java.util.concurrent.Semaphore) TopicName(org.apache.pulsar.common.naming.TopicName)

Aggregations

ByteBuf (io.netty.buffer.ByteBuf)2 InetSocketAddress (java.net.InetSocketAddress)2 SocketAddress (java.net.SocketAddress)2 Optional (java.util.Optional)2 Semaphore (java.util.concurrent.Semaphore)2 StringUtils (org.apache.commons.lang3.StringUtils)2 PulsarClientException (org.apache.pulsar.client.api.PulsarClientException)2 CommandGetSchema (org.apache.pulsar.common.api.proto.CommandGetSchema)2 CommandGetTopicsOfNamespace (org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace)2 CommandLookupTopic (org.apache.pulsar.common.api.proto.CommandLookupTopic)2 CommandPartitionedTopicMetadata (org.apache.pulsar.common.api.proto.CommandPartitionedTopicMetadata)2 ServerError (org.apache.pulsar.common.api.proto.ServerError)2 TopicName (org.apache.pulsar.common.naming.TopicName)2 Commands (org.apache.pulsar.common.protocol.Commands)2 SchemaVersion (org.apache.pulsar.common.protocol.schema.SchemaVersion)2 Logger (org.slf4j.Logger)2 LoggerFactory (org.slf4j.LoggerFactory)2 VisibleForTesting (com.google.common.annotations.VisibleForTesting)1 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)1 Strings (com.google.common.base.Strings)1