Search in sources :

Example 1 with SchemaData

use of org.apache.pulsar.common.protocol.schema.SchemaData in project pulsar by apache.

the class AbstractTopic method addSchema.

@Override
public CompletableFuture<SchemaVersion> addSchema(SchemaData schema) {
    if (schema == null) {
        return CompletableFuture.completedFuture(SchemaVersion.Empty);
    }
    String base = TopicName.get(getName()).getPartitionedTopicName();
    String id = TopicName.get(base).getSchemaName();
    SchemaRegistryService schemaRegistryService = brokerService.pulsar().getSchemaRegistryService();
    if (allowAutoUpdateSchema()) {
        return schemaRegistryService.putSchemaIfAbsent(id, schema, getSchemaCompatibilityStrategy());
    } else {
        return schemaRegistryService.trimDeletedSchemaAndGetList(id).thenCompose(schemaAndMetadataList -> schemaRegistryService.getSchemaVersionBySchemaData(schemaAndMetadataList, schema).thenCompose(schemaVersion -> {
            if (schemaVersion == null) {
                return FutureUtil.failedFuture(new IncompatibleSchemaException("Schema not found and schema auto updating is disabled."));
            } else {
                return CompletableFuture.completedFuture(schemaVersion);
            }
        }));
    }
}
Also used : StatsBuckets(org.apache.bookkeeper.mledger.util.StatsBuckets) Arrays(java.util.Arrays) BookkeeperSchemaStorage(org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage) LoggerFactory(org.slf4j.LoggerFactory) DelayedDeliveryPolicies(org.apache.pulsar.common.policies.data.DelayedDeliveryPolicies) ProducerBusyException(org.apache.pulsar.broker.service.BrokerServiceException.ProducerBusyException) SubscribeRate(org.apache.pulsar.common.policies.data.SubscribeRate) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) Pair(org.apache.commons.lang3.tuple.Pair) ResourceGroup(org.apache.pulsar.broker.resourcegroup.ResourceGroup) Map(java.util.Map) DispatchRateImpl(org.apache.pulsar.common.policies.data.impl.DispatchRateImpl) EnumSet(java.util.EnumSet) InactiveTopicPolicies(org.apache.pulsar.common.policies.data.InactiveTopicPolicies) SchemaCompatibilityStrategy(org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy) ResourceGroupPublishLimiter(org.apache.pulsar.broker.resourcegroup.ResourceGroupPublishLimiter) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) PublishRate(org.apache.pulsar.common.policies.data.PublishRate) HierarchyTopicPolicies(org.apache.pulsar.common.policies.data.HierarchyTopicPolicies) Objects(java.util.Objects) List(java.util.List) FutureUtil(org.apache.pulsar.common.util.FutureUtil) TopicTerminatedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicTerminatedException) Optional(java.util.Optional) ENTRY_LATENCY_BUCKETS_USEC(org.apache.bookkeeper.mledger.impl.ManagedLedgerMBeanImpl.ENTRY_LATENCY_BUCKETS_USEC) Queue(java.util.Queue) IncompatibleSchemaException(org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException) ConcurrentLinkedQueue(java.util.concurrent.ConcurrentLinkedQueue) LongAdder(java.util.concurrent.atomic.LongAdder) ProducerFencedException(org.apache.pulsar.broker.service.BrokerServiceException.ProducerFencedException) TopicName(org.apache.pulsar.common.naming.TopicName) Getter(lombok.Getter) SchemaRegistryService(org.apache.pulsar.broker.service.schema.SchemaRegistryService) BacklogQuota(org.apache.pulsar.common.policies.data.BacklogQuota) CompletableFuture(java.util.concurrent.CompletableFuture) ReentrantReadWriteLock(java.util.concurrent.locks.ReentrantReadWriteLock) CollectionUtils(org.apache.commons.collections4.CollectionUtils) SubType(org.apache.pulsar.common.api.proto.CommandSubscribe.SubType) Lists(com.google.common.collect.Lists) MapUtils(org.apache.commons.collections4.MapUtils) RetentionPolicies(org.apache.pulsar.common.policies.data.RetentionPolicies) Summary(org.apache.pulsar.broker.stats.prometheus.metrics.Summary) Logger(org.slf4j.Logger) ServiceConfiguration(org.apache.pulsar.broker.ServiceConfiguration) MoreObjects(com.google.common.base.MoreObjects) AtomicLongFieldUpdater(java.util.concurrent.atomic.AtomicLongFieldUpdater) TopicPolicies(org.apache.pulsar.common.policies.data.TopicPolicies) SchemaVersion(org.apache.pulsar.common.protocol.schema.SchemaVersion) TimeUnit(java.util.concurrent.TimeUnit) Policies(org.apache.pulsar.common.policies.data.Policies) ConsumerBusyException(org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException) SchemaData(org.apache.pulsar.common.protocol.schema.SchemaData) Collections(java.util.Collections) IncompatibleSchemaException(org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException) SchemaRegistryService(org.apache.pulsar.broker.service.schema.SchemaRegistryService)

Example 2 with SchemaData

use of org.apache.pulsar.common.protocol.schema.SchemaData in project pulsar by apache.

the class PersistentTopicE2ETest method testDeleteSchema.

@Test
public void testDeleteSchema() throws Exception {
    @Cleanup PulsarClientImpl httpProtocolClient = (PulsarClientImpl) PulsarClient.builder().serviceUrl(brokerUrl.toString()).build();
    PulsarClientImpl binaryProtocolClient = (PulsarClientImpl) pulsarClient;
    LookupService binaryLookupService = binaryProtocolClient.getLookup();
    LookupService httpLookupService = httpProtocolClient.getLookup();
    String topicName = "persistent://prop/ns-abc/topic-1";
    // Topic is not GCed with live connection
    @Cleanup Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    Optional<Topic> topic = getTopic(topicName);
    assertTrue(topic.isPresent());
    byte[] data = JSONSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()).getSchemaInfo().getSchema();
    SchemaData schemaData = SchemaData.builder().data(data).type(SchemaType.BYTES).user("foo").build();
    topic.get().addSchema(schemaData).join();
    assertTrue(topicHasSchema(topicName));
    Assert.assertEquals(admin.schemas().getAllSchemas(topicName).size(), 1);
    assertTrue(httpLookupService.getSchema(TopicName.get(topicName), ByteBuffer.allocate(8).putLong(0).array()).get().isPresent());
    assertTrue(binaryLookupService.getSchema(TopicName.get(topicName), ByteBuffer.allocate(8).putLong(0).array()).get().isPresent());
    topic.get().deleteSchema().join();
    Assert.assertEquals(admin.schemas().getAllSchemas(topicName).size(), 0);
    assertFalse(httpLookupService.getSchema(TopicName.get(topicName), ByteBuffer.allocate(8).putLong(0).array()).get().isPresent());
    assertFalse(binaryLookupService.getSchema(TopicName.get(topicName), ByteBuffer.allocate(8).putLong(0).array()).get().isPresent());
    assertFalse(topicHasSchema(topicName));
}
Also used : LookupService(org.apache.pulsar.client.impl.LookupService) SchemaData(org.apache.pulsar.common.protocol.schema.SchemaData) PulsarClientImpl(org.apache.pulsar.client.impl.PulsarClientImpl) ToString(lombok.ToString) PersistentTopic(org.apache.pulsar.broker.service.persistent.PersistentTopic) Cleanup(lombok.Cleanup) Test(org.testng.annotations.Test)

Example 3 with SchemaData

use of org.apache.pulsar.common.protocol.schema.SchemaData in project pulsar by apache.

the class NonPersistentTopicE2ETest method testGCWillDeleteSchema.

@Test(groups = "broker")
public void testGCWillDeleteSchema() throws Exception {
    // 1. Simple successful GC
    String topicName = "non-persistent://prop/ns-abc/topic-1";
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    producer.close();
    Optional<Topic> topic = getTopic(topicName);
    assertTrue(topic.isPresent());
    byte[] data = JSONSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()).getSchemaInfo().getSchema();
    SchemaData schemaData = SchemaData.builder().data(data).type(SchemaType.BYTES).user("foo").build();
    topic.get().addSchema(schemaData).join();
    assertTrue(topicHasSchema(topicName));
    runGC();
    topic = getTopic(topicName);
    assertFalse(topic.isPresent());
    assertFalse(topicHasSchema(topicName));
    // 1a. Topic that add/removes subscription can be GC'd
    topicName = "non-persistent://prop/ns-abc/topic-1a";
    String subName = "sub1";
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
    topic = getTopic(topicName);
    assertTrue(topic.isPresent());
    topic.get().addSchema(schemaData).join();
    assertTrue(topicHasSchema(topicName));
    admin.topics().deleteSubscription(topicName, subName);
    consumer.close();
    runGC();
    topic = getTopic(topicName);
    assertFalse(topic.isPresent());
    assertFalse(topicHasSchema(topicName));
    // 2. Topic is not GCed with live connection
    topicName = "non-persistent://prop/ns-abc/topic-2";
    subName = "sub1";
    consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
    topic = getTopic(topicName);
    assertTrue(topic.isPresent());
    topic.get().addSchema(schemaData).join();
    assertTrue(topicHasSchema(topicName));
    runGC();
    topic = getTopic(topicName);
    assertTrue(topic.isPresent());
    assertTrue(topicHasSchema(topicName));
    // 3. Topic with subscription is not GCed even with no connections
    consumer.close();
    runGC();
    topic = getTopic(topicName);
    assertTrue(topic.isPresent());
    assertTrue(topicHasSchema(topicName));
    // 4. Topic can be GCed after unsubscribe
    admin.topics().deleteSubscription(topicName, subName);
    runGC();
    topic = getTopic(topicName);
    assertFalse(topic.isPresent());
    assertFalse(topicHasSchema(topicName));
}
Also used : SchemaData(org.apache.pulsar.common.protocol.schema.SchemaData) Test(org.testng.annotations.Test)

Example 4 with SchemaData

use of org.apache.pulsar.common.protocol.schema.SchemaData in project pulsar by apache.

the class ServerCnx method handleProducer.

@Override
protected void handleProducer(final CommandProducer cmdProducer) {
    checkArgument(state == State.Connected);
    final long producerId = cmdProducer.getProducerId();
    final long requestId = cmdProducer.getRequestId();
    // Use producer name provided by client if present
    final String producerName = cmdProducer.hasProducerName() ? cmdProducer.getProducerName() : service.generateUniqueProducerName();
    final long epoch = cmdProducer.getEpoch();
    final boolean userProvidedProducerName = cmdProducer.isUserProvidedProducerName();
    final boolean isEncrypted = cmdProducer.isEncrypted();
    final Map<String, String> metadata = CommandUtils.metadataFromCommand(cmdProducer);
    final SchemaData schema = cmdProducer.hasSchema() ? getSchema(cmdProducer.getSchema()) : null;
    final ProducerAccessMode producerAccessMode = cmdProducer.getProducerAccessMode();
    final Optional<Long> topicEpoch = cmdProducer.hasTopicEpoch() ? Optional.of(cmdProducer.getTopicEpoch()) : Optional.empty();
    final boolean isTxnEnabled = cmdProducer.isTxnEnabled();
    final String initialSubscriptionName = cmdProducer.hasInitialSubscriptionName() ? cmdProducer.getInitialSubscriptionName() : null;
    final boolean supportsPartialProducer = supportsPartialProducer();
    TopicName topicName = validateTopicName(cmdProducer.getTopic(), requestId, cmdProducer);
    if (topicName == null) {
        return;
    }
    if (invalidOriginalPrincipal(originalPrincipal)) {
        final String msg = "Valid Proxy Client role should be provided while creating producer ";
        log.warn("[{}] {} with role {} and proxyClientAuthRole {} on topic {}", remoteAddress, msg, authRole, originalPrincipal, topicName);
        commandSender.sendErrorResponse(requestId, ServerError.AuthorizationError, msg);
        return;
    }
    CompletableFuture<Boolean> isAuthorizedFuture = isTopicOperationAllowed(topicName, TopicOperation.PRODUCE);
    if (!Strings.isNullOrEmpty(initialSubscriptionName)) {
        isAuthorizedFuture = isAuthorizedFuture.thenCombine(isTopicOperationAllowed(topicName, TopicOperation.SUBSCRIBE), (canProduce, canSubscribe) -> canProduce && canSubscribe);
    }
    isAuthorizedFuture.thenApply(isAuthorized -> {
        if (!isAuthorized) {
            String msg = "Client is not authorized to Produce";
            log.warn("[{}] {} with role {}", remoteAddress, msg, getPrincipal());
            ctx.writeAndFlush(Commands.newError(requestId, ServerError.AuthorizationError, msg));
            return null;
        }
        if (log.isDebugEnabled()) {
            log.debug("[{}] Client is authorized to Produce with role {}", remoteAddress, getPrincipal());
        }
        CompletableFuture<Producer> producerFuture = new CompletableFuture<>();
        CompletableFuture<Producer> existingProducerFuture = producers.putIfAbsent(producerId, producerFuture);
        if (existingProducerFuture != null) {
            if (existingProducerFuture.isDone() && !existingProducerFuture.isCompletedExceptionally()) {
                Producer producer = existingProducerFuture.getNow(null);
                log.info("[{}] Producer with the same id is already created:" + " producerId={}, producer={}", remoteAddress, producerId, producer);
                commandSender.sendProducerSuccessResponse(requestId, producer.getProducerName(), producer.getSchemaVersion());
                return null;
            } else {
                // There was an early request to create a producer with same producerId.
                // This can happen when client timeout is lower than the broker timeouts.
                // We need to wait until the previous producer creation request
                // either complete or fails.
                ServerError error = null;
                if (!existingProducerFuture.isDone()) {
                    error = ServerError.ServiceNotReady;
                } else {
                    error = getErrorCode(existingProducerFuture);
                    // remove producer with producerId as it's already completed with exception
                    producers.remove(producerId, existingProducerFuture);
                }
                log.warn("[{}][{}] Producer with id is already present on the connection, producerId={}", remoteAddress, topicName, producerId);
                commandSender.sendErrorResponse(requestId, error, "Producer is already present on the connection");
                return null;
            }
        }
        log.info("[{}][{}] Creating producer. producerId={}", remoteAddress, topicName, producerId);
        service.getOrCreateTopic(topicName.toString()).thenCompose((Topic topic) -> {
            // Before creating producer, check if backlog quota exceeded
            // on topic for size based limit and time based limit
            CompletableFuture<Void> backlogQuotaCheckFuture = CompletableFuture.allOf(topic.checkBacklogQuotaExceeded(producerName, BacklogQuotaType.destination_storage), topic.checkBacklogQuotaExceeded(producerName, BacklogQuotaType.message_age));
            backlogQuotaCheckFuture.thenRun(() -> {
                // Check whether the producer will publish encrypted messages or not
                if ((topic.isEncryptionRequired() || encryptionRequireOnProducer) && !isEncrypted) {
                    String msg = String.format("Encryption is required in %s", topicName);
                    log.warn("[{}] {}", remoteAddress, msg);
                    if (producerFuture.completeExceptionally(new ServerMetadataException(msg))) {
                        commandSender.sendErrorResponse(requestId, ServerError.MetadataError, msg);
                    }
                    producers.remove(producerId, producerFuture);
                    return;
                }
                disableTcpNoDelayIfNeeded(topicName.toString(), producerName);
                CompletableFuture<SchemaVersion> schemaVersionFuture = tryAddSchema(topic, schema);
                schemaVersionFuture.exceptionally(exception -> {
                    if (producerFuture.completeExceptionally(exception)) {
                        String message = exception.getMessage();
                        if (exception.getCause() != null) {
                            message += (" caused by " + exception.getCause());
                        }
                        commandSender.sendErrorResponse(requestId, BrokerServiceException.getClientErrorCode(exception), message);
                    }
                    producers.remove(producerId, producerFuture);
                    return null;
                });
                schemaVersionFuture.thenAccept(schemaVersion -> {
                    topic.checkIfTransactionBufferRecoverCompletely(isTxnEnabled).thenAccept(future -> {
                        CompletableFuture<Subscription> createInitSubFuture;
                        if (!Strings.isNullOrEmpty(initialSubscriptionName) && topic.isPersistent() && !topic.getSubscriptions().containsKey(initialSubscriptionName)) {
                            if (!this.getBrokerService().isAllowAutoSubscriptionCreation(topicName)) {
                                String msg = "Could not create the initial subscription due to the auto subscription " + "creation is not allowed.";
                                if (producerFuture.completeExceptionally(new BrokerServiceException.NotAllowedException(msg))) {
                                    log.warn("[{}] {} initialSubscriptionName: {}, topic: {}", remoteAddress, msg, initialSubscriptionName, topicName);
                                    commandSender.sendErrorResponse(requestId, ServerError.NotAllowedError, msg);
                                }
                                producers.remove(producerId, producerFuture);
                                return;
                            }
                            createInitSubFuture = topic.createSubscription(initialSubscriptionName, InitialPosition.Earliest, false);
                        } else {
                            createInitSubFuture = CompletableFuture.completedFuture(null);
                        }
                        createInitSubFuture.whenComplete((sub, ex) -> {
                            if (ex != null) {
                                String msg = "Failed to create the initial subscription: " + ex.getCause().getMessage();
                                log.warn("[{}] {} initialSubscriptionName: {}, topic: {}", remoteAddress, msg, initialSubscriptionName, topicName);
                                if (producerFuture.completeExceptionally(ex)) {
                                    commandSender.sendErrorResponse(requestId, BrokerServiceException.getClientErrorCode(ex), msg);
                                }
                                producers.remove(producerId, producerFuture);
                                return;
                            }
                            buildProducerAndAddTopic(topic, producerId, producerName, requestId, isEncrypted, metadata, schemaVersion, epoch, userProvidedProducerName, topicName, producerAccessMode, topicEpoch, supportsPartialProducer, producerFuture);
                        });
                    }).exceptionally(exception -> {
                        Throwable cause = exception.getCause();
                        log.error("producerId {}, requestId {} : TransactionBuffer recover failed", producerId, requestId, exception);
                        if (producerFuture.completeExceptionally(exception)) {
                            commandSender.sendErrorResponse(requestId, ServiceUnitNotReadyException.getClientErrorCode(cause), cause.getMessage());
                        }
                        producers.remove(producerId, producerFuture);
                        return null;
                    });
                });
            });
            return backlogQuotaCheckFuture;
        }).exceptionally(exception -> {
            Throwable cause = exception.getCause();
            if (cause instanceof BrokerServiceException.TopicBacklogQuotaExceededException) {
                BrokerServiceException.TopicBacklogQuotaExceededException tbqe = (BrokerServiceException.TopicBacklogQuotaExceededException) cause;
                IllegalStateException illegalStateException = new IllegalStateException(tbqe);
                BacklogQuota.RetentionPolicy retentionPolicy = tbqe.getRetentionPolicy();
                if (producerFuture.completeExceptionally(illegalStateException)) {
                    if (retentionPolicy == BacklogQuota.RetentionPolicy.producer_request_hold) {
                        commandSender.sendErrorResponse(requestId, ServerError.ProducerBlockedQuotaExceededError, illegalStateException.getMessage());
                    } else if (retentionPolicy == BacklogQuota.RetentionPolicy.producer_exception) {
                        commandSender.sendErrorResponse(requestId, ServerError.ProducerBlockedQuotaExceededException, illegalStateException.getMessage());
                    }
                }
                producers.remove(producerId, producerFuture);
                return null;
            }
            // Do not print stack traces for expected exceptions
            if (cause instanceof NoSuchElementException) {
                cause = new TopicNotFoundException("Topic Not Found.");
                log.info("[{}] Failed to load topic {}, producerId={}: Topic not found", remoteAddress, topicName, producerId);
            } else if (!Exceptions.areExceptionsPresentInChain(cause, ServiceUnitNotReadyException.class, ManagedLedgerException.class)) {
                log.error("[{}] Failed to create topic {}, producerId={}", remoteAddress, topicName, producerId, exception);
            }
            // client, only if not completed already.
            if (producerFuture.completeExceptionally(exception)) {
                commandSender.sendErrorResponse(requestId, BrokerServiceException.getClientErrorCode(cause), cause.getMessage());
            }
            producers.remove(producerId, producerFuture);
            return null;
        });
        return null;
    }).exceptionally(ex -> {
        logAuthException(remoteAddress, "producer", getPrincipal(), Optional.of(topicName), ex);
        commandSender.sendErrorResponse(requestId, ServerError.AuthorizationError, ex.getMessage());
        return null;
    });
}
Also used : CommandAuthResponse(org.apache.pulsar.common.api.proto.CommandAuthResponse) CommandUnsubscribe(org.apache.pulsar.common.api.proto.CommandUnsubscribe) ServiceUnitNotReadyException(org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException) CommandProducer(org.apache.pulsar.common.api.proto.CommandProducer) MessageIdData(org.apache.pulsar.common.api.proto.MessageIdData) ByteBufPair(org.apache.pulsar.common.protocol.ByteBufPair) StringUtils(org.apache.commons.lang3.StringUtils) ProtocolVersion(org.apache.pulsar.common.api.proto.ProtocolVersion) TxnID(org.apache.pulsar.client.api.transaction.TxnID) MLTransactionMetadataStore(org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStore) TopicOperation(org.apache.pulsar.common.policies.data.TopicOperation) MutableLong(org.apache.commons.lang3.mutable.MutableLong) Map(java.util.Map) BrokerInterceptor(org.apache.pulsar.broker.intercept.BrokerInterceptor) RestException(org.apache.pulsar.broker.web.RestException) NamespaceOperation(org.apache.pulsar.common.policies.data.NamespaceOperation) BaseCommand(org.apache.pulsar.common.api.proto.BaseCommand) CommandAck(org.apache.pulsar.common.api.proto.CommandAck) PositionImpl(org.apache.bookkeeper.mledger.impl.PositionImpl) InterceptException(org.apache.pulsar.common.intercept.InterceptException) CommandFlow(org.apache.pulsar.common.api.proto.CommandFlow) CommandConsumerStats(org.apache.pulsar.common.api.proto.CommandConsumerStats) ServerError(org.apache.pulsar.common.api.proto.ServerError) Set(java.util.Set) CommandNewTxn(org.apache.pulsar.common.api.proto.CommandNewTxn) StringUtils.isNotBlank(org.apache.commons.lang3.StringUtils.isNotBlank) BatchMessageIdImpl(org.apache.pulsar.client.impl.BatchMessageIdImpl) MessageMetadata(org.apache.pulsar.common.api.proto.MessageMetadata) SafeRun(org.apache.bookkeeper.mledger.util.SafeRun) SslHandler(io.netty.handler.ssl.SslHandler) AsyncCallbacks(org.apache.bookkeeper.mledger.AsyncCallbacks) ExceptionUtils(org.apache.commons.lang3.exception.ExceptionUtils) ClientCnx(org.apache.pulsar.client.impl.ClientCnx) ChannelOption(io.netty.channel.ChannelOption) SchemaRegistryService(org.apache.pulsar.broker.service.schema.SchemaRegistryService) AuthenticationState(org.apache.pulsar.broker.authentication.AuthenticationState) CommandGetTopicsOfNamespace(org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace) TopicNotFoundException(org.apache.pulsar.broker.service.BrokerServiceException.TopicNotFoundException) SchemaType(org.apache.pulsar.common.schema.SchemaType) Commands(org.apache.pulsar.common.protocol.Commands) CommandCloseProducer(org.apache.pulsar.common.api.proto.CommandCloseProducer) Strings(com.google.common.base.Strings) SubType(org.apache.pulsar.common.api.proto.CommandSubscribe.SubType) ServerMetadataException(org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException) SSLSession(javax.net.ssl.SSLSession) CommandGetOrCreateSchema(org.apache.pulsar.common.api.proto.CommandGetOrCreateSchema) ProtocolVersion.v5(org.apache.pulsar.common.api.proto.ProtocolVersion.v5) CommandGetSchema(org.apache.pulsar.common.api.proto.CommandGetSchema) SchemaInfoUtil(org.apache.pulsar.client.impl.schema.SchemaInfoUtil) CommandRedeliverUnacknowledgedMessages(org.apache.pulsar.common.api.proto.CommandRedeliverUnacknowledgedMessages) Metadata(org.apache.pulsar.common.naming.Metadata) Promise(io.netty.util.concurrent.Promise) AuthenticationProvider(org.apache.pulsar.broker.authentication.AuthenticationProvider) InitialPosition(org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition) lombok.val(lombok.val) PulsarService(org.apache.pulsar.broker.PulsarService) KeySharedMode(org.apache.pulsar.common.api.proto.KeySharedMode) CommandEndTxnOnSubscription(org.apache.pulsar.common.api.proto.CommandEndTxnOnSubscription) CommandConnect(org.apache.pulsar.common.api.proto.CommandConnect) SchemaData(org.apache.pulsar.common.protocol.schema.SchemaData) ProducerAccessMode(org.apache.pulsar.common.api.proto.ProducerAccessMode) CommandLookupTopic(org.apache.pulsar.common.api.proto.CommandLookupTopic) MutableInt(org.apache.commons.lang3.mutable.MutableInt) SocketAddress(java.net.SocketAddress) CommandAddPartitionToTxn(org.apache.pulsar.common.api.proto.CommandAddPartitionToTxn) LoggerFactory(org.slf4j.LoggerFactory) AuthData(org.apache.pulsar.common.api.AuthData) Exceptions(org.apache.pulsar.functions.utils.Exceptions) AuthenticationException(javax.naming.AuthenticationException) KeyValue(org.apache.pulsar.common.api.proto.KeyValue) PersistentTopicsBase.unsafeGetPartitionedTopicMetadataAsync(org.apache.pulsar.broker.admin.impl.PersistentTopicsBase.unsafeGetPartitionedTopicMetadataAsync) CommandEndTxnOnPartition(org.apache.pulsar.common.api.proto.CommandEndTxnOnPartition) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) BacklogQuotaType(org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType) Gauge(io.prometheus.client.Gauge) TxnAction(org.apache.pulsar.common.api.proto.TxnAction) NamespaceName(org.apache.pulsar.common.naming.NamespaceName) Schema(org.apache.pulsar.common.api.proto.Schema) TransactionMetadataStoreService(org.apache.pulsar.broker.TransactionMetadataStoreService) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) IdentityHashMap(java.util.IdentityHashMap) SubscriptionNotFoundException(org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionNotFoundException) Position(org.apache.bookkeeper.mledger.Position) InetSocketAddress(java.net.InetSocketAddress) Collectors(java.util.stream.Collectors) CommandEndTxn(org.apache.pulsar.common.api.proto.CommandEndTxn) AuthenticationDataSource(org.apache.pulsar.broker.authentication.AuthenticationDataSource) Objects(java.util.Objects) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) PersistentTopic(org.apache.pulsar.broker.service.persistent.PersistentTopic) FutureUtil(org.apache.pulsar.common.util.FutureUtil) PersistentSubscription(org.apache.pulsar.broker.service.persistent.PersistentSubscription) Optional(java.util.Optional) IncompatibleSchemaException(org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException) PulsarHandler(org.apache.pulsar.common.protocol.PulsarHandler) CommandTcClientConnectRequest(org.apache.pulsar.common.api.proto.CommandTcClientConnectRequest) TopicName(org.apache.pulsar.common.naming.TopicName) CommandAddSubscriptionToTxn(org.apache.pulsar.common.api.proto.CommandAddSubscriptionToTxn) Entry(org.apache.bookkeeper.mledger.Entry) CommandSend(org.apache.pulsar.common.api.proto.CommandSend) BacklogQuota(org.apache.pulsar.common.policies.data.BacklogQuota) CompletableFuture(java.util.concurrent.CompletableFuture) TopicLookupBase.lookupTopicAsync(org.apache.pulsar.broker.lookup.TopicLookupBase.lookupTopicAsync) KeySharedMeta(org.apache.pulsar.common.api.proto.KeySharedMeta) CommandSubscribe(org.apache.pulsar.common.api.proto.CommandSubscribe) ChannelHandlerContext(io.netty.channel.ChannelHandlerContext) ByteBuf(io.netty.buffer.ByteBuf) CoordinatorException(org.apache.pulsar.transaction.coordinator.exceptions.CoordinatorException) FastThreadLocal(io.netty.util.concurrent.FastThreadLocal) CommandPartitionedTopicMetadata(org.apache.pulsar.common.api.proto.CommandPartitionedTopicMetadata) CommandSeek(org.apache.pulsar.common.api.proto.CommandSeek) ConsumerStatsImpl(org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl) NoSuchElementException(java.util.NoSuchElementException) CommandGetLastMessageId(org.apache.pulsar.common.api.proto.CommandGetLastMessageId) CommandUtils(org.apache.pulsar.common.protocol.CommandUtils) ManagedLedgerImpl(org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl) AuthenticationDataCommand(org.apache.pulsar.broker.authentication.AuthenticationDataCommand) DEFAULT_CONSUMER_EPOCH(org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH) Logger(org.slf4j.Logger) Semaphore(java.util.concurrent.Semaphore) ServiceConfiguration(org.apache.pulsar.broker.ServiceConfiguration) FeatureFlags(org.apache.pulsar.common.api.proto.FeatureFlags) CommandCloseConsumer(org.apache.pulsar.common.api.proto.CommandCloseConsumer) SchemaVersion(org.apache.pulsar.common.protocol.schema.SchemaVersion) Commands.newLookupErrorResponse(org.apache.pulsar.common.protocol.Commands.newLookupErrorResponse) TimeUnit(java.util.concurrent.TimeUnit) HAProxyMessage(io.netty.handler.codec.haproxy.HAProxyMessage) ConcurrentLongHashMap(org.apache.pulsar.common.util.collections.ConcurrentLongHashMap) ConsumerBusyException(org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException) ChannelHandler(io.netty.channel.ChannelHandler) VisibleForTesting(com.google.common.annotations.VisibleForTesting) TransactionCoordinatorID(org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID) Collections(java.util.Collections) SchemaData(org.apache.pulsar.common.protocol.schema.SchemaData) TopicNotFoundException(org.apache.pulsar.broker.service.BrokerServiceException.TopicNotFoundException) ServiceUnitNotReadyException(org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException) CompletableFuture(java.util.concurrent.CompletableFuture) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) ProducerAccessMode(org.apache.pulsar.common.api.proto.ProducerAccessMode) CommandLookupTopic(org.apache.pulsar.common.api.proto.CommandLookupTopic) PersistentTopic(org.apache.pulsar.broker.service.persistent.PersistentTopic) SchemaVersion(org.apache.pulsar.common.protocol.schema.SchemaVersion) ServerError(org.apache.pulsar.common.api.proto.ServerError) TopicName(org.apache.pulsar.common.naming.TopicName) CommandProducer(org.apache.pulsar.common.api.proto.CommandProducer) CommandCloseProducer(org.apache.pulsar.common.api.proto.CommandCloseProducer) ServerMetadataException(org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException) MutableLong(org.apache.commons.lang3.mutable.MutableLong) NoSuchElementException(java.util.NoSuchElementException)

Example 5 with SchemaData

use of org.apache.pulsar.common.protocol.schema.SchemaData in project pulsar by apache.

the class AvroSchemaBasedCompatibilityCheck method checkCompatible.

@Override
public void checkCompatible(Iterable<SchemaData> from, SchemaData to, SchemaCompatibilityStrategy strategy) throws IncompatibleSchemaException {
    LinkedList<Schema> fromList = new LinkedList<>();
    checkArgument(from != null, "check compatibility list is null");
    try {
        for (SchemaData schemaData : from) {
            Schema.Parser parser = new Schema.Parser();
            parser.setValidateDefaults(false);
            fromList.addFirst(parser.parse(new String(schemaData.getData(), UTF_8)));
        }
        Schema.Parser parser = new Schema.Parser();
        parser.setValidateDefaults(false);
        Schema toSchema = parser.parse(new String(to.getData(), UTF_8));
        SchemaValidator schemaValidator = createSchemaValidator(strategy);
        schemaValidator.validate(toSchema, fromList);
    } catch (SchemaParseException e) {
        log.warn("Error during schema parsing: {}", e.getMessage());
        throw new IncompatibleSchemaException(e);
    } catch (SchemaValidationException e) {
        log.warn("Error during schema compatibility check: {}", e.getMessage());
        throw new IncompatibleSchemaException(e);
    }
}
Also used : IncompatibleSchemaException(org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException) SchemaValidationException(org.apache.avro.SchemaValidationException) SchemaData(org.apache.pulsar.common.protocol.schema.SchemaData) SchemaParseException(org.apache.avro.SchemaParseException) Schema(org.apache.avro.Schema) SchemaValidator(org.apache.avro.SchemaValidator) LinkedList(java.util.LinkedList)

Aggregations

SchemaData (org.apache.pulsar.common.protocol.schema.SchemaData)61 Test (org.testng.annotations.Test)52 CompletableFuture (java.util.concurrent.CompletableFuture)10 MockedPulsarServiceBaseTest (org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest)8 IncompatibleSchemaException (org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException)7 SchemaVersion (org.apache.pulsar.common.protocol.schema.SchemaVersion)7 Optional (java.util.Optional)6 TopicName (org.apache.pulsar.common.naming.TopicName)6 FutureUtil (org.apache.pulsar.common.util.FutureUtil)6 Collections (java.util.Collections)5 PersistentTopic (org.apache.pulsar.broker.service.persistent.PersistentTopic)5 SchemaType (org.apache.pulsar.common.schema.SchemaType)5 Logger (org.slf4j.Logger)5 LoggerFactory (org.slf4j.LoggerFactory)5 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)4 ByteBuf (io.netty.buffer.ByteBuf)4 InetSocketAddress (java.net.InetSocketAddress)4 PulsarClientException (org.apache.pulsar.client.api.PulsarClientException)4 VisibleForTesting (com.google.common.annotations.VisibleForTesting)3 Strings (com.google.common.base.Strings)3