use of org.apache.pulsar.common.protocol.schema.SchemaData in project pulsar by apache.
the class AbstractTopic method addSchema.
@Override
public CompletableFuture<SchemaVersion> addSchema(SchemaData schema) {
if (schema == null) {
return CompletableFuture.completedFuture(SchemaVersion.Empty);
}
String base = TopicName.get(getName()).getPartitionedTopicName();
String id = TopicName.get(base).getSchemaName();
SchemaRegistryService schemaRegistryService = brokerService.pulsar().getSchemaRegistryService();
if (allowAutoUpdateSchema()) {
return schemaRegistryService.putSchemaIfAbsent(id, schema, getSchemaCompatibilityStrategy());
} else {
return schemaRegistryService.trimDeletedSchemaAndGetList(id).thenCompose(schemaAndMetadataList -> schemaRegistryService.getSchemaVersionBySchemaData(schemaAndMetadataList, schema).thenCompose(schemaVersion -> {
if (schemaVersion == null) {
return FutureUtil.failedFuture(new IncompatibleSchemaException("Schema not found and schema auto updating is disabled."));
} else {
return CompletableFuture.completedFuture(schemaVersion);
}
}));
}
}
use of org.apache.pulsar.common.protocol.schema.SchemaData in project pulsar by apache.
the class PersistentTopicE2ETest method testDeleteSchema.
@Test
public void testDeleteSchema() throws Exception {
@Cleanup PulsarClientImpl httpProtocolClient = (PulsarClientImpl) PulsarClient.builder().serviceUrl(brokerUrl.toString()).build();
PulsarClientImpl binaryProtocolClient = (PulsarClientImpl) pulsarClient;
LookupService binaryLookupService = binaryProtocolClient.getLookup();
LookupService httpLookupService = httpProtocolClient.getLookup();
String topicName = "persistent://prop/ns-abc/topic-1";
// Topic is not GCed with live connection
@Cleanup Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
Optional<Topic> topic = getTopic(topicName);
assertTrue(topic.isPresent());
byte[] data = JSONSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()).getSchemaInfo().getSchema();
SchemaData schemaData = SchemaData.builder().data(data).type(SchemaType.BYTES).user("foo").build();
topic.get().addSchema(schemaData).join();
assertTrue(topicHasSchema(topicName));
Assert.assertEquals(admin.schemas().getAllSchemas(topicName).size(), 1);
assertTrue(httpLookupService.getSchema(TopicName.get(topicName), ByteBuffer.allocate(8).putLong(0).array()).get().isPresent());
assertTrue(binaryLookupService.getSchema(TopicName.get(topicName), ByteBuffer.allocate(8).putLong(0).array()).get().isPresent());
topic.get().deleteSchema().join();
Assert.assertEquals(admin.schemas().getAllSchemas(topicName).size(), 0);
assertFalse(httpLookupService.getSchema(TopicName.get(topicName), ByteBuffer.allocate(8).putLong(0).array()).get().isPresent());
assertFalse(binaryLookupService.getSchema(TopicName.get(topicName), ByteBuffer.allocate(8).putLong(0).array()).get().isPresent());
assertFalse(topicHasSchema(topicName));
}
use of org.apache.pulsar.common.protocol.schema.SchemaData in project pulsar by apache.
the class NonPersistentTopicE2ETest method testGCWillDeleteSchema.
@Test(groups = "broker")
public void testGCWillDeleteSchema() throws Exception {
// 1. Simple successful GC
String topicName = "non-persistent://prop/ns-abc/topic-1";
Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
producer.close();
Optional<Topic> topic = getTopic(topicName);
assertTrue(topic.isPresent());
byte[] data = JSONSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()).getSchemaInfo().getSchema();
SchemaData schemaData = SchemaData.builder().data(data).type(SchemaType.BYTES).user("foo").build();
topic.get().addSchema(schemaData).join();
assertTrue(topicHasSchema(topicName));
runGC();
topic = getTopic(topicName);
assertFalse(topic.isPresent());
assertFalse(topicHasSchema(topicName));
// 1a. Topic that add/removes subscription can be GC'd
topicName = "non-persistent://prop/ns-abc/topic-1a";
String subName = "sub1";
Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
topic = getTopic(topicName);
assertTrue(topic.isPresent());
topic.get().addSchema(schemaData).join();
assertTrue(topicHasSchema(topicName));
admin.topics().deleteSubscription(topicName, subName);
consumer.close();
runGC();
topic = getTopic(topicName);
assertFalse(topic.isPresent());
assertFalse(topicHasSchema(topicName));
// 2. Topic is not GCed with live connection
topicName = "non-persistent://prop/ns-abc/topic-2";
subName = "sub1";
consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
topic = getTopic(topicName);
assertTrue(topic.isPresent());
topic.get().addSchema(schemaData).join();
assertTrue(topicHasSchema(topicName));
runGC();
topic = getTopic(topicName);
assertTrue(topic.isPresent());
assertTrue(topicHasSchema(topicName));
// 3. Topic with subscription is not GCed even with no connections
consumer.close();
runGC();
topic = getTopic(topicName);
assertTrue(topic.isPresent());
assertTrue(topicHasSchema(topicName));
// 4. Topic can be GCed after unsubscribe
admin.topics().deleteSubscription(topicName, subName);
runGC();
topic = getTopic(topicName);
assertFalse(topic.isPresent());
assertFalse(topicHasSchema(topicName));
}
use of org.apache.pulsar.common.protocol.schema.SchemaData in project pulsar by apache.
the class ServerCnx method handleProducer.
@Override
protected void handleProducer(final CommandProducer cmdProducer) {
checkArgument(state == State.Connected);
final long producerId = cmdProducer.getProducerId();
final long requestId = cmdProducer.getRequestId();
// Use producer name provided by client if present
final String producerName = cmdProducer.hasProducerName() ? cmdProducer.getProducerName() : service.generateUniqueProducerName();
final long epoch = cmdProducer.getEpoch();
final boolean userProvidedProducerName = cmdProducer.isUserProvidedProducerName();
final boolean isEncrypted = cmdProducer.isEncrypted();
final Map<String, String> metadata = CommandUtils.metadataFromCommand(cmdProducer);
final SchemaData schema = cmdProducer.hasSchema() ? getSchema(cmdProducer.getSchema()) : null;
final ProducerAccessMode producerAccessMode = cmdProducer.getProducerAccessMode();
final Optional<Long> topicEpoch = cmdProducer.hasTopicEpoch() ? Optional.of(cmdProducer.getTopicEpoch()) : Optional.empty();
final boolean isTxnEnabled = cmdProducer.isTxnEnabled();
final String initialSubscriptionName = cmdProducer.hasInitialSubscriptionName() ? cmdProducer.getInitialSubscriptionName() : null;
final boolean supportsPartialProducer = supportsPartialProducer();
TopicName topicName = validateTopicName(cmdProducer.getTopic(), requestId, cmdProducer);
if (topicName == null) {
return;
}
if (invalidOriginalPrincipal(originalPrincipal)) {
final String msg = "Valid Proxy Client role should be provided while creating producer ";
log.warn("[{}] {} with role {} and proxyClientAuthRole {} on topic {}", remoteAddress, msg, authRole, originalPrincipal, topicName);
commandSender.sendErrorResponse(requestId, ServerError.AuthorizationError, msg);
return;
}
CompletableFuture<Boolean> isAuthorizedFuture = isTopicOperationAllowed(topicName, TopicOperation.PRODUCE);
if (!Strings.isNullOrEmpty(initialSubscriptionName)) {
isAuthorizedFuture = isAuthorizedFuture.thenCombine(isTopicOperationAllowed(topicName, TopicOperation.SUBSCRIBE), (canProduce, canSubscribe) -> canProduce && canSubscribe);
}
isAuthorizedFuture.thenApply(isAuthorized -> {
if (!isAuthorized) {
String msg = "Client is not authorized to Produce";
log.warn("[{}] {} with role {}", remoteAddress, msg, getPrincipal());
ctx.writeAndFlush(Commands.newError(requestId, ServerError.AuthorizationError, msg));
return null;
}
if (log.isDebugEnabled()) {
log.debug("[{}] Client is authorized to Produce with role {}", remoteAddress, getPrincipal());
}
CompletableFuture<Producer> producerFuture = new CompletableFuture<>();
CompletableFuture<Producer> existingProducerFuture = producers.putIfAbsent(producerId, producerFuture);
if (existingProducerFuture != null) {
if (existingProducerFuture.isDone() && !existingProducerFuture.isCompletedExceptionally()) {
Producer producer = existingProducerFuture.getNow(null);
log.info("[{}] Producer with the same id is already created:" + " producerId={}, producer={}", remoteAddress, producerId, producer);
commandSender.sendProducerSuccessResponse(requestId, producer.getProducerName(), producer.getSchemaVersion());
return null;
} else {
// There was an early request to create a producer with same producerId.
// This can happen when client timeout is lower than the broker timeouts.
// We need to wait until the previous producer creation request
// either complete or fails.
ServerError error = null;
if (!existingProducerFuture.isDone()) {
error = ServerError.ServiceNotReady;
} else {
error = getErrorCode(existingProducerFuture);
// remove producer with producerId as it's already completed with exception
producers.remove(producerId, existingProducerFuture);
}
log.warn("[{}][{}] Producer with id is already present on the connection, producerId={}", remoteAddress, topicName, producerId);
commandSender.sendErrorResponse(requestId, error, "Producer is already present on the connection");
return null;
}
}
log.info("[{}][{}] Creating producer. producerId={}", remoteAddress, topicName, producerId);
service.getOrCreateTopic(topicName.toString()).thenCompose((Topic topic) -> {
// Before creating producer, check if backlog quota exceeded
// on topic for size based limit and time based limit
CompletableFuture<Void> backlogQuotaCheckFuture = CompletableFuture.allOf(topic.checkBacklogQuotaExceeded(producerName, BacklogQuotaType.destination_storage), topic.checkBacklogQuotaExceeded(producerName, BacklogQuotaType.message_age));
backlogQuotaCheckFuture.thenRun(() -> {
// Check whether the producer will publish encrypted messages or not
if ((topic.isEncryptionRequired() || encryptionRequireOnProducer) && !isEncrypted) {
String msg = String.format("Encryption is required in %s", topicName);
log.warn("[{}] {}", remoteAddress, msg);
if (producerFuture.completeExceptionally(new ServerMetadataException(msg))) {
commandSender.sendErrorResponse(requestId, ServerError.MetadataError, msg);
}
producers.remove(producerId, producerFuture);
return;
}
disableTcpNoDelayIfNeeded(topicName.toString(), producerName);
CompletableFuture<SchemaVersion> schemaVersionFuture = tryAddSchema(topic, schema);
schemaVersionFuture.exceptionally(exception -> {
if (producerFuture.completeExceptionally(exception)) {
String message = exception.getMessage();
if (exception.getCause() != null) {
message += (" caused by " + exception.getCause());
}
commandSender.sendErrorResponse(requestId, BrokerServiceException.getClientErrorCode(exception), message);
}
producers.remove(producerId, producerFuture);
return null;
});
schemaVersionFuture.thenAccept(schemaVersion -> {
topic.checkIfTransactionBufferRecoverCompletely(isTxnEnabled).thenAccept(future -> {
CompletableFuture<Subscription> createInitSubFuture;
if (!Strings.isNullOrEmpty(initialSubscriptionName) && topic.isPersistent() && !topic.getSubscriptions().containsKey(initialSubscriptionName)) {
if (!this.getBrokerService().isAllowAutoSubscriptionCreation(topicName)) {
String msg = "Could not create the initial subscription due to the auto subscription " + "creation is not allowed.";
if (producerFuture.completeExceptionally(new BrokerServiceException.NotAllowedException(msg))) {
log.warn("[{}] {} initialSubscriptionName: {}, topic: {}", remoteAddress, msg, initialSubscriptionName, topicName);
commandSender.sendErrorResponse(requestId, ServerError.NotAllowedError, msg);
}
producers.remove(producerId, producerFuture);
return;
}
createInitSubFuture = topic.createSubscription(initialSubscriptionName, InitialPosition.Earliest, false);
} else {
createInitSubFuture = CompletableFuture.completedFuture(null);
}
createInitSubFuture.whenComplete((sub, ex) -> {
if (ex != null) {
String msg = "Failed to create the initial subscription: " + ex.getCause().getMessage();
log.warn("[{}] {} initialSubscriptionName: {}, topic: {}", remoteAddress, msg, initialSubscriptionName, topicName);
if (producerFuture.completeExceptionally(ex)) {
commandSender.sendErrorResponse(requestId, BrokerServiceException.getClientErrorCode(ex), msg);
}
producers.remove(producerId, producerFuture);
return;
}
buildProducerAndAddTopic(topic, producerId, producerName, requestId, isEncrypted, metadata, schemaVersion, epoch, userProvidedProducerName, topicName, producerAccessMode, topicEpoch, supportsPartialProducer, producerFuture);
});
}).exceptionally(exception -> {
Throwable cause = exception.getCause();
log.error("producerId {}, requestId {} : TransactionBuffer recover failed", producerId, requestId, exception);
if (producerFuture.completeExceptionally(exception)) {
commandSender.sendErrorResponse(requestId, ServiceUnitNotReadyException.getClientErrorCode(cause), cause.getMessage());
}
producers.remove(producerId, producerFuture);
return null;
});
});
});
return backlogQuotaCheckFuture;
}).exceptionally(exception -> {
Throwable cause = exception.getCause();
if (cause instanceof BrokerServiceException.TopicBacklogQuotaExceededException) {
BrokerServiceException.TopicBacklogQuotaExceededException tbqe = (BrokerServiceException.TopicBacklogQuotaExceededException) cause;
IllegalStateException illegalStateException = new IllegalStateException(tbqe);
BacklogQuota.RetentionPolicy retentionPolicy = tbqe.getRetentionPolicy();
if (producerFuture.completeExceptionally(illegalStateException)) {
if (retentionPolicy == BacklogQuota.RetentionPolicy.producer_request_hold) {
commandSender.sendErrorResponse(requestId, ServerError.ProducerBlockedQuotaExceededError, illegalStateException.getMessage());
} else if (retentionPolicy == BacklogQuota.RetentionPolicy.producer_exception) {
commandSender.sendErrorResponse(requestId, ServerError.ProducerBlockedQuotaExceededException, illegalStateException.getMessage());
}
}
producers.remove(producerId, producerFuture);
return null;
}
// Do not print stack traces for expected exceptions
if (cause instanceof NoSuchElementException) {
cause = new TopicNotFoundException("Topic Not Found.");
log.info("[{}] Failed to load topic {}, producerId={}: Topic not found", remoteAddress, topicName, producerId);
} else if (!Exceptions.areExceptionsPresentInChain(cause, ServiceUnitNotReadyException.class, ManagedLedgerException.class)) {
log.error("[{}] Failed to create topic {}, producerId={}", remoteAddress, topicName, producerId, exception);
}
// client, only if not completed already.
if (producerFuture.completeExceptionally(exception)) {
commandSender.sendErrorResponse(requestId, BrokerServiceException.getClientErrorCode(cause), cause.getMessage());
}
producers.remove(producerId, producerFuture);
return null;
});
return null;
}).exceptionally(ex -> {
logAuthException(remoteAddress, "producer", getPrincipal(), Optional.of(topicName), ex);
commandSender.sendErrorResponse(requestId, ServerError.AuthorizationError, ex.getMessage());
return null;
});
}
use of org.apache.pulsar.common.protocol.schema.SchemaData in project pulsar by apache.
the class AvroSchemaBasedCompatibilityCheck method checkCompatible.
@Override
public void checkCompatible(Iterable<SchemaData> from, SchemaData to, SchemaCompatibilityStrategy strategy) throws IncompatibleSchemaException {
LinkedList<Schema> fromList = new LinkedList<>();
checkArgument(from != null, "check compatibility list is null");
try {
for (SchemaData schemaData : from) {
Schema.Parser parser = new Schema.Parser();
parser.setValidateDefaults(false);
fromList.addFirst(parser.parse(new String(schemaData.getData(), UTF_8)));
}
Schema.Parser parser = new Schema.Parser();
parser.setValidateDefaults(false);
Schema toSchema = parser.parse(new String(to.getData(), UTF_8));
SchemaValidator schemaValidator = createSchemaValidator(strategy);
schemaValidator.validate(toSchema, fromList);
} catch (SchemaParseException e) {
log.warn("Error during schema parsing: {}", e.getMessage());
throw new IncompatibleSchemaException(e);
} catch (SchemaValidationException e) {
log.warn("Error during schema compatibility check: {}", e.getMessage());
throw new IncompatibleSchemaException(e);
}
}
Aggregations