Examples with KeyValueSchemaImpl org.apache.pulsar.client.impl.schema.KeyValueSchemaImpl used on opensource projects

Example 1 with KeyValueSchemaImpl

use of org.apache.pulsar.client.impl.schema.KeyValueSchemaImpl in project pulsar by apache.

the class SimpleSchemaTest method testAutoKeyValueConsumeGenericObject.

@Test
public void testAutoKeyValueConsumeGenericObject() throws Exception {
    String topic = "my-property/my-ns/schema-test-auto-keyvalue-consume-" + UUID.randomUUID();
    Schema<KeyValue<V1Data, V1Data>> pojoSchema = Schema.KeyValue(Schema.AVRO(V1Data.class), Schema.AVRO(V1Data.class), KeyValueEncodingType.SEPARATED);
    try (Producer<KeyValue<V1Data, V1Data>> p = pulsarClient.newProducer(pojoSchema).topic(topic).create();
        Consumer<GenericRecord> c0 = pulsarClient.newConsumer(Schema.AUTO_CONSUME()).topic(topic).subscriptionName("sub0").subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe()) {
        int numMessages = 10;
        for (int i = 0; i < numMessages; i++) {
            p.sendAsync(new KeyValue<>(new V1Data(i * 100), new V1Data(i * 1000)));
        }
        p.flush();
        // verify c0
        for (int i = 0; i < numMessages; i++) {
            Message<GenericRecord> wrapper = c0.receive();
            KeyValue<GenericRecord, GenericRecord> data = (KeyValue<GenericRecord, GenericRecord>) wrapper.getValue().getNativeObject();
            assertNotNull(wrapper.getSchemaVersion());
            assertEquals(data.getKey().getField("i"), i * 100);
            assertEquals(data.getValue().getField("i"), i * 1000);
            c0.acknowledge(wrapper);
            KeyValueSchemaImpl keyValueSchema = (KeyValueSchemaImpl) wrapper.getReaderSchema().get();
            assertNotNull(keyValueSchema.getKeySchema());
            assertNotNull(keyValueSchema.getValueSchema());
            assertTrue(keyValueSchema.getKeySchema().getSchemaInfo().getSchemaDefinition().contains("V1Data"));
            assertTrue(keyValueSchema.getValueSchema().getSchemaInfo().getSchemaDefinition().contains("V1Data"));
            assertTrue(keyValueSchema.getKeySchema().getNativeSchema().isPresent());
            assertTrue(keyValueSchema.getValueSchema().getNativeSchema().isPresent());
        }
        // new schema version
        Schema<KeyValue<V2Data, V2Data>> pojoSchemaV2 = Schema.KeyValue(Schema.AVRO(V2Data.class), Schema.AVRO(V2Data.class), KeyValueEncodingType.SEPARATED);
        try (Producer<KeyValue<V2Data, V2Data>> p2 = pulsarClient.newProducer(pojoSchemaV2).topic(topic).create()) {
            for (int i = 0; i < numMessages; i++) {
                p2.sendAsync(new KeyValue<>(new V2Data(i * 100, i), new V2Data(i * 1000, i * 20)));
            }
            p2.flush();
            // verify c0
            for (int i = 0; i < numMessages; i++) {
                Message<GenericRecord> wrapper = c0.receive();
                KeyValue<GenericRecord, GenericRecord> data = (KeyValue<GenericRecord, GenericRecord>) wrapper.getValue().getNativeObject();
                assertNotNull(wrapper.getSchemaVersion());
                assertEquals(data.getKey().getField("i"), i * 100);
                assertEquals(data.getValue().getField("i"), i * 1000);
                assertEquals(data.getKey().getField("j"), i);
                assertEquals(data.getValue().getField("j"), i * 20);
                KeyValueSchemaImpl keyValueSchema = (KeyValueSchemaImpl) wrapper.getReaderSchema().get();
                assertNotNull(keyValueSchema.getKeySchema());
                assertNotNull(keyValueSchema.getValueSchema());
                assertTrue(keyValueSchema.getKeySchema().getSchemaInfo().getSchemaDefinition().contains("V2Data"));
                assertTrue(keyValueSchema.getValueSchema().getSchemaInfo().getSchemaDefinition().contains("V2Data"));
                assertTrue(keyValueSchema.getKeySchema().getNativeSchema().isPresent());
                assertTrue(keyValueSchema.getValueSchema().getNativeSchema().isPresent());
            }
        }
    }
}

Example 2 with KeyValueSchemaImpl

use of org.apache.pulsar.client.impl.schema.KeyValueSchemaImpl in project pulsar by apache.

the class MessageImpl method getKeyValueBySchemaVersion.

private T getKeyValueBySchemaVersion() {
    KeyValueSchemaImpl kvSchema = getKeyValueSchema();
    byte[] schemaVersion = getSchemaVersion();
    if (kvSchema.getKeyValueEncodingType() == KeyValueEncodingType.SEPARATED) {
        org.apache.pulsar.common.schema.KeyValue keyValue = (org.apache.pulsar.common.schema.KeyValue) kvSchema.decode(getKeyBytes(), getData(), schemaVersion);
        if (schema instanceof AutoConsumeSchema) {
            return (T) AutoConsumeSchema.wrapPrimitiveObject(keyValue, ((AutoConsumeSchema) schema).getSchemaInfo(schemaVersion).getType(), schemaVersion);
        } else {
            return (T) keyValue;
        }
    } else {
        return decode(schemaVersion);
    }
}

Example 3 with KeyValueSchemaImpl

use of org.apache.pulsar.client.impl.schema.KeyValueSchemaImpl in project pulsar by apache.

the class SchemaTest method testKeyValueSchemaWithStructs.

private void testKeyValueSchemaWithStructs(KeyValueEncodingType keyValueEncodingType) throws Exception {
    final String tenant = PUBLIC_TENANT;
    final String namespace = "test-namespace-" + randomName(16);
    final String topicName = "test-kv-schema-" + randomName(16);
    final String topic = TopicName.get(TopicDomain.persistent.value(), tenant, namespace, topicName).toString();
    admin.namespaces().createNamespace(tenant + "/" + namespace, Sets.newHashSet(CLUSTER_NAME));
    admin.topics().createPartitionedTopic(topic, 2);
    Producer<KeyValue<Schemas.PersonOne, Schemas.PersonTwo>> producer = pulsarClient.newProducer(Schema.KeyValue(Schema.AVRO(Schemas.PersonOne.class), Schema.AVRO(Schemas.PersonTwo.class), keyValueEncodingType)).topic(topic).create();
    Consumer<KeyValue<Schemas.PersonOne, Schemas.PersonTwo>> consumer = pulsarClient.newConsumer(Schema.KeyValue(Schema.AVRO(Schemas.PersonOne.class), Schema.AVRO(Schemas.PersonTwo.class), keyValueEncodingType)).subscriptionName("test-sub").topic(topic).subscribe();
    Consumer<GenericRecord> consumer2 = // keyValueEncodingType autodetected
    pulsarClient.newConsumer(Schema.AUTO_CONSUME()).subscriptionName("test-sub2").topic(topic).subscribe();
    Schemas.PersonOne key = new Schemas.PersonOne(8787);
    Schemas.PersonTwo value = new Schemas.PersonTwo(323, "foo");
    producer.send(new KeyValue<>(key, value));
    Message<KeyValue<Schemas.PersonOne, Schemas.PersonTwo>> message = consumer.receive();
    Message<GenericRecord> message2 = consumer2.receive();
    log.info("message: {},{}", message.getValue(), message.getValue().getClass());
    log.info("message2: {},{}", message2.getValue().getNativeObject(), message2.getValue().getNativeObject().getClass());
    KeyValue<GenericRecord, GenericRecord> keyValue2 = (KeyValue<GenericRecord, GenericRecord>) message2.getValue().getNativeObject();
    assertEquals(message.getValue().getKey().id, keyValue2.getKey().getField("id"));
    assertEquals(message.getValue().getValue().id, keyValue2.getValue().getField("id"));
    assertEquals(message.getValue().getValue().name, keyValue2.getValue().getField("name"));
    Schema<?> schema = message.getReaderSchema().get();
    Schema<?> schemaFromGenericRecord = message.getReaderSchema().get();
    KeyValueSchemaImpl keyValueSchema = (KeyValueSchemaImpl) schema;
    KeyValueSchemaImpl keyValueSchemaFromGenericRecord = (KeyValueSchemaImpl) schemaFromGenericRecord;
    assertEquals(keyValueSchema.getSchemaInfo(), keyValueSchemaFromGenericRecord.getSchemaInfo());
    if (keyValueEncodingType == KeyValueEncodingType.SEPARATED) {
        // with "SEPARATED encoding the routing key is the key of the KeyValue
        assertNotNull(message.getKeyBytes());
        assertNotNull(message2.getKeyBytes());
    } else {
        assertNull(message.getKey());
        assertNull(message2.getKey());
    }
    producer.close();
    consumer.close();
    consumer2.close();
}

Example 4 with KeyValueSchemaImpl

use of org.apache.pulsar.client.impl.schema.KeyValueSchemaImpl in project pulsar by apache.

the class TypedMessageBuilderImpl method keyBytes.

@Override
public TypedMessageBuilder<T> keyBytes(byte[] key) {
    if (schema instanceof KeyValueSchemaImpl && schema.getSchemaInfo().getType() == SchemaType.KEY_VALUE) {
        KeyValueSchemaImpl kvSchema = (KeyValueSchemaImpl) schema;
        checkArgument(!(kvSchema.getKeyValueEncodingType() == KeyValueEncodingType.SEPARATED), "This method is not allowed to set keys when in encoding type is SEPARATED");
        if (key == null) {
            msgMetadata.setNullPartitionKey(true);
            return this;
        }
    }
    msgMetadata.setPartitionKey(Base64.getEncoder().encodeToString(key));
    msgMetadata.setPartitionKeyB64Encoded(true);
    return this;
}

Example 5 with KeyValueSchemaImpl

use of org.apache.pulsar.client.impl.schema.KeyValueSchemaImpl in project pulsar by apache.

the class TypedMessageBuilderImpl method value.

@Override
public TypedMessageBuilder<T> value(T value) {
    if (value == null) {
        msgMetadata.setNullValue(true);
        return this;
    }
    if (value instanceof org.apache.pulsar.common.schema.KeyValue && schema.getSchemaInfo() != null && schema.getSchemaInfo().getType() == SchemaType.KEY_VALUE) {
        KeyValueSchemaImpl kvSchema = (KeyValueSchemaImpl) schema;
        org.apache.pulsar.common.schema.KeyValue kv = (org.apache.pulsar.common.schema.KeyValue) value;
        if (kvSchema.getKeyValueEncodingType() == KeyValueEncodingType.SEPARATED) {
            // set key as the message key
            if (kv.getKey() != null) {
                msgMetadata.setPartitionKey(Base64.getEncoder().encodeToString(kvSchema.getKeySchema().encode(kv.getKey())));
                msgMetadata.setPartitionKeyB64Encoded(true);
            } else {
                this.msgMetadata.setNullPartitionKey(true);
            }
            // set value as the payload
            if (kv.getValue() != null) {
                this.content = ByteBuffer.wrap(kvSchema.getValueSchema().encode(kv.getValue()));
            } else {
                this.msgMetadata.setNullValue(true);
            }
            return this;
        }
    }
    this.content = ByteBuffer.wrap(schema.encode(value));
    return this;
}