Examples with AvroWriter org.apache.pulsar.client.impl.schema.writer.AvroWriter used on opensource projects

Example 1 with AvroWriter

use of org.apache.pulsar.client.impl.schema.writer.AvroWriter in project pulsar by apache.

the class SimpleSchemaTest method newNativeAvroProducerForMessageSchemaOnTopicWithMultiVersionSchema.

@Test
public void newNativeAvroProducerForMessageSchemaOnTopicWithMultiVersionSchema() throws Exception {
    String topic = "my-property/my-ns/schema-test";
    Schema<V1Data> v1Schema = Schema.AVRO(V1Data.class);
    byte[] v1SchemaBytes = v1Schema.getSchemaInfo().getSchema();
    org.apache.avro.Schema v1SchemaAvroNative = new Parser().parse(new ByteArrayInputStream(v1SchemaBytes));
    AvroWriter<V1Data> v1Writer = new AvroWriter<>(v1SchemaAvroNative);
    Schema<V2Data> v2Schema = Schema.AVRO(V2Data.class);
    byte[] v2SchemaBytes = v2Schema.getSchemaInfo().getSchema();
    org.apache.avro.Schema v2SchemaAvroNative = new Parser().parse(new ByteArrayInputStream(v2SchemaBytes));
    AvroWriter<V2Data> v2Writer = new AvroWriter<>(v2SchemaAvroNative);
    V1Data dataV1 = new V1Data(2);
    V2Data dataV2 = new V2Data(3, 5);
    byte[] contentV1 = v1Writer.write(dataV1);
    byte[] contentV2 = v2Writer.write(dataV2);
    try (Producer<byte[]> ignored = pulsarClient.newProducer(Schema.NATIVE_AVRO(v1SchemaAvroNative)).topic(topic).create()) {
    }
    try (Producer<byte[]> p = pulsarClient.newProducer(Schema.NATIVE_AVRO(v2SchemaAvroNative)).topic(topic).create()) {
        p.send(contentV2);
    }
    try (Producer<byte[]> p = pulsarClient.newProducer(Schema.NATIVE_AVRO(v1SchemaAvroNative)).topic(topic).create();
        Consumer<V2Data> c = pulsarClient.newConsumer(v2Schema).topic(topic).subscriptionName("sub1").subscribe()) {
        p.newMessage(Schema.NATIVE_AVRO(v1SchemaAvroNative)).value(contentV1).send();
        p.newMessage(Schema.NATIVE_AVRO(v2SchemaAvroNative)).value(contentV2).send();
        Message<V2Data> msg1 = c.receive();
        V2Data msg1Value = msg1.getValue();
        Assert.assertEquals(dataV1.i, msg1Value.i);
        assertNull(msg1Value.j);
        Assert.assertEquals(msg1.getSchemaVersion(), new LongSchemaVersion(0).bytes());
        Message<V2Data> msg2 = c.receive();
        Assert.assertEquals(dataV2, msg2.getValue());
        Assert.assertEquals(msg2.getSchemaVersion(), new LongSchemaVersion(1).bytes());
        try {
            p.newMessage(Schema.BYTES).value(contentV1).send();
            if (schemaValidationEnforced) {
                Assert.fail("Shouldn't be able to send to a schema'd topic with no schema" + " if SchemaValidationEnabled is enabled");
            }
            Message<V2Data> msg3 = c.receive();
            Assert.assertEquals(msg3.getSchemaVersion(), SchemaVersion.Empty.bytes());
        } catch (PulsarClientException e) {
            if (schemaValidationEnforced) {
                Assert.assertTrue(e instanceof IncompatibleSchemaException);
            } else {
                Assert.fail("Shouldn't throw IncompatibleSchemaException" + " if SchemaValidationEnforced is disabled");
            }
        }
    }
}

Example 2 with AvroWriter

use of org.apache.pulsar.client.impl.schema.writer.AvroWriter in project pulsar by yahoo.

the class SimpleProducerConsumerTest method testNativeAvroSendCompressedWithDeferredSchemaSetup.

@Test(timeOut = 100000, dataProvider = "enableBatching")
public void testNativeAvroSendCompressedWithDeferredSchemaSetup(boolean enableBatching) throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String topic = "persistent://my-property/my-ns/deferredSchemaCompressed";
    Consumer<GenericRecord> consumer = pulsarClient.newConsumer(Schema.AUTO_CONSUME()).topic(topic).subscriptionName("testsub").subscribe();
    // initially we are not setting a Schema in the producer
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topic).enableBatching(enableBatching).compressionType(CompressionType.LZ4).create();
    MyBean payload = new MyBean();
    payload.setField("aaaaaaaaaaaaaaaaaaaaaaaaa");
    // now we send with a schema, but we have enabled compression and batching
    // the producer will have to setup the schema and resume the send
    Schema<MyBean> myBeanSchema = Schema.AVRO(MyBean.class);
    byte[] schemaBytes = myBeanSchema.getSchemaInfo().getSchema();
    org.apache.avro.Schema schemaAvroNative = new Parser().parse(new ByteArrayInputStream(schemaBytes));
    AvroWriter<MyBean> writer = new AvroWriter<>(schemaAvroNative);
    byte[] content = writer.write(payload);
    producer.newMessage(Schema.NATIVE_AVRO(schemaAvroNative)).value(content).send();
    producer.close();
    GenericRecord res = consumer.receive(RECEIVE_TIMEOUT_SECONDS, TimeUnit.SECONDS).getValue();
    consumer.close();
    assertEquals(SchemaType.AVRO, res.getSchemaType());
    org.apache.avro.generic.GenericRecord nativeRecord = (org.apache.avro.generic.GenericRecord) res.getNativeObject();
    org.apache.avro.Schema schema = nativeRecord.getSchema();
    for (org.apache.pulsar.client.api.schema.Field f : res.getFields()) {
        log.info("field {} {}", f.getName(), res.getField(f));
        assertEquals("field", f.getName());
        assertEquals("aaaaaaaaaaaaaaaaaaaaaaaaa", res.getField(f));
        assertEquals("aaaaaaaaaaaaaaaaaaaaaaaaa", nativeRecord.get(f.getName()).toString());
    }
    assertEquals(1, res.getFields().size());
}

Example 3 with AvroWriter

use of org.apache.pulsar.client.impl.schema.writer.AvroWriter in project pulsar by yahoo.

the class SimpleSchemaTest method newNativeAvroProducerForMessageSchemaWithBatch.

@Test
public void newNativeAvroProducerForMessageSchemaWithBatch() throws Exception {
    String topic = "my-property/my-ns/schema-test";
    Schema<V1Data> v1Schema = Schema.AVRO(V1Data.class);
    byte[] v1SchemaBytes = v1Schema.getSchemaInfo().getSchema();
    org.apache.avro.Schema v1SchemaAvroNative = new Parser().parse(new ByteArrayInputStream(v1SchemaBytes));
    AvroWriter<V1Data> v1Writer = new AvroWriter<>(v1SchemaAvroNative);
    Schema<V2Data> v2Schema = Schema.AVRO(V2Data.class);
    byte[] v2SchemaBytes = v2Schema.getSchemaInfo().getSchema();
    org.apache.avro.Schema v2SchemaAvroNative = new Parser().parse(new ByteArrayInputStream(v2SchemaBytes));
    AvroWriter<V2Data> v2Writer = new AvroWriter<>(v2SchemaAvroNative);
    Consumer<byte[]> c = pulsarClient.newConsumer(Schema.BYTES).topic(topic).subscriptionName("sub1").subscribe();
    Producer<byte[]> p = pulsarClient.newProducer(Schema.NATIVE_AVRO(v1SchemaAvroNative)).topic(topic).enableBatching(true).batchingMaxPublishDelay(10, TimeUnit.SECONDS).create();
    AvroWriter<V1Data> v1DataAvroWriter = new AvroWriter<>(ReflectData.AllowNull.get().getSchema(V1Data.class));
    AvroWriter<V2Data> v2DataAvroWriter = new AvroWriter<>(ReflectData.AllowNull.get().getSchema(V2Data.class));
    AvroWriter<IncompatibleData> incompatibleDataAvroWriter = new AvroWriter<>(ReflectData.AllowNull.get().getSchema(IncompatibleData.class));
    int total = 20;
    int batch = 5;
    int incompatible = 3;
    for (int i = 0; i < total; ++i) {
        if (i / batch % 2 == 0) {
            byte[] content = v1DataAvroWriter.write(new V1Data(i));
            p.newMessage(Schema.NATIVE_AVRO(v1SchemaAvroNative)).value(content).sendAsync();
        } else {
            byte[] content = v2DataAvroWriter.write(new V2Data(i, i + total));
            p.newMessage(Schema.NATIVE_AVRO(v2SchemaAvroNative)).value(content).sendAsync();
        }
        if ((i + 1) % incompatible == 0) {
            Schema<IncompatibleData> incompatibleSchema = Schema.AVRO(IncompatibleData.class);
            byte[] incompatibleSchemaBytes = incompatibleSchema.getSchemaInfo().getSchema();
            org.apache.avro.Schema incompatibleSchemaAvroNative = new Parser().parse(new ByteArrayInputStream(incompatibleSchemaBytes));
            byte[] content = incompatibleDataAvroWriter.write(new IncompatibleData(-i, -i));
            try {
                p.newMessage(Schema.NATIVE_AVRO(incompatibleSchemaAvroNative)).value(content).send();
            } catch (Exception e) {
                Assert.assertTrue(e instanceof IncompatibleSchemaException, e.getMessage());
            }
        }
    }
    p.flush();
    for (int i = 0; i < total; ++i) {
        byte[] raw = c.receive().getData();
        if (i / batch % 2 == 0) {
            AvroReader<V1Data> reader = new AvroReader<>(v1SchemaAvroNative);
            V1Data value = reader.read(raw);
            Assert.assertEquals(value.i, i);
        } else {
            AvroReader<V2Data> reader = new AvroReader<>(v2SchemaAvroNative);
            V2Data value = reader.read(raw);
            Assert.assertEquals(value, new V2Data(i, i + total));
        }
    }
    c.close();
}

Example 4 with AvroWriter

use of org.apache.pulsar.client.impl.schema.writer.AvroWriter in project pulsar by yahoo.

the class SimpleSchemaTest method newNativeAvroProducerForMessageSchemaOnTopicWithMultiVersionSchema.

@Test
public void newNativeAvroProducerForMessageSchemaOnTopicWithMultiVersionSchema() throws Exception {
    String topic = "my-property/my-ns/schema-test";
    Schema<V1Data> v1Schema = Schema.AVRO(V1Data.class);
    byte[] v1SchemaBytes = v1Schema.getSchemaInfo().getSchema();
    org.apache.avro.Schema v1SchemaAvroNative = new Parser().parse(new ByteArrayInputStream(v1SchemaBytes));
    AvroWriter<V1Data> v1Writer = new AvroWriter<>(v1SchemaAvroNative);
    Schema<V2Data> v2Schema = Schema.AVRO(V2Data.class);
    byte[] v2SchemaBytes = v2Schema.getSchemaInfo().getSchema();
    org.apache.avro.Schema v2SchemaAvroNative = new Parser().parse(new ByteArrayInputStream(v2SchemaBytes));
    AvroWriter<V2Data> v2Writer = new AvroWriter<>(v2SchemaAvroNative);
    V1Data dataV1 = new V1Data(2);
    V2Data dataV2 = new V2Data(3, 5);
    byte[] contentV1 = v1Writer.write(dataV1);
    byte[] contentV2 = v2Writer.write(dataV2);
    try (Producer<byte[]> ignored = pulsarClient.newProducer(Schema.NATIVE_AVRO(v1SchemaAvroNative)).topic(topic).create()) {
    }
    try (Producer<byte[]> p = pulsarClient.newProducer(Schema.NATIVE_AVRO(v2SchemaAvroNative)).topic(topic).create()) {
        p.send(contentV2);
    }
    try (Producer<byte[]> p = pulsarClient.newProducer(Schema.NATIVE_AVRO(v1SchemaAvroNative)).topic(topic).create();
        Consumer<V2Data> c = pulsarClient.newConsumer(v2Schema).topic(topic).subscriptionName("sub1").subscribe()) {
        p.newMessage(Schema.NATIVE_AVRO(v1SchemaAvroNative)).value(contentV1).send();
        p.newMessage(Schema.NATIVE_AVRO(v2SchemaAvroNative)).value(contentV2).send();
        Message<V2Data> msg1 = c.receive();
        V2Data msg1Value = msg1.getValue();
        Assert.assertEquals(dataV1.i, msg1Value.i);
        assertNull(msg1Value.j);
        Assert.assertEquals(msg1.getSchemaVersion(), new LongSchemaVersion(0).bytes());
        Message<V2Data> msg2 = c.receive();
        Assert.assertEquals(dataV2, msg2.getValue());
        Assert.assertEquals(msg2.getSchemaVersion(), new LongSchemaVersion(1).bytes());
        try {
            p.newMessage(Schema.BYTES).value(contentV1).send();
            if (schemaValidationEnforced) {
                Assert.fail("Shouldn't be able to send to a schema'd topic with no schema" + " if SchemaValidationEnabled is enabled");
            }
            Message<V2Data> msg3 = c.receive();
            assertNull(msg3.getSchemaVersion());
            try {
                msg3.getValue();
                fail("Schema should be incompatible");
            } catch (SchemaSerializationException e) {
                assertTrue(e.getCause() instanceof EOFException);
            }
        } catch (PulsarClientException e) {
            if (schemaValidationEnforced) {
                Assert.assertTrue(e instanceof IncompatibleSchemaException);
            } else {
                Assert.fail("Shouldn't throw IncompatibleSchemaException" + " if SchemaValidationEnforced is disabled");
            }
        }
    }
}

Example 5 with AvroWriter

use of org.apache.pulsar.client.impl.schema.writer.AvroWriter in project incubator-pulsar by apache.

the class SchemaTest method testProducerMultipleSchemaMessages.

@Test
public void testProducerMultipleSchemaMessages() throws Exception {
    final String tenant = PUBLIC_TENANT;
    final String namespace = "test-namespace-" + randomName(16);
    final String topicName = "auto_schema_test";
    String ns = tenant + "/" + namespace;
    admin.namespaces().createNamespace(ns, Sets.newHashSet(CLUSTER_NAME));
    admin.namespaces().setSchemaCompatibilityStrategy(ns, SchemaCompatibilityStrategy.ALWAYS_COMPATIBLE);
    final String topic = TopicName.get(TopicDomain.persistent.value(), tenant, namespace, topicName).toString();
    @Cleanup Producer<byte[]> producer = pulsarClient.newProducer(Schema.AUTO_PRODUCE_BYTES()).topic(topic).create();
    producer.newMessage(Schema.STRING).value("test").send();
    producer.newMessage(Schema.JSON(Schemas.PersonThree.class)).value(new Schemas.PersonThree(0, "ran")).send();
    producer.newMessage(Schema.AVRO(Schemas.PersonThree.class)).value(new Schemas.PersonThree(0, "ran")).send();
    producer.newMessage(Schema.AVRO(Schemas.PersonOne.class)).value(new Schemas.PersonOne(0)).send();
    producer.newMessage(Schema.JSON(Schemas.PersonThree.class)).value(new Schemas.PersonThree(1, "tang")).send();
    producer.newMessage(Schema.BYTES).value("test".getBytes(StandardCharsets.UTF_8)).send();
    producer.newMessage(Schema.BYTES).value("test".getBytes(StandardCharsets.UTF_8)).send();
    producer.newMessage(Schema.BOOL).value(true).send();
    Schema<Schemas.PersonThree> personThreeSchema = Schema.AVRO(Schemas.PersonThree.class);
    byte[] personThreeSchemaBytes = personThreeSchema.getSchemaInfo().getSchema();
    org.apache.avro.Schema personThreeSchemaAvroNative = new Parser().parse(new ByteArrayInputStream(personThreeSchemaBytes));
    AvroWriter<Schemas.PersonThree> writer = new AvroWriter<>(personThreeSchemaAvroNative);
    byte[] content = writer.write(new Schemas.PersonThree(0, "ran"));
    producer.newMessage(Schema.NATIVE_AVRO(personThreeSchemaAvroNative)).value(content).send();
    List<SchemaInfo> allSchemas = admin.schemas().getAllSchemas(topic);
    Assert.assertEquals(allSchemas.size(), 5);
    Assert.assertEquals(allSchemas.get(0), Schema.STRING.getSchemaInfo());
    Assert.assertEquals(allSchemas.get(1), Schema.JSON(Schemas.PersonThree.class).getSchemaInfo());
    Assert.assertEquals(allSchemas.get(2), Schema.AVRO(Schemas.PersonThree.class).getSchemaInfo());
    Assert.assertEquals(allSchemas.get(3), Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo());
    Assert.assertEquals(allSchemas.get(4), Schema.BOOL.getSchemaInfo());
}