Examples with LongSchemaVersion org.apache.pulsar.common.schema.LongSchemaVersion used on opensource projects

Example 1 with LongSchemaVersion

use of org.apache.pulsar.common.schema.LongSchemaVersion in project pulsar by apache.

the class GenericSchemaImplTest method testEncodeAndDecodeGenericRecord.

private void testEncodeAndDecodeGenericRecord(Schema<Foo> encodeSchema, Schema<GenericRecord> decodeSchema) {
    int numRecords = 10;
    for (int i = 0; i < numRecords; i++) {
        Foo foo = newFoo(i);
        byte[] data = encodeSchema.encode(foo);
        log.info("Decoding : {}", new String(data, UTF_8));
        GenericRecord record;
        if (decodeSchema instanceof AutoConsumeSchema) {
            record = decodeSchema.decode(data, new LongSchemaVersion(0L).bytes());
        } else {
            record = decodeSchema.decode(data);
        }
        verifyFooRecord(record, i);
    }
}

Example 2 with LongSchemaVersion

use of org.apache.pulsar.common.schema.LongSchemaVersion in project pulsar by apache.

the class SimpleSchemaTest method newNativeAvroProducerForMessageSchemaOnTopicWithMultiVersionSchema.

@Test
public void newNativeAvroProducerForMessageSchemaOnTopicWithMultiVersionSchema() throws Exception {
    String topic = "my-property/my-ns/schema-test";
    Schema<V1Data> v1Schema = Schema.AVRO(V1Data.class);
    byte[] v1SchemaBytes = v1Schema.getSchemaInfo().getSchema();
    org.apache.avro.Schema v1SchemaAvroNative = new Parser().parse(new ByteArrayInputStream(v1SchemaBytes));
    AvroWriter<V1Data> v1Writer = new AvroWriter<>(v1SchemaAvroNative);
    Schema<V2Data> v2Schema = Schema.AVRO(V2Data.class);
    byte[] v2SchemaBytes = v2Schema.getSchemaInfo().getSchema();
    org.apache.avro.Schema v2SchemaAvroNative = new Parser().parse(new ByteArrayInputStream(v2SchemaBytes));
    AvroWriter<V2Data> v2Writer = new AvroWriter<>(v2SchemaAvroNative);
    V1Data dataV1 = new V1Data(2);
    V2Data dataV2 = new V2Data(3, 5);
    byte[] contentV1 = v1Writer.write(dataV1);
    byte[] contentV2 = v2Writer.write(dataV2);
    try (Producer<byte[]> ignored = pulsarClient.newProducer(Schema.NATIVE_AVRO(v1SchemaAvroNative)).topic(topic).create()) {
    }
    try (Producer<byte[]> p = pulsarClient.newProducer(Schema.NATIVE_AVRO(v2SchemaAvroNative)).topic(topic).create()) {
        p.send(contentV2);
    }
    try (Producer<byte[]> p = pulsarClient.newProducer(Schema.NATIVE_AVRO(v1SchemaAvroNative)).topic(topic).create();
        Consumer<V2Data> c = pulsarClient.newConsumer(v2Schema).topic(topic).subscriptionName("sub1").subscribe()) {
        p.newMessage(Schema.NATIVE_AVRO(v1SchemaAvroNative)).value(contentV1).send();
        p.newMessage(Schema.NATIVE_AVRO(v2SchemaAvroNative)).value(contentV2).send();
        Message<V2Data> msg1 = c.receive();
        V2Data msg1Value = msg1.getValue();
        Assert.assertEquals(dataV1.i, msg1Value.i);
        assertNull(msg1Value.j);
        Assert.assertEquals(msg1.getSchemaVersion(), new LongSchemaVersion(0).bytes());
        Message<V2Data> msg2 = c.receive();
        Assert.assertEquals(dataV2, msg2.getValue());
        Assert.assertEquals(msg2.getSchemaVersion(), new LongSchemaVersion(1).bytes());
        try {
            p.newMessage(Schema.BYTES).value(contentV1).send();
            if (schemaValidationEnforced) {
                Assert.fail("Shouldn't be able to send to a schema'd topic with no schema" + " if SchemaValidationEnabled is enabled");
            }
            Message<V2Data> msg3 = c.receive();
            Assert.assertEquals(msg3.getSchemaVersion(), SchemaVersion.Empty.bytes());
        } catch (PulsarClientException e) {
            if (schemaValidationEnforced) {
                Assert.assertTrue(e instanceof IncompatibleSchemaException);
            } else {
                Assert.fail("Shouldn't throw IncompatibleSchemaException" + " if SchemaValidationEnforced is disabled");
            }
        }
    }
}

Example 3 with LongSchemaVersion

use of org.apache.pulsar.common.schema.LongSchemaVersion in project pulsar by apache.

the class BookkeeperSchemaStorage method getSchema.

@NotNull
private CompletableFuture<StoredSchema> getSchema(String schemaId) {
    // There's already a schema read operation in progress. Just piggyback on that
    return readSchemaOperations.computeIfAbsent(schemaId, key -> {
        if (log.isDebugEnabled()) {
            log.debug("[{}] Fetching schema from store", schemaId);
        }
        CompletableFuture<StoredSchema> future = new CompletableFuture<>();
        getSchemaLocator(getSchemaPath(schemaId)).thenCompose(locator -> {
            if (log.isDebugEnabled()) {
                log.debug("[{}] Got schema locator {}", schemaId, locator);
            }
            if (!locator.isPresent()) {
                return completedFuture(null);
            }
            SchemaStorageFormat.SchemaLocator schemaLocator = locator.get().locator;
            return readSchemaEntry(schemaLocator.getInfo().getPosition()).thenApply(entry -> new StoredSchema(entry.getSchemaData().toByteArray(), new LongSchemaVersion(schemaLocator.getInfo().getVersion())));
        }).handleAsync((res, ex) -> {
            if (log.isDebugEnabled()) {
                log.debug("[{}] Get operation completed. res={} -- ex={}", schemaId, res, ex);
            }
            // Cleanup the pending ops from the map
            readSchemaOperations.remove(schemaId, future);
            if (ex != null) {
                future.completeExceptionally(ex);
            } else {
                future.complete(res);
            }
            return null;
        });
        return future;
    });
}

Example 4 with LongSchemaVersion

use of org.apache.pulsar.common.schema.LongSchemaVersion in project pulsar by apache.

the class GenericSchemaTest method testEncodeAndDecodeKeyValues.

private void testEncodeAndDecodeKeyValues(Schema<KeyValue<Foo, Foo>> encodeSchema, Schema<KeyValue<GenericRecord, GenericRecord>> decodeSchema) {
    int numRecords = 10;
    for (int i = 0; i < numRecords; i++) {
        Foo foo = newFoo(i);
        byte[] data = encodeSchema.encode(new KeyValue<>(foo, foo));
        KeyValue<GenericRecord, GenericRecord> kv = decodeSchema.decode(data, new LongSchemaVersion(1L).bytes());
        verifyFooRecord(kv.getKey(), i);
        verifyFooRecord(kv.getValue(), i);
    }
}

Example 5 with LongSchemaVersion

use of org.apache.pulsar.common.schema.LongSchemaVersion in project pulsar by apache.

the class TestPulsarRecordCursor method testGetSchemaInfo.

@Test
public void testGetSchemaInfo() throws Exception {
    String topic = "get-schema-test";
    PulsarSplit pulsarSplit = Mockito.mock(PulsarSplit.class);
    Mockito.when(pulsarSplit.getTableName()).thenReturn(TopicName.get(topic).getLocalName());
    Mockito.when(pulsarSplit.getSchemaName()).thenReturn("public/default");
    PulsarAdmin pulsarAdmin = Mockito.mock(PulsarAdmin.class);
    Schemas schemas = Mockito.mock(Schemas.class);
    Mockito.when(pulsarAdmin.schemas()).thenReturn(schemas);
    PulsarConnectorConfig connectorConfig = spy(PulsarConnectorConfig.class);
    Mockito.when(connectorConfig.getPulsarAdmin()).thenReturn(pulsarAdmin);
    PulsarRecordCursor pulsarRecordCursor = spy(new PulsarRecordCursor(new ArrayList<>(), pulsarSplit, connectorConfig, Mockito.mock(ManagedLedgerFactory.class), new ManagedLedgerConfig(), null, null));
    Class<PulsarRecordCursor> clazz = PulsarRecordCursor.class;
    Method getSchemaInfo = clazz.getDeclaredMethod("getSchemaInfo", PulsarSplit.class);
    getSchemaInfo.setAccessible(true);
    Field currentMessage = clazz.getDeclaredField("currentMessage");
    currentMessage.setAccessible(true);
    RawMessage rawMessage = Mockito.mock(RawMessage.class);
    currentMessage.set(pulsarRecordCursor, rawMessage);
    // If the schemaType of pulsarSplit is NONE or BYTES, using bytes schema
    Mockito.when(pulsarSplit.getSchemaType()).thenReturn(SchemaType.NONE);
    SchemaInfo schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
    assertEquals(SchemaType.BYTES, schemaInfo.getType());
    Mockito.when(pulsarSplit.getSchemaType()).thenReturn(SchemaType.BYTES);
    schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
    assertEquals(SchemaType.BYTES, schemaInfo.getType());
    Mockito.when(pulsarSplit.getSchemaName()).thenReturn(Schema.BYTEBUFFER.getSchemaInfo().getName());
    schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
    assertEquals(SchemaType.BYTES, schemaInfo.getType());
    // If the schemaVersion of the message is not null, try to get the schema.
    Mockito.when(pulsarSplit.getSchemaType()).thenReturn(SchemaType.AVRO);
    Mockito.when(rawMessage.getSchemaVersion()).thenReturn(new LongSchemaVersion(0).bytes());
    Mockito.when(schemas.getSchemaInfo(anyString(), eq(0L))).thenReturn(Schema.AVRO(Foo.class).getSchemaInfo());
    schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
    assertEquals(SchemaType.AVRO, schemaInfo.getType());
    String schemaTopic = "persistent://public/default/" + topic;
    // If the schemaVersion of the message is null and the schema of pulsarSplit is null, throw runtime exception.
    Mockito.when(pulsarSplit.getSchemaInfo()).thenReturn(null);
    Mockito.when(rawMessage.getSchemaVersion()).thenReturn(null);
    try {
        schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
        fail("The message schema version is null and the latest schema is null, should fail.");
    } catch (InvocationTargetException e) {
        assertTrue(e.getCause() instanceof RuntimeException);
        assertTrue(e.getCause().getMessage().contains("schema of the table " + topic + " is null"));
    }
    // If the schemaVersion of the message is null, try to get the latest schema.
    Mockito.when(rawMessage.getSchemaVersion()).thenReturn(null);
    Mockito.when(pulsarSplit.getSchemaInfo()).thenReturn(Schema.AVRO(Foo.class).getSchemaInfo());
    schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
    assertEquals(Schema.AVRO(Foo.class).getSchemaInfo(), schemaInfo);
    // If the specific version schema is null, throw runtime exception.
    Mockito.when(rawMessage.getSchemaVersion()).thenReturn(new LongSchemaVersion(1L).bytes());
    Mockito.when(schemas.getSchemaInfo(schemaTopic, 1)).thenReturn(null);
    try {
        schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
        fail("The specific version " + 1 + " schema is null, should fail.");
    } catch (InvocationTargetException e) {
        String schemaVersion = BytesSchemaVersion.of(new LongSchemaVersion(1L).bytes()).toString();
        assertTrue(e.getCause() instanceof RuntimeException);
        assertTrue(e.getCause().getMessage().contains("schema of the topic " + schemaTopic + " is null"));
    }
    // Get the specific version schema.
    Mockito.when(rawMessage.getSchemaVersion()).thenReturn(new LongSchemaVersion(2L).bytes());
    Mockito.when(schemas.getSchemaInfo(schemaTopic, 2)).thenReturn(Schema.AVRO(Foo.class).getSchemaInfo());
    schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
    assertEquals(Schema.AVRO(Foo.class).getSchemaInfo(), schemaInfo);
}