Examples with GenericAvroRecord org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord used on opensource projects

Example 6 with GenericAvroRecord

use of org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord in project pulsar-flink by streamnative.

the class PulsarSerializer method newConverter.

private BiFunction<PositionedGetter, Integer, Object> newConverter(DataType dataType, Schema avroType) throws IncompatibleSchemaException {
    LogicalTypeRoot tpe = dataType.getLogicalType().getTypeRoot();
    Schema.Type atpe = avroType.getType();
    if (tpe == LogicalTypeRoot.NULL && atpe == Schema.Type.NULL) {
        return (getter, ordinal) -> null;
    } else if ((tpe == LogicalTypeRoot.BOOLEAN && atpe == Schema.Type.BOOLEAN) || (tpe == LogicalTypeRoot.TINYINT && atpe == Schema.Type.INT) || (tpe == LogicalTypeRoot.SMALLINT && atpe == Schema.Type.INT) || (tpe == LogicalTypeRoot.INTEGER && atpe == Schema.Type.INT) || (tpe == LogicalTypeRoot.BIGINT && atpe == Schema.Type.LONG) || (tpe == LogicalTypeRoot.FLOAT && atpe == Schema.Type.FLOAT) || (tpe == LogicalTypeRoot.DOUBLE && atpe == Schema.Type.DOUBLE) || (tpe == LogicalTypeRoot.VARBINARY && atpe == Schema.Type.BYTES)) {
        return (getter, ordinal) -> getter.getField(ordinal);
    } else if (tpe == LogicalTypeRoot.DECIMAL && (atpe == Schema.Type.FIXED || atpe == Schema.Type.BYTES)) {
        DecimalType d = (DecimalType) dataType.getLogicalType();
        if (avroType.getLogicalType() == LogicalTypes.decimal(d.getPrecision(), d.getScale())) {
            return (getter, ordinal) -> {
                java.math.BigDecimal decimal = (java.math.BigDecimal) getter.getField(ordinal);
                return decimalConversion.toFixed(decimal, avroType, LogicalTypes.decimal(d.getPrecision(), d.getScale()));
            };
        } else {
            throw new IncompatibleSchemaException("Cannot convert flink decimal type to Avro logical type");
        }
    } else if (tpe == LogicalTypeRoot.BIGINT && atpe == Schema.Type.BYTES) {
        return (getter, ordinal) -> ByteBuffer.wrap((byte[]) getter.getField(ordinal));
    } else if (tpe == LogicalTypeRoot.DATE && atpe == Schema.Type.INT) {
        return (getter, ordinal) -> ((LocalDate) getter.getField(ordinal)).toEpochDay();
    } else if (tpe == LogicalTypeRoot.TIMESTAMP_WITH_TIME_ZONE && atpe == Schema.Type.LONG) {
        LogicalType altpe = avroType.getLogicalType();
        if (altpe instanceof LogicalTypes.TimestampMillis) {
            return (getter, ordinal) -> DateTimeUtils.fromJavaTimestamp(java.sql.Timestamp.valueOf((LocalDateTime) getter.getField(ordinal))) / 1000;
        } else if (altpe instanceof LogicalTypes.TimestampMicros) {
            return (getter, ordinal) -> DateTimeUtils.fromJavaTimestamp(java.sql.Timestamp.valueOf((LocalDateTime) getter.getField(ordinal)));
        } else {
            throw new IncompatibleSchemaException("Cannot convert flink timestamp to avro logical type " + altpe.toString());
        }
    } else if (tpe == LogicalTypeRoot.VARCHAR && atpe == Schema.Type.STRING) {
        return (getter, ordinal) -> new Utf8((String) getter.getField(ordinal));
    } else if (tpe == LogicalTypeRoot.VARCHAR && atpe == Schema.Type.ENUM) {
        HashSet<String> enumSymbols = new HashSet<>(avroType.getEnumSymbols());
        return (getter, ordinal) -> {
            String data = (String) getter.getField(ordinal);
            if (!enumSymbols.contains(data)) {
                throw new IllegalArgumentException(String.format("Cannot write %s since it's not defined in enum %s", data, String.join(", ", enumSymbols)));
            }
            return new GenericData.EnumSymbol(avroType, data);
        };
    } else if (tpe == LogicalTypeRoot.ARRAY && atpe == Schema.Type.ARRAY && dataType instanceof CollectionDataType) {
        DataType et = ((CollectionDataType) dataType).getElementDataType();
        boolean containsNull = et.getLogicalType().isNullable();
        BiFunction<PositionedGetter, Integer, Object> elementConverter = newConverter(et, resolveNullableType(avroType.getElementType(), containsNull));
        return (getter, ordinal) -> {
            Object[] arrayData = (Object[]) getter.getField(ordinal);
            int len = arrayData.length;
            Object[] result = new Object[len];
            for (int i = 0; i < len; i++) {
                if (containsNull && arrayData[i] == null) {
                    result[i] = null;
                } else {
                    result[i] = elementConverter.apply(new PositionedGetter(arrayData), i);
                }
            }
            // `ArrayList` backed by the specified array without data copying.
            return Arrays.asList(result);
        };
    } else if (tpe == LogicalTypeRoot.MAP && atpe == Schema.Type.MAP && ((KeyValueDataType) dataType).getKeyDataType().getLogicalType().getTypeRoot() == LogicalTypeRoot.VARCHAR) {
        return (getter, ordinal) -> getter.getField(ordinal);
    } else if (tpe == LogicalTypeRoot.ROW && atpe == Schema.Type.RECORD) {
        FieldsDataType st = (FieldsDataType) dataType;
        Function<Object, Object> structConverter = newStructConverter(st, avroType);
        return (getter, ordinal) -> ((GenericAvroRecord) structConverter.apply(getter.getField(ordinal))).getAvroRecord();
    } else {
        throw new IncompatibleSchemaException(String.format("Cannot convert flink type %s to avro type %s", dataType.toString(), avroType.toString(true)));
    }
}

Example 7 with GenericAvroRecord

use of org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord in project pulsar by apache.

the class TestAvroDecoder method testCompositeType.

@Test
public void testCompositeType() {
    DecoderTestMessage message = new DecoderTestMessage();
    DecoderTestMessage.NestedRow nestedRow = new DecoderTestMessage.NestedRow();
    nestedRow.longField = 222L;
    nestedRow.stringField = "message_2_nestedRow";
    DecoderTestMessage.CompositeRow compositeRow = new DecoderTestMessage.CompositeRow();
    DecoderTestMessage.NestedRow nestedRow1 = new DecoderTestMessage.NestedRow();
    nestedRow1.longField = 2;
    nestedRow1.stringField = "nestedRow_1";
    DecoderTestMessage.NestedRow nestedRow2 = new DecoderTestMessage.NestedRow();
    nestedRow2.longField = 2;
    nestedRow2.stringField = "nestedRow_2";
    compositeRow.arrayField = Arrays.asList(nestedRow1, nestedRow2);
    compositeRow.stringField = "compositeRow_1";
    compositeRow.mapField = new HashMap<String, DecoderTestMessage.NestedRow>() {

        {
            put("key1", nestedRow1);
            put("key2", nestedRow2);
        }
    };
    compositeRow.nestedRow = nestedRow;
    new HashMap<String, Long>() {

        {
            put("key1_1", 2L);
            put("key1_2", 22L);
        }
    };
    compositeRow.structedField = new HashMap<String, List<Long>>() {

        {
            put("key2_1", Arrays.asList(2L, 3L));
            put("key2_2", Arrays.asList(2L, 3L));
            put("key2_3", Arrays.asList(2L, 3L));
        }
    };
    message.compositeRow = compositeRow;
    byte[] bytes = schema.encode(message);
    ByteBuf payload = io.netty.buffer.Unpooled.copiedBuffer(bytes);
    GenericAvroRecord genericRecord = (GenericAvroRecord) GenericAvroSchema.of(schemaInfo).decode(bytes);
    Object fieldValue = genericRecord.getAvroRecord().get("compositeRow");
    Map<DecoderColumnHandle, FieldValueProvider> decodedRow = pulsarRowDecoder.decodeRow(payload).get();
    RowType columnType = RowType.from(ImmutableList.<RowType.Field>builder().add(RowType.field("arrayField", new ArrayType(RowType.from(ImmutableList.<RowType.Field>builder().add(RowType.field("longField", BIGINT)).add(RowType.field("stringField", VARCHAR)).build())))).add(RowType.field("mapField", decoderFactory.getTypeManager().getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.typeParameter(VarcharType.VARCHAR.getTypeSignature()), TypeSignatureParameter.typeParameter(RowType.from(ImmutableList.<RowType.Field>builder().add(RowType.field("longField", BIGINT)).add(RowType.field("stringField", VARCHAR)).build()).getTypeSignature()))))).add(RowType.field("nestedRow", RowType.from(ImmutableList.<RowType.Field>builder().add(RowType.field("longField", BIGINT)).add(RowType.field("stringField", VARCHAR)).build()))).add(RowType.field("stringField", VARCHAR)).add(RowType.field("structedField", decoderFactory.getTypeManager().getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.typeParameter(VarcharType.VARCHAR.getTypeSignature()), TypeSignatureParameter.typeParameter(new ArrayType(BIGINT).getTypeSignature()))))).build());
    PulsarColumnHandle columnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), "compositeRow", columnType, false, false, "compositeRow", null, null, PulsarColumnHandle.HandleKeyValueType.NONE);
    checkRowValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue);
}

Example 8 with GenericAvroRecord

use of org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord in project pulsar by apache.

the class TopicsBase method encodeWithSchema.

// Encode message with corresponding schema, do necessary conversion before encoding
private byte[] encodeWithSchema(String input, Schema schema) {
    try {
        switch(schema.getSchemaInfo().getType()) {
            case INT8:
                return schema.encode(Byte.parseByte(input));
            case INT16:
                return schema.encode(Short.parseShort(input));
            case INT32:
                return schema.encode(Integer.parseInt(input));
            case INT64:
                return schema.encode(Long.parseLong(input));
            case STRING:
                return schema.encode(input);
            case FLOAT:
                return schema.encode(Float.parseFloat(input));
            case DOUBLE:
                return schema.encode(Double.parseDouble(input));
            case BOOLEAN:
                return schema.encode(Boolean.parseBoolean(input));
            case BYTES:
                return schema.encode(input.getBytes());
            case DATE:
                return schema.encode(DateFormat.getDateInstance().parse(input));
            case TIME:
                return schema.encode(new Time(Long.parseLong(input)));
            case TIMESTAMP:
                return schema.encode(new Timestamp(Long.parseLong(input)));
            case INSTANT:
                return schema.encode(Instant.parse(input));
            case LOCAL_DATE:
                return schema.encode(LocalDate.parse(input));
            case LOCAL_TIME:
                return schema.encode(LocalTime.parse(input));
            case LOCAL_DATE_TIME:
                return schema.encode(LocalDateTime.parse(input));
            case JSON:
                GenericJsonWriter jsonWriter = new GenericJsonWriter();
                return jsonWriter.write(new GenericJsonRecord(null, null, ObjectMapperFactory.getThreadLocal().readTree(input), schema.getSchemaInfo()));
            case AVRO:
                AvroBaseStructSchema avroSchema = ((AvroBaseStructSchema) schema);
                Decoder decoder = DecoderFactory.get().jsonDecoder(avroSchema.getAvroSchema(), input);
                DatumReader<GenericData.Record> reader = new GenericDatumReader(avroSchema.getAvroSchema());
                GenericRecord genericRecord = reader.read(null, decoder);
                GenericAvroWriter avroWriter = new GenericAvroWriter(avroSchema.getAvroSchema());
                return avroWriter.write(new GenericAvroRecord(null, avroSchema.getAvroSchema(), null, genericRecord));
            case PROTOBUF_NATIVE:
            case KEY_VALUE:
            default:
                throw new PulsarClientException.InvalidMessageException("");
        }
    } catch (Exception e) {
        if (log.isDebugEnabled()) {
            log.debug("Fail to encode value {} with schema {} for rest produce request", input, new String(schema.getSchemaInfo().getSchema()));
        }
        return new byte[0];
    }
}

Example 9 with GenericAvroRecord

use of org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord in project pulsar by apache.

the class TopicsTest method testProduceWithAvroSchema.

@Test
public void testProduceWithAvroSchema() throws Exception {
    String topicName = "persistent://" + testTenant + "/" + testNamespace + "/" + testTopicName;
    admin.topics().createNonPartitionedTopic(topicName);
    AsyncResponse asyncResponse = mock(AsyncResponse.class);
    GenericSchemaImpl avroSchema = GenericAvroSchema.of(AvroSchema.of(SchemaDefinition.builder().withPojo(PC.class).build()).getSchemaInfo());
    PC pc = new PC("dell", "alienware", 2021, GPU.AMD, new Seller("WA", "main street", 98004));
    PC anotherPc = new PC("asus", "rog", 2020, GPU.NVIDIA, new Seller("CA", "back street", 90232));
    Consumer consumer = pulsarClient.newConsumer(avroSchema).topic(topicName).subscriptionName("my-sub").subscriptionType(SubscriptionType.Exclusive).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe();
    ProducerMessages producerMessages = new ProducerMessages();
    producerMessages.setValueSchema(ObjectMapperFactory.getThreadLocal().writeValueAsString(avroSchema.getSchemaInfo()));
    ReflectDatumWriter<PC> datumWriter = new ReflectDatumWriter(avroSchema.getAvroSchema());
    ByteArrayOutputStream outputStream1 = new ByteArrayOutputStream();
    ByteArrayOutputStream outputStream2 = new ByteArrayOutputStream();
    JsonEncoder encoder1 = EncoderFactory.get().jsonEncoder(avroSchema.getAvroSchema(), outputStream1);
    JsonEncoder encoder2 = EncoderFactory.get().jsonEncoder(avroSchema.getAvroSchema(), outputStream2);
    datumWriter.write(pc, encoder1);
    encoder1.flush();
    datumWriter.write(anotherPc, encoder2);
    encoder2.flush();
    String message = "[" + "{\"key\":\"my-key\",\"payload\":\"" + outputStream1.toString().replace("\"", "\\\"") + "\",\"eventTime\":1603045262772,\"sequenceId\":1}," + "{\"key\":\"my-key\",\"payload\":\"" + outputStream2.toString().replace("\"", "\\\"") + "\",\"eventTime\":1603045262772,\"sequenceId\":2}]";
    producerMessages.setMessages(ObjectMapperFactory.getThreadLocal().readValue(message, new TypeReference<List<ProducerMessage>>() {
    }));
    topics.produceOnPersistentTopic(asyncResponse, testTenant, testNamespace, testTopicName, false, producerMessages);
    ArgumentCaptor<Response> responseCaptor = ArgumentCaptor.forClass(Response.class);
    verify(asyncResponse, timeout(5000).times(1)).resume(responseCaptor.capture());
    Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.OK.getStatusCode());
    Object responseEntity = responseCaptor.getValue().getEntity();
    Assert.assertTrue(responseEntity instanceof ProducerAcks);
    ProducerAcks response = (ProducerAcks) responseEntity;
    Assert.assertEquals(response.getMessagePublishResults().size(), 2);
    Assert.assertEquals(response.getSchemaVersion(), 0);
    for (int index = 0; index < response.getMessagePublishResults().size(); index++) {
        Assert.assertEquals(Integer.parseInt(response.getMessagePublishResults().get(index).getMessageId().split(":")[2]), -1);
        Assert.assertEquals(response.getMessagePublishResults().get(index).getErrorCode(), 0);
        Assert.assertTrue(response.getMessagePublishResults().get(index).getMessageId().length() > 0);
    }
    List<PC> expected = Arrays.asList(pc, anotherPc);
    Message<String> msg = null;
    // Assert all messages published by REST producer can be received by consumer in expected order.
    for (int i = 0; i < 2; i++) {
        msg = consumer.receive(2, TimeUnit.SECONDS);
        GenericAvroRecord avroRecord = (GenericAvroRecord) avroSchema.decode(msg.getData());
        Assert.assertEquals(((Utf8) avroRecord.getAvroRecord().get("brand")).toString(), expected.get(i).brand);
        Assert.assertEquals(((Utf8) avroRecord.getAvroRecord().get("model")).toString(), expected.get(i).model);
        Assert.assertEquals((int) avroRecord.getAvroRecord().get("year"), expected.get(i).year);
        Assert.assertEquals(((GenericData.EnumSymbol) avroRecord.getAvroRecord().get("gpu")).toString(), expected.get(i).gpu.toString());
        Assert.assertEquals(((Utf8) ((GenericRecord) avroRecord.getAvroRecord().get("seller")).get("state")).toString(), expected.get(i).seller.state);
        Assert.assertEquals(((Utf8) ((GenericRecord) avroRecord.getAvroRecord().get("seller")).get("street")).toString(), expected.get(i).seller.street);
        Assert.assertEquals(((GenericRecord) avroRecord.getAvroRecord().get("seller")).get("zipCode"), expected.get(i).seller.zipCode);
        Assert.assertEquals("my-key", msg.getKey());
    }
}