Examples with GenericAvroRecord org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord used on opensource projects

Example 1 with GenericAvroRecord

use of org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord in project pulsar by apache.

the class TestAvroDecoder method testRow.

@Test
public void testRow() {
    DecoderTestMessage message = new DecoderTestMessage();
    message.stringField = "message_2";
    DecoderTestMessage.TestRow testRow = new DecoderTestMessage.TestRow();
    message.rowField = testRow;
    testRow.intField = 22;
    testRow.stringField = "message_2_testRow";
    DecoderTestMessage.NestedRow nestedRow = new DecoderTestMessage.NestedRow();
    nestedRow.longField = 222L;
    nestedRow.stringField = "message_2_nestedRow";
    testRow.nestedRow = nestedRow;
    byte[] bytes = schema.encode(message);
    ByteBuf payload = io.netty.buffer.Unpooled.copiedBuffer(bytes);
    GenericAvroRecord genericRecord = (GenericAvroRecord) GenericAvroSchema.of(schemaInfo).decode(bytes);
    Object fieldValue = genericRecord.getAvroRecord().get("rowField");
    Map<DecoderColumnHandle, FieldValueProvider> decodedRow = pulsarRowDecoder.decodeRow(payload).get();
    RowType columnType = RowType.from(ImmutableList.<RowType.Field>builder().add(RowType.field("intField", INTEGER)).add(RowType.field("nestedRow", RowType.from(ImmutableList.<RowType.Field>builder().add(RowType.field("longField", BIGINT)).add(RowType.field("stringField", VARCHAR)).build()))).add(RowType.field("stringField", VARCHAR)).build());
    PulsarColumnHandle columnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), "rowField", columnType, false, false, "rowField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE);
    checkRowValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue);
}

Example 2 with GenericAvroRecord

use of org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord in project pulsar by apache.

the class TestAvroDecoder method testMap.

@Test
public void testMap() {
    DecoderTestMessage message = new DecoderTestMessage();
    message.mapField = new HashMap<String, Long>() {

        {
            put("key1", 2L);
            put("key2", 22L);
        }
    };
    byte[] bytes = schema.encode(message);
    ByteBuf payload = io.netty.buffer.Unpooled.copiedBuffer(bytes);
    GenericAvroRecord genericRecord = (GenericAvroRecord) GenericAvroSchema.of(schemaInfo).decode(bytes);
    Object fieldValue = genericRecord.getAvroRecord().get("mapField");
    Map<DecoderColumnHandle, FieldValueProvider> decodedRow = pulsarRowDecoder.decodeRow(payload).get();
    Type columnType = decoderFactory.getTypeManager().getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.typeParameter(VarcharType.VARCHAR.getTypeSignature()), TypeSignatureParameter.typeParameter(BigintType.BIGINT.getTypeSignature())));
    PulsarColumnHandle columnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), "mapField", columnType, false, false, "mapField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE);
    checkMapValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue);
}

Example 3 with GenericAvroRecord

use of org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord in project pulsar by apache.

the class TestAvroDecoder method testArray.

@Test
public void testArray() {
    DecoderTestMessage message = new DecoderTestMessage();
    message.arrayField = Arrays.asList("message_1", "message_2", "message_3");
    byte[] bytes = schema.encode(message);
    ByteBuf payload = io.netty.buffer.Unpooled.copiedBuffer(bytes);
    GenericAvroRecord genericRecord = (GenericAvroRecord) GenericAvroSchema.of(schemaInfo).decode(bytes);
    Object fieldValue = genericRecord.getAvroRecord().get("arrayField");
    Map<DecoderColumnHandle, FieldValueProvider> decodedRow = pulsarRowDecoder.decodeRow(payload).get();
    ArrayType columnType = new ArrayType(VARCHAR);
    PulsarColumnHandle columnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), "arrayField", columnType, false, false, "arrayField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE);
    checkArrayValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue);
}

Example 4 with GenericAvroRecord

use of org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord in project pulsar by apache.

the class PulsarAvroRowDecoder method decodeRow.

/**
 * decode ByteBuf by {@link org.apache.pulsar.client.api.schema.GenericSchema}.
 * @param byteBuf
 * @return
 */
@Override
public Optional<Map<DecoderColumnHandle, FieldValueProvider>> decodeRow(ByteBuf byteBuf) {
    GenericRecord avroRecord;
    try {
        GenericAvroRecord record = (GenericAvroRecord) genericAvroSchema.decode(byteBuf);
        avroRecord = record.getAvroRecord();
    } catch (Exception e) {
        e.printStackTrace();
        throw new PrestoException(GENERIC_INTERNAL_ERROR, "Decoding avro record failed.", e);
    }
    return Optional.of(columnDecoders.entrySet().stream().collect(toImmutableMap(Map.Entry::getKey, entry -> entry.getValue().decodeField(avroRecord))));
}

Example 5 with GenericAvroRecord

use of org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord in project pulsar-flink by streamnative.

the class PulsarSerializer method newStructConverter.

private Function<Object, Object> newStructConverter(FieldsDataType dataType, Schema avroStruct) throws IncompatibleSchemaException {
    if (avroStruct.getType() != Schema.Type.RECORD || avroStruct.getFields().size() != dataType.getChildren().size()) {
        throw new IncompatibleSchemaException(String.format("Cannot convert Flink type %s to Avro type %s.", dataType.toString(), avroStruct.toString(true)));
    }
    RowType rowType = (RowType) dataType.getLogicalType();
    List<RowType.RowField> fields = rowType.getFields();
    List<BiFunction<PositionedGetter, Integer, Object>> fieldConverters = new ArrayList<>();
    for (int i = 0; i < fields.size(); i++) {
        org.apache.flink.table.types.logical.LogicalType logicalType = rowType.getTypeAt(i);
        DataType dt = TypeConversions.fromLogicalToDataType(logicalType);
        Schema.Field at = avroStruct.getFields().get(i);
        fieldConverters.add(newConverter(dt, resolveNullableType(at.schema(), dt.getLogicalType().isNullable())));
    }
    int numFields = dataType.getChildren().size();
    return row -> {
        GenericSchema<GenericRecord> pSchema = SchemaUtils.avroSchema2PulsarSchema(avroStruct);
        GenericRecordBuilder builder = pSchema.newRecordBuilder();
        Row rowX = (Row) row;
        for (int i = 0; i < numFields; i++) {
            if (rowX.getField(i) == null) {
                builder.set(pSchema.getFields().get(i), null);
            } else {
                builder.set(pSchema.getFields().get(i), fieldConverters.get(i).apply(new PositionedGetter(rowX), i));
            }
        }
        return (GenericAvroRecord) builder.build();
    };
}