Examples with GenericJsonRecord org.apache.pulsar.client.impl.schema.generic.GenericJsonRecord used on opensource projects

Example 6 with GenericJsonRecord

use of org.apache.pulsar.client.impl.schema.generic.GenericJsonRecord in project incubator-pulsar by apache.

the class TopicsBase method encodeWithSchema.

// Encode message with corresponding schema, do necessary conversion before encoding
private byte[] encodeWithSchema(String input, Schema schema) {
    try {
        switch(schema.getSchemaInfo().getType()) {
            case INT8:
                return schema.encode(Byte.parseByte(input));
            case INT16:
                return schema.encode(Short.parseShort(input));
            case INT32:
                return schema.encode(Integer.parseInt(input));
            case INT64:
                return schema.encode(Long.parseLong(input));
            case STRING:
                return schema.encode(input);
            case FLOAT:
                return schema.encode(Float.parseFloat(input));
            case DOUBLE:
                return schema.encode(Double.parseDouble(input));
            case BOOLEAN:
                return schema.encode(Boolean.parseBoolean(input));
            case BYTES:
                return schema.encode(input.getBytes());
            case DATE:
                return schema.encode(DateFormat.getDateInstance().parse(input));
            case TIME:
                return schema.encode(new Time(Long.parseLong(input)));
            case TIMESTAMP:
                return schema.encode(new Timestamp(Long.parseLong(input)));
            case INSTANT:
                return schema.encode(Instant.parse(input));
            case LOCAL_DATE:
                return schema.encode(LocalDate.parse(input));
            case LOCAL_TIME:
                return schema.encode(LocalTime.parse(input));
            case LOCAL_DATE_TIME:
                return schema.encode(LocalDateTime.parse(input));
            case JSON:
                GenericJsonWriter jsonWriter = new GenericJsonWriter();
                return jsonWriter.write(new GenericJsonRecord(null, null, ObjectMapperFactory.getThreadLocal().readTree(input), schema.getSchemaInfo()));
            case AVRO:
                AvroBaseStructSchema avroSchema = ((AvroBaseStructSchema) schema);
                Decoder decoder = DecoderFactory.get().jsonDecoder(avroSchema.getAvroSchema(), input);
                DatumReader<GenericData.Record> reader = new GenericDatumReader(avroSchema.getAvroSchema());
                GenericRecord genericRecord = reader.read(null, decoder);
                GenericAvroWriter avroWriter = new GenericAvroWriter(avroSchema.getAvroSchema());
                return avroWriter.write(new GenericAvroRecord(null, avroSchema.getAvroSchema(), null, genericRecord));
            case PROTOBUF_NATIVE:
            case KEY_VALUE:
            default:
                throw new PulsarClientException.InvalidMessageException("");
        }
    } catch (Exception e) {
        if (log.isDebugEnabled()) {
            log.debug("Fail to encode value {} with schema {} for rest produce request", input, new String(schema.getSchemaInfo().getSchema()));
        }
        return new byte[0];
    }
}

Example 7 with GenericJsonRecord

use of org.apache.pulsar.client.impl.schema.generic.GenericJsonRecord in project incubator-pulsar by apache.

the class TestJsonDecoder method testCompositeType.

@Test
public void testCompositeType() {
    DecoderTestMessage message = new DecoderTestMessage();
    DecoderTestMessage.NestedRow nestedRow = new DecoderTestMessage.NestedRow();
    nestedRow.longField = 222L;
    nestedRow.stringField = "message_2_nestedRow";
    DecoderTestMessage.CompositeRow compositeRow = new DecoderTestMessage.CompositeRow();
    DecoderTestMessage.NestedRow nestedRow1 = new DecoderTestMessage.NestedRow();
    nestedRow1.longField = 2;
    nestedRow1.stringField = "nestedRow_1";
    DecoderTestMessage.NestedRow nestedRow2 = new DecoderTestMessage.NestedRow();
    nestedRow2.longField = 2;
    nestedRow2.stringField = "nestedRow_2";
    compositeRow.arrayField = Arrays.asList(nestedRow1, nestedRow2);
    compositeRow.stringField = "compositeRow_1";
    compositeRow.mapField = new HashMap<String, DecoderTestMessage.NestedRow>() {

        {
            put("key1", nestedRow1);
            put("key2", nestedRow2);
        }
    };
    compositeRow.nestedRow = nestedRow;
    new HashMap<String, Long>() {

        {
            put("key1_1", 2L);
            put("key1_2", 22L);
        }
    };
    compositeRow.structedField = new HashMap<String, List<Long>>() {

        {
            put("key2_1", Arrays.asList(2L, 3L));
            put("key2_2", Arrays.asList(2L, 3L));
            put("key2_3", Arrays.asList(2L, 3L));
        }
    };
    message.compositeRow = compositeRow;
    byte[] bytes = schema.encode(message);
    ByteBuf payload = io.netty.buffer.Unpooled.copiedBuffer(bytes);
    GenericJsonRecord genericRecord = (GenericJsonRecord) GenericJsonSchema.of(schemaInfo).decode(bytes);
    Object fieldValue = genericRecord.getJsonNode().get("compositeRow");
    Map<DecoderColumnHandle, FieldValueProvider> decodedRow = pulsarRowDecoder.decodeRow(payload).get();
    RowType columnType = RowType.from(ImmutableList.<RowType.Field>builder().add(RowType.field("arrayField", new ArrayType(RowType.from(ImmutableList.<RowType.Field>builder().add(RowType.field("longField", BIGINT)).add(RowType.field("stringField", VARCHAR)).build())))).add(RowType.field("mapField", decoderFactory.getTypeManager().getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.typeParameter(VarcharType.VARCHAR.getTypeSignature()), TypeSignatureParameter.typeParameter(RowType.from(ImmutableList.<RowType.Field>builder().add(RowType.field("longField", BIGINT)).add(RowType.field("stringField", VARCHAR)).build()).getTypeSignature()))))).add(RowType.field("nestedRow", RowType.from(ImmutableList.<RowType.Field>builder().add(RowType.field("longField", BIGINT)).add(RowType.field("stringField", VARCHAR)).build()))).add(RowType.field("stringField", VARCHAR)).add(RowType.field("structedField", decoderFactory.getTypeManager().getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.typeParameter(VarcharType.VARCHAR.getTypeSignature()), TypeSignatureParameter.typeParameter(new ArrayType(BIGINT).getTypeSignature()))))).build());
    PulsarColumnHandle columnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), "compositeRow", columnType, false, false, "compositeRow", null, null, PulsarColumnHandle.HandleKeyValueType.NONE);
    checkRowValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue);
}

Example 8 with GenericJsonRecord

use of org.apache.pulsar.client.impl.schema.generic.GenericJsonRecord in project incubator-pulsar by apache.

the class JSONSchemaTest method testJsonGenericRecordBuilder.

@Test
public void testJsonGenericRecordBuilder() {
    JSONSchema<Seller> sellerJsonSchema = JSONSchema.of(Seller.class);
    RecordSchemaBuilder sellerSchemaBuilder = SchemaBuilder.record("seller");
    sellerSchemaBuilder.field("state").type(SchemaType.STRING);
    sellerSchemaBuilder.field("street").type(SchemaType.STRING);
    sellerSchemaBuilder.field("zipCode").type(SchemaType.INT64);
    SchemaInfo sellerSchemaInfo = sellerSchemaBuilder.build(SchemaType.JSON);
    GenericSchemaImpl sellerGenericSchema = GenericSchemaImpl.of(sellerSchemaInfo);
    JSONSchema<PC> pcJsonSchema = JSONSchema.of(PC.class);
    RecordSchemaBuilder pcSchemaBuilder = SchemaBuilder.record("pc");
    pcSchemaBuilder.field("brand").type(SchemaType.STRING);
    pcSchemaBuilder.field("model").type(SchemaType.STRING);
    pcSchemaBuilder.field("gpu").type(SchemaType.STRING);
    pcSchemaBuilder.field("year").type(SchemaType.INT64);
    pcSchemaBuilder.field("seller", sellerGenericSchema).type(SchemaType.JSON).optional();
    SchemaInfo pcGenericSchemaInfo = pcSchemaBuilder.build(SchemaType.JSON);
    GenericSchemaImpl pcGenericSchema = GenericSchemaImpl.of(pcGenericSchemaInfo);
    Seller seller = new Seller("USA", "oakstreet", 9999);
    PC pc = new PC("dell", "g3", 2020, GPU.AMD, seller);
    byte[] bytes = pcJsonSchema.encode(pc);
    Assert.assertTrue(bytes.length > 0);
    Object pc2 = pcJsonSchema.decode(bytes);
    assertEquals(pc, pc2);
    GenericRecord sellerRecord = sellerGenericSchema.newRecordBuilder().set("state", "USA").set("street", "oakstreet").set("zipCode", 9999).build();
    GenericRecord pcRecord = pcGenericSchema.newRecordBuilder().set("brand", "dell").set("model", "g3").set("year", 2020).set("gpu", GPU.AMD).set("seller", sellerRecord).build();
    byte[] bytes3 = pcGenericSchema.encode(pcRecord);
    Assert.assertTrue(bytes3.length > 0);
    GenericRecord pc3Record = pcGenericSchema.decode(bytes3);
    for (Field field : pc3Record.getFields()) {
        assertTrue(pcGenericSchema.getFields().contains(field));
    }
    assertEquals("dell", pc3Record.getField("brand"));
    assertEquals("g3", pc3Record.getField("model"));
    assertEquals(2020, pc3Record.getField("year"));
    assertEquals(GPU.AMD.toString(), pc3Record.getField("gpu"));
    GenericRecord seller3Record = (GenericRecord) pc3Record.getField("seller");
    assertEquals("USA", seller3Record.getField("state"));
    assertEquals("oakstreet", seller3Record.getField("street"));
    assertEquals(9999, seller3Record.getField("zipCode"));
    assertTrue(pc3Record instanceof GenericJsonRecord);
    Assertions.assertThatCode(() -> pc3Record.getField("I_DO_NOT_EXIST")).doesNotThrowAnyException();
}

Example 9 with GenericJsonRecord

use of org.apache.pulsar.client.impl.schema.generic.GenericJsonRecord in project incubator-pulsar by apache.

the class TestJsonDecoder method testMap.

@Test
public void testMap() {
    DecoderTestMessage message = new DecoderTestMessage();
    message.mapField = new HashMap<String, Long>() {

        {
            put("key1", 2L);
            put("key2", 22L);
        }
    };
    byte[] bytes = schema.encode(message);
    ByteBuf payload = io.netty.buffer.Unpooled.copiedBuffer(bytes);
    GenericJsonRecord genericRecord = (GenericJsonRecord) GenericJsonSchema.of(schemaInfo).decode(bytes);
    Object fieldValue = genericRecord.getJsonNode().get("mapField");
    Map<DecoderColumnHandle, FieldValueProvider> decodedRow = pulsarRowDecoder.decodeRow(payload).get();
    Type columnType = decoderFactory.getTypeManager().getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.typeParameter(VarcharType.VARCHAR.getTypeSignature()), TypeSignatureParameter.typeParameter(BigintType.BIGINT.getTypeSignature())));
    PulsarColumnHandle columnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), "mapField", columnType, false, false, "mapField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE);
    checkMapValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue);
}

Example 10 with GenericJsonRecord

use of org.apache.pulsar.client.impl.schema.generic.GenericJsonRecord in project incubator-pulsar by apache.

the class TestJsonDecoder method testRow.

@Test
public void testRow() {
    DecoderTestMessage message = new DecoderTestMessage();
    message.stringField = "message_2";
    DecoderTestMessage.TestRow testRow = new DecoderTestMessage.TestRow();
    message.rowField = testRow;
    testRow.intField = 22;
    testRow.stringField = "message_2_testRow";
    DecoderTestMessage.NestedRow nestedRow = new DecoderTestMessage.NestedRow();
    nestedRow.longField = 222L;
    nestedRow.stringField = "message_2_nestedRow";
    testRow.nestedRow = nestedRow;
    byte[] bytes = schema.encode(message);
    ByteBuf payload = io.netty.buffer.Unpooled.copiedBuffer(bytes);
    GenericJsonRecord genericRecord = (GenericJsonRecord) GenericJsonSchema.of(schemaInfo).decode(bytes);
    Object fieldValue = genericRecord.getJsonNode().get("rowField");
    Map<DecoderColumnHandle, FieldValueProvider> decodedRow = pulsarRowDecoder.decodeRow(payload).get();
    RowType columnType = RowType.from(ImmutableList.<RowType.Field>builder().add(RowType.field("intField", INTEGER)).add(RowType.field("nestedRow", RowType.from(ImmutableList.<RowType.Field>builder().add(RowType.field("longField", BIGINT)).add(RowType.field("stringField", VARCHAR)).build()))).add(RowType.field("stringField", VARCHAR)).build());
    PulsarColumnHandle columnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), "rowField", columnType, false, false, "rowField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE);
    checkRowValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue);
}