Examples with GenericSchema org.apache.pulsar.client.api.schema.GenericSchema used on opensource projects

Example 6 with GenericSchema

use of org.apache.pulsar.client.api.schema.GenericSchema in project pulsar-flink by streamnative.

the class PulsarSerializer method newStructConverter.

private Function<Object, Object> newStructConverter(FieldsDataType dataType, Schema avroStruct) throws IncompatibleSchemaException {
    if (avroStruct.getType() != Schema.Type.RECORD || avroStruct.getFields().size() != dataType.getChildren().size()) {
        throw new IncompatibleSchemaException(String.format("Cannot convert Flink type %s to Avro type %s.", dataType.toString(), avroStruct.toString(true)));
    }
    RowType rowType = (RowType) dataType.getLogicalType();
    List<RowType.RowField> fields = rowType.getFields();
    List<BiFunction<PositionedGetter, Integer, Object>> fieldConverters = new ArrayList<>();
    for (int i = 0; i < fields.size(); i++) {
        org.apache.flink.table.types.logical.LogicalType logicalType = rowType.getTypeAt(i);
        DataType dt = TypeConversions.fromLogicalToDataType(logicalType);
        Schema.Field at = avroStruct.getFields().get(i);
        fieldConverters.add(newConverter(dt, resolveNullableType(at.schema(), dt.getLogicalType().isNullable())));
    }
    int numFields = dataType.getChildren().size();
    return row -> {
        GenericSchema<GenericRecord> pSchema = SchemaUtils.avroSchema2PulsarSchema(avroStruct);
        GenericRecordBuilder builder = pSchema.newRecordBuilder();
        Row rowX = (Row) row;
        for (int i = 0; i < numFields; i++) {
            if (rowX.getField(i) == null) {
                builder.set(pSchema.getFields().get(i), null);
            } else {
                builder.set(pSchema.getFields().get(i), fieldConverters.get(i).apply(new PositionedGetter(rowX), i));
            }
        }
        return (GenericAvroRecord) builder.build();
    };
}

Example 7 with GenericSchema

use of org.apache.pulsar.client.api.schema.GenericSchema in project pulsar-sink by datastax.

the class StructEndToEndCCMIT method struct_optional_fields_missing.

@Test
void struct_optional_fields_missing() {
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, intcol=value.int, doublecol=value.double"));
    RecordSchemaBuilder builderRoot = org.apache.pulsar.client.api.schema.SchemaBuilder.record("MyBean");
    builderRoot.field("bigint").type(SchemaType.INT64).optional();
    builderRoot.field("boolean").type(SchemaType.BOOLEAN).optional();
    builderRoot.field("double").type(SchemaType.DOUBLE).optional();
    builderRoot.field("float").type(SchemaType.FLOAT).optional();
    builderRoot.field("int").type(SchemaType.INT32).optional();
    builderRoot.field("text").type(SchemaType.STRING).optional();
    builderRoot.field("blob").type(SchemaType.BYTES).optional();
    GenericSchema schema = org.apache.pulsar.client.api.Schema.generic(builderRoot.build(SchemaType.AVRO));
    Long baseValue = 98761234L;
    GenericRecordImpl value = new GenericRecordImpl().put("bigint", baseValue).put("int", baseValue.intValue());
    runTaskWithRecords(new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, value, schema));
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT * FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(baseValue);
    assertThat(row.getInt("intcol")).isEqualTo(baseValue.intValue());
}

Example 8 with GenericSchema

use of org.apache.pulsar.client.api.schema.GenericSchema in project pulsar-sink by datastax.

the class StructEndToEndCCMIT method struct_optional_fields_with_values.

@Test
void struct_optional_fields_with_values() {
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, " + "booleancol=value.boolean, " + "doublecol=value.double, " + "floatcol=value.float, " + "intcol=value.int, " + "textcol=value.text, " + "blobcol=value.blob"));
    RecordSchemaBuilder builderRoot = org.apache.pulsar.client.api.schema.SchemaBuilder.record("MyBean");
    builderRoot.field("bigint").type(SchemaType.INT64).optional();
    builderRoot.field("boolean").type(SchemaType.BOOLEAN).optional();
    builderRoot.field("double").type(SchemaType.DOUBLE).optional();
    builderRoot.field("float").type(SchemaType.FLOAT).optional();
    builderRoot.field("int").type(SchemaType.INT32).optional();
    builderRoot.field("text").type(SchemaType.STRING).optional();
    builderRoot.field("blob").type(SchemaType.BYTES).optional();
    GenericSchema schema = org.apache.pulsar.client.api.Schema.generic(builderRoot.build(SchemaType.AVRO));
    byte[] blobValue = new byte[] { 12, 22, 32 };
    Long baseValue = 98761234L;
    GenericRecordImpl value = new GenericRecordImpl().put("bigint", baseValue).put("boolean", (baseValue.intValue() & 1) == 1).put("double", (double) baseValue + 0.123).put("float", baseValue.floatValue() + 0.987f).put("int", baseValue.intValue()).put("smallint", baseValue.shortValue()).put("text", baseValue.toString()).put("blob", blobValue);
    runTaskWithRecords(new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, value, schema));
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT * FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(baseValue);
    assertThat(row.getBoolean("booleancol")).isEqualTo((baseValue.intValue() & 1) == 1);
    assertThat(row.getDouble("doublecol")).isEqualTo((double) baseValue + 0.123);
    assertThat(row.getFloat("floatcol")).isEqualTo(baseValue.floatValue() + 0.987f);
    assertThat(row.getInt("intcol")).isEqualTo(baseValue.intValue());
    assertThat(row.getString("textcol")).isEqualTo(baseValue.toString());
    ByteBuffer blobcol = row.getByteBuffer("blobcol");
    assertThat(blobcol).isNotNull();
    assertThat(Bytes.getArray(blobcol)).isEqualTo(blobValue);
}

Example 9 with GenericSchema

use of org.apache.pulsar.client.api.schema.GenericSchema in project pulsar by yahoo.

the class TopicsTest method testProduceWithJsonSchema.

@Test
public void testProduceWithJsonSchema() throws Exception {
    String topicName = "persistent://" + testTenant + "/" + testNamespace + "/" + testTopicName;
    admin.topics().createNonPartitionedTopic(topicName);
    AsyncResponse asyncResponse = mock(AsyncResponse.class);
    GenericSchema jsonSchema = GenericJsonSchema.of(JSONSchema.of(SchemaDefinition.builder().withPojo(PC.class).build()).getSchemaInfo());
    PC pc = new PC("dell", "alienware", 2021, GPU.AMD, new Seller("WA", "main street", 98004));
    PC anotherPc = new PC("asus", "rog", 2020, GPU.NVIDIA, new Seller("CA", "back street", 90232));
    Consumer consumer = pulsarClient.newConsumer(jsonSchema).topic(topicName).subscriptionName("my-sub").subscriptionType(SubscriptionType.Exclusive).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe();
    ProducerMessages producerMessages = new ProducerMessages();
    producerMessages.setValueSchema(ObjectMapperFactory.getThreadLocal().writeValueAsString(jsonSchema.getSchemaInfo()));
    String message = "[" + "{\"key\":\"my-key\",\"payload\":\"" + ObjectMapperFactory.getThreadLocal().writeValueAsString(pc).replace("\"", "\\\"") + "\",\"eventTime\":1603045262772,\"sequenceId\":1}," + "{\"key\":\"my-key\",\"payload\":\"" + ObjectMapperFactory.getThreadLocal().writeValueAsString(anotherPc).replace("\"", "\\\"") + "\",\"eventTime\":1603045262772,\"sequenceId\":2}]";
    producerMessages.setMessages(ObjectMapperFactory.getThreadLocal().readValue(message, new TypeReference<List<ProducerMessage>>() {
    }));
    topics.produceOnPersistentTopic(asyncResponse, testTenant, testNamespace, testTopicName, false, producerMessages);
    ArgumentCaptor<Response> responseCaptor = ArgumentCaptor.forClass(Response.class);
    verify(asyncResponse, timeout(5000).times(1)).resume(responseCaptor.capture());
    Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.OK.getStatusCode());
    Object responseEntity = responseCaptor.getValue().getEntity();
    Assert.assertTrue(responseEntity instanceof ProducerAcks);
    ProducerAcks response = (ProducerAcks) responseEntity;
    Assert.assertEquals(response.getMessagePublishResults().size(), 2);
    Assert.assertEquals(response.getSchemaVersion(), 0);
    for (int index = 0; index < response.getMessagePublishResults().size(); index++) {
        Assert.assertEquals(Integer.parseInt(response.getMessagePublishResults().get(index).getMessageId().split(":")[2]), -1);
        Assert.assertEquals(response.getMessagePublishResults().get(index).getErrorCode(), 0);
        Assert.assertTrue(response.getMessagePublishResults().get(index).getMessageId().length() > 0);
    }
    List<PC> expected = Arrays.asList(pc, anotherPc);
    Message<String> msg = null;
    // Assert all messages published by REST producer can be received by consumer in expected order.
    for (int i = 0; i < 2; i++) {
        msg = consumer.receive(2, TimeUnit.SECONDS);
        PC msgPc = ObjectMapperFactory.getThreadLocal().treeToValue(((GenericJsonRecord) jsonSchema.decode(msg.getData())).getJsonNode(), PC.class);
        Assert.assertEquals(msgPc.brand, expected.get(i).brand);
        Assert.assertEquals(msgPc.model, expected.get(i).model);
        Assert.assertEquals(msgPc.year, expected.get(i).year);
        Assert.assertEquals(msgPc.gpu, expected.get(i).gpu);
        Assert.assertEquals(msgPc.seller.state, expected.get(i).seller.state);
        Assert.assertEquals(msgPc.seller.street, expected.get(i).seller.street);
        Assert.assertEquals(msgPc.seller.zipCode, expected.get(i).seller.zipCode);
        Assert.assertEquals("my-key", msg.getKey());
    }
}

Example 10 with GenericSchema

use of org.apache.pulsar.client.api.schema.GenericSchema in project pulsar by yahoo.

the class FieldSchemaBuilderImpl method build.

Field build() {
    requireNonNull(type, "Schema type is not provided");
    // verify the default value and object
    SchemaUtils.validateFieldSchema(fieldName, type, defaultVal);
    final Schema baseSchema;
    switch(type) {
        case INT32:
            baseSchema = SchemaBuilder.builder().intType();
            break;
        case INT64:
            baseSchema = SchemaBuilder.builder().longType();
            break;
        case STRING:
            baseSchema = SchemaBuilder.builder().stringType();
            break;
        case FLOAT:
            baseSchema = SchemaBuilder.builder().floatType();
            break;
        case DOUBLE:
            baseSchema = SchemaBuilder.builder().doubleType();
            break;
        case BOOLEAN:
            baseSchema = SchemaBuilder.builder().booleanType();
            break;
        case BYTES:
            baseSchema = SchemaBuilder.builder().bytesType();
            break;
        // DATE, TIME, TIMESTAMP support from generic record
        case DATE:
            baseSchema = LogicalTypes.date().addToSchema(Schema.create(Schema.Type.INT));
            break;
        case TIME:
            baseSchema = LogicalTypes.timeMillis().addToSchema(Schema.create(Schema.Type.INT));
            break;
        case TIMESTAMP:
            baseSchema = LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG));
            break;
        case JSON:
            checkArgument(genericSchema.getSchemaInfo().getType() == SchemaType.JSON, "The field is expected to be using JSON schema but " + genericSchema.getSchemaInfo().getType() + " schema is found");
            AvroBaseStructSchema genericJsonSchema = (AvroBaseStructSchema) genericSchema;
            baseSchema = genericJsonSchema.getAvroSchema();
            break;
        case AVRO:
            checkArgument(genericSchema.getSchemaInfo().getType() == SchemaType.AVRO, "The field is expected to be using AVRO schema but " + genericSchema.getSchemaInfo().getType() + " schema is found");
            GenericAvroSchema genericAvroSchema = (GenericAvroSchema) genericSchema;
            baseSchema = genericAvroSchema.getAvroSchema();
            break;
        default:
            throw new RuntimeException("Schema `" + type + "` is not supported to be used as a field for now");
    }
    for (Map.Entry<String, String> entry : properties.entrySet()) {
        baseSchema.addProp(entry.getKey(), entry.getValue());
    }
    if (null != aliases) {
        for (String alias : aliases) {
            baseSchema.addAlias(alias);
        }
    }
    final Schema finalSchema;
    if (optional) {
        if (defaultVal != null) {
            finalSchema = SchemaBuilder.builder().unionOf().type(baseSchema).and().nullType().endUnion();
        } else {
            finalSchema = SchemaBuilder.builder().unionOf().nullType().and().type(baseSchema).endUnion();
        }
    } else {
        finalSchema = baseSchema;
    }
    final Object finalDefaultValue;
    if (defaultVal != null) {
        finalDefaultValue = SchemaUtils.toAvroObject(defaultVal);
    } else {
        if (optional) {
            finalDefaultValue = JsonProperties.NULL_VALUE;
        } else {
            finalDefaultValue = null;
        }
    }
    return new Field(fieldName, finalSchema, doc, finalDefaultValue);
}