Examples with GenericAvroSchema org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema used on opensource projects

Example 1 with GenericAvroSchema

use of org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema in project pulsar by apache.

the class FieldSchemaBuilderImpl method build.

Field build() {
    requireNonNull(type, "Schema type is not provided");
    // verify the default value and object
    SchemaUtils.validateFieldSchema(fieldName, type, defaultVal);
    final Schema baseSchema;
    switch(type) {
        case INT32:
            baseSchema = SchemaBuilder.builder().intType();
            break;
        case INT64:
            baseSchema = SchemaBuilder.builder().longType();
            break;
        case STRING:
            baseSchema = SchemaBuilder.builder().stringType();
            break;
        case FLOAT:
            baseSchema = SchemaBuilder.builder().floatType();
            break;
        case DOUBLE:
            baseSchema = SchemaBuilder.builder().doubleType();
            break;
        case BOOLEAN:
            baseSchema = SchemaBuilder.builder().booleanType();
            break;
        case BYTES:
            baseSchema = SchemaBuilder.builder().bytesType();
            break;
        // DATE, TIME, TIMESTAMP support from generic record
        case DATE:
            baseSchema = LogicalTypes.date().addToSchema(Schema.create(Schema.Type.INT));
            break;
        case TIME:
            baseSchema = LogicalTypes.timeMillis().addToSchema(Schema.create(Schema.Type.INT));
            break;
        case TIMESTAMP:
            baseSchema = LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG));
            break;
        case JSON:
            checkArgument(genericSchema.getSchemaInfo().getType() == SchemaType.JSON, "The field is expected to be using JSON schema but " + genericSchema.getSchemaInfo().getType() + " schema is found");
            AvroBaseStructSchema genericJsonSchema = (AvroBaseStructSchema) genericSchema;
            baseSchema = genericJsonSchema.getAvroSchema();
            break;
        case AVRO:
            checkArgument(genericSchema.getSchemaInfo().getType() == SchemaType.AVRO, "The field is expected to be using AVRO schema but " + genericSchema.getSchemaInfo().getType() + " schema is found");
            GenericAvroSchema genericAvroSchema = (GenericAvroSchema) genericSchema;
            baseSchema = genericAvroSchema.getAvroSchema();
            break;
        default:
            throw new RuntimeException("Schema `" + type + "` is not supported to be used as a field for now");
    }
    for (Map.Entry<String, String> entry : properties.entrySet()) {
        baseSchema.addProp(entry.getKey(), entry.getValue());
    }
    if (null != aliases) {
        for (String alias : aliases) {
            baseSchema.addAlias(alias);
        }
    }
    final Schema finalSchema;
    if (optional) {
        if (defaultVal != null) {
            finalSchema = SchemaBuilder.builder().unionOf().type(baseSchema).and().nullType().endUnion();
        } else {
            finalSchema = SchemaBuilder.builder().unionOf().nullType().and().type(baseSchema).endUnion();
        }
    } else {
        finalSchema = baseSchema;
    }
    final Object finalDefaultValue;
    if (defaultVal != null) {
        finalDefaultValue = SchemaUtils.toAvroObject(defaultVal);
    } else {
        if (optional) {
            finalDefaultValue = JsonProperties.NULL_VALUE;
        } else {
            finalDefaultValue = null;
        }
    }
    return new Field(fieldName, finalSchema, doc, finalDefaultValue);
}

Example 2 with GenericAvroSchema

use of org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema in project pulsar by apache.

the class SqliteJdbcSinkTest method testOpenAndWriteSink.

private void testOpenAndWriteSink(Map<String, String> actionProperties) throws Exception {
    Message<GenericRecord> insertMessage = mock(MessageImpl.class);
    GenericSchema<GenericRecord> genericAvroSchema;
    // prepare a foo Record
    Foo insertObj = new Foo();
    insertObj.setField1("ValueOfField1");
    insertObj.setField2("ValueOfField2");
    insertObj.setField3(3);
    AvroSchema<Foo> schema = AvroSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).build());
    byte[] insertBytes = schema.encode(insertObj);
    CompletableFuture<Void> future = new CompletableFuture<>();
    Record<GenericRecord> insertRecord = PulsarRecord.<GenericRecord>builder().message(insertMessage).topicName("fake_topic_name").ackFunction(() -> future.complete(null)).build();
    genericAvroSchema = new GenericAvroSchema(schema.getSchemaInfo());
    when(insertMessage.getValue()).thenReturn(genericAvroSchema.decode(insertBytes));
    when(insertMessage.getProperties()).thenReturn(actionProperties);
    log.info("foo:{}, Message.getValue: {}, record.getValue: {}", insertObj.toString(), insertMessage.getValue().toString(), insertRecord.getValue().toString());
    // write should success.
    jdbcSink.write(insertRecord);
    log.info("executed write");
    // sleep to wait backend flush complete
    future.get(1, TimeUnit.SECONDS);
    // value has been written to db, read it out and verify.
    String querySql = "SELECT * FROM " + tableName + " WHERE field3=3";
    int count = sqliteUtils.select(querySql, (resultSet) -> {
        Assert.assertEquals(insertObj.getField1(), resultSet.getString(1));
        Assert.assertEquals(insertObj.getField2(), resultSet.getString(2));
        Assert.assertEquals(insertObj.getField3(), resultSet.getInt(3));
    });
    Assert.assertEquals(count, 1);
}

Example 3 with GenericAvroSchema

use of org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema in project pulsar by apache.

the class SqliteJdbcSinkTest method testOpenAndWriteSinkNullValue.

private void testOpenAndWriteSinkNullValue(Map<String, String> actionProperties) throws Exception {
    Message<GenericRecord> insertMessage = mock(MessageImpl.class);
    GenericSchema<GenericRecord> genericAvroSchema;
    // prepare a foo Record
    Foo insertObj = new Foo();
    insertObj.setField1("ValueOfField1");
    // Not setting field2
    // Field1 is the key and field3 is used for selecting records
    insertObj.setField3(3);
    AvroSchema<Foo> schema = AvroSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).withAlwaysAllowNull(true).build());
    byte[] insertBytes = schema.encode(insertObj);
    CompletableFuture<Void> future = new CompletableFuture<>();
    Record<GenericRecord> insertRecord = PulsarRecord.<GenericRecord>builder().message(insertMessage).topicName("fake_topic_name").ackFunction(() -> future.complete(null)).build();
    genericAvroSchema = new GenericAvroSchema(schema.getSchemaInfo());
    when(insertMessage.getValue()).thenReturn(genericAvroSchema.decode(insertBytes));
    when(insertMessage.getProperties()).thenReturn(actionProperties);
    log.info("foo:{}, Message.getValue: {}, record.getValue: {}", insertObj.toString(), insertMessage.getValue().toString(), insertRecord.getValue().toString());
    // write should success.
    jdbcSink.write(insertRecord);
    log.info("executed write");
    // sleep to wait backend flush complete
    future.get(1, TimeUnit.SECONDS);
    // value has been written to db, read it out and verify.
    String querySql = "SELECT * FROM " + tableName + " WHERE field3=3";
    int count = sqliteUtils.select(querySql, (resultSet) -> {
        Assert.assertEquals(insertObj.getField1(), resultSet.getString(1));
        Assert.assertNull(insertObj.getField2());
        Assert.assertEquals(insertObj.getField3(), resultSet.getInt(3));
    });
    Assert.assertEquals(count, 1);
}

Example 4 with GenericAvroSchema

use of org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema in project pulsar by apache.

the class SqliteJdbcSinkTest method TestUpdateAction.

@Test
public void TestUpdateAction() throws Exception {
    AvroSchema<Foo> schema = AvroSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).build());
    Foo updateObj = new Foo();
    updateObj.setField1("ValueOfField3");
    updateObj.setField2("ValueOfField3");
    updateObj.setField3(4);
    byte[] updateBytes = schema.encode(updateObj);
    Message<GenericRecord> updateMessage = mock(MessageImpl.class);
    CompletableFuture<Void> future = new CompletableFuture<>();
    Record<GenericRecord> updateRecord = PulsarRecord.<GenericRecord>builder().message(updateMessage).topicName("fake_topic_name").ackFunction(() -> future.complete(null)).build();
    GenericSchema<GenericRecord> updateGenericAvroSchema;
    updateGenericAvroSchema = new GenericAvroSchema(schema.getSchemaInfo());
    Map<String, String> updateProperties = Maps.newHashMap();
    updateProperties.put("ACTION", "UPDATE");
    when(updateMessage.getValue()).thenReturn(updateGenericAvroSchema.decode(updateBytes));
    when(updateMessage.getProperties()).thenReturn(updateProperties);
    log.info("foo:{}, Message.getValue: {}, record.getValue: {}", updateObj.toString(), updateMessage.getValue().toString(), updateRecord.getValue().toString());
    jdbcSink.write(updateRecord);
    future.get(1, TimeUnit.SECONDS);
    // value has been written to db, read it out and verify.
    String updateQuerySql = "SELECT * FROM " + tableName + " WHERE field3=4";
    sqliteUtils.select(updateQuerySql, (resultSet) -> {
        Assert.assertEquals(updateObj.getField1(), resultSet.getString(1));
        Assert.assertEquals(updateObj.getField2(), resultSet.getString(2));
        Assert.assertEquals(updateObj.getField3(), resultSet.getInt(3));
    });
}

Example 5 with GenericAvroSchema

use of org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema in project pulsar by apache.

the class InfluxDBSinkTest method testAvroSchema.

@Test
public void testAvroSchema() {
    AvroSchema<Cpu> schema = AvroSchema.of(Cpu.class);
    AutoConsumeSchema autoConsumeSchema = new AutoConsumeSchema();
    autoConsumeSchema.setSchema(GenericSchemaImpl.of(schema.getSchemaInfo()));
    GenericSchema<GenericRecord> genericAvroSchema = GenericSchemaImpl.of(autoConsumeSchema.getSchemaInfo());
    assertTrue(genericAvroSchema instanceof GenericAvroSchema);
    byte[] bytes = schema.encode(cpu);
    GenericRecord record = genericAvroSchema.decode(bytes);
    assertEquals(record.getField("measurement"), "cpu");
    assertEquals(record.getField("timestamp"), timestamp);
    assertEquals(((Map) record.getField("tags")).get(new Utf8("host")).toString(), "server-1");
    assertEquals(((Map) record.getField("fields")).get(new Utf8("value")), 10);
}