Examples with GenericRecord org.apache.pulsar.client.api.schema.GenericRecord used on opensource projects

Example 16 with GenericRecord

use of org.apache.pulsar.client.api.schema.GenericRecord in project pulsar by apache.

the class ElasticSearchExtractTests method testKeyValueGenericRecord.

@Test(dataProvider = "schemaType")
public void testKeyValueGenericRecord(SchemaType schemaType) throws Exception {
    RecordSchemaBuilder keySchemaBuilder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("key");
    keySchemaBuilder.field("a").type(SchemaType.STRING).optional().defaultValue(null);
    keySchemaBuilder.field("b").type(SchemaType.INT32).optional().defaultValue(null);
    GenericSchema<GenericRecord> keySchema = Schema.generic(keySchemaBuilder.build(schemaType));
    GenericRecord keyGenericRecord = keySchema.newRecordBuilder().set("a", "1").set("b", 1).build();
    RecordSchemaBuilder valueSchemaBuilder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("value");
    valueSchemaBuilder.field("c").type(SchemaType.STRING).optional().defaultValue(null);
    valueSchemaBuilder.field("d").type(SchemaType.INT32).optional().defaultValue(null);
    RecordSchemaBuilder udtSchemaBuilder = SchemaBuilder.record("type1");
    udtSchemaBuilder.field("a").type(SchemaType.STRING).optional().defaultValue(null);
    udtSchemaBuilder.field("b").type(SchemaType.BOOLEAN).optional().defaultValue(null);
    udtSchemaBuilder.field("d").type(SchemaType.DOUBLE).optional().defaultValue(null);
    udtSchemaBuilder.field("f").type(SchemaType.FLOAT).optional().defaultValue(null);
    udtSchemaBuilder.field("i").type(SchemaType.INT32).optional().defaultValue(null);
    udtSchemaBuilder.field("l").type(SchemaType.INT64).optional().defaultValue(null);
    GenericSchema<GenericRecord> udtGenericSchema = Schema.generic(udtSchemaBuilder.build(schemaType));
    valueSchemaBuilder.field("e", udtGenericSchema).type(schemaType).optional().defaultValue(null);
    GenericSchema<GenericRecord> valueSchema = Schema.generic(valueSchemaBuilder.build(schemaType));
    GenericRecord valueGenericRecord = valueSchema.newRecordBuilder().set("c", "1").set("d", 1).set("e", udtGenericSchema.newRecordBuilder().set("a", "a").set("b", true).set("d", 1.0).set("f", 1.0f).set("i", 1).set("l", 10L).build()).build();
    Schema<KeyValue<GenericRecord, GenericRecord>> keyValueSchema = Schema.KeyValue(keySchema, valueSchema, KeyValueEncodingType.INLINE);
    KeyValue<GenericRecord, GenericRecord> keyValue = new KeyValue<>(keyGenericRecord, valueGenericRecord);
    GenericObject genericObject = new GenericObject() {

        @Override
        public SchemaType getSchemaType() {
            return SchemaType.KEY_VALUE;
        }

        @Override
        public Object getNativeObject() {
            return keyValue;
        }
    };
    Record<GenericObject> genericObjectRecord = new Record<GenericObject>() {

        @Override
        public Optional<String> getTopicName() {
            return Optional.of("data-ks1.table1");
        }

        @Override
        public org.apache.pulsar.client.api.Schema getSchema() {
            return keyValueSchema;
        }

        @Override
        public GenericObject getValue() {
            return genericObject;
        }
    };
    ElasticSearchSink elasticSearchSink = new ElasticSearchSink();
    elasticSearchSink.open(ImmutableMap.of("elasticSearchUrl", "http://localhost:9200", "schemaEnable", "true", "keyIgnore", "false"), null);
    Pair<String, String> pair = elasticSearchSink.extractIdAndDocument(genericObjectRecord);
    assertEquals(pair.getLeft(), "[\"1\",1]");
    assertEquals(pair.getRight(), "{\"c\":\"1\",\"d\":1,\"e\":{\"a\":\"a\",\"b\":true,\"d\":1.0,\"f\":1.0,\"i\":1,\"l\":10}}");
    ElasticSearchSink elasticSearchSink2 = new ElasticSearchSink();
    elasticSearchSink2.open(ImmutableMap.of("elasticSearchUrl", "http://localhost:9200", "schemaEnable", "true"), null);
    Pair<String, String> pair2 = elasticSearchSink2.extractIdAndDocument(genericObjectRecord);
    assertNull(pair2.getLeft());
    assertEquals(pair2.getRight(), "{\"c\":\"1\",\"d\":1,\"e\":{\"a\":\"a\",\"b\":true,\"d\":1.0,\"f\":1.0,\"i\":1,\"l\":10}}");
    // test null value
    ElasticSearchSink elasticSearchSink3 = new ElasticSearchSink();
    elasticSearchSink3.open(ImmutableMap.of("elasticSearchUrl", "http://localhost:9200", "schemaEnable", "true", "keyIgnore", "false"), null);
    Pair<String, String> pair3 = elasticSearchSink.extractIdAndDocument(new Record<GenericObject>() {

        @Override
        public Optional<String> getTopicName() {
            return Optional.of("data-ks1.table1");
        }

        @Override
        public org.apache.pulsar.client.api.Schema getSchema() {
            return keyValueSchema;
        }

        @Override
        public GenericObject getValue() {
            return new GenericObject() {

                @Override
                public SchemaType getSchemaType() {
                    return SchemaType.KEY_VALUE;
                }

                @Override
                public Object getNativeObject() {
                    return new KeyValue<>(keyGenericRecord, null);
                }
            };
        }
    });
    assertEquals(pair3.getLeft(), "[\"1\",1]");
    assertNull(pair3.getRight());
}

Example 17 with GenericRecord

use of org.apache.pulsar.client.api.schema.GenericRecord in project pulsar by apache.

the class ElasticSearchSinkTests method testKeepNullNodes.

@Test
public void testKeepNullNodes() throws Exception {
    map.put("stripNulls", false);
    sink.open(map, mockSinkContext);
    GenericRecord genericRecord = genericSchema.newRecordBuilder().set("name", null).set("userName", "boby").set("email", null).build();
    String json = sink.stringifyValue(valueSchema, genericRecord);
    assertEquals(json, "{\"name\":null,\"userName\":\"boby\",\"email\":null}");
}

Example 18 with GenericRecord

use of org.apache.pulsar.client.api.schema.GenericRecord in project pulsar by apache.

the class SqliteJdbcSinkTest method testOpenAndWriteSink.

private void testOpenAndWriteSink(Map<String, String> actionProperties) throws Exception {
    Message<GenericRecord> insertMessage = mock(MessageImpl.class);
    GenericSchema<GenericRecord> genericAvroSchema;
    // prepare a foo Record
    Foo insertObj = new Foo();
    insertObj.setField1("ValueOfField1");
    insertObj.setField2("ValueOfField2");
    insertObj.setField3(3);
    AvroSchema<Foo> schema = AvroSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).build());
    byte[] insertBytes = schema.encode(insertObj);
    CompletableFuture<Void> future = new CompletableFuture<>();
    Record<GenericRecord> insertRecord = PulsarRecord.<GenericRecord>builder().message(insertMessage).topicName("fake_topic_name").ackFunction(() -> future.complete(null)).build();
    genericAvroSchema = new GenericAvroSchema(schema.getSchemaInfo());
    when(insertMessage.getValue()).thenReturn(genericAvroSchema.decode(insertBytes));
    when(insertMessage.getProperties()).thenReturn(actionProperties);
    log.info("foo:{}, Message.getValue: {}, record.getValue: {}", insertObj.toString(), insertMessage.getValue().toString(), insertRecord.getValue().toString());
    // write should success.
    jdbcSink.write(insertRecord);
    log.info("executed write");
    // sleep to wait backend flush complete
    future.get(1, TimeUnit.SECONDS);
    // value has been written to db, read it out and verify.
    String querySql = "SELECT * FROM " + tableName + " WHERE field3=3";
    int count = sqliteUtils.select(querySql, (resultSet) -> {
        Assert.assertEquals(insertObj.getField1(), resultSet.getString(1));
        Assert.assertEquals(insertObj.getField2(), resultSet.getString(2));
        Assert.assertEquals(insertObj.getField3(), resultSet.getInt(3));
    });
    Assert.assertEquals(count, 1);
}

Example 19 with GenericRecord

use of org.apache.pulsar.client.api.schema.GenericRecord in project pulsar by apache.

the class SqliteJdbcSinkTest method testOpenAndWriteSinkNullValue.

private void testOpenAndWriteSinkNullValue(Map<String, String> actionProperties) throws Exception {
    Message<GenericRecord> insertMessage = mock(MessageImpl.class);
    GenericSchema<GenericRecord> genericAvroSchema;
    // prepare a foo Record
    Foo insertObj = new Foo();
    insertObj.setField1("ValueOfField1");
    // Not setting field2
    // Field1 is the key and field3 is used for selecting records
    insertObj.setField3(3);
    AvroSchema<Foo> schema = AvroSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).withAlwaysAllowNull(true).build());
    byte[] insertBytes = schema.encode(insertObj);
    CompletableFuture<Void> future = new CompletableFuture<>();
    Record<GenericRecord> insertRecord = PulsarRecord.<GenericRecord>builder().message(insertMessage).topicName("fake_topic_name").ackFunction(() -> future.complete(null)).build();
    genericAvroSchema = new GenericAvroSchema(schema.getSchemaInfo());
    when(insertMessage.getValue()).thenReturn(genericAvroSchema.decode(insertBytes));
    when(insertMessage.getProperties()).thenReturn(actionProperties);
    log.info("foo:{}, Message.getValue: {}, record.getValue: {}", insertObj.toString(), insertMessage.getValue().toString(), insertRecord.getValue().toString());
    // write should success.
    jdbcSink.write(insertRecord);
    log.info("executed write");
    // sleep to wait backend flush complete
    future.get(1, TimeUnit.SECONDS);
    // value has been written to db, read it out and verify.
    String querySql = "SELECT * FROM " + tableName + " WHERE field3=3";
    int count = sqliteUtils.select(querySql, (resultSet) -> {
        Assert.assertEquals(insertObj.getField1(), resultSet.getString(1));
        Assert.assertNull(insertObj.getField2());
        Assert.assertEquals(insertObj.getField3(), resultSet.getInt(3));
    });
    Assert.assertEquals(count, 1);
}

Example 20 with GenericRecord

use of org.apache.pulsar.client.api.schema.GenericRecord in project pulsar by apache.

the class SqliteJdbcSinkTest method TestUpdateAction.

@Test
public void TestUpdateAction() throws Exception {
    AvroSchema<Foo> schema = AvroSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).build());
    Foo updateObj = new Foo();
    updateObj.setField1("ValueOfField3");
    updateObj.setField2("ValueOfField3");
    updateObj.setField3(4);
    byte[] updateBytes = schema.encode(updateObj);
    Message<GenericRecord> updateMessage = mock(MessageImpl.class);
    CompletableFuture<Void> future = new CompletableFuture<>();
    Record<GenericRecord> updateRecord = PulsarRecord.<GenericRecord>builder().message(updateMessage).topicName("fake_topic_name").ackFunction(() -> future.complete(null)).build();
    GenericSchema<GenericRecord> updateGenericAvroSchema;
    updateGenericAvroSchema = new GenericAvroSchema(schema.getSchemaInfo());
    Map<String, String> updateProperties = Maps.newHashMap();
    updateProperties.put("ACTION", "UPDATE");
    when(updateMessage.getValue()).thenReturn(updateGenericAvroSchema.decode(updateBytes));
    when(updateMessage.getProperties()).thenReturn(updateProperties);
    log.info("foo:{}, Message.getValue: {}, record.getValue: {}", updateObj.toString(), updateMessage.getValue().toString(), updateRecord.getValue().toString());
    jdbcSink.write(updateRecord);
    future.get(1, TimeUnit.SECONDS);
    // value has been written to db, read it out and verify.
    String updateQuerySql = "SELECT * FROM " + tableName + " WHERE field3=4";
    sqliteUtils.select(updateQuerySql, (resultSet) -> {
        Assert.assertEquals(updateObj.getField1(), resultSet.getString(1));
        Assert.assertEquals(updateObj.getField2(), resultSet.getString(2));
        Assert.assertEquals(updateObj.getField3(), resultSet.getInt(3));
    });
}