Examples with Record org.apache.pulsar.functions.api.Record used on opensource projects

Example 16 with Record

use of org.apache.pulsar.functions.api.Record in project incubator-pulsar by apache.

the class SqliteJdbcSinkTest method testKeyValueSchema.

@Test(dataProvider = "schemaType")
public void testKeyValueSchema(SchemaType schemaType) throws Exception {
    RecordSchemaBuilder keySchemaBuilder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("key");
    keySchemaBuilder.field("key").type(SchemaType.STRING).optional().defaultValue(null);
    GenericSchema<GenericRecord> keySchema = Schema.generic(keySchemaBuilder.build(schemaType));
    GenericRecord keyGenericRecord = keySchema.newRecordBuilder().set("key", "mykey").build();
    RecordSchemaBuilder valueSchemaBuilder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("value");
    valueSchemaBuilder.field("string").type(SchemaType.STRING).optional().defaultValue(null);
    valueSchemaBuilder.field("stringutf8").type(SchemaType.STRING).optional().defaultValue(null);
    valueSchemaBuilder.field("int").type(SchemaType.INT32).optional().defaultValue(null);
    valueSchemaBuilder.field("bool").type(SchemaType.BOOLEAN).optional().defaultValue(null);
    valueSchemaBuilder.field("double").type(SchemaType.DOUBLE).optional().defaultValue(null);
    valueSchemaBuilder.field("float").type(SchemaType.FLOAT).optional().defaultValue(null);
    valueSchemaBuilder.field("long").type(SchemaType.INT64).optional().defaultValue(null);
    GenericSchema<GenericRecord> valueSchema = Schema.generic(valueSchemaBuilder.build(schemaType));
    GenericRecord valueGenericRecord = valueSchema.newRecordBuilder().set("string", "thestring").set("stringutf8", schemaType == SchemaType.AVRO ? new Utf8("thestringutf8") : "thestringutf8").set("int", Integer.MAX_VALUE).set("bool", true).set("double", Double.MAX_VALUE).set("float", Float.MAX_VALUE).set("long", Long.MIN_VALUE).build();
    Schema<KeyValue<GenericRecord, GenericRecord>> keyValueSchema = Schema.KeyValue(keySchema, valueSchema, KeyValueEncodingType.INLINE);
    KeyValue<GenericRecord, GenericRecord> keyValue = new KeyValue<>(keyGenericRecord, valueGenericRecord);
    GenericObject genericObject = new GenericObject() {

        @Override
        public SchemaType getSchemaType() {
            return SchemaType.KEY_VALUE;
        }

        @Override
        public Object getNativeObject() {
            return keyValue;
        }
    };
    Record<GenericObject> genericObjectRecord = new Record<>() {

        @Override
        public Optional<String> getTopicName() {
            return Optional.of("topic");
        }

        @Override
        public org.apache.pulsar.client.api.Schema getSchema() {
            return keyValueSchema;
        }

        @Override
        public GenericObject getValue() {
            return genericObject;
        }
    };
    jdbcSink.close();
    sqliteUtils.createTable("CREATE TABLE kvtable (" + "    key  TEXT," + "    int  INTEGER," + "    string TEXT," + "    stringutf8 TEXT," + "    nulltext  TEXT," + "    bool  NUMERIC," + "    double NUMERIC," + "    float NUMERIC," + "    long INTEGER," + "PRIMARY KEY (key));");
    String jdbcUrl = sqliteUtils.sqliteUri();
    Map<String, Object> conf = Maps.newHashMap();
    conf.put("jdbcUrl", jdbcUrl);
    conf.put("tableName", "kvtable");
    conf.put("key", "key");
    conf.put("nonKey", "long,int,double,float,bool,nulltext,string,stringutf8");
    // change batchSize to 1, to flush on each write.
    conf.put("batchSize", 1);
    try (SqliteJdbcAutoSchemaSink kvSchemaJdbcSink = new SqliteJdbcAutoSchemaSink()) {
        kvSchemaJdbcSink.open(conf, null);
        kvSchemaJdbcSink.write(genericObjectRecord);
        Awaitility.await().untilAsserted(() -> {
            final int count = sqliteUtils.select("select int,string,stringutf8,bool,double,float," + "long,nulltext from kvtable where key='mykey'", (resultSet) -> {
                int index = 1;
                Assert.assertEquals(resultSet.getInt(index++), Integer.MAX_VALUE);
                Assert.assertEquals(resultSet.getString(index++), "thestring");
                Assert.assertEquals(resultSet.getString(index++), "thestringutf8");
                Assert.assertEquals(resultSet.getBoolean(index++), true);
                Assert.assertEquals(resultSet.getDouble(index++), Double.MAX_VALUE);
                Assert.assertEquals(resultSet.getFloat(index++), Float.MAX_VALUE);
                Assert.assertEquals(resultSet.getLong(index++), Long.MIN_VALUE);
                Assert.assertNull(resultSet.getString(index++));
            });
            Assert.assertEquals(count, 1);
        });
    }
}

Example 17 with Record

use of org.apache.pulsar.functions.api.Record in project incubator-pulsar by apache.

the class KafkaConnectSinkTest method genericRecordCastTest.

@Test
public void genericRecordCastTest() throws Exception {
    props.put("kafkaConnectorSinkClass", SchemaedFileStreamSinkConnector.class.getCanonicalName());
    KafkaConnectSink sink = new KafkaConnectSink();
    sink.open(props, context);
    AvroSchema<PulsarSchemaToKafkaSchemaTest.StructWithAnnotations> pulsarAvroSchema = AvroSchema.of(PulsarSchemaToKafkaSchemaTest.StructWithAnnotations.class);
    final GenericData.Record obj = new GenericData.Record(pulsarAvroSchema.getAvroSchema());
    // schema type INT32
    obj.put("field1", (byte) 10);
    // schema type STRING
    obj.put("field2", "test");
    // schema type INT64
    obj.put("field3", (short) 100);
    final GenericRecord rec = getGenericRecord(obj, pulsarAvroSchema);
    Message msg = mock(MessageImpl.class);
    when(msg.getValue()).thenReturn(rec);
    when(msg.getKey()).thenReturn("key");
    when(msg.hasKey()).thenReturn(true);
    when(msg.getMessageId()).thenReturn(new MessageIdImpl(1, 0, 0));
    final AtomicInteger status = new AtomicInteger(0);
    Record<GenericObject> record = PulsarRecord.<String>builder().topicName("fake-topic").message(msg).schema(pulsarAvroSchema).ackFunction(status::incrementAndGet).failFunction(status::decrementAndGet).build();
    SinkRecord sinkRecord = sink.toSinkRecord(record);
    Struct out = (Struct) sinkRecord.value();
    Assert.assertEquals(out.get("field1").getClass(), Integer.class);
    Assert.assertEquals(out.get("field2").getClass(), String.class);
    Assert.assertEquals(out.get("field3").getClass(), Long.class);
    Assert.assertEquals(out.get("field1"), 10);
    Assert.assertEquals(out.get("field2"), "test");
    Assert.assertEquals(out.get("field3"), 100L);
    sink.close();
}

Example 18 with Record

use of org.apache.pulsar.functions.api.Record in project incubator-pulsar by apache.

the class InfluxDBGenericRecordSink method buildPoint.

@Override
protected Point buildPoint(Record<GenericRecord> message) throws Exception {
    Map<String, String> tags;
    Map<String, Object> fields = Maps.newHashMap();
    GenericRecord record = message.getValue();
    Object measurementField = getFiled(record, "measurement");
    if (null == measurementField) {
        throw new SchemaSerializationException("measurement is a required field.");
    }
    String measurement = measurementField.toString();
    // Looking for tags
    Object tagsField = getFiled(record, "tags");
    if (null == tagsField) {
        tags = ImmutableMap.of();
    } else if (Map.class.isAssignableFrom(tagsField.getClass())) {
        tags = ((Map<Object, Object>) tagsField).entrySet().stream().collect(Collectors.toMap(entry -> entry.getKey().toString(), entry -> entry.getValue().toString()));
    } else {
        // Field 'tags' that is not of Map type will be ignored
        tags = ImmutableMap.of();
    }
    // Just insert the current time millis
    long timestamp = System.currentTimeMillis();
    for (Field field : record.getFields()) {
        String fieldName = field.getName();
        if (FIELDS_TO_SKIP.contains(fieldName)) {
            continue;
        }
        Object fieldValue = record.getField(field);
        if (null != fieldValue) {
            fields.put(fieldName, fieldValue);
        }
    }
    Point.Builder builder = Point.measurement(measurement).time(timestamp, TimeUnit.MILLISECONDS).tag(tags).fields(fields);
    return builder.build();
}

Example 19 with Record

use of org.apache.pulsar.functions.api.Record in project incubator-pulsar by apache.

the class ElasticSearchExtractTests method getKeyValueGenericObject.

private Record<GenericObject> getKeyValueGenericObject(SchemaType schemaType, GenericSchema<GenericRecord> keySchema, GenericRecord keyGenericRecord) {
    RecordSchemaBuilder valueSchemaBuilder = SchemaBuilder.record("value");
    valueSchemaBuilder.field("value").type(SchemaType.STRING);
    GenericSchema<GenericRecord> valueSchema = Schema.generic(valueSchemaBuilder.build(schemaType));
    GenericRecord valueGenericRecord = valueSchema.newRecordBuilder().set("value", "value").build();
    Schema<KeyValue<GenericRecord, GenericRecord>> keyValueSchema = Schema.KeyValue(keySchema, valueSchema, KeyValueEncodingType.INLINE);
    KeyValue<GenericRecord, GenericRecord> keyValue = new KeyValue<>(keyGenericRecord, valueGenericRecord);
    GenericObject genericObject = new GenericObject() {

        @Override
        public SchemaType getSchemaType() {
            return SchemaType.KEY_VALUE;
        }

        @Override
        public Object getNativeObject() {
            return keyValue;
        }
    };
    Record<GenericObject> genericObjectRecord = new Record<GenericObject>() {

        @Override
        public Optional<String> getTopicName() {
            return Optional.of("data-ks1.table1");
        }

        @Override
        public Schema getSchema() {
            return keyValueSchema;
        }

        @Override
        public GenericObject getValue() {
            return genericObject;
        }
    };
    return genericObjectRecord;
}

Example 20 with Record

use of org.apache.pulsar.functions.api.Record in project incubator-pulsar by apache.

the class ElasticSearchExtractTests method testGenericRecord.

@Test(dataProvider = "schemaType")
public void testGenericRecord(SchemaType schemaType) throws Exception {
    RecordSchemaBuilder valueSchemaBuilder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("value");
    valueSchemaBuilder.field("c").type(SchemaType.STRING).optional().defaultValue(null);
    valueSchemaBuilder.field("d").type(SchemaType.INT32).optional().defaultValue(null);
    RecordSchemaBuilder udtSchemaBuilder = SchemaBuilder.record("type1");
    udtSchemaBuilder.field("a").type(SchemaType.STRING).optional().defaultValue(null);
    udtSchemaBuilder.field("b").type(SchemaType.BOOLEAN).optional().defaultValue(null);
    udtSchemaBuilder.field("d").type(SchemaType.DOUBLE).optional().defaultValue(null);
    udtSchemaBuilder.field("f").type(SchemaType.FLOAT).optional().defaultValue(null);
    udtSchemaBuilder.field("i").type(SchemaType.INT32).optional().defaultValue(null);
    udtSchemaBuilder.field("l").type(SchemaType.INT64).optional().defaultValue(null);
    GenericSchema<GenericRecord> udtGenericSchema = Schema.generic(udtSchemaBuilder.build(schemaType));
    valueSchemaBuilder.field("e", udtGenericSchema).type(schemaType).optional().defaultValue(null);
    GenericSchema<GenericRecord> valueSchema = Schema.generic(valueSchemaBuilder.build(schemaType));
    GenericRecord valueGenericRecord = valueSchema.newRecordBuilder().set("c", "1").set("d", 1).set("e", udtGenericSchema.newRecordBuilder().set("a", "a").set("b", true).set("d", 1.0).set("f", 1.0f).set("i", 1).set("l", 10L).build()).build();
    Record<GenericObject> genericObjectRecord = new Record<GenericObject>() {

        @Override
        public Optional<String> getTopicName() {
            return Optional.of("data-ks1.table1");
        }

        @Override
        public org.apache.pulsar.client.api.Schema getSchema() {
            return valueSchema;
        }

        @Override
        public GenericObject getValue() {
            return valueGenericRecord;
        }
    };
    // single field PK
    ElasticSearchSink elasticSearchSink = new ElasticSearchSink();
    elasticSearchSink.open(ImmutableMap.of("elasticSearchUrl", "http://localhost:9200", "compatibilityMode", "ELASTICSEARCH", "primaryFields", "c", "schemaEnable", "true", "keyIgnore", "true"), null);
    Pair<String, String> pair = elasticSearchSink.extractIdAndDocument(genericObjectRecord);
    assertEquals(pair.getLeft(), "1");
    assertEquals(pair.getRight(), "{\"c\":\"1\",\"d\":1,\"e\":{\"a\":\"a\",\"b\":true,\"d\":1.0,\"f\":1.0,\"i\":1,\"l\":10}}");
    // two fields PK
    ElasticSearchSink elasticSearchSink2 = new ElasticSearchSink();
    elasticSearchSink2.open(ImmutableMap.of("elasticSearchUrl", "http://localhost:9200", "compatibilityMode", "ELASTICSEARCH", "primaryFields", "c,d", "schemaEnable", "true", "keyIgnore", "true"), null);
    Pair<String, String> pair2 = elasticSearchSink2.extractIdAndDocument(genericObjectRecord);
    assertEquals(pair2.getLeft(), "[\"1\",1]");
    assertEquals(pair2.getRight(), "{\"c\":\"1\",\"d\":1,\"e\":{\"a\":\"a\",\"b\":true,\"d\":1.0,\"f\":1.0,\"i\":1,\"l\":10}}");
    // default config with null PK => indexed with auto generated _id
    ElasticSearchSink elasticSearchSink3 = new ElasticSearchSink();
    elasticSearchSink3.open(ImmutableMap.of("elasticSearchUrl", "http://localhost:9200", "schemaEnable", "true", "compatibilityMode", "ELASTICSEARCH"), null);
    Pair<String, String> pair3 = elasticSearchSink3.extractIdAndDocument(genericObjectRecord);
    assertNull(pair3.getLeft());
    assertEquals(pair3.getRight(), "{\"c\":\"1\",\"d\":1,\"e\":{\"a\":\"a\",\"b\":true,\"d\":1.0,\"f\":1.0,\"i\":1,\"l\":10}}");
    // default config with null PK + null value
    ElasticSearchSink elasticSearchSink4 = new ElasticSearchSink();
    elasticSearchSink4.open(ImmutableMap.of("elasticSearchUrl", "http://localhost:9200", "compatibilityMode", "ELASTICSEARCH", "schemaEnable", "true"), null);
    Pair<String, String> pair4 = elasticSearchSink3.extractIdAndDocument(new Record<GenericObject>() {

        @Override
        public Optional<String> getTopicName() {
            return Optional.of("data-ks1.table1");
        }

        @Override
        public org.apache.pulsar.client.api.Schema getSchema() {
            return valueSchema;
        }

        @Override
        public GenericObject getValue() {
            return null;
        }
    });
    assertNull(pair4.getLeft());
    assertNull(pair4.getRight());
}