Examples with Record org.apache.pulsar.functions.api.Record used on opensource projects

Example 1 with Record

use of org.apache.pulsar.functions.api.Record in project pulsar by apache.

the class ElasticSearchExtractTests method testKeyValueGenericRecord.

@Test(dataProvider = "schemaType")
public void testKeyValueGenericRecord(SchemaType schemaType) throws Exception {
    RecordSchemaBuilder keySchemaBuilder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("key");
    keySchemaBuilder.field("a").type(SchemaType.STRING).optional().defaultValue(null);
    keySchemaBuilder.field("b").type(SchemaType.INT32).optional().defaultValue(null);
    GenericSchema<GenericRecord> keySchema = Schema.generic(keySchemaBuilder.build(schemaType));
    GenericRecord keyGenericRecord = keySchema.newRecordBuilder().set("a", "1").set("b", 1).build();
    RecordSchemaBuilder valueSchemaBuilder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("value");
    valueSchemaBuilder.field("c").type(SchemaType.STRING).optional().defaultValue(null);
    valueSchemaBuilder.field("d").type(SchemaType.INT32).optional().defaultValue(null);
    RecordSchemaBuilder udtSchemaBuilder = SchemaBuilder.record("type1");
    udtSchemaBuilder.field("a").type(SchemaType.STRING).optional().defaultValue(null);
    udtSchemaBuilder.field("b").type(SchemaType.BOOLEAN).optional().defaultValue(null);
    udtSchemaBuilder.field("d").type(SchemaType.DOUBLE).optional().defaultValue(null);
    udtSchemaBuilder.field("f").type(SchemaType.FLOAT).optional().defaultValue(null);
    udtSchemaBuilder.field("i").type(SchemaType.INT32).optional().defaultValue(null);
    udtSchemaBuilder.field("l").type(SchemaType.INT64).optional().defaultValue(null);
    GenericSchema<GenericRecord> udtGenericSchema = Schema.generic(udtSchemaBuilder.build(schemaType));
    valueSchemaBuilder.field("e", udtGenericSchema).type(schemaType).optional().defaultValue(null);
    GenericSchema<GenericRecord> valueSchema = Schema.generic(valueSchemaBuilder.build(schemaType));
    GenericRecord valueGenericRecord = valueSchema.newRecordBuilder().set("c", "1").set("d", 1).set("e", udtGenericSchema.newRecordBuilder().set("a", "a").set("b", true).set("d", 1.0).set("f", 1.0f).set("i", 1).set("l", 10L).build()).build();
    Schema<KeyValue<GenericRecord, GenericRecord>> keyValueSchema = Schema.KeyValue(keySchema, valueSchema, KeyValueEncodingType.INLINE);
    KeyValue<GenericRecord, GenericRecord> keyValue = new KeyValue<>(keyGenericRecord, valueGenericRecord);
    GenericObject genericObject = new GenericObject() {

        @Override
        public SchemaType getSchemaType() {
            return SchemaType.KEY_VALUE;
        }

        @Override
        public Object getNativeObject() {
            return keyValue;
        }
    };
    Record<GenericObject> genericObjectRecord = new Record<GenericObject>() {

        @Override
        public Optional<String> getTopicName() {
            return Optional.of("data-ks1.table1");
        }

        @Override
        public org.apache.pulsar.client.api.Schema getSchema() {
            return keyValueSchema;
        }

        @Override
        public GenericObject getValue() {
            return genericObject;
        }
    };
    ElasticSearchSink elasticSearchSink = new ElasticSearchSink();
    elasticSearchSink.open(ImmutableMap.of("elasticSearchUrl", "http://localhost:9200", "schemaEnable", "true", "keyIgnore", "false"), null);
    Pair<String, String> pair = elasticSearchSink.extractIdAndDocument(genericObjectRecord);
    assertEquals(pair.getLeft(), "[\"1\",1]");
    assertEquals(pair.getRight(), "{\"c\":\"1\",\"d\":1,\"e\":{\"a\":\"a\",\"b\":true,\"d\":1.0,\"f\":1.0,\"i\":1,\"l\":10}}");
    ElasticSearchSink elasticSearchSink2 = new ElasticSearchSink();
    elasticSearchSink2.open(ImmutableMap.of("elasticSearchUrl", "http://localhost:9200", "schemaEnable", "true"), null);
    Pair<String, String> pair2 = elasticSearchSink2.extractIdAndDocument(genericObjectRecord);
    assertNull(pair2.getLeft());
    assertEquals(pair2.getRight(), "{\"c\":\"1\",\"d\":1,\"e\":{\"a\":\"a\",\"b\":true,\"d\":1.0,\"f\":1.0,\"i\":1,\"l\":10}}");
    // test null value
    ElasticSearchSink elasticSearchSink3 = new ElasticSearchSink();
    elasticSearchSink3.open(ImmutableMap.of("elasticSearchUrl", "http://localhost:9200", "schemaEnable", "true", "keyIgnore", "false"), null);
    Pair<String, String> pair3 = elasticSearchSink.extractIdAndDocument(new Record<GenericObject>() {

        @Override
        public Optional<String> getTopicName() {
            return Optional.of("data-ks1.table1");
        }

        @Override
        public org.apache.pulsar.client.api.Schema getSchema() {
            return keyValueSchema;
        }

        @Override
        public GenericObject getValue() {
            return new GenericObject() {

                @Override
                public SchemaType getSchemaType() {
                    return SchemaType.KEY_VALUE;
                }

                @Override
                public Object getNativeObject() {
                    return new KeyValue<>(keyGenericRecord, null);
                }
            };
        }
    });
    assertEquals(pair3.getLeft(), "[\"1\",1]");
    assertNull(pair3.getRight());
}

Example 2 with Record

use of org.apache.pulsar.functions.api.Record in project pulsar by apache.

the class IndexNameFormatterTest method testIndexFormats.

@Test(dataProvider = "indexFormats")
public void testIndexFormats(String format, String result) {
    Record record = Mockito.mock(Record.class);
    when(record.getEventTime()).thenReturn(Optional.of(1645182000000L));
    IndexNameFormatter formatter = new IndexNameFormatter(format);
    assertEquals(formatter.indexName(record), result);
}

Example 3 with Record

use of org.apache.pulsar.functions.api.Record in project pulsar by apache.

the class InfluxDBGenericRecordSink method buildPoint.

@Override
protected Point buildPoint(Record<GenericRecord> message) throws Exception {
    Map<String, String> tags;
    Map<String, Object> fields = Maps.newHashMap();
    GenericRecord record = message.getValue();
    Object measurementField = getFiled(record, "measurement");
    if (null == measurementField) {
        throw new SchemaSerializationException("measurement is a required field.");
    }
    String measurement = measurementField.toString();
    // Looking for tags
    Object tagsField = getFiled(record, "tags");
    if (null == tagsField) {
        tags = ImmutableMap.of();
    } else if (Map.class.isAssignableFrom(tagsField.getClass())) {
        tags = ((Map<Object, Object>) tagsField).entrySet().stream().collect(Collectors.toMap(entry -> entry.getKey().toString(), entry -> entry.getValue().toString()));
    } else {
        // Field 'tags' that is not of Map type will be ignored
        tags = ImmutableMap.of();
    }
    // Just insert the current time millis
    long timestamp = System.currentTimeMillis();
    for (Field field : record.getFields()) {
        String fieldName = field.getName();
        if (FIELDS_TO_SKIP.contains(fieldName)) {
            continue;
        }
        Object fieldValue = record.getField(field);
        if (null != fieldValue) {
            fields.put(fieldName, fieldValue);
        }
    }
    Point.Builder builder = Point.measurement(measurement).time(timestamp, TimeUnit.MILLISECONDS).tag(tags).fields(fields);
    return builder.build();
}

Example 4 with Record

use of org.apache.pulsar.functions.api.Record in project pulsar by apache.

the class JavaInstanceTest method testNullReturningAsyncFunction.

@Test
public void testNullReturningAsyncFunction() throws Exception {
    InstanceConfig instanceConfig = new InstanceConfig();
    @Cleanup("shutdownNow") ExecutorService executor = Executors.newCachedThreadPool();
    Function<String, CompletableFuture<String>> function = (input, context) -> {
        log.info("input string: {}", input);
        CompletableFuture<String> result = new CompletableFuture<>();
        executor.submit(() -> {
            try {
                Thread.sleep(500);
                result.complete(null);
            } catch (Exception e) {
                result.completeExceptionally(e);
            }
        });
        return result;
    };
    JavaInstance instance = new JavaInstance(mock(ContextImpl.class), function, instanceConfig);
    String testString = "ABC123";
    CompletableFuture<JavaExecutionResult> resultHolder = new CompletableFuture<>();
    JavaExecutionResult result = instance.handleMessage(mock(Record.class), testString, (record, javaResult) -> resultHolder.complete(javaResult), cause -> {
    });
    assertNull(result);
    assertNotNull(resultHolder.get());
    instance.close();
}

Example 5 with Record

use of org.apache.pulsar.functions.api.Record in project pulsar by apache.

the class GenericRecordSource method read.

@Override
public Record<GenericRecord> read() throws Exception {
    // slow down the production of values
    Thread.sleep(20);
    int value = count.incrementAndGet();
    GenericRecord record = schema.newRecordBuilder().set("number", value).set("text", "value-" + value).build();
    log.info("produced {}", record);
    return new Record<GenericRecord>() {

        @Override
        public GenericRecord getValue() {
            return record;
        }

        @Override
        public Schema<GenericRecord> getSchema() {
            return schema;
        }
    };
}