Examples with GenericRecord org.apache.pulsar.client.api.schema.GenericRecord used on opensource projects

Example 21 with GenericRecord

use of org.apache.pulsar.client.api.schema.GenericRecord in project pulsar by apache.

the class KafkaConnectSinkTest method smokeTest.

@Test
public void smokeTest() throws Exception {
    KafkaConnectSink sink = new KafkaConnectSink();
    sink.open(props, context);
    final GenericRecord rec = getGenericRecord("value", Schema.STRING);
    Message msg = mock(MessageImpl.class);
    when(msg.getValue()).thenReturn(rec);
    when(msg.getMessageId()).thenReturn(new MessageIdImpl(1, 0, 0));
    final AtomicInteger status = new AtomicInteger(0);
    Record<GenericObject> record = PulsarRecord.<String>builder().topicName("fake-topic").message(msg).ackFunction(status::incrementAndGet).failFunction(status::decrementAndGet).schema(Schema.STRING).build();
    sink.write(record);
    sink.flush();
    assertEquals(status.get(), 1);
    sink.close();
    List<String> lines = Files.readAllLines(file, StandardCharsets.US_ASCII);
    assertEquals(lines.get(0), "value");
}

Example 22 with GenericRecord

use of org.apache.pulsar.client.api.schema.GenericRecord in project pulsar by apache.

the class KafkaConnectSinkTest method unknownRecordSchemaTest.

@Test
public void unknownRecordSchemaTest() throws Exception {
    Object obj = new Object();
    props.put("kafkaConnectorSinkClass", SchemaedFileStreamSinkConnector.class.getCanonicalName());
    KafkaConnectSink sink = new KafkaConnectSink();
    sink.open(props, context);
    final GenericRecord rec = getGenericRecord(obj, null);
    Message msg = mock(MessageImpl.class);
    when(msg.getValue()).thenReturn(rec);
    when(msg.getKey()).thenReturn("key");
    when(msg.hasKey()).thenReturn(true);
    when(msg.getMessageId()).thenReturn(new MessageIdImpl(0, 0, 0));
    final AtomicInteger status = new AtomicInteger(0);
    Record<GenericObject> record = PulsarRecord.<String>builder().topicName("fake-topic").message(msg).ackFunction(status::incrementAndGet).failFunction(status::decrementAndGet).build();
    sink.write(record);
    sink.flush();
    assertEquals(status.get(), -1, "write should fail for unsupported schema");
    sink.close();
}

Example 23 with GenericRecord

use of org.apache.pulsar.client.api.schema.GenericRecord in project pulsar by apache.

the class InfluxDBSinkTest method testAvroSchema.

@Test
public void testAvroSchema() {
    AvroSchema<Cpu> schema = AvroSchema.of(Cpu.class);
    AutoConsumeSchema autoConsumeSchema = new AutoConsumeSchema();
    autoConsumeSchema.setSchema(GenericSchemaImpl.of(schema.getSchemaInfo()));
    GenericSchema<GenericRecord> genericAvroSchema = GenericSchemaImpl.of(autoConsumeSchema.getSchemaInfo());
    assertTrue(genericAvroSchema instanceof GenericAvroSchema);
    byte[] bytes = schema.encode(cpu);
    GenericRecord record = genericAvroSchema.decode(bytes);
    assertEquals(record.getField("measurement"), "cpu");
    assertEquals(record.getField("timestamp"), timestamp);
    assertEquals(((Map) record.getField("tags")).get(new Utf8("host")).toString(), "server-1");
    assertEquals(((Map) record.getField("fields")).get(new Utf8("value")), 10);
}

Example 24 with GenericRecord

use of org.apache.pulsar.client.api.schema.GenericRecord in project pulsar by apache.

the class InfluxDBGenericRecordSink method buildPoint.

@Override
protected Point buildPoint(Record<GenericRecord> message) throws Exception {
    Map<String, String> tags;
    Map<String, Object> fields = Maps.newHashMap();
    GenericRecord record = message.getValue();
    Object measurementField = getFiled(record, "measurement");
    if (null == measurementField) {
        throw new SchemaSerializationException("measurement is a required field.");
    }
    String measurement = measurementField.toString();
    // Looking for tags
    Object tagsField = getFiled(record, "tags");
    if (null == tagsField) {
        tags = ImmutableMap.of();
    } else if (Map.class.isAssignableFrom(tagsField.getClass())) {
        tags = ((Map<Object, Object>) tagsField).entrySet().stream().collect(Collectors.toMap(entry -> entry.getKey().toString(), entry -> entry.getValue().toString()));
    } else {
        // Field 'tags' that is not of Map type will be ignored
        tags = ImmutableMap.of();
    }
    // Just insert the current time millis
    long timestamp = System.currentTimeMillis();
    for (Field field : record.getFields()) {
        String fieldName = field.getName();
        if (FIELDS_TO_SKIP.contains(fieldName)) {
            continue;
        }
        Object fieldValue = record.getField(field);
        if (null != fieldValue) {
            fields.put(fieldName, fieldValue);
        }
    }
    Point.Builder builder = Point.measurement(measurement).time(timestamp, TimeUnit.MILLISECONDS).tag(tags).fields(fields);
    return builder.build();
}

Example 25 with GenericRecord

use of org.apache.pulsar.client.api.schema.GenericRecord in project pulsar by apache.

the class SchemaUpdateStrategyTest method testAutoUpdateBackward.

private void testAutoUpdateBackward(String namespace, String topicName) throws Exception {
    ContainerExecResult result = pulsarCluster.runAdminCommandOnAnyBroker("namespaces", "get-schema-autoupdate-strategy", namespace);
    Assert.assertEquals(result.getStdout().trim(), "FULL");
    pulsarCluster.runAdminCommandOnAnyBroker("namespaces", "set-schema-autoupdate-strategy", "--compatibility", "BACKWARD", namespace);
    try (PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(pulsarCluster.getPlainTextServiceUrl()).build()) {
        V1Data v1Data = new V1Data("test1", 1);
        try (Producer<V1Data> p = pulsarClient.newProducer(Schema.AVRO(V1Data.class)).topic(topicName).create()) {
            p.send(v1Data);
        }
        log.info("try with forward compat, should fail");
        try (Producer<V3Data> p = pulsarClient.newProducer(Schema.AVRO(V3Data.class)).topic(topicName).create()) {
            Assert.fail("Forward compat schema should be rejected");
        } catch (PulsarClientException e) {
            Assert.assertTrue(e.getMessage().contains("IncompatibleSchemaException"));
        }
        log.info("try with backward compat, should succeed");
        V2Data v2Data = new V2Data("test2");
        try (Producer<V2Data> p = pulsarClient.newProducer(Schema.AVRO(V2Data.class)).topic(topicName).create()) {
            p.send(v2Data);
        }
        Schema<GenericRecord> schema = Schema.AUTO_CONSUME();
        try (Consumer<GenericRecord> consumer = pulsarClient.newConsumer(schema).topic(topicName).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscriptionName("sub").subscribe()) {
            log.info("Schema Info : {}", schema.getSchemaInfo().getSchemaDefinition());
            Message<GenericRecord> msg1 = consumer.receive();
            v1Data.assertEqualToRecord(msg1.getValue());
            Message<GenericRecord> msg2 = consumer.receive();
            v2Data.assertEqualToRecord(msg2.getValue());
        }
    }
}