Examples with Field org.apache.pulsar.client.api.schema.Field used on opensource projects

Example 11 with Field

use of org.apache.pulsar.client.api.schema.Field in project pulsar-sink by datastax.

the class CassandraSinkTask method write.

@Override
public void write(Record<T> record) throws Exception {
    if (log.isDebugEnabled()) {
        log.debug("write {} props {}", record, record.getProperties());
        Object rawvalue = record.getValue();
        if (rawvalue instanceof GenericRecord) {
            GenericRecord value = (GenericRecord) rawvalue;
            for (Field field : value.getFields()) {
                Object v = value.getField(field);
                String clazz = v != null ? v.getClass().toGenericString() : "";
                log.debug("field {} value {} class {}", field, v, clazz);
            }
        } else {
            log.debug("write {}", rawvalue);
        }
    }
    if (verbose) {
        Object rawvalue = record.getValue();
        log.info("received {} with headers {}", record, record.getProperties());
        if (rawvalue instanceof GenericRecord) {
            GenericRecord value = (GenericRecord) rawvalue;
            log.info("received GenericRecord {} native object {}", record, value.getNativeObject());
            for (Field field : value.getFields()) {
                Object v = value.getField(field);
                String clazz = v != null ? v.getClass().toGenericString() : "";
                log.info("field {} value {} class {}", field, v, clazz);
            }
        } else {
            log.info("raw {}", rawvalue);
        }
    }
    int number;
    synchronized (this) {
        incomingList.add(record);
        number = incomingList.size();
    }
    if (number == batchSize) {
        flushExecutor.submit(this::flush);
    }
}

Example 12 with Field

use of org.apache.pulsar.client.api.schema.Field in project pulsar by yahoo.

the class SourceTester method validateSourceResultAvro.

public void validateSourceResultAvro(Consumer<KeyValue<GenericRecord, GenericRecord>> consumer, int number, String eventType) throws Exception {
    int recordsNumber = 0;
    Message<KeyValue<GenericRecord, GenericRecord>> msg = consumer.receive(initialDelayForMsgReceive(), TimeUnit.SECONDS);
    while (msg != null) {
        recordsNumber++;
        GenericRecord keyRecord = msg.getValue().getKey();
        Assert.assertNotNull(keyRecord.getFields());
        Assert.assertTrue(keyRecord.getFields().size() > 0);
        GenericRecord valueRecord = msg.getValue().getValue();
        Assert.assertNotNull(valueRecord.getFields());
        Assert.assertTrue(valueRecord.getFields().size() > 0);
        log.info("Received message: key = {}, value = {}.", keyRecord.getNativeObject(), valueRecord.getNativeObject());
        for (Field field : valueRecord.getFields()) {
            log.info("validating field {}", field.getName());
            Assert.assertTrue(DEBEZIUM_FIELD_SET.contains(field.getName()));
        }
        if (eventType != null) {
            String op = valueRecord.getField("op").toString();
            Assert.assertEquals(this.eventContains(eventType, false), op);
        }
        consumer.acknowledge(msg);
        msg = consumer.receive(1, TimeUnit.SECONDS);
    }
    Assert.assertEquals(recordsNumber, number);
    log.info("Stop {} server container. topic: {} has {} records.", getSourceType(), consumer.getTopic(), recordsNumber);
}

Example 13 with Field

use of org.apache.pulsar.client.api.schema.Field in project pulsar by yahoo.

the class AvroKafkaSourceTest method validateSourceResultAvro.

public void validateSourceResultAvro(Consumer<GenericRecord> consumer, List<MyBean> beans) throws Exception {
    int recordsNumber = 0;
    Message<GenericRecord> msg = consumer.receive(10, TimeUnit.SECONDS);
    while (msg != null) {
        GenericRecord valueRecord = msg.getValue();
        Assert.assertNotNull(valueRecord.getFields());
        Assert.assertTrue(valueRecord.getFields().size() > 0);
        for (Field field : valueRecord.getFields()) {
            log.info("field {} value {}", field, valueRecord.getField(field));
        }
        assertEquals(beans.get(recordsNumber).field, valueRecord.getField("field"));
        consumer.acknowledge(msg);
        recordsNumber++;
        msg = consumer.receive(10, TimeUnit.SECONDS);
    }
    Assert.assertEquals(recordsNumber, beans.size());
    log.info("Stop {} server container. topic: {} has {} records.", kafkaContainerName, consumer.getTopic(), recordsNumber);
}

Example 14 with Field

use of org.apache.pulsar.client.api.schema.Field in project pulsar by yahoo.

the class CmdConsume method genericRecordToMap.

private static Map<String, Object> genericRecordToMap(GenericRecord value, boolean displayHex) throws IOException {
    Map<String, Object> res = new HashMap<>();
    for (Field f : value.getFields()) {
        Object fieldValue = value.getField(f);
        if (fieldValue instanceof GenericRecord) {
            fieldValue = genericRecordToMap((GenericRecord) fieldValue, displayHex);
        } else if (fieldValue == null) {
            fieldValue = "NULL";
        } else if (fieldValue instanceof byte[]) {
            fieldValue = interpretByteArray(displayHex, (byte[]) fieldValue);
        }
        res.put(f.getName(), fieldValue);
    }
    return res;
}

Example 15 with Field

use of org.apache.pulsar.client.api.schema.Field in project pulsar by yahoo.

the class InfluxDBSink method buildPoint.

@Override
protected final Point buildPoint(Record<GenericRecord> record) {
    val genericRecord = record.getValue();
    // looking for measurement
    val measurementField = genericRecord.getField("measurement");
    if (null == measurementField) {
        throw new SchemaSerializationException("measurement is a required field.");
    }
    val measurement = (String) measurementField;
    // looking for timestamp
    long timestamp;
    val timestampField = getFiled(genericRecord, "timestamp");
    if (null == timestampField) {
        timestamp = System.currentTimeMillis();
    } else if (timestampField instanceof Number) {
        timestamp = ((Number) timestampField).longValue();
    } else if (timestampField instanceof String) {
        timestamp = Long.parseLong((String) timestampField);
    } else {
        throw new SchemaSerializationException("Invalid timestamp field");
    }
    val point = Point.measurement(measurement).time(timestamp, writePrecision);
    // Looking for tag fields
    val tagsField = getFiled(genericRecord, "tags");
    if (null != tagsField) {
        if (tagsField instanceof GenericRecord) {
            // JSONSchema
            GenericRecord tagsRecord = (GenericRecord) tagsField;
            for (Field field : tagsRecord.getFields()) {
                val fieldName = field.getName();
                val value = tagsRecord.getField(field);
                point.addTag(fieldName, (String) value);
            }
        } else if (Map.class.isAssignableFrom(tagsField.getClass())) {
            // AvroSchema
            Map<Object, Object> tagsMap = (Map<Object, Object>) tagsField;
            tagsMap.forEach((key, value) -> point.addTag(key.toString(), value.toString()));
        } else {
            throw new SchemaSerializationException("Unknown type for 'tags'");
        }
    }
    // Looking for sensor fields
    val columnsField = genericRecord.getField("fields");
    if (columnsField instanceof GenericRecord) {
        // JSONSchema
        val columnsRecord = (GenericRecord) columnsField;
        for (Field field : columnsRecord.getFields()) {
            val fieldName = field.getName();
            val value = columnsRecord.getField(field);
            addPointField(point, fieldName, value);
        }
    } else if (Map.class.isAssignableFrom(columnsField.getClass())) {
        // AvroSchema
        val columnsMap = (Map<Object, Object>) columnsField;
        columnsMap.forEach((key, value) -> addPointField(point, key.toString(), value));
    } else {
        throw new SchemaSerializationException("Unknown type for 'fields'");
    }
    return point;
}