Examples with GenericRecordBuilder org.apache.pulsar.client.api.schema.GenericRecordBuilder used on opensource projects

Example 1 with GenericRecordBuilder

use of org.apache.pulsar.client.api.schema.GenericRecordBuilder in project pulsar-flink by streamnative.

the class PulsarSerializer method newStructConverter.

private Function<Object, Object> newStructConverter(FieldsDataType dataType, Schema avroStruct) throws IncompatibleSchemaException {
    if (avroStruct.getType() != Schema.Type.RECORD || avroStruct.getFields().size() != dataType.getChildren().size()) {
        throw new IncompatibleSchemaException(String.format("Cannot convert Flink type %s to Avro type %s.", dataType.toString(), avroStruct.toString(true)));
    }
    RowType rowType = (RowType) dataType.getLogicalType();
    List<RowType.RowField> fields = rowType.getFields();
    List<BiFunction<PositionedGetter, Integer, Object>> fieldConverters = new ArrayList<>();
    for (int i = 0; i < fields.size(); i++) {
        org.apache.flink.table.types.logical.LogicalType logicalType = rowType.getTypeAt(i);
        DataType dt = TypeConversions.fromLogicalToDataType(logicalType);
        Schema.Field at = avroStruct.getFields().get(i);
        fieldConverters.add(newConverter(dt, resolveNullableType(at.schema(), dt.getLogicalType().isNullable())));
    }
    int numFields = dataType.getChildren().size();
    return row -> {
        GenericSchema<GenericRecord> pSchema = SchemaUtils.avroSchema2PulsarSchema(avroStruct);
        GenericRecordBuilder builder = pSchema.newRecordBuilder();
        Row rowX = (Row) row;
        for (int i = 0; i < numFields; i++) {
            if (rowX.getField(i) == null) {
                builder.set(pSchema.getFields().get(i), null);
            } else {
                builder.set(pSchema.getFields().get(i), fieldConverters.get(i).apply(new PositionedGetter(rowX), i));
            }
        }
        return (GenericAvroRecord) builder.build();
    };
}

Example 2 with GenericRecordBuilder

use of org.apache.pulsar.client.api.schema.GenericRecordBuilder in project pulsar by yahoo.

the class PulsarSinkTest method testWriteGenericRecords.

private void testWriteGenericRecords(ProcessingGuarantees guarantees) throws Exception {
    String defaultTopic = "default";
    PulsarSinkConfig sinkConfig = getPulsarConfigs();
    sinkConfig.setTopic(defaultTopic);
    sinkConfig.setTypeClassName(GenericRecord.class.getName());
    sinkConfig.setProcessingGuarantees(guarantees);
    PulsarClient client = getPulsarClient();
    PulsarSink pulsarSink = new PulsarSink(client, sinkConfig, new HashMap<>(), mock(ComponentStatsManager.class), Thread.currentThread().getContextClassLoader());
    pulsarSink.open(new HashMap<>(), mock(SinkContext.class));
    if (ProcessingGuarantees.ATMOST_ONCE == guarantees) {
        assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkAtMostOnceProcessor);
    } else if (ProcessingGuarantees.ATLEAST_ONCE == guarantees) {
        assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkAtLeastOnceProcessor);
    } else {
        assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkEffectivelyOnceProcessor);
    }
    PulsarSinkProcessorBase processor = (PulsarSinkProcessorBase) pulsarSink.pulsarSinkProcessor;
    assertFalse(processor.publishProducers.containsKey(defaultTopic));
    String[] topics = { "topic-1", "topic-2", "topic-3" };
    for (String topic : topics) {
        RecordSchemaBuilder builder = SchemaBuilder.record("MyRecord");
        builder.field("number").type(SchemaType.INT32);
        builder.field("text").type(SchemaType.STRING);
        GenericSchema<GenericRecord> schema = Schema.generic(builder.build(SchemaType.AVRO));
        GenericRecordBuilder recordBuilder = schema.newRecordBuilder();
        recordBuilder.set("number", 1);
        recordBuilder.set("text", topic);
        GenericRecord genericRecord = recordBuilder.build();
        SinkRecord<GenericRecord> record = new SinkRecord<>(new Record<GenericRecord>() {

            @Override
            public Optional<String> getDestinationTopic() {
                return Optional.of(topic);
            }

            @Override
            public Schema<GenericRecord> getSchema() {
                return schema;
            }

            @Override
            public GenericRecord getValue() {
                return genericRecord;
            }

            @Override
            public Optional<String> getPartitionId() {
                return Optional.of(topic + "-id-1");
            }

            @Override
            public Optional<Long> getRecordSequence() {
                return Optional.of(1L);
            }
        }, genericRecord);
        pulsarSink.write(record);
        if (ProcessingGuarantees.EFFECTIVELY_ONCE == guarantees) {
            assertTrue(processor.publishProducers.containsKey(String.format("%s-%s-id-1", topic, topic)));
        } else {
            assertTrue(processor.publishProducers.containsKey(topic));
        }
        verify(client.newProducer(), times(1)).topic(argThat(otherTopic -> topic != null ? topic.equals(otherTopic) : defaultTopic.equals(otherTopic)));
        verify(client, times(1)).newProducer(argThat(otherSchema -> Objects.equals(otherSchema, schema)));
    }
}

Example 3 with GenericRecordBuilder

use of org.apache.pulsar.client.api.schema.GenericRecordBuilder in project nosqlbench by nosqlbench.

the class AvroUtil method GetGenericRecord_PulsarAvro.

// Get a Pulsar Avro record from an OSS Avro schema record, matching a specific Pulsar Avro schema
public static GenericRecord GetGenericRecord_PulsarAvro(GenericAvroSchema pulsarGenericAvroSchema, org.apache.avro.generic.GenericRecord apacheAvroGenericRecord) {
    GenericRecordBuilder recordBuilder = pulsarGenericAvroSchema.newRecordBuilder();
    List<Field> fieldList = pulsarGenericAvroSchema.getFields();
    for (Field field : fieldList) {
        String fieldName = field.getName();
        recordBuilder.set(fieldName, apacheAvroGenericRecord.get(fieldName));
    }
    return recordBuilder.build();
}

Example 4 with GenericRecordBuilder

use of org.apache.pulsar.client.api.schema.GenericRecordBuilder in project incubator-pulsar by apache.

the class PulsarSinkTest method testWriteGenericRecords.

private void testWriteGenericRecords(ProcessingGuarantees guarantees) throws Exception {
    String defaultTopic = "default";
    PulsarSinkConfig sinkConfig = getPulsarConfigs();
    sinkConfig.setTopic(defaultTopic);
    sinkConfig.setTypeClassName(GenericRecord.class.getName());
    sinkConfig.setProcessingGuarantees(guarantees);
    PulsarClient client = getPulsarClient();
    PulsarSink pulsarSink = new PulsarSink(client, sinkConfig, new HashMap<>(), mock(ComponentStatsManager.class), Thread.currentThread().getContextClassLoader());
    pulsarSink.open(new HashMap<>(), mock(SinkContext.class));
    if (ProcessingGuarantees.ATMOST_ONCE == guarantees) {
        assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkAtMostOnceProcessor);
    } else if (ProcessingGuarantees.ATLEAST_ONCE == guarantees) {
        assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkAtLeastOnceProcessor);
    } else {
        assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkEffectivelyOnceProcessor);
    }
    PulsarSinkProcessorBase processor = (PulsarSinkProcessorBase) pulsarSink.pulsarSinkProcessor;
    assertFalse(processor.publishProducers.containsKey(defaultTopic));
    String[] topics = { "topic-1", "topic-2", "topic-3" };
    for (String topic : topics) {
        RecordSchemaBuilder builder = SchemaBuilder.record("MyRecord");
        builder.field("number").type(SchemaType.INT32);
        builder.field("text").type(SchemaType.STRING);
        GenericSchema<GenericRecord> schema = Schema.generic(builder.build(SchemaType.AVRO));
        GenericRecordBuilder recordBuilder = schema.newRecordBuilder();
        recordBuilder.set("number", 1);
        recordBuilder.set("text", topic);
        GenericRecord genericRecord = recordBuilder.build();
        SinkRecord<GenericRecord> record = new SinkRecord<>(new Record<GenericRecord>() {

            @Override
            public Optional<String> getDestinationTopic() {
                return Optional.of(topic);
            }

            @Override
            public Schema<GenericRecord> getSchema() {
                return schema;
            }

            @Override
            public GenericRecord getValue() {
                return genericRecord;
            }

            @Override
            public Optional<String> getPartitionId() {
                return Optional.of(topic + "-id-1");
            }

            @Override
            public Optional<Long> getRecordSequence() {
                return Optional.of(1L);
            }
        }, genericRecord);
        pulsarSink.write(record);
        if (ProcessingGuarantees.EFFECTIVELY_ONCE == guarantees) {
            assertTrue(processor.publishProducers.containsKey(String.format("%s-%s-id-1", topic, topic)));
        } else {
            assertTrue(processor.publishProducers.containsKey(topic));
        }
        verify(client.newProducer(), times(1)).topic(argThat(otherTopic -> topic != null ? topic.equals(otherTopic) : defaultTopic.equals(otherTopic)));
        verify(client, times(1)).newProducer(argThat(otherSchema -> Objects.equals(otherSchema, schema)));
    }
}

Example 5 with GenericRecordBuilder

use of org.apache.pulsar.client.api.schema.GenericRecordBuilder in project pulsar by apache.

the class PulsarSinkTest method testWriteGenericRecords.

private void testWriteGenericRecords(ProcessingGuarantees guarantees) throws Exception {
    String defaultTopic = "default";
    PulsarSinkConfig sinkConfig = getPulsarConfigs();
    sinkConfig.setTopic(defaultTopic);
    sinkConfig.setTypeClassName(GenericRecord.class.getName());
    sinkConfig.setProcessingGuarantees(guarantees);
    PulsarClient client = getPulsarClient();
    PulsarSink pulsarSink = new PulsarSink(client, sinkConfig, new HashMap<>(), mock(ComponentStatsManager.class), Thread.currentThread().getContextClassLoader());
    pulsarSink.open(new HashMap<>(), mock(SinkContext.class));
    if (ProcessingGuarantees.ATMOST_ONCE == guarantees) {
        assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkAtMostOnceProcessor);
    } else if (ProcessingGuarantees.ATLEAST_ONCE == guarantees) {
        assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkAtLeastOnceProcessor);
    } else {
        assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkEffectivelyOnceProcessor);
    }
    PulsarSinkProcessorBase processor = (PulsarSinkProcessorBase) pulsarSink.pulsarSinkProcessor;
    assertFalse(processor.publishProducers.containsKey(defaultTopic));
    String[] topics = { "topic-1", "topic-2", "topic-3" };
    for (String topic : topics) {
        RecordSchemaBuilder builder = SchemaBuilder.record("MyRecord");
        builder.field("number").type(SchemaType.INT32);
        builder.field("text").type(SchemaType.STRING);
        GenericSchema<GenericRecord> schema = Schema.generic(builder.build(SchemaType.AVRO));
        GenericRecordBuilder recordBuilder = schema.newRecordBuilder();
        recordBuilder.set("number", 1);
        recordBuilder.set("text", topic);
        GenericRecord genericRecord = recordBuilder.build();
        SinkRecord<GenericRecord> record = new SinkRecord<>(new Record<GenericRecord>() {

            @Override
            public Optional<String> getDestinationTopic() {
                return Optional.of(topic);
            }

            @Override
            public Schema<GenericRecord> getSchema() {
                return schema;
            }

            @Override
            public GenericRecord getValue() {
                return genericRecord;
            }

            @Override
            public Optional<String> getPartitionId() {
                return Optional.of(topic + "-id-1");
            }

            @Override
            public Optional<Long> getRecordSequence() {
                return Optional.of(1L);
            }
        }, genericRecord);
        pulsarSink.write(record);
        if (ProcessingGuarantees.EFFECTIVELY_ONCE == guarantees) {
            assertTrue(processor.publishProducers.containsKey(String.format("%s-%s-id-1", topic, topic)));
        } else {
            assertTrue(processor.publishProducers.containsKey(topic));
        }
        verify(client.newProducer(), times(1)).topic(argThat(otherTopic -> topic != null ? topic.equals(otherTopic) : defaultTopic.equals(otherTopic)));
        verify(client, times(1)).newProducer(argThat(otherSchema -> Objects.equals(otherSchema, schema)));
    }
}