Examples with PulsarSinkProcessorBase org.apache.pulsar.functions.sink.PulsarSink.PulsarSinkProcessorBase used on opensource projects

Example 1 with PulsarSinkProcessorBase

use of org.apache.pulsar.functions.sink.PulsarSink.PulsarSinkProcessorBase in project pulsar by yahoo.

the class PulsarSinkTest method testWriteGenericRecords.

private void testWriteGenericRecords(ProcessingGuarantees guarantees) throws Exception {
    String defaultTopic = "default";
    PulsarSinkConfig sinkConfig = getPulsarConfigs();
    sinkConfig.setTopic(defaultTopic);
    sinkConfig.setTypeClassName(GenericRecord.class.getName());
    sinkConfig.setProcessingGuarantees(guarantees);
    PulsarClient client = getPulsarClient();
    PulsarSink pulsarSink = new PulsarSink(client, sinkConfig, new HashMap<>(), mock(ComponentStatsManager.class), Thread.currentThread().getContextClassLoader());
    pulsarSink.open(new HashMap<>(), mock(SinkContext.class));
    if (ProcessingGuarantees.ATMOST_ONCE == guarantees) {
        assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkAtMostOnceProcessor);
    } else if (ProcessingGuarantees.ATLEAST_ONCE == guarantees) {
        assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkAtLeastOnceProcessor);
    } else {
        assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkEffectivelyOnceProcessor);
    }
    PulsarSinkProcessorBase processor = (PulsarSinkProcessorBase) pulsarSink.pulsarSinkProcessor;
    assertFalse(processor.publishProducers.containsKey(defaultTopic));
    String[] topics = { "topic-1", "topic-2", "topic-3" };
    for (String topic : topics) {
        RecordSchemaBuilder builder = SchemaBuilder.record("MyRecord");
        builder.field("number").type(SchemaType.INT32);
        builder.field("text").type(SchemaType.STRING);
        GenericSchema<GenericRecord> schema = Schema.generic(builder.build(SchemaType.AVRO));
        GenericRecordBuilder recordBuilder = schema.newRecordBuilder();
        recordBuilder.set("number", 1);
        recordBuilder.set("text", topic);
        GenericRecord genericRecord = recordBuilder.build();
        SinkRecord<GenericRecord> record = new SinkRecord<>(new Record<GenericRecord>() {

            @Override
            public Optional<String> getDestinationTopic() {
                return Optional.of(topic);
            }

            @Override
            public Schema<GenericRecord> getSchema() {
                return schema;
            }

            @Override
            public GenericRecord getValue() {
                return genericRecord;
            }

            @Override
            public Optional<String> getPartitionId() {
                return Optional.of(topic + "-id-1");
            }

            @Override
            public Optional<Long> getRecordSequence() {
                return Optional.of(1L);
            }
        }, genericRecord);
        pulsarSink.write(record);
        if (ProcessingGuarantees.EFFECTIVELY_ONCE == guarantees) {
            assertTrue(processor.publishProducers.containsKey(String.format("%s-%s-id-1", topic, topic)));
        } else {
            assertTrue(processor.publishProducers.containsKey(topic));
        }
        verify(client.newProducer(), times(1)).topic(argThat(otherTopic -> topic != null ? topic.equals(otherTopic) : defaultTopic.equals(otherTopic)));
        verify(client, times(1)).newProducer(argThat(otherSchema -> Objects.equals(otherSchema, schema)));
    }
}

Example 2 with PulsarSinkProcessorBase

use of org.apache.pulsar.functions.sink.PulsarSink.PulsarSinkProcessorBase in project incubator-pulsar by apache.

the class PulsarSinkTest method testWriteGenericRecords.

private void testWriteGenericRecords(ProcessingGuarantees guarantees) throws Exception {
    String defaultTopic = "default";
    PulsarSinkConfig sinkConfig = getPulsarConfigs();
    sinkConfig.setTopic(defaultTopic);
    sinkConfig.setTypeClassName(GenericRecord.class.getName());
    sinkConfig.setProcessingGuarantees(guarantees);
    PulsarClient client = getPulsarClient();
    PulsarSink pulsarSink = new PulsarSink(client, sinkConfig, new HashMap<>(), mock(ComponentStatsManager.class), Thread.currentThread().getContextClassLoader());
    pulsarSink.open(new HashMap<>(), mock(SinkContext.class));
    if (ProcessingGuarantees.ATMOST_ONCE == guarantees) {
        assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkAtMostOnceProcessor);
    } else if (ProcessingGuarantees.ATLEAST_ONCE == guarantees) {
        assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkAtLeastOnceProcessor);
    } else {
        assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkEffectivelyOnceProcessor);
    }
    PulsarSinkProcessorBase processor = (PulsarSinkProcessorBase) pulsarSink.pulsarSinkProcessor;
    assertFalse(processor.publishProducers.containsKey(defaultTopic));
    String[] topics = { "topic-1", "topic-2", "topic-3" };
    for (String topic : topics) {
        RecordSchemaBuilder builder = SchemaBuilder.record("MyRecord");
        builder.field("number").type(SchemaType.INT32);
        builder.field("text").type(SchemaType.STRING);
        GenericSchema<GenericRecord> schema = Schema.generic(builder.build(SchemaType.AVRO));
        GenericRecordBuilder recordBuilder = schema.newRecordBuilder();
        recordBuilder.set("number", 1);
        recordBuilder.set("text", topic);
        GenericRecord genericRecord = recordBuilder.build();
        SinkRecord<GenericRecord> record = new SinkRecord<>(new Record<GenericRecord>() {

            @Override
            public Optional<String> getDestinationTopic() {
                return Optional.of(topic);
            }

            @Override
            public Schema<GenericRecord> getSchema() {
                return schema;
            }

            @Override
            public GenericRecord getValue() {
                return genericRecord;
            }

            @Override
            public Optional<String> getPartitionId() {
                return Optional.of(topic + "-id-1");
            }

            @Override
            public Optional<Long> getRecordSequence() {
                return Optional.of(1L);
            }
        }, genericRecord);
        pulsarSink.write(record);
        if (ProcessingGuarantees.EFFECTIVELY_ONCE == guarantees) {
            assertTrue(processor.publishProducers.containsKey(String.format("%s-%s-id-1", topic, topic)));
        } else {
            assertTrue(processor.publishProducers.containsKey(topic));
        }
        verify(client.newProducer(), times(1)).topic(argThat(otherTopic -> topic != null ? topic.equals(otherTopic) : defaultTopic.equals(otherTopic)));
        verify(client, times(1)).newProducer(argThat(otherSchema -> Objects.equals(otherSchema, schema)));
    }
}

Example 3 with PulsarSinkProcessorBase

use of org.apache.pulsar.functions.sink.PulsarSink.PulsarSinkProcessorBase in project pulsar by apache.

the class PulsarSinkTest method testWriteGenericRecords.

private void testWriteGenericRecords(ProcessingGuarantees guarantees) throws Exception {
    String defaultTopic = "default";
    PulsarSinkConfig sinkConfig = getPulsarConfigs();
    sinkConfig.setTopic(defaultTopic);
    sinkConfig.setTypeClassName(GenericRecord.class.getName());
    sinkConfig.setProcessingGuarantees(guarantees);
    PulsarClient client = getPulsarClient();
    PulsarSink pulsarSink = new PulsarSink(client, sinkConfig, new HashMap<>(), mock(ComponentStatsManager.class), Thread.currentThread().getContextClassLoader());
    pulsarSink.open(new HashMap<>(), mock(SinkContext.class));
    if (ProcessingGuarantees.ATMOST_ONCE == guarantees) {
        assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkAtMostOnceProcessor);
    } else if (ProcessingGuarantees.ATLEAST_ONCE == guarantees) {
        assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkAtLeastOnceProcessor);
    } else {
        assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkEffectivelyOnceProcessor);
    }
    PulsarSinkProcessorBase processor = (PulsarSinkProcessorBase) pulsarSink.pulsarSinkProcessor;
    assertFalse(processor.publishProducers.containsKey(defaultTopic));
    String[] topics = { "topic-1", "topic-2", "topic-3" };
    for (String topic : topics) {
        RecordSchemaBuilder builder = SchemaBuilder.record("MyRecord");
        builder.field("number").type(SchemaType.INT32);
        builder.field("text").type(SchemaType.STRING);
        GenericSchema<GenericRecord> schema = Schema.generic(builder.build(SchemaType.AVRO));
        GenericRecordBuilder recordBuilder = schema.newRecordBuilder();
        recordBuilder.set("number", 1);
        recordBuilder.set("text", topic);
        GenericRecord genericRecord = recordBuilder.build();
        SinkRecord<GenericRecord> record = new SinkRecord<>(new Record<GenericRecord>() {

            @Override
            public Optional<String> getDestinationTopic() {
                return Optional.of(topic);
            }

            @Override
            public Schema<GenericRecord> getSchema() {
                return schema;
            }

            @Override
            public GenericRecord getValue() {
                return genericRecord;
            }

            @Override
            public Optional<String> getPartitionId() {
                return Optional.of(topic + "-id-1");
            }

            @Override
            public Optional<Long> getRecordSequence() {
                return Optional.of(1L);
            }
        }, genericRecord);
        pulsarSink.write(record);
        if (ProcessingGuarantees.EFFECTIVELY_ONCE == guarantees) {
            assertTrue(processor.publishProducers.containsKey(String.format("%s-%s-id-1", topic, topic)));
        } else {
            assertTrue(processor.publishProducers.containsKey(topic));
        }
        verify(client.newProducer(), times(1)).topic(argThat(otherTopic -> topic != null ? topic.equals(otherTopic) : defaultTopic.equals(otherTopic)));
        verify(client, times(1)).newProducer(argThat(otherSchema -> Objects.equals(otherSchema, schema)));
    }
}