Examples with SinkRecord org.apache.pulsar.functions.instance.SinkRecord used on opensource projects

Example 1 with SinkRecord

use of org.apache.pulsar.functions.instance.SinkRecord in project pulsar by yahoo.

the class PulsarSinkTest method testWriteGenericRecords.

private void testWriteGenericRecords(ProcessingGuarantees guarantees) throws Exception {
    String defaultTopic = "default";
    PulsarSinkConfig sinkConfig = getPulsarConfigs();
    sinkConfig.setTopic(defaultTopic);
    sinkConfig.setTypeClassName(GenericRecord.class.getName());
    sinkConfig.setProcessingGuarantees(guarantees);
    PulsarClient client = getPulsarClient();
    PulsarSink pulsarSink = new PulsarSink(client, sinkConfig, new HashMap<>(), mock(ComponentStatsManager.class), Thread.currentThread().getContextClassLoader());
    pulsarSink.open(new HashMap<>(), mock(SinkContext.class));
    if (ProcessingGuarantees.ATMOST_ONCE == guarantees) {
        assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkAtMostOnceProcessor);
    } else if (ProcessingGuarantees.ATLEAST_ONCE == guarantees) {
        assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkAtLeastOnceProcessor);
    } else {
        assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkEffectivelyOnceProcessor);
    }
    PulsarSinkProcessorBase processor = (PulsarSinkProcessorBase) pulsarSink.pulsarSinkProcessor;
    assertFalse(processor.publishProducers.containsKey(defaultTopic));
    String[] topics = { "topic-1", "topic-2", "topic-3" };
    for (String topic : topics) {
        RecordSchemaBuilder builder = SchemaBuilder.record("MyRecord");
        builder.field("number").type(SchemaType.INT32);
        builder.field("text").type(SchemaType.STRING);
        GenericSchema<GenericRecord> schema = Schema.generic(builder.build(SchemaType.AVRO));
        GenericRecordBuilder recordBuilder = schema.newRecordBuilder();
        recordBuilder.set("number", 1);
        recordBuilder.set("text", topic);
        GenericRecord genericRecord = recordBuilder.build();
        SinkRecord<GenericRecord> record = new SinkRecord<>(new Record<GenericRecord>() {

            @Override
            public Optional<String> getDestinationTopic() {
                return Optional.of(topic);
            }

            @Override
            public Schema<GenericRecord> getSchema() {
                return schema;
            }

            @Override
            public GenericRecord getValue() {
                return genericRecord;
            }

            @Override
            public Optional<String> getPartitionId() {
                return Optional.of(topic + "-id-1");
            }

            @Override
            public Optional<Long> getRecordSequence() {
                return Optional.of(1L);
            }
        }, genericRecord);
        pulsarSink.write(record);
        if (ProcessingGuarantees.EFFECTIVELY_ONCE == guarantees) {
            assertTrue(processor.publishProducers.containsKey(String.format("%s-%s-id-1", topic, topic)));
        } else {
            assertTrue(processor.publishProducers.containsKey(topic));
        }
        verify(client.newProducer(), times(1)).topic(argThat(otherTopic -> topic != null ? topic.equals(otherTopic) : defaultTopic.equals(otherTopic)));
        verify(client, times(1)).newProducer(argThat(otherSchema -> Objects.equals(otherSchema, schema)));
    }
}

Example 2 with SinkRecord

use of org.apache.pulsar.functions.instance.SinkRecord in project incubator-pulsar by apache.

the class PulsarSinkTest method testSinkAndMessageRouting.

@Test
public void testSinkAndMessageRouting() throws Exception {
    String[] topics = { "topic-1", "topic-2", "topic-3", null };
    String defaultTopic = "default";
    PulsarSinkConfig pulsarConfig = getPulsarConfigs();
    pulsarConfig.setTopic(defaultTopic);
    PulsarClient pulsarClient;
    /**
     * test At-least-once *
     */
    pulsarClient = getPulsarClient();
    pulsarConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
    PulsarSink pulsarSink = new PulsarSink(pulsarClient, pulsarConfig, new HashMap<>(), mock(ComponentStatsManager.class), Thread.currentThread().getContextClassLoader());
    pulsarSink.open(new HashMap<>(), mock(SinkContext.class));
    for (String topic : topics) {
        SinkRecord<String> record = new SinkRecord<>(new Record<String>() {

            @Override
            public String getValue() {
                return "in1";
            }

            @Override
            public Optional<String> getDestinationTopic() {
                return getTopicOptional(topic);
            }
        }, "out1");
        pulsarSink.write(record);
        Assert.assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkAtLeastOnceProcessor);
        PulsarSink.PulsarSinkAtLeastOnceProcessor pulsarSinkAtLeastOnceProcessor = (PulsarSink.PulsarSinkAtLeastOnceProcessor) pulsarSink.pulsarSinkProcessor;
        if (topic != null) {
            Assert.assertTrue(pulsarSinkAtLeastOnceProcessor.publishProducers.containsKey(topic));
        } else {
            Assert.assertTrue(pulsarSinkAtLeastOnceProcessor.publishProducers.containsKey(defaultTopic));
        }
        verify(pulsarClient.newProducer(), times(1)).topic(argThat(otherTopic -> {
            if (topic != null) {
                return topic.equals(otherTopic);
            } else {
                return defaultTopic.equals(otherTopic);
            }
        }));
    }
    /**
     * test At-most-once *
     */
    pulsarClient = getPulsarClient();
    pulsarConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATMOST_ONCE);
    pulsarSink = new PulsarSink(pulsarClient, pulsarConfig, new HashMap<>(), mock(ComponentStatsManager.class), Thread.currentThread().getContextClassLoader());
    pulsarSink.open(new HashMap<>(), mock(SinkContext.class));
    for (String topic : topics) {
        SinkRecord<String> record = new SinkRecord<>(new Record<String>() {

            @Override
            public Optional<String> getKey() {
                return Optional.empty();
            }

            @Override
            public String getValue() {
                return "in1";
            }

            @Override
            public Optional<String> getDestinationTopic() {
                return getTopicOptional(topic);
            }
        }, "out1");
        pulsarSink.write(record);
        Assert.assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkAtMostOnceProcessor);
        PulsarSink.PulsarSinkAtMostOnceProcessor pulsarSinkAtLeastOnceProcessor = (PulsarSink.PulsarSinkAtMostOnceProcessor) pulsarSink.pulsarSinkProcessor;
        if (topic != null) {
            Assert.assertTrue(pulsarSinkAtLeastOnceProcessor.publishProducers.containsKey(topic));
        } else {
            Assert.assertTrue(pulsarSinkAtLeastOnceProcessor.publishProducers.containsKey(defaultTopic));
        }
        verify(pulsarClient.newProducer(), times(1)).topic(argThat(o -> {
            return getTopicEquals(o, topic, defaultTopic);
        }));
    }
    /**
     * test Effectively-once *
     */
    pulsarClient = getPulsarClient();
    pulsarConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE);
    pulsarSink = new PulsarSink(pulsarClient, pulsarConfig, new HashMap<>(), mock(ComponentStatsManager.class), Thread.currentThread().getContextClassLoader());
    pulsarSink.open(new HashMap<>(), mock(SinkContext.class));
    for (String topic : topics) {
        SinkRecord<String> record = new SinkRecord<>(new Record<String>() {

            @Override
            public Optional<String> getKey() {
                return Optional.empty();
            }

            @Override
            public String getValue() {
                return "in1";
            }

            @Override
            public Optional<String> getDestinationTopic() {
                return getTopicOptional(topic);
            }

            @Override
            public Optional<String> getPartitionId() {
                if (topic != null) {
                    return Optional.of(topic + "-id-1");
                } else {
                    return Optional.of(defaultTopic + "-id-1");
                }
            }

            @Override
            public Optional<Integer> getPartitionIndex() {
                return Optional.of(1);
            }

            @Override
            public Optional<Long> getRecordSequence() {
                return Optional.of(1L);
            }
        }, "out1");
        assertEquals(1, record.getPartitionIndex().get().intValue());
        pulsarSink.write(record);
        Assert.assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkEffectivelyOnceProcessor);
        PulsarSink.PulsarSinkEffectivelyOnceProcessor pulsarSinkEffectivelyOnceProcessor = (PulsarSink.PulsarSinkEffectivelyOnceProcessor) pulsarSink.pulsarSinkProcessor;
        if (topic != null) {
            Assert.assertTrue(pulsarSinkEffectivelyOnceProcessor.publishProducers.containsKey(String.format("%s-%s-id-1", topic, topic)));
        } else {
            Assert.assertTrue(pulsarSinkEffectivelyOnceProcessor.publishProducers.containsKey(String.format("%s-%s-id-1", defaultTopic, defaultTopic)));
        }
        verify(pulsarClient.newProducer(), times(1)).topic(argThat(o -> {
            return getTopicEquals(o, topic, defaultTopic);
        }));
        verify(pulsarClient.newProducer(), times(1)).producerName(argThat(o -> {
            if (topic != null) {
                return String.format("%s-id-1", topic).equals(o);
            } else {
                return String.format("%s-id-1", defaultTopic).equals(o);
            }
        }));
    }
}

Example 3 with SinkRecord

use of org.apache.pulsar.functions.instance.SinkRecord in project incubator-pulsar by apache.

the class PulsarSinkTest method testWriteGenericRecords.

private void testWriteGenericRecords(ProcessingGuarantees guarantees) throws Exception {
    String defaultTopic = "default";
    PulsarSinkConfig sinkConfig = getPulsarConfigs();
    sinkConfig.setTopic(defaultTopic);
    sinkConfig.setTypeClassName(GenericRecord.class.getName());
    sinkConfig.setProcessingGuarantees(guarantees);
    PulsarClient client = getPulsarClient();
    PulsarSink pulsarSink = new PulsarSink(client, sinkConfig, new HashMap<>(), mock(ComponentStatsManager.class), Thread.currentThread().getContextClassLoader());
    pulsarSink.open(new HashMap<>(), mock(SinkContext.class));
    if (ProcessingGuarantees.ATMOST_ONCE == guarantees) {
        assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkAtMostOnceProcessor);
    } else if (ProcessingGuarantees.ATLEAST_ONCE == guarantees) {
        assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkAtLeastOnceProcessor);
    } else {
        assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkEffectivelyOnceProcessor);
    }
    PulsarSinkProcessorBase processor = (PulsarSinkProcessorBase) pulsarSink.pulsarSinkProcessor;
    assertFalse(processor.publishProducers.containsKey(defaultTopic));
    String[] topics = { "topic-1", "topic-2", "topic-3" };
    for (String topic : topics) {
        RecordSchemaBuilder builder = SchemaBuilder.record("MyRecord");
        builder.field("number").type(SchemaType.INT32);
        builder.field("text").type(SchemaType.STRING);
        GenericSchema<GenericRecord> schema = Schema.generic(builder.build(SchemaType.AVRO));
        GenericRecordBuilder recordBuilder = schema.newRecordBuilder();
        recordBuilder.set("number", 1);
        recordBuilder.set("text", topic);
        GenericRecord genericRecord = recordBuilder.build();
        SinkRecord<GenericRecord> record = new SinkRecord<>(new Record<GenericRecord>() {

            @Override
            public Optional<String> getDestinationTopic() {
                return Optional.of(topic);
            }

            @Override
            public Schema<GenericRecord> getSchema() {
                return schema;
            }

            @Override
            public GenericRecord getValue() {
                return genericRecord;
            }

            @Override
            public Optional<String> getPartitionId() {
                return Optional.of(topic + "-id-1");
            }

            @Override
            public Optional<Long> getRecordSequence() {
                return Optional.of(1L);
            }
        }, genericRecord);
        pulsarSink.write(record);
        if (ProcessingGuarantees.EFFECTIVELY_ONCE == guarantees) {
            assertTrue(processor.publishProducers.containsKey(String.format("%s-%s-id-1", topic, topic)));
        } else {
            assertTrue(processor.publishProducers.containsKey(topic));
        }
        verify(client.newProducer(), times(1)).topic(argThat(otherTopic -> topic != null ? topic.equals(otherTopic) : defaultTopic.equals(otherTopic)));
        verify(client, times(1)).newProducer(argThat(otherSchema -> Objects.equals(otherSchema, schema)));
    }
}

Example 4 with SinkRecord

use of org.apache.pulsar.functions.instance.SinkRecord in project pulsar by yahoo.

the class PulsarSinkTest method testSinkAndMessageRouting.

@Test
public void testSinkAndMessageRouting() throws Exception {
    String[] topics = { "topic-1", "topic-2", "topic-3", null };
    String defaultTopic = "default";
    PulsarSinkConfig pulsarConfig = getPulsarConfigs();
    pulsarConfig.setTopic(defaultTopic);
    PulsarClient pulsarClient;
    /**
     * test At-least-once *
     */
    pulsarClient = getPulsarClient();
    pulsarConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
    PulsarSink pulsarSink = new PulsarSink(pulsarClient, pulsarConfig, new HashMap<>(), mock(ComponentStatsManager.class), Thread.currentThread().getContextClassLoader());
    pulsarSink.open(new HashMap<>(), mock(SinkContext.class));
    for (String topic : topics) {
        SinkRecord<String> record = new SinkRecord<>(new Record<String>() {

            @Override
            public String getValue() {
                return "in1";
            }

            @Override
            public Optional<String> getDestinationTopic() {
                return getTopicOptional(topic);
            }
        }, "out1");
        pulsarSink.write(record);
        Assert.assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkAtLeastOnceProcessor);
        PulsarSink.PulsarSinkAtLeastOnceProcessor pulsarSinkAtLeastOnceProcessor = (PulsarSink.PulsarSinkAtLeastOnceProcessor) pulsarSink.pulsarSinkProcessor;
        if (topic != null) {
            Assert.assertTrue(pulsarSinkAtLeastOnceProcessor.publishProducers.containsKey(topic));
        } else {
            Assert.assertTrue(pulsarSinkAtLeastOnceProcessor.publishProducers.containsKey(defaultTopic));
        }
        verify(pulsarClient.newProducer(), times(1)).topic(argThat(otherTopic -> {
            if (topic != null) {
                return topic.equals(otherTopic);
            } else {
                return defaultTopic.equals(otherTopic);
            }
        }));
    }
    /**
     * test At-most-once *
     */
    pulsarClient = getPulsarClient();
    pulsarConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATMOST_ONCE);
    pulsarSink = new PulsarSink(pulsarClient, pulsarConfig, new HashMap<>(), mock(ComponentStatsManager.class), Thread.currentThread().getContextClassLoader());
    pulsarSink.open(new HashMap<>(), mock(SinkContext.class));
    for (String topic : topics) {
        SinkRecord<String> record = new SinkRecord<>(new Record<String>() {

            @Override
            public Optional<String> getKey() {
                return Optional.empty();
            }

            @Override
            public String getValue() {
                return "in1";
            }

            @Override
            public Optional<String> getDestinationTopic() {
                return getTopicOptional(topic);
            }
        }, "out1");
        pulsarSink.write(record);
        Assert.assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkAtMostOnceProcessor);
        PulsarSink.PulsarSinkAtMostOnceProcessor pulsarSinkAtLeastOnceProcessor = (PulsarSink.PulsarSinkAtMostOnceProcessor) pulsarSink.pulsarSinkProcessor;
        if (topic != null) {
            Assert.assertTrue(pulsarSinkAtLeastOnceProcessor.publishProducers.containsKey(topic));
        } else {
            Assert.assertTrue(pulsarSinkAtLeastOnceProcessor.publishProducers.containsKey(defaultTopic));
        }
        verify(pulsarClient.newProducer(), times(1)).topic(argThat(o -> {
            return getTopicEquals(o, topic, defaultTopic);
        }));
    }
    /**
     * test Effectively-once *
     */
    pulsarClient = getPulsarClient();
    pulsarConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE);
    pulsarSink = new PulsarSink(pulsarClient, pulsarConfig, new HashMap<>(), mock(ComponentStatsManager.class), Thread.currentThread().getContextClassLoader());
    pulsarSink.open(new HashMap<>(), mock(SinkContext.class));
    for (String topic : topics) {
        SinkRecord<String> record = new SinkRecord<>(new Record<String>() {

            @Override
            public Optional<String> getKey() {
                return Optional.empty();
            }

            @Override
            public String getValue() {
                return "in1";
            }

            @Override
            public Optional<String> getDestinationTopic() {
                return getTopicOptional(topic);
            }

            @Override
            public Optional<String> getPartitionId() {
                if (topic != null) {
                    return Optional.of(topic + "-id-1");
                } else {
                    return Optional.of(defaultTopic + "-id-1");
                }
            }

            @Override
            public Optional<Integer> getPartitionIndex() {
                return Optional.of(1);
            }

            @Override
            public Optional<Long> getRecordSequence() {
                return Optional.of(1L);
            }
        }, "out1");
        assertEquals(1, record.getPartitionIndex().get().intValue());
        pulsarSink.write(record);
        Assert.assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkEffectivelyOnceProcessor);
        PulsarSink.PulsarSinkEffectivelyOnceProcessor pulsarSinkEffectivelyOnceProcessor = (PulsarSink.PulsarSinkEffectivelyOnceProcessor) pulsarSink.pulsarSinkProcessor;
        if (topic != null) {
            Assert.assertTrue(pulsarSinkEffectivelyOnceProcessor.publishProducers.containsKey(String.format("%s-%s-id-1", topic, topic)));
        } else {
            Assert.assertTrue(pulsarSinkEffectivelyOnceProcessor.publishProducers.containsKey(String.format("%s-%s-id-1", defaultTopic, defaultTopic)));
        }
        verify(pulsarClient.newProducer(), times(1)).topic(argThat(o -> {
            return getTopicEquals(o, topic, defaultTopic);
        }));
        verify(pulsarClient.newProducer(), times(1)).producerName(argThat(o -> {
            if (topic != null) {
                return String.format("%s-id-1", topic).equals(o);
            } else {
                return String.format("%s-id-1", defaultTopic).equals(o);
            }
        }));
    }
}

Example 5 with SinkRecord

use of org.apache.pulsar.functions.instance.SinkRecord in project pulsar by apache.

the class PulsarSinkTest method testWriteGenericRecords.

private void testWriteGenericRecords(ProcessingGuarantees guarantees) throws Exception {
    String defaultTopic = "default";
    PulsarSinkConfig sinkConfig = getPulsarConfigs();
    sinkConfig.setTopic(defaultTopic);
    sinkConfig.setTypeClassName(GenericRecord.class.getName());
    sinkConfig.setProcessingGuarantees(guarantees);
    PulsarClient client = getPulsarClient();
    PulsarSink pulsarSink = new PulsarSink(client, sinkConfig, new HashMap<>(), mock(ComponentStatsManager.class), Thread.currentThread().getContextClassLoader());
    pulsarSink.open(new HashMap<>(), mock(SinkContext.class));
    if (ProcessingGuarantees.ATMOST_ONCE == guarantees) {
        assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkAtMostOnceProcessor);
    } else if (ProcessingGuarantees.ATLEAST_ONCE == guarantees) {
        assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkAtLeastOnceProcessor);
    } else {
        assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkEffectivelyOnceProcessor);
    }
    PulsarSinkProcessorBase processor = (PulsarSinkProcessorBase) pulsarSink.pulsarSinkProcessor;
    assertFalse(processor.publishProducers.containsKey(defaultTopic));
    String[] topics = { "topic-1", "topic-2", "topic-3" };
    for (String topic : topics) {
        RecordSchemaBuilder builder = SchemaBuilder.record("MyRecord");
        builder.field("number").type(SchemaType.INT32);
        builder.field("text").type(SchemaType.STRING);
        GenericSchema<GenericRecord> schema = Schema.generic(builder.build(SchemaType.AVRO));
        GenericRecordBuilder recordBuilder = schema.newRecordBuilder();
        recordBuilder.set("number", 1);
        recordBuilder.set("text", topic);
        GenericRecord genericRecord = recordBuilder.build();
        SinkRecord<GenericRecord> record = new SinkRecord<>(new Record<GenericRecord>() {

            @Override
            public Optional<String> getDestinationTopic() {
                return Optional.of(topic);
            }

            @Override
            public Schema<GenericRecord> getSchema() {
                return schema;
            }

            @Override
            public GenericRecord getValue() {
                return genericRecord;
            }

            @Override
            public Optional<String> getPartitionId() {
                return Optional.of(topic + "-id-1");
            }

            @Override
            public Optional<Long> getRecordSequence() {
                return Optional.of(1L);
            }
        }, genericRecord);
        pulsarSink.write(record);
        if (ProcessingGuarantees.EFFECTIVELY_ONCE == guarantees) {
            assertTrue(processor.publishProducers.containsKey(String.format("%s-%s-id-1", topic, topic)));
        } else {
            assertTrue(processor.publishProducers.containsKey(topic));
        }
        verify(client.newProducer(), times(1)).topic(argThat(otherTopic -> topic != null ? topic.equals(otherTopic) : defaultTopic.equals(otherTopic)));
        verify(client, times(1)).newProducer(argThat(otherSchema -> Objects.equals(otherSchema, schema)));
    }
}