Examples with Processor org.apache.kafka.streams.processor.Processor used on opensource projects

Example 6 with Processor

use of org.apache.kafka.streams.processor.Processor in project apache-kafka-on-k8s by banzaicloud.

the class SimpleBenchmark method createKafkaStreamsWithStateStore.

private KafkaStreams createKafkaStreamsWithStateStore(String topic, final CountDownLatch latch, boolean enableCaching) {
    setStreamProperties("simple-benchmark-streams-with-store" + enableCaching);
    StreamsBuilder builder = new StreamsBuilder();
    final StoreBuilder<KeyValueStore<Integer, byte[]>> storeBuilder = Stores.keyValueStoreBuilder(Stores.persistentKeyValueStore("store"), Serdes.Integer(), Serdes.ByteArray());
    if (enableCaching) {
        builder.addStateStore(storeBuilder.withCachingEnabled());
    } else {
        builder.addStateStore(storeBuilder);
    }
    KStream<Integer, byte[]> source = builder.stream(topic, Consumed.with(INTEGER_SERDE, BYTE_SERDE));
    source.process(new ProcessorSupplier<Integer, byte[]>() {

        @Override
        public Processor<Integer, byte[]> get() {
            return new AbstractProcessor<Integer, byte[]>() {

                KeyValueStore<Integer, byte[]> store;

                @SuppressWarnings("unchecked")
                @Override
                public void init(ProcessorContext context) {
                    store = (KeyValueStore<Integer, byte[]>) context.getStateStore("store");
                }

                @Override
                public void process(Integer key, byte[] value) {
                    store.put(key, value);
                    processedRecords.getAndIncrement();
                    processedBytes += value.length + Integer.SIZE;
                    if (processedRecords.get() == numRecords) {
                        latch.countDown();
                    }
                }

                @Override
                public void punctuate(long timestamp) {
                }

                @Override
                public void close() {
                }
            };
        }
    }, "store");
    return createKafkaStreamsWithExceptionHandler(builder, props);
}

Example 7 with Processor

use of org.apache.kafka.streams.processor.Processor in project kafka by apache.

the class KStreamTransformValuesTest method shouldNotAllowValueTransformerToCallInternalProcessorContextMethods.

@Test
public void shouldNotAllowValueTransformerToCallInternalProcessorContextMethods() {
    final KStreamTransformValues<Integer, Integer, Integer> transformValue = new KStreamTransformValues<>(new ValueTransformerSupplier<Integer, Integer>() {

        @Override
        public ValueTransformer<Integer, Integer> get() {
            return new BadValueTransformer();
        }
    });
    final Processor transformValueProcessor = transformValue.get();
    transformValueProcessor.init(null);
    try {
        transformValueProcessor.process(null, 0);
        fail("should not allow call to context.forward() within ValueTransformer");
    } catch (final StreamsException e) {
    // expected
    }
    try {
        transformValueProcessor.process(null, 1);
        fail("should not allow call to context.forward() within ValueTransformer");
    } catch (final StreamsException e) {
    // expected
    }
    try {
        transformValueProcessor.process(null, 2);
        fail("should not allow call to context.forward() within ValueTransformer");
    } catch (final StreamsException e) {
    // expected
    }
    try {
        transformValueProcessor.punctuate(0);
        fail("should not allow ValueTransformer#puntuate() to return not-null value");
    } catch (final StreamsException e) {
    // expected
    }
}

Example 8 with Processor

use of org.apache.kafka.streams.processor.Processor in project kafka by apache.

the class SimpleBenchmark method createKafkaStreamsWithSink.

private KafkaStreams createKafkaStreamsWithSink(String topic, final CountDownLatch latch) {
    final Properties props = setStreamProperties("simple-benchmark-streams-with-sink");
    KStreamBuilder builder = new KStreamBuilder();
    KStream<Integer, byte[]> source = builder.stream(INTEGER_SERDE, BYTE_SERDE, topic);
    source.to(INTEGER_SERDE, BYTE_SERDE, SINK_TOPIC);
    source.process(new ProcessorSupplier<Integer, byte[]>() {

        @Override
        public Processor<Integer, byte[]> get() {
            return new AbstractProcessor<Integer, byte[]>() {

                @Override
                public void init(ProcessorContext context) {
                }

                @Override
                public void process(Integer key, byte[] value) {
                    processedRecords++;
                    processedBytes += value.length + Integer.SIZE;
                    if (processedRecords == numRecords) {
                        latch.countDown();
                    }
                }

                @Override
                public void punctuate(long timestamp) {
                }

                @Override
                public void close() {
                }
            };
        }
    });
    return createKafkaStreamsWithExceptionHandler(builder, props);
}

Example 9 with Processor

use of org.apache.kafka.streams.processor.Processor in project kafka by apache.

the class SimpleBenchmark method createKafkaStreams.

private KafkaStreams createKafkaStreams(String topic, final CountDownLatch latch) {
    Properties props = setStreamProperties("simple-benchmark-streams");
    KStreamBuilder builder = new KStreamBuilder();
    KStream<Integer, byte[]> source = builder.stream(INTEGER_SERDE, BYTE_SERDE, topic);
    source.process(new ProcessorSupplier<Integer, byte[]>() {

        @Override
        public Processor<Integer, byte[]> get() {
            return new AbstractProcessor<Integer, byte[]>() {

                @Override
                public void init(ProcessorContext context) {
                }

                @Override
                public void process(Integer key, byte[] value) {
                    processedRecords++;
                    processedBytes += value.length + Integer.SIZE;
                    if (processedRecords == numRecords) {
                        latch.countDown();
                    }
                }

                @Override
                public void punctuate(long timestamp) {
                }

                @Override
                public void close() {
                }
            };
        }
    });
    return createKafkaStreamsWithExceptionHandler(builder, props);
}

Example 10 with Processor

use of org.apache.kafka.streams.processor.Processor in project apache-kafka-on-k8s by banzaicloud.

the class TopologyTestDriverTest method shouldCleanUpPersistentStateStoresOnClose.

@Test
public void shouldCleanUpPersistentStateStoresOnClose() {
    final Topology topology = new Topology();
    topology.addSource("sourceProcessor", "input-topic");
    topology.addProcessor("storeProcessor", new ProcessorSupplier() {

        @Override
        public Processor get() {
            return new Processor<String, Long>() {

                private KeyValueStore<String, Long> store;

                @Override
                public void init(final ProcessorContext context) {
                    // noinspection unchecked
                    this.store = (KeyValueStore<String, Long>) context.getStateStore("storeProcessorStore");
                }

                @Override
                public void process(final String key, final Long value) {
                    store.put(key, value);
                }

                @Override
                public void punctuate(final long timestamp) {
                }

                @Override
                public void close() {
                }
            };
        }
    }, "sourceProcessor");
    topology.addStateStore(Stores.keyValueStoreBuilder(Stores.persistentKeyValueStore("storeProcessorStore"), Serdes.String(), Serdes.Long()), "storeProcessor");
    final Properties config = new Properties();
    config.put(StreamsConfig.APPLICATION_ID_CONFIG, "test-TopologyTestDriver-cleanup");
    config.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "dummy:1234");
    config.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
    config.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
    config.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.Long().getClass().getName());
    {
        final TopologyTestDriver testDriver = new TopologyTestDriver(topology, config);
        Assert.assertNull(testDriver.getKeyValueStore("storeProcessorStore").get("a"));
        testDriver.pipeInput(recordFactory.create("input-topic", "a", 1L));
        Assert.assertEquals(1L, testDriver.getKeyValueStore("storeProcessorStore").get("a"));
        testDriver.close();
    }
    {
        final TopologyTestDriver testDriver = new TopologyTestDriver(topology, config);
        Assert.assertNull("Closing the prior test driver should have cleaned up this store and value.", testDriver.getKeyValueStore("storeProcessorStore").get("a"));
    }
}