Examples with Processor org.apache.kafka.streams.processor.Processor used on opensource projects

Example 1 with Processor

use of org.apache.kafka.streams.processor.Processor in project kafka by apache.

the class SimpleBenchmark method createKafkaStreamsWithStateStore.

private KafkaStreams createKafkaStreamsWithStateStore(String topic, final CountDownLatch latch, boolean enableCaching) {
    Properties props = setStreamProperties("simple-benchmark-streams-with-store" + enableCaching);
    KStreamBuilder builder = new KStreamBuilder();
    if (enableCaching) {
        builder.addStateStore(Stores.create("store").withIntegerKeys().withByteArrayValues().persistent().enableCaching().build());
    } else {
        builder.addStateStore(Stores.create("store").withIntegerKeys().withByteArrayValues().persistent().build());
    }
    KStream<Integer, byte[]> source = builder.stream(INTEGER_SERDE, BYTE_SERDE, topic);
    source.process(new ProcessorSupplier<Integer, byte[]>() {

        @Override
        public Processor<Integer, byte[]> get() {
            return new AbstractProcessor<Integer, byte[]>() {

                KeyValueStore<Integer, byte[]> store;

                @SuppressWarnings("unchecked")
                @Override
                public void init(ProcessorContext context) {
                    store = (KeyValueStore<Integer, byte[]>) context.getStateStore("store");
                }

                @Override
                public void process(Integer key, byte[] value) {
                    store.put(key, value);
                    processedRecords++;
                    processedBytes += value.length + Integer.SIZE;
                    if (processedRecords == numRecords) {
                        latch.countDown();
                    }
                }

                @Override
                public void punctuate(long timestamp) {
                }

                @Override
                public void close() {
                }
            };
        }
    }, "store");
    return createKafkaStreamsWithExceptionHandler(builder, props);
}

Example 2 with Processor

use of org.apache.kafka.streams.processor.Processor in project apache-kafka-on-k8s by banzaicloud.

the class RestoreIntegrationTest method shouldProcessDataFromStoresWithLoggingDisabled.

@Test
public void shouldProcessDataFromStoresWithLoggingDisabled() throws InterruptedException, ExecutionException {
    IntegrationTestUtils.produceKeyValuesSynchronously(INPUT_STREAM_2, Arrays.asList(KeyValue.pair(1, 1), KeyValue.pair(2, 2), KeyValue.pair(3, 3)), TestUtils.producerConfig(CLUSTER.bootstrapServers(), IntegerSerializer.class, IntegerSerializer.class), CLUSTER.time);
    final KeyValueBytesStoreSupplier lruMapSupplier = Stores.lruMap(INPUT_STREAM_2, 10);
    final StoreBuilder<KeyValueStore<Integer, Integer>> storeBuilder = new KeyValueStoreBuilder<>(lruMapSupplier, Serdes.Integer(), Serdes.Integer(), CLUSTER.time).withLoggingDisabled();
    final StreamsBuilder streamsBuilder = new StreamsBuilder();
    streamsBuilder.addStateStore(storeBuilder);
    final KStream<Integer, Integer> stream = streamsBuilder.stream(INPUT_STREAM_2);
    final CountDownLatch processorLatch = new CountDownLatch(3);
    stream.process(new ProcessorSupplier<Integer, Integer>() {

        @Override
        public Processor<Integer, Integer> get() {
            return new KeyValueStoreProcessor(INPUT_STREAM_2, processorLatch);
        }
    }, INPUT_STREAM_2);
    final Topology topology = streamsBuilder.build();
    kafkaStreams = new KafkaStreams(topology, props(applicationId + "-logging-disabled"));
    final CountDownLatch latch = new CountDownLatch(1);
    kafkaStreams.setStateListener(new KafkaStreams.StateListener() {

        @Override
        public void onChange(final KafkaStreams.State newState, final KafkaStreams.State oldState) {
            if (newState == KafkaStreams.State.RUNNING && oldState == KafkaStreams.State.REBALANCING) {
                latch.countDown();
            }
        }
    });
    kafkaStreams.start();
    latch.await(30, TimeUnit.SECONDS);
    assertTrue(processorLatch.await(30, TimeUnit.SECONDS));
}

Example 3 with Processor

use of org.apache.kafka.streams.processor.Processor in project apache-kafka-on-k8s by banzaicloud.

the class TopologyTestDriverTest method shouldReturnAllStores.

@Test
public void shouldReturnAllStores() {
    final Topology topology = setupSourceSinkTopology();
    topology.addStateStore(new KeyValueStoreBuilder<>(Stores.inMemoryKeyValueStore("store"), Serdes.ByteArray(), Serdes.ByteArray(), new SystemTime()).withLoggingDisabled());
    topology.addGlobalStore(new KeyValueStoreBuilder<>(Stores.inMemoryKeyValueStore("globalStore"), Serdes.ByteArray(), Serdes.ByteArray(), new SystemTime()).withLoggingDisabled(), "sourceProcessorName", Serdes.ByteArray().deserializer(), Serdes.ByteArray().deserializer(), "globalTopicName", "globalProcessorName", new ProcessorSupplier() {

        @Override
        public Processor get() {
            return null;
        }
    });
    testDriver = new TopologyTestDriver(topology, config);
    final Set<String> expectedStoreNames = new HashSet<>();
    expectedStoreNames.add("store");
    expectedStoreNames.add("globalStore");
    assertThat(testDriver.getAllStateStores().keySet(), equalTo(expectedStoreNames));
}

Example 4 with Processor

use of org.apache.kafka.streams.processor.Processor in project apache-kafka-on-k8s by banzaicloud.

the class MockProcessorContextTest method shouldCapturePunctuator.

@Test
public void shouldCapturePunctuator() {
    final Processor<String, Long> processor = new Processor<String, Long>() {

        @Override
        public void init(final ProcessorContext context) {
            context.schedule(1000L, PunctuationType.WALL_CLOCK_TIME, new Punctuator() {

                @Override
                public void punctuate(final long timestamp) {
                    context.commit();
                }
            });
        }

        @Override
        public void process(final String key, final Long value) {
        }

        @Override
        public void punctuate(final long timestamp) {
        }

        @Override
        public void close() {
        }
    };
    final MockProcessorContext context = new MockProcessorContext();
    processor.init(context);
    final MockProcessorContext.CapturedPunctuator capturedPunctuator = context.scheduledPunctuators().get(0);
    assertEquals(1000L, capturedPunctuator.getIntervalMs());
    assertEquals(PunctuationType.WALL_CLOCK_TIME, capturedPunctuator.getType());
    assertFalse(capturedPunctuator.cancelled());
    final Punctuator punctuator = capturedPunctuator.getPunctuator();
    assertFalse(context.committed());
    punctuator.punctuate(1234L);
    assertTrue(context.committed());
}

Example 5 with Processor

use of org.apache.kafka.streams.processor.Processor in project apache-kafka-on-k8s by banzaicloud.

the class StreamThreadTest method shouldPunctuateActiveTask.

@Test
public void shouldPunctuateActiveTask() {
    final List<Long> punctuatedStreamTime = new ArrayList<>();
    final List<Long> punctuatedWallClockTime = new ArrayList<>();
    final ProcessorSupplier<Object, Object> punctuateProcessor = new ProcessorSupplier<Object, Object>() {

        @Override
        public Processor<Object, Object> get() {
            return new Processor<Object, Object>() {

                @Override
                public void init(ProcessorContext context) {
                    context.schedule(100L, PunctuationType.STREAM_TIME, new Punctuator() {

                        @Override
                        public void punctuate(long timestamp) {
                            punctuatedStreamTime.add(timestamp);
                        }
                    });
                    context.schedule(100L, PunctuationType.WALL_CLOCK_TIME, new Punctuator() {

                        @Override
                        public void punctuate(long timestamp) {
                            punctuatedWallClockTime.add(timestamp);
                        }
                    });
                }

                @Override
                public void process(Object key, Object value) {
                }

                @SuppressWarnings("deprecation")
                @Override
                public void punctuate(long timestamp) {
                }

                @Override
                public void close() {
                }
            };
        }
    };
    internalStreamsBuilder.stream(Collections.singleton(topic1), consumed).process(punctuateProcessor);
    final StreamThread thread = createStreamThread(clientId, config, false);
    thread.setState(StreamThread.State.RUNNING);
    thread.rebalanceListener.onPartitionsRevoked(null);
    final List<TopicPartition> assignedPartitions = new ArrayList<>();
    final Map<TaskId, Set<TopicPartition>> activeTasks = new HashMap<>();
    // assign single partition
    assignedPartitions.add(t1p1);
    activeTasks.put(task1, Collections.singleton(t1p1));
    thread.taskManager().setAssignmentMetadata(activeTasks, Collections.<TaskId, Set<TopicPartition>>emptyMap());
    clientSupplier.consumer.assign(assignedPartitions);
    clientSupplier.consumer.updateBeginningOffsets(Collections.singletonMap(t1p1, 0L));
    thread.rebalanceListener.onPartitionsAssigned(assignedPartitions);
    thread.runOnce(-1);
    assertEquals(0, punctuatedStreamTime.size());
    assertEquals(0, punctuatedWallClockTime.size());
    mockTime.sleep(100L);
    for (long i = 0L; i < 10L; i++) {
        clientSupplier.consumer.addRecord(new ConsumerRecord<>(topic1, 1, i, i * 100L, TimestampType.CREATE_TIME, ConsumerRecord.NULL_CHECKSUM, ("K" + i).getBytes().length, ("V" + i).getBytes().length, ("K" + i).getBytes(), ("V" + i).getBytes()));
    }
    thread.runOnce(-1);
    assertEquals(1, punctuatedStreamTime.size());
    assertEquals(1, punctuatedWallClockTime.size());
    mockTime.sleep(100L);
    thread.runOnce(-1);
    // we should skip stream time punctuation, only trigger wall-clock time punctuation
    assertEquals(1, punctuatedStreamTime.size());
    assertEquals(2, punctuatedWallClockTime.size());
}