Examples with Topology org.apache.kafka.streams.Topology used on opensource projects

Example 81 with Topology

use of org.apache.kafka.streams.Topology in project brave by openzipkin.

the class ITKafkaStreamsTracing method should_create_spans_from_stream_with_tracing_mapValues_withKey.

@Test
public void should_create_spans_from_stream_with_tracing_mapValues_withKey() {
    String inputTopic = testName.getMethodName() + "-input";
    String outputTopic = testName.getMethodName() + "-output";
    StreamsBuilder builder = new StreamsBuilder();
    builder.stream(inputTopic, Consumed.with(Serdes.String(), Serdes.String())).transformValues(kafkaStreamsTracing.mapValues("mapValue-1", (key, value) -> {
        try {
            Thread.sleep(100L);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        return value;
    })).to(outputTopic, Produced.with(Serdes.String(), Serdes.String()));
    Topology topology = builder.build();
    KafkaStreams streams = buildKafkaStreams(topology);
    send(new ProducerRecord<>(inputTopic, TEST_KEY, TEST_VALUE));
    waitForStreamToRun(streams);
    MutableSpan spanInput = testSpanHandler.takeRemoteSpan(CONSUMER);
    assertThat(spanInput.tags()).containsEntry("kafka.topic", inputTopic);
    MutableSpan spanProcessor = testSpanHandler.takeLocalSpan();
    assertChildOf(spanProcessor, spanInput);
    MutableSpan spanOutput = testSpanHandler.takeRemoteSpan(PRODUCER);
    assertThat(spanOutput.tags()).containsEntry("kafka.topic", outputTopic);
    assertChildOf(spanOutput, spanProcessor);
    streams.close();
    streams.cleanUp();
}

Example 82 with Topology

use of org.apache.kafka.streams.Topology in project brave by openzipkin.

the class ITKafkaStreamsTracing method should_create_spans_from_stream_with_tracing_mark.

@Test
public void should_create_spans_from_stream_with_tracing_mark() {
    String inputTopic = testName.getMethodName() + "-input";
    String outputTopic = testName.getMethodName() + "-output";
    StreamsBuilder builder = new StreamsBuilder();
    builder.stream(inputTopic, Consumed.with(Serdes.String(), Serdes.String())).transformValues(kafkaStreamsTracing.mark("mark-1")).to(outputTopic, Produced.with(Serdes.String(), Serdes.String()));
    Topology topology = builder.build();
    KafkaStreams streams = buildKafkaStreams(topology);
    send(new ProducerRecord<>(inputTopic, TEST_KEY, TEST_VALUE));
    waitForStreamToRun(streams);
    MutableSpan spanInput = testSpanHandler.takeRemoteSpan(CONSUMER);
    assertThat(spanInput.tags()).containsEntry("kafka.topic", inputTopic);
    MutableSpan spanProcessor = testSpanHandler.takeLocalSpan();
    assertChildOf(spanProcessor, spanInput);
    MutableSpan spanOutput = testSpanHandler.takeRemoteSpan(PRODUCER);
    assertThat(spanOutput.tags()).containsEntry("kafka.topic", outputTopic);
    assertChildOf(spanOutput, spanProcessor);
    streams.close();
    streams.cleanUp();
}

Example 83 with Topology

use of org.apache.kafka.streams.Topology in project brave by openzipkin.

the class ITKafkaStreamsTracing method should_create_spans_from_stream_with_tracing_mark_as_not_filtered_predicate_true.

@Test
public void should_create_spans_from_stream_with_tracing_mark_as_not_filtered_predicate_true() {
    String inputTopic = testName.getMethodName() + "-input";
    String outputTopic = testName.getMethodName() + "-output";
    StreamsBuilder builder = new StreamsBuilder();
    builder.stream(inputTopic, Consumed.with(Serdes.String(), Serdes.String())).transformValues(kafkaStreamsTracing.markAsNotFiltered("filterNot-1", (key, value) -> true)).filterNot((k, v) -> Objects.isNull(v)).to(outputTopic, Produced.with(Serdes.String(), Serdes.String()));
    Topology topology = builder.build();
    KafkaStreams streams = buildKafkaStreams(topology);
    send(new ProducerRecord<>(inputTopic, TEST_KEY, TEST_VALUE));
    waitForStreamToRun(streams);
    MutableSpan spanInput = testSpanHandler.takeRemoteSpan(CONSUMER);
    assertThat(spanInput.tags()).containsEntry("kafka.topic", inputTopic);
    MutableSpan spanProcessor = testSpanHandler.takeLocalSpan();
    assertChildOf(spanProcessor, spanInput);
    assertThat(spanProcessor.tags()).containsEntry(KAFKA_STREAMS_FILTERED_TAG, "true");
    // the filterNot transformer returns true so record is dropped
    streams.close();
    streams.cleanUp();
}

Example 84 with Topology

use of org.apache.kafka.streams.Topology in project brave by openzipkin.

the class ITKafkaStreamsTracing method should_create_one_span_from_stream_input_topic_whenSharingEnabled.

@Test
public void should_create_one_span_from_stream_input_topic_whenSharingEnabled() {
    String inputTopic = testName.getMethodName() + "-input";
    StreamsBuilder builder = new StreamsBuilder();
    builder.stream(inputTopic).foreach((k, v) -> {
    });
    Topology topology = builder.build();
    MessagingTracing messagingTracing = MessagingTracing.create(tracing);
    KafkaStreamsTracing kafkaStreamsTracing = KafkaStreamsTracing.newBuilder(messagingTracing).singleRootSpanOnReceiveBatch(true).build();
    KafkaStreams streams = kafkaStreamsTracing.kafkaStreams(topology, streamsProperties());
    send(new ProducerRecord<>(inputTopic, TEST_KEY, TEST_VALUE));
    send(new ProducerRecord<>(inputTopic, TEST_KEY, TEST_VALUE));
    send(new ProducerRecord<>(inputTopic, TEST_KEY, TEST_VALUE));
    waitForStreamToRun(streams);
    MutableSpan spanInput = testSpanHandler.takeRemoteSpan(CONSUMER);
    assertThat(spanInput.tags()).containsEntry("kafka.topic", inputTopic);
    streams.close();
    streams.cleanUp();
}

Example 85 with Topology

use of org.apache.kafka.streams.Topology in project apache-kafka-on-k8s by banzaicloud.

the class WordCountProcessorDemo method main.

public static void main(String[] args) throws Exception {
    Properties props = new Properties();
    props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-wordcount-processor");
    props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
    props.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0);
    props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
    props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
    // setting offset reset to earliest so that we can re-run the demo code with the same pre-loaded data
    props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    Topology builder = new Topology();
    builder.addSource("Source", "streams-plaintext-input");
    builder.addProcessor("Process", new MyProcessorSupplier(), "Source");
    builder.addStateStore(Stores.keyValueStoreBuilder(Stores.inMemoryKeyValueStore("Counts"), Serdes.String(), Serdes.Integer()), "Process");
    builder.addSink("Sink", "streams-wordcount-processor-output", "Process");
    final KafkaStreams streams = new KafkaStreams(builder, props);
    final CountDownLatch latch = new CountDownLatch(1);
    // attach shutdown handler to catch control-c
    Runtime.getRuntime().addShutdownHook(new Thread("streams-wordcount-shutdown-hook") {

        @Override
        public void run() {
            streams.close();
            latch.countDown();
        }
    });
    try {
        streams.start();
        latch.await();
    } catch (Throwable e) {
        System.exit(1);
    }
    System.exit(0);
}