Search in sources :

Example 66 with Topology

use of org.apache.kafka.streams.Topology in project kafkastreams-cep by fhussonnois.

the class CEPStockKStreamsIntegrationTest method test.

@Test
public void test() throws ExecutionException, InterruptedException {
    final Collection<KeyValue<String, String>> batch1 = Arrays.asList(new KeyValue<>(null, "{\"name\":\"e1\",\"price\":100,\"volume\":1010}"), new KeyValue<>(null, "{\"name\":\"e2\",\"price\":120,\"volume\":990}"), new KeyValue<>(null, "{\"name\":\"e3\",\"price\":120,\"volume\":1005}"), new KeyValue<>(null, "{\"name\":\"e4\",\"price\":121,\"volume\":999}"), new KeyValue<>(null, "{\"name\":\"e5\",\"price\":120,\"volume\":999}"), new KeyValue<>(null, "{\"name\":\"e6\",\"price\":125,\"volume\":750}"), new KeyValue<>(null, "{\"name\":\"e7\",\"price\":120,\"volume\":950}"), new KeyValue<>(null, "{\"name\":\"e8\",\"price\":120,\"volume\":700}"));
    IntegrationTestUtils.produceKeyValuesSynchronously(INPUT_STREAM, batch1, TestUtils.producerConfig(CLUSTER.bootstrapServers(), StringSerializer.class, StringSerializer.class, new Properties()), mockTime);
    // build query
    ComplexStreamsBuilder builder = new ComplexStreamsBuilder();
    CEPStream<String, StockEvent> stream = builder.stream(INPUT_STREAM);
    KStream<String, Sequence<String, StockEvent>> stocks = stream.query("Stocks", Patterns.STOCKS);
    stocks.mapValues(seq -> {
        JSONObject json = new JSONObject();
        seq.asMap().forEach((k, v) -> {
            JSONArray events = new JSONArray();
            json.put(k, events);
            List<String> collect = v.stream().map(e -> e.value.name).collect(Collectors.toList());
            Collections.reverse(collect);
            collect.forEach(events::add);
        });
        return json.toJSONString();
    }).through(OUTPUT_STREAM, Produced.with(null, Serdes.String())).print(Printed.toSysOut());
    Topology topology = builder.build();
    kafkaStreams = new KafkaStreams(topology, streamsConfiguration);
    kafkaStreams.start();
    final Properties consumerConfig = TestUtils.consumerConfig(CLUSTER.bootstrapServers(), StringDeserializer.class, StringDeserializer.class);
    List<KeyValue<String, String>> result = IntegrationTestUtils.readKeyValues(OUTPUT_STREAM, consumerConfig, TimeUnit.SECONDS.toMillis(10), 4);
    Assert.assertEquals(4, result.size());
    Assert.assertEquals("{\"0\":[\"e1\"],\"1\":[\"e2\",\"e3\",\"e4\",\"e5\"],\"2\":[\"e6\"]}", result.get(0).value);
    Assert.assertEquals("{\"0\":[\"e3\"],\"1\":[\"e4\"],\"2\":[\"e6\"]}", result.get(1).value);
    Assert.assertEquals("{\"0\":[\"e1\"],\"1\":[\"e2\",\"e3\",\"e4\",\"e5\",\"e6\",\"e7\"],\"2\":[\"e8\"]}", result.get(2).value);
    Assert.assertEquals("{\"0\":[\"e3\"],\"1\":[\"e4\",\"e6\"],\"2\":[\"e8\"]}", result.get(3).value);
}
Also used : Sequence(com.github.fhuss.kafka.streams.cep.Sequence) StreamsConfig(org.apache.kafka.streams.StreamsConfig) Arrays(java.util.Arrays) Produced(org.apache.kafka.streams.kstream.Produced) CEPStream(com.github.fhuss.kafka.streams.cep.CEPStream) KStream(org.apache.kafka.streams.kstream.KStream) JSONArray(org.json.simple.JSONArray) MockTime(kafka.utils.MockTime) StringDeserializer(org.apache.kafka.common.serialization.StringDeserializer) EmbeddedKafkaCluster(org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster) After(org.junit.After) Serdes(org.apache.kafka.common.serialization.Serdes) StringSerializer(org.apache.kafka.common.serialization.StringSerializer) ClassRule(org.junit.ClassRule) Pattern(com.github.fhuss.kafka.streams.cep.pattern.Pattern) Printed(org.apache.kafka.streams.kstream.Printed) Before(org.junit.Before) Properties(java.util.Properties) TestUtils(org.apache.kafka.test.TestUtils) Collection(java.util.Collection) KeyValue(org.apache.kafka.streams.KeyValue) ConsumerConfig(org.apache.kafka.clients.consumer.ConsumerConfig) Test(org.junit.Test) IOException(java.io.IOException) ComplexStreamsBuilder(com.github.fhuss.kafka.streams.cep.ComplexStreamsBuilder) Collectors(java.util.stream.Collectors) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) IntegrationTestUtils(org.apache.kafka.streams.integration.utils.IntegrationTestUtils) List(java.util.List) JSONObject(org.json.simple.JSONObject) QueryBuilder(com.github.fhuss.kafka.streams.cep.pattern.QueryBuilder) KafkaStreams(org.apache.kafka.streams.KafkaStreams) Assert(org.junit.Assert) Collections(java.util.Collections) Topology(org.apache.kafka.streams.Topology) KafkaStreams(org.apache.kafka.streams.KafkaStreams) KeyValue(org.apache.kafka.streams.KeyValue) ComplexStreamsBuilder(com.github.fhuss.kafka.streams.cep.ComplexStreamsBuilder) JSONArray(org.json.simple.JSONArray) Sequence(com.github.fhuss.kafka.streams.cep.Sequence) Topology(org.apache.kafka.streams.Topology) Properties(java.util.Properties) JSONObject(org.json.simple.JSONObject) StringSerializer(org.apache.kafka.common.serialization.StringSerializer) Test(org.junit.Test)

Example 67 with Topology

use of org.apache.kafka.streams.Topology in project brave by openzipkin.

the class ITKafkaStreamsTracing method should_create_spans_from_stream_with_tracing_filter_predicate_false.

@Test
public void should_create_spans_from_stream_with_tracing_filter_predicate_false() {
    String inputTopic = testName.getMethodName() + "-input";
    String outputTopic = testName.getMethodName() + "-output";
    StreamsBuilder builder = new StreamsBuilder();
    builder.stream(inputTopic, Consumed.with(Serdes.String(), Serdes.String())).transform(kafkaStreamsTracing.filter("filter-2", (key, value) -> false)).to(outputTopic, Produced.with(Serdes.String(), Serdes.String()));
    Topology topology = builder.build();
    KafkaStreams streams = buildKafkaStreams(topology);
    send(new ProducerRecord<>(inputTopic, TEST_KEY, TEST_VALUE));
    waitForStreamToRun(streams);
    MutableSpan spanInput = testSpanHandler.takeRemoteSpan(CONSUMER);
    assertThat(spanInput.tags()).containsEntry("kafka.topic", inputTopic);
    MutableSpan spanProcessor = testSpanHandler.takeLocalSpan();
    assertChildOf(spanProcessor, spanInput);
    assertThat(spanProcessor.tags()).containsEntry(KAFKA_STREAMS_FILTERED_TAG, "true");
    // the filter transformer returns false so record is dropped
    streams.close();
    streams.cleanUp();
}
Also used : StreamsBuilder(org.apache.kafka.streams.StreamsBuilder) KafkaStreams(org.apache.kafka.streams.KafkaStreams) MutableSpan(brave.handler.MutableSpan) Topology(org.apache.kafka.streams.Topology) Test(org.junit.Test)

Example 68 with Topology

use of org.apache.kafka.streams.Topology in project brave by openzipkin.

the class ITKafkaStreamsTracing method should_throw_exception_upwards.

@Test
public void should_throw_exception_upwards() {
    TransformerSupplier<String, String, KeyValue<String, String>> transformerSupplier = kafkaStreamsTracing.transformer("exception-transformer", () -> new Transformer<String, String, KeyValue<String, String>>() {

        ProcessorContext context;

        @Override
        public void init(ProcessorContext context) {
            this.context = context;
        }

        @Override
        public KeyValue<String, String> transform(String key, String value) {
            throw new IllegalArgumentException("illegal-argument");
        }

        @Override
        public void close() {
        }
    });
    String inputTopic = testName.getMethodName() + "-input";
    String outputTopic = testName.getMethodName() + "-output";
    StreamsBuilder builder = new StreamsBuilder();
    builder.stream(inputTopic, Consumed.with(Serdes.String(), Serdes.String())).transform(transformerSupplier).to(outputTopic, Produced.with(Serdes.String(), Serdes.String()));
    Topology topology = builder.build();
    KafkaStreams streams = buildKafkaStreams(topology);
    send(new ProducerRecord<>(inputTopic, TEST_KEY, TEST_VALUE));
    waitForStreamToRun(streams);
    MutableSpan spanInput = testSpanHandler.takeRemoteSpan(CONSUMER);
    assertThat(spanInput.tags()).containsEntry("kafka.topic", inputTopic);
    MutableSpan spanProcessor = testSpanHandler.takeLocalSpan();
    assertThat(spanProcessor.error()).hasMessage("illegal-argument");
    assertChildOf(spanProcessor, spanInput);
    assertThat(!streams.state().isRunningOrRebalancing());
    streams.close();
    streams.cleanUp();
}
Also used : StreamsBuilder(org.apache.kafka.streams.StreamsBuilder) KafkaStreams(org.apache.kafka.streams.KafkaStreams) KeyValue(org.apache.kafka.streams.KeyValue) MutableSpan(brave.handler.MutableSpan) Topology(org.apache.kafka.streams.Topology) ProcessorContext(org.apache.kafka.streams.processor.ProcessorContext) Test(org.junit.Test)

Example 69 with Topology

use of org.apache.kafka.streams.Topology in project brave by openzipkin.

the class ITKafkaStreamsTracing method should_create_spans_from_stream_with_tracing_mark_as_filtered_predicate_false.

@Test
public void should_create_spans_from_stream_with_tracing_mark_as_filtered_predicate_false() {
    String inputTopic = testName.getMethodName() + "-input";
    String outputTopic = testName.getMethodName() + "-output";
    StreamsBuilder builder = new StreamsBuilder();
    builder.stream(inputTopic, Consumed.with(Serdes.String(), Serdes.String())).transformValues(kafkaStreamsTracing.markAsFiltered("filter-2", (key, value) -> false)).filterNot((k, v) -> Objects.isNull(v)).to(outputTopic, Produced.with(Serdes.String(), Serdes.String()));
    Topology topology = builder.build();
    KafkaStreams streams = buildKafkaStreams(topology);
    send(new ProducerRecord<>(inputTopic, TEST_KEY, TEST_VALUE));
    waitForStreamToRun(streams);
    MutableSpan spanInput = testSpanHandler.takeRemoteSpan(CONSUMER);
    assertThat(spanInput.tags()).containsEntry("kafka.topic", inputTopic);
    MutableSpan spanProcessor = testSpanHandler.takeLocalSpan();
    assertChildOf(spanProcessor, spanInput);
    assertThat(spanProcessor.tags()).containsEntry(KAFKA_STREAMS_FILTERED_TAG, "true");
    // the filter transformer returns false so record is dropped
    streams.close();
    streams.cleanUp();
}
Also used : StreamsBuilder(org.apache.kafka.streams.StreamsBuilder) StreamsConfig(org.apache.kafka.streams.StreamsConfig) TEST_KEY(brave.kafka.streams.KafkaStreamsTracingTest.TEST_KEY) Arrays(java.util.Arrays) ProducerRecord(org.apache.kafka.clients.producer.ProducerRecord) ValueTransformerSupplier(org.apache.kafka.streams.kstream.ValueTransformerSupplier) Produced(org.apache.kafka.streams.kstream.Produced) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) ProcessorSupplier(org.apache.kafka.streams.processor.ProcessorSupplier) ArrayList(java.util.ArrayList) TransformerSupplier(org.apache.kafka.streams.kstream.TransformerSupplier) KafkaJunitRule(com.github.charithe.kafka.KafkaJunitRule) After(org.junit.After) Serdes(org.apache.kafka.common.serialization.Serdes) ValueTransformer(org.apache.kafka.streams.kstream.ValueTransformer) KafkaTracing(brave.kafka.clients.KafkaTracing) ClassRule(org.junit.ClassRule) PRODUCER(brave.Span.Kind.PRODUCER) Consumer(org.apache.kafka.clients.consumer.Consumer) CommonClientConfigs(org.apache.kafka.clients.CommonClientConfigs) TopicPartition(org.apache.kafka.common.TopicPartition) StreamsBuilder(org.apache.kafka.streams.StreamsBuilder) Properties(java.util.Properties) Producer(org.apache.kafka.clients.producer.Producer) Consumed(org.apache.kafka.streams.kstream.Consumed) Transformer(org.apache.kafka.streams.kstream.Transformer) KeyValue(org.apache.kafka.streams.KeyValue) ConsumerConfig(org.apache.kafka.clients.consumer.ConsumerConfig) Test(org.junit.Test) MessagingTracing(brave.messaging.MessagingTracing) TEST_VALUE(brave.kafka.streams.KafkaStreamsTracingTest.TEST_VALUE) TraceContext(brave.propagation.TraceContext) ValueTransformerWithKeySupplier(org.apache.kafka.streams.kstream.ValueTransformerWithKeySupplier) Assertions.entry(org.assertj.core.api.Assertions.entry) FileNotFoundException(java.io.FileNotFoundException) EphemeralKafkaBroker(com.github.charithe.kafka.EphemeralKafkaBroker) Objects(java.util.Objects) ProcessorContext(org.apache.kafka.streams.processor.ProcessorContext) List(java.util.List) MutableSpan(brave.handler.MutableSpan) KAFKA_STREAMS_FILTERED_TAG(brave.kafka.streams.KafkaStreamsTags.KAFKA_STREAMS_FILTERED_TAG) ValueTransformerWithKey(org.apache.kafka.streams.kstream.ValueTransformerWithKey) KafkaStreams(org.apache.kafka.streams.KafkaStreams) AbstractProcessor(org.apache.kafka.streams.processor.AbstractProcessor) CONSUMER(brave.Span.Kind.CONSUMER) Topology(org.apache.kafka.streams.Topology) KafkaConsumer(org.apache.kafka.clients.consumer.KafkaConsumer) KafkaStreams(org.apache.kafka.streams.KafkaStreams) MutableSpan(brave.handler.MutableSpan) Topology(org.apache.kafka.streams.Topology) Test(org.junit.Test)

Example 70 with Topology

use of org.apache.kafka.streams.Topology in project brave by openzipkin.

the class ITKafkaStreamsTracing method should_create_multiple_span_from_stream_input_topic_whenSharingDisabled.

@Test
public void should_create_multiple_span_from_stream_input_topic_whenSharingDisabled() {
    String inputTopic = testName.getMethodName() + "-input";
    StreamsBuilder builder = new StreamsBuilder();
    builder.stream(inputTopic).foreach((k, v) -> {
    });
    Topology topology = builder.build();
    KafkaStreamsTracing kafkaStreamsTracing = KafkaStreamsTracing.newBuilder(tracing).singleRootSpanOnReceiveBatch(false).build();
    KafkaStreams streams = kafkaStreamsTracing.kafkaStreams(topology, streamsProperties());
    send(new ProducerRecord<>(inputTopic, TEST_KEY, TEST_VALUE));
    send(new ProducerRecord<>(inputTopic, TEST_KEY, TEST_VALUE));
    send(new ProducerRecord<>(inputTopic, TEST_KEY, TEST_VALUE));
    waitForStreamToRun(streams);
    for (int i = 0; i < 3; i++) {
        MutableSpan spanInput = testSpanHandler.takeRemoteSpan(CONSUMER);
        assertThat(spanInput.tags()).containsEntry("kafka.topic", inputTopic);
    }
    streams.close();
    streams.cleanUp();
}
Also used : StreamsBuilder(org.apache.kafka.streams.StreamsBuilder) KafkaStreams(org.apache.kafka.streams.KafkaStreams) MutableSpan(brave.handler.MutableSpan) Topology(org.apache.kafka.streams.Topology) Test(org.junit.Test)

Aggregations

Topology (org.apache.kafka.streams.Topology)127 Test (org.junit.Test)106 StreamsBuilder (org.apache.kafka.streams.StreamsBuilder)93 KafkaStreams (org.apache.kafka.streams.KafkaStreams)53 TopologyTestDriver (org.apache.kafka.streams.TopologyTestDriver)53 Properties (java.util.Properties)47 StringSerializer (org.apache.kafka.common.serialization.StringSerializer)46 KeyValue (org.apache.kafka.streams.KeyValue)40 Serdes (org.apache.kafka.common.serialization.Serdes)39 StreamsConfig (org.apache.kafka.streams.StreamsConfig)33 List (java.util.List)29 MutableSpan (brave.handler.MutableSpan)28 Consumed (org.apache.kafka.streams.kstream.Consumed)28 Produced (org.apache.kafka.streams.kstream.Produced)26 Arrays (java.util.Arrays)25 StringDeserializer (org.apache.kafka.common.serialization.StringDeserializer)25 ArrayList (java.util.ArrayList)23 ProcessorContext (org.apache.kafka.streams.processor.ProcessorContext)23 Duration (java.time.Duration)22 KStream (org.apache.kafka.streams.kstream.KStream)22