Examples with KStreamBuilder org.apache.kafka.streams.kstream.KStreamBuilder used on opensource projects

Example 56 with KStreamBuilder

use of org.apache.kafka.streams.kstream.KStreamBuilder in project kafka by apache.

the class KafkaStreamsTest method testLegalMetricsConfig.

@Test
public void testLegalMetricsConfig() {
    final Properties props = new Properties();
    props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "appId");
    props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
    props.setProperty(StreamsConfig.METRICS_RECORDING_LEVEL_CONFIG, Sensor.RecordingLevel.INFO.toString());
    final KStreamBuilder builder1 = new KStreamBuilder();
    final KafkaStreams streams1 = new KafkaStreams(builder1, props);
    streams1.close();
    props.setProperty(StreamsConfig.METRICS_RECORDING_LEVEL_CONFIG, Sensor.RecordingLevel.DEBUG.toString());
    final KStreamBuilder builder2 = new KStreamBuilder();
    final KafkaStreams streams2 = new KafkaStreams(builder2, props);
}

Example 57 with KStreamBuilder

use of org.apache.kafka.streams.kstream.KStreamBuilder in project kafka by apache.

the class KafkaStreamsTest method testCleanup.

@Test
public void testCleanup() throws Exception {
    final Properties props = new Properties();
    props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "testLocalCleanup");
    props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
    final KStreamBuilder builder = new KStreamBuilder();
    final KafkaStreams streams = new KafkaStreams(builder, props);
    streams.cleanUp();
    streams.start();
    streams.close();
    streams.cleanUp();
}

Example 58 with KStreamBuilder

use of org.apache.kafka.streams.kstream.KStreamBuilder in project kafka by apache.

the class InternalTopicIntegrationTest method shouldUseCompactAndDeleteForWindowStoreChangelogs.

@Test
public void shouldUseCompactAndDeleteForWindowStoreChangelogs() throws Exception {
    KStreamBuilder builder = new KStreamBuilder();
    KStream<String, String> textLines = builder.stream(DEFAULT_INPUT_TOPIC);
    final int durationMs = 2000;
    textLines.flatMapValues(new ValueMapper<String, Iterable<String>>() {

        @Override
        public Iterable<String> apply(String value) {
            return Arrays.asList(value.toLowerCase(Locale.getDefault()).split("\\W+"));
        }
    }).groupBy(MockKeyValueMapper.<String, String>SelectValueMapper()).count(TimeWindows.of(1000).until(durationMs), "CountWindows").toStream();
    // Remove any state from previous test runs
    IntegrationTestUtils.purgeLocalStreamsState(streamsConfiguration);
    KafkaStreams streams = new KafkaStreams(builder, streamsConfiguration);
    streams.start();
    //
    // Step 2: Produce some input data to the input topic.
    //
    produceData(Arrays.asList("hello", "world", "world", "hello world"));
    //
    // Step 3: Verify the state changelog topics are compact
    //
    streams.close();
    final Properties properties = getTopicConfigProperties(ProcessorStateManager.storeChangelogTopic(applicationId, "CountWindows"));
    final List<String> policies = Arrays.asList(properties.getProperty(LogConfig.CleanupPolicyProp()).split(","));
    assertEquals(2, policies.size());
    assertTrue(policies.contains(LogConfig.Compact()));
    assertTrue(policies.contains(LogConfig.Delete()));
    // retention should be 1 day + the window duration
    final long retention = TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS) + durationMs;
    assertEquals(retention, Long.parseLong(properties.getProperty(LogConfig.RetentionMsProp())));
}

Example 59 with KStreamBuilder

use of org.apache.kafka.streams.kstream.KStreamBuilder in project kafka by apache.

the class KStreamKStreamLeftJoinTest method testLeftJoin.

@Test
public void testLeftJoin() throws Exception {
    final KStreamBuilder builder = new KStreamBuilder();
    final int[] expectedKeys = new int[] { 0, 1, 2, 3 };
    final KStream<Integer, String> stream1;
    final KStream<Integer, String> stream2;
    final KStream<Integer, String> joined;
    final MockProcessorSupplier<Integer, String> processor;
    processor = new MockProcessorSupplier<>();
    stream1 = builder.stream(intSerde, stringSerde, topic1);
    stream2 = builder.stream(intSerde, stringSerde, topic2);
    joined = stream1.leftJoin(stream2, MockValueJoiner.TOSTRING_JOINER, JoinWindows.of(100), intSerde, stringSerde, stringSerde);
    joined.process(processor);
    final Collection<Set<String>> copartitionGroups = builder.copartitionGroups();
    assertEquals(1, copartitionGroups.size());
    assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next());
    driver = new KStreamTestDriver(builder, stateDir);
    driver.setTime(0L);
    for (int i = 0; i < 2; i++) {
        driver.process(topic1, expectedKeys[i], "X" + expectedKeys[i]);
    }
    driver.flushState();
    processor.checkAndClearProcessResult("0:X0+null", "1:X1+null");
    for (int i = 0; i < 2; i++) {
        driver.process(topic2, expectedKeys[i], "Y" + expectedKeys[i]);
    }
    driver.flushState();
    processor.checkAndClearProcessResult("0:X0+Y0", "1:X1+Y1");
    for (int i = 0; i < 3; i++) {
        driver.process(topic1, expectedKeys[i], "X" + expectedKeys[i]);
    }
    driver.flushState();
    processor.checkAndClearProcessResult("0:X0+Y0", "1:X1+Y1", "2:X2+null");
    for (int expectedKey : expectedKeys) {
        driver.process(topic2, expectedKey, "YY" + expectedKey);
    }
    driver.flushState();
    processor.checkAndClearProcessResult("0:X0+YY0", "0:X0+YY0", "1:X1+YY1", "1:X1+YY1", "2:X2+YY2");
    for (int expectedKey : expectedKeys) {
        driver.process(topic1, expectedKey, "XX" + expectedKey);
    }
    driver.flushState();
    processor.checkAndClearProcessResult("0:XX0+Y0", "0:XX0+YY0", "1:XX1+Y1", "1:XX1+YY1", "2:XX2+YY2", "3:XX3+YY3");
}

Example 60 with KStreamBuilder

use of org.apache.kafka.streams.kstream.KStreamBuilder in project kafka by apache.

the class KStreamMapTest method testTypeVariance.

@Test
public void testTypeVariance() throws Exception {
    KeyValueMapper<Number, Object, KeyValue<Number, String>> stringify = new KeyValueMapper<Number, Object, KeyValue<Number, String>>() {

        @Override
        public KeyValue<Number, String> apply(Number key, Object value) {
            return KeyValue.pair(key, key + ":" + value);
        }
    };
    new KStreamBuilder().<Integer, String>stream("numbers").map(stringify).to("strings");
}