Examples with ValueMapper org.apache.kafka.streams.kstream.ValueMapper used on opensource projects

Example 16 with ValueMapper

use of org.apache.kafka.streams.kstream.ValueMapper in project kafka by apache.

the class InternalTopicIntegrationTest method shouldUseCompactAndDeleteForWindowStoreChangelogs.

@Test
public void shouldUseCompactAndDeleteForWindowStoreChangelogs() throws Exception {
    KStreamBuilder builder = new KStreamBuilder();
    KStream<String, String> textLines = builder.stream(DEFAULT_INPUT_TOPIC);
    final int durationMs = 2000;
    textLines.flatMapValues(new ValueMapper<String, Iterable<String>>() {

        @Override
        public Iterable<String> apply(String value) {
            return Arrays.asList(value.toLowerCase(Locale.getDefault()).split("\\W+"));
        }
    }).groupBy(MockKeyValueMapper.<String, String>SelectValueMapper()).count(TimeWindows.of(1000).until(durationMs), "CountWindows").toStream();
    // Remove any state from previous test runs
    IntegrationTestUtils.purgeLocalStreamsState(streamsConfiguration);
    KafkaStreams streams = new KafkaStreams(builder, streamsConfiguration);
    streams.start();
    //
    // Step 2: Produce some input data to the input topic.
    //
    produceData(Arrays.asList("hello", "world", "world", "hello world"));
    //
    // Step 3: Verify the state changelog topics are compact
    //
    streams.close();
    final Properties properties = getTopicConfigProperties(ProcessorStateManager.storeChangelogTopic(applicationId, "CountWindows"));
    final List<String> policies = Arrays.asList(properties.getProperty(LogConfig.CleanupPolicyProp()).split(","));
    assertEquals(2, policies.size());
    assertTrue(policies.contains(LogConfig.Compact()));
    assertTrue(policies.contains(LogConfig.Delete()));
    // retention should be 1 day + the window duration
    final long retention = TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS) + durationMs;
    assertEquals(retention, Long.parseLong(properties.getProperty(LogConfig.RetentionMsProp())));
}

Example 17 with ValueMapper

use of org.apache.kafka.streams.kstream.ValueMapper in project ksql by confluentinc.

the class SchemaKTable method into.

@SuppressWarnings("unchecked")
@Override
public SchemaKTable into(final String kafkaTopicName, final Serde<GenericRow> topicValueSerDe, Set<Integer> rowkeyIndexes) {
    if (isWindowed) {
        final Serde<Windowed<String>> windowedSerde = WindowedSerdes.timeWindowedSerdeFrom(String.class);
        ktable.toStream().mapValues((ValueMapper<GenericRow, GenericRow>) row -> {
            if (row == null) {
                return null;
            }
            List columns = new ArrayList();
            for (int i = 0; i < row.getColumns().size(); i++) {
                if (!rowkeyIndexes.contains(i)) {
                    columns.add(row.getColumns().get(i));
                }
            }
            return new GenericRow(columns);
        }).to(kafkaTopicName, Produced.with(windowedSerde, topicValueSerDe));
    } else {
        ktable.toStream().mapValues((ValueMapper<GenericRow, GenericRow>) row -> {
            if (row == null) {
                return null;
            }
            List columns = new ArrayList();
            for (int i = 0; i < row.getColumns().size(); i++) {
                if (!rowkeyIndexes.contains(i)) {
                    columns.add(row.getColumns().get(i));
                }
            }
            return new GenericRow(columns);
        }).to(kafkaTopicName, Produced.with(Serdes.String(), topicValueSerDe));
    }
    return this;
}

Example 18 with ValueMapper

use of org.apache.kafka.streams.kstream.ValueMapper in project apache-kafka-on-k8s by banzaicloud.

the class BrokerCompatibilityTest method main.

public static void main(final String[] args) throws IOException {
    System.out.println("StreamsTest instance started");
    final String kafka = args.length > 0 ? args[0] : "localhost:9092";
    final String propFileName = args.length > 1 ? args[1] : null;
    final boolean eosEnabled = args.length > 2 ? Boolean.parseBoolean(args[2]) : false;
    final Properties streamsProperties = Utils.loadProps(propFileName);
    streamsProperties.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, kafka);
    streamsProperties.put(StreamsConfig.APPLICATION_ID_CONFIG, "kafka-streams-system-test-broker-compatibility");
    streamsProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    streamsProperties.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
    streamsProperties.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
    streamsProperties.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 100);
    streamsProperties.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0);
    if (eosEnabled) {
        streamsProperties.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE);
    }
    final int timeout = 6000;
    streamsProperties.put(StreamsConfig.consumerPrefix(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG), timeout);
    streamsProperties.put(StreamsConfig.consumerPrefix(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG), timeout);
    streamsProperties.put(StreamsConfig.REQUEST_TIMEOUT_MS_CONFIG, timeout + 1);
    // TODO remove this config or set to smaller value when KIP-91 is merged
    streamsProperties.put(StreamsConfig.producerPrefix(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG), 60000);
    Serde<String> stringSerde = Serdes.String();
    final StreamsBuilder builder = new StreamsBuilder();
    builder.<String, String>stream(SOURCE_TOPIC).groupByKey(Serialized.with(stringSerde, stringSerde)).count().toStream().mapValues(new ValueMapper<Long, String>() {

        @Override
        public String apply(Long value) {
            return value.toString();
        }
    }).to(SINK_TOPIC);
    final KafkaStreams streams = new KafkaStreams(builder.build(), streamsProperties);
    streams.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {

        @Override
        public void uncaughtException(final Thread t, final Throwable e) {
            Throwable cause = e;
            if (cause instanceof StreamsException) {
                while (cause.getCause() != null) {
                    cause = cause.getCause();
                }
            }
            System.err.println("FATAL: An unexpected exception " + cause);
            e.printStackTrace(System.err);
            System.err.flush();
            streams.close(30, TimeUnit.SECONDS);
        }
    });
    System.out.println("start Kafka Streams");
    streams.start();
    System.out.println("send data");
    final Properties producerProperties = new Properties();
    producerProperties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka);
    producerProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
    producerProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
    try {
        try (final KafkaProducer<String, String> producer = new KafkaProducer<>(producerProperties)) {
            producer.send(new ProducerRecord<>(SOURCE_TOPIC, "key", "value"));
            System.out.println("wait for result");
            loopUntilRecordReceived(kafka, eosEnabled);
            System.out.println("close Kafka Streams");
            streams.close();
        }
    } catch (final RuntimeException e) {
        System.err.println("Non-Streams exception occurred: ");
        e.printStackTrace(System.err);
        System.err.flush();
    }
}

Example 19 with ValueMapper

use of org.apache.kafka.streams.kstream.ValueMapper in project apache-kafka-on-k8s by banzaicloud.

the class KStreamMapValuesTest method testFlatMapValues.

@Test
public void testFlatMapValues() {
    StreamsBuilder builder = new StreamsBuilder();
    ValueMapper<CharSequence, Integer> mapper = new ValueMapper<CharSequence, Integer>() {

        @Override
        public Integer apply(CharSequence value) {
            return value.length();
        }
    };
    final int[] expectedKeys = { 1, 10, 100, 1000 };
    KStream<Integer, String> stream;
    MockProcessorSupplier<Integer, Integer> processor = new MockProcessorSupplier<>();
    stream = builder.stream(topicName, Consumed.with(intSerde, stringSerde));
    stream.mapValues(mapper).process(processor);
    driver.setUp(builder);
    for (int expectedKey : expectedKeys) {
        driver.process(topicName, expectedKey, Integer.toString(expectedKey));
    }
    String[] expected = { "1:1", "10:2", "100:3", "1000:4" };
    assertArrayEquals(expected, processor.processed.toArray());
}

Example 20 with ValueMapper

use of org.apache.kafka.streams.kstream.ValueMapper in project apache-kafka-on-k8s by banzaicloud.

the class KTableImplTest method testStateStore.

@Test
public void testStateStore() {
    String topic1 = "topic1";
    String topic2 = "topic2";
    final StreamsBuilder builder = new StreamsBuilder();
    KTableImpl<String, String, String> table1 = (KTableImpl<String, String, String>) builder.table(topic1, consumed);
    KTableImpl<String, String, String> table2 = (KTableImpl<String, String, String>) builder.table(topic2, consumed);
    KTableImpl<String, String, Integer> table1Mapped = (KTableImpl<String, String, Integer>) table1.mapValues(new ValueMapper<String, Integer>() {

        @Override
        public Integer apply(String value) {
            return new Integer(value);
        }
    });
    KTableImpl<String, Integer, Integer> table1MappedFiltered = (KTableImpl<String, Integer, Integer>) table1Mapped.filter(new Predicate<String, Integer>() {

        @Override
        public boolean test(String key, Integer value) {
            return (value % 2) == 0;
        }
    });
    table2.join(table1MappedFiltered, new ValueJoiner<String, Integer, String>() {

        @Override
        public String apply(String v1, Integer v2) {
            return v1 + v2;
        }
    });
    driver.setUp(builder, stateDir, null, null);
    driver.setTime(0L);
    // two state store should be created
    assertEquals(2, driver.allStateStores().size());
}