Examples with Serde org.apache.kafka.common.serialization.Serde used on opensource projects

Example 16 with Serde

use of org.apache.kafka.common.serialization.Serde in project ksql by confluentinc.

the class SchemaKStream method groupBy.

public SchemaKGroupedStream groupBy(final Serde<String> keySerde, final Serde<GenericRow> valSerde, final List<Expression> groupByExpressions) {
    boolean rekey = rekeyRequired(groupByExpressions);
    if (!rekey) {
        KGroupedStream kgroupedStream = kstream.groupByKey(Serialized.with(keySerde, valSerde));
        return new SchemaKGroupedStream(schema, kgroupedStream, keyField, Collections.singletonList(this), functionRegistry, schemaRegistryClient);
    }
    // Collect the column indexes, and build the new key as <column1>+<column2>+...
    StringBuilder aggregateKeyName = new StringBuilder();
    List<Integer> newKeyIndexes = new ArrayList<>();
    boolean addSeparator = false;
    for (Expression groupByExpr : groupByExpressions) {
        if (addSeparator) {
            aggregateKeyName.append("|+|");
        } else {
            addSeparator = true;
        }
        aggregateKeyName.append(groupByExpr.toString());
        newKeyIndexes.add(SchemaUtil.getIndexInSchema(groupByExpr.toString(), getSchema()));
    }
    KGroupedStream kgroupedStream = kstream.filter((key, value) -> value != null).groupBy((key, value) -> {
        StringBuilder newKey = new StringBuilder();
        boolean addSeparator1 = false;
        for (int index : newKeyIndexes) {
            if (addSeparator1) {
                newKey.append("|+|");
            } else {
                addSeparator1 = true;
            }
            newKey.append(String.valueOf(value.getColumns().get(index)));
        }
        return newKey.toString();
    }, Serialized.with(keySerde, valSerde));
    // TODO: if the key is a prefix of the grouping columns then we can
    // use the repartition reflection hack to tell streams not to
    // repartition.
    Field newKeyField = new Field(aggregateKeyName.toString(), -1, Schema.STRING_SCHEMA);
    return new SchemaKGroupedStream(schema, kgroupedStream, newKeyField, Collections.singletonList(this), functionRegistry, schemaRegistryClient);
}

Example 17 with Serde

use of org.apache.kafka.common.serialization.Serde in project tutorials by eugenp.

the class KafkaStreamsLiveTest method shouldTestKafkaStreams.

@Test
@Ignore("it needs to have kafka broker running on local")
public void shouldTestKafkaStreams() throws InterruptedException {
    // given
    String inputTopic = "inputTopic";
    Properties streamsConfiguration = new Properties();
    streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, "wordcount-live-test");
    streamsConfiguration.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
    streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
    streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
    streamsConfiguration.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000);
    streamsConfiguration.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    // Use a temporary directory for storing state, which will be automatically removed after the test.
    streamsConfiguration.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath());
    // when
    KStreamBuilder builder = new KStreamBuilder();
    KStream<String, String> textLines = builder.stream(inputTopic);
    Pattern pattern = Pattern.compile("\\W+", Pattern.UNICODE_CHARACTER_CLASS);
    KTable<String, Long> wordCounts = textLines.flatMapValues(value -> Arrays.asList(pattern.split(value.toLowerCase()))).groupBy((key, word) -> word).count();
    wordCounts.foreach((word, count) -> System.out.println("word: " + word + " -> " + count));
    String outputTopic = "outputTopic";
    final Serde<String> stringSerde = Serdes.String();
    final Serde<Long> longSerde = Serdes.Long();
    wordCounts.to(stringSerde, longSerde, outputTopic);
    KafkaStreams streams = new KafkaStreams(builder, streamsConfiguration);
    streams.start();
    // then
    Thread.sleep(30000);
    streams.close();
}

Example 18 with Serde

use of org.apache.kafka.common.serialization.Serde in project apache-kafka-on-k8s by banzaicloud.

the class InMemoryKeyValueLoggedStoreTest method createKeyValueStore.

@SuppressWarnings("unchecked")
@Override
protected <K, V> KeyValueStore<K, V> createKeyValueStore(final ProcessorContext context) {
    final StoreBuilder storeBuilder = Stores.keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), (Serde<K>) context.keySerde(), (Serde<V>) context.valueSerde()).withLoggingEnabled(Collections.singletonMap("retention.ms", "1000"));
    final StateStore store = storeBuilder.build();
    store.init(context, store);
    return (KeyValueStore<K, V>) store;
}

Example 19 with Serde

use of org.apache.kafka.common.serialization.Serde in project apache-kafka-on-k8s by banzaicloud.

the class TimeWindowedDeserializer method configure.

@SuppressWarnings("unchecked")
@Override
public void configure(final Map<String, ?> configs, final boolean isKey) {
    if (inner == null) {
        final String propertyName = isKey ? StreamsConfig.DEFAULT_WINDOWED_KEY_SERDE_INNER_CLASS : StreamsConfig.DEFAULT_WINDOWED_VALUE_SERDE_INNER_CLASS;
        final String value = (String) configs.get(propertyName);
        try {
            inner = Serde.class.cast(Utils.newInstance(value, Serde.class)).deserializer();
            inner.configure(configs, isKey);
        } catch (final ClassNotFoundException e) {
            throw new ConfigException(propertyName, value, "Serde class " + value + " could not be found.");
        }
    }
}

Example 20 with Serde

use of org.apache.kafka.common.serialization.Serde in project apache-kafka-on-k8s by banzaicloud.

the class CachingKeyValueStoreTest method createKeyValueStore.

@SuppressWarnings("unchecked")
@Override
protected <K, V> KeyValueStore<K, V> createKeyValueStore(final ProcessorContext context) {
    final StoreBuilder storeBuilder = Stores.keyValueStoreBuilder(Stores.persistentKeyValueStore("cache-store"), (Serde<K>) context.keySerde(), (Serde<V>) context.valueSerde()).withCachingEnabled();
    final KeyValueStore<K, V> store = (KeyValueStore<K, V>) storeBuilder.build();
    final CacheFlushListenerStub<K, V> cacheFlushListener = new CacheFlushListenerStub<>();
    final CachedStateStore inner = (CachedStateStore) ((WrappedStateStore) store).wrappedStore();
    inner.setFlushListener(cacheFlushListener, false);
    store.init(context, store);
    return store;
}