Examples with StringSerde org.apache.samza.serializers.StringSerde used on opensource projects

Example 26 with StringSerde

use of org.apache.samza.serializers.StringSerde in project samza by apache.

the class TestLargeMessageSafeKeyValueStores method setup.

@Before
public void setup() {
    KeyValueStore<byte[], byte[]> kvStore;
    switch(typeOfStore) {
        case "inmemory":
            {
                kvStore = new InMemoryKeyValueStore(keyValueStoreMetrics);
                break;
            }
        case "rocksdb":
            {
                kvStore = new RocksDbKeyValueStore(dir, new org.rocksdb.Options().setCreateIfMissing(true).setCompressionType(org.rocksdb.CompressionType.SNAPPY_COMPRESSION), new MapConfig(), false, storeName, new WriteOptions(), new FlushOptions(), keyValueStoreMetrics);
                break;
            }
        default:
            throw new IllegalArgumentException("Type of store undefined: " + typeOfStore);
    }
    MessageCollector collector = envelope -> {
        int messageLength = ((byte[]) envelope.getMessage()).length;
        if (messageLength > maxMessageSize) {
            throw new SamzaException("Logged store message size " + messageLength + " for store " + storeName + " was larger than the maximum allowed message size " + maxMessageSize + ".");
        }
    };
    loggedStore = new LoggedStore<>(kvStore, systemStreamPartition, collector, loggedStoreMetrics);
    switch(storeConfig) {
        case "serde":
            {
                KeyValueStore<byte[], byte[]> largeMessageSafeStore = new LargeMessageSafeStore(loggedStore, storeName, dropLargeMessage, maxMessageSize);
                store = new SerializedKeyValueStore<>(largeMessageSafeStore, stringSerde, stringSerde, serializedKeyValueStoreMetrics);
                break;
            }
        case "cache-then-serde":
            {
                KeyValueStore<byte[], byte[]> toBeSerializedStore = loggedStore;
                if (dropLargeMessage) {
                    toBeSerializedStore = new LargeMessageSafeStore(loggedStore, storeName, dropLargeMessage, maxMessageSize);
                }
                KeyValueStore<String, String> serializedStore = new SerializedKeyValueStore<>(toBeSerializedStore, stringSerde, stringSerde, serializedKeyValueStoreMetrics);
                store = new CachedStore<>(serializedStore, cacheSize, batchSize, cachedStoreMetrics);
                break;
            }
        // large messages are expected and StorageConfig.DISALLOW_LARGE_MESSAGES is true.
        case "serde-then-cache":
            {
                KeyValueStore<byte[], byte[]> cachedStore = new CachedStore<>(loggedStore, cacheSize, batchSize, cachedStoreMetrics);
                KeyValueStore<byte[], byte[]> largeMessageSafeStore = new LargeMessageSafeStore(cachedStore, storeName, dropLargeMessage, maxMessageSize);
                store = new SerializedKeyValueStore<>(largeMessageSafeStore, stringSerde, stringSerde, serializedKeyValueStoreMetrics);
                break;
            }
        default:
            throw new IllegalArgumentException("Store config undefined: " + storeConfig);
    }
    store = new NullSafeKeyValueStore<>(store);
}

Example 27 with StringSerde

use of org.apache.samza.serializers.StringSerde in project samza by apache.

the class SessionWindowApp method describe.

@Override
public void describe(StreamApplicationDescriptor appDescriptor) {
    JsonSerdeV2<PageView> inputSerde = new JsonSerdeV2<>(PageView.class);
    KVSerde<String, Integer> outputSerde = KVSerde.of(new StringSerde(), new IntegerSerde());
    KafkaSystemDescriptor ksd = new KafkaSystemDescriptor(SYSTEM);
    KafkaInputDescriptor<PageView> id = ksd.getInputDescriptor(INPUT_TOPIC, inputSerde);
    KafkaOutputDescriptor<KV<String, Integer>> od = ksd.getOutputDescriptor(OUTPUT_TOPIC, outputSerde);
    MessageStream<PageView> pageViews = appDescriptor.getInputStream(id);
    OutputStream<KV<String, Integer>> outputStream = appDescriptor.getOutputStream(od);
    pageViews.filter(m -> !FILTER_KEY.equals(m.getUserId())).window(Windows.keyedSessionWindow(PageView::getUserId, Duration.ofSeconds(3), new StringSerde(), new JsonSerdeV2<>(PageView.class)), "sessionWindow").map(m -> KV.of(m.getKey().getKey(), m.getMessage().size())).sendTo(outputStream);
}

Example 28 with StringSerde

use of org.apache.samza.serializers.StringSerde in project samza by apache.

the class TumblingWindowApp method describe.

@Override
public void describe(StreamApplicationDescriptor appDescriptor) {
    JsonSerdeV2<PageView> inputSerde = new JsonSerdeV2<>(PageView.class);
    KVSerde<String, Integer> outputSerde = KVSerde.of(new StringSerde(), new IntegerSerde());
    KafkaSystemDescriptor ksd = new KafkaSystemDescriptor(SYSTEM);
    KafkaInputDescriptor<PageView> id = ksd.getInputDescriptor(INPUT_TOPIC, inputSerde);
    KafkaOutputDescriptor<KV<String, Integer>> od = ksd.getOutputDescriptor(OUTPUT_TOPIC, outputSerde);
    MessageStream<PageView> pageViews = appDescriptor.getInputStream(id);
    OutputStream<KV<String, Integer>> outputStream = appDescriptor.getOutputStream(od);
    pageViews.filter(m -> !FILTER_KEY.equals(m.getUserId())).window(Windows.keyedTumblingWindow(PageView::getUserId, Duration.ofSeconds(3), new StringSerde(), new JsonSerdeV2<>(PageView.class)), "tumblingWindow").map(m -> KV.of(m.getKey().getKey(), m.getMessage().size())).sendTo(outputStream);
}

Example 29 with StringSerde

use of org.apache.samza.serializers.StringSerde in project samza by apache.

the class MergeExample method describe.

@Override
public void describe(StreamApplicationDescriptor appDescriptor) {
    KVSerde<String, PageViewEvent> serde = KVSerde.of(new StringSerde("UTF-8"), new JsonSerdeV2<>(PageViewEvent.class));
    KafkaSystemDescriptor trackingSystem = new KafkaSystemDescriptor("tracking");
    KafkaInputDescriptor<KV<String, PageViewEvent>> isd1 = trackingSystem.getInputDescriptor("pageViewStream1", serde);
    KafkaInputDescriptor<KV<String, PageViewEvent>> isd2 = trackingSystem.getInputDescriptor("pageViewStream2", serde);
    KafkaInputDescriptor<KV<String, PageViewEvent>> isd3 = trackingSystem.getInputDescriptor("pageViewStream3", serde);
    KafkaOutputDescriptor<KV<String, PageViewEvent>> osd = trackingSystem.getOutputDescriptor("mergedStream", serde);
    MessageStream.mergeAll(ImmutableList.of(appDescriptor.getInputStream(isd1), appDescriptor.getInputStream(isd2), appDescriptor.getInputStream(isd3))).sendTo(appDescriptor.getOutputStream(osd));
}

Example 30 with StringSerde

use of org.apache.samza.serializers.StringSerde in project samza by apache.

the class PageViewCounterExample method describe.

@Override
public void describe(StreamApplicationDescriptor appDescriptor) {
    KafkaSystemDescriptor trackingSystem = new KafkaSystemDescriptor("tracking");
    KafkaInputDescriptor<PageViewEvent> inputStreamDescriptor = trackingSystem.getInputDescriptor("pageViewEvent", new JsonSerdeV2<>(PageViewEvent.class));
    KafkaOutputDescriptor<KV<String, PageViewCount>> outputStreamDescriptor = trackingSystem.getOutputDescriptor("pageViewEventPerMember", KVSerde.of(new StringSerde(), new JsonSerdeV2<>(PageViewCount.class)));
    MessageStream<PageViewEvent> pageViewEvents = appDescriptor.getInputStream(inputStreamDescriptor);
    OutputStream<KV<String, PageViewCount>> pageViewEventPerMemberStream = appDescriptor.getOutputStream(outputStreamDescriptor);
    SupplierFunction<Integer> initialValue = () -> 0;
    FoldLeftFunction<PageViewEvent, Integer> foldLeftFn = (m, c) -> c + 1;
    pageViewEvents.window(Windows.keyedTumblingWindow(PageViewEvent::getMemberId, Duration.ofSeconds(10), initialValue, foldLeftFn, null, null).setEarlyTrigger(Triggers.repeat(Triggers.count(5))).setAccumulationMode(AccumulationMode.DISCARDING), "tumblingWindow").map(windowPane -> KV.of(windowPane.getKey().getKey(), buildPageViewCount(windowPane))).sendTo(pageViewEventPerMemberStream);
}