use of org.apache.samza.operators.KV in project samza by apache.
the class MergeExample method describe.
@Override
public void describe(StreamApplicationDescriptor appDescriptor) {
KVSerde<String, PageViewEvent> serde = KVSerde.of(new StringSerde("UTF-8"), new JsonSerdeV2<>(PageViewEvent.class));
KafkaSystemDescriptor trackingSystem = new KafkaSystemDescriptor("tracking");
KafkaInputDescriptor<KV<String, PageViewEvent>> isd1 = trackingSystem.getInputDescriptor("pageViewStream1", serde);
KafkaInputDescriptor<KV<String, PageViewEvent>> isd2 = trackingSystem.getInputDescriptor("pageViewStream2", serde);
KafkaInputDescriptor<KV<String, PageViewEvent>> isd3 = trackingSystem.getInputDescriptor("pageViewStream3", serde);
KafkaOutputDescriptor<KV<String, PageViewEvent>> osd = trackingSystem.getOutputDescriptor("mergedStream", serde);
MessageStream.mergeAll(ImmutableList.of(appDescriptor.getInputStream(isd1), appDescriptor.getInputStream(isd2), appDescriptor.getInputStream(isd3))).sendTo(appDescriptor.getOutputStream(osd));
}
use of org.apache.samza.operators.KV in project samza by apache.
the class PageViewCounterExample method describe.
@Override
public void describe(StreamApplicationDescriptor appDescriptor) {
KafkaSystemDescriptor trackingSystem = new KafkaSystemDescriptor("tracking");
KafkaInputDescriptor<PageViewEvent> inputStreamDescriptor = trackingSystem.getInputDescriptor("pageViewEvent", new JsonSerdeV2<>(PageViewEvent.class));
KafkaOutputDescriptor<KV<String, PageViewCount>> outputStreamDescriptor = trackingSystem.getOutputDescriptor("pageViewEventPerMember", KVSerde.of(new StringSerde(), new JsonSerdeV2<>(PageViewCount.class)));
MessageStream<PageViewEvent> pageViewEvents = appDescriptor.getInputStream(inputStreamDescriptor);
OutputStream<KV<String, PageViewCount>> pageViewEventPerMemberStream = appDescriptor.getOutputStream(outputStreamDescriptor);
SupplierFunction<Integer> initialValue = () -> 0;
FoldLeftFunction<PageViewEvent, Integer> foldLeftFn = (m, c) -> c + 1;
pageViewEvents.window(Windows.keyedTumblingWindow(PageViewEvent::getMemberId, Duration.ofSeconds(10), initialValue, foldLeftFn, null, null).setEarlyTrigger(Triggers.repeat(Triggers.count(5))).setAccumulationMode(AccumulationMode.DISCARDING), "tumblingWindow").map(windowPane -> KV.of(windowPane.getKey().getKey(), buildPageViewCount(windowPane))).sendTo(pageViewEventPerMemberStream);
}
use of org.apache.samza.operators.KV in project samza by apache.
the class TestSamzaSqlRemoteTableJoinFunction method testWithInnerJoinWithTableOnRight.
@Test
public void testWithInnerJoinWithTableOnRight() {
Map<String, String> props = new HashMap<>();
SystemStream ss = new SystemStream("test", "nestedRecord");
props.put(String.format(ConfigBasedAvroRelSchemaProviderFactory.CFG_SOURCE_SCHEMA, ss.getSystem(), ss.getStream()), SimpleRecord.SCHEMA$.toString());
ConfigBasedAvroRelSchemaProviderFactory factory = new ConfigBasedAvroRelSchemaProviderFactory();
AvroRelSchemaProvider schemaProvider = (AvroRelSchemaProvider) factory.create(ss, new MapConfig(props));
AvroRelConverter relConverter = new AvroRelConverter(ss, schemaProvider, new MapConfig());
SamzaRelTableKeyConverter relTableKeyConverter = new SampleRelTableKeyConverter();
String remoteTableName = "testDb.testTable.$table";
GenericData.Record tableRecord = new GenericData.Record(SimpleRecord.SCHEMA$);
tableRecord.put("id", 1);
tableRecord.put("name", "name1");
SamzaSqlRelMessage streamMsg = new SamzaSqlRelMessage(streamFieldNames, streamFieldValues, new SamzaSqlRelMsgMetadata(0L, 0L));
SamzaSqlRelMessage tableMsg = relConverter.convertToRelMessage(new KV(tableRecord.get("id"), tableRecord));
JoinRelType joinRelType = JoinRelType.INNER;
List<Integer> streamKeyIds = Arrays.asList(1);
List<Integer> tableKeyIds = Arrays.asList(0);
KV<Object, GenericRecord> record = KV.of(tableRecord.get("id"), tableRecord);
JoinInputNode mockTableInputNode = mock(JoinInputNode.class);
when(mockTableInputNode.getKeyIds()).thenReturn(tableKeyIds);
when(mockTableInputNode.isPosOnRight()).thenReturn(true);
when(mockTableInputNode.getFieldNames()).thenReturn(tableMsg.getSamzaSqlRelRecord().getFieldNames());
when(mockTableInputNode.getSourceName()).thenReturn(remoteTableName);
JoinInputNode mockStreamInputNode = mock(JoinInputNode.class);
when(mockStreamInputNode.getKeyIds()).thenReturn(streamKeyIds);
when(mockStreamInputNode.isPosOnRight()).thenReturn(false);
when(mockStreamInputNode.getFieldNames()).thenReturn(streamFieldNames);
SamzaSqlRemoteTableJoinFunction joinFn = new SamzaSqlRemoteTableJoinFunction(relConverter, relTableKeyConverter, mockStreamInputNode, mockTableInputNode, joinRelType, 0);
SamzaSqlRelMessage outMsg = joinFn.apply(streamMsg, record);
Assert.assertEquals(outMsg.getSamzaSqlRelRecord().getFieldValues().size(), outMsg.getSamzaSqlRelRecord().getFieldNames().size());
List<String> expectedFieldNames = new ArrayList<>(streamFieldNames);
expectedFieldNames.addAll(tableMsg.getSamzaSqlRelRecord().getFieldNames());
List<Object> expectedFieldValues = new ArrayList<>(streamFieldValues);
expectedFieldValues.addAll(tableMsg.getSamzaSqlRelRecord().getFieldValues());
Assert.assertEquals(expectedFieldNames, outMsg.getSamzaSqlRelRecord().getFieldNames());
Assert.assertEquals(expectedFieldValues, outMsg.getSamzaSqlRelRecord().getFieldValues());
}
use of org.apache.samza.operators.KV in project samza by apache.
the class KeyValueStoreExample method describe.
@Override
public void describe(StreamApplicationDescriptor appDescriptor) {
KafkaSystemDescriptor trackingSystem = new KafkaSystemDescriptor("tracking");
KafkaInputDescriptor<PageViewEvent> inputStreamDescriptor = trackingSystem.getInputDescriptor("pageViewEvent", new JsonSerdeV2<>(PageViewEvent.class));
KafkaOutputDescriptor<KV<String, StatsOutput>> outputStreamDescriptor = trackingSystem.getOutputDescriptor("pageViewEventPerMember", KVSerde.of(new StringSerde(), new JsonSerdeV2<>(StatsOutput.class)));
appDescriptor.withDefaultSystem(trackingSystem);
MessageStream<PageViewEvent> pageViewEvents = appDescriptor.getInputStream(inputStreamDescriptor);
OutputStream<KV<String, StatsOutput>> pageViewEventPerMember = appDescriptor.getOutputStream(outputStreamDescriptor);
pageViewEvents.partitionBy(pve -> pve.getMemberId(), pve -> pve, KVSerde.of(new StringSerde(), new JsonSerdeV2<>(PageViewEvent.class)), "partitionBy").map(KV::getValue).flatMap(new MyStatsCounter()).map(stats -> KV.of(stats.memberId, stats)).sendTo(pageViewEventPerMember);
}
use of org.apache.samza.operators.KV in project beam by apache.
the class SamzaTestStreamTranslator method createInputDescriptor.
@SuppressWarnings("unchecked")
private static <T> GenericInputDescriptor<KV<?, OpMessage<T>>> createInputDescriptor(String id, String encodedTestStream, SerializableFunction<String, TestStream<T>> testStreamDecoder) {
final Map<String, String> systemConfig = ImmutableMap.of(ENCODED_TEST_STREAM, encodedTestStream, TEST_STREAM_DECODER, Base64Serializer.serializeUnchecked(testStreamDecoder));
final GenericSystemDescriptor systemDescriptor = new GenericSystemDescriptor(id, SamzaTestStreamSystemFactory.class.getName()).withSystemConfigs(systemConfig);
// The KvCoder is needed here for Samza not to crop the key.
final Serde<KV<?, OpMessage<T>>> kvSerde = KVSerde.of(new NoOpSerde(), new NoOpSerde<>());
return systemDescriptor.getInputDescriptor(id, kvSerde);
}
Aggregations