Examples with KVSerde org.apache.samza.serializers.KVSerde used on opensource projects

Example 21 with KVSerde

use of org.apache.samza.serializers.KVSerde in project samza by apache.

the class TestEventHubsOutputDescriptor method testStreamDescriptorContainsKVserde.

@Test
public void testStreamDescriptorContainsKVserde() {
    String systemName = "eventHub";
    String streamId = "output-stream";
    EventHubsSystemDescriptor systemDescriptor = new EventHubsSystemDescriptor(systemName);
    EventHubsOutputDescriptor<KV<String, String>> outputDescriptor = systemDescriptor.getOutputDescriptor(streamId, "entity-namespace", "entity3", new StringSerde());
    assertTrue(outputDescriptor.getSerde() instanceof KVSerde);
    assertTrue(((KVSerde) outputDescriptor.getSerde()).getKeySerde() instanceof NoOpSerde);
    assertTrue(((KVSerde) outputDescriptor.getSerde()).getValueSerde() instanceof StringSerde);
}

Example 22 with KVSerde

use of org.apache.samza.serializers.KVSerde in project samza by apache.

the class TestEventHubsInputDescriptor method testStreamDescriptorContainsKVserde.

@Test
public void testStreamDescriptorContainsKVserde() {
    String systemName = "eventHub";
    String streamId = "input-stream";
    EventHubsSystemDescriptor systemDescriptor = new EventHubsSystemDescriptor(systemName);
    EventHubsInputDescriptor<KV<String, String>> outputDescriptor = systemDescriptor.getInputDescriptor(streamId, "entity-namespace", "entity3", new StringSerde());
    assertTrue(outputDescriptor.getSerde() instanceof KVSerde);
    assertTrue(((KVSerde) outputDescriptor.getSerde()).getKeySerde() instanceof NoOpSerde);
    assertTrue(((KVSerde) outputDescriptor.getSerde()).getValueSerde() instanceof StringSerde);
}

Example 23 with KVSerde

use of org.apache.samza.serializers.KVSerde in project samza by apache.

the class TestStreamApplicationDescriptorImpl method testGetOutputStreamWithKeyValueSerde.

@Test
public void testGetOutputStreamWithKeyValueSerde() {
    String streamId = "test-stream-1";
    KVSerde mockKVSerde = mock(KVSerde.class);
    Serde mockKeySerde = mock(Serde.class);
    Serde mockValueSerde = mock(Serde.class);
    doReturn(mockKeySerde).when(mockKVSerde).getKeySerde();
    doReturn(mockValueSerde).when(mockKVSerde).getValueSerde();
    GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
    GenericOutputDescriptor osd = sd.getOutputDescriptor(streamId, mockKVSerde);
    StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> {
        appDesc.getOutputStream(osd);
    }, getConfig());
    OutputStreamImpl<TestMessageEnvelope> outputStreamImpl = streamAppDesc.getOutputStreams().get(streamId);
    assertEquals(streamId, outputStreamImpl.getStreamId());
    assertEquals(osd, streamAppDesc.getOutputDescriptors().get(streamId));
    assertEquals(mockKeySerde, outputStreamImpl.getKeySerde());
    assertEquals(mockValueSerde, outputStreamImpl.getValueSerde());
}

Example 24 with KVSerde

use of org.apache.samza.serializers.KVSerde in project beam by apache.

the class ReadTranslator method translate.

@Override
public void translate(PTransform<PBegin, PCollection<T>> transform, TransformHierarchy.Node node, TranslationContext ctx) {
    final PCollection<T> output = ctx.getOutput(transform);
    final Coder<WindowedValue<T>> coder = SamzaCoders.of(output);
    final Source<?> source = transform instanceof SplittableParDo.PrimitiveBoundedRead ? ((SplittableParDo.PrimitiveBoundedRead) transform).getSource() : ((SplittableParDo.PrimitiveUnboundedRead) transform).getSource();
    final String id = ctx.getIdForPValue(output);
    // Create system descriptor
    final GenericSystemDescriptor systemDescriptor;
    if (source instanceof BoundedSource) {
        systemDescriptor = new GenericSystemDescriptor(id, BoundedSourceSystem.Factory.class.getName());
    } else {
        systemDescriptor = new GenericSystemDescriptor(id, UnboundedSourceSystem.Factory.class.getName());
    }
    final Map<String, String> systemConfig = ImmutableMap.of("source", Base64Serializer.serializeUnchecked(source), "coder", Base64Serializer.serializeUnchecked(coder), "stepName", node.getFullName());
    systemDescriptor.withSystemConfigs(systemConfig);
    // Create stream descriptor
    @SuppressWarnings("unchecked") final Serde<KV<?, OpMessage<T>>> kvSerde = (Serde) KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>());
    final GenericInputDescriptor<KV<?, OpMessage<T>>> inputDescriptor = systemDescriptor.getInputDescriptor(id, kvSerde);
    if (source instanceof BoundedSource) {
        inputDescriptor.isBounded();
    }
    ctx.registerInputMessageStream(output, inputDescriptor);
}

Example 25 with KVSerde

use of org.apache.samza.serializers.KVSerde in project samza by apache.

the class TestExecutionPlanner method createStreamGraphWithStreamTableJoin.

private StreamApplicationDescriptorImpl createStreamGraphWithStreamTableJoin() {
    /**
     * Example stream-table join app. Expected partition counts of intermediate streams introduced
     * by partitionBy operations are enclosed in quotes.
     *
     *    input2 (16) -> partitionBy ("32") -> send-to-table t
     *
     *                                      join-table t —————
     *                                       |                |
     *    input1 (64) -> partitionBy ("32") _|                |
     *                                                       join -> output1 (8)
     *                                                        |
     *                                      input3 (32) ——————
     */
    return new StreamApplicationDescriptorImpl(appDesc -> {
        MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor);
        MessageStream<KV<Object, Object>> messageStream2 = appDesc.getInputStream(input2Descriptor);
        MessageStream<KV<Object, Object>> messageStream3 = appDesc.getInputStream(input3Descriptor);
        OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
        TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor("table-id", new KVSerde(new StringSerde(), new StringSerde()));
        Table table = appDesc.getTable(tableDescriptor);
        messageStream2.partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p1").sendTo(table);
        messageStream1.partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p2").join(table, mock(StreamTableJoinFunction.class)).join(messageStream3, mock(JoinFunction.class), mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofHours(1), "j2").sendTo(output1);
    }, config);
}