Examples with MessageType org.apache.samza.operators.data.MessageType used on opensource projects

Example 1 with MessageType

use of org.apache.samza.operators.data.MessageType in project samza by apache.

the class TestStreamGraphImpl method testGetOutputStream.

@Test
public void testGetOutputStream() {
    ApplicationRunner mockRunner = mock(ApplicationRunner.class);
    Config mockConfig = mock(Config.class);
    StreamSpec testStreamSpec = new StreamSpec("test-stream-1", "physical-stream-1", "test-system");
    when(mockRunner.getStreamSpec("test-stream-1")).thenReturn(testStreamSpec);
    class MyMessageType extends MessageType {

        public final String outputId;

        public MyMessageType(String value, long eventTime, String outputId) {
            super(value, eventTime);
            this.outputId = outputId;
        }
    }
    StreamGraphImpl graph = new StreamGraphImpl(mockRunner, mockConfig);
    Function<TestMessageEnvelope, String> xKeyExtractor = x -> x.getKey();
    Function<TestMessageEnvelope, MyMessageType> xMsgExtractor = x -> new MyMessageType(x.getMessage().getValue(), x.getMessage().getEventTime(), "test-output-id-1");
    OutputStream<String, MyMessageType, TestInputMessageEnvelope> mOutputStream = graph.getOutputStream("test-stream-1", xKeyExtractor, xMsgExtractor);
    assertEquals(graph.getOutputStreams().get(testStreamSpec), mOutputStream);
    assertTrue(mOutputStream instanceof OutputStreamInternalImpl);
    assertEquals(((OutputStreamInternalImpl) mOutputStream).getKeyExtractor(), xKeyExtractor);
    assertEquals(((OutputStreamInternalImpl) mOutputStream).getMsgExtractor(), xMsgExtractor);
    TestInputMessageEnvelope xInputMsg = new TestInputMessageEnvelope("test-key-1", "test-msg-1", 33333L, "input-id-1");
    assertEquals(((OutputStreamInternalImpl<String, MyMessageType, TestInputMessageEnvelope>) mOutputStream).getKeyExtractor().apply(xInputMsg), "test-key-1");
    assertEquals(((OutputStreamInternalImpl<String, MyMessageType, TestInputMessageEnvelope>) mOutputStream).getMsgExtractor().apply(xInputMsg).getValue(), "test-msg-1");
    assertEquals(((OutputStreamInternalImpl<String, MyMessageType, TestInputMessageEnvelope>) mOutputStream).getMsgExtractor().apply(xInputMsg).getEventTime(), 33333L);
    assertEquals(((OutputStreamInternalImpl<String, MyMessageType, TestInputMessageEnvelope>) mOutputStream).getMsgExtractor().apply(xInputMsg).outputId, "test-output-id-1");
}

Example 2 with MessageType

use of org.apache.samza.operators.data.MessageType in project samza by apache.

the class TestStreamGraphImpl method testGetIntermediateStream.

@Test
public void testGetIntermediateStream() {
    ApplicationRunner mockRunner = mock(ApplicationRunner.class);
    Config mockConfig = mock(Config.class);
    StreamSpec testStreamSpec = new StreamSpec("myJob-i001-test-stream-1", "physical-stream-1", "test-system");
    when(mockRunner.getStreamSpec("myJob-i001-test-stream-1")).thenReturn(testStreamSpec);
    when(mockConfig.get(JobConfig.JOB_NAME())).thenReturn("myJob");
    when(mockConfig.get(JobConfig.JOB_ID(), "1")).thenReturn("i001");
    class MyMessageType extends MessageType {

        public final String outputId;

        public MyMessageType(String value, long eventTime, String outputId) {
            super(value, eventTime);
            this.outputId = outputId;
        }
    }
    StreamGraphImpl graph = new StreamGraphImpl(mockRunner, mockConfig);
    Function<TestMessageEnvelope, String> xKeyExtractor = x -> x.getKey();
    Function<TestMessageEnvelope, MyMessageType> xMsgExtractor = x -> new MyMessageType(x.getMessage().getValue(), x.getMessage().getEventTime(), "test-output-id-1");
    BiFunction<String, MessageType, TestInputMessageEnvelope> xMsgBuilder = (k, v) -> new TestInputMessageEnvelope(k, v.getValue(), v.getEventTime(), "input-id-1");
    MessageStream<TestMessageEnvelope> mIntermediateStream = graph.getIntermediateStream("test-stream-1", xKeyExtractor, xMsgExtractor, xMsgBuilder);
    assertEquals(graph.getOutputStreams().get(testStreamSpec), mIntermediateStream);
    assertTrue(mIntermediateStream instanceof IntermediateStreamInternalImpl);
    assertEquals(((IntermediateStreamInternalImpl) mIntermediateStream).getKeyExtractor(), xKeyExtractor);
    assertEquals(((IntermediateStreamInternalImpl) mIntermediateStream).getMsgExtractor(), xMsgExtractor);
    assertEquals(((IntermediateStreamInternalImpl) mIntermediateStream).getMsgBuilder(), xMsgBuilder);
    TestMessageEnvelope xInputMsg = new TestMessageEnvelope("test-key-1", "test-msg-1", 33333L);
    assertEquals(((IntermediateStreamInternalImpl<String, MessageType, TestMessageEnvelope>) mIntermediateStream).getKeyExtractor().apply(xInputMsg), "test-key-1");
    assertEquals(((IntermediateStreamInternalImpl<String, MessageType, TestMessageEnvelope>) mIntermediateStream).getMsgExtractor().apply(xInputMsg).getValue(), "test-msg-1");
    assertEquals(((IntermediateStreamInternalImpl<String, MessageType, TestMessageEnvelope>) mIntermediateStream).getMsgExtractor().apply(xInputMsg).getEventTime(), 33333L);
    assertEquals(((IntermediateStreamInternalImpl<String, MessageType, TestMessageEnvelope>) mIntermediateStream).getMsgBuilder().apply("test-key-1", new MyMessageType("test-msg-1", 33333L, "test-output-id-1")).getKey(), "test-key-1");
    assertEquals(((IntermediateStreamInternalImpl<String, MessageType, TestMessageEnvelope>) mIntermediateStream).getMsgBuilder().apply("test-key-1", new MyMessageType("test-msg-1", 33333L, "test-output-id-1")).getMessage().getValue(), "test-msg-1");
    assertEquals(((IntermediateStreamInternalImpl<String, MessageType, TestMessageEnvelope>) mIntermediateStream).getMsgBuilder().apply("test-key-1", new MyMessageType("test-msg-1", 33333L, "test-output-id-1")).getMessage().getEventTime(), 33333L);
}

Example 3 with MessageType

use of org.apache.samza.operators.data.MessageType in project samza by apache.

the class TestOperatorSpecs method testCreateSinkOperator.

@Test
public void testCreateSinkOperator() {
    SystemStream testStream = new SystemStream("test-sys", "test-stream");
    SinkFunction<TestMessageEnvelope> sinkFn = (TestMessageEnvelope message, MessageCollector messageCollector, TaskCoordinator taskCoordinator) -> {
        messageCollector.send(new OutgoingMessageEnvelope(testStream, message.getKey(), message.getMessage()));
    };
    SinkOperatorSpec<TestMessageEnvelope> sinkOp = OperatorSpecs.createSinkOperatorSpec(sinkFn, 1);
    assertEquals(sinkOp.getSinkFn(), sinkFn);
    TestMessageEnvelope mockInput = mock(TestMessageEnvelope.class);
    when(mockInput.getKey()).thenReturn("my-test-msg-key");
    MessageType mockMsgBody = mock(MessageType.class);
    when(mockInput.getMessage()).thenReturn(mockMsgBody);
    final List<OutgoingMessageEnvelope> outputMsgs = new ArrayList<>();
    MessageCollector mockCollector = mock(MessageCollector.class);
    doAnswer(invocation -> {
        outputMsgs.add((OutgoingMessageEnvelope) invocation.getArguments()[0]);
        return null;
    }).when(mockCollector).send(any());
    sinkOp.getSinkFn().apply(mockInput, mockCollector, null);
    assertEquals(1, outputMsgs.size());
    assertEquals(outputMsgs.get(0).getKey(), "my-test-msg-key");
    assertEquals(outputMsgs.get(0).getMessage(), mockMsgBody);
    assertEquals(sinkOp.getOpCode(), OperatorSpec.OpCode.SINK);
    assertEquals(sinkOp.getNextStream(), null);
}

Example 4 with MessageType

use of org.apache.samza.operators.data.MessageType in project samza by apache.

the class TestStreamGraphImpl method testGetInputStream.

@Test
public void testGetInputStream() {
    ApplicationRunner mockRunner = mock(ApplicationRunner.class);
    Config mockConfig = mock(Config.class);
    StreamSpec testStreamSpec = new StreamSpec("test-stream-1", "physical-stream-1", "test-system");
    when(mockRunner.getStreamSpec("test-stream-1")).thenReturn(testStreamSpec);
    StreamGraphImpl graph = new StreamGraphImpl(mockRunner, mockConfig);
    BiFunction<String, MessageType, TestInputMessageEnvelope> xMsgBuilder = (k, v) -> new TestInputMessageEnvelope(k, v.getValue(), v.getEventTime(), "input-id-1");
    MessageStream<TestMessageEnvelope> mInputStream = graph.getInputStream("test-stream-1", xMsgBuilder);
    assertEquals(graph.getInputStreams().get(testStreamSpec), mInputStream);
    assertTrue(mInputStream instanceof InputStreamInternalImpl);
    assertEquals(((InputStreamInternalImpl) mInputStream).getMsgBuilder(), xMsgBuilder);
    String key = "test-input-key";
    MessageType msgBody = new MessageType("test-msg-value", 333333L);
    TestMessageEnvelope xInputMsg = ((InputStreamInternalImpl<String, MessageType, TestMessageEnvelope>) mInputStream).getMsgBuilder().apply(key, msgBody);
    assertEquals(xInputMsg.getKey(), key);
    assertEquals(xInputMsg.getMessage().getValue(), msgBody.getValue());
    assertEquals(xInputMsg.getMessage().getEventTime(), msgBody.getEventTime());
    assertEquals(((TestInputMessageEnvelope) xInputMsg).getInputId(), "input-id-1");
}

Example 5 with MessageType

use of org.apache.samza.operators.data.MessageType in project samza by apache.

the class TestMessageStreamImpl method testFilter.

@Test
public void testFilter() {
    MessageStreamImpl<TestMessageEnvelope> inputStream = new MessageStreamImpl<>(mockGraph);
    FilterFunction<TestMessageEnvelope> xFilter = (TestMessageEnvelope m) -> m.getMessage().getEventTime() > 123456L;
    MessageStream<TestMessageEnvelope> outputStream = inputStream.filter(xFilter);
    Collection<OperatorSpec> subs = inputStream.getRegisteredOperatorSpecs();
    assertEquals(subs.size(), 1);
    OperatorSpec<TestMessageEnvelope> filterOp = subs.iterator().next();
    assertTrue(filterOp instanceof StreamOperatorSpec);
    assertEquals(filterOp.getNextStream(), outputStream);
    // assert that the transformation function is what we defined above
    FlatMapFunction<TestMessageEnvelope, TestMessageEnvelope> txfmFn = ((StreamOperatorSpec<TestMessageEnvelope, TestMessageEnvelope>) filterOp).getTransformFn();
    TestMessageEnvelope mockMsg = mock(TestMessageEnvelope.class);
    MessageType mockInnerTestMessage = mock(MessageType.class);
    when(mockMsg.getMessage()).thenReturn(mockInnerTestMessage);
    when(mockInnerTestMessage.getEventTime()).thenReturn(11111L);
    Collection<TestMessageEnvelope> output = txfmFn.apply(mockMsg);
    assertTrue(output.isEmpty());
    when(mockMsg.getMessage()).thenReturn(mockInnerTestMessage);
    when(mockInnerTestMessage.getEventTime()).thenReturn(999999L);
    output = txfmFn.apply(mockMsg);
    assertEquals(output.size(), 1);
    assertEquals(output.iterator().next(), mockMsg);
}