Search in sources :

Example 6 with TestMessageEnvelope

use of org.apache.samza.operators.data.TestMessageEnvelope in project samza by apache.

the class TestOperatorSpecs method testCreateMergeOperator.

@Test
public void testCreateMergeOperator() {
    StreamGraphImpl mockGraph = mock(StreamGraphImpl.class);
    MessageStreamImpl<TestMessageEnvelope> output = TestMessageStreamImplUtil.<TestMessageEnvelope>getMessageStreamImpl(mockGraph);
    StreamOperatorSpec<TestMessageEnvelope, TestMessageEnvelope> mergeOp = OperatorSpecs.createMergeOperatorSpec(output, 1);
    Function<TestMessageEnvelope, Collection<TestMessageEnvelope>> mergeFn = t -> new ArrayList<TestMessageEnvelope>() {

        {
            this.add(t);
        }
    };
    TestMessageEnvelope t = mock(TestMessageEnvelope.class);
    assertEquals(mergeOp.getTransformFn().apply(t), mergeFn.apply(t));
    assertEquals(mergeOp.getNextStream(), output);
}
Also used : PartialJoinFunction(org.apache.samza.operators.functions.PartialJoinFunction) TestInputMessageEnvelope(org.apache.samza.operators.data.TestInputMessageEnvelope) Function(java.util.function.Function) Supplier(java.util.function.Supplier) WindowPane(org.apache.samza.operators.windows.WindowPane) ArrayList(java.util.ArrayList) OutputStreamInternalImpl(org.apache.samza.operators.stream.OutputStreamInternalImpl) MessageCollector(org.apache.samza.task.MessageCollector) MessageStreamImpl(org.apache.samza.operators.MessageStreamImpl) SystemStream(org.apache.samza.system.SystemStream) Mockito.doAnswer(org.mockito.Mockito.doAnswer) TestOutputMessageEnvelope(org.apache.samza.operators.data.TestOutputMessageEnvelope) TestMessageEnvelope(org.apache.samza.operators.data.TestMessageEnvelope) WindowType(org.apache.samza.operators.windows.internal.WindowType) MessageType(org.apache.samza.operators.data.MessageType) Assert.assertNotNull(org.junit.Assert.assertNotNull) Collection(java.util.Collection) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) FlatMapFunction(org.apache.samza.operators.functions.FlatMapFunction) TaskCoordinator(org.apache.samza.task.TaskCoordinator) StreamGraphImpl(org.apache.samza.operators.StreamGraphImpl) Matchers.any(org.mockito.Matchers.any) List(java.util.List) SinkFunction(org.apache.samza.operators.functions.SinkFunction) OutgoingMessageEnvelope(org.apache.samza.system.OutgoingMessageEnvelope) TestMessageStreamImplUtil(org.apache.samza.operators.TestMessageStreamImplUtil) WindowInternal(org.apache.samza.operators.windows.internal.WindowInternal) FoldLeftFunction(org.apache.samza.operators.functions.FoldLeftFunction) Assert.assertEquals(org.junit.Assert.assertEquals) Mockito.mock(org.mockito.Mockito.mock) TestMessageEnvelope(org.apache.samza.operators.data.TestMessageEnvelope) ArrayList(java.util.ArrayList) StreamGraphImpl(org.apache.samza.operators.StreamGraphImpl) Collection(java.util.Collection) Test(org.junit.Test)

Example 7 with TestMessageEnvelope

use of org.apache.samza.operators.data.TestMessageEnvelope in project samza by apache.

the class TestStreamGraphImpl method testGetOutputStream.

@Test
public void testGetOutputStream() {
    ApplicationRunner mockRunner = mock(ApplicationRunner.class);
    Config mockConfig = mock(Config.class);
    StreamSpec testStreamSpec = new StreamSpec("test-stream-1", "physical-stream-1", "test-system");
    when(mockRunner.getStreamSpec("test-stream-1")).thenReturn(testStreamSpec);
    class MyMessageType extends MessageType {

        public final String outputId;

        public MyMessageType(String value, long eventTime, String outputId) {
            super(value, eventTime);
            this.outputId = outputId;
        }
    }
    StreamGraphImpl graph = new StreamGraphImpl(mockRunner, mockConfig);
    Function<TestMessageEnvelope, String> xKeyExtractor = x -> x.getKey();
    Function<TestMessageEnvelope, MyMessageType> xMsgExtractor = x -> new MyMessageType(x.getMessage().getValue(), x.getMessage().getEventTime(), "test-output-id-1");
    OutputStream<String, MyMessageType, TestInputMessageEnvelope> mOutputStream = graph.getOutputStream("test-stream-1", xKeyExtractor, xMsgExtractor);
    assertEquals(graph.getOutputStreams().get(testStreamSpec), mOutputStream);
    assertTrue(mOutputStream instanceof OutputStreamInternalImpl);
    assertEquals(((OutputStreamInternalImpl) mOutputStream).getKeyExtractor(), xKeyExtractor);
    assertEquals(((OutputStreamInternalImpl) mOutputStream).getMsgExtractor(), xMsgExtractor);
    TestInputMessageEnvelope xInputMsg = new TestInputMessageEnvelope("test-key-1", "test-msg-1", 33333L, "input-id-1");
    assertEquals(((OutputStreamInternalImpl<String, MyMessageType, TestInputMessageEnvelope>) mOutputStream).getKeyExtractor().apply(xInputMsg), "test-key-1");
    assertEquals(((OutputStreamInternalImpl<String, MyMessageType, TestInputMessageEnvelope>) mOutputStream).getMsgExtractor().apply(xInputMsg).getValue(), "test-msg-1");
    assertEquals(((OutputStreamInternalImpl<String, MyMessageType, TestInputMessageEnvelope>) mOutputStream).getMsgExtractor().apply(xInputMsg).getEventTime(), 33333L);
    assertEquals(((OutputStreamInternalImpl<String, MyMessageType, TestInputMessageEnvelope>) mOutputStream).getMsgExtractor().apply(xInputMsg).outputId, "test-output-id-1");
}
Also used : TestMessageEnvelope(org.apache.samza.operators.data.TestMessageEnvelope) ApplicationRunner(org.apache.samza.runtime.ApplicationRunner) MessageType(org.apache.samza.operators.data.MessageType) BiFunction(java.util.function.BiFunction) JobConfig(org.apache.samza.config.JobConfig) TestInputMessageEnvelope(org.apache.samza.operators.data.TestInputMessageEnvelope) Assert.assertTrue(org.junit.Assert.assertTrue) StreamSpec(org.apache.samza.system.StreamSpec) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) Function(java.util.function.Function) OutputStreamInternalImpl(org.apache.samza.operators.stream.OutputStreamInternalImpl) Config(org.apache.samza.config.Config) IntermediateStreamInternalImpl(org.apache.samza.operators.stream.IntermediateStreamInternalImpl) InputStreamInternalImpl(org.apache.samza.operators.stream.InputStreamInternalImpl) Assert.assertEquals(org.junit.Assert.assertEquals) Mockito.mock(org.mockito.Mockito.mock) StreamSpec(org.apache.samza.system.StreamSpec) JobConfig(org.apache.samza.config.JobConfig) Config(org.apache.samza.config.Config) TestInputMessageEnvelope(org.apache.samza.operators.data.TestInputMessageEnvelope) OutputStreamInternalImpl(org.apache.samza.operators.stream.OutputStreamInternalImpl) TestMessageEnvelope(org.apache.samza.operators.data.TestMessageEnvelope) ApplicationRunner(org.apache.samza.runtime.ApplicationRunner) MessageType(org.apache.samza.operators.data.MessageType) Test(org.junit.Test)

Example 8 with TestMessageEnvelope

use of org.apache.samza.operators.data.TestMessageEnvelope in project samza by apache.

the class TestStreamGraphImpl method testGetIntermediateStream.

@Test
public void testGetIntermediateStream() {
    ApplicationRunner mockRunner = mock(ApplicationRunner.class);
    Config mockConfig = mock(Config.class);
    StreamSpec testStreamSpec = new StreamSpec("myJob-i001-test-stream-1", "physical-stream-1", "test-system");
    when(mockRunner.getStreamSpec("myJob-i001-test-stream-1")).thenReturn(testStreamSpec);
    when(mockConfig.get(JobConfig.JOB_NAME())).thenReturn("myJob");
    when(mockConfig.get(JobConfig.JOB_ID(), "1")).thenReturn("i001");
    class MyMessageType extends MessageType {

        public final String outputId;

        public MyMessageType(String value, long eventTime, String outputId) {
            super(value, eventTime);
            this.outputId = outputId;
        }
    }
    StreamGraphImpl graph = new StreamGraphImpl(mockRunner, mockConfig);
    Function<TestMessageEnvelope, String> xKeyExtractor = x -> x.getKey();
    Function<TestMessageEnvelope, MyMessageType> xMsgExtractor = x -> new MyMessageType(x.getMessage().getValue(), x.getMessage().getEventTime(), "test-output-id-1");
    BiFunction<String, MessageType, TestInputMessageEnvelope> xMsgBuilder = (k, v) -> new TestInputMessageEnvelope(k, v.getValue(), v.getEventTime(), "input-id-1");
    MessageStream<TestMessageEnvelope> mIntermediateStream = graph.getIntermediateStream("test-stream-1", xKeyExtractor, xMsgExtractor, xMsgBuilder);
    assertEquals(graph.getOutputStreams().get(testStreamSpec), mIntermediateStream);
    assertTrue(mIntermediateStream instanceof IntermediateStreamInternalImpl);
    assertEquals(((IntermediateStreamInternalImpl) mIntermediateStream).getKeyExtractor(), xKeyExtractor);
    assertEquals(((IntermediateStreamInternalImpl) mIntermediateStream).getMsgExtractor(), xMsgExtractor);
    assertEquals(((IntermediateStreamInternalImpl) mIntermediateStream).getMsgBuilder(), xMsgBuilder);
    TestMessageEnvelope xInputMsg = new TestMessageEnvelope("test-key-1", "test-msg-1", 33333L);
    assertEquals(((IntermediateStreamInternalImpl<String, MessageType, TestMessageEnvelope>) mIntermediateStream).getKeyExtractor().apply(xInputMsg), "test-key-1");
    assertEquals(((IntermediateStreamInternalImpl<String, MessageType, TestMessageEnvelope>) mIntermediateStream).getMsgExtractor().apply(xInputMsg).getValue(), "test-msg-1");
    assertEquals(((IntermediateStreamInternalImpl<String, MessageType, TestMessageEnvelope>) mIntermediateStream).getMsgExtractor().apply(xInputMsg).getEventTime(), 33333L);
    assertEquals(((IntermediateStreamInternalImpl<String, MessageType, TestMessageEnvelope>) mIntermediateStream).getMsgBuilder().apply("test-key-1", new MyMessageType("test-msg-1", 33333L, "test-output-id-1")).getKey(), "test-key-1");
    assertEquals(((IntermediateStreamInternalImpl<String, MessageType, TestMessageEnvelope>) mIntermediateStream).getMsgBuilder().apply("test-key-1", new MyMessageType("test-msg-1", 33333L, "test-output-id-1")).getMessage().getValue(), "test-msg-1");
    assertEquals(((IntermediateStreamInternalImpl<String, MessageType, TestMessageEnvelope>) mIntermediateStream).getMsgBuilder().apply("test-key-1", new MyMessageType("test-msg-1", 33333L, "test-output-id-1")).getMessage().getEventTime(), 33333L);
}
Also used : TestMessageEnvelope(org.apache.samza.operators.data.TestMessageEnvelope) ApplicationRunner(org.apache.samza.runtime.ApplicationRunner) MessageType(org.apache.samza.operators.data.MessageType) BiFunction(java.util.function.BiFunction) JobConfig(org.apache.samza.config.JobConfig) TestInputMessageEnvelope(org.apache.samza.operators.data.TestInputMessageEnvelope) Assert.assertTrue(org.junit.Assert.assertTrue) StreamSpec(org.apache.samza.system.StreamSpec) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) Function(java.util.function.Function) OutputStreamInternalImpl(org.apache.samza.operators.stream.OutputStreamInternalImpl) Config(org.apache.samza.config.Config) IntermediateStreamInternalImpl(org.apache.samza.operators.stream.IntermediateStreamInternalImpl) InputStreamInternalImpl(org.apache.samza.operators.stream.InputStreamInternalImpl) Assert.assertEquals(org.junit.Assert.assertEquals) Mockito.mock(org.mockito.Mockito.mock) StreamSpec(org.apache.samza.system.StreamSpec) IntermediateStreamInternalImpl(org.apache.samza.operators.stream.IntermediateStreamInternalImpl) JobConfig(org.apache.samza.config.JobConfig) Config(org.apache.samza.config.Config) TestInputMessageEnvelope(org.apache.samza.operators.data.TestInputMessageEnvelope) TestMessageEnvelope(org.apache.samza.operators.data.TestMessageEnvelope) ApplicationRunner(org.apache.samza.runtime.ApplicationRunner) MessageType(org.apache.samza.operators.data.MessageType) Test(org.junit.Test)

Example 9 with TestMessageEnvelope

use of org.apache.samza.operators.data.TestMessageEnvelope in project samza by apache.

the class TestOperatorSpecs method testCreateWindowOperatorWithRelaxedTypes.

@Test
public void testCreateWindowOperatorWithRelaxedTypes() throws Exception {
    Function<TestMessageEnvelope, String> keyExtractor = m -> m.getKey();
    FoldLeftFunction<TestMessageEnvelope, Integer> aggregator = (m, c) -> c + 1;
    Supplier<Integer> initialValue = () -> 0;
    //instantiate a window using reflection
    WindowInternal<TestInputMessageEnvelope, String, Integer> window = new WindowInternal(null, initialValue, aggregator, keyExtractor, null, WindowType.TUMBLING);
    MessageStreamImpl<WindowPane<String, Integer>> mockWndOut = mock(MessageStreamImpl.class);
    WindowOperatorSpec spec = OperatorSpecs.createWindowOperatorSpec(window, mockWndOut, 1);
    assertEquals(spec.getWindow(), window);
    assertEquals(spec.getWindow().getKeyExtractor(), keyExtractor);
    assertEquals(spec.getWindow().getFoldLeftFunction(), aggregator);
    // make sure that the functions with relaxed types work as expected
    TestInputMessageEnvelope inputMsg = new TestInputMessageEnvelope("test-input-key1", "test-value-1", 23456L, "input-id-1");
    assertEquals("test-input-key1", spec.getWindow().getKeyExtractor().apply(inputMsg));
    assertEquals(1, spec.getWindow().getFoldLeftFunction().apply(inputMsg, 0));
}
Also used : PartialJoinFunction(org.apache.samza.operators.functions.PartialJoinFunction) TestInputMessageEnvelope(org.apache.samza.operators.data.TestInputMessageEnvelope) Function(java.util.function.Function) Supplier(java.util.function.Supplier) WindowPane(org.apache.samza.operators.windows.WindowPane) ArrayList(java.util.ArrayList) OutputStreamInternalImpl(org.apache.samza.operators.stream.OutputStreamInternalImpl) MessageCollector(org.apache.samza.task.MessageCollector) MessageStreamImpl(org.apache.samza.operators.MessageStreamImpl) SystemStream(org.apache.samza.system.SystemStream) Mockito.doAnswer(org.mockito.Mockito.doAnswer) TestOutputMessageEnvelope(org.apache.samza.operators.data.TestOutputMessageEnvelope) TestMessageEnvelope(org.apache.samza.operators.data.TestMessageEnvelope) WindowType(org.apache.samza.operators.windows.internal.WindowType) MessageType(org.apache.samza.operators.data.MessageType) Assert.assertNotNull(org.junit.Assert.assertNotNull) Collection(java.util.Collection) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) FlatMapFunction(org.apache.samza.operators.functions.FlatMapFunction) TaskCoordinator(org.apache.samza.task.TaskCoordinator) StreamGraphImpl(org.apache.samza.operators.StreamGraphImpl) Matchers.any(org.mockito.Matchers.any) List(java.util.List) SinkFunction(org.apache.samza.operators.functions.SinkFunction) OutgoingMessageEnvelope(org.apache.samza.system.OutgoingMessageEnvelope) TestMessageStreamImplUtil(org.apache.samza.operators.TestMessageStreamImplUtil) WindowInternal(org.apache.samza.operators.windows.internal.WindowInternal) FoldLeftFunction(org.apache.samza.operators.functions.FoldLeftFunction) Assert.assertEquals(org.junit.Assert.assertEquals) Mockito.mock(org.mockito.Mockito.mock) WindowInternal(org.apache.samza.operators.windows.internal.WindowInternal) TestInputMessageEnvelope(org.apache.samza.operators.data.TestInputMessageEnvelope) WindowPane(org.apache.samza.operators.windows.WindowPane) TestMessageEnvelope(org.apache.samza.operators.data.TestMessageEnvelope) Test(org.junit.Test)

Example 10 with TestMessageEnvelope

use of org.apache.samza.operators.data.TestMessageEnvelope in project samza by apache.

the class TestOperatorSpecs method testCreateWindowOperator.

@Test
public void testCreateWindowOperator() throws Exception {
    Function<TestMessageEnvelope, String> keyExtractor = m -> "globalkey";
    FoldLeftFunction<TestMessageEnvelope, Integer> aggregator = (m, c) -> c + 1;
    Supplier<Integer> initialValue = () -> 0;
    //instantiate a window using reflection
    WindowInternal window = new WindowInternal(null, initialValue, aggregator, keyExtractor, null, WindowType.TUMBLING);
    MessageStreamImpl<WindowPane<String, Integer>> mockWndOut = mock(MessageStreamImpl.class);
    WindowOperatorSpec spec = OperatorSpecs.<TestMessageEnvelope, String, Integer>createWindowOperatorSpec(window, mockWndOut, 1);
    assertEquals(spec.getWindow(), window);
    assertEquals(spec.getWindow().getKeyExtractor(), keyExtractor);
    assertEquals(spec.getWindow().getFoldLeftFunction(), aggregator);
}
Also used : PartialJoinFunction(org.apache.samza.operators.functions.PartialJoinFunction) TestInputMessageEnvelope(org.apache.samza.operators.data.TestInputMessageEnvelope) Function(java.util.function.Function) Supplier(java.util.function.Supplier) WindowPane(org.apache.samza.operators.windows.WindowPane) ArrayList(java.util.ArrayList) OutputStreamInternalImpl(org.apache.samza.operators.stream.OutputStreamInternalImpl) MessageCollector(org.apache.samza.task.MessageCollector) MessageStreamImpl(org.apache.samza.operators.MessageStreamImpl) SystemStream(org.apache.samza.system.SystemStream) Mockito.doAnswer(org.mockito.Mockito.doAnswer) TestOutputMessageEnvelope(org.apache.samza.operators.data.TestOutputMessageEnvelope) TestMessageEnvelope(org.apache.samza.operators.data.TestMessageEnvelope) WindowType(org.apache.samza.operators.windows.internal.WindowType) MessageType(org.apache.samza.operators.data.MessageType) Assert.assertNotNull(org.junit.Assert.assertNotNull) Collection(java.util.Collection) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) FlatMapFunction(org.apache.samza.operators.functions.FlatMapFunction) TaskCoordinator(org.apache.samza.task.TaskCoordinator) StreamGraphImpl(org.apache.samza.operators.StreamGraphImpl) Matchers.any(org.mockito.Matchers.any) List(java.util.List) SinkFunction(org.apache.samza.operators.functions.SinkFunction) OutgoingMessageEnvelope(org.apache.samza.system.OutgoingMessageEnvelope) TestMessageStreamImplUtil(org.apache.samza.operators.TestMessageStreamImplUtil) WindowInternal(org.apache.samza.operators.windows.internal.WindowInternal) FoldLeftFunction(org.apache.samza.operators.functions.FoldLeftFunction) Assert.assertEquals(org.junit.Assert.assertEquals) Mockito.mock(org.mockito.Mockito.mock) TestMessageEnvelope(org.apache.samza.operators.data.TestMessageEnvelope) WindowInternal(org.apache.samza.operators.windows.internal.WindowInternal) WindowPane(org.apache.samza.operators.windows.WindowPane) Test(org.junit.Test)

Aggregations

TestMessageEnvelope (org.apache.samza.operators.data.TestMessageEnvelope)24 Test (org.junit.Test)23 TestOutputMessageEnvelope (org.apache.samza.operators.data.TestOutputMessageEnvelope)16 ArrayList (java.util.ArrayList)13 Config (org.apache.samza.config.Config)12 MessageType (org.apache.samza.operators.data.MessageType)12 OperatorSpec (org.apache.samza.operators.spec.OperatorSpec)11 PartialJoinOperatorSpec (org.apache.samza.operators.spec.PartialJoinOperatorSpec)11 SinkOperatorSpec (org.apache.samza.operators.spec.SinkOperatorSpec)11 StreamOperatorSpec (org.apache.samza.operators.spec.StreamOperatorSpec)11 Assert.assertEquals (org.junit.Assert.assertEquals)11 Mockito.mock (org.mockito.Mockito.mock)11 Mockito.when (org.mockito.Mockito.when)11 StreamGraphImpl (org.apache.samza.operators.StreamGraphImpl)9 TestInputMessageEnvelope (org.apache.samza.operators.data.TestInputMessageEnvelope)9 FlatMapFunction (org.apache.samza.operators.functions.FlatMapFunction)9 TaskContext (org.apache.samza.task.TaskContext)9 Function (java.util.function.Function)8 SinkFunction (org.apache.samza.operators.functions.SinkFunction)8 List (java.util.List)7