Examples with FlatMapFunction org.apache.samza.operators.functions.FlatMapFunction used on opensource projects

Example 6 with FlatMapFunction

use of org.apache.samza.operators.functions.FlatMapFunction in project samza by apache.

the class TestOperatorSpec method testStreamOperatorSpecWithMap.

@Test
public void testStreamOperatorSpecWithMap() {
    MapFunction<TestMessageEnvelope, TestOutputMessageEnvelope> mapFn = m -> new TestOutputMessageEnvelope(m.getKey(), m.getMessage().hashCode());
    StreamOperatorSpec<TestMessageEnvelope, TestOutputMessageEnvelope> streamOperatorSpec = OperatorSpecs.createMapOperatorSpec(mapFn, "op0");
    StreamOperatorSpec<TestMessageEnvelope, TestOutputMessageEnvelope> cloneOperatorSpec = (StreamOperatorSpec<TestMessageEnvelope, TestOutputMessageEnvelope>) OperatorSpecTestUtils.copyOpSpec(streamOperatorSpec);
    assertNotEquals(streamOperatorSpec, cloneOperatorSpec);
    assertTrue(streamOperatorSpec.isClone(cloneOperatorSpec));
    MapFunction userFn = (MapFunction) Whitebox.getInternalState(streamOperatorSpec, "mapFn");
    assertEquals(userFn, mapFn);
    assertNotEquals(streamOperatorSpec.getTransformFn(), cloneOperatorSpec.getTransformFn());
    MapFunction clonedUserFn = (MapFunction) Whitebox.getInternalState(cloneOperatorSpec, "mapFn");
    assertTrue(cloneOperatorSpec.getTransformFn() instanceof FlatMapFunction);
    assertTrue(clonedUserFn instanceof MapFunction);
    assertNotEquals(userFn, clonedUserFn);
    assertNull(streamOperatorSpec.getWatermarkFn());
    assertNull(cloneOperatorSpec.getWatermarkFn());
    assertNull(streamOperatorSpec.getScheduledFn());
    assertNull(cloneOperatorSpec.getScheduledFn());
}

Example 7 with FlatMapFunction

use of org.apache.samza.operators.functions.FlatMapFunction in project samza by apache.

the class TestOperatorSpec method testStreamOperatorSpecWithFilter.

@Test
public void testStreamOperatorSpecWithFilter() {
    FilterFunction<TestMessageEnvelope> filterFn = m -> m.getKey().equals("key1");
    StreamOperatorSpec<TestMessageEnvelope, TestMessageEnvelope> streamOperatorSpec = OperatorSpecs.createFilterOperatorSpec(filterFn, "op0");
    StreamOperatorSpec<TestMessageEnvelope, TestOutputMessageEnvelope> cloneOperatorSpec = (StreamOperatorSpec<TestMessageEnvelope, TestOutputMessageEnvelope>) OperatorSpecTestUtils.copyOpSpec(streamOperatorSpec);
    assertNotEquals(streamOperatorSpec, cloneOperatorSpec);
    assertTrue(streamOperatorSpec.isClone(cloneOperatorSpec));
    FilterFunction userFn = (FilterFunction) Whitebox.getInternalState(streamOperatorSpec, "filterFn");
    assertEquals(userFn, filterFn);
    assertNotEquals(streamOperatorSpec.getTransformFn(), cloneOperatorSpec.getTransformFn());
    FilterFunction clonedUserFn = (FilterFunction) Whitebox.getInternalState(cloneOperatorSpec, "filterFn");
    assertTrue(cloneOperatorSpec.getTransformFn() instanceof FlatMapFunction);
    assertTrue(clonedUserFn instanceof FilterFunction);
    assertNotEquals(userFn, clonedUserFn);
    assertNull(streamOperatorSpec.getWatermarkFn());
    assertNull(cloneOperatorSpec.getWatermarkFn());
    assertNull(streamOperatorSpec.getScheduledFn());
    assertNull(cloneOperatorSpec.getScheduledFn());
}

Example 8 with FlatMapFunction

use of org.apache.samza.operators.functions.FlatMapFunction in project samza by apache.

the class TestOperatorImpls method testCreateOperator.

@Test
public void testCreateOperator() throws NoSuchFieldException, IllegalAccessException, InvocationTargetException {
    // get window operator
    WindowOperatorSpec mockWnd = mock(WindowOperatorSpec.class);
    WindowInternal<TestMessageEnvelope, String, Integer> windowInternal = new WindowInternal<>(null, null, null, null, null, WindowType.TUMBLING);
    when(mockWnd.getWindow()).thenReturn(windowInternal);
    Config mockConfig = mock(Config.class);
    TaskContext mockContext = mock(TaskContext.class);
    OperatorImplGraph opGraph = new OperatorImplGraph();
    OperatorImpl<TestMessageEnvelope, ?> opImpl = (OperatorImpl<TestMessageEnvelope, ?>) createOpMethod.invoke(opGraph, mockWnd, mockConfig, mockContext);
    assertTrue(opImpl instanceof WindowOperatorImpl);
    Field wndInternalField = WindowOperatorImpl.class.getDeclaredField("window");
    wndInternalField.setAccessible(true);
    WindowInternal wndInternal = (WindowInternal) wndInternalField.get(opImpl);
    assertEquals(wndInternal, windowInternal);
    // get simple operator
    StreamOperatorSpec<TestMessageEnvelope, TestOutputMessageEnvelope> mockSimpleOp = mock(StreamOperatorSpec.class);
    FlatMapFunction<TestMessageEnvelope, TestOutputMessageEnvelope> mockTxfmFn = mock(FlatMapFunction.class);
    when(mockSimpleOp.getTransformFn()).thenReturn(mockTxfmFn);
    opImpl = (OperatorImpl<TestMessageEnvelope, ?>) createOpMethod.invoke(opGraph, mockSimpleOp, mockConfig, mockContext);
    assertTrue(opImpl instanceof StreamOperatorImpl);
    Field txfmFnField = StreamOperatorImpl.class.getDeclaredField("transformFn");
    txfmFnField.setAccessible(true);
    assertEquals(mockTxfmFn, txfmFnField.get(opImpl));
    // get sink operator
    SinkFunction<TestMessageEnvelope> sinkFn = (m, mc, tc) -> {
    };
    SinkOperatorSpec<TestMessageEnvelope> sinkOp = mock(SinkOperatorSpec.class);
    when(sinkOp.getSinkFn()).thenReturn(sinkFn);
    opImpl = (OperatorImpl<TestMessageEnvelope, ?>) createOpMethod.invoke(opGraph, sinkOp, mockConfig, mockContext);
    assertTrue(opImpl instanceof SinkOperatorImpl);
    Field sinkFnField = SinkOperatorImpl.class.getDeclaredField("sinkFn");
    sinkFnField.setAccessible(true);
    assertEquals(sinkFn, sinkFnField.get(opImpl));
    // get join operator
    PartialJoinOperatorSpec<String, TestMessageEnvelope, TestMessageEnvelope, TestOutputMessageEnvelope> joinOp = mock(PartialJoinOperatorSpec.class);
    opImpl = (OperatorImpl<TestMessageEnvelope, ?>) createOpMethod.invoke(opGraph, joinOp, mockConfig, mockContext);
    assertTrue(opImpl instanceof PartialJoinOperatorImpl);
}

Example 9 with FlatMapFunction

use of org.apache.samza.operators.functions.FlatMapFunction in project samza by apache.

the class TestOperatorSpecs method testCreateStreamOperator.

@Test
public void testCreateStreamOperator() {
    FlatMapFunction<Object, TestMessageEnvelope> transformFn = m -> new ArrayList<TestMessageEnvelope>() {

        {
            this.add(new TestMessageEnvelope(m.toString(), m.toString(), 12345L));
        }
    };
    MessageStreamImpl<TestMessageEnvelope> mockOutput = mock(MessageStreamImpl.class);
    StreamOperatorSpec<Object, TestMessageEnvelope> streamOp = OperatorSpecs.createStreamOperatorSpec(transformFn, mockOutput, 1);
    assertEquals(streamOp.getTransformFn(), transformFn);
    Object mockInput = mock(Object.class);
    when(mockInput.toString()).thenReturn("test-string-1");
    List<TestMessageEnvelope> outputs = (List<TestMessageEnvelope>) streamOp.getTransformFn().apply(mockInput);
    assertEquals(outputs.size(), 1);
    assertEquals(outputs.get(0).getKey(), "test-string-1");
    assertEquals(outputs.get(0).getMessage().getValue(), "test-string-1");
    assertEquals(outputs.get(0).getMessage().getEventTime(), 12345L);
    assertEquals(streamOp.getNextStream(), mockOutput);
}

Example 10 with FlatMapFunction

use of org.apache.samza.operators.functions.FlatMapFunction in project samza by apache.

the class TestMessageStreamImpl method testMap.

@Test
public void testMap() {
    StreamApplicationDescriptorImpl mockGraph = mock(StreamApplicationDescriptorImpl.class);
    OperatorSpec mockOpSpec = mock(OperatorSpec.class);
    MessageStreamImpl<TestMessageEnvelope> inputStream = new MessageStreamImpl<>(mockGraph, mockOpSpec);
    MapFunction<TestMessageEnvelope, TestOutputMessageEnvelope> mockMapFn = mock(MapFunction.class);
    inputStream.map(mockMapFn);
    ArgumentCaptor<OperatorSpec> registeredOpCaptor = ArgumentCaptor.forClass(OperatorSpec.class);
    verify(mockOpSpec).registerNextOperatorSpec(registeredOpCaptor.capture());
    OperatorSpec<?, TestMessageEnvelope> registeredOpSpec = registeredOpCaptor.getValue();
    assertTrue(registeredOpSpec instanceof StreamOperatorSpec);
    FlatMapFunction transformFn = ((StreamOperatorSpec) registeredOpSpec).getTransformFn();
    assertNotNull(transformFn);
    assertEquals(OpCode.MAP, registeredOpSpec.getOpCode());
    TestOutputMessageEnvelope mockOutput = mock(TestOutputMessageEnvelope.class);
    when(mockMapFn.apply(anyObject())).thenReturn(mockOutput);
    assertTrue(transformFn.apply(new Object()).contains(mockOutput));
    when(mockMapFn.apply(anyObject())).thenReturn(null);
    assertTrue(transformFn.apply(null).isEmpty());
}