Search in sources :

Example 6 with TestOutputMessageEnvelope

use of org.apache.samza.operators.data.TestOutputMessageEnvelope in project samza by apache.

the class TestOperatorSpec method testStreamOperatorSpecWithMapWithFunctionReference.

@Test
public void testStreamOperatorSpecWithMapWithFunctionReference() {
    MapFunction<KV<String, Object>, Object> mapFn = KV::getValue;
    StreamOperatorSpec<KV<String, Object>, Object> streamOperatorSpec = OperatorSpecs.createMapOperatorSpec(mapFn, "op0");
    StreamOperatorSpec<TestMessageEnvelope, TestOutputMessageEnvelope> cloneOperatorSpec = (StreamOperatorSpec<TestMessageEnvelope, TestOutputMessageEnvelope>) OperatorSpecTestUtils.copyOpSpec(streamOperatorSpec);
    assertNotEquals(streamOperatorSpec, cloneOperatorSpec);
    assertTrue(streamOperatorSpec.isClone(cloneOperatorSpec));
    MapFunction userFn = (MapFunction) Whitebox.getInternalState(streamOperatorSpec, "mapFn");
    assertEquals(userFn, mapFn);
    assertNotEquals(streamOperatorSpec.getTransformFn(), cloneOperatorSpec.getTransformFn());
    MapFunction clonedUserFn = (MapFunction) Whitebox.getInternalState(cloneOperatorSpec, "mapFn");
    assertTrue(cloneOperatorSpec.getTransformFn() instanceof FlatMapFunction);
    assertTrue(clonedUserFn instanceof MapFunction);
    assertNotEquals(userFn, clonedUserFn);
}
Also used : TestMessageEnvelope(org.apache.samza.operators.data.TestMessageEnvelope) FlatMapFunction(org.apache.samza.operators.functions.FlatMapFunction) TestOutputMessageEnvelope(org.apache.samza.operators.data.TestOutputMessageEnvelope) KV(org.apache.samza.operators.KV) FlatMapFunction(org.apache.samza.operators.functions.FlatMapFunction) MapFunction(org.apache.samza.operators.functions.MapFunction) Test(org.junit.Test)

Example 7 with TestOutputMessageEnvelope

use of org.apache.samza.operators.data.TestOutputMessageEnvelope in project samza by apache.

the class TestOperatorSpec method testStreamOperatorSpecWithMapWithEnum.

@Test
public void testStreamOperatorSpecWithMapWithEnum() {
    MapFunction<TestMessageEnvelope, TestOutputMessageEnvelope> mapFn = new MapWithEnum(OperatorSpecTestUtils.TestEnum.One);
    StreamOperatorSpec<TestMessageEnvelope, TestOutputMessageEnvelope> streamOperatorSpec = OperatorSpecs.createMapOperatorSpec(mapFn, "op0");
    assertTrue(streamOperatorSpec instanceof MapOperatorSpec);
    StreamOperatorSpec<TestMessageEnvelope, TestOutputMessageEnvelope> cloneOperatorSpec = (StreamOperatorSpec<TestMessageEnvelope, TestOutputMessageEnvelope>) OperatorSpecTestUtils.copyOpSpec(streamOperatorSpec);
    assertNotEquals(streamOperatorSpec, cloneOperatorSpec);
    assertTrue(streamOperatorSpec.isClone(cloneOperatorSpec));
    MapFunction userFn = (MapFunction) Whitebox.getInternalState(streamOperatorSpec, "mapFn");
    assertEquals(userFn, mapFn);
    assertNotEquals(streamOperatorSpec.getTransformFn(), cloneOperatorSpec.getTransformFn());
    MapFunction clonedUserFn = (MapFunction) Whitebox.getInternalState(cloneOperatorSpec, "mapFn");
    assertTrue(cloneOperatorSpec.getTransformFn() instanceof FlatMapFunction);
    assertTrue(clonedUserFn instanceof MapWithEnum);
    assertNotEquals(userFn, clonedUserFn);
    // originally the types should be the same
    assertTrue(((MapWithEnum) userFn).getType() == ((MapWithEnum) clonedUserFn).getType());
    // after changing the type of the cloned user function, the types are different now
    ((MapWithEnum) clonedUserFn).setType(OperatorSpecTestUtils.TestEnum.Two);
    assertTrue(((MapWithEnum) userFn).getType() != ((MapWithEnum) clonedUserFn).getType());
}
Also used : TestMessageEnvelope(org.apache.samza.operators.data.TestMessageEnvelope) FlatMapFunction(org.apache.samza.operators.functions.FlatMapFunction) TestOutputMessageEnvelope(org.apache.samza.operators.data.TestOutputMessageEnvelope) FlatMapFunction(org.apache.samza.operators.functions.FlatMapFunction) MapFunction(org.apache.samza.operators.functions.MapFunction) Test(org.junit.Test)

Example 8 with TestOutputMessageEnvelope

use of org.apache.samza.operators.data.TestOutputMessageEnvelope in project samza by apache.

the class TestOperatorSpec method testStreamOperatorSpecWithMapAndListInClosure.

@Test
public void testStreamOperatorSpecWithMapAndListInClosure() {
    List<Integer> integers = new ArrayList<>(1);
    integers.add(0, 100);
    List<String> keys = new ArrayList<>(1);
    keys.add(0, "test-1");
    MapFunction<TestMessageEnvelope, TestOutputMessageEnvelope> mapFn = m -> new TestOutputMessageEnvelope(keys.get(m.getKey().hashCode() % 1), integers.get(m.getMessage().hashCode() % 1));
    StreamOperatorSpec<TestMessageEnvelope, TestOutputMessageEnvelope> streamOperatorSpec = OperatorSpecs.createMapOperatorSpec(mapFn, "op0");
    StreamOperatorSpec<TestMessageEnvelope, TestOutputMessageEnvelope> cloneOperatorSpec = (StreamOperatorSpec<TestMessageEnvelope, TestOutputMessageEnvelope>) OperatorSpecTestUtils.copyOpSpec(streamOperatorSpec);
    assertNotEquals(streamOperatorSpec, cloneOperatorSpec);
    assertTrue(streamOperatorSpec.isClone(cloneOperatorSpec));
    MapFunction userFn = (MapFunction) Whitebox.getInternalState(streamOperatorSpec, "mapFn");
    assertEquals(userFn, mapFn);
    assertNotEquals(streamOperatorSpec.getTransformFn(), cloneOperatorSpec.getTransformFn());
    MapFunction clonedUserFn = (MapFunction) Whitebox.getInternalState(cloneOperatorSpec, "mapFn");
    assertTrue(cloneOperatorSpec.getTransformFn() instanceof FlatMapFunction);
    assertTrue(clonedUserFn instanceof MapFunction);
    assertNotEquals(userFn, clonedUserFn);
    // verify changing the values in the original keys and integers list will change the result of the original map function
    TestMessageEnvelope mockImsg = new TestMessageEnvelope("input-key-x", new String("value-x"));
    assertEquals(((MapFunction) userFn).apply(mockImsg), new TestOutputMessageEnvelope("test-1", 100));
    integers.set(0, 200);
    keys.set(0, "test-2");
    assertEquals(((MapFunction) userFn).apply(mockImsg), new TestOutputMessageEnvelope("test-2", 200));
    // verify that the cloned map function uses a different copy of lists and still yields the same result
    assertEquals(((MapFunction) clonedUserFn).apply(mockImsg), new TestOutputMessageEnvelope("test-1", 100));
}
Also used : TestMessageEnvelope(org.apache.samza.operators.data.TestMessageEnvelope) FilterFunction(org.apache.samza.operators.functions.FilterFunction) ScheduledFunction(org.apache.samza.operators.functions.ScheduledFunction) StreamTableJoinFunction(org.apache.samza.operators.functions.StreamTableJoinFunction) Collection(java.util.Collection) Scheduler(org.apache.samza.operators.Scheduler) Serde(org.apache.samza.serializers.Serde) Test(org.junit.Test) JoinFunction(org.apache.samza.operators.functions.JoinFunction) FlatMapFunction(org.apache.samza.operators.functions.FlatMapFunction) ArrayList(java.util.ArrayList) MapFunction(org.apache.samza.operators.functions.MapFunction) WatermarkFunction(org.apache.samza.operators.functions.WatermarkFunction) StringSerde(org.apache.samza.serializers.StringSerde) List(java.util.List) Whitebox(org.mockito.internal.util.reflection.Whitebox) TestOutputMessageEnvelope(org.apache.samza.operators.data.TestOutputMessageEnvelope) SinkFunction(org.apache.samza.operators.functions.SinkFunction) JsonSerdeV2(org.apache.samza.serializers.JsonSerdeV2) Assert(org.junit.Assert) KV(org.apache.samza.operators.KV) NoOpSerde(org.apache.samza.serializers.NoOpSerde) ArrayList(java.util.ArrayList) FlatMapFunction(org.apache.samza.operators.functions.FlatMapFunction) MapFunction(org.apache.samza.operators.functions.MapFunction) TestMessageEnvelope(org.apache.samza.operators.data.TestMessageEnvelope) FlatMapFunction(org.apache.samza.operators.functions.FlatMapFunction) TestOutputMessageEnvelope(org.apache.samza.operators.data.TestOutputMessageEnvelope) Test(org.junit.Test)

Example 9 with TestOutputMessageEnvelope

use of org.apache.samza.operators.data.TestOutputMessageEnvelope in project samza by apache.

the class TestOperatorSpec method testStreamOperatorSpecWithMap.

@Test
public void testStreamOperatorSpecWithMap() {
    MapFunction<TestMessageEnvelope, TestOutputMessageEnvelope> mapFn = m -> new TestOutputMessageEnvelope(m.getKey(), m.getMessage().hashCode());
    StreamOperatorSpec<TestMessageEnvelope, TestOutputMessageEnvelope> streamOperatorSpec = OperatorSpecs.createMapOperatorSpec(mapFn, "op0");
    StreamOperatorSpec<TestMessageEnvelope, TestOutputMessageEnvelope> cloneOperatorSpec = (StreamOperatorSpec<TestMessageEnvelope, TestOutputMessageEnvelope>) OperatorSpecTestUtils.copyOpSpec(streamOperatorSpec);
    assertNotEquals(streamOperatorSpec, cloneOperatorSpec);
    assertTrue(streamOperatorSpec.isClone(cloneOperatorSpec));
    MapFunction userFn = (MapFunction) Whitebox.getInternalState(streamOperatorSpec, "mapFn");
    assertEquals(userFn, mapFn);
    assertNotEquals(streamOperatorSpec.getTransformFn(), cloneOperatorSpec.getTransformFn());
    MapFunction clonedUserFn = (MapFunction) Whitebox.getInternalState(cloneOperatorSpec, "mapFn");
    assertTrue(cloneOperatorSpec.getTransformFn() instanceof FlatMapFunction);
    assertTrue(clonedUserFn instanceof MapFunction);
    assertNotEquals(userFn, clonedUserFn);
    assertNull(streamOperatorSpec.getWatermarkFn());
    assertNull(cloneOperatorSpec.getWatermarkFn());
    assertNull(streamOperatorSpec.getScheduledFn());
    assertNull(cloneOperatorSpec.getScheduledFn());
}
Also used : TestMessageEnvelope(org.apache.samza.operators.data.TestMessageEnvelope) FilterFunction(org.apache.samza.operators.functions.FilterFunction) ScheduledFunction(org.apache.samza.operators.functions.ScheduledFunction) StreamTableJoinFunction(org.apache.samza.operators.functions.StreamTableJoinFunction) Collection(java.util.Collection) Scheduler(org.apache.samza.operators.Scheduler) Serde(org.apache.samza.serializers.Serde) Test(org.junit.Test) JoinFunction(org.apache.samza.operators.functions.JoinFunction) FlatMapFunction(org.apache.samza.operators.functions.FlatMapFunction) ArrayList(java.util.ArrayList) MapFunction(org.apache.samza.operators.functions.MapFunction) WatermarkFunction(org.apache.samza.operators.functions.WatermarkFunction) StringSerde(org.apache.samza.serializers.StringSerde) List(java.util.List) Whitebox(org.mockito.internal.util.reflection.Whitebox) TestOutputMessageEnvelope(org.apache.samza.operators.data.TestOutputMessageEnvelope) SinkFunction(org.apache.samza.operators.functions.SinkFunction) JsonSerdeV2(org.apache.samza.serializers.JsonSerdeV2) Assert(org.junit.Assert) KV(org.apache.samza.operators.KV) NoOpSerde(org.apache.samza.serializers.NoOpSerde) TestMessageEnvelope(org.apache.samza.operators.data.TestMessageEnvelope) FlatMapFunction(org.apache.samza.operators.functions.FlatMapFunction) TestOutputMessageEnvelope(org.apache.samza.operators.data.TestOutputMessageEnvelope) FlatMapFunction(org.apache.samza.operators.functions.FlatMapFunction) MapFunction(org.apache.samza.operators.functions.MapFunction) Test(org.junit.Test)

Example 10 with TestOutputMessageEnvelope

use of org.apache.samza.operators.data.TestOutputMessageEnvelope in project samza by apache.

the class TestOperatorSpec method testStreamOperatorSpecWithFilter.

@Test
public void testStreamOperatorSpecWithFilter() {
    FilterFunction<TestMessageEnvelope> filterFn = m -> m.getKey().equals("key1");
    StreamOperatorSpec<TestMessageEnvelope, TestMessageEnvelope> streamOperatorSpec = OperatorSpecs.createFilterOperatorSpec(filterFn, "op0");
    StreamOperatorSpec<TestMessageEnvelope, TestOutputMessageEnvelope> cloneOperatorSpec = (StreamOperatorSpec<TestMessageEnvelope, TestOutputMessageEnvelope>) OperatorSpecTestUtils.copyOpSpec(streamOperatorSpec);
    assertNotEquals(streamOperatorSpec, cloneOperatorSpec);
    assertTrue(streamOperatorSpec.isClone(cloneOperatorSpec));
    FilterFunction userFn = (FilterFunction) Whitebox.getInternalState(streamOperatorSpec, "filterFn");
    assertEquals(userFn, filterFn);
    assertNotEquals(streamOperatorSpec.getTransformFn(), cloneOperatorSpec.getTransformFn());
    FilterFunction clonedUserFn = (FilterFunction) Whitebox.getInternalState(cloneOperatorSpec, "filterFn");
    assertTrue(cloneOperatorSpec.getTransformFn() instanceof FlatMapFunction);
    assertTrue(clonedUserFn instanceof FilterFunction);
    assertNotEquals(userFn, clonedUserFn);
    assertNull(streamOperatorSpec.getWatermarkFn());
    assertNull(cloneOperatorSpec.getWatermarkFn());
    assertNull(streamOperatorSpec.getScheduledFn());
    assertNull(cloneOperatorSpec.getScheduledFn());
}
Also used : TestMessageEnvelope(org.apache.samza.operators.data.TestMessageEnvelope) FilterFunction(org.apache.samza.operators.functions.FilterFunction) ScheduledFunction(org.apache.samza.operators.functions.ScheduledFunction) StreamTableJoinFunction(org.apache.samza.operators.functions.StreamTableJoinFunction) Collection(java.util.Collection) Scheduler(org.apache.samza.operators.Scheduler) Serde(org.apache.samza.serializers.Serde) Test(org.junit.Test) JoinFunction(org.apache.samza.operators.functions.JoinFunction) FlatMapFunction(org.apache.samza.operators.functions.FlatMapFunction) ArrayList(java.util.ArrayList) MapFunction(org.apache.samza.operators.functions.MapFunction) WatermarkFunction(org.apache.samza.operators.functions.WatermarkFunction) StringSerde(org.apache.samza.serializers.StringSerde) List(java.util.List) Whitebox(org.mockito.internal.util.reflection.Whitebox) TestOutputMessageEnvelope(org.apache.samza.operators.data.TestOutputMessageEnvelope) SinkFunction(org.apache.samza.operators.functions.SinkFunction) JsonSerdeV2(org.apache.samza.serializers.JsonSerdeV2) Assert(org.junit.Assert) KV(org.apache.samza.operators.KV) NoOpSerde(org.apache.samza.serializers.NoOpSerde) FilterFunction(org.apache.samza.operators.functions.FilterFunction) TestMessageEnvelope(org.apache.samza.operators.data.TestMessageEnvelope) FlatMapFunction(org.apache.samza.operators.functions.FlatMapFunction) TestOutputMessageEnvelope(org.apache.samza.operators.data.TestOutputMessageEnvelope) Test(org.junit.Test)

Aggregations

TestOutputMessageEnvelope (org.apache.samza.operators.data.TestOutputMessageEnvelope)20 Test (org.junit.Test)19 TestMessageEnvelope (org.apache.samza.operators.data.TestMessageEnvelope)15 FlatMapFunction (org.apache.samza.operators.functions.FlatMapFunction)7 ArrayList (java.util.ArrayList)5 Config (org.apache.samza.config.Config)5 JoinFunction (org.apache.samza.operators.functions.JoinFunction)5 MapFunction (org.apache.samza.operators.functions.MapFunction)5 Serde (org.apache.samza.serializers.Serde)5 StringSerde (org.apache.samza.serializers.StringSerde)5 MessageCollector (org.apache.samza.task.MessageCollector)5 TaskContext (org.apache.samza.task.TaskContext)5 TaskCoordinator (org.apache.samza.task.TaskCoordinator)5 KV (org.apache.samza.operators.KV)4 SinkFunction (org.apache.samza.operators.functions.SinkFunction)4 OperatorSpec (org.apache.samza.operators.spec.OperatorSpec)4 SinkOperatorSpec (org.apache.samza.operators.spec.SinkOperatorSpec)4 StreamOperatorSpec (org.apache.samza.operators.spec.StreamOperatorSpec)4 WindowOperatorSpec (org.apache.samza.operators.spec.WindowOperatorSpec)4 NoOpSerde (org.apache.samza.serializers.NoOpSerde)4