Examples with TaskCoordinator org.apache.samza.task.TaskCoordinator used on opensource projects

Example 6 with TaskCoordinator

use of org.apache.samza.task.TaskCoordinator in project samza by apache.

the class TestJoinOperator method joinRetainsLatestMessageForKeyReverse.

@Test
public void joinRetainsLatestMessageForKeyReverse() throws Exception {
    StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), new TestJoinStreamApplication(new TestJoinFunction()));
    List<Integer> output = new ArrayList<>();
    MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage());
    // push messages to second stream
    numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator));
    // push messages to second stream again with same keys but different values
    numbers.forEach(n -> sot.process(new SecondStreamIME(n, 2 * n), messageCollector, taskCoordinator));
    // push messages to first stream with same key
    numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator));
    int outputSum = output.stream().reduce(0, (s, m) -> s + m);
    // should use latest messages in the second stream
    assertEquals(165, outputSum);
}

Example 7 with TaskCoordinator

use of org.apache.samza.task.TaskCoordinator in project samza by apache.

the class TestJoinOperator method joinReverse.

@Test
public void joinReverse() throws Exception {
    StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), new TestJoinStreamApplication(new TestJoinFunction()));
    List<Integer> output = new ArrayList<>();
    MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage());
    // push messages to second stream
    numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator));
    // push messages to first stream with same keys
    numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator));
    int outputSum = output.stream().reduce(0, (s, m) -> s + m);
    assertEquals(110, outputSum);
}

Example 8 with TaskCoordinator

use of org.apache.samza.task.TaskCoordinator in project samza by apache.

the class TestOperatorImpl method testOnTimerUpdatesMetrics.

@Test
public void testOnTimerUpdatesMetrics() {
    TaskContext mockTaskContext = mock(TaskContext.class);
    MetricsRegistry mockMetricsRegistry = mock(MetricsRegistry.class);
    when(mockTaskContext.getMetricsRegistry()).thenReturn(mockMetricsRegistry);
    Counter mockMessageCounter = mock(Counter.class);
    Timer mockTimer = mock(Timer.class);
    when(mockMetricsRegistry.newCounter(anyString(), anyString())).thenReturn(mockMessageCounter);
    when(mockMetricsRegistry.newTimer(anyString(), anyString())).thenReturn(mockTimer);
    Object mockTestOpImplOutput = mock(Object.class);
    OperatorImpl<Object, Object> opImpl = new TestOpImpl(mockTestOpImplOutput);
    opImpl.init(mock(Config.class), mockTaskContext);
    // send a message to this operator
    MessageCollector mockCollector = mock(MessageCollector.class);
    TaskCoordinator mockCoordinator = mock(TaskCoordinator.class);
    opImpl.onTimer(mockCollector, mockCoordinator);
    // verify that it updates metrics
    verify(mockMessageCounter, times(0)).inc();
    verify(mockTimer, times(1)).update(anyLong());
}

Example 9 with TaskCoordinator

use of org.apache.samza.task.TaskCoordinator in project samza by apache.

the class TestStreamOperatorImpl method testSimpleOperator.

@Test
@SuppressWarnings("unchecked")
public void testSimpleOperator() {
    StreamOperatorSpec<TestMessageEnvelope, TestOutputMessageEnvelope> mockOp = mock(StreamOperatorSpec.class);
    FlatMapFunction<TestMessageEnvelope, TestOutputMessageEnvelope> txfmFn = mock(FlatMapFunction.class);
    when(mockOp.getTransformFn()).thenReturn(txfmFn);
    Config mockConfig = mock(Config.class);
    TaskContext mockContext = mock(TaskContext.class);
    StreamOperatorImpl<TestMessageEnvelope, TestOutputMessageEnvelope> opImpl = spy(new StreamOperatorImpl<>(mockOp, mockConfig, mockContext));
    TestMessageEnvelope inMsg = mock(TestMessageEnvelope.class);
    TestOutputMessageEnvelope outMsg = mock(TestOutputMessageEnvelope.class);
    Collection<TestOutputMessageEnvelope> mockOutputs = new ArrayList() {

        {
            this.add(outMsg);
        }
    };
    when(txfmFn.apply(inMsg)).thenReturn(mockOutputs);
    MessageCollector mockCollector = mock(MessageCollector.class);
    TaskCoordinator mockCoordinator = mock(TaskCoordinator.class);
    Collection<TestOutputMessageEnvelope> results = opImpl.handleMessage(inMsg, mockCollector, mockCoordinator);
    verify(txfmFn, times(1)).apply(inMsg);
    assertEquals(results, mockOutputs);
}

Example 10 with TaskCoordinator

use of org.apache.samza.task.TaskCoordinator in project samza by apache.

the class TestWindowOperator method testTumblingWindowsAccumulatingMode.

@Test
public void testTumblingWindowsAccumulatingMode() throws Exception {
    StreamApplication sgb = new KeyedTumblingWindowStreamApplication(AccumulationMode.ACCUMULATING, Duration.ofSeconds(1), Triggers.repeat(Triggers.count(2)));
    List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>();
    TestClock testClock = new TestClock();
    StreamOperatorTask task = new StreamOperatorTask(sgb, runner, testClock);
    task.init(config, taskContext);
    MessageCollector messageCollector = envelope -> windowPanes.add((WindowPane<Integer, Collection<IntegerEnvelope>>) envelope.getMessage());
    integers.forEach(n -> task.process(new IntegerEnvelope(n), messageCollector, taskCoordinator));
    testClock.advanceTime(Duration.ofSeconds(1));
    task.window(messageCollector, taskCoordinator);
    Assert.assertEquals(windowPanes.size(), 7);
    Assert.assertEquals(windowPanes.get(0).getKey().getKey(), new Integer(1));
    Assert.assertEquals((windowPanes.get(0).getMessage()).size(), 2);
    Assert.assertEquals(windowPanes.get(1).getKey().getKey(), new Integer(2));
    Assert.assertEquals((windowPanes.get(1).getMessage()).size(), 2);
    Assert.assertEquals(windowPanes.get(2).getKey().getKey(), new Integer(1));
    Assert.assertEquals((windowPanes.get(2).getMessage()).size(), 4);
    Assert.assertEquals(windowPanes.get(3).getKey().getKey(), new Integer(2));
    Assert.assertEquals((windowPanes.get(3).getMessage()).size(), 4);
}