Examples with TaskCallback org.apache.samza.task.TaskCallback used on opensource projects

Example 11 with TaskCallback

use of org.apache.samza.task.TaskCallback in project samza by apache.

the class TestWindowOperator method testEndOfStreamFlushesWithNoTriggerFirings.

@Test
public void testEndOfStreamFlushesWithNoTriggerFirings() throws Exception {
    OperatorSpecGraph sgb = this.getKeyedSessionWindowStreamGraph(AccumulationMode.DISCARDING, Duration.ofMillis(500)).getOperatorSpecGraph();
    TestClock testClock = new TestClock();
    List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>();
    StreamOperatorTask task = new StreamOperatorTask(sgb, testClock);
    task.init(this.context);
    MessageCollector messageCollector = envelope -> windowPanes.add((WindowPane<Integer, Collection<IntegerEnvelope>>) envelope.getMessage());
    task.processAsync(new IntegerEnvelope(1), messageCollector, taskCoordinator, taskCallback);
    task.processAsync(new IntegerEnvelope(1), messageCollector, taskCoordinator, taskCallback);
    task.processAsync(new IntegerEnvelope(1), messageCollector, taskCoordinator, taskCallback);
    task.processAsync(new IntegerEnvelope(1), messageCollector, taskCoordinator, taskCallback);
    final IncomingMessageEnvelope endOfStream = IncomingMessageEnvelope.buildEndOfStreamEnvelope(new SystemStreamPartition("kafka", "integers", new Partition(0)));
    task.processAsync(endOfStream, messageCollector, taskCoordinator, taskCallback);
    Assert.assertEquals(windowPanes.size(), 1);
    Assert.assertEquals(windowPanes.get(0).getMessage().size(), 4);
    verify(taskCoordinator, times(1)).commit(TaskCoordinator.RequestScope.CURRENT_TASK);
    verify(taskCoordinator, times(1)).shutdown(TaskCoordinator.RequestScope.CURRENT_TASK);
}

Example 12 with TaskCallback

use of org.apache.samza.task.TaskCallback in project samza by apache.

the class TestJoinOperator method joinRetainsMatchedMessages.

@Test
public void joinRetainsMatchedMessages() throws Exception {
    StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction());
    StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc);
    List<Integer> output = new ArrayList<>();
    MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage());
    // push messages to first stream
    numbers.forEach(n -> sot.processAsync(new FirstStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));
    // push messages to second stream with same key
    numbers.forEach(n -> sot.processAsync(new SecondStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));
    int outputSum = output.stream().reduce(0, (s, m) -> s + m);
    assertEquals(110, outputSum);
    output.clear();
    // push messages to first stream with same keys once again.
    numbers.forEach(n -> sot.processAsync(new FirstStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));
    int newOutputSum = output.stream().reduce(0, (s, m) -> s + m);
    // should produce the same output as before
    assertEquals(110, newOutputSum);
}

Example 13 with TaskCallback

use of org.apache.samza.task.TaskCallback in project samza by apache.

the class TestJoinOperator method joinRetainsLatestMessageForKey.

@Test
public void joinRetainsLatestMessageForKey() throws Exception {
    StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction());
    StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc);
    List<Integer> output = new ArrayList<>();
    MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage());
    // push messages to first stream
    numbers.forEach(n -> sot.processAsync(new FirstStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));
    // push messages to first stream again with same keys but different values
    numbers.forEach(n -> sot.processAsync(new FirstStreamIME(n, 2 * n), messageCollector, taskCoordinator, taskCallback));
    // push messages to second stream with same key
    numbers.forEach(n -> sot.processAsync(new SecondStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));
    int outputSum = output.stream().reduce(0, (s, m) -> s + m);
    // should use latest messages in the first stream
    assertEquals(165, outputSum);
}

Example 14 with TaskCallback

use of org.apache.samza.task.TaskCallback in project samza by apache.

the class TestJoinOperator method joinReverse.

@Test
public void joinReverse() throws Exception {
    StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction());
    StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc);
    List<Integer> output = new ArrayList<>();
    MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage());
    // push messages to second stream
    numbers.forEach(n -> sot.processAsync(new SecondStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));
    // push messages to first stream with same keys
    numbers.forEach(n -> sot.processAsync(new FirstStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));
    int outputSum = output.stream().reduce(0, (s, m) -> s + m);
    assertEquals(110, outputSum);
}

Example 15 with TaskCallback

use of org.apache.samza.task.TaskCallback in project samza by apache.

the class TestJoinOperator method joinRetainsLatestMessageForKeyReverse.

@Test
public void joinRetainsLatestMessageForKeyReverse() throws Exception {
    StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction());
    StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc);
    List<Integer> output = new ArrayList<>();
    MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage());
    // push messages to second stream
    numbers.forEach(n -> sot.processAsync(new SecondStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));
    // push messages to second stream again with same keys but different values
    numbers.forEach(n -> sot.processAsync(new SecondStreamIME(n, 2 * n), messageCollector, taskCoordinator, taskCallback));
    // push messages to first stream with same key
    numbers.forEach(n -> sot.processAsync(new FirstStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));
    int outputSum = output.stream().reduce(0, (s, m) -> s + m);
    // should use latest messages in the second stream
    assertEquals(165, outputSum);
}