use of org.apache.samza.testUtils.TestClock in project samza by apache.
the class TestJoinOperator method joinRemovesExpiredMessagesReverse.
@Test
public void joinRemovesExpiredMessagesReverse() throws Exception {
TestClock testClock = new TestClock();
StreamOperatorTask sot = createStreamOperatorTask(testClock, new TestJoinStreamApplication(new TestJoinFunction()));
List<Integer> output = new ArrayList<>();
MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage());
// push messages to second stream
numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator));
// 1 minute after ttl
testClock.advanceTime(JOIN_TTL.plus(Duration.ofMinutes(1)));
// should expire second stream messages
sot.window(messageCollector, taskCoordinator);
// push messages to first stream with same key
numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator));
assertTrue(output.isEmpty());
}
use of org.apache.samza.testUtils.TestClock in project samza by apache.
the class TestJoinOperator method joinRemovesExpiredMessages.
@Test
public void joinRemovesExpiredMessages() throws Exception {
TestClock testClock = new TestClock();
StreamOperatorTask sot = createStreamOperatorTask(testClock, new TestJoinStreamApplication(new TestJoinFunction()));
List<Integer> output = new ArrayList<>();
MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage());
// push messages to first stream
numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator));
// 1 minute after ttl
testClock.advanceTime(JOIN_TTL.plus(Duration.ofMinutes(1)));
// should expire first stream messages
sot.window(messageCollector, taskCoordinator);
// push messages to second stream with same key
numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator));
assertTrue(output.isEmpty());
}
use of org.apache.samza.testUtils.TestClock in project samza by apache.
the class TestWindowOperator method testTumblingWindowsAccumulatingMode.
@Test
public void testTumblingWindowsAccumulatingMode() throws Exception {
StreamApplication sgb = new KeyedTumblingWindowStreamApplication(AccumulationMode.ACCUMULATING, Duration.ofSeconds(1), Triggers.repeat(Triggers.count(2)));
List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>();
TestClock testClock = new TestClock();
StreamOperatorTask task = new StreamOperatorTask(sgb, runner, testClock);
task.init(config, taskContext);
MessageCollector messageCollector = envelope -> windowPanes.add((WindowPane<Integer, Collection<IntegerEnvelope>>) envelope.getMessage());
integers.forEach(n -> task.process(new IntegerEnvelope(n), messageCollector, taskCoordinator));
testClock.advanceTime(Duration.ofSeconds(1));
task.window(messageCollector, taskCoordinator);
Assert.assertEquals(windowPanes.size(), 7);
Assert.assertEquals(windowPanes.get(0).getKey().getKey(), new Integer(1));
Assert.assertEquals((windowPanes.get(0).getMessage()).size(), 2);
Assert.assertEquals(windowPanes.get(1).getKey().getKey(), new Integer(2));
Assert.assertEquals((windowPanes.get(1).getMessage()).size(), 2);
Assert.assertEquals(windowPanes.get(2).getKey().getKey(), new Integer(1));
Assert.assertEquals((windowPanes.get(2).getMessage()).size(), 4);
Assert.assertEquals(windowPanes.get(3).getKey().getKey(), new Integer(2));
Assert.assertEquals((windowPanes.get(3).getMessage()).size(), 4);
}
use of org.apache.samza.testUtils.TestClock in project samza by apache.
the class TestWindowOperator method testTumblingWindowsDiscardingMode.
@Test
public void testTumblingWindowsDiscardingMode() throws Exception {
StreamApplication sgb = new KeyedTumblingWindowStreamApplication(AccumulationMode.DISCARDING, Duration.ofSeconds(1), Triggers.repeat(Triggers.count(2)));
List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>();
TestClock testClock = new TestClock();
StreamOperatorTask task = new StreamOperatorTask(sgb, runner, testClock);
task.init(config, taskContext);
MessageCollector messageCollector = envelope -> windowPanes.add((WindowPane<Integer, Collection<IntegerEnvelope>>) envelope.getMessage());
integers.forEach(n -> task.process(new IntegerEnvelope(n), messageCollector, taskCoordinator));
testClock.advanceTime(Duration.ofSeconds(1));
task.window(messageCollector, taskCoordinator);
Assert.assertEquals(windowPanes.size(), 5);
Assert.assertEquals(windowPanes.get(0).getKey().getKey(), new Integer(1));
Assert.assertEquals((windowPanes.get(0).getMessage()).size(), 2);
Assert.assertEquals(windowPanes.get(1).getKey().getKey(), new Integer(2));
Assert.assertEquals((windowPanes.get(1).getMessage()).size(), 2);
Assert.assertEquals(windowPanes.get(2).getKey().getKey(), new Integer(1));
Assert.assertEquals((windowPanes.get(2).getMessage()).size(), 2);
Assert.assertEquals(windowPanes.get(3).getKey().getKey(), new Integer(2));
Assert.assertEquals((windowPanes.get(3).getMessage()).size(), 2);
Assert.assertEquals(windowPanes.get(4).getKey().getKey(), new Integer(3));
Assert.assertEquals((windowPanes.get(4).getMessage()).size(), 1);
}
use of org.apache.samza.testUtils.TestClock in project samza by apache.
the class TestWindowOperator method testSessionWindowsDiscardingMode.
@Test
public void testSessionWindowsDiscardingMode() throws Exception {
StreamApplication sgb = new KeyedSessionWindowStreamApplication(AccumulationMode.DISCARDING, Duration.ofMillis(500));
TestClock testClock = new TestClock();
List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>();
StreamOperatorTask task = new StreamOperatorTask(sgb, runner, testClock);
task.init(config, taskContext);
MessageCollector messageCollector = envelope -> windowPanes.add((WindowPane<Integer, Collection<IntegerEnvelope>>) envelope.getMessage());
task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator);
task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator);
testClock.advanceTime(Duration.ofSeconds(1));
task.window(messageCollector, taskCoordinator);
Assert.assertEquals(windowPanes.size(), 1);
Assert.assertEquals(windowPanes.get(0).getKey().getPaneId(), "1");
Assert.assertEquals(windowPanes.get(0).getKey().getKey(), new Integer(1));
task.process(new IntegerEnvelope(2), messageCollector, taskCoordinator);
task.process(new IntegerEnvelope(2), messageCollector, taskCoordinator);
task.process(new IntegerEnvelope(3), messageCollector, taskCoordinator);
task.process(new IntegerEnvelope(3), messageCollector, taskCoordinator);
testClock.advanceTime(Duration.ofSeconds(1));
task.window(messageCollector, taskCoordinator);
Assert.assertEquals(windowPanes.size(), 3);
Assert.assertEquals(windowPanes.get(0).getKey().getPaneId(), "1");
Assert.assertEquals(windowPanes.get(1).getKey().getPaneId(), "1001");
Assert.assertEquals(windowPanes.get(2).getKey().getPaneId(), "1001");
Assert.assertEquals((windowPanes.get(0).getMessage()).size(), 2);
Assert.assertEquals((windowPanes.get(1).getMessage()).size(), 2);
Assert.assertEquals((windowPanes.get(2).getMessage()).size(), 2);
task.process(new IntegerEnvelope(2), messageCollector, taskCoordinator);
task.process(new IntegerEnvelope(2), messageCollector, taskCoordinator);
testClock.advanceTime(Duration.ofSeconds(1));
task.window(messageCollector, taskCoordinator);
Assert.assertEquals(windowPanes.size(), 4);
Assert.assertEquals(windowPanes.get(3).getKey().getKey(), new Integer(2));
Assert.assertEquals(windowPanes.get(3).getKey().getPaneId(), "2001");
Assert.assertEquals((windowPanes.get(3).getMessage()).size(), 2);
}
Aggregations