use of org.apache.samza.operators.windows.WindowPane in project samza by apache.
the class TestWindowOperator method testCancelationOfRepeatingNestedTriggers.
@Test
public void testCancelationOfRepeatingNestedTriggers() throws Exception {
OperatorSpecGraph sgb = this.getKeyedTumblingWindowStreamGraph(AccumulationMode.ACCUMULATING, Duration.ofSeconds(1), Triggers.repeat(Triggers.any(Triggers.count(2), Triggers.timeSinceFirstMessage(Duration.ofMillis(500))))).getOperatorSpecGraph();
List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>();
MessageCollector messageCollector = envelope -> windowPanes.add((WindowPane<Integer, Collection<IntegerEnvelope>>) envelope.getMessage());
TestClock testClock = new TestClock();
StreamOperatorTask task = new StreamOperatorTask(sgb, testClock);
task.init(this.context);
task.processAsync(new IntegerEnvelope(1), messageCollector, taskCoordinator, taskCallback);
task.processAsync(new IntegerEnvelope(1), messageCollector, taskCoordinator, taskCallback);
// assert that the count trigger fired
Assert.assertEquals(windowPanes.size(), 1);
// advance the timer to enable the potential triggering of the inner timeSinceFirstMessage trigger
task.processAsync(new IntegerEnvelope(1), messageCollector, taskCoordinator, taskCallback);
testClock.advanceTime(Duration.ofMillis(500));
// assert that the triggering of the count trigger cancelled the inner timeSinceFirstMessage trigger
task.window(messageCollector, taskCoordinator);
Assert.assertEquals(windowPanes.size(), 2);
task.processAsync(new IntegerEnvelope(1), messageCollector, taskCoordinator, taskCallback);
task.processAsync(new IntegerEnvelope(1), messageCollector, taskCoordinator, taskCallback);
Assert.assertEquals(windowPanes.size(), 3);
task.processAsync(new IntegerEnvelope(1), messageCollector, taskCoordinator, taskCallback);
// advance timer by 500 more millis to enable the default trigger
testClock.advanceTime(Duration.ofMillis(500));
task.window(messageCollector, taskCoordinator);
// assert that the default trigger fired
Assert.assertEquals(windowPanes.size(), 4);
}
use of org.apache.samza.operators.windows.WindowPane in project samza by apache.
the class TestWindowOperator method testEndOfStreamFlushesWithDefaultTriggerFirings.
@Test
public void testEndOfStreamFlushesWithDefaultTriggerFirings() throws Exception {
OperatorSpecGraph sgb = this.getKeyedSessionWindowStreamGraph(AccumulationMode.DISCARDING, Duration.ofMillis(500)).getOperatorSpecGraph();
TestClock testClock = new TestClock();
List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>();
StreamOperatorTask task = new StreamOperatorTask(sgb, testClock);
task.init(this.context);
MessageCollector messageCollector = envelope -> windowPanes.add((WindowPane<Integer, Collection<IntegerEnvelope>>) envelope.getMessage());
task.processAsync(new IntegerEnvelope(1), messageCollector, taskCoordinator, taskCallback);
task.processAsync(new IntegerEnvelope(1), messageCollector, taskCoordinator, taskCallback);
testClock.advanceTime(1000);
task.window(messageCollector, taskCoordinator);
Assert.assertEquals(windowPanes.size(), 1);
task.processAsync(new IntegerEnvelope(1), messageCollector, taskCoordinator, taskCallback);
task.processAsync(new IntegerEnvelope(1), messageCollector, taskCoordinator, taskCallback);
final IncomingMessageEnvelope endOfStream = IncomingMessageEnvelope.buildEndOfStreamEnvelope(new SystemStreamPartition("kafka", "integers", new Partition(0)));
task.processAsync(endOfStream, messageCollector, taskCoordinator, taskCallback);
Assert.assertEquals(windowPanes.size(), 2);
Assert.assertEquals(windowPanes.get(0).getMessage().size(), 2);
verify(taskCoordinator, times(1)).commit(TaskCoordinator.RequestScope.CURRENT_TASK);
verify(taskCoordinator, times(1)).shutdown(TaskCoordinator.RequestScope.CURRENT_TASK);
}
use of org.apache.samza.operators.windows.WindowPane in project samza by apache.
the class TestWindowOperator method testSessionWindowsAccumulatingMode.
@Test
public void testSessionWindowsAccumulatingMode() throws Exception {
OperatorSpecGraph sgb = this.getKeyedSessionWindowStreamGraph(AccumulationMode.DISCARDING, Duration.ofMillis(500)).getOperatorSpecGraph();
TestClock testClock = new TestClock();
StreamOperatorTask task = new StreamOperatorTask(sgb, testClock);
List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>();
MessageCollector messageCollector = envelope -> windowPanes.add((WindowPane<Integer, Collection<IntegerEnvelope>>) envelope.getMessage());
task.init(this.context);
task.processAsync(new IntegerEnvelope(1), messageCollector, taskCoordinator, taskCallback);
task.processAsync(new IntegerEnvelope(1), messageCollector, taskCoordinator, taskCallback);
testClock.advanceTime(Duration.ofSeconds(1));
task.processAsync(new IntegerEnvelope(2), messageCollector, taskCoordinator, taskCallback);
task.processAsync(new IntegerEnvelope(2), messageCollector, taskCoordinator, taskCallback);
task.processAsync(new IntegerEnvelope(2), messageCollector, taskCoordinator, taskCallback);
task.processAsync(new IntegerEnvelope(2), messageCollector, taskCoordinator, taskCallback);
testClock.advanceTime(Duration.ofSeconds(1));
task.window(messageCollector, taskCoordinator);
Assert.assertEquals(windowPanes.size(), 2);
Assert.assertEquals((windowPanes.get(0).getMessage()).size(), 2);
Assert.assertEquals(windowPanes.get(0).getKey().getKey(), new Integer(1));
Assert.assertEquals(windowPanes.get(1).getKey().getKey(), new Integer(2));
Assert.assertEquals((windowPanes.get(0).getMessage()).size(), 2);
Assert.assertEquals((windowPanes.get(1).getMessage()).size(), 4);
}
use of org.apache.samza.operators.windows.WindowPane in project samza by apache.
the class TestWindowOperator method testEndOfStreamFlushesWithNoTriggerFirings.
@Test
public void testEndOfStreamFlushesWithNoTriggerFirings() throws Exception {
OperatorSpecGraph sgb = this.getKeyedSessionWindowStreamGraph(AccumulationMode.DISCARDING, Duration.ofMillis(500)).getOperatorSpecGraph();
TestClock testClock = new TestClock();
List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>();
StreamOperatorTask task = new StreamOperatorTask(sgb, testClock);
task.init(this.context);
MessageCollector messageCollector = envelope -> windowPanes.add((WindowPane<Integer, Collection<IntegerEnvelope>>) envelope.getMessage());
task.processAsync(new IntegerEnvelope(1), messageCollector, taskCoordinator, taskCallback);
task.processAsync(new IntegerEnvelope(1), messageCollector, taskCoordinator, taskCallback);
task.processAsync(new IntegerEnvelope(1), messageCollector, taskCoordinator, taskCallback);
task.processAsync(new IntegerEnvelope(1), messageCollector, taskCoordinator, taskCallback);
final IncomingMessageEnvelope endOfStream = IncomingMessageEnvelope.buildEndOfStreamEnvelope(new SystemStreamPartition("kafka", "integers", new Partition(0)));
task.processAsync(endOfStream, messageCollector, taskCoordinator, taskCallback);
Assert.assertEquals(windowPanes.size(), 1);
Assert.assertEquals(windowPanes.get(0).getMessage().size(), 4);
verify(taskCoordinator, times(1)).commit(TaskCoordinator.RequestScope.CURRENT_TASK);
verify(taskCoordinator, times(1)).shutdown(TaskCoordinator.RequestScope.CURRENT_TASK);
}
use of org.apache.samza.operators.windows.WindowPane in project samza by apache.
the class TestMessageStreamImpl method testWindowWithRelaxedTypes.
@Test
public void testWindowWithRelaxedTypes() throws Exception {
StreamApplicationDescriptorImpl mockGraph = mock(StreamApplicationDescriptorImpl.class);
OperatorSpec mockOpSpec = mock(OperatorSpec.class);
MessageStream<TestInputMessageEnvelope> inputStream = new MessageStreamImpl<>(mockGraph, mockOpSpec);
MapFunction<TestMessageEnvelope, String> keyExtractor = m -> m.getKey();
FoldLeftFunction<TestMessageEnvelope, Integer> aggregator = (m, c) -> c + 1;
SupplierFunction<Integer> initialValue = () -> 0;
// should compile since TestMessageEnvelope (input for functions) is base class of TestInputMessageEnvelope (M)
Window<TestInputMessageEnvelope, String, Integer> window = Windows.keyedTumblingWindow(keyExtractor, Duration.ofHours(1), initialValue, aggregator, null, mock(Serde.class));
MessageStream<WindowPane<String, Integer>> windowedStream = inputStream.window(window, "w1");
ArgumentCaptor<OperatorSpec> registeredOpCaptor = ArgumentCaptor.forClass(OperatorSpec.class);
verify(mockOpSpec).registerNextOperatorSpec(registeredOpCaptor.capture());
OperatorSpec<?, TestMessageEnvelope> registeredOpSpec = registeredOpCaptor.getValue();
assertTrue(registeredOpSpec instanceof WindowOperatorSpec);
assertEquals(OpCode.WINDOW, registeredOpSpec.getOpCode());
assertEquals(window, ((WindowOperatorSpec) registeredOpSpec).getWindow());
}
Aggregations