Examples with Punctuator org.apache.kafka.streams.processor.Punctuator used on opensource projects

Example 1 with Punctuator

use of org.apache.kafka.streams.processor.Punctuator in project apache-kafka-on-k8s by banzaicloud.

the class MockProcessorContextTest method shouldCapturePunctuator.

@Test
public void shouldCapturePunctuator() {
    final Processor<String, Long> processor = new Processor<String, Long>() {

        @Override
        public void init(final ProcessorContext context) {
            context.schedule(1000L, PunctuationType.WALL_CLOCK_TIME, new Punctuator() {

                @Override
                public void punctuate(final long timestamp) {
                    context.commit();
                }
            });
        }

        @Override
        public void process(final String key, final Long value) {
        }

        @Override
        public void punctuate(final long timestamp) {
        }

        @Override
        public void close() {
        }
    };
    final MockProcessorContext context = new MockProcessorContext();
    processor.init(context);
    final MockProcessorContext.CapturedPunctuator capturedPunctuator = context.scheduledPunctuators().get(0);
    assertEquals(1000L, capturedPunctuator.getIntervalMs());
    assertEquals(PunctuationType.WALL_CLOCK_TIME, capturedPunctuator.getType());
    assertFalse(capturedPunctuator.cancelled());
    final Punctuator punctuator = capturedPunctuator.getPunctuator();
    assertFalse(context.committed());
    punctuator.punctuate(1234L);
    assertTrue(context.committed());
}

Example 2 with Punctuator

use of org.apache.kafka.streams.processor.Punctuator in project apache-kafka-on-k8s by banzaicloud.

the class StreamTaskTest method shouldWrapKafkaExceptionsWithStreamsExceptionAndAddContextWhenPunctuatingStreamTime.

@Test
public void shouldWrapKafkaExceptionsWithStreamsExceptionAndAddContextWhenPunctuatingStreamTime() {
    task = createStatelessTask(false);
    task.initializeStateStores();
    task.initializeTopology();
    try {
        task.punctuate(processorStreamTime, 1, PunctuationType.STREAM_TIME, new Punctuator() {

            @Override
            public void punctuate(long timestamp) {
                throw new KafkaException("KABOOM!");
            }
        });
        fail("Should've thrown StreamsException");
    } catch (final StreamsException e) {
        final String message = e.getMessage();
        assertTrue("message=" + message + " should contain processor", message.contains("processor '" + processorStreamTime.name() + "'"));
        assertThat(task.processorContext.currentNode(), nullValue());
    }
}

Example 3 with Punctuator

use of org.apache.kafka.streams.processor.Punctuator in project apache-kafka-on-k8s by banzaicloud.

the class StreamTaskTest method shouldThrowIllegalStateExceptionOnScheduleIfCurrentNodeIsNull.

@Test(expected = IllegalStateException.class)
public void shouldThrowIllegalStateExceptionOnScheduleIfCurrentNodeIsNull() {
    task = createStatelessTask(false);
    task.schedule(1, PunctuationType.STREAM_TIME, new Punctuator() {

        @Override
        public void punctuate(long timestamp) {
        // no-op
        }
    });
}

Example 4 with Punctuator

use of org.apache.kafka.streams.processor.Punctuator in project apache-kafka-on-k8s by banzaicloud.

the class StreamThreadTest method shouldPunctuateActiveTask.

@Test
public void shouldPunctuateActiveTask() {
    final List<Long> punctuatedStreamTime = new ArrayList<>();
    final List<Long> punctuatedWallClockTime = new ArrayList<>();
    final ProcessorSupplier<Object, Object> punctuateProcessor = new ProcessorSupplier<Object, Object>() {

        @Override
        public Processor<Object, Object> get() {
            return new Processor<Object, Object>() {

                @Override
                public void init(ProcessorContext context) {
                    context.schedule(100L, PunctuationType.STREAM_TIME, new Punctuator() {

                        @Override
                        public void punctuate(long timestamp) {
                            punctuatedStreamTime.add(timestamp);
                        }
                    });
                    context.schedule(100L, PunctuationType.WALL_CLOCK_TIME, new Punctuator() {

                        @Override
                        public void punctuate(long timestamp) {
                            punctuatedWallClockTime.add(timestamp);
                        }
                    });
                }

                @Override
                public void process(Object key, Object value) {
                }

                @SuppressWarnings("deprecation")
                @Override
                public void punctuate(long timestamp) {
                }

                @Override
                public void close() {
                }
            };
        }
    };
    internalStreamsBuilder.stream(Collections.singleton(topic1), consumed).process(punctuateProcessor);
    final StreamThread thread = createStreamThread(clientId, config, false);
    thread.setState(StreamThread.State.RUNNING);
    thread.rebalanceListener.onPartitionsRevoked(null);
    final List<TopicPartition> assignedPartitions = new ArrayList<>();
    final Map<TaskId, Set<TopicPartition>> activeTasks = new HashMap<>();
    // assign single partition
    assignedPartitions.add(t1p1);
    activeTasks.put(task1, Collections.singleton(t1p1));
    thread.taskManager().setAssignmentMetadata(activeTasks, Collections.<TaskId, Set<TopicPartition>>emptyMap());
    clientSupplier.consumer.assign(assignedPartitions);
    clientSupplier.consumer.updateBeginningOffsets(Collections.singletonMap(t1p1, 0L));
    thread.rebalanceListener.onPartitionsAssigned(assignedPartitions);
    thread.runOnce(-1);
    assertEquals(0, punctuatedStreamTime.size());
    assertEquals(0, punctuatedWallClockTime.size());
    mockTime.sleep(100L);
    for (long i = 0L; i < 10L; i++) {
        clientSupplier.consumer.addRecord(new ConsumerRecord<>(topic1, 1, i, i * 100L, TimestampType.CREATE_TIME, ConsumerRecord.NULL_CHECKSUM, ("K" + i).getBytes().length, ("V" + i).getBytes().length, ("K" + i).getBytes(), ("V" + i).getBytes()));
    }
    thread.runOnce(-1);
    assertEquals(1, punctuatedStreamTime.size());
    assertEquals(1, punctuatedWallClockTime.size());
    mockTime.sleep(100L);
    thread.runOnce(-1);
    // we should skip stream time punctuation, only trigger wall-clock time punctuation
    assertEquals(1, punctuatedStreamTime.size());
    assertEquals(2, punctuatedWallClockTime.size());
}

Example 5 with Punctuator

use of org.apache.kafka.streams.processor.Punctuator in project apache-kafka-on-k8s by banzaicloud.

the class PunctuationQueueTest method testPunctuationIntervalCustomAlignment.

@Test
public void testPunctuationIntervalCustomAlignment() {
    final TestProcessor processor = new TestProcessor();
    final ProcessorNode<String, String> node = new ProcessorNode<>("test", processor, null);
    final PunctuationQueue queue = new PunctuationQueue();
    final Punctuator punctuator = new Punctuator() {

        @Override
        public void punctuate(long timestamp) {
            node.processor().punctuate(timestamp);
        }
    };
    final PunctuationSchedule sched = new PunctuationSchedule(node, 50L, 100L, punctuator);
    final long now = sched.timestamp - 50L;
    queue.schedule(sched);
    ProcessorNodePunctuator processorNodePunctuator = new ProcessorNodePunctuator() {

        @Override
        public void punctuate(ProcessorNode node, long time, PunctuationType type, Punctuator punctuator) {
            punctuator.punctuate(time);
        }
    };
    queue.mayPunctuate(now, PunctuationType.STREAM_TIME, processorNodePunctuator);
    assertEquals(0, processor.punctuatedAt.size());
    queue.mayPunctuate(now + 49L, PunctuationType.STREAM_TIME, processorNodePunctuator);
    assertEquals(0, processor.punctuatedAt.size());
    queue.mayPunctuate(now + 50L, PunctuationType.STREAM_TIME, processorNodePunctuator);
    assertEquals(1, processor.punctuatedAt.size());
    queue.mayPunctuate(now + 149L, PunctuationType.STREAM_TIME, processorNodePunctuator);
    assertEquals(1, processor.punctuatedAt.size());
    queue.mayPunctuate(now + 150L, PunctuationType.STREAM_TIME, processorNodePunctuator);
    assertEquals(2, processor.punctuatedAt.size());
    queue.mayPunctuate(now + 1051L, PunctuationType.STREAM_TIME, processorNodePunctuator);
    assertEquals(3, processor.punctuatedAt.size());
    queue.mayPunctuate(now + 1052L, PunctuationType.STREAM_TIME, processorNodePunctuator);
    assertEquals(3, processor.punctuatedAt.size());
    queue.mayPunctuate(now + 1150L, PunctuationType.STREAM_TIME, processorNodePunctuator);
    assertEquals(4, processor.punctuatedAt.size());
}