Examples with Record org.apache.kafka.streams.processor.api.Record used on opensource projects

Example 16 with Record

use of org.apache.kafka.streams.processor.api.Record in project kafka by apache.

the class GlobalStateUpdateTask method update.

@SuppressWarnings("unchecked")
@Override
public void update(final ConsumerRecord<byte[], byte[]> record) {
    final RecordDeserializer sourceNodeAndDeserializer = deserializers.get(record.topic());
    final ConsumerRecord<Object, Object> deserialized = sourceNodeAndDeserializer.deserialize(processorContext, record);
    if (deserialized != null) {
        final ProcessorRecordContext recordContext = new ProcessorRecordContext(deserialized.timestamp(), deserialized.offset(), deserialized.partition(), deserialized.topic(), deserialized.headers());
        processorContext.setRecordContext(recordContext);
        processorContext.setCurrentNode(sourceNodeAndDeserializer.sourceNode());
        final Record<Object, Object> toProcess = new Record<>(deserialized.key(), deserialized.value(), processorContext.timestamp(), processorContext.headers());
        ((SourceNode<Object, Object>) sourceNodeAndDeserializer.sourceNode()).process(toProcess);
    }
    offsets.put(new TopicPartition(record.topic(), record.partition()), record.offset() + 1);
}

Example 17 with Record

use of org.apache.kafka.streams.processor.api.Record in project kafka by apache.

the class ProcessorNodeTest method testTopologyLevelClassCastExceptionDirect.

@Test
public void testTopologyLevelClassCastExceptionDirect() {
    final Metrics metrics = new Metrics();
    final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, "test-client", StreamsConfig.METRICS_LATEST, new MockTime());
    final InternalMockProcessorContext<Object, Object> context = new InternalMockProcessorContext<>(streamsMetrics);
    final ProcessorNode<Object, Object, Object, Object> node = new ProcessorNode<>("pname", new ClassCastProcessor(), Collections.emptySet());
    node.init(context);
    final StreamsException se = assertThrows(StreamsException.class, () -> node.process(new Record<>("aKey", "aValue", 0)));
    assertThat(se.getCause(), instanceOf(ClassCastException.class));
    assertThat(se.getMessage(), containsString("default Serdes"));
    assertThat(se.getMessage(), containsString("input types"));
    assertThat(se.getMessage(), containsString("pname"));
}

Example 18 with Record

use of org.apache.kafka.streams.processor.api.Record in project kafka by apache.

the class WordCountTransformerTest method test.

@Test
public void test() {
    final MockProcessorContext<String, String> context = new MockProcessorContext<>();
    // Create and initialize the transformer under test; including its provided store
    final WordCountTransformerDemo.MyTransformerSupplier supplier = new WordCountTransformerDemo.MyTransformerSupplier();
    for (final StoreBuilder<?> storeBuilder : supplier.stores()) {
        final StateStore store = storeBuilder.withLoggingDisabled().build();
        store.init(context.getStateStoreContext(), store);
        context.getStateStoreContext().register(store, null);
    }
    final Transformer<String, String, KeyValue<String, String>> transformer = supplier.get();
    transformer.init(new org.apache.kafka.streams.processor.MockProcessorContext() {

        @Override
        public <S extends StateStore> S getStateStore(final String name) {
            return context.getStateStore(name);
        }

        @Override
        public <K, V> void forward(final K key, final V value) {
            context.forward(new Record<>((String) key, (String) value, 0L));
        }

        @Override
        public Cancellable schedule(final Duration interval, final PunctuationType type, final Punctuator callback) {
            return context.schedule(interval, type, callback);
        }
    });
    // send a record to the transformer
    transformer.transform("key", "alpha beta\tgamma\n\talpha");
    // note that the transformer does not forward during transform()
    assertTrue(context.forwarded().isEmpty());
    // now, we trigger the punctuator, which iterates over the state store and forwards the contents.
    context.scheduledPunctuators().get(0).getPunctuator().punctuate(0L);
    // finally, we can verify the output.
    final List<MockProcessorContext.CapturedForward<? extends String, ? extends String>> capturedForwards = context.forwarded();
    final List<MockProcessorContext.CapturedForward<? extends String, ? extends String>> expected = asList(new MockProcessorContext.CapturedForward<>(new Record<>("alpha", "2", 0L)), new MockProcessorContext.CapturedForward<>(new Record<>("beta", "1", 0L)), new MockProcessorContext.CapturedForward<>(new Record<>("gamma", "1", 0L)));
    assertThat(capturedForwards, is(expected));
}

Example 19 with Record

use of org.apache.kafka.streams.processor.api.Record in project kafka by apache.

the class KTableSuppressProcessorTest method finalResultsWithZeroGraceShouldStillBufferUntilTheWindowEnd.

/**
 * Testing a special case of final results: that even with a grace period of 0,
 * it will still buffer events and emit only after the end of the window.
 * As opposed to emitting immediately the way regular suppression would with a time limit of 0.
 */
@Test
public void finalResultsWithZeroGraceShouldStillBufferUntilTheWindowEnd() {
    final Harness<Windowed<String>, Long> harness = new Harness<>(finalResults(ofMillis(0L)), timeWindowedSerdeFrom(String.class, 100L), Long());
    final MockInternalNewProcessorContext<Windowed<String>, Change<Long>> context = harness.context;
    // note the record is in the past, but the window end is in the future, so we still have to buffer,
    // even though the grace period is 0.
    final long timestamp = 5L;
    final long windowEnd = 100L;
    context.setRecordMetadata("", 0, 0L);
    context.setTimestamp(timestamp);
    final Windowed<String> key = new Windowed<>("hey", new TimeWindow(0, windowEnd));
    final Change<Long> value = ARBITRARY_CHANGE;
    harness.processor.process(new Record<>(key, value, timestamp));
    assertThat(context.forwarded(), hasSize(0));
    context.setRecordMetadata("", 0, 1L);
    context.setTimestamp(windowEnd);
    harness.processor.process(new Record<>(new Windowed<>("dummyKey", new TimeWindow(windowEnd, windowEnd + 100L)), ARBITRARY_CHANGE, windowEnd));
    assertThat(context.forwarded(), hasSize(1));
    final MockProcessorContext.CapturedForward capturedForward = context.forwarded().get(0);
    assertThat(capturedForward.record(), is(new Record<>(key, value, timestamp)));
}

Example 20 with Record

use of org.apache.kafka.streams.processor.api.Record in project kafka by apache.

the class KTableSuppressProcessorTest method suppressShouldNotDropTombstonesForSessionWindows.

/**
 * It's NOT OK to drop tombstones for non-final-results windowed streams, since we may have emitted some results for
 * the window before getting the tombstone (see the {@link SuppressedInternal} javadoc).
 */
@Test
public void suppressShouldNotDropTombstonesForSessionWindows() {
    final Harness<Windowed<String>, Long> harness = new Harness<>(untilTimeLimit(ofMillis(0), maxRecords(0)), sessionWindowedSerdeFrom(String.class), Long());
    final MockInternalNewProcessorContext<Windowed<String>, Change<Long>> context = harness.context;
    final long timestamp = 100L;
    context.setRecordMetadata("", 0, 0L);
    context.setTimestamp(timestamp);
    final Windowed<String> key = new Windowed<>("hey", new SessionWindow(0L, 0L));
    final Change<Long> value = new Change<>(null, ARBITRARY_LONG);
    harness.processor.process(new Record<>(key, value, timestamp));
    assertThat(context.forwarded(), hasSize(1));
    final MockProcessorContext.CapturedForward capturedForward = context.forwarded().get(0);
    assertThat(capturedForward.record(), is(new Record<>(key, value, timestamp)));
}