Search in sources :

Example 1 with RecordBatchingStateRestoreCallback

use of org.apache.kafka.streams.processor.internals.RecordBatchingStateRestoreCallback in project kafka by apache.

the class InMemoryWindowStore method init.

@Deprecated
@Override
public void init(final ProcessorContext context, final StateStore root) {
    this.context = context;
    final StreamsMetricsImpl metrics = ProcessorContextUtils.getMetricsImpl(context);
    final String threadId = Thread.currentThread().getName();
    final String taskName = context.taskId().toString();
    expiredRecordSensor = TaskMetrics.droppedRecordsSensor(threadId, taskName, metrics);
    if (root != null) {
        final boolean consistencyEnabled = StreamsConfig.InternalConfig.getBoolean(context.appConfigs(), IQ_CONSISTENCY_OFFSET_VECTOR_ENABLED, false);
        context.register(root, (RecordBatchingStateRestoreCallback) records -> {
            for (final ConsumerRecord<byte[], byte[]> record : records) {
                put(Bytes.wrap(extractStoreKeyBytes(record.key())), record.value(), extractStoreTimestamp(record.key()));
                ChangelogRecordDeserializationHelper.applyChecksAndUpdatePosition(record, consistencyEnabled, position);
            }
        });
    }
    open = true;
}
Also used : StreamsConfig(org.apache.kafka.streams.StreamsConfig) TaskMetrics(org.apache.kafka.streams.processor.internals.metrics.TaskMetrics) LoggerFactory(org.slf4j.LoggerFactory) PositionBound(org.apache.kafka.streams.query.PositionBound) RecordBatchingStateRestoreCallback(org.apache.kafka.streams.processor.internals.RecordBatchingStateRestoreCallback) WindowStore(org.apache.kafka.streams.state.WindowStore) ByteBuffer(java.nio.ByteBuffer) StoreToProcessorContextAdapter(org.apache.kafka.streams.processor.internals.StoreToProcessorContextAdapter) StateStoreContext(org.apache.kafka.streams.processor.StateStoreContext) Windowed(org.apache.kafka.streams.kstream.Windowed) Map(java.util.Map) ProcessorContextUtils(org.apache.kafka.streams.processor.internals.ProcessorContextUtils) StreamsMetricsImpl(org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl) NoSuchElementException(java.util.NoSuchElementException) QueryResult(org.apache.kafka.streams.query.QueryResult) Sensor(org.apache.kafka.common.metrics.Sensor) QueryConfig(org.apache.kafka.streams.query.QueryConfig) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) Position(org.apache.kafka.streams.query.Position) Query(org.apache.kafka.streams.query.Query) WindowKeySchema.extractStoreTimestamp(org.apache.kafka.streams.state.internals.WindowKeySchema.extractStoreTimestamp) ConcurrentNavigableMap(java.util.concurrent.ConcurrentNavigableMap) KeyValue(org.apache.kafka.streams.KeyValue) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) ChangelogRecordDeserializationHelper(org.apache.kafka.streams.processor.internals.ChangelogRecordDeserializationHelper) Bytes(org.apache.kafka.common.utils.Bytes) Objects(java.util.Objects) ProcessorContext(org.apache.kafka.streams.processor.ProcessorContext) KeyValueIterator(org.apache.kafka.streams.state.KeyValueIterator) ConcurrentSkipListMap(java.util.concurrent.ConcurrentSkipListMap) WindowKeySchema.extractStoreKeyBytes(org.apache.kafka.streams.state.internals.WindowKeySchema.extractStoreKeyBytes) StateStore(org.apache.kafka.streams.processor.StateStore) WindowStoreIterator(org.apache.kafka.streams.state.WindowStoreIterator) ConsumerRecord(org.apache.kafka.clients.consumer.ConsumerRecord) TimeWindow(org.apache.kafka.streams.kstream.internals.TimeWindow) IQ_CONSISTENCY_OFFSET_VECTOR_ENABLED(org.apache.kafka.streams.StreamsConfig.InternalConfig.IQ_CONSISTENCY_OFFSET_VECTOR_ENABLED) StreamsMetricsImpl(org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl) ConsumerRecord(org.apache.kafka.clients.consumer.ConsumerRecord)

Example 2 with RecordBatchingStateRestoreCallback

use of org.apache.kafka.streams.processor.internals.RecordBatchingStateRestoreCallback in project kafka by apache.

the class InMemorySessionStore method init.

@Deprecated
@Override
public void init(final ProcessorContext context, final StateStore root) {
    final String threadId = Thread.currentThread().getName();
    final String taskName = context.taskId().toString();
    // If it doesn't, we can't record this metric.
    if (context instanceof InternalProcessorContext) {
        this.context = (InternalProcessorContext) context;
        final StreamsMetricsImpl metrics = this.context.metrics();
        expiredRecordSensor = TaskMetrics.droppedRecordsSensor(threadId, taskName, metrics);
    } else {
        this.context = null;
        expiredRecordSensor = null;
    }
    if (root != null) {
        final boolean consistencyEnabled = StreamsConfig.InternalConfig.getBoolean(context.appConfigs(), IQ_CONSISTENCY_OFFSET_VECTOR_ENABLED, false);
        context.register(root, (RecordBatchingStateRestoreCallback) records -> {
            for (final ConsumerRecord<byte[], byte[]> record : records) {
                put(SessionKeySchema.from(Bytes.wrap(record.key())), record.value());
                ChangelogRecordDeserializationHelper.applyChecksAndUpdatePosition(record, consistencyEnabled, position);
            }
        });
    }
    open = true;
}
Also used : StreamsConfig(org.apache.kafka.streams.StreamsConfig) TaskMetrics(org.apache.kafka.streams.processor.internals.metrics.TaskMetrics) LoggerFactory(org.slf4j.LoggerFactory) PositionBound(org.apache.kafka.streams.query.PositionBound) RecordBatchingStateRestoreCallback(org.apache.kafka.streams.processor.internals.RecordBatchingStateRestoreCallback) StoreToProcessorContextAdapter(org.apache.kafka.streams.processor.internals.StoreToProcessorContextAdapter) StateStoreContext(org.apache.kafka.streams.processor.StateStoreContext) Windowed(org.apache.kafka.streams.kstream.Windowed) Map(java.util.Map) StreamsMetricsImpl(org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl) SessionStore(org.apache.kafka.streams.state.SessionStore) NoSuchElementException(java.util.NoSuchElementException) QueryResult(org.apache.kafka.streams.query.QueryResult) Sensor(org.apache.kafka.common.metrics.Sensor) QueryConfig(org.apache.kafka.streams.query.QueryConfig) Logger(org.slf4j.Logger) InternalProcessorContext(org.apache.kafka.streams.processor.internals.InternalProcessorContext) Iterator(java.util.Iterator) Position(org.apache.kafka.streams.query.Position) Query(org.apache.kafka.streams.query.Query) ConcurrentNavigableMap(java.util.concurrent.ConcurrentNavigableMap) KeyValue(org.apache.kafka.streams.KeyValue) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) ChangelogRecordDeserializationHelper(org.apache.kafka.streams.processor.internals.ChangelogRecordDeserializationHelper) Bytes(org.apache.kafka.common.utils.Bytes) Objects(java.util.Objects) ProcessorContext(org.apache.kafka.streams.processor.ProcessorContext) KeyValueIterator(org.apache.kafka.streams.state.KeyValueIterator) ConcurrentSkipListMap(java.util.concurrent.ConcurrentSkipListMap) StateStore(org.apache.kafka.streams.processor.StateStore) ConsumerRecord(org.apache.kafka.clients.consumer.ConsumerRecord) Entry(java.util.Map.Entry) SessionWindow(org.apache.kafka.streams.kstream.internals.SessionWindow) IQ_CONSISTENCY_OFFSET_VECTOR_ENABLED(org.apache.kafka.streams.StreamsConfig.InternalConfig.IQ_CONSISTENCY_OFFSET_VECTOR_ENABLED) InternalProcessorContext(org.apache.kafka.streams.processor.internals.InternalProcessorContext) StreamsMetricsImpl(org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl) ConsumerRecord(org.apache.kafka.clients.consumer.ConsumerRecord)

Example 3 with RecordBatchingStateRestoreCallback

use of org.apache.kafka.streams.processor.internals.RecordBatchingStateRestoreCallback in project kafka by apache.

the class AbstractRocksDBSegmentedBytesStore method init.

@Deprecated
@Override
public void init(final ProcessorContext context, final StateStore root) {
    this.context = context;
    final StreamsMetricsImpl metrics = ProcessorContextUtils.getMetricsImpl(context);
    final String threadId = Thread.currentThread().getName();
    final String taskName = context.taskId().toString();
    expiredRecordSensor = TaskMetrics.droppedRecordsSensor(threadId, taskName, metrics);
    segments.openExisting(this.context, observedStreamTime);
    final File positionCheckpointFile = new File(context.stateDir(), name() + ".position");
    this.positionCheckpoint = new OffsetCheckpoint(positionCheckpointFile);
    this.position = StoreQueryUtils.readPositionFromCheckpoint(positionCheckpoint);
    // register and possibly restore the state from the logs
    stateStoreContext.register(root, (RecordBatchingStateRestoreCallback) this::restoreAllInternal, () -> StoreQueryUtils.checkpointPosition(positionCheckpoint, position));
    open = true;
    consistencyEnabled = StreamsConfig.InternalConfig.getBoolean(context.appConfigs(), IQ_CONSISTENCY_OFFSET_VECTOR_ENABLED, false);
}
Also used : RecordBatchingStateRestoreCallback(org.apache.kafka.streams.processor.internals.RecordBatchingStateRestoreCallback) StreamsMetricsImpl(org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl) File(java.io.File)

Example 4 with RecordBatchingStateRestoreCallback

use of org.apache.kafka.streams.processor.internals.RecordBatchingStateRestoreCallback in project kafka by apache.

the class TimeOrderedKeyValueBufferTest method shouldRestoreV3Format.

@Test
public void shouldRestoreV3Format() {
    final TimeOrderedKeyValueBuffer<String, String> buffer = bufferSupplier.apply(testName);
    final MockInternalProcessorContext context = makeContext();
    buffer.init((StateStoreContext) context, buffer);
    final RecordBatchingStateRestoreCallback stateRestoreCallback = (RecordBatchingStateRestoreCallback) context.stateRestoreCallback(testName);
    context.setRecordContext(new ProcessorRecordContext(0, 0, 0, "", new RecordHeaders()));
    final RecordHeaders headers = new RecordHeaders(new Header[] { new RecordHeader("v", new byte[] { (byte) 3 }) });
    // These serialized formats were captured by running version 2.4 code.
    // They verify that an upgrade from 2.4 will work.
    // Do not change them.
    final String toDeleteBinary = "0000000000000000000000000000000000000005746F70696300000000FFFFFFFFFFFFFFFFFFFFFFFF00000006646F6F6D65640000000000000000";
    final String asdfBinary = "0000000000000001000000000000000000000005746F70696300000000FFFFFFFFFFFFFFFFFFFFFFFF00000004717765720000000000000002";
    final String zxcvBinary1 = "0000000000000002000000000000000000000005746F70696300000000FFFFFFFF0000000870726576696F75730000000749474E4F52454400000005336F34696D0000000000000001";
    final String zxcvBinary2 = "0000000000000003000000000000000000000005746F70696300000000FFFFFFFF0000000870726576696F757300000005336F34696D000000046E6578740000000000000001";
    stateRestoreCallback.restoreBatch(asList(new ConsumerRecord<>("changelog-topic", 0, 0, 999, TimestampType.CREATE_TIME, -1, -1, "todelete".getBytes(UTF_8), hexStringToByteArray(toDeleteBinary), headers, Optional.empty()), new ConsumerRecord<>("changelog-topic", 0, 1, 9999, TimestampType.CREATE_TIME, -1, -1, "asdf".getBytes(UTF_8), hexStringToByteArray(asdfBinary), headers, Optional.empty()), new ConsumerRecord<>("changelog-topic", 0, 2, 99, TimestampType.CREATE_TIME, -1, -1, "zxcv".getBytes(UTF_8), hexStringToByteArray(zxcvBinary1), headers, Optional.empty()), new ConsumerRecord<>("changelog-topic", 0, 2, 100, TimestampType.CREATE_TIME, -1, -1, "zxcv".getBytes(UTF_8), hexStringToByteArray(zxcvBinary2), headers, Optional.empty())));
    assertThat(buffer.numRecords(), is(3));
    assertThat(buffer.minTimestamp(), is(0L));
    assertThat(buffer.bufferSize(), is(142L));
    stateRestoreCallback.restoreBatch(singletonList(new ConsumerRecord<>("changelog-topic", 0, 3, 3, TimestampType.CREATE_TIME, -1, -1, "todelete".getBytes(UTF_8), null, new RecordHeaders(), Optional.empty())));
    assertThat(buffer.numRecords(), is(2));
    assertThat(buffer.minTimestamp(), is(1L));
    assertThat(buffer.bufferSize(), is(95L));
    assertThat(buffer.priorValueForBuffered("todelete"), is(Maybe.undefined()));
    assertThat(buffer.priorValueForBuffered("asdf"), is(Maybe.defined(null)));
    assertThat(buffer.priorValueForBuffered("zxcv"), is(Maybe.defined(ValueAndTimestamp.make("previous", -1))));
    // flush the buffer into a list in buffer order so we can make assertions about the contents.
    final List<Eviction<String, String>> evicted = new LinkedList<>();
    buffer.evictWhile(() -> true, evicted::add);
    // Several things to note:
    // * The buffered records are ordered according to their buffer time (serialized in the value of the changelog)
    // * The record timestamps are properly restored, and not conflated with the record's buffer time.
    // * The keys and values are properly restored
    // * The record topic is set to the original input topic, *not* the changelog topic
    // * The record offset preserves the original input record's offset, *not* the offset of the changelog record
    assertThat(evicted, is(asList(new Eviction<>("zxcv", new Change<>("next", "3o4im"), getContext(3L)), new Eviction<>("asdf", new Change<>("qwer", null), getContext(1L)))));
    cleanup(context, buffer);
}
Also used : Change(org.apache.kafka.streams.kstream.internals.Change) ConsumerRecord(org.apache.kafka.clients.consumer.ConsumerRecord) LinkedList(java.util.LinkedList) RecordBatchingStateRestoreCallback(org.apache.kafka.streams.processor.internals.RecordBatchingStateRestoreCallback) RecordHeaders(org.apache.kafka.common.header.internals.RecordHeaders) ProcessorRecordContext(org.apache.kafka.streams.processor.internals.ProcessorRecordContext) Eviction(org.apache.kafka.streams.state.internals.TimeOrderedKeyValueBuffer.Eviction) MockInternalProcessorContext(org.apache.kafka.test.MockInternalProcessorContext) RecordHeader(org.apache.kafka.common.header.internals.RecordHeader) Test(org.junit.Test)

Example 5 with RecordBatchingStateRestoreCallback

use of org.apache.kafka.streams.processor.internals.RecordBatchingStateRestoreCallback in project kafka by apache.

the class TimeOrderedKeyValueBufferTest method shouldRestoreV1Format.

@Test
public void shouldRestoreV1Format() {
    final TimeOrderedKeyValueBuffer<String, String> buffer = bufferSupplier.apply(testName);
    final MockInternalProcessorContext context = makeContext();
    buffer.init((StateStoreContext) context, buffer);
    final RecordBatchingStateRestoreCallback stateRestoreCallback = (RecordBatchingStateRestoreCallback) context.stateRestoreCallback(testName);
    context.setRecordContext(new ProcessorRecordContext(0, 0, 0, "", new RecordHeaders()));
    final RecordHeaders v1FlagHeaders = new RecordHeaders(new Header[] { new RecordHeader("v", new byte[] { (byte) 1 }) });
    // These serialized formats were captured by running version 2.2 code.
    // They verify that an upgrade from 2.2 will work.
    // Do not change them.
    final String toDeleteBinary = "00000000000000000000000000000000000000000000000000000005746F70696300000000FFFFFFFF0000000EFFFFFFFF00000006646F6F6D6564";
    final String asdfBinary = "00000000000000020000000000000001000000000000000000000005746F70696300000000FFFFFFFF0000000CFFFFFFFF0000000471776572";
    final String zxcvBinary1 = "00000000000000010000000000000002000000000000000000000005746F70696300000000FFFFFFFF000000150000000870726576696F757300000005336F34696D";
    final String zxcvBinary2 = "00000000000000010000000000000003000000000000000000000005746F70696300000000FFFFFFFF0000001100000005336F34696D000000046E657874";
    stateRestoreCallback.restoreBatch(asList(new ConsumerRecord<>("changelog-topic", 0, 0, 999, TimestampType.CREATE_TIME, -1, -1, "todelete".getBytes(UTF_8), hexStringToByteArray(toDeleteBinary), v1FlagHeaders, Optional.empty()), new ConsumerRecord<>("changelog-topic", 0, 1, 9999, TimestampType.CREATE_TIME, -1, -1, "asdf".getBytes(UTF_8), hexStringToByteArray(asdfBinary), v1FlagHeaders, Optional.empty()), new ConsumerRecord<>("changelog-topic", 0, 2, 99, TimestampType.CREATE_TIME, -1, -1, "zxcv".getBytes(UTF_8), hexStringToByteArray(zxcvBinary1), v1FlagHeaders, Optional.empty()), new ConsumerRecord<>("changelog-topic", 0, 3, 100, TimestampType.CREATE_TIME, -1, -1, "zxcv".getBytes(UTF_8), hexStringToByteArray(zxcvBinary2), v1FlagHeaders, Optional.empty())));
    assertThat(buffer.numRecords(), is(3));
    assertThat(buffer.minTimestamp(), is(0L));
    assertThat(buffer.bufferSize(), is(142L));
    stateRestoreCallback.restoreBatch(singletonList(new ConsumerRecord<>("changelog-topic", 0, 3, 3, TimestampType.CREATE_TIME, -1, -1, "todelete".getBytes(UTF_8), null, new RecordHeaders(), Optional.empty())));
    assertThat(buffer.numRecords(), is(2));
    assertThat(buffer.minTimestamp(), is(1L));
    assertThat(buffer.bufferSize(), is(95L));
    assertThat(buffer.priorValueForBuffered("todelete"), is(Maybe.undefined()));
    assertThat(buffer.priorValueForBuffered("asdf"), is(Maybe.defined(null)));
    assertThat(buffer.priorValueForBuffered("zxcv"), is(Maybe.defined(ValueAndTimestamp.make("previous", -1))));
    // flush the buffer into a list in buffer order so we can make assertions about the contents.
    final List<Eviction<String, String>> evicted = new LinkedList<>();
    buffer.evictWhile(() -> true, evicted::add);
    // Several things to note:
    // * The buffered records are ordered according to their buffer time (serialized in the value of the changelog)
    // * The record timestamps are properly restored, and not conflated with the record's buffer time.
    // * The keys and values are properly restored
    // * The record topic is set to the original input topic, *not* the changelog topic
    // * The record offset preserves the original input record's offset, *not* the offset of the changelog record
    assertThat(evicted, is(asList(new Eviction<>("zxcv", new Change<>("next", "3o4im"), getContext(3L)), new Eviction<>("asdf", new Change<>("qwer", null), getContext(1L)))));
    cleanup(context, buffer);
}
Also used : Change(org.apache.kafka.streams.kstream.internals.Change) ConsumerRecord(org.apache.kafka.clients.consumer.ConsumerRecord) LinkedList(java.util.LinkedList) RecordBatchingStateRestoreCallback(org.apache.kafka.streams.processor.internals.RecordBatchingStateRestoreCallback) RecordHeaders(org.apache.kafka.common.header.internals.RecordHeaders) ProcessorRecordContext(org.apache.kafka.streams.processor.internals.ProcessorRecordContext) Eviction(org.apache.kafka.streams.state.internals.TimeOrderedKeyValueBuffer.Eviction) MockInternalProcessorContext(org.apache.kafka.test.MockInternalProcessorContext) RecordHeader(org.apache.kafka.common.header.internals.RecordHeader) Test(org.junit.Test)

Aggregations

RecordBatchingStateRestoreCallback (org.apache.kafka.streams.processor.internals.RecordBatchingStateRestoreCallback)12 ConsumerRecord (org.apache.kafka.clients.consumer.ConsumerRecord)9 RecordHeaders (org.apache.kafka.common.header.internals.RecordHeaders)6 ProcessorRecordContext (org.apache.kafka.streams.processor.internals.ProcessorRecordContext)6 MockInternalProcessorContext (org.apache.kafka.test.MockInternalProcessorContext)6 Test (org.junit.Test)6 LinkedList (java.util.LinkedList)5 RecordHeader (org.apache.kafka.common.header.internals.RecordHeader)5 Change (org.apache.kafka.streams.kstream.internals.Change)5 Eviction (org.apache.kafka.streams.state.internals.TimeOrderedKeyValueBuffer.Eviction)5 StreamsMetricsImpl (org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl)3 File (java.io.File)2 Iterator (java.util.Iterator)2 Map (java.util.Map)2 NoSuchElementException (java.util.NoSuchElementException)2 Objects (java.util.Objects)2 Set (java.util.Set)2 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)2 ConcurrentNavigableMap (java.util.concurrent.ConcurrentNavigableMap)2 ConcurrentSkipListMap (java.util.concurrent.ConcurrentSkipListMap)2