Search in sources :

Example 1 with IQ_CONSISTENCY_OFFSET_VECTOR_ENABLED

use of org.apache.kafka.streams.StreamsConfig.InternalConfig.IQ_CONSISTENCY_OFFSET_VECTOR_ENABLED in project kafka by apache.

the class InMemoryWindowStore method init.

@Deprecated
@Override
public void init(final ProcessorContext context, final StateStore root) {
    this.context = context;
    final StreamsMetricsImpl metrics = ProcessorContextUtils.getMetricsImpl(context);
    final String threadId = Thread.currentThread().getName();
    final String taskName = context.taskId().toString();
    expiredRecordSensor = TaskMetrics.droppedRecordsSensor(threadId, taskName, metrics);
    if (root != null) {
        final boolean consistencyEnabled = StreamsConfig.InternalConfig.getBoolean(context.appConfigs(), IQ_CONSISTENCY_OFFSET_VECTOR_ENABLED, false);
        context.register(root, (RecordBatchingStateRestoreCallback) records -> {
            for (final ConsumerRecord<byte[], byte[]> record : records) {
                put(Bytes.wrap(extractStoreKeyBytes(record.key())), record.value(), extractStoreTimestamp(record.key()));
                ChangelogRecordDeserializationHelper.applyChecksAndUpdatePosition(record, consistencyEnabled, position);
            }
        });
    }
    open = true;
}
Also used : StreamsConfig(org.apache.kafka.streams.StreamsConfig) TaskMetrics(org.apache.kafka.streams.processor.internals.metrics.TaskMetrics) LoggerFactory(org.slf4j.LoggerFactory) PositionBound(org.apache.kafka.streams.query.PositionBound) RecordBatchingStateRestoreCallback(org.apache.kafka.streams.processor.internals.RecordBatchingStateRestoreCallback) WindowStore(org.apache.kafka.streams.state.WindowStore) ByteBuffer(java.nio.ByteBuffer) StoreToProcessorContextAdapter(org.apache.kafka.streams.processor.internals.StoreToProcessorContextAdapter) StateStoreContext(org.apache.kafka.streams.processor.StateStoreContext) Windowed(org.apache.kafka.streams.kstream.Windowed) Map(java.util.Map) ProcessorContextUtils(org.apache.kafka.streams.processor.internals.ProcessorContextUtils) StreamsMetricsImpl(org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl) NoSuchElementException(java.util.NoSuchElementException) QueryResult(org.apache.kafka.streams.query.QueryResult) Sensor(org.apache.kafka.common.metrics.Sensor) QueryConfig(org.apache.kafka.streams.query.QueryConfig) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) Position(org.apache.kafka.streams.query.Position) Query(org.apache.kafka.streams.query.Query) WindowKeySchema.extractStoreTimestamp(org.apache.kafka.streams.state.internals.WindowKeySchema.extractStoreTimestamp) ConcurrentNavigableMap(java.util.concurrent.ConcurrentNavigableMap) KeyValue(org.apache.kafka.streams.KeyValue) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) ChangelogRecordDeserializationHelper(org.apache.kafka.streams.processor.internals.ChangelogRecordDeserializationHelper) Bytes(org.apache.kafka.common.utils.Bytes) Objects(java.util.Objects) ProcessorContext(org.apache.kafka.streams.processor.ProcessorContext) KeyValueIterator(org.apache.kafka.streams.state.KeyValueIterator) ConcurrentSkipListMap(java.util.concurrent.ConcurrentSkipListMap) WindowKeySchema.extractStoreKeyBytes(org.apache.kafka.streams.state.internals.WindowKeySchema.extractStoreKeyBytes) StateStore(org.apache.kafka.streams.processor.StateStore) WindowStoreIterator(org.apache.kafka.streams.state.WindowStoreIterator) ConsumerRecord(org.apache.kafka.clients.consumer.ConsumerRecord) TimeWindow(org.apache.kafka.streams.kstream.internals.TimeWindow) IQ_CONSISTENCY_OFFSET_VECTOR_ENABLED(org.apache.kafka.streams.StreamsConfig.InternalConfig.IQ_CONSISTENCY_OFFSET_VECTOR_ENABLED) StreamsMetricsImpl(org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl) ConsumerRecord(org.apache.kafka.clients.consumer.ConsumerRecord)

Example 2 with IQ_CONSISTENCY_OFFSET_VECTOR_ENABLED

use of org.apache.kafka.streams.StreamsConfig.InternalConfig.IQ_CONSISTENCY_OFFSET_VECTOR_ENABLED in project kafka by apache.

the class InMemorySessionStore method init.

@Deprecated
@Override
public void init(final ProcessorContext context, final StateStore root) {
    final String threadId = Thread.currentThread().getName();
    final String taskName = context.taskId().toString();
    // If it doesn't, we can't record this metric.
    if (context instanceof InternalProcessorContext) {
        this.context = (InternalProcessorContext) context;
        final StreamsMetricsImpl metrics = this.context.metrics();
        expiredRecordSensor = TaskMetrics.droppedRecordsSensor(threadId, taskName, metrics);
    } else {
        this.context = null;
        expiredRecordSensor = null;
    }
    if (root != null) {
        final boolean consistencyEnabled = StreamsConfig.InternalConfig.getBoolean(context.appConfigs(), IQ_CONSISTENCY_OFFSET_VECTOR_ENABLED, false);
        context.register(root, (RecordBatchingStateRestoreCallback) records -> {
            for (final ConsumerRecord<byte[], byte[]> record : records) {
                put(SessionKeySchema.from(Bytes.wrap(record.key())), record.value());
                ChangelogRecordDeserializationHelper.applyChecksAndUpdatePosition(record, consistencyEnabled, position);
            }
        });
    }
    open = true;
}
Also used : StreamsConfig(org.apache.kafka.streams.StreamsConfig) TaskMetrics(org.apache.kafka.streams.processor.internals.metrics.TaskMetrics) LoggerFactory(org.slf4j.LoggerFactory) PositionBound(org.apache.kafka.streams.query.PositionBound) RecordBatchingStateRestoreCallback(org.apache.kafka.streams.processor.internals.RecordBatchingStateRestoreCallback) StoreToProcessorContextAdapter(org.apache.kafka.streams.processor.internals.StoreToProcessorContextAdapter) StateStoreContext(org.apache.kafka.streams.processor.StateStoreContext) Windowed(org.apache.kafka.streams.kstream.Windowed) Map(java.util.Map) StreamsMetricsImpl(org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl) SessionStore(org.apache.kafka.streams.state.SessionStore) NoSuchElementException(java.util.NoSuchElementException) QueryResult(org.apache.kafka.streams.query.QueryResult) Sensor(org.apache.kafka.common.metrics.Sensor) QueryConfig(org.apache.kafka.streams.query.QueryConfig) Logger(org.slf4j.Logger) InternalProcessorContext(org.apache.kafka.streams.processor.internals.InternalProcessorContext) Iterator(java.util.Iterator) Position(org.apache.kafka.streams.query.Position) Query(org.apache.kafka.streams.query.Query) ConcurrentNavigableMap(java.util.concurrent.ConcurrentNavigableMap) KeyValue(org.apache.kafka.streams.KeyValue) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) ChangelogRecordDeserializationHelper(org.apache.kafka.streams.processor.internals.ChangelogRecordDeserializationHelper) Bytes(org.apache.kafka.common.utils.Bytes) Objects(java.util.Objects) ProcessorContext(org.apache.kafka.streams.processor.ProcessorContext) KeyValueIterator(org.apache.kafka.streams.state.KeyValueIterator) ConcurrentSkipListMap(java.util.concurrent.ConcurrentSkipListMap) StateStore(org.apache.kafka.streams.processor.StateStore) ConsumerRecord(org.apache.kafka.clients.consumer.ConsumerRecord) Entry(java.util.Map.Entry) SessionWindow(org.apache.kafka.streams.kstream.internals.SessionWindow) IQ_CONSISTENCY_OFFSET_VECTOR_ENABLED(org.apache.kafka.streams.StreamsConfig.InternalConfig.IQ_CONSISTENCY_OFFSET_VECTOR_ENABLED) InternalProcessorContext(org.apache.kafka.streams.processor.internals.InternalProcessorContext) StreamsMetricsImpl(org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl) ConsumerRecord(org.apache.kafka.clients.consumer.ConsumerRecord)

Aggregations

Iterator (java.util.Iterator)2 Map (java.util.Map)2 NoSuchElementException (java.util.NoSuchElementException)2 Objects (java.util.Objects)2 Set (java.util.Set)2 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)2 ConcurrentNavigableMap (java.util.concurrent.ConcurrentNavigableMap)2 ConcurrentSkipListMap (java.util.concurrent.ConcurrentSkipListMap)2 ConsumerRecord (org.apache.kafka.clients.consumer.ConsumerRecord)2 Sensor (org.apache.kafka.common.metrics.Sensor)2 Bytes (org.apache.kafka.common.utils.Bytes)2 KeyValue (org.apache.kafka.streams.KeyValue)2 StreamsConfig (org.apache.kafka.streams.StreamsConfig)2 IQ_CONSISTENCY_OFFSET_VECTOR_ENABLED (org.apache.kafka.streams.StreamsConfig.InternalConfig.IQ_CONSISTENCY_OFFSET_VECTOR_ENABLED)2 Windowed (org.apache.kafka.streams.kstream.Windowed)2 ProcessorContext (org.apache.kafka.streams.processor.ProcessorContext)2 StateStore (org.apache.kafka.streams.processor.StateStore)2 StateStoreContext (org.apache.kafka.streams.processor.StateStoreContext)2 ChangelogRecordDeserializationHelper (org.apache.kafka.streams.processor.internals.ChangelogRecordDeserializationHelper)2 RecordBatchingStateRestoreCallback (org.apache.kafka.streams.processor.internals.RecordBatchingStateRestoreCallback)2