Search in sources :

Example 1 with ContextualProcessor

use of org.apache.kafka.streams.processor.api.ContextualProcessor in project kafka by apache.

the class GlobalStreamThreadTest method before.

@Before
public void before() {
    final MaterializedInternal<Object, Object, KeyValueStore<Bytes, byte[]>> materialized = new MaterializedInternal<>(Materialized.with(null, null), new InternalNameProvider() {

        @Override
        public String newProcessorName(final String prefix) {
            return "processorName";
        }

        @Override
        public String newStoreName(final String prefix) {
            return GLOBAL_STORE_NAME;
        }
    }, "store-");
    final ProcessorSupplier<Object, Object, Void, Void> processorSupplier = () -> new ContextualProcessor<Object, Object, Void, Void>() {

        @Override
        public void process(final Record<Object, Object> record) {
        }
    };
    builder.addGlobalStore(new TimestampedKeyValueStoreMaterializer<>(materialized).materialize().withLoggingDisabled(), "sourceName", null, null, null, GLOBAL_STORE_TOPIC_NAME, "processorName", processorSupplier);
    baseDirectoryName = TestUtils.tempDirectory().getAbsolutePath();
    final HashMap<String, Object> properties = new HashMap<>();
    properties.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "blah");
    properties.put(StreamsConfig.APPLICATION_ID_CONFIG, "testAppId");
    properties.put(StreamsConfig.STATE_DIR_CONFIG, baseDirectoryName);
    properties.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.ByteArraySerde.class.getName());
    properties.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.ByteArraySerde.class.getName());
    config = new StreamsConfig(properties);
    globalStreamThread = new GlobalStreamThread(builder.rewriteTopology(config).buildGlobalStateTopology(), config, mockConsumer, new StateDirectory(config, time, true, false), 0, new StreamsMetricsImpl(new Metrics(), "test-client", StreamsConfig.METRICS_LATEST, time), time, "clientId", stateRestoreListener, e -> {
    });
}
Also used : CoreMatchers.is(org.hamcrest.CoreMatchers.is) StreamsConfig(org.apache.kafka.streams.StreamsConfig) MockTime(org.apache.kafka.common.utils.MockTime) InternalNameProvider(org.apache.kafka.streams.kstream.internals.InternalNameProvider) IsInstanceOf.instanceOf(org.hamcrest.core.IsInstanceOf.instanceOf) MockConsumer(org.apache.kafka.clients.consumer.MockConsumer) CoreMatchers.equalTo(org.hamcrest.CoreMatchers.equalTo) HashMap(java.util.HashMap) StreamsException(org.apache.kafka.streams.errors.StreamsException) OffsetResetStrategy(org.apache.kafka.clients.consumer.OffsetResetStrategy) DEAD(org.apache.kafka.streams.processor.internals.GlobalStreamThread.State.DEAD) ContextualProcessor(org.apache.kafka.streams.processor.api.ContextualProcessor) ProcessorSupplier(org.apache.kafka.streams.processor.api.ProcessorSupplier) KeyValueStore(org.apache.kafka.streams.state.KeyValueStore) Serdes(org.apache.kafka.common.serialization.Serdes) Record(org.apache.kafka.streams.processor.api.Record) StreamsMetricsImpl(org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl) Assert.fail(org.junit.Assert.fail) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Before(org.junit.Before) TopicPartition(org.apache.kafka.common.TopicPartition) RUNNING(org.apache.kafka.streams.processor.internals.GlobalStreamThread.State.RUNNING) MockStateRestoreListener(org.apache.kafka.test.MockStateRestoreListener) TestUtils(org.apache.kafka.test.TestUtils) Set(java.util.Set) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) PartitionInfo(org.apache.kafka.common.PartitionInfo) MaterializedInternal(org.apache.kafka.streams.kstream.internals.MaterializedInternal) File(java.io.File) Bytes(org.apache.kafka.common.utils.Bytes) ConsumerRecordUtil.record(org.apache.kafka.streams.processor.internals.testutil.ConsumerRecordUtil.record) List(java.util.List) Metrics(org.apache.kafka.common.metrics.Metrics) TimestampedKeyValueStoreMaterializer(org.apache.kafka.streams.kstream.internals.TimestampedKeyValueStoreMaterializer) StateStore(org.apache.kafka.streams.processor.StateStore) Assert.assertFalse(org.junit.Assert.assertFalse) Materialized(org.apache.kafka.streams.kstream.Materialized) InvalidOffsetException(org.apache.kafka.clients.consumer.InvalidOffsetException) Node(org.apache.kafka.common.Node) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) HashMap(java.util.HashMap) KeyValueStore(org.apache.kafka.streams.state.KeyValueStore) InternalNameProvider(org.apache.kafka.streams.kstream.internals.InternalNameProvider) Metrics(org.apache.kafka.common.metrics.Metrics) Record(org.apache.kafka.streams.processor.api.Record) ContextualProcessor(org.apache.kafka.streams.processor.api.ContextualProcessor) MaterializedInternal(org.apache.kafka.streams.kstream.internals.MaterializedInternal) StreamsMetricsImpl(org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl) StreamsConfig(org.apache.kafka.streams.StreamsConfig) Before(org.junit.Before)

Example 2 with ContextualProcessor

use of org.apache.kafka.streams.processor.api.ContextualProcessor in project kafka by apache.

the class SubscriptionStoreReceiveProcessorSupplier method get.

@Override
public Processor<KO, SubscriptionWrapper<K>, CombinedKey<KO, K>, Change<ValueAndTimestamp<SubscriptionWrapper<K>>>> get() {
    return new ContextualProcessor<KO, SubscriptionWrapper<K>, CombinedKey<KO, K>, Change<ValueAndTimestamp<SubscriptionWrapper<K>>>>() {

        private TimestampedKeyValueStore<Bytes, SubscriptionWrapper<K>> store;

        private Sensor droppedRecordsSensor;

        @Override
        public void init(final ProcessorContext<CombinedKey<KO, K>, Change<ValueAndTimestamp<SubscriptionWrapper<K>>>> context) {
            super.init(context);
            final InternalProcessorContext<?, ?> internalProcessorContext = (InternalProcessorContext<?, ?>) context;
            droppedRecordsSensor = TaskMetrics.droppedRecordsSensor(Thread.currentThread().getName(), internalProcessorContext.taskId().toString(), internalProcessorContext.metrics());
            store = internalProcessorContext.getStateStore(storeBuilder);
            keySchema.init(context);
        }

        @Override
        public void process(final Record<KO, SubscriptionWrapper<K>> record) {
            if (record.key() == null) {
                if (context().recordMetadata().isPresent()) {
                    final RecordMetadata recordMetadata = context().recordMetadata().get();
                    LOG.warn("Skipping record due to null foreign key. " + "topic=[{}] partition=[{}] offset=[{}]", recordMetadata.topic(), recordMetadata.partition(), recordMetadata.offset());
                } else {
                    LOG.warn("Skipping record due to null foreign key. Topic, partition, and offset not known.");
                }
                droppedRecordsSensor.record();
                return;
            }
            if (record.value().getVersion() != SubscriptionWrapper.CURRENT_VERSION) {
                // from older SubscriptionWrapper versions to newer versions.
                throw new UnsupportedVersionException("SubscriptionWrapper is of an incompatible version.");
            }
            final Bytes subscriptionKey = keySchema.toBytes(record.key(), record.value().getPrimaryKey());
            final ValueAndTimestamp<SubscriptionWrapper<K>> newValue = ValueAndTimestamp.make(record.value(), record.timestamp());
            final ValueAndTimestamp<SubscriptionWrapper<K>> oldValue = store.get(subscriptionKey);
            // This store is used by the prefix scanner in ForeignJoinSubscriptionProcessorSupplier
            if (record.value().getInstruction().equals(SubscriptionWrapper.Instruction.DELETE_KEY_AND_PROPAGATE) || record.value().getInstruction().equals(SubscriptionWrapper.Instruction.DELETE_KEY_NO_PROPAGATE)) {
                store.delete(subscriptionKey);
            } else {
                store.put(subscriptionKey, newValue);
            }
            final Change<ValueAndTimestamp<SubscriptionWrapper<K>>> change = new Change<>(newValue, oldValue);
            // note: key is non-nullable
            // note: newValue is non-nullable
            context().forward(record.withKey(new CombinedKey<>(record.key(), record.value().getPrimaryKey())).withValue(change).withTimestamp(newValue.timestamp()));
        }
    };
}
Also used : InternalProcessorContext(org.apache.kafka.streams.processor.internals.InternalProcessorContext) Change(org.apache.kafka.streams.kstream.internals.Change) ProcessorContext(org.apache.kafka.streams.processor.api.ProcessorContext) InternalProcessorContext(org.apache.kafka.streams.processor.internals.InternalProcessorContext) ValueAndTimestamp(org.apache.kafka.streams.state.ValueAndTimestamp) RecordMetadata(org.apache.kafka.streams.processor.api.RecordMetadata) Bytes(org.apache.kafka.common.utils.Bytes) TimestampedKeyValueStore(org.apache.kafka.streams.state.TimestampedKeyValueStore) Record(org.apache.kafka.streams.processor.api.Record) ContextualProcessor(org.apache.kafka.streams.processor.api.ContextualProcessor) Sensor(org.apache.kafka.common.metrics.Sensor) UnsupportedVersionException(org.apache.kafka.common.errors.UnsupportedVersionException)

Aggregations

Bytes (org.apache.kafka.common.utils.Bytes)2 ContextualProcessor (org.apache.kafka.streams.processor.api.ContextualProcessor)2 Record (org.apache.kafka.streams.processor.api.Record)2 File (java.io.File)1 Collections (java.util.Collections)1 HashMap (java.util.HashMap)1 List (java.util.List)1 Set (java.util.Set)1 InvalidOffsetException (org.apache.kafka.clients.consumer.InvalidOffsetException)1 MockConsumer (org.apache.kafka.clients.consumer.MockConsumer)1 OffsetResetStrategy (org.apache.kafka.clients.consumer.OffsetResetStrategy)1 Node (org.apache.kafka.common.Node)1 PartitionInfo (org.apache.kafka.common.PartitionInfo)1 TopicPartition (org.apache.kafka.common.TopicPartition)1 UnsupportedVersionException (org.apache.kafka.common.errors.UnsupportedVersionException)1 Metrics (org.apache.kafka.common.metrics.Metrics)1 Sensor (org.apache.kafka.common.metrics.Sensor)1 Serdes (org.apache.kafka.common.serialization.Serdes)1 MockTime (org.apache.kafka.common.utils.MockTime)1 StreamsConfig (org.apache.kafka.streams.StreamsConfig)1