Search in sources :

Example 1 with ProcessorSupplier

use of org.apache.kafka.streams.processor.api.ProcessorSupplier in project kafka by apache.

the class GlobalStreamThreadTest method before.

@Before
public void before() {
    final MaterializedInternal<Object, Object, KeyValueStore<Bytes, byte[]>> materialized = new MaterializedInternal<>(Materialized.with(null, null), new InternalNameProvider() {

        @Override
        public String newProcessorName(final String prefix) {
            return "processorName";
        }

        @Override
        public String newStoreName(final String prefix) {
            return GLOBAL_STORE_NAME;
        }
    }, "store-");
    final ProcessorSupplier<Object, Object, Void, Void> processorSupplier = () -> new ContextualProcessor<Object, Object, Void, Void>() {

        @Override
        public void process(final Record<Object, Object> record) {
        }
    };
    builder.addGlobalStore(new TimestampedKeyValueStoreMaterializer<>(materialized).materialize().withLoggingDisabled(), "sourceName", null, null, null, GLOBAL_STORE_TOPIC_NAME, "processorName", processorSupplier);
    baseDirectoryName = TestUtils.tempDirectory().getAbsolutePath();
    final HashMap<String, Object> properties = new HashMap<>();
    properties.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "blah");
    properties.put(StreamsConfig.APPLICATION_ID_CONFIG, "testAppId");
    properties.put(StreamsConfig.STATE_DIR_CONFIG, baseDirectoryName);
    properties.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.ByteArraySerde.class.getName());
    properties.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.ByteArraySerde.class.getName());
    config = new StreamsConfig(properties);
    globalStreamThread = new GlobalStreamThread(builder.rewriteTopology(config).buildGlobalStateTopology(), config, mockConsumer, new StateDirectory(config, time, true, false), 0, new StreamsMetricsImpl(new Metrics(), "test-client", StreamsConfig.METRICS_LATEST, time), time, "clientId", stateRestoreListener, e -> {
    });
}
Also used : CoreMatchers.is(org.hamcrest.CoreMatchers.is) StreamsConfig(org.apache.kafka.streams.StreamsConfig) MockTime(org.apache.kafka.common.utils.MockTime) InternalNameProvider(org.apache.kafka.streams.kstream.internals.InternalNameProvider) IsInstanceOf.instanceOf(org.hamcrest.core.IsInstanceOf.instanceOf) MockConsumer(org.apache.kafka.clients.consumer.MockConsumer) CoreMatchers.equalTo(org.hamcrest.CoreMatchers.equalTo) HashMap(java.util.HashMap) StreamsException(org.apache.kafka.streams.errors.StreamsException) OffsetResetStrategy(org.apache.kafka.clients.consumer.OffsetResetStrategy) DEAD(org.apache.kafka.streams.processor.internals.GlobalStreamThread.State.DEAD) ContextualProcessor(org.apache.kafka.streams.processor.api.ContextualProcessor) ProcessorSupplier(org.apache.kafka.streams.processor.api.ProcessorSupplier) KeyValueStore(org.apache.kafka.streams.state.KeyValueStore) Serdes(org.apache.kafka.common.serialization.Serdes) Record(org.apache.kafka.streams.processor.api.Record) StreamsMetricsImpl(org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl) Assert.fail(org.junit.Assert.fail) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Before(org.junit.Before) TopicPartition(org.apache.kafka.common.TopicPartition) RUNNING(org.apache.kafka.streams.processor.internals.GlobalStreamThread.State.RUNNING) MockStateRestoreListener(org.apache.kafka.test.MockStateRestoreListener) TestUtils(org.apache.kafka.test.TestUtils) Set(java.util.Set) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) PartitionInfo(org.apache.kafka.common.PartitionInfo) MaterializedInternal(org.apache.kafka.streams.kstream.internals.MaterializedInternal) File(java.io.File) Bytes(org.apache.kafka.common.utils.Bytes) ConsumerRecordUtil.record(org.apache.kafka.streams.processor.internals.testutil.ConsumerRecordUtil.record) List(java.util.List) Metrics(org.apache.kafka.common.metrics.Metrics) TimestampedKeyValueStoreMaterializer(org.apache.kafka.streams.kstream.internals.TimestampedKeyValueStoreMaterializer) StateStore(org.apache.kafka.streams.processor.StateStore) Assert.assertFalse(org.junit.Assert.assertFalse) Materialized(org.apache.kafka.streams.kstream.Materialized) InvalidOffsetException(org.apache.kafka.clients.consumer.InvalidOffsetException) Node(org.apache.kafka.common.Node) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) HashMap(java.util.HashMap) KeyValueStore(org.apache.kafka.streams.state.KeyValueStore) InternalNameProvider(org.apache.kafka.streams.kstream.internals.InternalNameProvider) Metrics(org.apache.kafka.common.metrics.Metrics) Record(org.apache.kafka.streams.processor.api.Record) ContextualProcessor(org.apache.kafka.streams.processor.api.ContextualProcessor) MaterializedInternal(org.apache.kafka.streams.kstream.internals.MaterializedInternal) StreamsMetricsImpl(org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl) StreamsConfig(org.apache.kafka.streams.StreamsConfig) Before(org.junit.Before)

Example 2 with ProcessorSupplier

use of org.apache.kafka.streams.processor.api.ProcessorSupplier in project kafka by apache.

the class GraphGraceSearchUtil method extractGracePeriod.

@SuppressWarnings("rawtypes")
private static Long extractGracePeriod(final GraphNode node) {
    if (node instanceof StatefulProcessorNode) {
        final ProcessorSupplier processorSupplier = ((StatefulProcessorNode) node).processorParameters().processorSupplier();
        if (processorSupplier instanceof KStreamWindowAggregate) {
            final KStreamWindowAggregate kStreamWindowAggregate = (KStreamWindowAggregate) processorSupplier;
            final Windows windows = kStreamWindowAggregate.windows();
            return windows.gracePeriodMs();
        } else if (processorSupplier instanceof KStreamSessionWindowAggregate) {
            final KStreamSessionWindowAggregate kStreamSessionWindowAggregate = (KStreamSessionWindowAggregate) processorSupplier;
            final SessionWindows windows = kStreamSessionWindowAggregate.windows();
            return windows.gracePeriodMs() + windows.inactivityGap();
        } else if (processorSupplier instanceof KStreamSlidingWindowAggregate) {
            final KStreamSlidingWindowAggregate kStreamSlidingWindowAggregate = (KStreamSlidingWindowAggregate) processorSupplier;
            final SlidingWindows windows = kStreamSlidingWindowAggregate.windows();
            return windows.gracePeriodMs();
        } else {
            return null;
        }
    } else {
        return null;
    }
}
Also used : SessionWindows(org.apache.kafka.streams.kstream.SessionWindows) ProcessorSupplier(org.apache.kafka.streams.processor.api.ProcessorSupplier) SessionWindows(org.apache.kafka.streams.kstream.SessionWindows) Windows(org.apache.kafka.streams.kstream.Windows) SlidingWindows(org.apache.kafka.streams.kstream.SlidingWindows) KStreamSessionWindowAggregate(org.apache.kafka.streams.kstream.internals.KStreamSessionWindowAggregate) KStreamWindowAggregate(org.apache.kafka.streams.kstream.internals.KStreamWindowAggregate) KStreamSlidingWindowAggregate(org.apache.kafka.streams.kstream.internals.KStreamSlidingWindowAggregate) SlidingWindows(org.apache.kafka.streams.kstream.SlidingWindows)

Aggregations

ProcessorSupplier (org.apache.kafka.streams.processor.api.ProcessorSupplier)2 File (java.io.File)1 Collections (java.util.Collections)1 HashMap (java.util.HashMap)1 List (java.util.List)1 Set (java.util.Set)1 InvalidOffsetException (org.apache.kafka.clients.consumer.InvalidOffsetException)1 MockConsumer (org.apache.kafka.clients.consumer.MockConsumer)1 OffsetResetStrategy (org.apache.kafka.clients.consumer.OffsetResetStrategy)1 Node (org.apache.kafka.common.Node)1 PartitionInfo (org.apache.kafka.common.PartitionInfo)1 TopicPartition (org.apache.kafka.common.TopicPartition)1 Metrics (org.apache.kafka.common.metrics.Metrics)1 Serdes (org.apache.kafka.common.serialization.Serdes)1 Bytes (org.apache.kafka.common.utils.Bytes)1 MockTime (org.apache.kafka.common.utils.MockTime)1 StreamsConfig (org.apache.kafka.streams.StreamsConfig)1 StreamsException (org.apache.kafka.streams.errors.StreamsException)1 Materialized (org.apache.kafka.streams.kstream.Materialized)1 SessionWindows (org.apache.kafka.streams.kstream.SessionWindows)1