Examples with StreamsConfig org.apache.kafka.streams.StreamsConfig used on opensource projects

Example 91 with StreamsConfig

use of org.apache.kafka.streams.StreamsConfig in project kafka by apache.

the class InternalTopologyBuilderTest method shouldOverrideGlobalStreamsConfigWhenGivenNamedTopologyProps.

@Test
public void shouldOverrideGlobalStreamsConfigWhenGivenNamedTopologyProps() {
    final Properties topologyOverrides = new Properties();
    topologyOverrides.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 12345L);
    topologyOverrides.put(StreamsConfig.MAX_TASK_IDLE_MS_CONFIG, 500L);
    topologyOverrides.put(StreamsConfig.TASK_TIMEOUT_MS_CONFIG, 1000L);
    topologyOverrides.put(StreamsConfig.BUFFERED_RECORDS_PER_PARTITION_CONFIG, 15);
    topologyOverrides.put(StreamsConfig.DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG, MockTimestampExtractor.class);
    topologyOverrides.put(StreamsConfig.DEFAULT_DESERIALIZATION_EXCEPTION_HANDLER_CLASS_CONFIG, LogAndContinueExceptionHandler.class);
    final StreamsConfig config = new StreamsConfig(StreamsTestUtils.getStreamsConfig());
    final InternalTopologyBuilder topologyBuilder = new InternalTopologyBuilder(new TopologyConfig("my-topology", config, topologyOverrides));
    assertThat(topologyBuilder.topologyConfigs().cacheSize, is(12345L));
    assertThat(topologyBuilder.topologyConfigs().getTaskConfig().maxTaskIdleMs, equalTo(500L));
    assertThat(topologyBuilder.topologyConfigs().getTaskConfig().taskTimeoutMs, equalTo(1000L));
    assertThat(topologyBuilder.topologyConfigs().getTaskConfig().maxBufferedSize, equalTo(15));
    assertThat(topologyBuilder.topologyConfigs().getTaskConfig().timestampExtractor.getClass(), equalTo(MockTimestampExtractor.class));
    assertThat(topologyBuilder.topologyConfigs().getTaskConfig().deserializationExceptionHandler.getClass(), equalTo(LogAndContinueExceptionHandler.class));
}

Example 92 with StreamsConfig

use of org.apache.kafka.streams.StreamsConfig in project kafka by apache.

the class InternalTopologyBuilderTest method shouldNotOverrideGlobalStreamsConfigWhenGivenUnnamedTopologyProps.

@Test
public void shouldNotOverrideGlobalStreamsConfigWhenGivenUnnamedTopologyProps() {
    final Properties streamsProps = StreamsTestUtils.getStreamsConfig();
    streamsProps.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 12345L);
    streamsProps.put(StreamsConfig.MAX_TASK_IDLE_MS_CONFIG, 500L);
    streamsProps.put(StreamsConfig.TASK_TIMEOUT_MS_CONFIG, 1000L);
    streamsProps.put(StreamsConfig.BUFFERED_RECORDS_PER_PARTITION_CONFIG, 15);
    streamsProps.put(StreamsConfig.DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG, MockTimestampExtractor.class);
    streamsProps.put(StreamsConfig.DEFAULT_DESERIALIZATION_EXCEPTION_HANDLER_CLASS_CONFIG, LogAndContinueExceptionHandler.class);
    final StreamsConfig config = new StreamsConfig(streamsProps);
    final InternalTopologyBuilder topologyBuilder = new InternalTopologyBuilder(new TopologyConfig("my-topology", config, new Properties()));
    assertThat(topologyBuilder.topologyConfigs().cacheSize, is(12345L));
    assertThat(topologyBuilder.topologyConfigs().getTaskConfig().maxTaskIdleMs, is(500L));
    assertThat(topologyBuilder.topologyConfigs().getTaskConfig().taskTimeoutMs, is(1000L));
    assertThat(topologyBuilder.topologyConfigs().getTaskConfig().maxBufferedSize, is(15));
    assertThat(topologyBuilder.topologyConfigs().getTaskConfig().timestampExtractor.getClass(), is(MockTimestampExtractor.class));
    assertThat(topologyBuilder.topologyConfigs().getTaskConfig().deserializationExceptionHandler.getClass(), is(LogAndContinueExceptionHandler.class));
}

Example 93 with StreamsConfig

use of org.apache.kafka.streams.StreamsConfig in project kafka by apache.

the class InternalTopologyBuilderTest method shouldAddTimestampExtractorPerSource.

@Test
public void shouldAddTimestampExtractorPerSource() {
    builder.addSource(null, "source", new MockTimestampExtractor(), null, null, "topic");
    final ProcessorTopology processorTopology = builder.rewriteTopology(new StreamsConfig(StreamsTestUtils.getStreamsConfig())).buildTopology();
    assertThat(processorTopology.source("topic").getTimestampExtractor(), instanceOf(MockTimestampExtractor.class));
}

Example 94 with StreamsConfig

use of org.apache.kafka.streams.StreamsConfig in project kafka by apache.

the class GlobalStateManagerImplTest method shouldNotRetryWhenPartitionsForThrowsTimeoutExceptionAndTaskTimeoutIsZero.

@Test
public void shouldNotRetryWhenPartitionsForThrowsTimeoutExceptionAndTaskTimeoutIsZero() {
    final AtomicInteger numberOfCalls = new AtomicInteger(0);
    consumer = new MockConsumer<byte[], byte[]>(OffsetResetStrategy.EARLIEST) {

        @Override
        public List<PartitionInfo> partitionsFor(final String topic) {
            numberOfCalls.incrementAndGet();
            throw new TimeoutException("KABOOM!");
        }
    };
    initializeConsumer(0, 0, t1, t2, t3, t4);
    streamsConfig = new StreamsConfig(mkMap(mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, "appId"), mkEntry(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "dummy:1234"), mkEntry(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath()), mkEntry(StreamsConfig.TASK_TIMEOUT_MS_CONFIG, 0L)));
    stateManager = new GlobalStateManagerImpl(new LogContext("mock"), time, topology, consumer, stateDirectory, stateRestoreListener, streamsConfig);
    processorContext.setStateManger(stateManager);
    stateManager.setGlobalProcessorContext(processorContext);
    final StreamsException expected = assertThrows(StreamsException.class, () -> stateManager.initialize());
    final Throwable cause = expected.getCause();
    assertThat(cause, instanceOf(TimeoutException.class));
    assertThat(cause.getMessage(), equalTo("KABOOM!"));
    assertEquals(numberOfCalls.get(), 1);
}

Example 95 with StreamsConfig

use of org.apache.kafka.streams.StreamsConfig in project kafka by apache.

the class GlobalStateManagerImplTest method shouldRetryWhenPartitionsForThrowsTimeoutExceptionUntilTaskTimeoutExpires.

@Test
public void shouldRetryWhenPartitionsForThrowsTimeoutExceptionUntilTaskTimeoutExpires() {
    final AtomicInteger numberOfCalls = new AtomicInteger(0);
    consumer = new MockConsumer<byte[], byte[]>(OffsetResetStrategy.EARLIEST) {

        @Override
        public List<PartitionInfo> partitionsFor(final String topic) {
            time.sleep(100L);
            numberOfCalls.incrementAndGet();
            throw new TimeoutException("KABOOM!");
        }
    };
    initializeConsumer(0, 0, t1, t2, t3, t4);
    streamsConfig = new StreamsConfig(mkMap(mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, "appId"), mkEntry(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "dummy:1234"), mkEntry(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath()), mkEntry(StreamsConfig.TASK_TIMEOUT_MS_CONFIG, 1000L)));
    stateManager = new GlobalStateManagerImpl(new LogContext("mock"), time, topology, consumer, stateDirectory, stateRestoreListener, streamsConfig);
    processorContext.setStateManger(stateManager);
    stateManager.setGlobalProcessorContext(processorContext);
    final TimeoutException expected = assertThrows(TimeoutException.class, () -> stateManager.initialize());
    assertThat(expected.getMessage(), equalTo("Global task did not make progress to restore state within 1000 ms. Adjust `task.timeout.ms` if needed."));
    assertEquals(numberOfCalls.get(), 11);
}