Examples with StreamsConfig org.apache.kafka.streams.StreamsConfig used on opensource projects

Example 96 with StreamsConfig

use of org.apache.kafka.streams.StreamsConfig in project kafka by apache.

the class GlobalStateManagerImplTest method shouldRetryAtLeastOnceWhenPartitionsForThrowsTimeoutException.

@Test
public void shouldRetryAtLeastOnceWhenPartitionsForThrowsTimeoutException() {
    final AtomicInteger numberOfCalls = new AtomicInteger(0);
    consumer = new MockConsumer<byte[], byte[]>(OffsetResetStrategy.EARLIEST) {

        @Override
        public List<PartitionInfo> partitionsFor(final String topic) {
            time.sleep(100L);
            numberOfCalls.incrementAndGet();
            throw new TimeoutException("KABOOM!");
        }
    };
    initializeConsumer(0, 0, t1, t2, t3, t4);
    streamsConfig = new StreamsConfig(mkMap(mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, "appId"), mkEntry(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "dummy:1234"), mkEntry(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath()), mkEntry(StreamsConfig.TASK_TIMEOUT_MS_CONFIG, 1L)));
    stateManager = new GlobalStateManagerImpl(new LogContext("mock"), time, topology, consumer, stateDirectory, stateRestoreListener, streamsConfig);
    processorContext.setStateManger(stateManager);
    stateManager.setGlobalProcessorContext(processorContext);
    final TimeoutException expected = assertThrows(TimeoutException.class, () -> stateManager.initialize());
    assertThat(expected.getMessage(), equalTo("Global task did not make progress to restore state within 100 ms. Adjust `task.timeout.ms` if needed."));
    assertEquals(numberOfCalls.get(), 2);
}

Example 97 with StreamsConfig

use of org.apache.kafka.streams.StreamsConfig in project kafka by apache.

the class AbstractRocksDBSegmentedBytesStoreTest method shouldRestoreRecordsAndConsistencyVectorMultipleTopics.

@Test
public void shouldRestoreRecordsAndConsistencyVectorMultipleTopics() {
    final Properties props = StreamsTestUtils.getStreamsConfig();
    props.put(InternalConfig.IQ_CONSISTENCY_OFFSET_VECTOR_ENABLED, true);
    final File dir = TestUtils.tempDirectory();
    context = new InternalMockProcessorContext<>(dir, Serdes.String(), Serdes.String(), new StreamsMetricsImpl(new Metrics(), "mock", StreamsConfig.METRICS_LATEST, new MockTime()), new StreamsConfig(props), MockRecordCollector::new, new ThreadCache(new LogContext("testCache "), 0, new MockStreamsMetrics(new Metrics())), Time.SYSTEM);
    bytesStore = getBytesStore();
    bytesStore.init((StateStoreContext) context, bytesStore);
    // 0 segments initially.
    assertEquals(0, bytesStore.getSegments().size());
    bytesStore.restoreAllInternal(getChangelogRecordsMultipleTopics());
    // 2 segments are created during restoration.
    assertEquals(2, bytesStore.getSegments().size());
    final String key = "a";
    final List<KeyValue<Windowed<String>, Long>> expected = new ArrayList<>();
    expected.add(new KeyValue<>(new Windowed<>(key, windows[0]), 50L));
    expected.add(new KeyValue<>(new Windowed<>(key, windows[2]), 100L));
    expected.add(new KeyValue<>(new Windowed<>(key, windows[3]), 200L));
    final List<KeyValue<Windowed<String>, Long>> results = toList(bytesStore.all());
    assertEquals(expected, results);
    assertThat(bytesStore.getPosition(), Matchers.notNullValue());
    assertThat(bytesStore.getPosition().getPartitionPositions("A"), Matchers.notNullValue());
    assertThat(bytesStore.getPosition().getPartitionPositions("A"), hasEntry(0, 3L));
    assertThat(bytesStore.getPosition().getPartitionPositions("B"), Matchers.notNullValue());
    assertThat(bytesStore.getPosition().getPartitionPositions("B"), hasEntry(0, 2L));
}

Example 98 with StreamsConfig

use of org.apache.kafka.streams.StreamsConfig in project kafka by apache.

the class AbstractRocksDBSegmentedBytesStoreTest method shouldNotThrowWhenRestoringOnMissingHeaders.

@Test
public void shouldNotThrowWhenRestoringOnMissingHeaders() {
    final Properties props = StreamsTestUtils.getStreamsConfig();
    props.put(InternalConfig.IQ_CONSISTENCY_OFFSET_VECTOR_ENABLED, true);
    final File dir = TestUtils.tempDirectory();
    context = new InternalMockProcessorContext<>(dir, Serdes.String(), Serdes.String(), new StreamsMetricsImpl(new Metrics(), "mock", StreamsConfig.METRICS_LATEST, new MockTime()), new StreamsConfig(props), MockRecordCollector::new, new ThreadCache(new LogContext("testCache "), 0, new MockStreamsMetrics(new Metrics())), Time.SYSTEM);
    bytesStore = getBytesStore();
    bytesStore.init((StateStoreContext) context, bytesStore);
    bytesStore.restoreAllInternal(getChangelogRecordsWithoutHeaders());
    assertThat(bytesStore.getPosition(), is(Position.emptyPosition()));
}

Example 99 with StreamsConfig

use of org.apache.kafka.streams.StreamsConfig in project kafka by apache.

the class AbstractWindowBytesStoreTest method shouldLogAndMeasureExpiredRecords.

@Test
public void shouldLogAndMeasureExpiredRecords() {
    final Properties streamsConfig = StreamsTestUtils.getStreamsConfig();
    final WindowStore<Integer, String> windowStore = buildWindowStore(RETENTION_PERIOD, WINDOW_SIZE, false, Serdes.Integer(), Serdes.String());
    final InternalMockProcessorContext context = new InternalMockProcessorContext(TestUtils.tempDirectory(), new StreamsConfig(streamsConfig), recordCollector);
    final Time time = new SystemTime();
    context.setSystemTimeMs(time.milliseconds());
    context.setTime(1L);
    windowStore.init((StateStoreContext) context, windowStore);
    try (final LogCaptureAppender appender = LogCaptureAppender.createAndRegister()) {
        // Advance stream time by inserting record with large enough timestamp that records with timestamp 0 are expired
        windowStore.put(1, "initial record", 2 * RETENTION_PERIOD);
        // Try inserting a record with timestamp 0 -- should be dropped
        windowStore.put(1, "late record", 0L);
        windowStore.put(1, "another on-time record", RETENTION_PERIOD + 1);
        final List<String> messages = appender.getMessages();
        assertThat(messages, hasItem("Skipping record for expired segment."));
    }
    final Map<MetricName, ? extends Metric> metrics = context.metrics().metrics();
    final String threadId = Thread.currentThread().getName();
    final Metric dropTotal;
    final Metric dropRate;
    dropTotal = metrics.get(new MetricName("dropped-records-total", "stream-task-metrics", "", mkMap(mkEntry("thread-id", threadId), mkEntry("task-id", "0_0"))));
    dropRate = metrics.get(new MetricName("dropped-records-rate", "stream-task-metrics", "", mkMap(mkEntry("thread-id", threadId), mkEntry("task-id", "0_0"))));
    assertEquals(1.0, dropTotal.metricValue());
    assertNotEquals(0.0, dropRate.metricValue());
    windowStore.close();
}

Example 100 with StreamsConfig

use of org.apache.kafka.streams.StreamsConfig in project kafka by apache.

the class ChangeLoggingKeyValueBytesStoreTest method streamsConfigMock.

private StreamsConfig streamsConfigMock() {
    final StreamsConfig streamsConfig = mock(StreamsConfig.class);
    final Map<String, Object> myValues = new HashMap<>();
    myValues.put(InternalConfig.IQ_CONSISTENCY_OFFSET_VECTOR_ENABLED, true);
    expect(streamsConfig.originals()).andStubReturn(myValues);
    expect(streamsConfig.values()).andStubReturn(Collections.emptyMap());
    expect(streamsConfig.getString(StreamsConfig.APPLICATION_ID_CONFIG)).andStubReturn("add-id");
    expect(streamsConfig.defaultValueSerde()).andStubReturn(Serdes.ByteArray());
    expect(streamsConfig.defaultKeySerde()).andStubReturn(Serdes.ByteArray());
    replay(streamsConfig);
    return streamsConfig;
}