Examples with StreamsMetricsImpl org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl used on opensource projects

Example 41 with StreamsMetricsImpl

use of org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl in project kafka by apache.

the class AbstractRocksDBSegmentedBytesStoreTest method shouldHandleTombstoneRecords.

@Test
public void shouldHandleTombstoneRecords() {
    final Properties props = StreamsTestUtils.getStreamsConfig();
    props.put(InternalConfig.IQ_CONSISTENCY_OFFSET_VECTOR_ENABLED, true);
    final File dir = TestUtils.tempDirectory();
    context = new InternalMockProcessorContext<>(dir, Serdes.String(), Serdes.String(), new StreamsMetricsImpl(new Metrics(), "mock", StreamsConfig.METRICS_LATEST, new MockTime()), new StreamsConfig(props), MockRecordCollector::new, new ThreadCache(new LogContext("testCache "), 0, new MockStreamsMetrics(new Metrics())), Time.SYSTEM);
    bytesStore = getBytesStore();
    bytesStore.init((StateStoreContext) context, bytesStore);
    // 0 segments initially.
    assertEquals(0, bytesStore.getSegments().size());
    bytesStore.restoreAllInternal(getChangelogRecordsWithTombstones());
    // 1 segments are created during restoration.
    assertEquals(1, bytesStore.getSegments().size());
    final String key = "a";
    final List<KeyValue<Windowed<String>, Long>> expected = new ArrayList<>();
    expected.add(new KeyValue<>(new Windowed<>(key, windows[0]), 50L));
    final List<KeyValue<Windowed<String>, Long>> results = toList(bytesStore.all());
    assertEquals(expected, results);
    assertThat(bytesStore.getPosition(), Matchers.notNullValue());
    assertThat(bytesStore.getPosition().getPartitionPositions("A"), hasEntry(0, 2L));
}

Example 42 with StreamsMetricsImpl

use of org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl in project kafka by apache.

the class StreamThreadTest method shouldNotEnforceRebalanceWhenTaskCorruptedExceptionIsThrownForAnInactiveTask.

@Test
@SuppressWarnings("unchecked")
public void shouldNotEnforceRebalanceWhenTaskCorruptedExceptionIsThrownForAnInactiveTask() {
    final TaskManager taskManager = EasyMock.createNiceMock(TaskManager.class);
    expect(taskManager.producerClientIds()).andStubReturn(Collections.emptySet());
    final Consumer<byte[], byte[]> consumer = mock(Consumer.class);
    final ConsumerGroupMetadata consumerGroupMetadata = mock(ConsumerGroupMetadata.class);
    expect(consumer.groupMetadata()).andStubReturn(consumerGroupMetadata);
    expect(consumerGroupMetadata.groupInstanceId()).andReturn(Optional.empty());
    consumer.subscribe((Collection<String>) anyObject(), anyObject());
    EasyMock.expectLastCall().anyTimes();
    consumer.unsubscribe();
    EasyMock.expectLastCall().anyTimes();
    EasyMock.replay(consumerGroupMetadata);
    final Task task1 = mock(Task.class);
    final Task task2 = mock(Task.class);
    final TaskId taskId1 = new TaskId(0, 0);
    final TaskId taskId2 = new TaskId(0, 2);
    final Set<TaskId> corruptedTasks = singleton(taskId1);
    expect(task1.state()).andReturn(Task.State.CLOSED).anyTimes();
    expect(task1.id()).andReturn(taskId1).anyTimes();
    expect(task2.state()).andReturn(Task.State.CLOSED).anyTimes();
    expect(task2.id()).andReturn(taskId2).anyTimes();
    expect(taskManager.handleCorruption(corruptedTasks)).andReturn(false);
    EasyMock.replay(task1, task2, taskManager, consumer);
    final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST, mockTime);
    final TopologyMetadata topologyMetadata = new TopologyMetadata(internalTopologyBuilder, config);
    topologyMetadata.buildAndRewriteTopology();
    final StreamThread thread = new StreamThread(mockTime, eosEnabledConfig, null, consumer, consumer, null, null, taskManager, streamsMetrics, topologyMetadata, CLIENT_ID, new LogContext(""), new AtomicInteger(), new AtomicLong(Long.MAX_VALUE), new LinkedList<>(), null, HANDLER, null) {

        @Override
        void runOnce() {
            setState(State.PENDING_SHUTDOWN);
            throw new TaskCorruptedException(corruptedTasks);
        }
    }.updateThreadMetadata(getSharedAdminClientId(CLIENT_ID));
    thread.setState(StreamThread.State.STARTING);
    thread.runLoop();
    verify(taskManager);
    verify(consumer);
}

Example 43 with StreamsMetricsImpl

use of org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl in project kafka by apache.

the class StreamThreadTest method createStandbyTask.

// TODO: change return type to `StandbyTask`
private Collection<Task> createStandbyTask() {
    final LogContext logContext = new LogContext("test");
    final Logger log = logContext.logger(StreamThreadTest.class);
    final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST, mockTime);
    final StandbyTaskCreator standbyTaskCreator = new StandbyTaskCreator(new TopologyMetadata(internalTopologyBuilder, config), config, streamsMetrics, stateDirectory, new MockChangelogReader(), CLIENT_ID, log);
    return standbyTaskCreator.createTasks(singletonMap(new TaskId(1, 2), emptySet()));
}

Example 44 with StreamsMetricsImpl

use of org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl in project kafka by apache.

the class StreamThreadTest method shouldNotCommitNonRunningNonRestoringTasks.

@Test
public void shouldNotCommitNonRunningNonRestoringTasks() {
    final TaskManager taskManager = EasyMock.createNiceMock(TaskManager.class);
    final Consumer<byte[], byte[]> consumer = mock(Consumer.class);
    final ConsumerGroupMetadata consumerGroupMetadata = mock(ConsumerGroupMetadata.class);
    expect(consumer.groupMetadata()).andStubReturn(consumerGroupMetadata);
    expect(consumerGroupMetadata.groupInstanceId()).andReturn(Optional.empty());
    EasyMock.replay(consumer, consumerGroupMetadata);
    final Task task1 = mock(Task.class);
    final Task task2 = mock(Task.class);
    final Task task3 = mock(Task.class);
    final TaskId taskId1 = new TaskId(0, 1);
    final TaskId taskId2 = new TaskId(0, 2);
    final TaskId taskId3 = new TaskId(0, 3);
    expect(task1.state()).andReturn(Task.State.RUNNING).anyTimes();
    expect(task1.id()).andReturn(taskId1).anyTimes();
    expect(task2.state()).andReturn(Task.State.RESTORING).anyTimes();
    expect(task2.id()).andReturn(taskId2).anyTimes();
    expect(task3.state()).andReturn(Task.State.CREATED).anyTimes();
    expect(task3.id()).andReturn(taskId3).anyTimes();
    expect(taskManager.tasks()).andReturn(mkMap(mkEntry(taskId1, task1), mkEntry(taskId2, task2), mkEntry(taskId3, task3))).anyTimes();
    // expect not to try and commit task3, because it's not running.
    expect(taskManager.commit(mkSet(task1, task2))).andReturn(2).times(1);
    final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST, mockTime);
    final TopologyMetadata topologyMetadata = new TopologyMetadata(internalTopologyBuilder, config);
    topologyMetadata.buildAndRewriteTopology();
    final StreamThread thread = buildStreamThread(consumer, taskManager, config, topologyMetadata);
    EasyMock.replay(task1, task2, task3, taskManager);
    thread.setNow(mockTime.milliseconds());
    thread.maybeCommit();
    verify(taskManager);
}

Example 45 with StreamsMetricsImpl

use of org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl in project kafka by apache.

the class StreamThreadTest method shouldCatchTimeoutExceptionFromHandleCorruptionAndInvokeExceptionHandler.

@Test
@SuppressWarnings("unchecked")
public void shouldCatchTimeoutExceptionFromHandleCorruptionAndInvokeExceptionHandler() {
    final TaskManager taskManager = EasyMock.createNiceMock(TaskManager.class);
    expect(taskManager.producerClientIds()).andStubReturn(Collections.emptySet());
    final Consumer<byte[], byte[]> consumer = mock(Consumer.class);
    final ConsumerGroupMetadata consumerGroupMetadata = mock(ConsumerGroupMetadata.class);
    expect(consumer.groupMetadata()).andStubReturn(consumerGroupMetadata);
    expect(consumerGroupMetadata.groupInstanceId()).andReturn(Optional.empty());
    consumer.subscribe((Collection<String>) anyObject(), anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    consumer.unsubscribe();
    EasyMock.expectLastCall().atLeastOnce();
    EasyMock.replay(consumerGroupMetadata);
    final Task task1 = mock(Task.class);
    final Task task2 = mock(Task.class);
    final TaskId taskId1 = new TaskId(0, 0);
    final TaskId taskId2 = new TaskId(0, 2);
    final Set<TaskId> corruptedTasks = singleton(taskId1);
    expect(task1.state()).andStubReturn(Task.State.RUNNING);
    expect(task1.id()).andStubReturn(taskId1);
    expect(task2.state()).andStubReturn(Task.State.RUNNING);
    expect(task2.id()).andStubReturn(taskId2);
    taskManager.handleCorruption(corruptedTasks);
    expectLastCall().andThrow(new TimeoutException());
    EasyMock.replay(task1, task2, taskManager, consumer);
    final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST, mockTime);
    final TopologyMetadata topologyMetadata = new TopologyMetadata(internalTopologyBuilder, config);
    topologyMetadata.buildAndRewriteTopology();
    final StreamThread thread = new StreamThread(mockTime, config, null, consumer, consumer, null, null, taskManager, streamsMetrics, topologyMetadata, CLIENT_ID, new LogContext(""), new AtomicInteger(), new AtomicLong(Long.MAX_VALUE), new LinkedList<>(), null, HANDLER, null) {

        @Override
        void runOnce() {
            setState(State.PENDING_SHUTDOWN);
            throw new TaskCorruptedException(corruptedTasks);
        }
    }.updateThreadMetadata(getSharedAdminClientId(CLIENT_ID));
    final AtomicBoolean exceptionHandlerInvoked = new AtomicBoolean(false);
    thread.setStreamsUncaughtExceptionHandler((e, b) -> exceptionHandlerInvoked.set(true));
    thread.run();
    verify(taskManager);
    assertThat(exceptionHandlerInvoked.get(), is(true));
}