Examples with ConsumerGroupMetadata org.apache.kafka.clients.consumer.ConsumerGroupMetadata used on opensource projects

Example 21 with ConsumerGroupMetadata

use of org.apache.kafka.clients.consumer.ConsumerGroupMetadata in project kafka by apache.

the class StreamThreadTest method shouldShutdownTaskManagerOnCloseWithoutStart.

@Test
public void shouldShutdownTaskManagerOnCloseWithoutStart() {
    final Consumer<byte[], byte[]> consumer = EasyMock.createNiceMock(Consumer.class);
    final ConsumerGroupMetadata consumerGroupMetadata = mock(ConsumerGroupMetadata.class);
    expect(consumer.groupMetadata()).andStubReturn(consumerGroupMetadata);
    expect(consumerGroupMetadata.groupInstanceId()).andReturn(Optional.empty());
    EasyMock.replay(consumerGroupMetadata);
    final TaskManager taskManager = EasyMock.createNiceMock(TaskManager.class);
    expect(taskManager.producerClientIds()).andStubReturn(Collections.emptySet());
    taskManager.shutdown(true);
    EasyMock.expectLastCall();
    EasyMock.replay(taskManager, consumer);
    final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST, mockTime);
    final TopologyMetadata topologyMetadata = new TopologyMetadata(internalTopologyBuilder, config);
    topologyMetadata.buildAndRewriteTopology();
    final StreamThread thread = buildStreamThread(consumer, taskManager, config, topologyMetadata).updateThreadMetadata(getSharedAdminClientId(CLIENT_ID));
    thread.shutdown();
    verify(taskManager);
}

Example 22 with ConsumerGroupMetadata

use of org.apache.kafka.clients.consumer.ConsumerGroupMetadata in project kafka by apache.

the class StreamThreadTest method runAndVerifyFailedStreamThreadRecording.

public void runAndVerifyFailedStreamThreadRecording(final boolean shouldFail) {
    final Consumer<byte[], byte[]> consumer = EasyMock.createNiceMock(Consumer.class);
    final ConsumerGroupMetadata consumerGroupMetadata = mock(ConsumerGroupMetadata.class);
    expect(consumer.groupMetadata()).andStubReturn(consumerGroupMetadata);
    expect(consumerGroupMetadata.groupInstanceId()).andReturn(Optional.empty());
    EasyMock.replay(consumer, consumerGroupMetadata);
    final TaskManager taskManager = EasyMock.createNiceMock(TaskManager.class);
    expect(taskManager.producerClientIds()).andStubReturn(Collections.emptySet());
    final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST, mockTime);
    final TopologyMetadata topologyMetadata = new TopologyMetadata(internalTopologyBuilder, config);
    topologyMetadata.buildAndRewriteTopology();
    final StreamThread thread = new StreamThread(mockTime, config, null, consumer, consumer, null, null, taskManager, streamsMetrics, topologyMetadata, CLIENT_ID, new LogContext(""), new AtomicInteger(), new AtomicLong(Long.MAX_VALUE), new LinkedList<>(), null, (e, b) -> {
    }, null) {

        @Override
        void runOnce() {
            setState(StreamThread.State.PENDING_SHUTDOWN);
            if (shouldFail) {
                throw new StreamsException(Thread.currentThread().getName());
            }
        }
    };
    EasyMock.replay(taskManager);
    thread.updateThreadMetadata("metadata");
    thread.run();
    final Metric failedThreads = StreamsTestUtils.getMetricByName(metrics.metrics(), "failed-stream-threads", "stream-metrics");
    assertThat(failedThreads.metricValue(), is(shouldFail ? 1.0 : 0.0));
}

Example 23 with ConsumerGroupMetadata

use of org.apache.kafka.clients.consumer.ConsumerGroupMetadata in project kafka by apache.

the class TaskManagerTest method shouldCommitAllActiveTasksThatNeedCommittingOnHandleRevocationWithEosV2.

@Test
public void shouldCommitAllActiveTasksThatNeedCommittingOnHandleRevocationWithEosV2() {
    final StreamsProducer producer = mock(StreamsProducer.class);
    setUpTaskManager(ProcessingMode.EXACTLY_ONCE_V2);
    final StateMachineTask task00 = new StateMachineTask(taskId00, taskId00Partitions, true);
    final Map<TopicPartition, OffsetAndMetadata> offsets00 = singletonMap(t1p0, new OffsetAndMetadata(0L, null));
    task00.setCommittableOffsetsAndMetadata(offsets00);
    task00.setCommitNeeded();
    final StateMachineTask task01 = new StateMachineTask(taskId01, taskId01Partitions, true);
    final Map<TopicPartition, OffsetAndMetadata> offsets01 = singletonMap(t1p1, new OffsetAndMetadata(1L, null));
    task01.setCommittableOffsetsAndMetadata(offsets01);
    task01.setCommitNeeded();
    final StateMachineTask task02 = new StateMachineTask(taskId02, taskId02Partitions, true);
    final Map<TopicPartition, OffsetAndMetadata> offsets02 = singletonMap(t1p2, new OffsetAndMetadata(2L, null));
    task02.setCommittableOffsetsAndMetadata(offsets02);
    final StateMachineTask task10 = new StateMachineTask(taskId10, taskId10Partitions, false);
    final Map<TopicPartition, OffsetAndMetadata> expectedCommittedOffsets = new HashMap<>();
    expectedCommittedOffsets.putAll(offsets00);
    expectedCommittedOffsets.putAll(offsets01);
    final Map<TaskId, Set<TopicPartition>> assignmentActive = mkMap(mkEntry(taskId00, taskId00Partitions), mkEntry(taskId01, taskId01Partitions), mkEntry(taskId02, taskId02Partitions));
    final Map<TaskId, Set<TopicPartition>> assignmentStandby = mkMap(mkEntry(taskId10, taskId10Partitions));
    expectRestoreToBeCompleted(consumer, changeLogReader);
    expect(activeTaskCreator.createTasks(anyObject(), eq(assignmentActive))).andReturn(asList(task00, task01, task02));
    expect(activeTaskCreator.threadProducer()).andReturn(producer);
    activeTaskCreator.closeAndRemoveTaskProducerIfNeeded(taskId00);
    expect(standbyTaskCreator.createTasks(eq(assignmentStandby))).andReturn(singletonList(task10));
    final ConsumerGroupMetadata groupMetadata = new ConsumerGroupMetadata("appId");
    expect(consumer.groupMetadata()).andReturn(groupMetadata);
    producer.commitTransaction(expectedCommittedOffsets, groupMetadata);
    expectLastCall();
    task00.committedOffsets();
    EasyMock.expectLastCall();
    task01.committedOffsets();
    EasyMock.expectLastCall();
    task02.committedOffsets();
    EasyMock.expectLastCall();
    task10.committedOffsets();
    EasyMock.expectLastCall();
    replay(activeTaskCreator, standbyTaskCreator, consumer, changeLogReader);
    taskManager.handleAssignment(assignmentActive, assignmentStandby);
    assertThat(taskManager.tryToCompleteRestoration(time.milliseconds(), null), is(true));
    assertThat(task00.state(), is(Task.State.RUNNING));
    assertThat(task01.state(), is(Task.State.RUNNING));
    assertThat(task02.state(), is(Task.State.RUNNING));
    assertThat(task10.state(), is(Task.State.RUNNING));
    taskManager.handleRevocation(taskId00Partitions);
    assertThat(task00.commitNeeded, is(false));
    assertThat(task01.commitNeeded, is(false));
    assertThat(task02.commitPrepared, is(false));
    assertThat(task10.commitPrepared, is(false));
}

Example 24 with ConsumerGroupMetadata

use of org.apache.kafka.clients.consumer.ConsumerGroupMetadata in project kafka by apache.

the class TaskManagerTest method shouldCommitViaProducerIfEosEnabled.

private void shouldCommitViaProducerIfEosEnabled(final ProcessingMode processingMode, final StreamsProducer producer, final Map<TopicPartition, OffsetAndMetadata> offsetsT01, final Map<TopicPartition, OffsetAndMetadata> offsetsT02) {
    setUpTaskManager(processingMode);
    final StateMachineTask task01 = new StateMachineTask(taskId01, taskId01Partitions, true);
    task01.setCommittableOffsetsAndMetadata(offsetsT01);
    task01.setCommitNeeded();
    taskManager.addTask(task01);
    final StateMachineTask task02 = new StateMachineTask(taskId02, taskId02Partitions, true);
    task02.setCommittableOffsetsAndMetadata(offsetsT02);
    task02.setCommitNeeded();
    taskManager.addTask(task02);
    reset(consumer);
    expect(consumer.groupMetadata()).andStubReturn(new ConsumerGroupMetadata("appId"));
    replay(activeTaskCreator, consumer, producer);
    taskManager.commitAll();
    verify(producer, consumer);
}

Example 25 with ConsumerGroupMetadata

use of org.apache.kafka.clients.consumer.ConsumerGroupMetadata in project kafka by apache.

the class StreamThreadTest method shouldTransmitTaskManagerMetrics.

@Test
public void shouldTransmitTaskManagerMetrics() {
    final Consumer<byte[], byte[]> consumer = EasyMock.createNiceMock(Consumer.class);
    final ConsumerGroupMetadata consumerGroupMetadata = mock(ConsumerGroupMetadata.class);
    expect(consumer.groupMetadata()).andStubReturn(consumerGroupMetadata);
    expect(consumerGroupMetadata.groupInstanceId()).andReturn(Optional.empty());
    EasyMock.replay(consumer, consumerGroupMetadata);
    final TaskManager taskManager = EasyMock.createNiceMock(TaskManager.class);
    final MetricName testMetricName = new MetricName("test_metric", "", "", new HashMap<>());
    final Metric testMetric = new KafkaMetric(new Object(), testMetricName, (Measurable) (config, now) -> 0, null, new MockTime());
    final Map<MetricName, Metric> dummyProducerMetrics = singletonMap(testMetricName, testMetric);
    expect(taskManager.producerMetrics()).andReturn(dummyProducerMetrics);
    EasyMock.replay(taskManager);
    final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST, mockTime);
    final TopologyMetadata topologyMetadata = new TopologyMetadata(internalTopologyBuilder, config);
    topologyMetadata.buildAndRewriteTopology();
    final StreamThread thread = buildStreamThread(consumer, taskManager, config, topologyMetadata);
    assertThat(dummyProducerMetrics, is(thread.producerMetrics()));
}