use of org.apache.kafka.common.utils.MockTime in project kafka by apache.
the class KafkaConsumerTest method testMeasureCommittedDuration.
@Test
public void testMeasureCommittedDuration() {
long offset1 = 10000;
Time time = new MockTime(Duration.ofSeconds(1).toMillis());
SubscriptionState subscription = new SubscriptionState(new LogContext(), OffsetResetStrategy.EARLIEST);
ConsumerMetadata metadata = createMetadata(subscription);
MockClient client = new MockClient(time, metadata);
initMetadata(client, Collections.singletonMap(topic, 2));
Node node = metadata.fetch().nodes().get(0);
KafkaConsumer<String, String> consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId);
consumer.assign(singletonList(tp0));
// lookup coordinator
client.prepareResponseFrom(FindCoordinatorResponse.prepareResponse(Errors.NONE, groupId, node), node);
Node coordinator = new Node(Integer.MAX_VALUE - node.id(), node.host(), node.port());
// fetch offset for one topic
client.prepareResponseFrom(offsetResponse(Collections.singletonMap(tp0, offset1), Errors.NONE), coordinator);
consumer.committed(Collections.singleton(tp0)).get(tp0).offset();
final Metric metric = consumer.metrics().get(consumer.metrics.metricName("committed-time-ns-total", "consumer-metrics"));
assertTrue((Double) metric.metricValue() >= Duration.ofMillis(999).toNanos());
}
use of org.apache.kafka.common.utils.MockTime in project kafka by apache.
the class KafkaConsumerTest method testClosingConsumerUnregistersConsumerMetrics.
@Test
public void testClosingConsumerUnregistersConsumerMetrics() {
Time time = new MockTime(1L);
ConsumerMetadata metadata = createMetadata(subscription);
MockClient client = new MockClient(time, metadata);
initMetadata(client, Collections.singletonMap(topic, 1));
KafkaConsumer<String, String> consumer = newConsumer(time, client, subscription, metadata, new RoundRobinAssignor(), true, groupInstanceId);
consumer.subscribe(singletonList(topic));
assertTrue(consumerMetricPresent(consumer, "last-poll-seconds-ago"));
assertTrue(consumerMetricPresent(consumer, "time-between-poll-avg"));
assertTrue(consumerMetricPresent(consumer, "time-between-poll-max"));
consumer.close();
assertFalse(consumerMetricPresent(consumer, "last-poll-seconds-ago"));
assertFalse(consumerMetricPresent(consumer, "time-between-poll-avg"));
assertFalse(consumerMetricPresent(consumer, "time-between-poll-max"));
}
use of org.apache.kafka.common.utils.MockTime in project kafka by apache.
the class KafkaConsumerTest method testReturnRecordsDuringRebalance.
@Test
public void testReturnRecordsDuringRebalance() throws InterruptedException {
Time time = new MockTime(1L);
ConsumerMetadata metadata = createMetadata(subscription);
MockClient client = new MockClient(time, metadata);
ConsumerPartitionAssignor assignor = new CooperativeStickyAssignor();
KafkaConsumer<String, String> consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId);
initMetadata(client, Utils.mkMap(Utils.mkEntry(topic, 1), Utils.mkEntry(topic2, 1), Utils.mkEntry(topic3, 1)));
consumer.subscribe(Arrays.asList(topic, topic2), getConsumerRebalanceListener(consumer));
Node node = metadata.fetch().nodes().get(0);
Node coordinator = prepareRebalance(client, node, assignor, Arrays.asList(tp0, t2p0), null);
// a poll with non-zero milliseconds would complete three round-trips (discover, join, sync)
TestUtils.waitForCondition(() -> {
consumer.poll(Duration.ofMillis(100L));
return consumer.assignment().equals(Utils.mkSet(tp0, t2p0));
}, "Does not complete rebalance in time");
assertEquals(Utils.mkSet(topic, topic2), consumer.subscription());
assertEquals(Utils.mkSet(tp0, t2p0), consumer.assignment());
// prepare a response of the outstanding fetch so that we have data available on the next poll
Map<TopicPartition, FetchInfo> fetches1 = new HashMap<>();
fetches1.put(tp0, new FetchInfo(0, 1));
fetches1.put(t2p0, new FetchInfo(0, 10));
client.respondFrom(fetchResponse(fetches1), node);
ConsumerRecords<String, String> records = consumer.poll(Duration.ZERO);
// verify that the fetch occurred as expected
assertEquals(11, records.count());
assertEquals(1L, consumer.position(tp0));
assertEquals(10L, consumer.position(t2p0));
// prepare the next response of the prefetch
fetches1.clear();
fetches1.put(tp0, new FetchInfo(1, 1));
fetches1.put(t2p0, new FetchInfo(10, 20));
client.respondFrom(fetchResponse(fetches1), node);
// subscription change
consumer.subscribe(Arrays.asList(topic, topic3), getConsumerRebalanceListener(consumer));
// verify that subscription has changed but assignment is still unchanged
assertEquals(Utils.mkSet(topic, topic3), consumer.subscription());
assertEquals(Utils.mkSet(tp0, t2p0), consumer.assignment());
// mock the offset commit response for to be revoked partitions
Map<TopicPartition, Long> partitionOffsets1 = new HashMap<>();
partitionOffsets1.put(t2p0, 10L);
AtomicBoolean commitReceived = prepareOffsetCommitResponse(client, coordinator, partitionOffsets1);
// poll once which would not complete the rebalance
records = consumer.poll(Duration.ZERO);
// clear out the prefetch so it doesn't interfere with the rest of the test
fetches1.clear();
fetches1.put(tp0, new FetchInfo(2, 1));
client.respondFrom(fetchResponse(fetches1), node);
// verify that the fetch still occurred as expected
assertEquals(Utils.mkSet(topic, topic3), consumer.subscription());
assertEquals(Collections.singleton(tp0), consumer.assignment());
assertEquals(1, records.count());
assertEquals(2L, consumer.position(tp0));
// verify that the offset commits occurred as expected
assertTrue(commitReceived.get());
// mock rebalance responses
client.respondFrom(joinGroupFollowerResponse(assignor, 2, "memberId", "leaderId", Errors.NONE), coordinator);
// we need to poll 1) for getting the join response, and then send the sync request;
// 2) for getting the sync response
records = consumer.poll(Duration.ZERO);
// should not finish the response yet
assertEquals(Utils.mkSet(topic, topic3), consumer.subscription());
assertEquals(Collections.singleton(tp0), consumer.assignment());
assertEquals(1, records.count());
assertEquals(3L, consumer.position(tp0));
fetches1.clear();
fetches1.put(tp0, new FetchInfo(3, 1));
client.respondFrom(fetchResponse(fetches1), node);
// now complete the rebalance
client.respondFrom(syncGroupResponse(Arrays.asList(tp0, t3p0), Errors.NONE), coordinator);
AtomicInteger count = new AtomicInteger(0);
TestUtils.waitForCondition(() -> {
ConsumerRecords<String, String> recs = consumer.poll(Duration.ofMillis(100L));
return consumer.assignment().equals(Utils.mkSet(tp0, t3p0)) && count.addAndGet(recs.count()) == 1;
}, "Does not complete rebalance in time");
// should have t3 but not sent yet the t3 records
assertEquals(Utils.mkSet(topic, topic3), consumer.subscription());
assertEquals(Utils.mkSet(tp0, t3p0), consumer.assignment());
assertEquals(4L, consumer.position(tp0));
assertEquals(0L, consumer.position(t3p0));
fetches1.clear();
fetches1.put(tp0, new FetchInfo(4, 1));
fetches1.put(t3p0, new FetchInfo(0, 100));
client.respondFrom(fetchResponse(fetches1), node);
count.set(0);
TestUtils.waitForCondition(() -> {
ConsumerRecords<String, String> recs = consumer.poll(Duration.ofMillis(100L));
return count.addAndGet(recs.count()) == 101;
}, "Does not complete rebalance in time");
assertEquals(5L, consumer.position(tp0));
assertEquals(100L, consumer.position(t3p0));
client.requests().clear();
consumer.unsubscribe();
consumer.close();
}
use of org.apache.kafka.common.utils.MockTime in project kafka by apache.
the class RetryTest method test.
@Test
public void test() throws ExecutionException {
Exception[] attempts = new Exception[] { new IOException("pretend connect error"), new IOException("pretend timeout error"), new IOException("pretend read error"), // success!
null };
long retryWaitMs = 1000;
long maxWaitMs = 10000;
Retryable<String> call = createRetryable(attempts);
Time time = new MockTime(0, 0, 0);
assertEquals(0L, time.milliseconds());
Retry<String> r = new Retry<>(time, retryWaitMs, maxWaitMs);
r.execute(call);
long secondWait = retryWaitMs * 2;
long thirdWait = retryWaitMs * 4;
long totalWait = retryWaitMs + secondWait + thirdWait;
assertEquals(totalWait, time.milliseconds());
}
use of org.apache.kafka.common.utils.MockTime in project kafka by apache.
the class RetryTest method testIOExceptionFailure.
@Test
public void testIOExceptionFailure() {
Exception[] attempts = new Exception[] { new IOException("pretend connect error"), new IOException("pretend timeout error"), new IOException("pretend read error"), new IOException("pretend another read error") };
long retryWaitMs = 1000;
long maxWaitMs = 1000 + 2000 + 3999;
Retryable<String> call = createRetryable(attempts);
Time time = new MockTime(0, 0, 0);
assertEquals(0L, time.milliseconds());
Retry<String> r = new Retry<>(time, retryWaitMs, maxWaitMs);
assertThrows(ExecutionException.class, () -> r.execute(call));
assertEquals(maxWaitMs, time.milliseconds());
}
Aggregations