Examples with ApiVersions org.apache.kafka.clients.ApiVersions used on opensource projects

Example 16 with ApiVersions

use of org.apache.kafka.clients.ApiVersions in project kafka by apache.

the class SenderTest method testQuotaMetrics.

/*
     * Send multiple requests. Verify that the client side quota metrics have the right values
     */
@SuppressWarnings("deprecation")
@Test
public void testQuotaMetrics() {
    MockSelector selector = new MockSelector(time);
    Sensor throttleTimeSensor = Sender.throttleTimeSensor(this.senderMetricsRegistry);
    Cluster cluster = TestUtils.singletonCluster("test", 1);
    Node node = cluster.nodes().get(0);
    NetworkClient client = new NetworkClient(selector, metadata, "mock", Integer.MAX_VALUE, 1000, 1000, 64 * 1024, 64 * 1024, 1000, 10 * 1000, 127 * 1000, time, true, new ApiVersions(), throttleTimeSensor, logContext);
    ApiVersionsResponse apiVersionsResponse = ApiVersionsResponse.defaultApiVersionsResponse(400, ApiMessageType.ListenerType.ZK_BROKER);
    ByteBuffer buffer = RequestTestUtils.serializeResponseWithHeader(apiVersionsResponse, ApiKeys.API_VERSIONS.latestVersion(), 0);
    selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer)));
    while (!client.ready(node, time.milliseconds())) {
        client.poll(1, time.milliseconds());
        // If a throttled response is received, advance the time to ensure progress.
        time.sleep(client.throttleDelayMs(node, time.milliseconds()));
    }
    selector.clear();
    for (int i = 1; i <= 3; i++) {
        int throttleTimeMs = 100 * i;
        ProduceRequest.Builder builder = ProduceRequest.forCurrentMagic(new ProduceRequestData().setTopicData(new ProduceRequestData.TopicProduceDataCollection()).setAcks((short) 1).setTimeoutMs(1000));
        ClientRequest request = client.newClientRequest(node.idString(), builder, time.milliseconds(), true);
        client.send(request, time.milliseconds());
        client.poll(1, time.milliseconds());
        ProduceResponse response = produceResponse(tp0, i, Errors.NONE, throttleTimeMs);
        buffer = RequestTestUtils.serializeResponseWithHeader(response, ApiKeys.PRODUCE.latestVersion(), request.correlationId());
        selector.completeReceive(new NetworkReceive(node.idString(), buffer));
        client.poll(1, time.milliseconds());
        // If a throttled response is received, advance the time to ensure progress.
        time.sleep(client.throttleDelayMs(node, time.milliseconds()));
        selector.clear();
    }
    Map<MetricName, KafkaMetric> allMetrics = metrics.metrics();
    KafkaMetric avgMetric = allMetrics.get(this.senderMetricsRegistry.produceThrottleTimeAvg);
    KafkaMetric maxMetric = allMetrics.get(this.senderMetricsRegistry.produceThrottleTimeMax);
    // Throttle times are ApiVersions=400, Produce=(100, 200, 300)
    assertEquals(250, (Double) avgMetric.metricValue(), EPS);
    assertEquals(400, (Double) maxMetric.metricValue(), EPS);
    client.close();
}

Example 17 with ApiVersions

use of org.apache.kafka.clients.ApiVersions in project kafka by apache.

the class SenderTest method testInitProducerIdWithMaxInFlightOne.

/**
 * Verifies that InitProducerId of transactional producer succeeds even if metadata requests
 * are pending with only one bootstrap node available and maxInFlight=1, where multiple
 * polls are necessary to send requests.
 */
@Test
public void testInitProducerIdWithMaxInFlightOne() throws Exception {
    final long producerId = 123456L;
    createMockClientWithMaxFlightOneMetadataPending();
    // Initialize transaction manager. InitProducerId will be queued up until metadata response
    // is processed and FindCoordinator can be sent to `leastLoadedNode`.
    TransactionManager transactionManager = new TransactionManager(new LogContext(), "testInitProducerIdWithPendingMetadataRequest", 60000, 100L, new ApiVersions());
    setupWithTransactionState(transactionManager, false, null, false);
    ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch(producerId, (short) 0);
    transactionManager.initializeTransactions();
    sender.runOnce();
    // Process metadata response, prepare FindCoordinator and InitProducerId responses.
    // Verify producerId after the sender is run to process responses.
    MetadataResponse metadataUpdate = RequestTestUtils.metadataUpdateWith(1, Collections.emptyMap());
    client.respond(metadataUpdate);
    prepareFindCoordinatorResponse(Errors.NONE, "testInitProducerIdWithPendingMetadataRequest");
    prepareInitProducerResponse(Errors.NONE, producerIdAndEpoch.producerId, producerIdAndEpoch.epoch);
    waitForProducerId(transactionManager, producerIdAndEpoch);
}

Example 18 with ApiVersions

use of org.apache.kafka.clients.ApiVersions in project kafka by apache.

the class FetcherTest method testOffsetValidationFencing.

@Test
public void testOffsetValidationFencing() {
    buildFetcher();
    assignFromUser(singleton(tp0));
    Map<String, Integer> partitionCounts = new HashMap<>();
    partitionCounts.put(tp0.topic(), 4);
    final int epochOne = 1;
    final int epochTwo = 2;
    final int epochThree = 3;
    // Start with metadata, epoch=1
    metadata.updateWithCurrentRequestVersion(RequestTestUtils.metadataUpdateWithIds("dummy", 1, Collections.emptyMap(), partitionCounts, tp -> epochOne, topicIds), false, 0L);
    // Offset validation requires OffsetForLeaderEpoch request v3 or higher
    Node node = metadata.fetch().nodes().get(0);
    apiVersions.update(node.idString(), NodeApiVersions.create());
    // Seek with a position and leader+epoch
    Metadata.LeaderAndEpoch leaderAndEpoch = new Metadata.LeaderAndEpoch(metadata.currentLeader(tp0).leader, Optional.of(epochOne));
    subscriptions.seekValidated(tp0, new SubscriptionState.FetchPosition(0, Optional.of(epochOne), leaderAndEpoch));
    // Update metadata to epoch=2, enter validation
    metadata.updateWithCurrentRequestVersion(RequestTestUtils.metadataUpdateWithIds("dummy", 1, Collections.emptyMap(), partitionCounts, tp -> epochTwo, topicIds), false, 0L);
    fetcher.validateOffsetsIfNeeded();
    assertTrue(subscriptions.awaitingValidation(tp0));
    // Update the position to epoch=3, as we would from a fetch
    subscriptions.completeValidation(tp0);
    SubscriptionState.FetchPosition nextPosition = new SubscriptionState.FetchPosition(10, Optional.of(epochTwo), new Metadata.LeaderAndEpoch(leaderAndEpoch.leader, Optional.of(epochTwo)));
    subscriptions.position(tp0, nextPosition);
    subscriptions.maybeValidatePositionForCurrentLeader(apiVersions, tp0, new Metadata.LeaderAndEpoch(leaderAndEpoch.leader, Optional.of(epochThree)));
    // Prepare offset list response from async validation with epoch=2
    client.prepareResponse(prepareOffsetsForLeaderEpochResponse(tp0, Errors.NONE, epochTwo, 10L));
    consumerClient.pollNoWakeup();
    assertTrue(subscriptions.awaitingValidation(tp0), "Expected validation to fail since leader epoch changed");
    // Next round of validation, should succeed in validating the position
    fetcher.validateOffsetsIfNeeded();
    client.prepareResponse(prepareOffsetsForLeaderEpochResponse(tp0, Errors.NONE, epochThree, 10L));
    consumerClient.pollNoWakeup();
    assertFalse(subscriptions.awaitingValidation(tp0), "Expected validation to succeed with latest epoch");
}

Example 19 with ApiVersions

use of org.apache.kafka.clients.ApiVersions in project kafka by apache.

the class SubscriptionStateTest method testSeekUnvalidatedWithNoOffsetEpoch.

@Test
public void testSeekUnvalidatedWithNoOffsetEpoch() {
    Node broker1 = new Node(1, "localhost", 9092);
    state.assignFromUser(Collections.singleton(tp0));
    // Seek with no offset epoch requires no validation no matter what the current leader is
    state.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(0L, Optional.empty(), new Metadata.LeaderAndEpoch(Optional.of(broker1), Optional.of(5))));
    assertTrue(state.hasValidPosition(tp0));
    assertFalse(state.awaitingValidation(tp0));
    ApiVersions apiVersions = new ApiVersions();
    apiVersions.update(broker1.idString(), NodeApiVersions.create());
    assertFalse(state.maybeValidatePositionForCurrentLeader(apiVersions, tp0, new Metadata.LeaderAndEpoch(Optional.of(broker1), Optional.empty())));
    assertTrue(state.hasValidPosition(tp0));
    assertFalse(state.awaitingValidation(tp0));
    assertFalse(state.maybeValidatePositionForCurrentLeader(apiVersions, tp0, new Metadata.LeaderAndEpoch(Optional.of(broker1), Optional.of(10))));
    assertTrue(state.hasValidPosition(tp0));
    assertFalse(state.awaitingValidation(tp0));
}

Example 20 with ApiVersions

use of org.apache.kafka.clients.ApiVersions in project kafka by apache.

the class SubscriptionStateTest method testSeekUnvalidatedWithOffsetEpoch.

@Test
public void testSeekUnvalidatedWithOffsetEpoch() {
    Node broker1 = new Node(1, "localhost", 9092);
    ApiVersions apiVersions = new ApiVersions();
    apiVersions.update(broker1.idString(), NodeApiVersions.create());
    state.assignFromUser(Collections.singleton(tp0));
    state.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(0L, Optional.of(2), new Metadata.LeaderAndEpoch(Optional.of(broker1), Optional.of(5))));
    assertFalse(state.hasValidPosition(tp0));
    assertTrue(state.awaitingValidation(tp0));
    // Update using the current leader and epoch
    assertTrue(state.maybeValidatePositionForCurrentLeader(apiVersions, tp0, new Metadata.LeaderAndEpoch(Optional.of(broker1), Optional.of(5))));
    assertFalse(state.hasValidPosition(tp0));
    assertTrue(state.awaitingValidation(tp0));
    // Update with a newer leader and epoch
    assertTrue(state.maybeValidatePositionForCurrentLeader(apiVersions, tp0, new Metadata.LeaderAndEpoch(Optional.of(broker1), Optional.of(15))));
    assertFalse(state.hasValidPosition(tp0));
    assertTrue(state.awaitingValidation(tp0));
    // If the updated leader has no epoch information, then skip validation and begin fetching
    assertFalse(state.maybeValidatePositionForCurrentLeader(apiVersions, tp0, new Metadata.LeaderAndEpoch(Optional.of(broker1), Optional.empty())));
    assertTrue(state.hasValidPosition(tp0));
    assertFalse(state.awaitingValidation(tp0));
}