Examples with ClientRequest org.apache.kafka.clients.ClientRequest used on opensource projects

Example 21 with ClientRequest

use of org.apache.kafka.clients.ClientRequest in project kafka by apache.

the class FetcherTest method testQuotaMetrics.

/*
     * Send multiple requests. Verify that the client side quota metrics have the right values
     */
@Test
public void testQuotaMetrics() {
    buildFetcher();
    MockSelector selector = new MockSelector(time);
    Sensor throttleTimeSensor = Fetcher.throttleTimeSensor(metrics, metricsRegistry);
    Cluster cluster = TestUtils.singletonCluster("test", 1);
    Node node = cluster.nodes().get(0);
    NetworkClient client = new NetworkClient(selector, metadata, "mock", Integer.MAX_VALUE, 1000, 1000, 64 * 1024, 64 * 1024, 1000, 10 * 1000, 127 * 1000, time, true, new ApiVersions(), throttleTimeSensor, new LogContext());
    ApiVersionsResponse apiVersionsResponse = ApiVersionsResponse.defaultApiVersionsResponse(400, ApiMessageType.ListenerType.ZK_BROKER);
    ByteBuffer buffer = RequestTestUtils.serializeResponseWithHeader(apiVersionsResponse, ApiKeys.API_VERSIONS.latestVersion(), 0);
    selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer)));
    while (!client.ready(node, time.milliseconds())) {
        client.poll(1, time.milliseconds());
        // If a throttled response is received, advance the time to ensure progress.
        time.sleep(client.throttleDelayMs(node, time.milliseconds()));
    }
    selector.clear();
    for (int i = 1; i <= 3; i++) {
        int throttleTimeMs = 100 * i;
        FetchRequest.Builder builder = FetchRequest.Builder.forConsumer(ApiKeys.FETCH.latestVersion(), 100, 100, new LinkedHashMap<>());
        builder.rackId("");
        ClientRequest request = client.newClientRequest(node.idString(), builder, time.milliseconds(), true);
        client.send(request, time.milliseconds());
        client.poll(1, time.milliseconds());
        FetchResponse response = fullFetchResponse(tidp0, nextRecords, Errors.NONE, i, throttleTimeMs);
        buffer = RequestTestUtils.serializeResponseWithHeader(response, ApiKeys.FETCH.latestVersion(), request.correlationId());
        selector.completeReceive(new NetworkReceive(node.idString(), buffer));
        client.poll(1, time.milliseconds());
        // If a throttled response is received, advance the time to ensure progress.
        time.sleep(client.throttleDelayMs(node, time.milliseconds()));
        selector.clear();
    }
    Map<MetricName, KafkaMetric> allMetrics = metrics.metrics();
    KafkaMetric avgMetric = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchThrottleTimeAvg));
    KafkaMetric maxMetric = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchThrottleTimeMax));
    // Throttle times are ApiVersions=400, Fetch=(100, 200, 300)
    assertEquals(250, (Double) avgMetric.metricValue(), EPSILON);
    assertEquals(400, (Double) maxMetric.metricValue(), EPSILON);
    client.close();
}

Example 22 with ClientRequest

use of org.apache.kafka.clients.ClientRequest in project kafka by apache.

the class StreamsKafkaClient method createTopics.

/**
     * Create a set of new topics using batch request.
     */
public void createTopics(final Map<InternalTopicConfig, Integer> topicsMap, final int replicationFactor, final long windowChangeLogAdditionalRetention, final MetadataResponse metadata) {
    final Map<String, CreateTopicsRequest.TopicDetails> topicRequestDetails = new HashMap<>();
    for (Map.Entry<InternalTopicConfig, Integer> entry : topicsMap.entrySet()) {
        InternalTopicConfig internalTopicConfig = entry.getKey();
        Integer partitions = entry.getValue();
        final Properties topicProperties = internalTopicConfig.toProperties(windowChangeLogAdditionalRetention);
        final Map<String, String> topicConfig = new HashMap<>();
        for (String key : topicProperties.stringPropertyNames()) {
            topicConfig.put(key, topicProperties.getProperty(key));
        }
        final CreateTopicsRequest.TopicDetails topicDetails = new CreateTopicsRequest.TopicDetails(partitions, (short) replicationFactor, topicConfig);
        topicRequestDetails.put(internalTopicConfig.name(), topicDetails);
    }
    final ClientRequest clientRequest = kafkaClient.newClientRequest(getControllerReadyBrokerId(metadata), new CreateTopicsRequest.Builder(topicRequestDetails, streamsConfig.getInt(StreamsConfig.REQUEST_TIMEOUT_MS_CONFIG)), Time.SYSTEM.milliseconds(), true);
    final ClientResponse clientResponse = sendRequest(clientRequest);
    if (!clientResponse.hasResponse()) {
        throw new StreamsException("Empty response for client request.");
    }
    if (!(clientResponse.responseBody() instanceof CreateTopicsResponse)) {
        throw new StreamsException("Inconsistent response type for internal topic creation request. " + "Expected CreateTopicsResponse but received " + clientResponse.responseBody().getClass().getName());
    }
    final CreateTopicsResponse createTopicsResponse = (CreateTopicsResponse) clientResponse.responseBody();
    for (InternalTopicConfig internalTopicConfig : topicsMap.keySet()) {
        CreateTopicsResponse.Error error = createTopicsResponse.errors().get(internalTopicConfig.name());
        if (!error.is(Errors.NONE) && !error.is(Errors.TOPIC_ALREADY_EXISTS)) {
            throw new StreamsException("Could not create topic: " + internalTopicConfig.name() + " due to " + error.messageWithFallback());
        }
    }
}

Example 23 with ClientRequest

use of org.apache.kafka.clients.ClientRequest in project kafka by apache.

the class StreamsKafkaClient method fetchMetadata.

/**
     * Fetch the metadata for all topics
     */
public MetadataResponse fetchMetadata() {
    final ClientRequest clientRequest = kafkaClient.newClientRequest(getAnyReadyBrokerId(), new MetadataRequest.Builder(null), Time.SYSTEM.milliseconds(), true);
    final ClientResponse clientResponse = sendRequest(clientRequest);
    if (!clientResponse.hasResponse()) {
        throw new StreamsException("Empty response for client request.");
    }
    if (!(clientResponse.responseBody() instanceof MetadataResponse)) {
        throw new StreamsException("Inconsistent response type for internal topic metadata request. " + "Expected MetadataResponse but received " + clientResponse.responseBody().getClass().getName());
    }
    final MetadataResponse metadataResponse = (MetadataResponse) clientResponse.responseBody();
    return metadataResponse;
}

Example 24 with ClientRequest

use of org.apache.kafka.clients.ClientRequest in project apache-kafka-on-k8s by banzaicloud.

the class KafkaConsumerTest method testManualAssignmentChangeWithAutoCommitDisabled.

@Test
public void testManualAssignmentChangeWithAutoCommitDisabled() {
    Time time = new MockTime();
    Map<String, Integer> tpCounts = new HashMap<>();
    tpCounts.put(topic, 1);
    tpCounts.put(topic2, 1);
    Cluster cluster = TestUtils.singletonCluster(tpCounts);
    Node node = cluster.nodes().get(0);
    Metadata metadata = createMetadata();
    metadata.update(cluster, Collections.<String>emptySet(), time.milliseconds());
    MockClient client = new MockClient(time, metadata);
    client.setNode(node);
    PartitionAssignor assignor = new RangeAssignor();
    KafkaConsumer<String, String> consumer = newConsumer(time, client, metadata, assignor, false);
    // lookup coordinator
    client.prepareResponseFrom(new FindCoordinatorResponse(Errors.NONE, node), node);
    Node coordinator = new Node(Integer.MAX_VALUE - node.id(), node.host(), node.port());
    // manual assignment
    consumer.assign(singleton(tp0));
    consumer.seekToBeginning(singleton(tp0));
    // fetch offset for one topic
    client.prepareResponseFrom(offsetResponse(Collections.singletonMap(tp0, 0L), Errors.NONE), coordinator);
    assertEquals(0, consumer.committed(tp0).offset());
    // verify that assignment immediately changes
    assertTrue(consumer.assignment().equals(singleton(tp0)));
    // there shouldn't be any need to lookup the coordinator or fetch committed offsets.
    // we just lookup the starting position and send the record fetch.
    client.prepareResponse(listOffsetsResponse(Collections.singletonMap(tp0, 10L)));
    client.prepareResponse(fetchResponse(tp0, 10L, 1));
    ConsumerRecords<String, String> records = consumer.poll(5);
    assertEquals(1, records.count());
    assertEquals(11L, consumer.position(tp0));
    // new manual assignment
    consumer.assign(singleton(t2p0));
    // verify that assignment immediately changes
    assertTrue(consumer.assignment().equals(singleton(t2p0)));
    // the auto commit is disabled, so no offset commit request should be sent
    for (ClientRequest req : client.requests()) assertTrue(req.requestBuilder().apiKey() != ApiKeys.OFFSET_COMMIT);
    client.requests().clear();
    consumer.close();
}

Example 25 with ClientRequest

use of org.apache.kafka.clients.ClientRequest in project apache-kafka-on-k8s by banzaicloud.

the class ConsumerNetworkClient method failUnsentRequests.

private void failUnsentRequests(Node node, RuntimeException e) {
    // clear unsent requests to node and fail their corresponding futures
    lock.lock();
    try {
        Collection<ClientRequest> unsentRequests = unsent.remove(node);
        for (ClientRequest unsentRequest : unsentRequests) {
            RequestFutureCompletionHandler handler = (RequestFutureCompletionHandler) unsentRequest.callback();
            handler.onFailure(e);
        }
    } finally {
        lock.unlock();
    }
}