Examples with ThrottlingQuotaExceededException org.apache.kafka.common.errors.ThrottlingQuotaExceededException used on opensource projects

Example 1 with ThrottlingQuotaExceededException

use of org.apache.kafka.common.errors.ThrottlingQuotaExceededException in project kafka by apache.

the class KafkaAdminClient method getDeleteTopicsCall.

private Call getDeleteTopicsCall(final DeleteTopicsOptions options, final Map<String, KafkaFutureImpl<Void>> futures, final List<String> topics, final Map<String, ThrottlingQuotaExceededException> quotaExceededExceptions, final long now, final long deadline) {
    return new Call("deleteTopics", deadline, new ControllerNodeProvider()) {

        @Override
        DeleteTopicsRequest.Builder createRequest(int timeoutMs) {
            return new DeleteTopicsRequest.Builder(new DeleteTopicsRequestData().setTopicNames(topics).setTimeoutMs(timeoutMs));
        }

        @Override
        void handleResponse(AbstractResponse abstractResponse) {
            // Check for controller change
            handleNotControllerError(abstractResponse);
            // Handle server responses for particular topics.
            final DeleteTopicsResponse response = (DeleteTopicsResponse) abstractResponse;
            final List<String> retryTopics = new ArrayList<>();
            final Map<String, ThrottlingQuotaExceededException> retryTopicQuotaExceededExceptions = new HashMap<>();
            for (DeletableTopicResult result : response.data().responses()) {
                KafkaFutureImpl<Void> future = futures.get(result.name());
                if (future == null) {
                    log.warn("Server response mentioned unknown topic {}", result.name());
                } else {
                    ApiError error = new ApiError(result.errorCode(), result.errorMessage());
                    if (error.isFailure()) {
                        if (error.is(Errors.THROTTLING_QUOTA_EXCEEDED)) {
                            ThrottlingQuotaExceededException quotaExceededException = new ThrottlingQuotaExceededException(response.throttleTimeMs(), error.messageWithFallback());
                            if (options.shouldRetryOnQuotaViolation()) {
                                retryTopics.add(result.name());
                                retryTopicQuotaExceededExceptions.put(result.name(), quotaExceededException);
                            } else {
                                future.completeExceptionally(quotaExceededException);
                            }
                        } else {
                            future.completeExceptionally(error.exception());
                        }
                    } else {
                        future.complete(null);
                    }
                }
            }
            // If there are topics to retry, retry them; complete unrealized futures otherwise.
            if (retryTopics.isEmpty()) {
                // The server should send back a response for every topic. But do a sanity check anyway.
                completeUnrealizedFutures(futures.entrySet().stream(), topic -> "The controller response did not contain a result for topic " + topic);
            } else {
                final long now = time.milliseconds();
                final Call call = getDeleteTopicsCall(options, futures, retryTopics, retryTopicQuotaExceededExceptions, now, deadline);
                runnable.call(call, now);
            }
        }

        @Override
        void handleFailure(Throwable throwable) {
            // If there were any topics retries due to a quota exceeded exception, we propagate
            // the initial error back to the caller if the request timed out.
            maybeCompleteQuotaExceededException(options.shouldRetryOnQuotaViolation(), throwable, futures, quotaExceededExceptions, (int) (time.milliseconds() - now));
            // Fail all the other remaining futures
            completeAllExceptionally(futures.values(), throwable);
        }
    };
}

Example 2 with ThrottlingQuotaExceededException

use of org.apache.kafka.common.errors.ThrottlingQuotaExceededException in project kafka by apache.

the class KafkaAdminClientTest method testCreatePartitionsRetryThrottlingExceptionWhenEnabledUntilRequestTimeOut.

@Test
public void testCreatePartitionsRetryThrottlingExceptionWhenEnabledUntilRequestTimeOut() throws Exception {
    long defaultApiTimeout = 60000;
    MockTime time = new MockTime();
    try (AdminClientUnitTestEnv env = mockClientEnv(time, AdminClientConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, String.valueOf(defaultApiTimeout))) {
        env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
        env.kafkaClient().prepareResponse(expectCreatePartitionsRequestWithTopics("topic1", "topic2", "topic3"), prepareCreatePartitionsResponse(1000, createPartitionsTopicResult("topic1", Errors.NONE), createPartitionsTopicResult("topic2", Errors.THROTTLING_QUOTA_EXCEEDED), createPartitionsTopicResult("topic3", Errors.TOPIC_ALREADY_EXISTS)));
        env.kafkaClient().prepareResponse(expectCreatePartitionsRequestWithTopics("topic2"), prepareCreatePartitionsResponse(1000, createPartitionsTopicResult("topic2", Errors.THROTTLING_QUOTA_EXCEEDED)));
        Map<String, NewPartitions> counts = new HashMap<>();
        counts.put("topic1", NewPartitions.increaseTo(1));
        counts.put("topic2", NewPartitions.increaseTo(2));
        counts.put("topic3", NewPartitions.increaseTo(3));
        CreatePartitionsResult result = env.adminClient().createPartitions(counts, new CreatePartitionsOptions().retryOnQuotaViolation(true));
        // Wait until the prepared attempts have consumed
        TestUtils.waitForCondition(() -> env.kafkaClient().numAwaitingResponses() == 0, "Failed awaiting CreatePartitions requests");
        // Wait until the next request is sent out
        TestUtils.waitForCondition(() -> env.kafkaClient().inFlightRequestCount() == 1, "Failed awaiting next CreatePartitions request");
        // Advance time past the default api timeout to time out the inflight request
        time.sleep(defaultApiTimeout + 1);
        assertNull(result.values().get("topic1").get());
        ThrottlingQuotaExceededException e = TestUtils.assertFutureThrows(result.values().get("topic2"), ThrottlingQuotaExceededException.class);
        assertEquals(0, e.throttleTimeMs());
        TestUtils.assertFutureThrows(result.values().get("topic3"), TopicExistsException.class);
    }
}

Example 3 with ThrottlingQuotaExceededException

use of org.apache.kafka.common.errors.ThrottlingQuotaExceededException in project kafka by apache.

the class KafkaAdminClientTest method testCreateTopicsDontRetryThrottlingExceptionWhenDisabled.

@Test
public void testCreateTopicsDontRetryThrottlingExceptionWhenDisabled() throws Exception {
    try (AdminClientUnitTestEnv env = mockClientEnv()) {
        env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
        env.kafkaClient().prepareResponse(expectCreateTopicsRequestWithTopics("topic1", "topic2", "topic3"), prepareCreateTopicsResponse(1000, creatableTopicResult("topic1", Errors.NONE), creatableTopicResult("topic2", Errors.THROTTLING_QUOTA_EXCEEDED), creatableTopicResult("topic3", Errors.TOPIC_ALREADY_EXISTS)));
        CreateTopicsResult result = env.adminClient().createTopics(asList(new NewTopic("topic1", 1, (short) 1), new NewTopic("topic2", 1, (short) 1), new NewTopic("topic3", 1, (short) 1)), new CreateTopicsOptions().retryOnQuotaViolation(false));
        assertNull(result.values().get("topic1").get());
        ThrottlingQuotaExceededException e = TestUtils.assertFutureThrows(result.values().get("topic2"), ThrottlingQuotaExceededException.class);
        assertEquals(1000, e.throttleTimeMs());
        TestUtils.assertFutureThrows(result.values().get("topic3"), TopicExistsException.class);
    }
}

Example 4 with ThrottlingQuotaExceededException

use of org.apache.kafka.common.errors.ThrottlingQuotaExceededException in project kafka by apache.

the class KafkaAdminClientTest method testCreateTopicsRetryThrottlingExceptionWhenEnabledUntilRequestTimeOut.

@Test
public void testCreateTopicsRetryThrottlingExceptionWhenEnabledUntilRequestTimeOut() throws Exception {
    long defaultApiTimeout = 60000;
    MockTime time = new MockTime();
    try (AdminClientUnitTestEnv env = mockClientEnv(time, AdminClientConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, String.valueOf(defaultApiTimeout))) {
        env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
        env.kafkaClient().prepareResponse(expectCreateTopicsRequestWithTopics("topic1", "topic2", "topic3"), prepareCreateTopicsResponse(1000, creatableTopicResult("topic1", Errors.NONE), creatableTopicResult("topic2", Errors.THROTTLING_QUOTA_EXCEEDED), creatableTopicResult("topic3", Errors.TOPIC_ALREADY_EXISTS)));
        env.kafkaClient().prepareResponse(expectCreateTopicsRequestWithTopics("topic2"), prepareCreateTopicsResponse(1000, creatableTopicResult("topic2", Errors.THROTTLING_QUOTA_EXCEEDED)));
        CreateTopicsResult result = env.adminClient().createTopics(asList(new NewTopic("topic1", 1, (short) 1), new NewTopic("topic2", 1, (short) 1), new NewTopic("topic3", 1, (short) 1)), new CreateTopicsOptions().retryOnQuotaViolation(true));
        // Wait until the prepared attempts have consumed
        TestUtils.waitForCondition(() -> env.kafkaClient().numAwaitingResponses() == 0, "Failed awaiting CreateTopics requests");
        // Wait until the next request is sent out
        TestUtils.waitForCondition(() -> env.kafkaClient().inFlightRequestCount() == 1, "Failed awaiting next CreateTopics request");
        // Advance time past the default api timeout to time out the inflight request
        time.sleep(defaultApiTimeout + 1);
        assertNull(result.values().get("topic1").get());
        ThrottlingQuotaExceededException e = TestUtils.assertFutureThrows(result.values().get("topic2"), ThrottlingQuotaExceededException.class);
        assertEquals(0, e.throttleTimeMs());
        TestUtils.assertFutureThrows(result.values().get("topic3"), TopicExistsException.class);
    }
}

Example 5 with ThrottlingQuotaExceededException

use of org.apache.kafka.common.errors.ThrottlingQuotaExceededException in project kafka by apache.

the class KafkaAdminClientTest method testDeleteTopicsRetryThrottlingExceptionWhenEnabledUntilRequestTimeOut.

@Test
public void testDeleteTopicsRetryThrottlingExceptionWhenEnabledUntilRequestTimeOut() throws Exception {
    long defaultApiTimeout = 60000;
    MockTime time = new MockTime();
    try (AdminClientUnitTestEnv env = mockClientEnv(time, AdminClientConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, String.valueOf(defaultApiTimeout))) {
        env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
        env.kafkaClient().prepareResponse(expectDeleteTopicsRequestWithTopics("topic1", "topic2", "topic3"), prepareDeleteTopicsResponse(1000, deletableTopicResult("topic1", Errors.NONE), deletableTopicResult("topic2", Errors.THROTTLING_QUOTA_EXCEEDED), deletableTopicResult("topic3", Errors.TOPIC_ALREADY_EXISTS)));
        env.kafkaClient().prepareResponse(expectDeleteTopicsRequestWithTopics("topic2"), prepareDeleteTopicsResponse(1000, deletableTopicResult("topic2", Errors.THROTTLING_QUOTA_EXCEEDED)));
        DeleteTopicsResult result = env.adminClient().deleteTopics(asList("topic1", "topic2", "topic3"), new DeleteTopicsOptions().retryOnQuotaViolation(true));
        // Wait until the prepared attempts have consumed
        TestUtils.waitForCondition(() -> env.kafkaClient().numAwaitingResponses() == 0, "Failed awaiting DeleteTopics requests");
        // Wait until the next request is sent out
        TestUtils.waitForCondition(() -> env.kafkaClient().inFlightRequestCount() == 1, "Failed awaiting next DeleteTopics request");
        // Advance time past the default api timeout to time out the inflight request
        time.sleep(defaultApiTimeout + 1);
        assertNull(result.topicNameValues().get("topic1").get());
        ThrottlingQuotaExceededException e = TestUtils.assertFutureThrows(result.topicNameValues().get("topic2"), ThrottlingQuotaExceededException.class);
        assertEquals(0, e.throttleTimeMs());
        TestUtils.assertFutureThrows(result.topicNameValues().get("topic3"), TopicExistsException.class);
        // With topic IDs
        Uuid topicId1 = Uuid.randomUuid();
        Uuid topicId2 = Uuid.randomUuid();
        Uuid topicId3 = Uuid.randomUuid();
        env.kafkaClient().prepareResponse(expectDeleteTopicsRequestWithTopicIds(topicId1, topicId2, topicId3), prepareDeleteTopicsResponse(1000, deletableTopicResultWithId(topicId1, Errors.NONE), deletableTopicResultWithId(topicId2, Errors.THROTTLING_QUOTA_EXCEEDED), deletableTopicResultWithId(topicId3, Errors.UNKNOWN_TOPIC_ID)));
        env.kafkaClient().prepareResponse(expectDeleteTopicsRequestWithTopicIds(topicId2), prepareDeleteTopicsResponse(1000, deletableTopicResultWithId(topicId2, Errors.THROTTLING_QUOTA_EXCEEDED)));
        DeleteTopicsResult resultIds = env.adminClient().deleteTopics(TopicCollection.ofTopicIds(asList(topicId1, topicId2, topicId3)), new DeleteTopicsOptions().retryOnQuotaViolation(true));
        // Wait until the prepared attempts have consumed
        TestUtils.waitForCondition(() -> env.kafkaClient().numAwaitingResponses() == 0, "Failed awaiting DeleteTopics requests");
        // Wait until the next request is sent out
        TestUtils.waitForCondition(() -> env.kafkaClient().inFlightRequestCount() == 1, "Failed awaiting next DeleteTopics request");
        // Advance time past the default api timeout to time out the inflight request
        time.sleep(defaultApiTimeout + 1);
        assertNull(resultIds.topicIdValues().get(topicId1).get());
        e = TestUtils.assertFutureThrows(resultIds.topicIdValues().get(topicId2), ThrottlingQuotaExceededException.class);
        assertEquals(0, e.throttleTimeMs());
        TestUtils.assertFutureThrows(resultIds.topicIdValues().get(topicId3), UnknownTopicIdException.class);
    }
}