Examples with BufferExhaustedException org.apache.kafka.clients.producer.BufferExhaustedException used on opensource projects

Example 1 with BufferExhaustedException

use of org.apache.kafka.clients.producer.BufferExhaustedException in project kafka by apache.

the class BufferPool method allocate.

/**
 * Allocate a buffer of the given size. This method blocks if there is not enough memory and the buffer pool
 * is configured with blocking mode.
 *
 * @param size The buffer size to allocate in bytes
 * @param maxTimeToBlockMs The maximum time in milliseconds to block for buffer memory to be available
 * @return The buffer
 * @throws InterruptedException If the thread is interrupted while blocked
 * @throws IllegalArgumentException if size is larger than the total memory controlled by the pool (and hence we would block
 *         forever)
 */
public ByteBuffer allocate(int size, long maxTimeToBlockMs) throws InterruptedException {
    if (size > this.totalMemory)
        throw new IllegalArgumentException("Attempt to allocate " + size + " bytes, but there is a hard limit of " + this.totalMemory + " on memory allocations.");
    ByteBuffer buffer = null;
    this.lock.lock();
    if (this.closed) {
        this.lock.unlock();
        throw new KafkaException("Producer closed while allocating memory");
    }
    try {
        // check if we have a free buffer of the right size pooled
        if (size == poolableSize && !this.free.isEmpty())
            return this.free.pollFirst();
        // now check if the request is immediately satisfiable with the
        // memory on hand or if we need to block
        int freeListSize = freeSize() * this.poolableSize;
        if (this.nonPooledAvailableMemory + freeListSize >= size) {
            // we have enough unallocated or pooled memory to immediately
            // satisfy the request, but need to allocate the buffer
            freeUp(size);
            this.nonPooledAvailableMemory -= size;
        } else {
            // we are out of memory and will have to block
            int accumulated = 0;
            Condition moreMemory = this.lock.newCondition();
            try {
                long remainingTimeToBlockNs = TimeUnit.MILLISECONDS.toNanos(maxTimeToBlockMs);
                this.waiters.addLast(moreMemory);
                // enough memory to allocate one
                while (accumulated < size) {
                    long startWaitNs = time.nanoseconds();
                    long timeNs;
                    boolean waitingTimeElapsed;
                    try {
                        waitingTimeElapsed = !moreMemory.await(remainingTimeToBlockNs, TimeUnit.NANOSECONDS);
                    } finally {
                        long endWaitNs = time.nanoseconds();
                        timeNs = Math.max(0L, endWaitNs - startWaitNs);
                        recordWaitTime(timeNs);
                    }
                    if (this.closed)
                        throw new KafkaException("Producer closed while allocating memory");
                    if (waitingTimeElapsed) {
                        this.metrics.sensor("buffer-exhausted-records").record();
                        throw new BufferExhaustedException("Failed to allocate " + size + " bytes within the configured max blocking time " + maxTimeToBlockMs + " ms. Total memory: " + totalMemory() + " bytes. Available memory: " + availableMemory() + " bytes. Poolable size: " + poolableSize() + " bytes");
                    }
                    remainingTimeToBlockNs -= timeNs;
                    // otherwise allocate memory
                    if (accumulated == 0 && size == this.poolableSize && !this.free.isEmpty()) {
                        // just grab a buffer from the free list
                        buffer = this.free.pollFirst();
                        accumulated = size;
                    } else {
                        // we'll need to allocate memory, but we may only get
                        // part of what we need on this iteration
                        freeUp(size - accumulated);
                        int got = (int) Math.min(size - accumulated, this.nonPooledAvailableMemory);
                        this.nonPooledAvailableMemory -= got;
                        accumulated += got;
                    }
                }
                // Don't reclaim memory on throwable since nothing was thrown
                accumulated = 0;
            } finally {
                // When this loop was not able to successfully terminate don't loose available memory
                this.nonPooledAvailableMemory += accumulated;
                this.waiters.remove(moreMemory);
            }
        }
    } finally {
        // over for them
        try {
            if (!(this.nonPooledAvailableMemory == 0 && this.free.isEmpty()) && !this.waiters.isEmpty())
                this.waiters.peekFirst().signal();
        } finally {
            // Another finally... otherwise find bugs complains
            lock.unlock();
        }
    }
    if (buffer == null)
        return safeAllocateByteBuffer(size);
    else
        return buffer;
}

Example 2 with BufferExhaustedException

use of org.apache.kafka.clients.producer.BufferExhaustedException in project kafka by apache.

the class BufferPoolTest method testBlockTimeout.

/**
 * Verify that a failed allocation attempt due to not enough memory finishes soon after the maxBlockTimeMs.
 */
@Test
public void testBlockTimeout() throws Exception {
    BufferPool pool = new BufferPool(10, 1, metrics, Time.SYSTEM, metricGroup);
    ByteBuffer buffer1 = pool.allocate(1, maxBlockTimeMs);
    ByteBuffer buffer2 = pool.allocate(1, maxBlockTimeMs);
    ByteBuffer buffer3 = pool.allocate(1, maxBlockTimeMs);
    // The first two buffers will be de-allocated within maxBlockTimeMs since the most recent allocation
    delayedDeallocate(pool, buffer1, maxBlockTimeMs / 2);
    delayedDeallocate(pool, buffer2, maxBlockTimeMs);
    // The third buffer will be de-allocated after maxBlockTimeMs since the most recent allocation
    delayedDeallocate(pool, buffer3, maxBlockTimeMs / 2 * 5);
    long beginTimeMs = Time.SYSTEM.milliseconds();
    try {
        pool.allocate(10, maxBlockTimeMs);
        fail("The buffer allocated more memory than its maximum value 10");
    } catch (BufferExhaustedException e) {
    // this is good
    }
    // Thread scheduling sometimes means that deallocation varies by this point
    assertTrue(pool.availableMemory() >= 7 && pool.availableMemory() <= 10, "available memory " + pool.availableMemory());
    long durationMs = Time.SYSTEM.milliseconds() - beginTimeMs;
    assertTrue(durationMs >= maxBlockTimeMs, "BufferExhaustedException should not throw before maxBlockTimeMs");
    assertTrue(durationMs < maxBlockTimeMs + 1000, "BufferExhaustedException should throw soon after maxBlockTimeMs");
}