Examples with Callback org.apache.kafka.clients.producer.Callback used on opensource projects

Example 1 with Callback

use of org.apache.kafka.clients.producer.Callback in project kafka by apache.

the class RecordAccumulatorTest method testAppendInExpiryCallback.

@Test
public void testAppendInExpiryCallback() throws InterruptedException {
    long retryBackoffMs = 100L;
    long lingerMs = 3000L;
    int requestTimeout = 60;
    int messagesPerBatch = 1024 / msgSize;
    final RecordAccumulator accum = new RecordAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, retryBackoffMs, metrics, time);
    final AtomicInteger expiryCallbackCount = new AtomicInteger();
    final AtomicReference<Exception> unexpectedException = new AtomicReference<Exception>();
    Callback callback = new Callback() {

        @Override
        public void onCompletion(RecordMetadata metadata, Exception exception) {
            if (exception instanceof TimeoutException) {
                expiryCallbackCount.incrementAndGet();
                try {
                    accum.append(tp1, 0L, key, value, null, maxBlockTimeMs);
                } catch (InterruptedException e) {
                    throw new RuntimeException("Unexpected interruption", e);
                }
            } else if (exception != null)
                unexpectedException.compareAndSet(null, exception);
        }
    };
    for (int i = 0; i < messagesPerBatch + 1; i++) accum.append(tp1, 0L, key, value, callback, maxBlockTimeMs);
    assertEquals(2, accum.batches().get(tp1).size());
    assertTrue("First batch not full", accum.batches().get(tp1).peekFirst().isFull());
    // Advance the clock to expire the first batch.
    time.sleep(requestTimeout + 1);
    List<ProducerBatch> expiredBatches = accum.abortExpiredBatches(requestTimeout, time.milliseconds());
    assertEquals("The batch was not expired", 1, expiredBatches.size());
    assertEquals("Callbacks not invoked for expiry", messagesPerBatch, expiryCallbackCount.get());
    assertNull("Unexpected exception", unexpectedException.get());
    assertEquals("Some messages not appended from expiry callbacks", 2, accum.batches().get(tp1).size());
    assertTrue("First batch not full after expiry callbacks with appends", accum.batches().get(tp1).peekFirst().isFull());
}

Example 2 with Callback

use of org.apache.kafka.clients.producer.Callback in project kafka by apache.

the class RecordCollectorTest method shouldThrowStreamsExceptionOnSubsequentCallIfASendFails.

@SuppressWarnings("unchecked")
@Test(expected = StreamsException.class)
public void shouldThrowStreamsExceptionOnSubsequentCallIfASendFails() throws Exception {
    final RecordCollector collector = new RecordCollectorImpl(new MockProducer(cluster, true, new DefaultPartitioner(), byteArraySerializer, byteArraySerializer) {

        @Override
        public synchronized Future<RecordMetadata> send(final ProducerRecord record, final Callback callback) {
            callback.onCompletion(null, new Exception());
            return null;
        }
    }, "test");
    collector.send("topic1", "3", "0", null, null, stringSerializer, stringSerializer, streamPartitioner);
    collector.send("topic1", "3", "0", null, null, stringSerializer, stringSerializer, streamPartitioner);
}

Example 3 with Callback

use of org.apache.kafka.clients.producer.Callback in project kafka by apache.

the class RecordCollectorTest method shouldThrowStreamsExceptionOnCloseIfASendFailed.

@SuppressWarnings("unchecked")
@Test(expected = StreamsException.class)
public void shouldThrowStreamsExceptionOnCloseIfASendFailed() throws Exception {
    final RecordCollector collector = new RecordCollectorImpl(new MockProducer(cluster, true, new DefaultPartitioner(), byteArraySerializer, byteArraySerializer) {

        @Override
        public synchronized Future<RecordMetadata> send(final ProducerRecord record, final Callback callback) {
            callback.onCompletion(null, new Exception());
            return null;
        }
    }, "test");
    collector.send("topic1", "3", "0", null, null, stringSerializer, stringSerializer, streamPartitioner);
    collector.close();
}

Example 4 with Callback

use of org.apache.kafka.clients.producer.Callback in project kafka by apache.

the class SmokeTestDriver method generate.

public static Map<String, Set<Integer>> generate(String kafka, final int numKeys, final int maxRecordsPerKey) throws Exception {
    Properties props = new Properties();
    props.put(ProducerConfig.CLIENT_ID_CONFIG, "SmokeTest");
    props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka);
    props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class);
    props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class);
    KafkaProducer<byte[], byte[]> producer = new KafkaProducer<>(props);
    int numRecordsProduced = 0;
    Map<String, Set<Integer>> allData = new HashMap<>();
    ValueList[] data = new ValueList[numKeys];
    for (int i = 0; i < numKeys; i++) {
        data[i] = new ValueList(i, i + maxRecordsPerKey - 1);
        allData.put(data[i].key, new HashSet<Integer>());
    }
    Random rand = new Random();
    int remaining = data.length;
    while (remaining > 0) {
        int index = rand.nextInt(remaining);
        String key = data[index].key;
        int value = data[index].next();
        if (value < 0) {
            remaining--;
            data[index] = data[remaining];
        } else {
            ProducerRecord<byte[], byte[]> record = new ProducerRecord<>("data", stringSerde.serializer().serialize("", key), intSerde.serializer().serialize("", value));
            producer.send(record, new Callback() {

                @Override
                public void onCompletion(final RecordMetadata metadata, final Exception exception) {
                    if (exception != null) {
                        exception.printStackTrace();
                        Exit.exit(1);
                    }
                }
            });
            numRecordsProduced++;
            allData.get(key).add(value);
            if (numRecordsProduced % 100 == 0)
                System.out.println(numRecordsProduced + " records produced");
            Utils.sleep(2);
        }
    }
    producer.close();
    return Collections.unmodifiableMap(allData);
}

Example 5 with Callback

use of org.apache.kafka.clients.producer.Callback in project storm by apache.

the class KafkaBoltTest method executeWithByteArrayKeyAndMessageFire.

/* test with fireAndForget option enabled */
@Test
public void executeWithByteArrayKeyAndMessageFire() {
    boolean async = true;
    boolean fireAndForget = true;
    bolt = generateDefaultSerializerBolt(async, fireAndForget, null);
    String keyString = "test-key";
    String messageString = "test-message";
    byte[] key = keyString.getBytes();
    byte[] message = messageString.getBytes();
    Tuple tuple = generateTestTuple(key, message);
    final ByteBufferMessageSet mockMsg = mockSingleMessage(key, message);
    simpleConsumer.close();
    simpleConsumer = mockSimpleConsumer(mockMsg);
    KafkaProducer<?, ?> producer = mock(KafkaProducer.class);
    // do not invoke the callback of send() in order to test whether the bolt handle the fireAndForget option
    // properly.
    doReturn(mock(Future.class)).when(producer).send(any(ProducerRecord.class), any(Callback.class));
    bolt.execute(tuple);
    verify(collector).ack(tuple);
    verifyMessage(keyString, messageString);
}