Examples with KafkaMessageId nl.minvenj.nfi.storm.kafka.util.KafkaMessageId used on opensource projects

Example 11 with KafkaMessageId

use of nl.minvenj.nfi.storm.kafka.util.KafkaMessageId in project kafka-spout by HolmesNL.

the class KafkaSpout method ack.

@Override
public void ack(final Object o) {
    if (o instanceof KafkaMessageId) {
        final KafkaMessageId id = (KafkaMessageId) o;
        // message corresponding to o is no longer pending
        _inProgress.remove(id);
        LOG.debug("kafka message {} acknowledged", id);
        if (_inProgress.isEmpty()) {
            // commit offsets to zookeeper when pending is now empty
            // (buffer will be filled on next call to nextTuple())
            LOG.debug("all pending messages acknowledged, committing client offsets");
            _consumer.commitOffsets();
        }
        // notify fail handler of tuple success
        _failHandler.ack(id);
    }
}

Example 12 with KafkaMessageId

use of nl.minvenj.nfi.storm.kafka.util.KafkaMessageId in project kafka-spout by HolmesNL.

the class KafkaSpoutBufferBehaviourTest method testRefillBothOnMessageAvailable.

@Test
public void testRefillBothOnMessageAvailable() {
    // NB: update the consumer mock for this test to return the single message stream
    when(_consumer.createMessageStreams(any(Map.class))).thenReturn(SINGLE_MESSAGE_STREAM);
    final KafkaMessageId id = new KafkaMessageId(1, 1234);
    // test whether the single message in the stream is added to both the buffer and pending
    _subject.fillBuffer();
    assertEquals(1, _subject._queue.size());
    assertEquals(id, _subject._queue.peek());
    assertEquals(1, _subject._inProgress.size());
    assertEquals(id, _subject._inProgress.firstKey());
}

Example 13 with KafkaMessageId

use of nl.minvenj.nfi.storm.kafka.util.KafkaMessageId in project kafka-spout by HolmesNL.

the class KafkaSpoutFailurePolicyTest method testReliableFailure.

@Test
public void testReliableFailure() {
    final FailHandler reliable = spy(new ReliableFailHandler());
    _subject._failHandler = reliable;
    final KafkaMessageId id = new KafkaMessageId(1, 1234);
    final byte[] message = { 1, 2, 3, 4 };
    _subject._inProgress.put(id, message);
    // fail the message
    _subject.fail(id);
    // verify decision delegate to fail handler and replay
    verify(reliable).shouldReplay(eq(id));
    assertEquals(id, _subject._queue.peek());
    assertTrue(_subject._inProgress.containsKey(id));
}

Example 14 with KafkaMessageId

use of nl.minvenj.nfi.storm.kafka.util.KafkaMessageId in project kafka-spout by HolmesNL.

the class KafkaSpoutFailurePolicyTest method testUnreliableFailure.

@Test
public void testUnreliableFailure() {
    final FailHandler unreliable = spy(new UnreliableFailHandler());
    _subject._failHandler = unreliable;
    final KafkaMessageId id = new KafkaMessageId(1, 1234);
    final byte[] message = { 1, 2, 3, 4 };
    _subject._inProgress.put(id, message);
    // fail the message
    _subject.fail(id);
    // verify decision and message delegate to fail handler and no replay
    verify(unreliable).shouldReplay(eq(id));
    verify(unreliable).fail(eq(id), eq(message));
    assertTrue(_subject._queue.isEmpty());
    assertFalse(_subject._inProgress.containsKey(id));
}

Example 15 with KafkaMessageId

use of nl.minvenj.nfi.storm.kafka.util.KafkaMessageId in project kafka-spout by HolmesNL.

the class ReliableFailHandlerTest method testShouldReplay.

@Test
public void testShouldReplay() {
    final FailHandler subject = new ReliableFailHandler();
    final KafkaMessageId id = new KafkaMessageId(1, 1234);
    // reliable handler should *always* tell the spout to replay
    assertTrue(subject.shouldReplay(id));
}