Examples with MessageAndOffset kafka.message.MessageAndOffset used on opensource projects

Example 21 with MessageAndOffset

use of kafka.message.MessageAndOffset in project storm by apache.

the class TridentKafkaEmitter method reEmitPartitionBatch.

/**
     * re-emit the batch described by the meta data provided
     *
     * @param attempt
     * @param collector
     * @param partition
     * @param meta
     */
private void reEmitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map meta) {
    LOG.info("re-emitting batch, attempt " + attempt);
    String instanceId = (String) meta.get("instanceId");
    if (!_config.ignoreZkOffsets || instanceId.equals(_topologyInstanceId)) {
        SimpleConsumer consumer = _connections.register(partition);
        long offset = (Long) meta.get("offset");
        long nextOffset = (Long) meta.get("nextOffset");
        ByteBufferMessageSet msgs = null;
        msgs = fetchMessages(consumer, partition, offset);
        if (msgs != null) {
            for (MessageAndOffset msg : msgs) {
                if (offset == nextOffset) {
                    break;
                }
                if (offset > nextOffset) {
                    throw new RuntimeException("Error when re-emitting batch. overshot the end offset");
                }
                emit(collector, msg.message(), partition, msg.offset(), attempt);
                offset = msg.nextOffset();
            }
        }
    }
}

Example 22 with MessageAndOffset

use of kafka.message.MessageAndOffset in project storm by apache.

the class TridentKafkaEmitter method doEmitNewPartitionBatch.

private Map doEmitNewPartitionBatch(SimpleConsumer consumer, Partition partition, TridentCollector collector, Map lastMeta, TransactionAttempt attempt) {
    LOG.debug("Emitting new partition batch - [transaction = {}], [lastMeta = {}]", attempt, lastMeta);
    long offset;
    if (lastMeta != null) {
        String lastInstanceId = null;
        Map lastTopoMeta = (Map) lastMeta.get("topology");
        if (lastTopoMeta != null) {
            lastInstanceId = (String) lastTopoMeta.get("id");
        }
        if (_config.ignoreZkOffsets && !_topologyInstanceId.equals(lastInstanceId)) {
            offset = KafkaUtils.getOffset(consumer, partition.topic, partition.partition, _config.startOffsetTime);
        } else {
            offset = (Long) lastMeta.get("nextOffset");
        }
    } else {
        offset = KafkaUtils.getOffset(consumer, partition.topic, partition.partition, _config);
    }
    LOG.debug("[transaction = {}], [OFFSET = {}]", attempt, offset);
    ByteBufferMessageSet msgs = null;
    try {
        msgs = fetchMessages(consumer, partition, offset);
    } catch (TopicOffsetOutOfRangeException e) {
        long newOffset = KafkaUtils.getOffset(consumer, partition.topic, partition.partition, kafka.api.OffsetRequest.EarliestTime());
        LOG.warn("OffsetOutOfRange: Updating offset from offset = " + offset + " to offset = " + newOffset);
        offset = newOffset;
        msgs = KafkaUtils.fetchMessages(_config, consumer, partition, offset);
    }
    long endoffset = offset;
    for (MessageAndOffset msg : msgs) {
        emit(collector, msg.message(), partition, msg.offset(), attempt);
        endoffset = msg.nextOffset();
    }
    Map newMeta = new HashMap();
    newMeta.put("offset", offset);
    newMeta.put("nextOffset", endoffset);
    newMeta.put("instanceId", _topologyInstanceId);
    newMeta.put("partition", partition.partition);
    newMeta.put("broker", ImmutableMap.of("host", partition.host.host, "port", partition.host.port));
    newMeta.put("topic", partition.topic);
    newMeta.put("topology", ImmutableMap.of("name", _topologyName, "id", _topologyInstanceId));
    LOG.debug("[transaction = {}], [newMeta = {}]", attempt, newMeta);
    return newMeta;
}

Example 23 with MessageAndOffset

use of kafka.message.MessageAndOffset in project storm by apache.

the class KafkaUtilsTest method generateTuplesWithValueAndStringMultiSchemeWithTopic.

@Test
public void generateTuplesWithValueAndStringMultiSchemeWithTopic() {
    config.scheme = new StringMultiSchemeWithTopic();
    String value = "value";
    createTopicAndSendMessage(value);
    ByteBufferMessageSet messageAndOffsets = getLastMessage();
    for (MessageAndOffset msg : messageAndOffsets) {
        Iterable<List<Object>> lists = KafkaUtils.generateTuples(config, msg.message(), config.topic);
        List<Object> list = lists.iterator().next();
        assertEquals(value, list.get(0));
        assertEquals(config.topic, list.get(1));
    }
}

Example 24 with MessageAndOffset

use of kafka.message.MessageAndOffset in project storm by apache.

the class KafkaUtilsTest method generateTuplesWithValueSchemeAndKeyValueMessage.

@Test
public void generateTuplesWithValueSchemeAndKeyValueMessage() {
    config.scheme = new SchemeAsMultiScheme(new StringScheme());
    String value = "value";
    String key = "key";
    createTopicAndSendMessage(key, value);
    ByteBufferMessageSet messageAndOffsets = getLastMessage();
    for (MessageAndOffset msg : messageAndOffsets) {
        Iterable<List<Object>> lists = KafkaUtils.generateTuples(config, msg.message(), config.topic);
        assertEquals(value, lists.iterator().next().get(0));
    }
}

Example 25 with MessageAndOffset

use of kafka.message.MessageAndOffset in project storm by apache.

the class KafkaUtilsTest method generateTuplesWithMessageAndMetadataScheme.

@Test
public void generateTuplesWithMessageAndMetadataScheme() {
    String value = "value";
    Partition mockPartition = Mockito.mock(Partition.class);
    mockPartition.partition = 0;
    long offset = 0L;
    MessageMetadataSchemeAsMultiScheme scheme = new MessageMetadataSchemeAsMultiScheme(new StringMessageAndMetadataScheme());
    createTopicAndSendMessage(null, value);
    ByteBufferMessageSet messageAndOffsets = getLastMessage();
    for (MessageAndOffset msg : messageAndOffsets) {
        Iterable<List<Object>> lists = KafkaUtils.generateTuples(scheme, msg.message(), mockPartition, offset);
        List<Object> values = lists.iterator().next();
        assertEquals("Message is incorrect", value, values.get(0));
        assertEquals("Partition is incorrect", mockPartition.partition, values.get(1));
        assertEquals("Offset is incorrect", offset, values.get(2));
    }
}