Examples with ByteBufferMessageSet kafka.javaapi.message.ByteBufferMessageSet used on opensource projects

Example 26 with ByteBufferMessageSet

use of kafka.javaapi.message.ByteBufferMessageSet in project storm by apache.

the class KafkaBoltTest method executeWithByteArrayKeyAndMessageAsync.

/* test asynchronous sending (default) */
@Test
public void executeWithByteArrayKeyAndMessageAsync() {
    boolean async = true;
    boolean fireAndForget = false;
    String keyString = "test-key";
    String messageString = "test-message";
    byte[] key = keyString.getBytes();
    byte[] message = messageString.getBytes();
    final Tuple tuple = generateTestTuple(key, message);
    final ByteBufferMessageSet mockMsg = mockSingleMessage(key, message);
    simpleConsumer.close();
    simpleConsumer = mockSimpleConsumer(mockMsg);
    KafkaProducer<?, ?> producer = mock(KafkaProducer.class);
    when(producer.send(any(ProducerRecord.class), any(Callback.class))).thenAnswer(new Answer<Future>() {

        @Override
        public Future answer(InvocationOnMock invocationOnMock) throws Throwable {
            Callback cb = (Callback) invocationOnMock.getArguments()[1];
            cb.onCompletion(null, null);
            return mock(Future.class);
        }
    });
    bolt = generateDefaultSerializerBolt(async, fireAndForget, producer);
    bolt.execute(tuple);
    verify(collector).ack(tuple);
    verifyMessage(keyString, messageString);
}

Example 27 with ByteBufferMessageSet

use of kafka.javaapi.message.ByteBufferMessageSet in project storm by apache.

the class KafkaBoltTest method verifyMessage.

private boolean verifyMessage(String key, String message) {
    long lastMessageOffset = KafkaUtils.getOffset(simpleConsumer, kafkaConfig.topic, 0, OffsetRequest.LatestTime()) - 1;
    ByteBufferMessageSet messageAndOffsets = KafkaUtils.fetchMessages(kafkaConfig, simpleConsumer, new Partition(Broker.fromString(broker.getBrokerConnectionString()), kafkaConfig.topic, 0), lastMessageOffset);
    MessageAndOffset messageAndOffset = messageAndOffsets.iterator().next();
    Message kafkaMessage = messageAndOffset.message();
    ByteBuffer messageKeyBuffer = kafkaMessage.key();
    String keyString = null;
    String messageString = new String(Utils.toByteArray(kafkaMessage.payload()));
    if (messageKeyBuffer != null) {
        keyString = new String(Utils.toByteArray(messageKeyBuffer));
    }
    assertEquals(key, keyString);
    assertEquals(message, messageString);
    return true;
}

Example 28 with ByteBufferMessageSet

use of kafka.javaapi.message.ByteBufferMessageSet in project jstorm by alibaba.

the class PartitionConsumer method fillMessages.

private void fillMessages() {
    ByteBufferMessageSet msgs;
    try {
        long start = System.currentTimeMillis();
        msgs = consumer.fetchMessages(partition, emittingOffset + 1);
        if (msgs == null) {
            LOG.error("fetch null message from offset {}", emittingOffset);
            return;
        }
        int count = 0;
        for (MessageAndOffset msg : msgs) {
            count += 1;
            emittingMessages.add(msg);
            emittingOffset = msg.offset();
            pendingOffsets.add(emittingOffset);
            LOG.debug("fillmessage fetched a message:{}, offset:{}", msg.message().toString(), msg.offset());
        }
        long end = System.currentTimeMillis();
        LOG.info("fetch message from partition:" + partition + ", offset:" + emittingOffset + ", size:" + msgs.sizeInBytes() + ", count:" + count + ", time:" + (end - start));
    } catch (Exception e) {
        e.printStackTrace();
        LOG.error(e.getMessage(), e);
    }
}

Example 29 with ByteBufferMessageSet

use of kafka.javaapi.message.ByteBufferMessageSet in project bagheera by mozilla-metrics.

the class ProducerTest method countMessages.

private int countMessages() throws InvalidProtocolBufferException {
    SimpleConsumer consumer = new SimpleConsumer("localhost", KAFKA_BROKER_PORT, 100, 1024);
    long offset = 0l;
    int messageCount = 0;
    for (int i = 0; i < BATCH_SIZE; i++) {
        ByteBufferMessageSet messageSet = consumer.fetch(new FetchRequest(KAFKA_TOPIC, 0, offset, 1024));
        Iterator<MessageAndOffset> iterator = messageSet.iterator();
        MessageAndOffset msgAndOff;
        while (iterator.hasNext()) {
            messageCount++;
            msgAndOff = iterator.next();
            offset = msgAndOff.offset();
            Message message2 = msgAndOff.message();
            BagheeraMessage bmsg = BagheeraMessage.parseFrom(ByteString.copyFrom(message2.payload()));
            String payload = new String(bmsg.getPayload().toByteArray());
            System.out.println(String.format("Message %d @%d: %s", messageCount, offset, payload));
        }
    }
    consumer.close();
    return messageCount;
}

Example 30 with ByteBufferMessageSet

use of kafka.javaapi.message.ByteBufferMessageSet in project heron by twitter.

the class KafkaUtilsTest method generateTuplesWithMessageAndMetadataScheme.

@Test
public void generateTuplesWithMessageAndMetadataScheme() {
    String value = "value";
    Partition mockPartition = Mockito.mock(Partition.class);
    mockPartition.partition = 0;
    long offset = 0L;
    MessageMetadataSchemeAsMultiScheme scheme = new MessageMetadataSchemeAsMultiScheme(new StringMessageAndMetadataScheme());
    createTopicAndSendMessage(null, value);
    ByteBufferMessageSet messageAndOffsets = getLastMessage();
    for (MessageAndOffset msg : messageAndOffsets) {
        Iterable<List<Object>> lists = KafkaUtils.generateTuples(scheme, msg.message(), mockPartition, offset);
        List<Object> values = lists.iterator().next();
        assertEquals("Message is incorrect", value, values.get(0));
        assertEquals("Partition is incorrect", mockPartition.partition, values.get(1));
        assertEquals("Offset is incorrect", offset, values.get(2));
    }
}