Examples with Message kafka.message.Message used on opensource projects

Example 11 with Message

use of kafka.message.Message in project avro-kafka-storm by ransilberman.

the class MainTest method testGenericRecord.

@Test
public void testGenericRecord() throws IOException, InterruptedException {
    Schema.Parser parser = new Schema.Parser();
    Schema schema = parser.parse(getClass().getResourceAsStream("LPEvent.avsc"));
    GenericRecord datum = new GenericData.Record(schema);
    datum.put("revision", 1L);
    datum.put("siteId", "28280110");
    datum.put("eventType", "PLine");
    datum.put("timeStamp", System.currentTimeMillis());
    datum.put("sessionId", "123456II");
    Map<String, Schema> unions = new HashMap<String, Schema>();
    List<Schema> typeList = schema.getField("subrecord").schema().getTypes();
    for (Schema sch : typeList) {
        unions.put(sch.getName(), sch);
    }
    GenericRecord plineDatum = new GenericData.Record(unions.get("pline"));
    plineDatum.put("text", "How can I help you?");
    plineDatum.put("lineType", 1);
    plineDatum.put("repId", "REPID12345");
    datum.put("subrecord", plineDatum);
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    DatumWriter<GenericRecord> writer = new GenericDatumWriter<GenericRecord>(schema);
    Encoder encoder = EncoderFactory.get().binaryEncoder(out, null);
    writer.write(datum, encoder);
    encoder.flush();
    out.close();
    Message message = new Message(out.toByteArray());
    Properties props = new Properties();
    props.put("zk.connect", zkConnection);
    Producer<Message, Message> producer = new kafka.javaapi.producer.Producer<Message, Message>(new ProducerConfig(props));
    producer.send(new ProducerData<Message, Message>(topic, message));
}

Example 12 with Message

use of kafka.message.Message in project bagheera by mozilla-metrics.

the class ProducerTest method countMessages.

private int countMessages() throws InvalidProtocolBufferException {
    SimpleConsumer consumer = new SimpleConsumer("localhost", KAFKA_BROKER_PORT, 100, 1024);
    long offset = 0l;
    int messageCount = 0;
    for (int i = 0; i < BATCH_SIZE; i++) {
        ByteBufferMessageSet messageSet = consumer.fetch(new FetchRequest(KAFKA_TOPIC, 0, offset, 1024));
        Iterator<MessageAndOffset> iterator = messageSet.iterator();
        MessageAndOffset msgAndOff;
        while (iterator.hasNext()) {
            messageCount++;
            msgAndOff = iterator.next();
            offset = msgAndOff.offset();
            Message message2 = msgAndOff.message();
            BagheeraMessage bmsg = BagheeraMessage.parseFrom(ByteString.copyFrom(message2.payload()));
            String payload = new String(bmsg.getPayload().toByteArray());
            System.out.println(String.format("Message %d @%d: %s", messageCount, offset, payload));
        }
    }
    consumer.close();
    return messageCount;
}

Example 13 with Message

use of kafka.message.Message in project graylog2-server by Graylog2.

the class KafkaJournal method write.

/**
     * Writes the list of entries to the journal.
     *
     * @param entries journal entries to be written
     * @return the last position written to in the journal
     */
@Override
public long write(List<Entry> entries) {
    try (Timer.Context ignored = writeTime.time()) {
        long payloadSize = 0L;
        long messageSetSize = 0L;
        long lastWriteOffset = 0L;
        final List<Message> messages = new ArrayList<>(entries.size());
        for (final Entry entry : entries) {
            final byte[] messageBytes = entry.getMessageBytes();
            final byte[] idBytes = entry.getIdBytes();
            payloadSize += messageBytes.length;
            final Message newMessage = new Message(messageBytes, idBytes);
            // Calculate the size of the new message in the message set by including the overhead for the log entry.
            final int newMessageSize = MessageSet.entrySize(newMessage);
            if (newMessageSize > maxMessageSize) {
                writeDiscardedMessages.mark();
                LOG.warn("Message with ID <{}> is too large to store in journal, skipping! (size: {} bytes / max: {} bytes)", new String(idBytes, StandardCharsets.UTF_8), newMessageSize, maxMessageSize);
                payloadSize = 0;
                continue;
            }
            // list of message to avoid a MessageSetSizeTooLargeException.
            if ((messageSetSize + newMessageSize) > maxSegmentSize) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Flushing {} bytes message set with {} messages to avoid overflowing segment with max size of {} bytes", messageSetSize, messages.size(), maxSegmentSize);
                }
                lastWriteOffset = flushMessages(messages, payloadSize);
                // Reset the messages list and size counters to start a new batch.
                messages.clear();
                messageSetSize = 0;
                payloadSize = 0;
            }
            messages.add(newMessage);
            messageSetSize += newMessageSize;
            if (LOG.isTraceEnabled()) {
                LOG.trace("Message {} contains bytes {}", bytesToHex(idBytes), bytesToHex(messageBytes));
            }
        }
        // Flush the rest of the messages.
        if (messages.size() > 0) {
            lastWriteOffset = flushMessages(messages, payloadSize);
        }
        return lastWriteOffset;
    }
}