Examples with IncomingMessageEnvelope org.apache.samza.system.IncomingMessageEnvelope used on opensource projects

Example 21 with IncomingMessageEnvelope

use of org.apache.samza.system.IncomingMessageEnvelope in project samza by apache.

the class TestMultiFileHdfsReader method testReaderReopen.

@Test
public void testReaderReopen() throws Exception {
    SystemStreamPartition ssp = new SystemStreamPartition("hdfs", "testStream", new Partition(0));
    // read until the middle of the first file
    MultiFileHdfsReader multiReader = new MultiFileHdfsReader(HdfsReaderFactory.ReaderType.AVRO, ssp, Arrays.asList(descriptors), "0:0");
    int index = 0;
    String offset = "0:0";
    for (; index < NUM_EVENTS / 2; index++) {
        IncomingMessageEnvelope envelope = multiReader.readNext();
        GenericRecord record = (GenericRecord) envelope.getMessage();
        Assert.assertEquals(index % NUM_EVENTS, record.get(FIELD_1));
        Assert.assertEquals("string_" + (index % NUM_EVENTS), record.get(FIELD_2).toString());
        offset = envelope.getOffset();
    }
    multiReader.close();
    // read until the middle of the second file
    multiReader = new MultiFileHdfsReader(HdfsReaderFactory.ReaderType.AVRO, ssp, Arrays.asList(descriptors), offset);
    // skip one duplicate event
    multiReader.readNext();
    for (; index < NUM_EVENTS + NUM_EVENTS / 2; index++) {
        IncomingMessageEnvelope envelope = multiReader.readNext();
        GenericRecord record = (GenericRecord) envelope.getMessage();
        Assert.assertEquals(index % NUM_EVENTS, record.get(FIELD_1));
        Assert.assertEquals("string_" + (index % NUM_EVENTS), record.get(FIELD_2).toString());
        offset = envelope.getOffset();
    }
    multiReader.close();
    // read the rest of all files
    multiReader = new MultiFileHdfsReader(HdfsReaderFactory.ReaderType.AVRO, ssp, Arrays.asList(descriptors), offset);
    // skip one duplicate event
    multiReader.readNext();
    while (multiReader.hasNext()) {
        IncomingMessageEnvelope envelope = multiReader.readNext();
        GenericRecord record = (GenericRecord) envelope.getMessage();
        Assert.assertEquals(index % NUM_EVENTS, record.get(FIELD_1));
        Assert.assertEquals("string_" + (index % NUM_EVENTS), record.get(FIELD_2).toString());
        index++;
        offset = envelope.getOffset();
    }
    Assert.assertEquals(3 * NUM_EVENTS, index);
    multiReader.close();
    // reopen with the offset of the last record
    multiReader = new MultiFileHdfsReader(HdfsReaderFactory.ReaderType.AVRO, ssp, Arrays.asList(descriptors), offset);
    // skip one duplicate event
    multiReader.readNext();
    Assert.assertFalse(multiReader.hasNext());
    multiReader.close();
}

Example 22 with IncomingMessageEnvelope

use of org.apache.samza.system.IncomingMessageEnvelope in project samza by apache.

the class TestMultiFileHdfsReader method testReachingMaxReconnect.

@Test(expected = SamzaException.class)
public void testReachingMaxReconnect() {
    int numMaxRetries = 3;
    SystemStreamPartition ssp = new SystemStreamPartition("hdfs", "testStream", new Partition(0));
    MultiFileHdfsReader multiReader = new MultiFileHdfsReader(HdfsReaderFactory.ReaderType.AVRO, ssp, Arrays.asList(descriptors), "0:0", numMaxRetries);
    // first read a few events, and then reconnect
    for (int i = 0; i < NUM_EVENTS / 2; i++) {
        multiReader.readNext();
    }
    for (int i = 0; i < numMaxRetries; i++) {
        IncomingMessageEnvelope envelope = multiReader.readNext();
        multiReader.reconnect();
        IncomingMessageEnvelope envelopeAfterReconnect = multiReader.readNext();
        Assert.assertEquals(envelope, envelopeAfterReconnect);
    }
    multiReader.readNext();
    multiReader.reconnect();
    Assert.fail();
}

Example 23 with IncomingMessageEnvelope

use of org.apache.samza.system.IncomingMessageEnvelope in project samza by apache.

the class CoordinatorStreamSystemConsumer method getUnreadMessages.

/**
   * returns all unread messages of a specific type, after an iterator on the stream
   *
   * @param iterator the iterator pointing to an offset in the coordinator stream. All unread messages after this iterator are returned
   * @param type     the type of the messages to be returned
   * @return a set of unread messages of a given type, after a given iterator
   */
public Set<CoordinatorStreamMessage> getUnreadMessages(SystemStreamPartitionIterator iterator, String type) {
    LinkedHashSet<CoordinatorStreamMessage> messages = new LinkedHashSet<CoordinatorStreamMessage>();
    while (iterator.hasNext()) {
        IncomingMessageEnvelope envelope = iterator.next();
        Object[] keyArray = keySerde.fromBytes((byte[]) envelope.getKey()).toArray();
        Map<String, Object> valueMap = null;
        if (envelope.getMessage() != null) {
            valueMap = messageSerde.fromBytes((byte[]) envelope.getMessage());
        }
        CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, valueMap);
        if (type == null || type.equals(coordinatorStreamMessage.getType())) {
            messages.add(coordinatorStreamMessage);
        }
    }
    return messages;
}