Examples with IncomingMessageEnvelope org.apache.samza.system.IncomingMessageEnvelope used on opensource projects

Example 16 with IncomingMessageEnvelope

use of org.apache.samza.system.IncomingMessageEnvelope in project samza by apache.

the class BlockingEnvelopeMap method poll.

/**
   * {@inheritDoc}
   */
public Map<SystemStreamPartition, List<IncomingMessageEnvelope>> poll(Set<SystemStreamPartition> systemStreamPartitions, long timeout) throws InterruptedException {
    long stopTime = clock.currentTimeMillis() + timeout;
    Map<SystemStreamPartition, List<IncomingMessageEnvelope>> messagesToReturn = new HashMap<SystemStreamPartition, List<IncomingMessageEnvelope>>();
    metrics.incPoll();
    for (SystemStreamPartition systemStreamPartition : systemStreamPartitions) {
        BlockingQueue<IncomingMessageEnvelope> queue = bufferedMessages.get(systemStreamPartition);
        List<IncomingMessageEnvelope> outgoingList = new ArrayList<IncomingMessageEnvelope>(queue.size());
        if (queue.size() > 0) {
            queue.drainTo(outgoingList);
        } else if (timeout != 0) {
            IncomingMessageEnvelope envelope = null;
            // How long we can legally block (if timeout > 0)
            long timeRemaining = stopTime - clock.currentTimeMillis();
            if (timeout == SystemConsumer.BLOCK_ON_OUTSTANDING_MESSAGES) {
                // the head of the stream.
                while (envelope == null && !isAtHead(systemStreamPartition)) {
                    metrics.incBlockingPoll(systemStreamPartition);
                    envelope = queue.poll(1000, TimeUnit.MILLISECONDS);
                }
            } else if (timeout > 0 && timeRemaining > 0) {
                // Block until we get at least one message.
                metrics.incBlockingTimeoutPoll(systemStreamPartition);
                envelope = queue.poll(timeRemaining, TimeUnit.MILLISECONDS);
            }
            // If we got a message, add it.
            if (envelope != null) {
                outgoingList.add(envelope);
                // Drain any remaining messages without blocking.
                queue.drainTo(outgoingList);
            }
        }
        if (outgoingList.size() > 0) {
            messagesToReturn.put(systemStreamPartition, outgoingList);
            subtractSizeOnQDrain(systemStreamPartition, outgoingList);
        }
    }
    return messagesToReturn;
}

Example 17 with IncomingMessageEnvelope

use of org.apache.samza.system.IncomingMessageEnvelope in project samza by apache.

the class AvroFileHdfsReader method readNext.

@Override
public IncomingMessageEnvelope readNext() {
    // get checkpoint for THIS record
    String checkpoint = nextOffset();
    GenericRecord record = fileReader.next();
    if (fileReader.previousSync() != curBlockStart) {
        curBlockStart = fileReader.previousSync();
        curRecordOffset = 0;
    } else {
        curRecordOffset++;
    }
    // avro schema doesn't necessarily have key field
    return new IncomingMessageEnvelope(systemStreamPartition, checkpoint, null, record);
}

Example 18 with IncomingMessageEnvelope

use of org.apache.samza.system.IncomingMessageEnvelope in project samza by apache.

the class TestTaskCallbackManager method testUpdateCallbackOutofOrder.

@Test
public void testUpdateCallbackOutofOrder() {
    TaskName taskName = new TaskName("Partition 0");
    SystemStreamPartition ssp = new SystemStreamPartition("kafka", "topic", new Partition(0));
    ReadableCoordinator coordinator = new ReadableCoordinator(taskName);
    // simulate out of order
    IncomingMessageEnvelope envelope2 = new IncomingMessageEnvelope(ssp, "2", null, null);
    TaskCallbackImpl callback2 = new TaskCallbackImpl(listener, taskName, envelope2, coordinator, 2, 0);
    List<TaskCallbackImpl> callbacksToUpdate = callbackManager.updateCallback(callback2);
    assertTrue(callbacksToUpdate.isEmpty());
    IncomingMessageEnvelope envelope1 = new IncomingMessageEnvelope(ssp, "1", null, null);
    TaskCallbackImpl callback1 = new TaskCallbackImpl(listener, taskName, envelope1, coordinator, 1, 0);
    callbacksToUpdate = callbackManager.updateCallback(callback1);
    assertTrue(callbacksToUpdate.isEmpty());
    IncomingMessageEnvelope envelope0 = new IncomingMessageEnvelope(ssp, "0", null, null);
    TaskCallbackImpl callback0 = new TaskCallbackImpl(listener, taskName, envelope0, coordinator, 0, 0);
    callbacksToUpdate = callbackManager.updateCallback(callback0);
    assertEquals(3, callbacksToUpdate.size());
    TaskCallbackImpl callback = callbacksToUpdate.get(0);
    assertTrue(callback.matchSeqNum(0));
    assertEquals(ssp, callback.envelope.getSystemStreamPartition());
    assertEquals("0", callback.envelope.getOffset());
    callback = callbacksToUpdate.get(1);
    assertTrue(callback.matchSeqNum(1));
    assertEquals(ssp, callback.envelope.getSystemStreamPartition());
    assertEquals("1", callback.envelope.getOffset());
    callback = callbacksToUpdate.get(2);
    assertTrue(callback.matchSeqNum(2));
    assertEquals(ssp, callback.envelope.getSystemStreamPartition());
    assertEquals("2", callback.envelope.getOffset());
}

Example 19 with IncomingMessageEnvelope

use of org.apache.samza.system.IncomingMessageEnvelope in project samza by apache.

the class TestAsyncRunLoop method testEndOfStreamOffsetManagement.

// TODO: Add assertions.
//@Test
public void testEndOfStreamOffsetManagement() throws Exception {
    //explicitly configure to disable commits inside process or window calls and invoke commit from end of stream
    TestTask mockStreamTask1 = new TestTask(true, false, false, null);
    TestTask mockStreamTask2 = new TestTask(true, false, false, null);
    Partition p1 = new Partition(1);
    Partition p2 = new Partition(2);
    SystemStreamPartition ssp1 = new SystemStreamPartition("system1", "stream1", p1);
    SystemStreamPartition ssp2 = new SystemStreamPartition("system1", "stream2", p2);
    IncomingMessageEnvelope envelope1 = new IncomingMessageEnvelope(ssp2, "1", "key1", "message1");
    IncomingMessageEnvelope envelope2 = new IncomingMessageEnvelope(ssp2, "2", "key1", "message1");
    IncomingMessageEnvelope envelope3 = IncomingMessageEnvelope.buildEndOfStreamEnvelope(ssp2);
    Map<SystemStreamPartition, List<IncomingMessageEnvelope>> sspMap = new HashMap<>();
    List<IncomingMessageEnvelope> messageList = new ArrayList<>();
    messageList.add(envelope1);
    messageList.add(envelope2);
    messageList.add(envelope3);
    sspMap.put(ssp2, messageList);
    SystemConsumer mockConsumer = mock(SystemConsumer.class);
    when(mockConsumer.poll(anyObject(), anyLong())).thenReturn(sspMap);
    HashMap<String, SystemConsumer> systemConsumerMap = new HashMap<>();
    systemConsumerMap.put("system1", mockConsumer);
    SystemConsumers consumers = TestSystemConsumers.getSystemConsumers(systemConsumerMap);
    TaskName taskName1 = new TaskName("task1");
    TaskName taskName2 = new TaskName("task2");
    Set<TaskName> taskNames = new HashSet<>();
    taskNames.add(taskName1);
    taskNames.add(taskName2);
    OffsetManager offsetManager = mock(OffsetManager.class);
    when(offsetManager.getLastProcessedOffset(taskName1, ssp1)).thenReturn(Option.apply("3"));
    when(offsetManager.getLastProcessedOffset(taskName2, ssp2)).thenReturn(Option.apply("0"));
    when(offsetManager.getStartingOffset(taskName1, ssp1)).thenReturn(Option.apply(IncomingMessageEnvelope.END_OF_STREAM_OFFSET));
    when(offsetManager.getStartingOffset(taskName2, ssp2)).thenReturn(Option.apply("1"));
    TaskInstance taskInstance1 = createTaskInstance(mockStreamTask1, taskName1, ssp1, offsetManager, consumers);
    TaskInstance taskInstance2 = createTaskInstance(mockStreamTask2, taskName2, ssp2, offsetManager, consumers);
    Map<TaskName, TaskInstance> tasks = new HashMap<>();
    tasks.put(taskName1, taskInstance1);
    tasks.put(taskName2, taskInstance2);
    taskInstance1.registerConsumers();
    taskInstance2.registerConsumers();
    consumers.start();
    int maxMessagesInFlight = 1;
    AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumers, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, false);
    runLoop.run();
}

Example 20 with IncomingMessageEnvelope

use of org.apache.samza.system.IncomingMessageEnvelope in project samza by apache.

the class TestAvroFileHdfsReader method testRandomRead.

@Test
public void testRandomRead() throws Exception {
    SystemStreamPartition ssp = new SystemStreamPartition("hdfs", "testStream", new Partition(0));
    SingleFileHdfsReader reader = new AvroFileHdfsReader(ssp);
    reader.open(AVRO_FILE, "0");
    for (int i = 0; i < NUM_EVENTS / 2; i++) {
        reader.readNext();
    }
    String offset = reader.nextOffset();
    IncomingMessageEnvelope envelope = reader.readNext();
    Assert.assertEquals(offset, envelope.getOffset());
    GenericRecord record1 = (GenericRecord) envelope.getMessage();
    for (int i = 0; i < 5; i++) reader.readNext();
    // seek to the offset within the same reader
    reader.seek(offset);
    Assert.assertEquals(offset, reader.nextOffset());
    envelope = reader.readNext();
    Assert.assertEquals(offset, envelope.getOffset());
    GenericRecord record2 = (GenericRecord) envelope.getMessage();
    Assert.assertEquals(record1, record2);
    reader.close();
    // open a new reader and initialize it with the offset
    reader = new AvroFileHdfsReader(ssp);
    reader.open(AVRO_FILE, offset);
    envelope = reader.readNext();
    Assert.assertEquals(offset, envelope.getOffset());
    GenericRecord record3 = (GenericRecord) envelope.getMessage();
    Assert.assertEquals(record1, record3);
    reader.close();
}