Examples with SystemConsumer org.apache.samza.system.SystemConsumer used on opensource projects

Example 16 with SystemConsumer

use of org.apache.samza.system.SystemConsumer in project samza by apache.

the class TestInMemorySystem method testConsumerRespectsOffset.

@Test
public void testConsumerRespectsOffset() {
    PageViewEvent event = new PageViewEvent(TEST_MEMBER_X, PAGE_ID_X, System.currentTimeMillis());
    PageViewEvent event1 = new PageViewEvent(TEST_MEMBER_Y, PAGE_ID_Y, System.currentTimeMillis());
    produceMessages(event);
    SystemConsumer consumer = systemFactory.getConsumer(SYSTEM_NAME, config, mockRegistry);
    Set<SystemStreamPartition> sspsToPoll = IntStream.range(0, PARTITION_COUNT).mapToObj(partition -> new SystemStreamPartition(SYSTEM_STREAM, new Partition(partition))).collect(Collectors.toSet());
    // register the consumer for ssps
    for (SystemStreamPartition ssp : sspsToPoll) {
        consumer.register(ssp, "0");
    }
    List<PageViewEvent> results = consumeMessages(consumer, sspsToPoll);
    assertEquals(1, results.size());
    assertTrue(results.contains(event));
    // nothing to poll
    results = consumeMessages(consumer, sspsToPoll);
    assertEquals(0, results.size());
    produceMessages(event1);
    // got new message. check if the offset has progressed
    results = consumeMessages(consumer, sspsToPoll);
    assertEquals(1, results.size());
    assertTrue(results.contains(event1));
}

Example 17 with SystemConsumer

use of org.apache.samza.system.SystemConsumer in project samza by apache.

the class TestInMemorySystem method testNullMessageWithValidMessageKey.

@Test
public void testNullMessageWithValidMessageKey() {
    final String messageKey = "validKey";
    SystemProducer systemProducer = systemFactory.getProducer(SYSTEM_NAME, config, mockRegistry);
    systemProducer.send(SOURCE, new OutgoingMessageEnvelope(SYSTEM_STREAM, messageKey, null));
    SystemConsumer consumer = systemFactory.getConsumer(SYSTEM_NAME, config, mockRegistry);
    Set<SystemStreamPartition> sspsToPoll = IntStream.range(0, PARTITION_COUNT).mapToObj(partition -> new SystemStreamPartition(SYSTEM_STREAM, new Partition(partition))).collect(Collectors.toSet());
    // register the consumer for ssps
    for (SystemStreamPartition ssp : sspsToPoll) {
        consumer.register(ssp, "0");
    }
    List<IncomingMessageEnvelope> results = consumeRawMessages(consumer, sspsToPoll);
    assertEquals(1, results.size());
    assertEquals(results.get(0).getKey(), messageKey);
    assertNull(results.get(0).getMessage());
}

Example 18 with SystemConsumer

use of org.apache.samza.system.SystemConsumer in project beam by apache.

the class SamzaImpulseSystemTest method testSamzaImpulseSystemConsumer.

@Test
public void testSamzaImpulseSystemConsumer() throws Exception {
    SystemConsumer consumer = new SamzaImpulseSystemFactory().getConsumer("default-system", new MapConfig(), null);
    Map<SystemStreamPartition, List<IncomingMessageEnvelope>> result = consumer.poll(Collections.singleton(sspForPartition(0)), 100);
    Assert.assertEquals(1, result.size());
    Assert.assertTrue(result.containsKey(sspForPartition(0)));
    List<IncomingMessageEnvelope> messageEnvelopes = result.get(sspForPartition(0));
    Assert.assertEquals(3, messageEnvelopes.size());
    Assert.assertTrue(messageEnvelopes.get(0).getMessage() instanceof OpMessage);
    OpMessage impulseEvent = (OpMessage) messageEnvelopes.get(0).getMessage();
    Assert.assertEquals(OpMessage.Type.ELEMENT, impulseEvent.getType());
    Assert.assertTrue(messageEnvelopes.get(1).getMessage() instanceof WatermarkMessage);
    Assert.assertTrue(messageEnvelopes.get(2).isEndOfStream());
}