Examples with SystemStream org.apache.samza.system.SystemStream used on opensource projects

Example 6 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class Emitter method window.

public void window(MessageCollector collector, TaskCoordinator coordinator) {
    Integer epoch = getInt(EPOCH);
    if (epoch == null) {
        resetEpoch();
        return;
    }
    int counter = getInt(COUNT);
    if (counter < max) {
        logger.info("Emitting: " + counter + ", epoch = " + epoch + ", task = " + taskName);
        OutgoingMessageEnvelope envelope = new OutgoingMessageEnvelope(new SystemStream("kafka", "emitted"), Integer.toString(counter), epoch + "-" + taskName.toString());
        collector.send(envelope);
        this.state.put(COUNT, Integer.toString(getInt(COUNT) + 1));
    }
}

Example 7 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class TestMessageStreamImpl method testSink.

@Test
public void testSink() {
    MessageStreamImpl<TestMessageEnvelope> inputStream = new MessageStreamImpl<>(mockGraph);
    SystemStream testStream = new SystemStream("test-sys", "test-stream");
    SinkFunction<TestMessageEnvelope> xSink = (TestMessageEnvelope m, MessageCollector mc, TaskCoordinator tc) -> {
        mc.send(new OutgoingMessageEnvelope(testStream, m.getMessage()));
        tc.commit(TaskCoordinator.RequestScope.CURRENT_TASK);
    };
    inputStream.sink(xSink);
    Collection<OperatorSpec> subs = inputStream.getRegisteredOperatorSpecs();
    assertEquals(subs.size(), 1);
    OperatorSpec<TestMessageEnvelope> sinkOp = subs.iterator().next();
    assertTrue(sinkOp instanceof SinkOperatorSpec);
    assertEquals(((SinkOperatorSpec) sinkOp).getSinkFn(), xSink);
    TestMessageEnvelope mockTest1 = mock(TestMessageEnvelope.class);
    MessageType mockMsgBody = mock(MessageType.class);
    when(mockTest1.getMessage()).thenReturn(mockMsgBody);
    final List<OutgoingMessageEnvelope> outMsgs = new ArrayList<>();
    MessageCollector mockCollector = mock(MessageCollector.class);
    doAnswer(invocation -> {
        outMsgs.add((OutgoingMessageEnvelope) invocation.getArguments()[0]);
        return null;
    }).when(mockCollector).send(any());
    TaskCoordinator mockCoordinator = mock(TaskCoordinator.class);
    ((SinkOperatorSpec) sinkOp).getSinkFn().apply(mockTest1, mockCollector, mockCoordinator);
    assertEquals(1, outMsgs.size());
    assertEquals(testStream, outMsgs.get(0).getSystemStream());
    assertEquals(mockMsgBody, outMsgs.get(0).getMessage());
}

Example 8 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class TestCoordinatorStreamSystemConsumer method testCoordinatorStreamSystemConsumer.

@Test
public void testCoordinatorStreamSystemConsumer() {
    Map<String, String> expectedConfig = new LinkedHashMap<String, String>();
    expectedConfig.put("job.id", "1234");
    SystemStream systemStream = new SystemStream("system", "stream");
    MockSystemConsumer systemConsumer = new MockSystemConsumer(new SystemStreamPartition(systemStream, new Partition(0)));
    CoordinatorStreamSystemConsumer consumer = new CoordinatorStreamSystemConsumer(systemStream, systemConsumer, new SinglePartitionWithoutOffsetsSystemAdmin());
    assertEquals(0, systemConsumer.getRegisterCount());
    consumer.register();
    assertEquals(1, systemConsumer.getRegisterCount());
    assertFalse(systemConsumer.isStarted());
    consumer.start();
    assertTrue(systemConsumer.isStarted());
    try {
        consumer.getConfig();
        fail("Should have failed when retrieving config before bootstrapping.");
    } catch (SamzaException e) {
    // Expected.
    }
    consumer.bootstrap();
    assertEquals(expectedConfig, consumer.getConfig());
    assertFalse(systemConsumer.isStopped());
    consumer.stop();
    assertTrue(systemConsumer.isStopped());
}

Example 9 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class TestCoordinatorStreamSystemConsumer method testCoordinatorStreamSystemConsumerRegisterOnceOnly.

@Test
public void testCoordinatorStreamSystemConsumerRegisterOnceOnly() throws Exception {
    Map<String, String> expectedConfig = new LinkedHashMap<String, String>();
    expectedConfig.put("job.id", "1234");
    SystemStream systemStream = new SystemStream("system", "stream");
    MockSystemConsumer systemConsumer = new MockSystemConsumer(new SystemStreamPartition(systemStream, new Partition(0)));
    CoordinatorStreamSystemConsumer consumer = new CoordinatorStreamSystemConsumer(systemStream, systemConsumer, new SinglePartitionWithoutOffsetsSystemAdmin());
    assertEquals(0, systemConsumer.getRegisterCount());
    consumer.register();
    assertEquals(1, systemConsumer.getRegisterCount());
    assertFalse(systemConsumer.isStarted());
    consumer.start();
    assertTrue(systemConsumer.isStarted());
    consumer.register();
    assertEquals(1, systemConsumer.getRegisterCount());
}

Example 10 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class TestStreamConfig method testContainsSamzaPropertyThrowsIfInvalidPropertyName.

@Test(expected = IllegalArgumentException.class)
public void testContainsSamzaPropertyThrowsIfInvalidPropertyName() {
    StreamConfig config = buildConfig("key1", "value1", "key2", "value2");
    config.containsSamzaProperty(new SystemStream("SysX", "StrX"), "key1");
}