Examples with SystemStream org.apache.samza.system.SystemStream used on opensource projects

Example 96 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class TestEventHubSystemProducer method testSendingToEventHubHashing.

@Test
public void testSendingToEventHubHashing() throws Exception {
    String systemName = "eventhubs";
    String streamName = "testStream";
    int numEvents = 10;
    String partitionId0 = "124";
    String partitionId1 = "235";
    TestMetricsRegistry testMetrics = new TestMetricsRegistry();
    Map<String, Interceptor> interceptor = new HashMap<>();
    interceptor.put(streamName, new PassThroughInterceptor());
    List<String> outgoingMessagesP0 = generateMessages(numEvents);
    List<String> outgoingMessagesP1 = generateMessages(numEvents);
    // Set configs
    Map<String, String> configMap = new HashMap<>();
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_LIST, systemName), streamName);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, streamName), EVENTHUB_NAMESPACE);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, streamName), EVENTHUB_KEY_NAME);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, streamName), EVENTHUB_KEY);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, streamName), EVENTHUB_ENTITY1);
    // mod 2 on the partitionid to simulate consistent hashing
    configMap.put(String.format(EventHubConfig.CONFIG_PRODUCER_PARTITION_METHOD, systemName), PartitioningMethod.EVENT_HUB_HASHING.toString());
    MapConfig config = new MapConfig(configMap);
    MockEventHubClientManagerFactory factory = new MockEventHubClientManagerFactory();
    EventHubSystemProducer producer = new EventHubSystemProducer(new EventHubConfig(config), systemName, factory, interceptor, testMetrics);
    SystemStream systemStream = new SystemStream(systemName, streamName);
    producer.register(SOURCE);
    producer.start();
    outgoingMessagesP0.forEach(message -> producer.send(SOURCE, new OutgoingMessageEnvelope(systemStream, partitionId0, null, message.getBytes())));
    outgoingMessagesP1.forEach(message -> producer.send(SOURCE, new OutgoingMessageEnvelope(systemStream, partitionId1, null, message.getBytes())));
    // Retrieve sent data
    List<String> receivedData0 = factory.getSentData(systemName, streamName, 0).stream().map(eventData -> new String(eventData.getBytes())).collect(Collectors.toList());
    List<String> receivedData1 = factory.getSentData(systemName, streamName, 1).stream().map(eventData -> new String(eventData.getBytes())).collect(Collectors.toList());
    Assert.assertTrue(outgoingMessagesP0.equals(receivedData0));
    Assert.assertTrue(outgoingMessagesP1.equals(receivedData1));
}

Example 97 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class TestAzureBlobSystemProducer method createOME.

private OutgoingMessageEnvelope createOME(String streamName) {
    SystemStream systemStream = new SystemStream(SYSTEM_NAME, streamName);
    DummyPageViewEvent record = new DummyPageViewEvent();
    return new OutgoingMessageEnvelope(systemStream, record);
}

Example 98 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class TestStreamOperatorTask method testExceptionIfInputOperatorMissing.

/**
 * Tests if the appropriate SamzaException is propagated to the TaskCallback if there is no InputOperator for a given
 * SystemStream in the OperatorGraph.
 */
@Test
public void testExceptionIfInputOperatorMissing() throws NoSuchFieldException, IllegalAccessException {
    IncomingMessageEnvelope mockIme = mock(IncomingMessageEnvelope.class, RETURNS_DEEP_STUBS);
    SystemStream testSystemStream = new SystemStream("foo", "bar");
    when(mockIme.getSystemStreamPartition().getSystemStream()).thenReturn(testSystemStream);
    OperatorImplGraph mockOperatorImplGraph = mock(OperatorImplGraph.class);
    when(mockOperatorImplGraph.getInputOperator(anyObject())).thenReturn(null);
    StreamOperatorTask operatorTask = new StreamOperatorTask(mock(OperatorSpecGraph.class));
    operatorTask.setOperatorImplGraph(mockOperatorImplGraph);
    TaskCallback mockTaskCallback = mock(TaskCallback.class);
    operatorTask.processAsync(mockIme, mock(MessageCollector.class), mock(TaskCoordinator.class), mockTaskCallback);
    ArgumentCaptor<Throwable> throwableCaptor = ArgumentCaptor.forClass(Throwable.class);
    verify(mockTaskCallback, only()).failure(throwableCaptor.capture());
    assertEquals(throwableCaptor.getValue().getClass(), SamzaException.class);
    String expectedErrMessage = String.format("InputOperator not found in OperatorGraph for %s. The available input" + " operators are: %s. Please check SystemStream configuration for the `SystemConsumer` and/or task.inputs" + " task configuration.", testSystemStream, mockOperatorImplGraph.getAllInputOperators());
    assertEquals(throwableCaptor.getValue().getMessage(), expectedErrMessage);
}

Example 99 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class IdentityStreamTask method process.

@Override
public void process(IncomingMessageEnvelope incomingMessageEnvelope, MessageCollector messageCollector, TaskCoordinator taskCoordinator) throws Exception {
    messageCollector.send(new OutgoingMessageEnvelope(new SystemStream(outputSystem, outputTopic), incomingMessageEnvelope.getMessage()));
    processedMessageCount++;
    if (processedMessageCount == expectedMessageCount) {
        taskCoordinator.shutdown(TaskCoordinator.RequestScope.ALL_TASKS_IN_CONTAINER);
    }
}

Example 100 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class IdentityStreamTask method process.

@Override
public void process(IncomingMessageEnvelope incomingMessageEnvelope, MessageCollector messageCollector, TaskCoordinator taskCoordinator) throws Exception {
    messageCollector.send(new OutgoingMessageEnvelope(new SystemStream(outputSystem, outputTopic), incomingMessageEnvelope.getMessage()));
    processedMessageCount++;
    if (processedMessageCount == expectedMessageCount) {
        taskCoordinator.shutdown(TaskCoordinator.RequestScope.ALL_TASKS_IN_CONTAINER);
    }
}