Examples with SystemStream org.apache.samza.system.SystemStream used on opensource projects

Example 1 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class StreamOperatorTask method init.

/**
   * Initializes this task during startup.
   * <p>
   * Implementation: Initializes the user-implemented {@link StreamApplication}. The {@link StreamApplication} sets
   * the input and output streams and the task-wide context manager using the {@link StreamGraphImpl} APIs,
   * and the logical transforms using the {@link org.apache.samza.operators.MessageStream} APIs.
   *<p>
   * It then uses the {@link StreamGraphImpl} to create the {@link OperatorImplGraph} corresponding to the logical
   * DAG. It also saves the mapping between input {@link SystemStream}s and their corresponding
   * {@link InputStreamInternal}s for delivering incoming messages to the appropriate sub-DAG.
   *
   * @param config allows accessing of fields in the configuration files that this StreamTask is specified in
   * @param context allows initializing and accessing contextual data of this StreamTask
   * @throws Exception in case of initialization errors
   */
@Override
public final void init(Config config, TaskContext context) throws Exception {
    StreamGraphImpl streamGraph = new StreamGraphImpl(runner, config);
    // initialize the user-implemented stream application.
    this.streamApplication.init(streamGraph, config);
    // get the user-implemented context manager and initialize it
    this.contextManager = streamGraph.getContextManager();
    if (this.contextManager != null) {
        this.contextManager.init(config, context);
    }
    // create the operator impl DAG corresponding to the logical operator spec DAG
    OperatorImplGraph operatorImplGraph = new OperatorImplGraph(clock);
    operatorImplGraph.init(streamGraph, config, context);
    this.operatorImplGraph = operatorImplGraph;
    // TODO: SAMZA-1118 - Remove mapping after SystemConsumer starts returning logical streamId with incoming messages
    inputSystemStreamToInputStream = new HashMap<>();
    streamGraph.getInputStreams().forEach((streamSpec, inputStream) -> {
        SystemStream systemStream = new SystemStream(streamSpec.getSystemName(), streamSpec.getPhysicalName());
        inputSystemStreamToInputStream.put(systemStream, inputStream);
    });
}

Example 2 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class TestOperatorSpecs method testCreateSinkOperator.

@Test
public void testCreateSinkOperator() {
    SystemStream testStream = new SystemStream("test-sys", "test-stream");
    SinkFunction<TestMessageEnvelope> sinkFn = (TestMessageEnvelope message, MessageCollector messageCollector, TaskCoordinator taskCoordinator) -> {
        messageCollector.send(new OutgoingMessageEnvelope(testStream, message.getKey(), message.getMessage()));
    };
    SinkOperatorSpec<TestMessageEnvelope> sinkOp = OperatorSpecs.createSinkOperatorSpec(sinkFn, 1);
    assertEquals(sinkOp.getSinkFn(), sinkFn);
    TestMessageEnvelope mockInput = mock(TestMessageEnvelope.class);
    when(mockInput.getKey()).thenReturn("my-test-msg-key");
    MessageType mockMsgBody = mock(MessageType.class);
    when(mockInput.getMessage()).thenReturn(mockMsgBody);
    final List<OutgoingMessageEnvelope> outputMsgs = new ArrayList<>();
    MessageCollector mockCollector = mock(MessageCollector.class);
    doAnswer(invocation -> {
        outputMsgs.add((OutgoingMessageEnvelope) invocation.getArguments()[0]);
        return null;
    }).when(mockCollector).send(any());
    sinkOp.getSinkFn().apply(mockInput, mockCollector, null);
    assertEquals(1, outputMsgs.size());
    assertEquals(outputMsgs.get(0).getKey(), "my-test-msg-key");
    assertEquals(outputMsgs.get(0).getMessage(), mockMsgBody);
    assertEquals(sinkOp.getOpCode(), OperatorSpec.OpCode.SINK);
    assertEquals(sinkOp.getNextStream(), null);
}

Example 3 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class TestEventHubSystemProducer method testSendingLargeMessage.

@Test
public void testSendingLargeMessage() throws Exception {
    String systemName = "eventhubs";
    String streamName = "testLMStream";
    int numEvents = 10;
    int partitionId0 = 0;
    TestMetricsRegistry testMetrics = new TestMetricsRegistry();
    Map<String, Interceptor> interceptor = new HashMap<>();
    interceptor.put(streamName, new PassThroughInterceptor());
    List<String> outgoingMessagesP0 = generateMessages(numEvents / 2);
    outgoingMessagesP0.add("1234567890123456789012345678901234567890");
    outgoingMessagesP0.addAll(generateMessages(numEvents / 2));
    // Set configs
    Map<String, String> configMap = new HashMap<>();
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_LIST, systemName), streamName);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, streamName), EVENTHUB_NAMESPACE);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, streamName), EVENTHUB_KEY_NAME);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, streamName), EVENTHUB_KEY);
    configMap.put(String.format(EventHubConfig.CONFIG_SKIP_MESSAGES_LARGER_THAN, systemName), "30");
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, streamName), EVENTHUB_ENTITY1);
    configMap.put(String.format(EventHubConfig.CONFIG_PRODUCER_PARTITION_METHOD, systemName), PartitioningMethod.PARTITION_KEY_AS_PARTITION.toString());
    MapConfig config = new MapConfig(configMap);
    MockEventHubClientManagerFactory factory = new MockEventHubClientManagerFactory();
    EventHubSystemProducer producer = new EventHubSystemProducer(new EventHubConfig(config), systemName, factory, interceptor, testMetrics);
    SystemStream systemStream = new SystemStream(systemName, streamName);
    producer.register(SOURCE);
    producer.start();
    outgoingMessagesP0.forEach(message -> producer.send(SOURCE, new OutgoingMessageEnvelope(systemStream, partitionId0, null, message.getBytes())));
    // Retrieve sent data
    List<String> receivedData0 = factory.getSentData(systemName, streamName, partitionId0).stream().map(eventData -> new String(eventData.getBytes())).collect(Collectors.toList());
    Assert.assertEquals(outgoingMessagesP0.size(), receivedData0.size() + 1);
}

Example 4 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class TestEventHubSystemProducer method testSkipLargeMessageCheck.

@Test
public void testSkipLargeMessageCheck() throws Exception {
    String systemName = "eventhubs";
    String streamName = "testLMStream";
    int numEvents = 10;
    int partitionId0 = 0;
    TestMetricsRegistry testMetrics = new TestMetricsRegistry();
    Map<String, Interceptor> interceptor = new HashMap<>();
    interceptor.put(streamName, new PassThroughInterceptor());
    List<String> outgoingMessagesP0 = generateMessages(numEvents / 2);
    outgoingMessagesP0.add("1234567890123456789012345678901234567890");
    outgoingMessagesP0.addAll(generateMessages(numEvents / 2));
    // Set configs
    Map<String, String> configMap = new HashMap<>();
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_LIST, systemName), streamName);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, streamName), EVENTHUB_NAMESPACE);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, streamName), EVENTHUB_KEY_NAME);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, streamName), EVENTHUB_KEY);
    configMap.put(String.format(EventHubConfig.CONFIG_SKIP_MESSAGES_LARGER_THAN, systemName), "-1");
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, streamName), EVENTHUB_ENTITY1);
    configMap.put(String.format(EventHubConfig.CONFIG_PRODUCER_PARTITION_METHOD, systemName), PartitioningMethod.PARTITION_KEY_AS_PARTITION.toString());
    MapConfig config = new MapConfig(configMap);
    MockEventHubClientManagerFactory factory = new MockEventHubClientManagerFactory();
    EventHubSystemProducer producer = new EventHubSystemProducer(new EventHubConfig(config), systemName, factory, interceptor, testMetrics);
    SystemStream systemStream = new SystemStream(systemName, streamName);
    producer.register(SOURCE);
    producer.start();
    outgoingMessagesP0.forEach(message -> producer.send(SOURCE, new OutgoingMessageEnvelope(systemStream, partitionId0, null, message.getBytes())));
    // Retrieve sent data
    List<String> receivedData0 = factory.getSentData(systemName, streamName, partitionId0).stream().map(eventData -> new String(eventData.getBytes())).collect(Collectors.toList());
    Assert.assertEquals(outgoingMessagesP0.size(), receivedData0.size());
}

Example 5 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class TestEventHubSystemProducer method testSendingToSpecificPartitions.

private void testSendingToSpecificPartitions(boolean perPartitionConnection) throws Exception {
    String systemName = "eventhubs";
    String streamName = "testStream";
    int numEvents = 10;
    int partitionId0 = 0;
    int partitionId1 = 1;
    TestMetricsRegistry testMetrics = new TestMetricsRegistry();
    Map<String, Interceptor> interceptor = new HashMap<>();
    interceptor.put(streamName, new PassThroughInterceptor());
    List<String> outgoingMessagesP0 = generateMessages(numEvents);
    List<String> outgoingMessagesP1 = generateMessages(numEvents);
    // Set configs
    Map<String, String> configMap = new HashMap<>();
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_LIST, systemName), streamName);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, streamName), EVENTHUB_NAMESPACE);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, streamName), EVENTHUB_KEY_NAME);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, streamName), EVENTHUB_KEY);
    configMap.put(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, streamName), EVENTHUB_ENTITY1);
    configMap.put(String.format(EventHubConfig.CONFIG_PRODUCER_PARTITION_METHOD, systemName), PartitioningMethod.PARTITION_KEY_AS_PARTITION.toString());
    configMap.put(String.format(EventHubConfig.CONFIG_PER_PARTITION_CONNECTION, systemName), String.valueOf(perPartitionConnection));
    MapConfig config = new MapConfig(configMap);
    MockEventHubClientManagerFactory factory = new MockEventHubClientManagerFactory();
    EventHubSystemProducer producer = new EventHubSystemProducer(new EventHubConfig(config), systemName, factory, interceptor, testMetrics);
    SystemStream systemStream = new SystemStream(systemName, streamName);
    producer.register(SOURCE);
    producer.start();
    outgoingMessagesP0.forEach(message -> producer.send(SOURCE, new OutgoingMessageEnvelope(systemStream, partitionId0, null, message.getBytes())));
    outgoingMessagesP1.forEach(message -> producer.send(SOURCE, new OutgoingMessageEnvelope(systemStream, partitionId1, null, message.getBytes())));
    // Retrieve sent data
    List<String> receivedData0 = factory.getSentData(systemName, streamName, partitionId0).stream().map(eventData -> new String(eventData.getBytes())).collect(Collectors.toList());
    List<String> receivedData1 = factory.getSentData(systemName, streamName, partitionId1).stream().map(eventData -> new String(eventData.getBytes())).collect(Collectors.toList());
    Assert.assertTrue(outgoingMessagesP0.equals(receivedData0));
    Assert.assertTrue(outgoingMessagesP1.equals(receivedData1));
    if (perPartitionConnection) {
        Assert.assertNotEquals("perPartitionConnection=true; partitions should not share the same client", producer.perPartitionEventHubClients.get(streamName).get(0), producer.perPartitionEventHubClients.get(streamName).get(1));
    } else {
        Assert.assertEquals("perPartitionConnection=false; partitions should share the same client", producer.perPartitionEventHubClients.get(streamName).get(0), producer.perPartitionEventHubClients.get(streamName).get(1));
    }
}