Examples with StreamApplicationDescriptorImpl org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl used on opensource projects

Example 21 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestJoinOperator method joinRetainsLatestMessageForKeyReverse.

@Test
public void joinRetainsLatestMessageForKeyReverse() throws Exception {
    StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction());
    StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc);
    List<Integer> output = new ArrayList<>();
    MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage());
    // push messages to second stream
    numbers.forEach(n -> sot.processAsync(new SecondStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));
    // push messages to second stream again with same keys but different values
    numbers.forEach(n -> sot.processAsync(new SecondStreamIME(n, 2 * n), messageCollector, taskCoordinator, taskCallback));
    // push messages to first stream with same key
    numbers.forEach(n -> sot.processAsync(new FirstStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));
    int outputSum = output.stream().reduce(0, (s, m) -> s + m);
    // should use latest messages in the second stream
    assertEquals(165, outputSum);
}

Example 22 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestJoinOperator method joinWithSelfThrowsException.

@Test(expected = SamzaException.class)
public void joinWithSelfThrowsException() throws Exception {
    Map<String, String> mapConfig = new HashMap<>();
    mapConfig.put("job.name", "jobName");
    mapConfig.put("job.id", "jobId");
    StreamTestUtils.addStreamConfigs(mapConfig, "inStream", "insystem", "instream");
    Config config = new MapConfig(mapConfig);
    StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> {
        IntegerSerde integerSerde = new IntegerSerde();
        KVSerde<Integer, Integer> kvSerde = KVSerde.of(integerSerde, integerSerde);
        GenericSystemDescriptor sd = new GenericSystemDescriptor("insystem", "mockFactoryClassName");
        GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("inStream", kvSerde);
        MessageStream<KV<Integer, Integer>> inStream = appDesc.getInputStream(inputDescriptor);
        inStream.join(inStream, new TestJoinFunction(), integerSerde, kvSerde, kvSerde, JOIN_TTL, "join");
    }, config);
    // should throw an exception
    createStreamOperatorTask(new SystemClock(), streamAppDesc);
}

Example 23 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestJoinOperator method joinFnInitAndClose.

@Test
public void joinFnInitAndClose() throws Exception {
    TestJoinFunction joinFn = new TestJoinFunction();
    StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(joinFn);
    StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc);
    MessageCollector messageCollector = mock(MessageCollector.class);
    // push messages to first stream
    numbers.forEach(n -> sot.processAsync(new FirstStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));
    // close should not be called till now
    sot.close();
    verify(messageCollector, times(0)).send(any(OutgoingMessageEnvelope.class));
    // Make sure the joinFn has been copied instead of directly referred by the task instance
    assertEquals(0, joinFn.getNumInitCalls());
    assertEquals(0, joinFn.getNumCloseCalls());
}

Example 24 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestJoinOperator method joinRetainsMatchedMessagesReverse.

@Test
public void joinRetainsMatchedMessagesReverse() throws Exception {
    StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction());
    StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc);
    List<Integer> output = new ArrayList<>();
    MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage());
    // push messages to first stream
    numbers.forEach(n -> sot.processAsync(new FirstStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));
    // push messages to second stream with same key
    numbers.forEach(n -> sot.processAsync(new SecondStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));
    int outputSum = output.stream().reduce(0, (s, m) -> s + m);
    assertEquals(110, outputSum);
    output.clear();
    // push messages to second stream with same keys once again.
    numbers.forEach(n -> sot.processAsync(new SecondStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));
    int newOutputSum = output.stream().reduce(0, (s, m) -> s + m);
    // should produce the same output as before
    assertEquals(110, newOutputSum);
}

Example 25 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestJoinOperator method joinRemovesExpiredMessagesReverse.

@Test
public void joinRemovesExpiredMessagesReverse() throws Exception {
    TestClock testClock = new TestClock();
    StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction());
    StreamOperatorTask sot = createStreamOperatorTask(testClock, streamAppDesc);
    List<Integer> output = new ArrayList<>();
    MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage());
    // push messages to second stream
    numbers.forEach(n -> sot.processAsync(new SecondStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));
    // 1 minute after ttl
    testClock.advanceTime(JOIN_TTL.plus(Duration.ofMinutes(1)));
    // should expire second stream messages
    sot.window(messageCollector, taskCoordinator);
    // push messages to first stream with same key
    numbers.forEach(n -> sot.processAsync(new FirstStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));
    assertTrue(output.isEmpty());
}