Examples with TStream com.ibm.streamsx.topology.TStream used on opensource projects

Example 41 with TStream

use of com.ibm.streamsx.topology.TStream in project streamsx.topology by IBMStreams.

the class StreamTest method testUnionNops.

@Test
public void testUnionNops() throws Exception {
    assumeTrue(isMainRun());
    final Topology f = newTopology("Union");
    TStream<String> s1 = f.strings("A1", "B1", "C1", "D1");
    Set<TStream<String>> empty = Collections.emptySet();
    assertSame(s1, s1.union(s1));
    assertSame(s1, s1.union(empty));
    assertSame(s1, s1.union(Collections.singleton(s1)));
}

Example 42 with TStream

use of com.ibm.streamsx.topology.TStream in project streamsx.topology by IBMStreams.

the class StreamTest method testSplit.

@Test
public void testSplit() throws Exception {
    final Topology topology = newTopology("testSplit");
    TStream<String> s1 = topology.strings("ch0", "ch1", "ch2", "omit", "another-ch2", "another-ch1", "another-ch0", "another-omit");
    List<TStream<String>> splits = s1.split(3, myStringSplitter());
    assertEquals("list size", 3, splits.size());
    Tester tester = topology.getTester();
    List<Condition<List<String>>> contents = new ArrayList<>();
    for (int i = 0; i < splits.size(); i++) {
        TStream<String> ch = splits.get(i);
        Condition<List<String>> chContents = tester.stringContents(ch, "ch" + i, "another-ch" + i);
        contents.add(chContents);
    }
    TStream<String> all = splits.get(0).union(new HashSet<>(splits.subList(1, splits.size())));
    Condition<Long> uCount = tester.tupleCount(all, 6);
    complete(tester, uCount, 10, TimeUnit.SECONDS);
    for (int i = 0; i < splits.size(); i++) {
        assertTrue("chContents[" + i + "]:" + contents.get(i), contents.get(i).valid());
    }
}

Example 43 with TStream

use of com.ibm.streamsx.topology.TStream in project streamsx.topology by IBMStreams.

the class StreamTest method testUnionSet.

@Test
public void testUnionSet() throws Exception {
    final Topology topology = newTopology();
    TStream<String> s1 = topology.strings("A1", "B1", "C1");
    TStream<String> s2 = topology.strings("A2", "B2", "C2", "D2");
    TStream<String> s3 = topology.strings("A3", "B3", "C3");
    Set<TStream<String>> streams = new HashSet<TStream<String>>();
    streams.add(s2);
    streams.add(s3);
    TStream<String> su = s1.union(streams);
    // TODO - testing doesn't work against union streams in embedded.
    su = su.filter(new AllowAll<String>());
    Tester tester = topology.getTester();
    Condition<Long> suCount = tester.tupleCount(su, 10);
    Condition<List<String>> suContents = tester.stringContentsUnordered(su, "A1", "B1", "C1", "A2", "B2", "C2", "D2", "A3", "B3", "C3");
    assertTrue(complete(tester, suCount, 10, TimeUnit.SECONDS));
    // assertTrue("SU:" + suContents, suContents.valid());
    assertTrue("SU:" + suCount, suCount.valid());
    assertTrue("SUContents:" + suContents, suContents.valid());
}

Example 44 with TStream

use of com.ibm.streamsx.topology.TStream in project streamsx.topology by IBMStreams.

the class FileStreamsTest method runDirectoryWatcher.

private void runDirectoryWatcher(final Topology t, int numberOfFiles, int repeat) throws Exception {
    // final Path dir = Files.createTempDirectory("testdw");
    final String[] files = new String[numberOfFiles];
    for (int i = 0; i < files.length; i++) {
        files[i] = PREFIX + (numberOfFiles - i);
    }
    String dir;
    dir = ".";
    this.getConfig().put(JobProperties.DATA_DIRECTORY, dir);
    TStream<String> rawFileNames = FileStreams.directoryWatcher(t, dir);
    TStream<String> fileNames = rawFileNames.modify(fn -> new File(fn).getName());
    fileNames = fileNames.filter(fn -> fn.startsWith(PREFIX));
    // fileNames.print();
    // Create the files from within the topology to ensure it works
    // in distributed and standalone.
    // 
    // Due to vagaries / delays that can occur in operator startup
    // in different processes (distributed mode), delay the initial
    // file creation process to give the watcher a chance to startup.
    // 
    // e.g., with numberOfFiles=20 & repeat=1, each group of files
    // only lasts 20*(10ms*2) => 200ms.  That can easily happen before
    // the watcher is started and has done its first dir.listFiles(),
    // with the result being not seeing/processing the expected number
    // of files.
    TSink creator = addStartupDelay(BeaconStreams.single(t)).forEach(createFiles(dir, files, repeat));
    creator.colocate(rawFileNames);
    Tester tester = t.getTester();
    Condition<Long> expectedCount = tester.tupleCount(fileNames, numberOfFiles * repeat);
    String[] expectedFileNames = files;
    if (repeat != 1) {
        expectedFileNames = new String[files.length * repeat];
        for (int r = 0; r < repeat; r++) System.arraycopy(files, 0, expectedFileNames, r * files.length, files.length);
    }
    Condition<List<String>> expectedNames = tester.stringContentsUnordered(fileNames.filter(x -> true), expectedFileNames);
    complete(tester, expectedCount, 60, TimeUnit.SECONDS);
    if (!".".equals(dir)) {
    // Files.d
    }
}

Example 45 with TStream

use of com.ibm.streamsx.topology.TStream in project streamsx.topology by IBMStreams.

the class FunctionalSubmissionParamsTest method testPeriodicSourceWithSubmissionParams.

@Test
public void testPeriodicSourceWithSubmissionParams() throws Exception {
    Topology topology = newTopology();
    Supplier<Integer> someInt = topology.createSubmissionParameter("someInt", Integer.class);
    Supplier<Integer> someIntD = topology.createSubmissionParameter("someIntD", 20);
    // The test's functional logic asserts it receives the expected SP value.
    // HEADS UP.  issue#213 - exceptions in the supplier get silently eaten
    // and the stream just ends up with 0 tuples.
    TStream<Integer> s = topology.periodicSource(sourceSupplier(someInt, 10), 100, TimeUnit.MILLISECONDS);
    TStream<Integer> sD = topology.periodicSource(sourceSupplier(someIntD, 20), 100, TimeUnit.MILLISECONDS);
    TStream<Integer> ms = topology.periodicMultiSource(sourceIterableSupplier(someInt, 10), 100, TimeUnit.MILLISECONDS);
    TStream<Integer> msD = topology.periodicMultiSource(sourceIterableSupplier(someIntD, 20), 100, TimeUnit.MILLISECONDS);
    Map<String, Object> params = new HashMap<>();
    params.put("someInt", 10);
    getConfig().put(ContextProperties.SUBMISSION_PARAMS, params);
    // //////////////////
    Set<TStream<Integer>> all = new HashSet<>(Arrays.asList(s, sD, ms, msD));
    TStream<Integer> union = s.union(all);
    Tester tester = topology.getTester();
    Condition<Long> expectedCount1 = tester.tupleCount(s, 5);
    Condition<Long> expectedCount2 = tester.tupleCount(sD, 5);
    Condition<Long> expectedCount3 = tester.tupleCount(ms, 5);
    Condition<Long> expectedCount4 = tester.tupleCount(msD, 5);
    Condition<Long> expectedCount = tester.tupleCount(union, 20);
    complete(tester, expectedCount, 15, TimeUnit.SECONDS);
    assertTrue(expectedCount1.toString(), expectedCount1.valid());
    assertTrue(expectedCount2.toString(), expectedCount2.valid());
    assertTrue(expectedCount3.toString(), expectedCount3.valid());
    assertTrue(expectedCount4.toString(), expectedCount4.valid());
}