Examples with Topology com.ibm.streamsx.topology.Topology used on opensource projects

Example 16 with Topology

use of com.ibm.streamsx.topology.Topology in project streamsx.topology by IBMStreams.

the class ParallelTest method testParallelSubmissionParam.

@Test
public void testParallelSubmissionParam() throws Exception {
    checkUdpSupported();
    Topology topology = newTopology("testParallelSubmissionParam");
    final int count = new Random().nextInt(1000) + 37;
    String submissionWidthName = "width";
    Integer submissionWidth = 5;
    TStream<BeaconTuple> fb = BeaconStreams.beacon(topology, count);
    TStream<BeaconTuple> pb = fb.parallel(topology.createSubmissionParameter(submissionWidthName, Integer.class));
    TStream<Integer> is = pb.transform(randomHashProducer());
    TStream<Integer> joined = is.endParallel();
    TStream<String> numRegions = joined.transform(uniqueIdentifierMap(count));
    Tester tester = topology.getTester();
    Condition<Long> expectedCount = tester.tupleCount(numRegions, 1);
    Condition<List<String>> regionCount = tester.stringContents(numRegions, submissionWidth.toString());
    Map<String, Object> params = new HashMap<>();
    params.put(submissionWidthName, submissionWidth);
    getConfig().put(ContextProperties.SUBMISSION_PARAMS, params);
    complete(tester, allConditions(regionCount, regionCount), 10, TimeUnit.SECONDS);
    assertTrue(expectedCount.valid());
    assertTrue(regionCount.valid());
}

Example 17 with Topology

use of com.ibm.streamsx.topology.Topology in project streamsx.topology by IBMStreams.

the class ParallelTest method testParallelSubmissionParamInner.

@Test
public void testParallelSubmissionParamInner() throws Exception {
    checkUdpSupported();
    Topology topology = newTopology("testParallelSubmissionParamInner");
    final int count = new Random().nextInt(1000) + 37;
    String submissionWidthName = "width";
    Integer submissionWidth = 5;
    String submissionAppendName = "append";
    boolean submissionAppend = true;
    String submissionFlushName = "flush";
    Integer submissionFlush = 1;
    String submissionThresholdName = "threshold";
    String submissionDefaultedThresholdName = "defaultedTreshold";
    Integer submissionThreshold = -1;
    // getConfig().put(ContextProperties.KEEP_ARTIFACTS, true);
    Supplier<Integer> threshold = topology.createSubmissionParameter(submissionThresholdName, Integer.class);
    Supplier<Integer> defaultedThreshold = topology.createSubmissionParameter(submissionDefaultedThresholdName, submissionThreshold);
    TStream<BeaconTuple> fb = BeaconStreams.beacon(topology, count);
    TStream<BeaconTuple> pb = fb.parallel(topology.createSubmissionParameter(submissionWidthName, Integer.class));
    TStream<Integer> is = pb.transform(randomHashProducer());
    // submission param use within a parallel region
    StreamSchema schema = getStreamSchema("tuple<int32 i>");
    SPLStream splStream = SPLStreams.convertStream(is, cvtMsgFunc(), schema);
    File tmpFile = File.createTempFile("parallelTest", null);
    tmpFile.deleteOnExit();
    Map<String, Object> splParams = new HashMap<>();
    splParams.put("file", tmpFile.getAbsolutePath());
    splParams.put("append", topology.createSubmissionParameter(submissionAppendName, submissionAppend));
    splParams.put("flush", SPL.createSubmissionParameter(topology, submissionFlushName, SPL.createValue(0, Type.MetaType.UINT32), false));
    SPL.invokeSink("spl.adapter::FileSink", splStream, splParams);
    // use a submission parameter in "inner" functional logic
    is = is.filter(thresholdFilter(threshold));
    is = is.filter(thresholdFilter(defaultedThreshold));
    // avoid another parallel impl limitation noted in issue#173
    is = is.filter(passthru());
    TStream<Integer> joined = is.endParallel();
    TStream<String> numRegions = joined.transform(uniqueIdentifierMap(count));
    Tester tester = topology.getTester();
    Condition<Long> expectedCount = tester.tupleCount(numRegions, 1);
    Condition<List<String>> regionCount = tester.stringContents(numRegions, submissionWidth.toString());
    Map<String, Object> params = new HashMap<>();
    params.put(submissionWidthName, submissionWidth);
    params.put(submissionFlushName, submissionFlush);
    params.put(submissionThresholdName, submissionThreshold);
    getConfig().put(ContextProperties.SUBMISSION_PARAMS, params);
    complete(tester, allConditions(expectedCount, regionCount), 10, TimeUnit.SECONDS);
    assertTrue(expectedCount.valid());
    assertTrue(regionCount.valid());
}

Example 18 with Topology

use of com.ibm.streamsx.topology.Topology in project streamsx.topology by IBMStreams.

the class ParallelTest method testParallelSubmissionParamDefault.

@Test
public void testParallelSubmissionParamDefault() throws Exception {
    checkUdpSupported();
    Topology topology = newTopology("testParallelSubmissionParamDefault");
    final int count = new Random().nextInt(1000) + 37;
    String submissionWidthName = "width";
    Integer submissionWidth = 5;
    TStream<BeaconTuple> fb = BeaconStreams.beacon(topology, count);
    TStream<BeaconTuple> pb = fb.parallel(topology.createSubmissionParameter(submissionWidthName, submissionWidth));
    TStream<Integer> is = pb.transform(randomHashProducer());
    TStream<Integer> joined = is.endParallel();
    TStream<String> numRegions = joined.transform(uniqueIdentifierMap(count));
    Tester tester = topology.getTester();
    Condition<Long> expectedCount = tester.tupleCount(numRegions, 1);
    Condition<List<String>> regionCount = tester.stringContents(numRegions, submissionWidth.toString());
    complete(tester, allConditions(regionCount, expectedCount), 10, TimeUnit.SECONDS);
    assertTrue(expectedCount.valid());
    assertTrue(regionCount.valid());
}

Example 19 with Topology

use of com.ibm.streamsx.topology.Topology in project streamsx.topology by IBMStreams.

the class ParallelTest method testParallelPreFanOut.

@Test
@Ignore("Issue #131")
public void testParallelPreFanOut() throws Exception {
    Topology topology = newTopology();
    TStream<String> strings = topology.strings("A", "B", "C", "D", "E");
    strings.print();
    TStream<String> stringsP = strings.parallel(3);
    stringsP = stringsP.filter(new AllowAll<String>());
    stringsP = stringsP.endParallel();
    Tester tester = topology.getTester();
    Condition<Long> fiveTuples = tester.tupleCount(stringsP, 5);
    Condition<List<String>> contents = tester.stringContentsUnordered(stringsP, "A", "B", "C", "D", "E");
    complete(tester, allConditions(fiveTuples, contents), 10, TimeUnit.SECONDS);
    assertTrue("contents: " + contents, contents.valid());
}

Example 20 with Topology

use of com.ibm.streamsx.topology.Topology in project streamsx.topology by IBMStreams.

the class ParallelTest method testAdjacentParallel.

@Test
public void testAdjacentParallel() throws Exception {
    checkUdpSupported();
    List<String> stringList = getListOfUniqueStrings(800);
    String[] stringArray = new String[800];
    stringArray = stringList.toArray(stringArray);
    Topology topology = newTopology("testAdj");
    TStream<String> out0 = topology.strings(stringArray).parallel(of(20), TStream.Routing.HASH_PARTITIONED);
    out0 = out0.transform(randomStringProducer("region1")).endParallel();
    TStream<String> out2 = out0.parallel(of(5), TStream.Routing.HASH_PARTITIONED);
    out2 = out2.transform(randomStringProducer("region2")).endParallel();
    TStream<String> numRegions = out2.multiTransform(uniqueStringCounter(800, "region"));
    Tester tester = topology.getTester();
    Condition<List<String>> assertFinished = tester.stringContentsUnordered(numRegions, "20", "5");
    Condition<Long> expectedCount = tester.tupleCount(out2, 800);
    complete(tester, allConditions(assertFinished, expectedCount), 60, TimeUnit.SECONDS);
    assertTrue(expectedCount.valid());
    assertTrue(assertFinished.valid());
}