Examples with TStream com.ibm.streamsx.topology.TStream used on opensource projects

Example 21 with TStream

use of com.ibm.streamsx.topology.TStream in project streamsx.topology by IBMStreams.

the class StreamTest method testUnionNops.

@Test
public void testUnionNops() throws Exception {
    assumeTrue(isMainRun());
    final Topology f = newTopology("Union");
    TStream<String> s1 = f.strings("A1", "B1", "C1", "D1");
    Set<TStream<String>> empty = Collections.emptySet();
    assertSame(s1, s1.union(s1));
    assertSame(s1, s1.union(empty));
    assertSame(s1, s1.union(Collections.singleton(s1)));
}

Example 22 with TStream

use of com.ibm.streamsx.topology.TStream in project streamsx.topology by IBMStreams.

the class StreamImpl method split.

@Override
public List<TStream<T>> split(int n, ToIntFunction<T> splitter) {
    if (n <= 0)
        throw new IllegalArgumentException("n");
    List<TStream<T>> l = new ArrayList<>(n);
    String opName = splitter.getClass().getSimpleName();
    if (opName.isEmpty()) {
        opName = getTupleName() + "Split";
    }
    BOperatorInvocation bop = JavaFunctional.addFunctionalOperator(this, opName, FunctionSplit.class, splitter);
    SourceInfo.setSourceInfo(bop, StreamImpl.class);
    connectTo(bop, true, null);
    Type outputType = refineType(ToIntFunction.class, 0, splitter);
    for (int i = 0; i < n; i++) {
        TStream<T> splitOutput = JavaFunctional.addJavaOutput(this, bop, outputType);
        l.add(splitOutput);
    }
    return l;
}

Example 23 with TStream

use of com.ibm.streamsx.topology.TStream in project streamsx.topology by IBMStreams.

the class StreamImpl method union.

@SuppressWarnings("unchecked")
@Override
public TStream<T> union(Set<TStream<T>> others) {
    if (others.isEmpty())
        return this;
    Set<TStream<T>> allStreams = new HashSet<>();
    allStreams.addAll(others);
    allStreams.add(this);
    // Check we don't have just a single stream.
    if (allStreams.size() == 1)
        return this;
    List<TStream<T>> sourceStreams = new ArrayList<>();
    sourceStreams.addAll(allStreams);
    StreamSchema schema = output().schema();
    Type tupleType = getTupleType();
    // Unwrap all streams so that we do not add the same stream twice
    // in multiple unions or explicitly and in a union.
    Set<BOutput> outputs = new HashSet<>();
    for (int i = 0; i < sourceStreams.size(); i++) {
        TStream<T> s = sourceStreams.get(i);
        // E..g TStream<String>.
        if (!schema.equals(s.output().schema())) {
            if (s.getTupleClass() != null) {
                // This stream has the direct schema!
                schema = s.output().schema();
                assert getTupleClass() == null;
                tupleType = s.getTupleClass();
                if (i != 0) {
                    // Didn't discover it first
                    // reset to go through the list
                    // again. Note this assumes there
                    // are just two options for the schema
                    // generic or direct
                    // to get back to 0.
                    i = -1;
                    outputs.clear();
                    continue;
                }
            } else {
                assert tupleType instanceof Class;
                s = s.asType((Class<T>) tupleType);
                assert s.output().schema().equals(schema);
                sourceStreams.set(i, s);
            }
        }
        outputs.add(s.output());
    }
    BOutput unionOutput = builder().addUnion(outputs);
    return new StreamImpl<T>(this, unionOutput, tupleType);
}

Example 24 with TStream

use of com.ibm.streamsx.topology.TStream in project streamsx.topology by IBMStreams.

the class LowLatencyTest method testLowLatencySplit.

@Test
public void testLowLatencySplit() throws Exception {
    // lowLatency().split() is an interesting case because split()
    // has >1 oports.
    final Topology topology = newTopology("testLowLatencySplit");
    int splitWidth = 3;
    String[] strs = { "ch0", "ch1", "ch2" };
    TStream<String> s1 = topology.strings(strs);
    s1 = s1.isolate();
    s1 = s1.lowLatency();
    /////////////////////////////////////
    // assume that if s1.modify and the split().[modify()] are
    // in the same PE, that s1.split() is in the same too
    TStream<String> s2 = s1.modify(unaryGetPEId());
    List<TStream<String>> splits = s1.split(splitWidth, roundRobinSplitter());
    List<TStream<String>> splitChResults = new ArrayList<>();
    for (int i = 0; i < splits.size(); i++) {
        splitChResults.add(splits.get(i).modify(unaryGetPEId()));
    }
    TStream<String> splitChFanin = splitChResults.get(0).union(new HashSet<>(splitChResults.subList(1, splitChResults.size())));
    /////////////////////////////////////
    TStream<String> all = splitChFanin.endLowLatency();
    Tester tester = topology.getTester();
    Condition<Long> uCount = tester.tupleCount(all, strs.length);
    Condition<List<String>> contents = tester.stringContents(all, "");
    Condition<List<String>> s2contents = tester.stringContents(s2, "");
    complete(tester, uCount, 10, TimeUnit.SECONDS);
    Set<String> peIds = new HashSet<>();
    peIds.addAll(contents.getResult());
    peIds.addAll(s2contents.getResult());
    assertEquals("peIds: " + peIds, 1, peIds.size());
}

Example 25 with TStream

use of com.ibm.streamsx.topology.TStream in project streamsx.kafka by IBMStreams.

the class KafkaConsumerFanInTest method kafkaFanInTest.

@Test
public void kafkaFanInTest() throws Exception {
    Topology topo = getTopology();
    // create the producer (produces tuples after a short delay)
    TStream<String> stringSrcStream = topo.strings(Constants.STRING_DATA).modify(new Delay<>(5000));
    SPL.invokeSink(Constants.KafkaProducerOp, KafkaSPLStreamsUtils.convertStreamToKafkaTuple(stringSrcStream), getKafkaParams());
    // create the consumer
    SPLStream consumerStream1 = SPL.invokeSource(topo, Constants.KafkaConsumerOp, getConsumerParams(1), KafkaSPLStreamsUtils.STRING_SCHEMA);
    SPLStream consumerStream2 = SPL.invokeSource(topo, Constants.KafkaConsumerOp, getConsumerParams(2), KafkaSPLStreamsUtils.STRING_SCHEMA);
    SPLStream unionStream = KafkaSPLStreamsUtils.union(Arrays.asList(consumerStream1, consumerStream2), KafkaSPLStreamsUtils.STRING_SCHEMA);
    SPLStream msgStream = SPLStreams.stringToSPLStream(unionStream.convert(t -> t.getString("message")));
    // test the output of the consumer
    StreamsContext<?> context = StreamsContextFactory.getStreamsContext(Type.DISTRIBUTED_TESTER);
    Tester tester = topo.getTester();
    // both consumers consume the same data, so each result is duplicated
    String[] expectedArr = KafkaSPLStreamsUtils.duplicateArrayEntries(Constants.STRING_DATA, 2);
    Condition<List<String>> condition = KafkaSPLStreamsUtils.stringContentsUnordered(tester, msgStream, expectedArr);
    tester.complete(context, new HashMap<>(), condition, 30, TimeUnit.SECONDS);
    // check the results
    Assert.assertTrue(condition.getResult().size() > 0);
    Assert.assertTrue(condition.getResult().toString(), condition.valid());
}