Examples with SPLStream com.ibm.streamsx.topology.spl.SPLStream used on opensource projects

Example 61 with SPLStream

use of com.ibm.streamsx.topology.spl.SPLStream in project streamsx.topology by IBMStreams.

the class KafkaConsumer method subscribe.

/**
 * Subscribe to a topic and create a stream of messages.
 * <p>
 * N.B., A topology that includes this will not support
 * {@code StreamsContext.Type.EMBEDDED}.
 * <p>
 * N.B. due to com.ibm.streamsx.messaging
 * <a href="https://github.com/IBMStreams/streamsx.messaging/issues/118">issue#118</a>,
 * multiple consumers will have issues in
 * {@code StreamsContext.Type.STANDALONE}.
 * <p>
 * N.B. due to com.ibm.streamsx.messaging
 * <a href="https://github.com/IBMStreams/streamsx.messaging/issues/117">issue#117</a>,
 * a consumer in {@code StreamsContext.Type.STANDALONE} subsequently results
 * in an orphaned @{code standalone} processes that continues as the lead
 * group/topic consumer thereby preventing subsequent instances of the
 * group/topic consumer from receiving messages.
 * <p>
 * N.B. due to com.ibm.streamsx.messaging
 * <a href="https://github.com/IBMStreams/streamsx.messaging/issues/114">issue#114</a>,
 * a consumer essentially ignores messages generated by producers where the
 * optional {@code key} is {@code null}.
 * e.g., Kafka's {@code kafka-console-producer.sh} tool generates
 * {@code key==null} messages.
 *
 * @param threadsPerTopic number of threads to allocate to processing each
 *        topic.  May be a submission parameter.
 * @param topic the topic to subscribe to.  May be a submission parameter.
 * @return TStream&lt;Message>
 *      The generated {@code Message} tuples have a non-null {@code topic}.
 *      The tuple's {@code key} will be null if the Kafka message
 *      lacked a key or it's key was the empty string.
 * @throws IllegalArgumentException if topic is null.
 * @throws IllegalArgumentException if threadsPerTopic is null.
 * @see Value
 * @see Topology#createSubmissionParameter(String, Class)
 */
public TStream<Message> subscribe(Supplier<Integer> threadsPerTopic, Supplier<String> topic) {
    if (topic == null)
        throw new IllegalArgumentException("topic");
    if (threadsPerTopic == null || (threadsPerTopic.get() != null && threadsPerTopic.get() <= 0))
        throw new IllegalArgumentException("threadsPerTopic");
    Map<String, Object> params = new HashMap<>();
    params.put("topic", topic);
    if (threadsPerTopic instanceof Value && threadsPerTopic.get() == 1)
        // The default is one.
        ;
    else
        params.put("threadsPerTopic", threadsPerTopic);
    if (!config.isEmpty())
        params.put("kafkaProperty", Util.toKafkaProperty(config));
    // workaround streamsx.messaging issue #107
    params.put("propertiesFile", PROP_FILE_PARAM);
    addPropertiesFile();
    // Use SPL.invoke to avoid adding a compile time dependency
    // to com.ibm.streamsx.messaging since JavaPrimitive.invoke*()
    // lack "kind" based variants.
    String kind = "com.ibm.streamsx.messaging.kafka::KafkaConsumer";
    String className = "com.ibm.streamsx.messaging.kafka.KafkaSource";
    SPLStream rawKafka = SPL.invokeSource(te, kind, params, KafkaSchemas.KAFKA);
    SPL.tagOpAsJavaPrimitive(toOp(rawKafka), kind, className);
    TStream<Message> rcvdMsgs = toMessageStream(rawKafka);
    rcvdMsgs.colocate(rawKafka);
    // workaround streamsx.messaging issue#118 w/java8
    // isolate even in the single consumer case since we don't
    // know if others may be subsequently created.
    rcvdMsgs = rcvdMsgs.isolate();
    return rcvdMsgs;
}

Example 62 with SPLStream

use of com.ibm.streamsx.topology.spl.SPLStream in project streamsx.topology by IBMStreams.

the class MqttStreams method publish.

/**
 * Publish {@code stream} tuples to one or more MQTT topics.
 * <p>
 * If {@code topic} is null, each tuple is published to the topic
 * specified by its {@link Message#getTopic()}.
 * Otherwise, all tuples are published to {@code topic}.
 * <p>
 * The messages added to MQTT include a topic and message.
 * The {@link Message#getKey()} field is ignored.
 * <p>
 * The message is handled with the quality of service
 * indicated by configuration property {@code defaultQOS}.
 *
 * @param stream the stream to publish
 * @param topic topic to publish to.  May be a submission parameter. May be null.
 * @return the sink element
 * @see Value
 * @see Topology#createSubmissionParameter(String, Class)
 */
public TSink publish(TStream<? extends Message> stream, Supplier<String> topic) {
    stream = stream.lowLatency();
    @SuppressWarnings("unchecked") SPLStream splStream = SPLStreams.convertStream((TStream<Message>) stream, cvtMsgFunc(topic), MqttSchemas.MQTT);
    Map<String, Object> params = new HashMap<String, Object>();
    params.put("reconnectionBound", -1);
    params.put("qos", 0);
    params.putAll(Util.configToSplParams(config));
    params.remove("messageQueueSize");
    if (topic == null)
        params.put("topicAttributeName", "topic");
    else
        params.put("topic", topic);
    params.put("dataAttributeName", "message");
    if (++opCnt > 1) {
        // each op requires its own clientID
        String clientId = (String) params.get("clientID");
        if (clientId != null && clientId.length() > 0)
            params.put("clientID", opCnt + "-" + clientId);
    }
    // Use SPL.invoke to avoid adding a compile time dependency
    // to com.ibm.streamsx.messaging since JavaPrimitive.invoke*()
    // lack "kind" based variants.
    String kind = "com.ibm.streamsx.messaging.mqtt::MQTTSink";
    String className = "com.ibm.streamsx.messaging.kafka.MqttSinkOperator";
    TSink sink = SPL.invokeSink(kind, splStream, params);
    SPL.tagOpAsJavaPrimitive(sink.operator(), kind, className);
    return sink;
}

Example 63 with SPLStream

use of com.ibm.streamsx.topology.spl.SPLStream in project streamsx.topology by IBMStreams.

the class Vwap method createVwapTopology.

/**
 * Create the Vwap topology, returning the resultant bargains stream.
 */
public static TStream<Bargain> createVwapTopology() {
    Topology topology = new Topology("Vwap");
    String vwapDataFile = System.getenv("STREAMS_INSTALL") + "/samples/spl/application/Vwap/data/TradesAndQuotes.csv.gz";
    TStream<String> vwapDataFileName = topology.strings(vwapDataFile);
    SPLStream tradeQuotes = FileSPLStreams.csvCompressedReader(vwapDataFileName, TQRecT, Compression.gzip);
    // Convert the SPLStreams into TStream<T> instances,
    // unpacking the SPL Tuple into Quote and Trade objects
    TStream<Trade> trades = Trade.getTrades(tradeQuotes);
    TStream<Quote> quotes = Quote.getQuotes(tradeQuotes);
    TStream<Bargain> bargains = VwapProcessing.bargains(trades, quotes);
    return bargains;
}

Example 64 with SPLStream

use of com.ibm.streamsx.topology.spl.SPLStream in project streamsx.topology by IBMStreams.

the class PublishSubscribeSPLTest method testSPLPublishNoFilterWithSubscribeSubmissionParams.

@Test
public void testSPLPublishNoFilterWithSubscribeSubmissionParams() throws Exception {
    final Topology t = new Topology();
    SPLStream source = SPLStreamsTest.testTupleStream(t);
    source = addStartupDelay(source);
    final String topic = uniqueTopic("SPLNoFilterSubmissionParams");
    Supplier<String> pubParam = t.createSubmissionParameter("PPSPL", String.class);
    source.publish(pubParam);
    Supplier<String> subParam = t.createSubmissionParameter("SPSPL", String.class);
    SPLStream sub = SPLStreams.subscribe(t, subParam, source.getSchema());
    TStream<String> subscribe = sub.transform(new GetTupleId());
    JobConfig jco = new JobConfig();
    jco.addSubmissionParameter("SPSPL", topic);
    jco.addSubmissionParameter("PPSPL", topic);
    jco.addToConfig(getConfig());
    completeAndValidate(subscribe, 20, "SPL:0", "SPL:1", "SPL:2", "SPL:3");
}

Example 65 with SPLStream

use of com.ibm.streamsx.topology.spl.SPLStream in project streamsx.topology by IBMStreams.

the class SimpleEmbeddedTest method testTwoStreams.

@Test
public void testTwoStreams() throws Exception {
    Topology topology = new Topology("testTwoStreams");
    TStream<String> hw = topology.strings("Hello", "World!", "Test!!");
    SPLStream hws = SPLStreams.stringToSPLStream(hw);
    TStream<String> hw2 = StringStreams.contains(hw, "e");
    SPLStream hw2s = SPLStreams.stringToSPLStream(hw2);
    Tester tester = topology.getTester();
    StreamCounter<Tuple> counter = tester.splHandler(hws, new StreamCounter<Tuple>());
    StreamCollector<LinkedList<Tuple>, Tuple> collector = tester.splHandler(hws, StreamCollector.newLinkedListCollector());
    StreamCounter<Tuple> counter2 = tester.splHandler(hw2s, new StreamCounter<Tuple>());
    StreamCollector<LinkedList<Tuple>, Tuple> collector2 = tester.splHandler(hw2s, StreamCollector.newLinkedListCollector());
    StreamsContextFactory.getStreamsContext(StreamsContext.Type.EMBEDDED_TESTER).submit(topology).get();
    assertEquals(3, counter.getTupleCount());
    assertEquals("Hello", collector.getTuples().get(0).getString(0));
    assertEquals("World!", collector.getTuples().get(1).getString(0));
    assertEquals("Test!!", collector.getTuples().get(2).getString(0));
    assertEquals(2, counter2.getTupleCount());
    assertEquals("Hello", collector2.getTuples().get(0).getString(0));
    assertEquals("Test!!", collector2.getTuples().get(1).getString(0));
}