Examples with TSink com.ibm.streamsx.topology.TSink used on opensource projects

Example 1 with TSink

use of com.ibm.streamsx.topology.TSink in project streamsx.topology by IBMStreams.

the class KafkaProducer method publish.

/**
 * Publish a stream of messages to a topic.
 * <p>
 * If {@code topic} is null, each tuple is published to the topic
 * specified by its {@link Message#getTopic()}.
 * Otherwise, all tuples are published to {@code topic}.
 * <p>
 * The messages added to Kafka include a topic, message and key.
 * If {@link Message#getKey()} is null, an empty key value is published.
 * <p>
 * N.B. there seem to be some issues with the underlying
 * com.ibm.streamsx.messaging library - e.g.,
 * <a href="https://github.com/IBMStreams/streamsx.messaging/issues/118">issue#118</a>.
 * If your application is experiencing odd Kafka behavior
 * try isolating the producer from its feeding streams.
 * e.g.,
 * <pre>
 * KafkaProducer pc = ...
 * TStream<Message> s = ...
 * pc.publish(s.isolate(), ...);
 * </pre>
 *
 * @param stream the stream to publish
 * @param topic topic to publish to.  May be null.
 * @return the sink element
 *
 * @throws IllegalArgumentException if a non-null empty {@code topic} is specified.
 */
public TSink publish(TStream<? extends Message> stream, Supplier<String> topic) {
    stream = stream.lowLatency();
    @SuppressWarnings("unchecked") SPLStream splStream = SPLStreams.convertStream((TStream<Message>) stream, cvtMsgFunc(topic), KafkaSchemas.KAFKA);
    Map<String, Object> params = new HashMap<String, Object>();
    if (!config.isEmpty())
        params.put("kafkaProperty", Util.toKafkaProperty(config));
    if (topic == null)
        params.put("topicAttribute", "topic");
    else
        params.put("topic", topic);
    // workaround streamsx.messaging issue #107
    params.put("propertiesFile", PROP_FILE_PARAM);
    addPropertiesFile();
    // Use SPL.invoke to avoid adding a compile time dependency
    // to com.ibm.streamsx.messaging since JavaPrimitive.invoke*()
    // lack "kind" based variants.
    String kind = "com.ibm.streamsx.messaging.kafka::KafkaProducer";
    String className = "com.ibm.streamsx.messaging.kafka.KafkaSink";
    TSink sink = SPL.invokeSink(kind, splStream, params);
    SPL.tagOpAsJavaPrimitive(sink.operator(), kind, className);
    return sink;
}

Example 2 with TSink

use of com.ibm.streamsx.topology.TSink in project streamsx.topology by IBMStreams.

the class MqttStreamsTest method testMultiPubSub.

@Test
public void testMultiPubSub() throws Exception {
    checkAssumes();
    setupDebug();
    Topology top = new Topology("testMultiPubSub");
    MsgGenerator mgen = new MsgGenerator(top.getName());
    String subClientId = newSubClientId(top.getName());
    String pubClientId = newPubClientId(top.getName());
    String[] topics = getMqttTopics();
    String topic1 = topics[0];
    String topic2 = topics[1];
    List<Message> topic1Msgs = createMsgs(mgen, topic1);
    List<Message> topic2Msgs = createMsgs(mgen, topic2);
    List<Message> msgs = new ArrayList<>(topic1Msgs);
    msgs.addAll(topic2Msgs);
    List<String> expectedAsString = mapList(msgs, msgToJSONStringFunc());
    // Multi pub / sub on a single connector
    MqttStreams producer = new MqttStreams(top, createConfig(pubClientId));
    MqttStreams consumer = new MqttStreams(top, createConfig(subClientId));
    TStream<Message> topic1MsgsToPublish = top.constants(topic1Msgs).modify(new InitialDelay<Message>(PUB_DELAY_MSEC));
    TStream<Message> topic2MsgsToPublish = top.constants(topic2Msgs).modify(new InitialDelay<Message>(PUB_DELAY_MSEC));
    TSink sink1 = producer.publish(topic1MsgsToPublish, new Value<String>(topic1));
    TSink sink2 = producer.publish(topic2MsgsToPublish, new Value<String>(topic2));
    TStream<Message> rcvdTopic1Msgs = consumer.subscribe(new Value<String>(topic1));
    TStream<Message> rcvdTopic2Msgs = consumer.subscribe(new Value<String>(topic2));
    // for validation...
    TStream<Message> rcvdMsgs = rcvdTopic1Msgs.union(rcvdTopic2Msgs);
    rcvdMsgs.print();
    // just our msgs
    rcvdMsgs = selectMsgs(rcvdMsgs, mgen.pattern());
    TStream<String> rcvdAsString = rcvdMsgs.transform(msgToJSONStringFunc());
    if (testBuildOnly(top))
        return;
    completeAndValidateUnordered(subClientId, top, rcvdAsString, SEC_TIMEOUT, expectedAsString.toArray(new String[0]));
    assertTrue(sink1 != null);
    assertTrue(sink2 != null);
}

Example 3 with TSink

use of com.ibm.streamsx.topology.TSink in project streamsx.topology by IBMStreams.

the class MqttStreamsTest method testExplicitTopicProducer.

@Test
public void testExplicitTopicProducer() throws Exception {
    checkAssumes();
    setupDebug();
    Topology top = new Topology("testNewExplicitTopicProducer");
    MsgGenerator mgen = new MsgGenerator(top.getName());
    String subClientId = newSubClientId(top.getName());
    String pubClientId = newPubClientId(top.getName());
    String topicVal = getMqttTopics()[0];
    Supplier<String> topic = new Value<String>(topicVal);
    List<Message> msgs = createMsgs(mgen, null);
    List<String> expectedAsString = mapList(modifyList(msgs, setTopic(topicVal)), msgToJSONStringFunc());
    // Test producer that takes an explicit topic and implicit config qos
    MqttStreams producer = new MqttStreams(top, createConfig(pubClientId));
    MqttStreams consumer = new MqttStreams(top, createConfig(subClientId));
    TStream<Message> msgsToPublish = top.constants(msgs).modify(new InitialDelay<Message>(PUB_DELAY_MSEC));
    TSink sink = producer.publish(msgsToPublish, topic);
    TStream<Message> rcvdMsgs = consumer.subscribe(topic);
    // for validation...
    rcvdMsgs.print();
    // just our msgs
    rcvdMsgs = selectMsgs(rcvdMsgs, mgen.pattern());
    TStream<String> rcvdAsString = rcvdMsgs.transform(msgToJSONStringFunc());
    if (testBuildOnly(top))
        return;
    completeAndValidate(subClientId, top, rcvdAsString, SEC_TIMEOUT, expectedAsString.toArray(new String[0]));
    assertTrue(sink != null);
}

Example 4 with TSink

use of com.ibm.streamsx.topology.TSink in project streamsx.topology by IBMStreams.

the class MqttStreamsTest method testReusableApp.

@Test
public void testReusableApp() throws Exception {
    checkAssumes();
    setupDebug();
    Topology top = new Topology("testReusableApp");
    MsgGenerator mgen = new MsgGenerator(top.getName());
    String subClientId = newSubClientId(top.getName());
    String pubClientId = newPubClientId(top.getName());
    String topic = getMqttTopics()[0];
    List<Message> msgs = createMsgs(mgen, null);
    // Test an app structured more as a "reusable asset" - i.e.,
    // where the mqtt connection info (URI, authInfo) and
    // topic are defined at submission time.
    // define/create the app's submission parameters
    ParameterHelper params = new ParameterHelper(top);
    params.definitions().put("mqtt.serverURI", String.class);
    params.definitions().put("mqtt.userID", System.getProperty("user.name"));
    params.definitions().put("mqtt.password", String.class);
    params.definitions().put("mqtt.pub.topic", String.class);
    params.definitions().put("mqtt.sub.topic", String.class);
    params.createAll();
    // add the actual param values for our call to submit()
    Map<String, Object> submitParams = new HashMap<>();
    submitParams.put("mqtt.serverURI", "tcp://localhost:1883");
    // submitParams.put("mqtt.userID", System.getProperty("user.name"));
    submitParams.put("mqtt.password", "myMosquittoPw");
    submitParams.put("mqtt.pub.topic", topic);
    submitParams.put("mqtt.sub.topic", topic);
    getConfig().put(ContextProperties.SUBMISSION_PARAMS, submitParams);
    // Produce and consume the msgs
    Map<String, Object> pconfig = createConfig(pubClientId);
    addMqttParams(pconfig, false, params);
    Map<String, Object> cconfig = createConfig(subClientId);
    addMqttParams(cconfig, true, params);
    MqttStreams producer = new MqttStreams(top, pconfig);
    MqttStreams consumer = new MqttStreams(top, cconfig);
    TStream<Message> msgsToPublish = top.constants(msgs).modify(new InitialDelay<Message>(PUB_DELAY_MSEC));
    TSink sink = producer.publish(msgsToPublish, params.getString("mqtt.pub.topic"));
    TStream<Message> rcvdMsgs = consumer.subscribe(params.getString("mqtt.sub.topic"));
    // for validation...
    rcvdMsgs.print();
    // just our msgs
    rcvdMsgs = selectMsgs(rcvdMsgs, mgen.pattern());
    TStream<String> rcvdAsString = rcvdMsgs.transform(msgToJSONStringFunc());
    msgs = modifyList(msgs, setTopic(topic));
    List<String> expectedAsString = mapList(msgs, msgToJSONStringFunc());
    if (testBuildOnly(top))
        return;
    completeAndValidate(subClientId, top, rcvdAsString, SEC_TIMEOUT, expectedAsString.toArray(new String[0]));
    assertTrue(sink != null);
}

Example 5 with TSink

use of com.ibm.streamsx.topology.TSink in project streamsx.topology by IBMStreams.

the class MqttStreamsTest method testSubtypeImplicitTopicProducer.

@Test
public void testSubtypeImplicitTopicProducer() throws Exception {
    checkAssumes();
    setupDebug();
    Topology top = new Topology("testSubtypeImplicitTopicProducer");
    MsgGenerator mgen = new MsgGenerator(top.getName());
    String subClientId = newSubClientId(top.getName());
    String pubClientId = newPubClientId(top.getName());
    String topic = getMqttTopics()[0];
    List<MyMsgSubtype> msgs = new ArrayList<>();
    for (Message m : createMsgs(mgen, topic)) {
        msgs.add(new MyMsgSubtype(m.getMessage(), m.getTopic()));
    }
    List<String> expectedAsString = mapList(msgs, subtypeMsgToJSONStringFunc(null));
    // Test producer that takes a TStream<MyMsgSubtype> implicit topic
    MqttStreams producer = new MqttStreams(top, createConfig(pubClientId));
    MqttStreams consumer = new MqttStreams(top, createConfig(subClientId));
    TStream<MyMsgSubtype> msgsToPublish = top.constants(msgs).asType(MyMsgSubtype.class).modify(new InitialDelay<MyMsgSubtype>(PUB_DELAY_MSEC));
    TSink sink = producer.publish(msgsToPublish);
    TStream<Message> rcvdMsgs = consumer.subscribe(new Value<String>(topic));
    // for validation...
    rcvdMsgs.print();
    // just our msgs
    rcvdMsgs = selectMsgs(rcvdMsgs, mgen.pattern());
    TStream<String> rcvdAsString = rcvdMsgs.transform(msgToJSONStringFunc());
    if (testBuildOnly(top))
        return;
    completeAndValidate(subClientId, top, rcvdAsString, SEC_TIMEOUT, expectedAsString.toArray(new String[0]));
    assertTrue(sink != null);
}