use of com.ibm.streamsx.topology.logic.Value in project streamsx.topology by IBMStreams.
the class MqttStreamsTest method testMsgImplProducer.
@Test
public void testMsgImplProducer() throws Exception {
checkAssumes();
setupDebug();
Topology top = new Topology("testMsgImplProducer");
MsgGenerator mgen = new MsgGenerator(top.getName());
String subClientId = newSubClientId(top.getName());
String pubClientId = newPubClientId(top.getName());
String topicVal = getMqttTopics()[0];
Supplier<String> topic = new Value<String>(topicVal);
List<Message> msgs = createMsgs(mgen, null);
List<String> expectedAsString = mapList(modifyList(msgs, setTopic(topicVal)), msgToJSONStringFunc());
// Test producer that takes TStream<SimpleMessage> and an explicit topic.
MqttStreams producer = new MqttStreams(top, createConfig(pubClientId));
MqttStreams consumer = new MqttStreams(top, createConfig(subClientId));
TStream<Message> msgsToPublish = top.constants(msgs).modify(new InitialDelay<Message>(PUB_DELAY_MSEC));
TSink sink = producer.publish(msgsToPublish, topic);
TStream<Message> rcvdMsgs = consumer.subscribe(topic);
// for validation...
rcvdMsgs.print();
// just our msgs
rcvdMsgs = selectMsgs(rcvdMsgs, mgen.pattern());
TStream<String> rcvdAsString = rcvdMsgs.transform(msgToJSONStringFunc());
if (testBuildOnly(top))
return;
completeAndValidate(subClientId, top, rcvdAsString, SEC_TIMEOUT, expectedAsString.toArray(new String[0]));
assertTrue(sink != null);
}
use of com.ibm.streamsx.topology.logic.Value in project streamsx.topology by IBMStreams.
the class KafkaStreamsTest method testSubtypeImplicitTopicProducer.
@Test
public void testSubtypeImplicitTopicProducer() throws Exception {
checkAssumes();
Topology top = new Topology("testSubtypeImplicitTopicProducer");
MsgGenerator mgen = new MsgGenerator(top.getName());
String groupId = newGroupId(top.getName());
String topicVal = getKafkaTopics()[0];
Supplier<String> topic = new Value<String>(topicVal);
KafkaProducer producer = new KafkaProducer(top, createProducerConfig());
KafkaConsumer consumer = new KafkaConsumer(top, createConsumerConfig(groupId));
// Test producer that takes a TStream<MyMsgSubtype> implicit topic
List<MyMsgSubtype> msgs = new ArrayList<>();
msgs.add(new MyMsgSubtype(mgen.create(topicVal, "Hello"), null, topicVal));
msgs.add(new MyMsgSubtype(mgen.create(topicVal, "key1", "Are you there?"), "key1", topicVal));
TStream<MyMsgSubtype> msgsToPublish = top.constants(msgs).asType(MyMsgSubtype.class);
msgsToPublish = msgsToPublish.modify(new InitialDelay<MyMsgSubtype>(PUB_DELAY_MSEC));
producer.publish(msgsToPublish);
TStream<Message> rcvdMsgs = consumer.subscribe(topic);
// for validation...
rcvdMsgs.print();
// just our msgs
rcvdMsgs = selectMsgs(rcvdMsgs, mgen.pattern());
TStream<String> rcvdAsString = rcvdMsgs.transform(msgToJSONStringFunc());
List<String> expectedAsString = mapList(msgs, subtypeMsgToJSONStringFunc(null));
setupDebug();
if (testBuildOnly(top))
return;
completeAndValidate(groupId, top, rcvdAsString, SEC_TIMEOUT, expectedAsString.toArray(new String[0]));
}
use of com.ibm.streamsx.topology.logic.Value in project streamsx.topology by IBMStreams.
the class KafkaStreamsTest method testExplicitTopicProducer.
@Test
public void testExplicitTopicProducer() throws Exception {
checkAssumes();
Topology top = new Topology("testNewExplicitTopicProducer");
MsgGenerator mgen = new MsgGenerator(top.getName());
String groupId = newGroupId(top.getName());
String topicVal = getKafkaTopics()[0];
Supplier<String> topic = new Value<String>(topicVal);
KafkaProducer producer = new KafkaProducer(top, createProducerConfig());
KafkaConsumer consumer = new KafkaConsumer(top, createConsumerConfig(groupId));
// Test producer that takes an arbitrary TStream<T> and explicit topic
List<Vals> msgs = new ArrayList<>();
msgs.add(new Vals(mgen.create(topicVal, "Hello"), null, null));
msgs.add(new Vals(mgen.create(topicVal, "key1", "Are you there?"), "key1", null));
msgs.add(new Vals(mgen.create(topicVal, "Msg with an empty key"), "", null));
msgs.add(new Vals("", mgen.create(topicVal, null, "Msg with an empty msg (this is the key)"), null));
TStream<Vals> valsToPublish = top.constants(msgs).asType(Vals.class);
TStream<Message> msgsToPublish = valsToPublish.transform(msgFromValsFunc(null));
msgsToPublish = msgsToPublish.modify(new InitialDelay<Message>(PUB_DELAY_MSEC));
producer.publish(msgsToPublish, topic);
TStream<Message> rcvdMsgs = consumer.subscribe(topic);
// for validation...
rcvdMsgs.print();
// just our msgs
rcvdMsgs = selectMsgs(rcvdMsgs, mgen.pattern());
TStream<String> rcvdAsString = rcvdMsgs.transform(msgToJSONStringFunc());
List<Message> expectedAsMessage = mapList(msgs, msgFromValsFunc(topicVal));
expectedAsMessage = modifyList(expectedAsMessage, adjustKey());
List<String> expectedAsString = mapList(expectedAsMessage, msgToJSONStringFunc());
setupDebug();
if (testBuildOnly(top))
return;
completeAndValidate(groupId, top, rcvdAsString, SEC_TIMEOUT, expectedAsString.toArray(new String[0]));
}
use of com.ibm.streamsx.topology.logic.Value in project streamsx.topology by IBMStreams.
the class KafkaConsumer method subscribe.
/**
* Subscribe to a topic and create a stream of messages.
* <p>
* N.B., A topology that includes this will not support
* {@code StreamsContext.Type.EMBEDDED}.
* <p>
* N.B. due to com.ibm.streamsx.messaging
* <a href="https://github.com/IBMStreams/streamsx.messaging/issues/118">issue#118</a>,
* multiple consumers will have issues in
* {@code StreamsContext.Type.STANDALONE}.
* <p>
* N.B. due to com.ibm.streamsx.messaging
* <a href="https://github.com/IBMStreams/streamsx.messaging/issues/117">issue#117</a>,
* a consumer in {@code StreamsContext.Type.STANDALONE} subsequently results
* in an orphaned @{code standalone} processes that continues as the lead
* group/topic consumer thereby preventing subsequent instances of the
* group/topic consumer from receiving messages.
* <p>
* N.B. due to com.ibm.streamsx.messaging
* <a href="https://github.com/IBMStreams/streamsx.messaging/issues/114">issue#114</a>,
* a consumer essentially ignores messages generated by producers where the
* optional {@code key} is {@code null}.
* e.g., Kafka's {@code kafka-console-producer.sh} tool generates
* {@code key==null} messages.
*
* @param threadsPerTopic number of threads to allocate to processing each
* topic. May be a submission parameter.
* @param topic the topic to subscribe to. May be a submission parameter.
* @return TStream<Message>
* The generated {@code Message} tuples have a non-null {@code topic}.
* The tuple's {@code key} will be null if the Kafka message
* lacked a key or it's key was the empty string.
* @throws IllegalArgumentException if topic is null.
* @throws IllegalArgumentException if threadsPerTopic is null.
* @see Value
* @see Topology#createSubmissionParameter(String, Class)
*/
public TStream<Message> subscribe(Supplier<Integer> threadsPerTopic, Supplier<String> topic) {
if (topic == null)
throw new IllegalArgumentException("topic");
if (threadsPerTopic == null || (threadsPerTopic.get() != null && threadsPerTopic.get() <= 0))
throw new IllegalArgumentException("threadsPerTopic");
Map<String, Object> params = new HashMap<>();
params.put("topic", topic);
if (threadsPerTopic instanceof Value && threadsPerTopic.get() == 1)
// The default is one.
;
else
params.put("threadsPerTopic", threadsPerTopic);
if (!config.isEmpty())
params.put("kafkaProperty", Util.toKafkaProperty(config));
// workaround streamsx.messaging issue #107
params.put("propertiesFile", PROP_FILE_PARAM);
addPropertiesFile();
// Use SPL.invoke to avoid adding a compile time dependency
// to com.ibm.streamsx.messaging since JavaPrimitive.invoke*()
// lack "kind" based variants.
String kind = "com.ibm.streamsx.messaging.kafka::KafkaConsumer";
String className = "com.ibm.streamsx.messaging.kafka.KafkaSource";
SPLStream rawKafka = SPL.invokeSource(te, kind, params, KafkaSchemas.KAFKA);
SPL.tagOpAsJavaPrimitive(toOp(rawKafka), kind, className);
TStream<Message> rcvdMsgs = toMessageStream(rawKafka);
rcvdMsgs.colocate(rawKafka);
// workaround streamsx.messaging issue#118 w/java8
// isolate even in the single consumer case since we don't
// know if others may be subsequently created.
rcvdMsgs = rcvdMsgs.isolate();
return rcvdMsgs;
}
use of com.ibm.streamsx.topology.logic.Value in project streamsx.topology by IBMStreams.
the class KafkaStreamsTest method testSubtypeExplicitTopicProducer.
@Test
public void testSubtypeExplicitTopicProducer() throws Exception {
checkAssumes();
Topology top = new Topology("testSubtypeExplicitTopicProducer");
MsgGenerator mgen = new MsgGenerator(top.getName());
String groupId = newGroupId(top.getName());
String topicVal = getKafkaTopics()[0];
Supplier<String> topic = new Value<String>(topicVal);
KafkaProducer producer = new KafkaProducer(top, createProducerConfig());
KafkaConsumer consumer = new KafkaConsumer(top, createConsumerConfig(groupId));
// Test producer that takes a TStream<MyMsgSubtype>
List<MyMsgSubtype> msgs = new ArrayList<>();
msgs.add(new MyMsgSubtype(mgen.create(topicVal, "Hello")));
msgs.add(new MyMsgSubtype(mgen.create(topicVal, "key1", "Are you there?"), "key1"));
TStream<MyMsgSubtype> msgsToPublish = top.constants(msgs).asType(MyMsgSubtype.class);
msgsToPublish = msgsToPublish.modify(new InitialDelay<MyMsgSubtype>(PUB_DELAY_MSEC));
producer.publish(msgsToPublish, topic);
TStream<Message> rcvdMsgs = consumer.subscribe(topic);
// for validation...
rcvdMsgs.print();
// just our msgs
rcvdMsgs = selectMsgs(rcvdMsgs, mgen.pattern());
TStream<String> rcvdAsString = rcvdMsgs.transform(msgToJSONStringFunc());
List<String> expectedAsString = mapList(msgs, subtypeMsgToJSONStringFunc(topicVal));
setupDebug();
if (testBuildOnly(top))
return;
completeAndValidate(groupId, top, rcvdAsString, SEC_TIMEOUT, expectedAsString.toArray(new String[0]));
}
Aggregations