use of com.ibm.streamsx.topology.logic.Value in project streamsx.topology by IBMStreams.
the class KafkaSample method publishSubscribe.
/**
* Publish some messages to a topic, scribe to the topic and report
* received messages.
* @param contextType string value of a {@code StreamsContext.Type}
* @throws Exception
*/
public void publishSubscribe(String contextType) throws Exception {
setupConfig();
identifyStreamsxMessagingVer();
Topology top = new Topology("kafkaSample");
String groupId = newGroupId(top.getName());
Supplier<String> topic = new Value<String>(TOPIC);
KafkaProducer producer = new KafkaProducer(top, createProducerConfig());
KafkaConsumer consumer = new KafkaConsumer(top, createConsumerConfig(groupId));
TStream<Message> msgs = makeStreamToPublish(top);
// for the sample, give the consumer a chance to become ready
msgs = msgs.modify(initialDelayFunc(PUB_DELAY_MSEC));
producer.publish(msgs, topic);
TStream<Message> rcvdMsgs = consumer.subscribe(topic);
// show what we received
rcvdMsgs.print();
// Execute the topology, to send and receive the messages.
Future<?> future = StreamsContextFactory.getStreamsContext(contextType).submit(top, config);
if (contextType.contains("DISTRIBUTED")) {
System.out.println("\nSee the job's PE console logs for the topology output.\n");
} else if (contextType.contains("STANDALONE") || contextType.contains("EMBEDDED")) {
Thread.sleep(15000);
future.cancel(true);
}
}
use of com.ibm.streamsx.topology.logic.Value in project streamsx.topology by IBMStreams.
the class KafkaStreamsTest method testImplicitTopicProducer.
@Test
public void testImplicitTopicProducer() throws Exception {
checkAssumes();
Topology top = new Topology("testImplicitTopicProducer");
MsgGenerator mgen = new MsgGenerator(top.getName());
String groupId = newGroupId(top.getName());
String topicVal = getKafkaTopics()[0];
Supplier<String> topic = new Value<String>(topicVal);
KafkaProducer producer = new KafkaProducer(top, createProducerConfig());
KafkaConsumer consumer = new KafkaConsumer(top, createConsumerConfig(groupId));
// Test producer that takes an arbitrary TStream<T> and implicit topic
List<Vals> msgs = new ArrayList<>();
msgs.add(new Vals(mgen.create(topicVal, "Hello"), null, null));
msgs.add(new Vals(mgen.create(topicVal, "key1", "Are you there?"), "key1", null));
TStream<Vals> valsToPublish = top.constants(msgs).asType(Vals.class);
TStream<Message> msgsToPublish = valsToPublish.transform(msgFromValsFunc(topicVal));
msgsToPublish = msgsToPublish.modify(new InitialDelay<Message>(PUB_DELAY_MSEC));
producer.publish(msgsToPublish);
TStream<Message> rcvdMsgs = consumer.subscribe(topic);
// for validation...
rcvdMsgs.print();
// just our msgs
rcvdMsgs = selectMsgs(rcvdMsgs, mgen.pattern());
TStream<String> rcvdAsString = rcvdMsgs.transform(msgToJSONStringFunc());
List<Message> expectedAsMessage = mapList(msgs, msgFromValsFunc(topicVal));
List<String> expectedAsString = mapList(expectedAsMessage, msgToJSONStringFunc());
setupDebug();
if (testBuildOnly(top))
return;
completeAndValidate(groupId, top, rcvdAsString, SEC_TIMEOUT, expectedAsString.toArray(new String[0]));
}
use of com.ibm.streamsx.topology.logic.Value in project streamsx.topology by IBMStreams.
the class MqttStreamsTest method testExplicitTopicProducer.
@Test
public void testExplicitTopicProducer() throws Exception {
checkAssumes();
setupDebug();
Topology top = new Topology("testNewExplicitTopicProducer");
MsgGenerator mgen = new MsgGenerator(top.getName());
String subClientId = newSubClientId(top.getName());
String pubClientId = newPubClientId(top.getName());
String topicVal = getMqttTopics()[0];
Supplier<String> topic = new Value<String>(topicVal);
List<Message> msgs = createMsgs(mgen, null);
List<String> expectedAsString = mapList(modifyList(msgs, setTopic(topicVal)), msgToJSONStringFunc());
// Test producer that takes an explicit topic and implicit config qos
MqttStreams producer = new MqttStreams(top, createConfig(pubClientId));
MqttStreams consumer = new MqttStreams(top, createConfig(subClientId));
TStream<Message> msgsToPublish = top.constants(msgs).modify(new InitialDelay<Message>(PUB_DELAY_MSEC));
TSink sink = producer.publish(msgsToPublish, topic);
TStream<Message> rcvdMsgs = consumer.subscribe(topic);
// for validation...
rcvdMsgs.print();
// just our msgs
rcvdMsgs = selectMsgs(rcvdMsgs, mgen.pattern());
TStream<String> rcvdAsString = rcvdMsgs.transform(msgToJSONStringFunc());
if (testBuildOnly(top))
return;
completeAndValidate(subClientId, top, rcvdAsString, SEC_TIMEOUT, expectedAsString.toArray(new String[0]));
assertTrue(sink != null);
}
use of com.ibm.streamsx.topology.logic.Value in project streamsx.topology by IBMStreams.
the class KafkaStreamsTest method testMultiTopicProducer.
@Test
public void testMultiTopicProducer() throws Exception {
checkAssumes();
// streamsx.messaging issue#118 prevents successful execution
// For standalone it seems to consistently get 0 topic1 msgs.
assumeTrue(getTesterType() != StreamsContext.Type.STANDALONE_TESTER);
Topology top = new Topology("testMultiTopicProducer");
MsgGenerator mgen = new MsgGenerator(top.getName());
String groupId = newGroupId(top.getName());
String[] topics = getKafkaTopics();
String topic1Val = topics[0];
String topic2Val = topics[1];
Supplier<String> topic1 = new Value<String>(topic1Val);
Supplier<String> topic2 = new Value<String>(topic2Val);
KafkaProducer producer = new KafkaProducer(top, createProducerConfig());
KafkaConsumer consumer = new KafkaConsumer(top, createConsumerConfig(groupId));
// Test producer that publishes to multiple topics (implies implicit topic)
List<Message> topic1Msgs = new ArrayList<>();
topic1Msgs.add(new SimpleMessage(mgen.create(topic1Val, "Hello"), null, topic1Val));
topic1Msgs.add(new SimpleMessage(mgen.create(topic1Val, "Are you there?"), null, topic1Val));
List<Message> topic2Msgs = new ArrayList<>();
topic2Msgs.add(new SimpleMessage(mgen.create(topic2Val, "Hello"), null, topic2Val));
topic2Msgs.add(new SimpleMessage(mgen.create(topic2Val, "Are you there?"), null, topic2Val));
List<Message> msgs = new ArrayList<>(topic1Msgs);
msgs.addAll(topic2Msgs);
TStream<Message> msgsToPublish = top.constants(msgs);
msgsToPublish = msgsToPublish.modify(new InitialDelay<Message>(PUB_DELAY_MSEC));
producer.publish(msgsToPublish);
TStream<Message> rcvdTopic1Msgs = consumer.subscribe(topic1);
TStream<Message> rcvdTopic2Msgs = consumer.subscribe(topic2);
// for validation...
TStream<Message> rcvdMsgs = rcvdTopic1Msgs.union(rcvdTopic2Msgs);
rcvdMsgs.print();
// just our msgs
rcvdMsgs = selectMsgs(rcvdMsgs, mgen.pattern());
TStream<String> rcvdAsString = rcvdMsgs.transform(msgToJSONStringFunc());
List<String> expectedAsString = mapList(msgs, msgToJSONStringFunc());
setupDebug();
if (testBuildOnly(top))
return;
completeAndValidateUnordered(groupId, top, rcvdAsString, SEC_TIMEOUT, expectedAsString.toArray(new String[0]));
}
use of com.ibm.streamsx.topology.logic.Value in project streamsx.topology by IBMStreams.
the class MqttStreamsTest method testConfigParams.
@Test
public void testConfigParams() throws Exception {
checkAssumes();
setupDebug();
Topology top = new Topology("testConfigParams");
MsgGenerator mgen = new MsgGenerator(top.getName());
String clientId = newPubClientId(top.getName());
String topicVal = getMqttTopics()[0];
Supplier<String> topic = new Value<String>(topicVal);
List<Message> msgs = createMsgs(mgen, null);
// Test more config properties to be sure we don't blow up
Map<String, Object> config = createConfig(clientId);
config.put("defaultQOS", 1);
config.put("keepAliveInterval", 20);
config.put("commandTimeoutMsec", 30000L);
config.put("reconnectDelayMsec", 5000L);
config.put("receiveBufferSize", 10);
config.put("reconnectionBound", 20);
config.put("retain", false);
config.put("userID", System.getProperty("user.name"));
config.put("password", "foobar");
config.put("trustStore", "/tmp/no-such-trustStore");
config.put("trustStorePassword", "woohoo");
config.put("keyStore", "/tmp/no-such-keyStore");
config.put("keyStorePassword", "woohoo");
MqttStreams mqtt = new MqttStreams(top, config);
Map<String, Object> pcfg = mqtt.getConfig();
for (String s : config.keySet()) assertEquals("property " + s, config.get(s), pcfg.get(s));
TStream<Message> msgsToPublish = top.constants(msgs).modify(new InitialDelay<Message>(PUB_DELAY_MSEC));
TSink sink = mqtt.publish(msgsToPublish, topic);
TStream<Message> rcvdMsgs = mqtt.subscribe(topic);
// for validation...
rcvdMsgs.print();
// bundle construction fails for unrecognized or incorrectly typed SPL op params
File actBundle = (File) StreamsContextFactory.getStreamsContext(StreamsContext.Type.BUNDLE).submit(top, getConfig()).get(15, TimeUnit.SECONDS);
System.out.println("bundle " + actBundle.getAbsolutePath());
assertTrue(actBundle != null);
actBundle.delete();
assertTrue(sink != null);
}
Aggregations