use of com.ibm.streamsx.topology.messaging.kafka.KafkaProducer in project streamsx.topology by IBMStreams.
the class KafkaStreamsTest method testSubtypeImplicitTopicProducer.
@Test
public void testSubtypeImplicitTopicProducer() throws Exception {
checkAssumes();
Topology top = new Topology("testSubtypeImplicitTopicProducer");
MsgGenerator mgen = new MsgGenerator(top.getName());
String groupId = newGroupId(top.getName());
String topicVal = getKafkaTopics()[0];
Supplier<String> topic = new Value<String>(topicVal);
KafkaProducer producer = new KafkaProducer(top, createProducerConfig());
KafkaConsumer consumer = new KafkaConsumer(top, createConsumerConfig(groupId));
// Test producer that takes a TStream<MyMsgSubtype> implicit topic
List<MyMsgSubtype> msgs = new ArrayList<>();
msgs.add(new MyMsgSubtype(mgen.create(topicVal, "Hello"), null, topicVal));
msgs.add(new MyMsgSubtype(mgen.create(topicVal, "key1", "Are you there?"), "key1", topicVal));
TStream<MyMsgSubtype> msgsToPublish = top.constants(msgs).asType(MyMsgSubtype.class);
msgsToPublish = msgsToPublish.modify(new InitialDelay<MyMsgSubtype>(PUB_DELAY_MSEC));
producer.publish(msgsToPublish);
TStream<Message> rcvdMsgs = consumer.subscribe(topic);
// for validation...
rcvdMsgs.print();
// just our msgs
rcvdMsgs = selectMsgs(rcvdMsgs, mgen.pattern());
TStream<String> rcvdAsString = rcvdMsgs.transform(msgToJSONStringFunc());
List<String> expectedAsString = mapList(msgs, subtypeMsgToJSONStringFunc(null));
setupDebug();
if (testBuildOnly(top))
return;
completeAndValidate(groupId, top, rcvdAsString, SEC_TIMEOUT, expectedAsString.toArray(new String[0]));
}
use of com.ibm.streamsx.topology.messaging.kafka.KafkaProducer in project streamsx.topology by IBMStreams.
the class KafkaStreamsTest method testExplicitTopicProducer.
@Test
public void testExplicitTopicProducer() throws Exception {
checkAssumes();
Topology top = new Topology("testNewExplicitTopicProducer");
MsgGenerator mgen = new MsgGenerator(top.getName());
String groupId = newGroupId(top.getName());
String topicVal = getKafkaTopics()[0];
Supplier<String> topic = new Value<String>(topicVal);
KafkaProducer producer = new KafkaProducer(top, createProducerConfig());
KafkaConsumer consumer = new KafkaConsumer(top, createConsumerConfig(groupId));
// Test producer that takes an arbitrary TStream<T> and explicit topic
List<Vals> msgs = new ArrayList<>();
msgs.add(new Vals(mgen.create(topicVal, "Hello"), null, null));
msgs.add(new Vals(mgen.create(topicVal, "key1", "Are you there?"), "key1", null));
msgs.add(new Vals(mgen.create(topicVal, "Msg with an empty key"), "", null));
msgs.add(new Vals("", mgen.create(topicVal, null, "Msg with an empty msg (this is the key)"), null));
TStream<Vals> valsToPublish = top.constants(msgs).asType(Vals.class);
TStream<Message> msgsToPublish = valsToPublish.transform(msgFromValsFunc(null));
msgsToPublish = msgsToPublish.modify(new InitialDelay<Message>(PUB_DELAY_MSEC));
producer.publish(msgsToPublish, topic);
TStream<Message> rcvdMsgs = consumer.subscribe(topic);
// for validation...
rcvdMsgs.print();
// just our msgs
rcvdMsgs = selectMsgs(rcvdMsgs, mgen.pattern());
TStream<String> rcvdAsString = rcvdMsgs.transform(msgToJSONStringFunc());
List<Message> expectedAsMessage = mapList(msgs, msgFromValsFunc(topicVal));
expectedAsMessage = modifyList(expectedAsMessage, adjustKey());
List<String> expectedAsString = mapList(expectedAsMessage, msgToJSONStringFunc());
setupDebug();
if (testBuildOnly(top))
return;
completeAndValidate(groupId, top, rcvdAsString, SEC_TIMEOUT, expectedAsString.toArray(new String[0]));
}
use of com.ibm.streamsx.topology.messaging.kafka.KafkaProducer in project streamsx.topology by IBMStreams.
the class KafkaStreamsTest method testSubtypeExplicitTopicProducer.
@Test
public void testSubtypeExplicitTopicProducer() throws Exception {
checkAssumes();
Topology top = new Topology("testSubtypeExplicitTopicProducer");
MsgGenerator mgen = new MsgGenerator(top.getName());
String groupId = newGroupId(top.getName());
String topicVal = getKafkaTopics()[0];
Supplier<String> topic = new Value<String>(topicVal);
KafkaProducer producer = new KafkaProducer(top, createProducerConfig());
KafkaConsumer consumer = new KafkaConsumer(top, createConsumerConfig(groupId));
// Test producer that takes a TStream<MyMsgSubtype>
List<MyMsgSubtype> msgs = new ArrayList<>();
msgs.add(new MyMsgSubtype(mgen.create(topicVal, "Hello")));
msgs.add(new MyMsgSubtype(mgen.create(topicVal, "key1", "Are you there?"), "key1"));
TStream<MyMsgSubtype> msgsToPublish = top.constants(msgs).asType(MyMsgSubtype.class);
msgsToPublish = msgsToPublish.modify(new InitialDelay<MyMsgSubtype>(PUB_DELAY_MSEC));
producer.publish(msgsToPublish, topic);
TStream<Message> rcvdMsgs = consumer.subscribe(topic);
// for validation...
rcvdMsgs.print();
// just our msgs
rcvdMsgs = selectMsgs(rcvdMsgs, mgen.pattern());
TStream<String> rcvdAsString = rcvdMsgs.transform(msgToJSONStringFunc());
List<String> expectedAsString = mapList(msgs, subtypeMsgToJSONStringFunc(topicVal));
setupDebug();
if (testBuildOnly(top))
return;
completeAndValidate(groupId, top, rcvdAsString, SEC_TIMEOUT, expectedAsString.toArray(new String[0]));
}
use of com.ibm.streamsx.topology.messaging.kafka.KafkaProducer in project streamsx.topology by IBMStreams.
the class KafkaStreamsTest method testMsgImplProducer.
@Test
public void testMsgImplProducer() throws Exception {
checkAssumes();
Topology top = new Topology("testMsgImplProducer");
MsgGenerator mgen = new MsgGenerator(top.getName());
String groupId = newGroupId(top.getName());
String topicVal = getKafkaTopics()[0];
Supplier<String> topic = new Value<String>(topicVal);
KafkaProducer producer = new KafkaProducer(top, createProducerConfig());
KafkaConsumer consumer = new KafkaConsumer(top, createConsumerConfig(groupId));
// Test producer that takes TStream<SimpleMessage> and an explicit topic.
List<Message> msgs = new ArrayList<>();
msgs.add(new SimpleMessage(mgen.create(topicVal, "Hello")));
msgs.add(new SimpleMessage(mgen.create(topicVal, "key1", "Are you there?"), "key1"));
TStream<Message> msgsToPublish = top.constants(msgs);
msgsToPublish = msgsToPublish.modify(new InitialDelay<Message>(PUB_DELAY_MSEC));
producer.publish(msgsToPublish, topic);
TStream<Message> rcvdMsgs = consumer.subscribe(topic);
// for validation...
rcvdMsgs.print();
// just our msgs
rcvdMsgs = selectMsgs(rcvdMsgs, mgen.pattern());
TStream<String> rcvdAsString = rcvdMsgs.transform(msgToJSONStringFunc());
msgs = modifyList(msgs, setTopic(topicVal));
List<String> expectedAsString = mapList(msgs, msgToJSONStringFunc());
setupDebug();
if (testBuildOnly(top))
return;
completeAndValidate(groupId, top, rcvdAsString, SEC_TIMEOUT, expectedAsString.toArray(new String[0]));
}
Aggregations