Examples with TStream com.ibm.streamsx.topology.TStream used on opensource projects

Example 26 with TStream

use of com.ibm.streamsx.topology.TStream in project streamsx.kafka by IBMStreams.

the class KafkaOperatorsAttrNameParamsTest method kafkaAttrNameParamsTest.

@Test
public void kafkaAttrNameParamsTest() throws Exception {
    Topology topo = getTopology();
    StreamSchema producerSchema = com.ibm.streams.operator.Type.Factory.getStreamSchema("tuple<int32 " + PROD_KEY_ATTR_NAME + ", rstring " + PROD_MSG_ATTR_NAME + ", rstring " + PROD_TOPIC_ATTR_NAME + ", int32 " + PROD_PARTITION_ATTR_NAME + ">");
    // create the producer (produces tuples after a short delay)
    Map<String, Object> producerProps = new HashMap<>();
    producerProps.put("propertiesFile", Constants.PROPERTIES_FILE_PATH);
    producerProps.put("messageAttribute", producerSchema.getAttribute(PROD_MSG_ATTR_NAME));
    producerProps.put("keyAttribute", producerSchema.getAttribute(PROD_KEY_ATTR_NAME));
    producerProps.put("topicAttribute", producerSchema.getAttribute(PROD_TOPIC_ATTR_NAME));
    producerProps.put("partitionAttribute", producerSchema.getAttribute(PROD_PARTITION_ATTR_NAME));
    TStream<String> srcStream = topo.strings(MSG).modify(new Delay<>(5000));
    SPL.invokeSink(Constants.KafkaProducerOp, SPLStreams.convertStream(srcStream, new ProducerConverter(), producerSchema), producerProps);
    // create the consumer
    StreamSchema consumerSchema = com.ibm.streams.operator.Type.Factory.getStreamSchema("tuple<int32 " + CONS_KEY_ATTR_NAME + ", rstring " + CONS_MSG_ATTR_NAME + ", rstring " + CONS_TOPIC_ATTR_NAME + ">");
    Map<String, Object> consumerProps = new HashMap<String, Object>();
    consumerProps.put("propertiesFile", Constants.PROPERTIES_FILE_PATH);
    consumerProps.put("outputMessageAttributeName", CONS_MSG_ATTR_NAME);
    consumerProps.put("outputKeyAttributeName", CONS_KEY_ATTR_NAME);
    consumerProps.put("outputTopicAttributeName", CONS_TOPIC_ATTR_NAME);
    consumerProps.put("topic", Constants.TOPIC_TEST);
    SPLStream consumerStream = SPL.invokeSource(topo, Constants.KafkaConsumerOp, consumerProps, consumerSchema);
    SPLStream msgStream = SPLStreams.stringToSPLStream(consumerStream.convert(t -> {
        return t.getString(CONS_TOPIC_ATTR_NAME) + ":" + t.getInt(CONS_KEY_ATTR_NAME) + ":" + t.getString(CONS_MSG_ATTR_NAME);
    }));
    // test the output of the consumer
    StreamsContext<?> context = StreamsContextFactory.getStreamsContext(Type.DISTRIBUTED_TESTER);
    Tester tester = topo.getTester();
    Condition<List<String>> condition = KafkaSPLStreamsUtils.stringContentsUnordered(tester, msgStream, Constants.TOPIC_TEST + ":" + KEY + ":" + MSG);
    tester.complete(context, new HashMap<>(), condition, 30, TimeUnit.SECONDS);
    // check the results
    Assert.assertTrue(condition.getResult().size() > 0);
    Assert.assertTrue(condition.getResult().toString(), condition.valid());
}

Example 27 with TStream

use of com.ibm.streamsx.topology.TStream in project streamsx.kafka by IBMStreams.

the class KafkaOperatorsIntTypeTest method kafkaIntTypeTest.

@Test
public void kafkaIntTypeTest() throws Exception {
    Topology topo = getTopology();
    StreamSchema schema = KafkaSPLStreamsUtils.INT_SCHEMA;
    // create the producer (produces tuples after a short delay)
    TStream<Integer> srcStream = topo.strings(DATA).transform(s -> Integer.valueOf(s)).modify(new Delay<>(5000));
    SPLStream splSrcStream = SPLStreams.convertStream(srcStream, new Converter(), schema);
    SPL.invokeSink(Constants.KafkaProducerOp, splSrcStream, getKafkaParams());
    // create the consumer
    SPLStream consumerStream = SPL.invokeSource(topo, Constants.KafkaConsumerOp, getKafkaParams(), schema);
    SPLStream msgStream = SPLStreams.stringToSPLStream(consumerStream.convert(t -> String.valueOf(t.getInt("message"))));
    // test the output of the consumer
    StreamsContext<?> context = StreamsContextFactory.getStreamsContext(Type.DISTRIBUTED_TESTER);
    Tester tester = topo.getTester();
    Condition<List<String>> condition = KafkaSPLStreamsUtils.stringContentsUnordered(tester, msgStream, DATA);
    tester.complete(context, new HashMap<>(), condition, 30, TimeUnit.SECONDS);
    // check the results
    Assert.assertTrue(condition.getResult().size() > 0);
    Assert.assertTrue(condition.getResult().toString(), condition.valid());
}

Example 28 with TStream

use of com.ibm.streamsx.topology.TStream in project streamsx.kafka by IBMStreams.

the class KafkaOperatorsNoKey method kafkaNoKeyTest.

@Test
public void kafkaNoKeyTest() throws Exception {
    Topology topo = getTopology();
    // create the producer (produces tuples after a short delay)
    TStream<String> stringSrcStream = topo.strings(Constants.STRING_DATA).modify(new Delay<>(5000));
    SPL.invokeSink(Constants.KafkaProducerOp, KafkaSPLStreamsUtils.convertStreamToKafkaTuple(stringSrcStream, false), getKafkaParams());
    // create the consumer
    SPLStream consumerStream = SPL.invokeSource(topo, Constants.KafkaConsumerOp, getKafkaParams(), KafkaSPLStreamsUtils.STRING_NOKEY_SCHEMA);
    SPLStream msgStream = SPLStreams.stringToSPLStream(consumerStream.convert(t -> t.getString("message")));
    // test the output of the consumer
    StreamsContext<?> context = StreamsContextFactory.getStreamsContext(Type.DISTRIBUTED_TESTER);
    Tester tester = topo.getTester();
    Condition<List<String>> condition = KafkaSPLStreamsUtils.stringContentsUnordered(tester, msgStream, Constants.STRING_DATA);
    tester.complete(context, new HashMap<>(), condition, 30, TimeUnit.SECONDS);
    // check the results
    Assert.assertTrue(condition.getResult().size() > 0);
    Assert.assertTrue(condition.getResult().toString(), condition.valid());
}

Example 29 with TStream

use of com.ibm.streamsx.topology.TStream in project streamsx.kafka by IBMStreams.

the class KafkaProducerFanOutTest method kafkaFanOutTest.

@Test
public void kafkaFanOutTest() throws Exception {
    Topology topo = getTopology();
    // create the producers (produces tuples after a short delay)
    TStream<String> stringSrcStream = topo.strings(Constants.STRING_DATA).modify(new Delay<>(5000)).lowLatency();
    SPL.invokeSink(Constants.KafkaProducerOp, KafkaSPLStreamsUtils.convertStreamToKafkaTuple(stringSrcStream), getKafkaParams());
    SPL.invokeSink(Constants.KafkaProducerOp, KafkaSPLStreamsUtils.convertStreamToKafkaTuple(stringSrcStream), getKafkaParams());
    // create the consumer
    SPLStream consumerStream = SPL.invokeSource(topo, Constants.KafkaConsumerOp, getKafkaParams(), KafkaSPLStreamsUtils.STRING_SCHEMA);
    SPLStream msgStream = SPLStreams.stringToSPLStream(consumerStream.convert(t -> t.getString("message")));
    // test the output of the consumer
    StreamsContext<?> context = StreamsContextFactory.getStreamsContext(Type.DISTRIBUTED_TESTER);
    Tester tester = topo.getTester();
    // both producers are sending the same data, so each result is duplicated
    String[] expectedArr = KafkaSPLStreamsUtils.duplicateArrayEntries(Constants.STRING_DATA, 2);
    Condition<List<String>> condition = KafkaSPLStreamsUtils.stringContentsUnordered(tester, msgStream, expectedArr);
    tester.complete(context, new HashMap<>(), condition, 30, TimeUnit.SECONDS);
    // check the results
    Assert.assertTrue(condition.getResult().size() > 0);
    Assert.assertTrue(condition.getResult().toString(), condition.valid());
}