Examples with StreamsContext com.ibm.streamsx.topology.context.StreamsContext used on opensource projects

Example 11 with StreamsContext

use of com.ibm.streamsx.topology.context.StreamsContext in project streamsx.kafka by IBMStreams.

the class KafkaConsumerFanInTest method kafkaFanInTest.

@Test
public void kafkaFanInTest() throws Exception {
    Topology topo = getTopology();
    // create the producer (produces tuples after a short delay)
    TStream<String> stringSrcStream = topo.strings(Constants.STRING_DATA).modify(new Delay<>(5000));
    SPL.invokeSink(Constants.KafkaProducerOp, KafkaSPLStreamsUtils.convertStreamToKafkaTuple(stringSrcStream), getKafkaParams());
    // create the consumer
    SPLStream consumerStream1 = SPL.invokeSource(topo, Constants.KafkaConsumerOp, getConsumerParams(1), KafkaSPLStreamsUtils.STRING_SCHEMA);
    SPLStream consumerStream2 = SPL.invokeSource(topo, Constants.KafkaConsumerOp, getConsumerParams(2), KafkaSPLStreamsUtils.STRING_SCHEMA);
    SPLStream unionStream = KafkaSPLStreamsUtils.union(Arrays.asList(consumerStream1, consumerStream2), KafkaSPLStreamsUtils.STRING_SCHEMA);
    SPLStream msgStream = SPLStreams.stringToSPLStream(unionStream.convert(t -> t.getString("message")));
    // test the output of the consumer
    StreamsContext<?> context = StreamsContextFactory.getStreamsContext(Type.DISTRIBUTED_TESTER);
    Tester tester = topo.getTester();
    // both consumers consume the same data, so each result is duplicated
    String[] expectedArr = KafkaSPLStreamsUtils.duplicateArrayEntries(Constants.STRING_DATA, 2);
    Condition<List<String>> stringContentsUnordered = tester.stringContentsUnordered(msgStream.toStringStream(), expectedArr);
    HashMap<String, Object> config = new HashMap<>();
    // config.put (ContextProperties.TRACING_LEVEL, java.util.logging.Level.FINE);
    // config.put(ContextProperties.KEEP_ARTIFACTS,  new Boolean(true));
    tester.complete(context, config, stringContentsUnordered, 60, TimeUnit.SECONDS);
    // check the results
    Assert.assertTrue(stringContentsUnordered.valid());
    Assert.assertTrue(stringContentsUnordered.getResult().size() == Constants.STRING_DATA.length * 2);
}

Example 12 with StreamsContext

use of com.ibm.streamsx.topology.context.StreamsContext in project streamsx.kafka by IBMStreams.

the class KafkaOperatorsAttrNameParamsTest method kafkaAttrNameParamsTest.

@Test
public void kafkaAttrNameParamsTest() throws Exception {
    Topology topo = getTopology();
    StreamSchema producerSchema = com.ibm.streams.operator.Type.Factory.getStreamSchema("tuple<int32 " + PROD_KEY_ATTR_NAME + ", rstring " + PROD_MSG_ATTR_NAME + ", rstring " + PROD_TOPIC_ATTR_NAME + ", int32 " + PROD_PARTITION_ATTR_NAME + ">");
    // create the producer (produces tuples after a short delay)
    Map<String, Object> producerProps = new HashMap<>();
    producerProps.put("propertiesFile", Constants.PROPERTIES_FILE_PATH);
    producerProps.put("messageAttribute", producerSchema.getAttribute(PROD_MSG_ATTR_NAME));
    producerProps.put("keyAttribute", producerSchema.getAttribute(PROD_KEY_ATTR_NAME));
    producerProps.put("topicAttribute", producerSchema.getAttribute(PROD_TOPIC_ATTR_NAME));
    producerProps.put("partitionAttribute", producerSchema.getAttribute(PROD_PARTITION_ATTR_NAME));
    TStream<String> srcStream = topo.strings(MSG).modify(new Delay<>(5000));
    SPL.invokeSink(Constants.KafkaProducerOp, SPLStreams.convertStream(srcStream, new ProducerConverter(), producerSchema), producerProps);
    // create the consumer
    StreamSchema consumerSchema = com.ibm.streams.operator.Type.Factory.getStreamSchema("tuple<int32 " + CONS_KEY_ATTR_NAME + ", rstring " + CONS_MSG_ATTR_NAME + ", rstring " + CONS_TOPIC_ATTR_NAME + ">");
    Map<String, Object> consumerProps = new HashMap<String, Object>();
    consumerProps.put("propertiesFile", Constants.PROPERTIES_FILE_PATH);
    consumerProps.put("outputMessageAttributeName", CONS_MSG_ATTR_NAME);
    consumerProps.put("outputKeyAttributeName", CONS_KEY_ATTR_NAME);
    consumerProps.put("outputTopicAttributeName", CONS_TOPIC_ATTR_NAME);
    consumerProps.put("topic", Constants.TOPIC_TEST);
    SPLStream consumerStream = SPL.invokeSource(topo, Constants.KafkaConsumerOp, consumerProps, consumerSchema);
    SPLStream msgStream = SPLStreams.stringToSPLStream(consumerStream.convert(t -> {
        return t.getString(CONS_TOPIC_ATTR_NAME) + ":" + t.getInt(CONS_KEY_ATTR_NAME) + ":" + t.getString(CONS_MSG_ATTR_NAME);
    }));
    // test the output of the consumer
    StreamsContext<?> context = StreamsContextFactory.getStreamsContext(Type.DISTRIBUTED_TESTER);
    Tester tester = topo.getTester();
    Condition<List<String>> stringContentsUnordered = tester.stringContentsUnordered(msgStream.toStringStream(), Constants.TOPIC_TEST + ":" + KEY + ":" + MSG);
    HashMap<String, Object> config = new HashMap<>();
    // config.put (ContextProperties.TRACING_LEVEL, java.util.logging.Level.FINE);
    // config.put(ContextProperties.KEEP_ARTIFACTS,  new Boolean(true));
    tester.complete(context, config, stringContentsUnordered, 60, TimeUnit.SECONDS);
    // check the results
    Assert.assertTrue(stringContentsUnordered.valid());
    Assert.assertTrue(stringContentsUnordered.getResult().size() > 0);
}

Example 13 with StreamsContext

use of com.ibm.streamsx.topology.context.StreamsContext in project streamsx.kafka by IBMStreams.

the class KafkaConsumerPatternSubscribeTest method kafkaPatternSubscribeTest.

@Test
public void kafkaPatternSubscribeTest() throws Exception {
    Topology topo = getTopology();
    // create the producer (produces tuples after a short delay)
    TStream<String> stringSrcStream = topo.strings(Constants.STRING_DATA).modify(new Delay<>(Constants.PRODUCER_DELAY));
    SPL.invokeSink(Constants.KafkaProducerOp, KafkaSPLStreamsUtils.convertStreamToKafkaTuple(stringSrcStream), getProducerKafkaParams());
    // create the consumer
    SPLStream consumerStream = SPL.invokeSource(topo, Constants.KafkaConsumerOp, getConsumerKafkaParams(), KafkaSPLStreamsUtils.STRING_SCHEMA);
    SPLStream msgStream = SPLStreams.stringToSPLStream(consumerStream.convert(t -> t.getString("message")));
    // test the output of the consumer
    StreamsContext<?> context = StreamsContextFactory.getStreamsContext(Type.DISTRIBUTED_TESTER);
    Tester tester = topo.getTester();
    String[] expectedArr = KafkaSPLStreamsUtils.duplicateArrayEntries(Constants.STRING_DATA, 2);
    Condition<List<String>> stringContentsUnordered = tester.stringContentsUnordered(msgStream.toStringStream(), expectedArr);
    HashMap<String, Object> config = new HashMap<>();
    // config.put (ContextProperties.KEEP_ARTIFACTS, new Boolean (true));
    // config.put (ContextProperties.TRACING_LEVEL, java.util.logging.Level.FINE);
    tester.complete(context, config, stringContentsUnordered, 60, TimeUnit.SECONDS);
    // check the results
    Assert.assertTrue(stringContentsUnordered.valid());
    Assert.assertTrue(stringContentsUnordered.getResult().size() == Constants.STRING_DATA.length * 2);
}

Example 14 with StreamsContext

use of com.ibm.streamsx.topology.context.StreamsContext in project streamsx.kafka by IBMStreams.

the class KafkaOperatorsIntTypeTest method kafkaIntTypeTest.

@Test
public void kafkaIntTypeTest() throws Exception {
    Topology topo = getTopology();
    StreamSchema schema = KafkaSPLStreamsUtils.INT_SCHEMA;
    // create the producer (produces tuples after a short delay)
    TStream<Integer> srcStream = topo.strings(DATA).transform(s -> Integer.valueOf(s)).modify(new Delay<>(5000));
    SPLStream splSrcStream = SPLStreams.convertStream(srcStream, new Converter(), schema);
    SPL.invokeSink(Constants.KafkaProducerOp, splSrcStream, getKafkaParams());
    // create the consumer
    SPLStream consumerStream = SPL.invokeSource(topo, Constants.KafkaConsumerOp, getKafkaParams(), schema);
    SPLStream msgStream = SPLStreams.stringToSPLStream(consumerStream.convert(t -> String.valueOf(t.getInt("message"))));
    // test the output of the consumer
    StreamsContext<?> context = StreamsContextFactory.getStreamsContext(Type.DISTRIBUTED_TESTER);
    Tester tester = topo.getTester();
    Condition<List<String>> stringContentsUnordered = tester.stringContentsUnordered(msgStream.toStringStream(), DATA);
    HashMap<String, Object> config = new HashMap<>();
    // config.put (ContextProperties.KEEP_ARTIFACTS, new Boolean (true));
    // config.put (ContextProperties.TRACING_LEVEL, java.util.logging.Level.FINE);
    tester.complete(context, config, stringContentsUnordered, 60, TimeUnit.SECONDS);
    // check the results
    Assert.assertTrue(stringContentsUnordered.valid());
    Assert.assertTrue(stringContentsUnordered.getResult().size() == DATA.length);
}

Example 15 with StreamsContext

use of com.ibm.streamsx.topology.context.StreamsContext in project streamsx.kafka by IBMStreams.

the class KafkaOperatorsNoKey method kafkaNoKeyTest.

@Test
public void kafkaNoKeyTest() throws Exception {
    Topology topo = getTopology();
    // create the producer (produces tuples after a short delay)
    TStream<String> stringSrcStream = topo.strings(Constants.STRING_DATA).modify(new Delay<>(5000));
    SPL.invokeSink(Constants.KafkaProducerOp, KafkaSPLStreamsUtils.convertStreamToKafkaTuple(stringSrcStream, false), getKafkaParams());
    // create the consumer
    SPLStream consumerStream = SPL.invokeSource(topo, Constants.KafkaConsumerOp, getKafkaParams(), KafkaSPLStreamsUtils.STRING_NOKEY_SCHEMA);
    SPLStream msgStream = SPLStreams.stringToSPLStream(consumerStream.convert(t -> t.getString("message")));
    // test the output of the consumer
    StreamsContext<?> context = StreamsContextFactory.getStreamsContext(Type.DISTRIBUTED_TESTER);
    Tester tester = topo.getTester();
    Condition<List<String>> stringContentsUnordered = tester.stringContentsUnordered(msgStream.toStringStream(), Constants.STRING_DATA);
    HashMap<String, Object> config = new HashMap<>();
    // config.put (ContextProperties.KEEP_ARTIFACTS, new Boolean (true));
    // config.put (ContextProperties.TRACING_LEVEL, java.util.logging.Level.FINE);
    tester.complete(context, config, stringContentsUnordered, 60, TimeUnit.SECONDS);
    // check the results
    Assert.assertTrue(stringContentsUnordered.valid());
    Assert.assertTrue(stringContentsUnordered.getResult().size() == Constants.STRING_DATA.length);
}