Examples with SPLStream com.ibm.streamsx.topology.spl.SPLStream used on opensource projects

Example 41 with SPLStream

use of com.ibm.streamsx.topology.spl.SPLStream in project streamsx.kafka by IBMStreams.

the class KafkaOperatorsStartPositionTest method kafkaStartPositionTest.

@Test
public void kafkaStartPositionTest() throws Exception {
    Topology producerTopo = createTopology("producerTopo");
    // create the producer (produces tuples after a short delay)
    Map<String, Object> producerProps = new HashMap<>();
    producerProps.put("topic", Constants.TOPIC_POS);
    producerProps.put("propertiesFile", Constants.PROPERTIES_FILE_PATH);
    TStream<String> stringSrcStream = producerTopo.strings(Constants.STRING_DATA);
    SPL.invokeSink(Constants.KafkaProducerOp, KafkaSPLStreamsUtils.convertStreamToKafkaTuple(stringSrcStream), producerProps);
    // Launch the producer and allow it to finish writing
    // the data to the queue. Consumer should run AFTER
    // producer is finished.
    @SuppressWarnings("unchecked") Future<BigInteger> future = (Future<BigInteger>) StreamsContextFactory.getStreamsContext(Type.STANDALONE).submit(producerTopo);
    future.get();
    Thread.sleep(TimeUnit.SECONDS.toMillis(20));
    if (!future.isDone()) {
        future.cancel(true);
    }
    // create the consumer
    Topology topo = getTopology();
    Map<String, Object> consumerParams = new HashMap<>();
    consumerParams.put("topic", Constants.TOPIC_POS);
    consumerParams.put("propertiesFile", Constants.PROPERTIES_FILE_PATH);
    consumerParams.put("startPosition", StartPosition.Beginning);
    SPLStream consumerStream = SPL.invokeSource(topo, Constants.KafkaConsumerOp, consumerParams, KafkaSPLStreamsUtils.STRING_SCHEMA);
    SPLStream msgStream = SPLStreams.stringToSPLStream(consumerStream.convert(t -> t.getString("message")));
    // test the output of the consumer
    StreamsContext<?> context = StreamsContextFactory.getStreamsContext(Type.DISTRIBUTED_TESTER);
    Tester tester = topo.getTester();
    Condition<List<String>> condition = KafkaSPLStreamsUtils.stringContentsUnordered(tester, msgStream, Constants.STRING_DATA);
    tester.complete(context, new HashMap<>(), condition, 30, TimeUnit.SECONDS);
    // check the results
    Assert.assertTrue(condition.getResult().size() > 0);
    Assert.assertTrue(condition.getResult().toString(), condition.valid());
}

Example 42 with SPLStream

use of com.ibm.streamsx.topology.spl.SPLStream in project streamsx.kafka by IBMStreams.

the class KafkaOperatorsTopicPartitionTest method kafkaTopicPartitionTest.

@Test
public void kafkaTopicPartitionTest() throws Exception {
    Topology topo = getTopology();
    topo.addFileDependency("etc/custom_partitioner.properties", "etc");
    topo.addFileDependency("etc/custompartitioner.jar", "etc");
    // create producer
    TStream<Message<Integer, String>> src = topo.limitedSource(new MySupplier(), 9).modify(new Delay<>(Constants.PRODUCER_DELAY));
    SPLStream outStream = SPLStreams.convertStream(src, new MessageConverter(), SCHEMA);
    SPL.invokeSink(Constants.KafkaProducerOp, outStream, getKafkaProducerParams());
    // create the consumers
    SPLStream msgStream1 = createConsumer(topo, 0);
    SPLStream msgStream2 = createConsumer(topo, 1);
    SPLStream msgStream3 = createConsumer(topo, 2);
    Set<TStream<Tuple>> s = new HashSet<>();
    s.add(msgStream2);
    s.add(msgStream3);
    TStream<String> unionStream = msgStream1.union(s).transform(t -> t.getString("message"));
    SPLStream msgStream = SPLStreams.stringToSPLStream(unionStream);
    StreamsContext<?> context = StreamsContextFactory.getStreamsContext(Type.DISTRIBUTED_TESTER);
    Tester tester = topo.getTester();
    String[] expectedArr = { "A0", "B1", "C2", "A3", "B4", "C5", "A6", "B7", "C8" };
    Condition<List<String>> condition = KafkaSPLStreamsUtils.stringContentsUnordered(tester, msgStream, expectedArr);
    tester.complete(context, new HashMap<>(), condition, 30, TimeUnit.SECONDS);
    // check the results
    Assert.assertTrue(condition.getResult().size() > 0);
    Assert.assertTrue(condition.getResult().toString(), condition.valid());
}

Example 43 with SPLStream

use of com.ibm.streamsx.topology.spl.SPLStream in project streamsx.kafka by IBMStreams.

the class KafkaSPLStreamsUtils method union.

public static SPLStream union(List<SPLStream> streams, StreamSchema schema) {
    if (streams.size() == 0)
        throw new IllegalArgumentException("At least one stream must be provided.");
    if (streams.size() == 1) {
        return streams.get(0);
    } else {
        SPLStream stream1 = streams.get(0);
        Set<TStream<Tuple>> streamSet = new HashSet<TStream<Tuple>>(streams);
        streamSet.remove(stream1);
        return SPLStreams.convertStream(stream1.union(streamSet), getTupleStreamConvert(), schema);
    }
}

Example 44 with SPLStream

use of com.ibm.streamsx.topology.spl.SPLStream in project streamsx.topology by IBMStreams.

the class HTTPStreams method getJSON.

/**
     * Periodically poll a web service using HTTP {@code GET} for
     * {@code application/json} data. Declares a source stream that will contain
     * a single tuple for each successful {@code GET}. The tuple is the complete
     * JSON ({@code application/json} content) returned by the request.
     * 
     * @param te
     *            Topology the source stream will be contained in.
     * @param url
     *            URL to poll.
     * @param period
     *            Poling period.
     * @param unit
     *            Unit for {@code period}.
     * @return Stream that will contain the JSON tuples from periodic HTTP
     *         {@code GET} requests.
     */
public static TStream<JSONObject> getJSON(TopologyElement te, String url, long period, TimeUnit unit) {
    Map<String, Object> params = new HashMap<>();
    params.put("url", url);
    double dperiod = (unit.toMillis(period) / 1000.0);
    params.put("period", dperiod);
    SPLStream rawJson = SPL.invokeSource(te, "com.ibm.streamsx.inet.http::HTTPGetJSONContent", params, JSONSchemas.JSON);
    TStream<String> string = SPLStreams.toStringStream(rawJson);
    return JSONStreams.deserialize(string);
}

Example 45 with SPLStream

use of com.ibm.streamsx.topology.spl.SPLStream in project streamsx.health by IBMStreams.

the class JsonPublisher method publish.

public static void publish(TStream<String> jsonInputStream, String topic) {
    SPLStream splStream = SPLStreams.convertStream(jsonInputStream, new JsonToSpl(), JSONSchemas.JSON);
    splStream.publish(topic);
}