Examples with SPLStream com.ibm.streamsx.topology.spl.SPLStream used on opensource projects

Example 6 with SPLStream

use of com.ibm.streamsx.topology.spl.SPLStream in project streamsx.topology by IBMStreams.

the class MqttStreams method subscribe.

/**
     * Subscribe to a MQTT topic and create a stream of messages.
     * <p>
     * The quality of service for handling each topic is
     * the value of configuration property {@code defaultQOS}.
     * <p> 
     * N.B., A topology that includes this will not support
     * {@code StreamsContext.Type.EMBEDDED}.
     * <p>
     * N.B. due to com.ibm.streamsx.messaging 
     * <a href="https://github.com/IBMStreams/streamsx.messaging/issues/124">issue#124</a>,
     * terminating a {@code StreamsContext.Type.STANDALONE} topology may result
     * in ERROR messages and a stranded standalone process.
     *
     * @param topic the MQTT topic.  May be a submission parameter.
     * @return TStream&lt;Message>
     *      The generated {@code Message} tuples have a non-null {@code topic}.
     *      The tuple's {@code key} will be null. 
     * @throws IllegalArgumentException if topic is null.
     * @see Value
     * @see Topology#createSubmissionParameter(String, Class)
    */
public TStream<Message> subscribe(Supplier<String> topic) {
    if (topic == null)
        throw new IllegalArgumentException("topic");
    Map<String, Object> params = new HashMap<>();
    params.put("reconnectionBound", -1);
    params.put("qos", 0);
    params.putAll(Util.configToSplParams(config));
    params.remove("retain");
    params.put("topics", topic);
    params.put("topicOutAttrName", "topic");
    params.put("dataAttributeName", "message");
    if (++opCnt > 1) {
        // each op requires its own clientID
        String clientId = (String) params.get("clientID");
        if (clientId != null && clientId.length() > 0)
            params.put("clientID", opCnt + "-" + clientId);
    }
    // Use SPL.invoke to avoid adding a compile time dependency
    // to com.ibm.streamsx.messaging since JavaPrimitive.invoke*()
    // lack "kind" based variants.
    String kind = "com.ibm.streamsx.messaging.mqtt::MQTTSource";
    String className = "com.ibm.streamsx.messaging.mqtt.MqttSourceOperator";
    SPLStream rawMqtt = SPL.invokeSource(te, kind, params, MqttSchemas.MQTT);
    SPL.tagOpAsJavaPrimitive(toOp(rawMqtt), kind, className);
    TStream<Message> rcvdMsgs = toMessageStream(rawMqtt);
    rcvdMsgs.colocate(rawMqtt);
    return rcvdMsgs;
}

Example 7 with SPLStream

use of com.ibm.streamsx.topology.spl.SPLStream in project streamsx.topology by IBMStreams.

the class ParallelTest method testParallelSubmissionParamInner.

@Test
public void testParallelSubmissionParamInner() throws Exception {
    checkUdpSupported();
    Topology topology = newTopology("testParallelSubmissionParamInner");
    final int count = new Random().nextInt(1000) + 37;
    String submissionWidthName = "width";
    Integer submissionWidth = 5;
    String submissionAppendName = "append";
    boolean submissionAppend = true;
    String submissionFlushName = "flush";
    Integer submissionFlush = 1;
    String submissionThresholdName = "threshold";
    String submissionDefaultedThresholdName = "defaultedTreshold";
    Integer submissionThreshold = -1;
    // getConfig().put(ContextProperties.KEEP_ARTIFACTS, true);
    Supplier<Integer> threshold = topology.createSubmissionParameter(submissionThresholdName, Integer.class);
    Supplier<Integer> defaultedThreshold = topology.createSubmissionParameter(submissionDefaultedThresholdName, submissionThreshold);
    TStream<BeaconTuple> fb = BeaconStreams.beacon(topology, count);
    TStream<BeaconTuple> pb = fb.parallel(topology.createSubmissionParameter(submissionWidthName, Integer.class));
    TStream<Integer> is = pb.transform(randomHashProducer());
    // submission param use within a parallel region
    StreamSchema schema = getStreamSchema("tuple<int32 i>");
    SPLStream splStream = SPLStreams.convertStream(is, cvtMsgFunc(), schema);
    File tmpFile = File.createTempFile("parallelTest", null);
    tmpFile.deleteOnExit();
    Map<String, Object> splParams = new HashMap<>();
    splParams.put("file", tmpFile.getAbsolutePath());
    splParams.put("append", topology.createSubmissionParameter(submissionAppendName, submissionAppend));
    splParams.put("flush", SPL.createSubmissionParameter(topology, submissionFlushName, SPL.createValue(0, Type.MetaType.UINT32), false));
    SPL.invokeSink("spl.adapter::FileSink", splStream, splParams);
    // use a submission parameter in "inner" functional logic
    is = is.filter(thresholdFilter(threshold));
    is = is.filter(thresholdFilter(defaultedThreshold));
    // avoid another parallel impl limitation noted in issue#173
    is = is.filter(passthru());
    TStream<Integer> joined = is.endParallel();
    TStream<String> numRegions = joined.transform(uniqueIdentifierMap(count));
    Tester tester = topology.getTester();
    Condition<Long> expectedCount = tester.tupleCount(numRegions, 1);
    Condition<List<String>> regionCount = tester.stringContents(numRegions, submissionWidth.toString());
    Map<String, Object> params = new HashMap<>();
    params.put(submissionWidthName, submissionWidth);
    params.put(submissionFlushName, submissionFlush);
    params.put(submissionThresholdName, submissionThreshold);
    getConfig().put(ContextProperties.SUBMISSION_PARAMS, params);
    complete(tester, allConditions(expectedCount, regionCount), 10, TimeUnit.SECONDS);
    assertTrue(expectedCount.valid());
    assertTrue(regionCount.valid());
}

Example 8 with SPLStream

use of com.ibm.streamsx.topology.spl.SPLStream in project streamsx.topology by IBMStreams.

the class JobConfigSubmissionTest method testItDirect.

private List<String> testItDirect(String topologyName, JobConfig config) throws Exception {
    // JobConfig only apply to DISTRIBUTED submit
    assumeTrue(getTesterType() == StreamsContext.Type.DISTRIBUTED_TESTER);
    assumeTrue(SC_OK);
    config.addToConfig(getConfig());
    Topology topology = newTopology(topologyName);
    topology.addClassDependency(JobPropertiesTestOp.class);
    SPLStream sourceSPL = JavaPrimitive.invokeJavaPrimitiveSource(topology, JobPropertiesTestOp.class, Schemas.STRING, null);
    TStream<String> source = sourceSPL.toStringStream();
    Condition<Long> end = topology.getTester().tupleCount(source, 4);
    Condition<List<String>> result = topology.getTester().stringContents(source);
    complete(topology.getTester(), end, 10, TimeUnit.SECONDS);
    return result.getResult();
}

Example 9 with SPLStream

use of com.ibm.streamsx.topology.spl.SPLStream in project streamsx.topology by IBMStreams.

the class JSONStreamsTest method checkJsonOutput.

private JSONArtifact checkJsonOutput(JSONArtifact expected, TStream<String> jsonString) throws Exception, IOException {
    assertEquals(String.class, jsonString.getTupleClass());
    assertEquals(String.class, jsonString.getTupleType());
    SPLStream splS = SPLStreams.stringToSPLStream(jsonString);
    MostRecent<Tuple> mr = jsonString.topology().getTester().splHandler(splS, new MostRecent<Tuple>());
    Condition<Long> singleTuple = jsonString.topology().getTester().tupleCount(splS, 1);
    complete(jsonString.topology().getTester(), singleTuple, 10, TimeUnit.SECONDS);
    JSONArtifact rv = JSON.parse(mr.getMostRecentTuple().getString(0));
    assertEquals(expected, rv);
    return rv;
}

Example 10 with SPLStream

use of com.ibm.streamsx.topology.spl.SPLStream in project streamsx.topology by IBMStreams.

the class SubscribeSPLDictTest method testSubscribeSPLDictMap.

@Test
public void testSubscribeSPLDictMap() throws Exception {
    Topology topology = new Topology("testSubscribeMap");
    // Publish a stream with all the SPL types supported by Python including sets
    SPLStream tuples = PythonFunctionalOperatorsTest.testTupleStream(topology, true);
    tuples = addStartupDelay(tuples);
    tuples.publish("pytest/spl/map");
    SPLStream viaSPL = SPL.invokeOperator("spl.relational::Functor", tuples, tuples.getSchema(), null);
    // Python that subscribes to the SPL tuple stream and then republishes as Json.
    includePythonApp(topology, "spl_map_json.py", "spl_map_json::spl_map_json");
    TStream<JSONObject> viaPythonJson = topology.subscribe("pytest/spl/map/result", JSONObject.class);
    SPLStream viaPythonJsonSpl = JSONStreams.toSPL(viaPythonJson.isolate());
    Tester tester = topology.getTester();
    Condition<Long> expectedCount = tester.tupleCount(viaPythonJsonSpl, TUPLE_COUNT);
    Condition<Long> expectedCountSpl = tester.tupleCount(viaSPL, TUPLE_COUNT);
    Condition<List<Tuple>> viaSPLResult = tester.tupleContents(viaSPL);
    Condition<List<Tuple>> viaPythonResult = tester.tupleContents(viaPythonJsonSpl);
    complete(tester, allConditions(expectedCount, expectedCountSpl), 60, TimeUnit.SECONDS);
    System.out.println(expectedCount.getResult());
    System.out.println(expectedCountSpl.getResult());
    assertTrue(expectedCount.valid());
    assertTrue(expectedCountSpl.valid());
    List<Tuple> splResults = viaSPLResult.getResult();
    List<Tuple> pyJsonResults = viaPythonResult.getResult();
    assertEquals(TUPLE_COUNT, splResults.size());
    assertEquals(TUPLE_COUNT, pyJsonResults.size());
    for (int i = 0; i < TUPLE_COUNT; i++) {
        Tuple spl = splResults.get(i);
        JSONObject json = (JSONObject) JSON.parse(pyJsonResults.get(i).getString("jsonString"));
        System.out.println(spl);
        System.out.println(pyJsonResults.get(i).getString("jsonString"));
        assertEquals(spl.getBoolean("b"), ((Boolean) json.get("b")).booleanValue());
        assertEquals(spl.getInt("i8"), ((Number) json.get("i8")).intValue());
        assertEquals(spl.getInt("i16"), ((Number) json.get("i16")).intValue());
        assertEquals(spl.getInt("i32"), ((Number) json.get("i32")).intValue());
        assertEquals(spl.getLong("i64"), ((Number) json.get("i64")).longValue());
        assertEquals(spl.getString("r"), json.get("r").toString());
        assertEquals(spl.getDouble("f32"), ((Number) json.get("f32")).doubleValue(), 0.1);
        assertEquals(spl.getDouble("f64"), ((Number) json.get("f64")).doubleValue(), 0.1);
        {
            List<?> ex = spl.getList("li32");
            JSONArray pya = (JSONArray) json.get("li32");
            assertEquals(ex.size(), pya.size());
            for (int j = 0; j < ex.size(); j++) {
                assertEquals(ex.get(j), ((Number) pya.get(j)).intValue());
            }
        }
        {
            Set<?> ex = spl.getSet("si32");
            JSONArray pya = (JSONArray) json.get("si32");
            assertEquals(ex.size(), pya.size());
            setAssertHelper(ex, pya);
        }
    }
}