Examples with SPLStream com.ibm.streamsx.topology.spl.SPLStream used on opensource projects

Example 71 with SPLStream

use of com.ibm.streamsx.topology.spl.SPLStream in project streamsx.topology by IBMStreams.

the class PythonFunctionalOperatorsTest method _testSchemaBuild.

/**
 * Just create a Beacon feeding a Python Noop to allow
 * checking of input and output schemas.
 */
private void _testSchemaBuild(StreamSchema input, StreamSchema output) throws Exception {
    Topology topology = new Topology("testNonHashableSet");
    SPLStream s = SPL.invokeSource(topology, "spl.utility::Beacon", Collections.emptyMap(), input);
    addTestToolkit(topology);
    SPL.invokeOperator("com.ibm.streamsx.topology.pysamples.positional::Noop", s, output, null);
    File bundle = (File) StreamsContextFactory.getStreamsContext(StreamsContext.Type.BUNDLE).submit(topology).get();
    bundle.delete();
}

Example 72 with SPLStream

use of com.ibm.streamsx.topology.spl.SPLStream in project streamsx.topology by IBMStreams.

the class PythonFunctionalOperatorsTest method testReturnDict.

/**
 * Test that specific values in Python
 * make their way into SPL correctly
 * when returning as a dictionary.
 * @throws Exception
 */
@Test
public void testReturnDict() throws Exception {
    Topology topology = new Topology("testReturnDict");
    // getConfig().put(ContextProperties.KEEP_ARTIFACTS, true);
    addTestToolkit(topology);
    StreamSchema schema = Type.Factory.getStreamSchema("tuple<int32 a, int32 b, int32 c, int32 d, int32 e>");
    SPLStream pyds = SPL.invokeSource(topology, "com.ibm.streamsx.topology.pytest.pysource::DictTuple", null, schema);
    SPLStream pydm = SPL.invokeOperator("com.ibm.streamsx.topology.pytest.pymap::DictTupleMap", pyds, schema, null);
    Tester tester = topology.getTester();
    Condition<?> expectedCount = tester.tupleCount(pyds, 4).and(tester.tupleCount(pydm, 4));
    Condition<List<Tuple>> outTuples = tester.tupleContents(pyds);
    Condition<List<Tuple>> outTuplesMap = tester.tupleContents(pydm);
    complete(tester, expectedCount, 20, TimeUnit.SECONDS);
    assertTrue(expectedCount.valid());
    // Dict tuple handling - source
    Tuple r1 = outTuples.getResult().get(0);
    assertEquals(3245, r1.getInt("a"));
    assertEquals(0, r1.getInt("b"));
    assertEquals(93, r1.getInt("c"));
    assertEquals(0, r1.getInt("d"));
    assertEquals(0, r1.getInt("e"));
    Tuple r2 = outTuples.getResult().get(1);
    assertEquals(831, r2.getInt("a"));
    assertEquals(421, r2.getInt("b"));
    assertEquals(0, r2.getInt("c"));
    assertEquals(-4455, r2.getInt("d"));
    assertEquals(0, r2.getInt("e"));
    Tuple r3 = outTuples.getResult().get(2);
    assertEquals(1, r3.getInt("a"));
    assertEquals(2, r3.getInt("b"));
    assertEquals(3, r3.getInt("c"));
    assertEquals(4, r3.getInt("d"));
    assertEquals(5, r3.getInt("e"));
    Tuple r4 = outTuples.getResult().get(3);
    assertEquals(0, r4.getInt("a"));
    assertEquals(-32, r4.getInt("b"));
    assertEquals(0, r4.getInt("c"));
    assertEquals(0, r4.getInt("d"));
    assertEquals(-64, r4.getInt("e"));
    // Now the map
    Tuple m1 = outTuplesMap.getResult().get(0);
    assertEquals(3245, m1.getInt("a"));
    assertEquals(120, m1.getInt("b"));
    assertEquals(93, m1.getInt("c"));
    assertEquals(0, m1.getInt("d"));
    assertEquals(0, m1.getInt("e"));
    Tuple m2 = outTuplesMap.getResult().get(1);
    assertEquals(1, m2.getInt("a"));
    assertEquals(2, m2.getInt("b"));
    assertEquals(3, m2.getInt("c"));
    assertEquals(4, m2.getInt("d"));
    assertEquals(5, m2.getInt("e"));
    Tuple m3 = outTuplesMap.getResult().get(2);
    assertEquals(1, m3.getInt("a"));
    assertEquals(2, m3.getInt("b"));
    assertEquals(23, m3.getInt("c"));
    assertEquals(24, m3.getInt("d"));
    assertEquals(25, m3.getInt("e"));
    Tuple m4 = outTuplesMap.getResult().get(3);
    assertEquals(0, m4.getInt("a"));
    assertEquals(-39, m4.getInt("b"));
    assertEquals(0, m4.getInt("c"));
    assertEquals(0, m4.getInt("d"));
    assertEquals(-64, m4.getInt("e"));
}

Example 73 with SPLStream

use of com.ibm.streamsx.topology.spl.SPLStream in project streamsx.topology by IBMStreams.

the class PythonFunctionalOperatorsTest method testStatefulOperator.

@Test
public void testStatefulOperator() throws Exception {
    Topology topology = new Topology("testPositionalSampleSimpleFilterUsingSPLType");
    SPLStream tuples = testTupleStream(topology, false);
    addTestToolkit(tuples);
    StreamSchema outSchema = tuples.getSchema().extend("int32", "sequence_using_py");
    SPLStream viaPython = SPL.invokeOperator("com.ibm.streamsx.topology.pysamples.positional::AddSeq", tuples, outSchema, null);
    // Add a second count to make sure that the states are independent.
    SPLStream filtered = tuples.filter(t -> t.getInt("i32") < 10000);
    SPLStream viaPythonFiltered = SPL.invokeOperator("com.ibm.streamsx.topology.pysamples.positional::AddSeq", filtered, outSchema, null);
    Tester tester = topology.getTester();
    Condition<Long> expectedCount = tester.tupleCount(viaPython, TUPLE_COUNT);
    Condition<List<Tuple>> outTuples = tester.tupleContents(viaPython);
    Condition<List<Tuple>> outFilteredTuples = tester.tupleContents(viaPythonFiltered);
    complete(tester, expectedCount, 60, TimeUnit.SECONDS);
    assertTrue(expectedCount.getResult().toString(), expectedCount.valid());
    List<Tuple> result = outTuples.getResult();
    assertEquals(TUPLE_COUNT, result.size());
    for (int i = 0; i < TUPLE_COUNT; i++) assertEquals(i, result.get(i).getInt("sequence_using_py"));
    List<Tuple> filteredResult = outFilteredTuples.getResult();
    assertTrue(filteredResult.size() <= TUPLE_COUNT);
    for (int i = 0; i < filteredResult.size(); i++) assertEquals(i, filteredResult.get(i).getInt("sequence_using_py"));
}

Example 74 with SPLStream

use of com.ibm.streamsx.topology.spl.SPLStream in project streamsx.topology by IBMStreams.

the class PythonFunctionalOperatorsTest method testPositionalSampleSimpleFilterUsingSPLType.

@Test
public void testPositionalSampleSimpleFilterUsingSPLType() throws Exception {
    Topology topology = new Topology("testPositionalSampleSimpleFilterUsingSPLType");
    SPLStream tuples = sampleFilterStream(topology);
    addTestToolkit(tuples);
    SPLStream viaPython = SPL.invokeOperator("testspl::SF", tuples, tuples.getSchema(), null);
    Tester tester = topology.getTester();
    Condition<Long> expectedCount = tester.tupleCount(viaPython, 2);
    // first attribute is the sum of the first and second input attributes
    // others are copied across from in to out.
    Tuple r1 = TEST_SCHEMA_SF.getTuple(new Object[] { 32, (short) 25, 34535L });
    Tuple r2 = TEST_SCHEMA_SF.getTuple(new Object[] { 5, (short) 3, 654932L });
    Condition<List<Tuple>> viaPythonResult = tester.tupleContents(viaPython, r1, r2);
    complete(tester, expectedCount, 10, TimeUnit.SECONDS);
    assertTrue(expectedCount.valid());
    assertTrue(viaPythonResult.toString(), viaPythonResult.valid());
}

Example 75 with SPLStream

use of com.ibm.streamsx.topology.spl.SPLStream in project streamsx.topology by IBMStreams.

the class PythonFunctionalOperatorsTest method testValuesForOptionalTypes.

/**
 * Test that specific values in Python for optional types
 * make their way into SPL correctly
 * when returning as a tuple.
 * @throws Exception
 */
@Test
public void testValuesForOptionalTypes() throws Exception {
    assumeOptionalTypes();
    Topology topology = new Topology("testValuesForOptionalTypes");
    addTestToolkit(topology);
    SPLStream pysrc = SPL.invokeSource(topology, "com.ibm.streamsx.topology.pytest.pysource.opttype::SpecificValues", null, Type.Factory.getStreamSchema(PYTHON_OPTIONAL_TYPES_SCHEMA_STRING));
    StreamSchema sparseSchema = Type.Factory.getStreamSchema("tuple<optional<int32> a, optional<int32> b, optional<int32> c, optional<int32> d, optional<int32> e, optional<int32> f, int32 g, int32 h, optional<int32> i>");
    StreamSchema sparseSchemaMap = Type.Factory.getStreamSchema("tuple<optional<int32> a, optional<int32> b, optional<int32> c, optional<int32> d, optional<int32> e, int32 f, int32 g, optional<int32> h, optional<int32> i>");
    Tester tester = topology.getTester();
    Condition<Long> expectedCount = tester.tupleCount(pysrc, 1);
    Condition<List<Tuple>> outTuples = tester.tupleContents(pysrc);
    SPLStream pysparse = SPL.invokeSource(topology, "com.ibm.streamsx.topology.pytest.pysource.opttype::SparseTuple", null, sparseSchema);
    SPLStream pysparsemap = SPL.invokeOperator("com.ibm.streamsx.topology.pytest.pymap.opttype::SparseTupleMap", pysparse, sparseSchemaMap.extend("optional<int32>", "j"), null);
    Condition<Long> expectedCountSparse = tester.tupleCount(pysparse, 1);
    Condition<List<Tuple>> sparseTupleOut = tester.tupleContents(pysparse);
    Condition<Long> expectedCountSparseMap = tester.tupleCount(pysparsemap, 1);
    Condition<List<Tuple>> sparseTupleMapOut = tester.tupleContents(pysparsemap);
    // getConfig().put(ContextProperties.TRACING_LEVEL, TraceLevel.DEBUG);
    complete(tester, expectedCount.and(expectedCountSparse, expectedCountSparseMap), 20, TimeUnit.SECONDS);
    assertTrue(expectedCount.valid());
    assertTrue(expectedCountSparse.valid());
    assertTrue(expectedCountSparseMap.valid());
    Tuple r1 = outTuples.getResult().get(0);
    // optional signed integers
    // 123, null
    assertEquals(r1.getObject("oi32v"), 123);
    assertNull(r1.getObject("oi32nv"));
    // ["a", "b"], null
    {
        @SuppressWarnings("unchecked") List<RString> lr = (List<RString>) r1.getObject("olrv");
        assertEquals(2, lr.size());
        assertEquals("a", lr.get(0).getString());
        assertEquals("b", lr.get(1).getString());
        assertNull(r1.getObject("olrnv"));
    }
    // Sparse tuple handling - source
    assertEquals(1, sparseTupleOut.getResult().size());
    Tuple st = sparseTupleOut.getResult().get(0);
    // set by op
    assertEquals(37, st.getObject("a"));
    // default as None in tuple
    assertEquals(null, st.getObject("b"));
    // set by op
    assertEquals(23, st.getObject("c"));
    // set by op
    assertEquals(-46, st.getObject("d"));
    // default as None in tuple
    assertEquals(null, st.getObject("e"));
    // set by op
    assertEquals(56, st.getObject("f"));
    // set by op
    assertEquals(67, st.getObject("g"));
    // set by op
    assertEquals(78, st.getObject("h"));
    // default as no value (short tuple)
    assertEquals(null, st.getObject("i"));
    // Sparse tuple handling - map
    assertEquals(1, sparseTupleMapOut.getResult().size());
    Tuple stm = sparseTupleMapOut.getResult().get(0);
    // set by op
    assertEquals(37 + 81, stm.getObject("a"));
    // match input as None in tuple
    assertEquals(null, stm.getObject("b"));
    // match input as None in tuple
    assertEquals(23, stm.getObject("c"));
    // default to matching input
    assertEquals(-46, stm.getObject("d"));
    // default to matching input
    assertEquals(null, stm.getObject("e"));
    // no match: opt to non-opt
    assertEquals(0, stm.getObject("f"));
    // match non-opt to non-opt
    assertEquals(67, stm.getObject("g"));
    // match non-opt to opt
    assertEquals(78, stm.getObject("h"));
    // default to matching input
    assertEquals(null, stm.getObject("i"));
    // default as no value (short tuple)
    assertEquals(null, stm.getObject("j"));
}