Examples with StreamSchema com.ibm.streams.operator.StreamSchema used on opensource projects

Example 41 with StreamSchema

use of com.ibm.streams.operator.StreamSchema in project streamsx.topology by IBMStreams.

the class SPL method invokeOperator.

/**
 * Invoke an SPL operator with an arbitrary number
 * of input and output ports.
 * <BR>
 * Each input stream or window in {@code inputs} results in
 * a input port for the operator with the input port index
 * matching the position of the input in {@code inputs}.
 * If {@code inputs} is {@code null} or empty then the operator will not
 * have any input ports.
 * <BR>
 * Each SPL schema in {@code outputSchemas} an output port
 * for the operator with the output port index
 * matching the position of the schema in {@code outputSchemas}.
 * If {@code outputSchemas} is {@code null} or empty then the operator will not
 * have any output ports.
 *
 * @param te Reference to Topology the operator will be in.
 * @param name Name for the operator invocation.
 * @param kind
 *            SPL kind of the operator to be invoked.
 * @param inputs Input streams to be connected to the operator. May be {@code null} if no input  streams are required.
 * @param outputSchemas Schemas of the output streams. May be {@code null} if no output streams are required.
 * @param params
 *            Parameters for the SPL Java Primitive operator, ignored if {@code null}.
 * @return List of {@code SPLStream} instances that represent the outputs of the operator.
 */
public static List<SPLStream> invokeOperator(TopologyElement te, String name, String kind, List<? extends SPLInput> inputs, List<StreamSchema> outputSchemas, Map<String, ? extends Object> params) {
    BOperatorInvocation op = te.builder().addSPLOperator(name, kind, fixParameters(params));
    SourceInfo.setSourceInfo(op, SPL.class);
    if (inputs != null && !inputs.isEmpty()) {
        for (SPLInput input : inputs) SPL.connectInputToOperator(input, op);
    }
    if (outputSchemas == null || outputSchemas.isEmpty())
        return Collections.emptyList();
    List<SPLStream> streams = new ArrayList<>(outputSchemas.size());
    for (StreamSchema outputSchema : outputSchemas) streams.add(newSPLStream(te, op, outputSchema, outputSchemas.size() == 1));
    return streams;
}

Example 42 with StreamSchema

use of com.ibm.streams.operator.StreamSchema in project streamsx.topology by IBMStreams.

the class JavaPrimitive method invokeJavaPrimitive.

/**
 * Invoke an SPL Java primitive operator with an arbitrary number
 * of input and output ports.
 * <BR>
 * Each input stream or window in {@code inputs} results in
 * a input port for the operator with the input port index
 * matching the position of the input in {@code inputs}.
 * If {@code inputs} is {@code null} or empty then the operator will not
 * have any input ports.
 * <BR>
 * Each SPL schema in {@code outputSchemas} an output port
 * for the operator with the output port index
 * matching the position of the schema in {@code outputSchemas}.
 * If {@code outputSchemas} is {@code null} or empty then the operator will not
 * have any output ports.
 *
 * @param te Reference to Topology the operator will be in.
 * @param opClass Class of the operator to be invoked.
 * @param inputs Input streams to be connected to the operator. May be {@code null} if no input  streams are required.
 * @param outputSchemas Schemas of the output streams. May be {@code null} if no output streams are required.
 * @param params
 *            Parameters for the SPL Java Primitive operator, ignored if {@code null}.
 * @return List of {@code SPLStream} instances that represent the outputs of the operator.
 */
public static List<SPLStream> invokeJavaPrimitive(TopologyElement te, Class<? extends Operator> opClass, List<? extends SPLInput> inputs, List<StreamSchema> outputSchemas, Map<String, ? extends Object> params) {
    BOperatorInvocation op = te.builder().addOperator(getInvocationName(opClass), getKind(opClass), fixParameters(params));
    op.setModel(MODEL_SPL, LANGUAGE_JAVA);
    op._json().addProperty(KIND_CLASS, opClass.getCanonicalName());
    SourceInfo.setSourceInfo(op, JavaPrimitive.class);
    if (inputs != null && !inputs.isEmpty()) {
        for (SPLInput input : inputs) SPL.connectInputToOperator(input, op);
    }
    if (outputSchemas == null || outputSchemas.isEmpty())
        return Collections.emptyList();
    List<SPLStream> streams = new ArrayList<>(outputSchemas.size());
    for (StreamSchema outputSchema : outputSchemas) streams.add(newSPLStream(te, op, outputSchema, outputSchemas.size() == 1));
    return streams;
}

Example 43 with StreamSchema

use of com.ibm.streams.operator.StreamSchema in project streamsx.topology by IBMStreams.

the class PythonFunctionalOperatorsTest method testReturnDict.

/**
 * Test that specific values in Python
 * make their way into SPL correctly
 * when returning as a dictionary.
 * @throws Exception
 */
@Test
public void testReturnDict() throws Exception {
    Topology topology = new Topology("testReturnDict");
    // getConfig().put(ContextProperties.KEEP_ARTIFACTS, true);
    addTestToolkit(topology);
    StreamSchema schema = Type.Factory.getStreamSchema("tuple<int32 a, int32 b, int32 c, int32 d, int32 e>");
    SPLStream pyds = SPL.invokeSource(topology, "com.ibm.streamsx.topology.pytest.pysource::DictTuple", null, schema);
    SPLStream pydm = SPL.invokeOperator("com.ibm.streamsx.topology.pytest.pymap::DictTupleMap", pyds, schema, null);
    Tester tester = topology.getTester();
    Condition<?> expectedCount = tester.tupleCount(pyds, 4).and(tester.tupleCount(pydm, 4));
    Condition<List<Tuple>> outTuples = tester.tupleContents(pyds);
    Condition<List<Tuple>> outTuplesMap = tester.tupleContents(pydm);
    complete(tester, expectedCount, 20, TimeUnit.SECONDS);
    assertTrue(expectedCount.valid());
    // Dict tuple handling - source
    Tuple r1 = outTuples.getResult().get(0);
    assertEquals(3245, r1.getInt("a"));
    assertEquals(0, r1.getInt("b"));
    assertEquals(93, r1.getInt("c"));
    assertEquals(0, r1.getInt("d"));
    assertEquals(0, r1.getInt("e"));
    Tuple r2 = outTuples.getResult().get(1);
    assertEquals(831, r2.getInt("a"));
    assertEquals(421, r2.getInt("b"));
    assertEquals(0, r2.getInt("c"));
    assertEquals(-4455, r2.getInt("d"));
    assertEquals(0, r2.getInt("e"));
    Tuple r3 = outTuples.getResult().get(2);
    assertEquals(1, r3.getInt("a"));
    assertEquals(2, r3.getInt("b"));
    assertEquals(3, r3.getInt("c"));
    assertEquals(4, r3.getInt("d"));
    assertEquals(5, r3.getInt("e"));
    Tuple r4 = outTuples.getResult().get(3);
    assertEquals(0, r4.getInt("a"));
    assertEquals(-32, r4.getInt("b"));
    assertEquals(0, r4.getInt("c"));
    assertEquals(0, r4.getInt("d"));
    assertEquals(-64, r4.getInt("e"));
    // Now the map
    Tuple m1 = outTuplesMap.getResult().get(0);
    assertEquals(3245, m1.getInt("a"));
    assertEquals(120, m1.getInt("b"));
    assertEquals(93, m1.getInt("c"));
    assertEquals(0, m1.getInt("d"));
    assertEquals(0, m1.getInt("e"));
    Tuple m2 = outTuplesMap.getResult().get(1);
    assertEquals(1, m2.getInt("a"));
    assertEquals(2, m2.getInt("b"));
    assertEquals(3, m2.getInt("c"));
    assertEquals(4, m2.getInt("d"));
    assertEquals(5, m2.getInt("e"));
    Tuple m3 = outTuplesMap.getResult().get(2);
    assertEquals(1, m3.getInt("a"));
    assertEquals(2, m3.getInt("b"));
    assertEquals(23, m3.getInt("c"));
    assertEquals(24, m3.getInt("d"));
    assertEquals(25, m3.getInt("e"));
    Tuple m4 = outTuplesMap.getResult().get(3);
    assertEquals(0, m4.getInt("a"));
    assertEquals(-39, m4.getInt("b"));
    assertEquals(0, m4.getInt("c"));
    assertEquals(0, m4.getInt("d"));
    assertEquals(-64, m4.getInt("e"));
}

Example 44 with StreamSchema

use of com.ibm.streams.operator.StreamSchema in project streamsx.topology by IBMStreams.

the class PythonFunctionalOperatorsTest method testStatefulOperator.

@Test
public void testStatefulOperator() throws Exception {
    Topology topology = new Topology("testPositionalSampleSimpleFilterUsingSPLType");
    SPLStream tuples = testTupleStream(topology, false);
    addTestToolkit(tuples);
    StreamSchema outSchema = tuples.getSchema().extend("int32", "sequence_using_py");
    SPLStream viaPython = SPL.invokeOperator("com.ibm.streamsx.topology.pysamples.positional::AddSeq", tuples, outSchema, null);
    // Add a second count to make sure that the states are independent.
    SPLStream filtered = tuples.filter(t -> t.getInt("i32") < 10000);
    SPLStream viaPythonFiltered = SPL.invokeOperator("com.ibm.streamsx.topology.pysamples.positional::AddSeq", filtered, outSchema, null);
    Tester tester = topology.getTester();
    Condition<Long> expectedCount = tester.tupleCount(viaPython, TUPLE_COUNT);
    Condition<List<Tuple>> outTuples = tester.tupleContents(viaPython);
    Condition<List<Tuple>> outFilteredTuples = tester.tupleContents(viaPythonFiltered);
    complete(tester, expectedCount, 60, TimeUnit.SECONDS);
    assertTrue(expectedCount.getResult().toString(), expectedCount.valid());
    List<Tuple> result = outTuples.getResult();
    assertEquals(TUPLE_COUNT, result.size());
    for (int i = 0; i < TUPLE_COUNT; i++) assertEquals(i, result.get(i).getInt("sequence_using_py"));
    List<Tuple> filteredResult = outFilteredTuples.getResult();
    assertTrue(filteredResult.size() <= TUPLE_COUNT);
    for (int i = 0; i < filteredResult.size(); i++) assertEquals(i, filteredResult.get(i).getInt("sequence_using_py"));
}

Example 45 with StreamSchema

use of com.ibm.streams.operator.StreamSchema in project streamsx.topology by IBMStreams.

the class PythonFunctionalOperatorsTest method testValuesForOptionalTypes.

/**
 * Test that specific values in Python for optional types
 * make their way into SPL correctly
 * when returning as a tuple.
 * @throws Exception
 */
@Test
public void testValuesForOptionalTypes() throws Exception {
    assumeOptionalTypes();
    Topology topology = new Topology("testValuesForOptionalTypes");
    addTestToolkit(topology);
    SPLStream pysrc = SPL.invokeSource(topology, "com.ibm.streamsx.topology.pytest.pysource.opttype::SpecificValues", null, Type.Factory.getStreamSchema(PYTHON_OPTIONAL_TYPES_SCHEMA_STRING));
    StreamSchema sparseSchema = Type.Factory.getStreamSchema("tuple<optional<int32> a, optional<int32> b, optional<int32> c, optional<int32> d, optional<int32> e, optional<int32> f, int32 g, int32 h, optional<int32> i>");
    StreamSchema sparseSchemaMap = Type.Factory.getStreamSchema("tuple<optional<int32> a, optional<int32> b, optional<int32> c, optional<int32> d, optional<int32> e, int32 f, int32 g, optional<int32> h, optional<int32> i>");
    Tester tester = topology.getTester();
    Condition<Long> expectedCount = tester.tupleCount(pysrc, 1);
    Condition<List<Tuple>> outTuples = tester.tupleContents(pysrc);
    SPLStream pysparse = SPL.invokeSource(topology, "com.ibm.streamsx.topology.pytest.pysource.opttype::SparseTuple", null, sparseSchema);
    SPLStream pysparsemap = SPL.invokeOperator("com.ibm.streamsx.topology.pytest.pymap.opttype::SparseTupleMap", pysparse, sparseSchemaMap.extend("optional<int32>", "j"), null);
    Condition<Long> expectedCountSparse = tester.tupleCount(pysparse, 1);
    Condition<List<Tuple>> sparseTupleOut = tester.tupleContents(pysparse);
    Condition<Long> expectedCountSparseMap = tester.tupleCount(pysparsemap, 1);
    Condition<List<Tuple>> sparseTupleMapOut = tester.tupleContents(pysparsemap);
    // getConfig().put(ContextProperties.TRACING_LEVEL, TraceLevel.DEBUG);
    complete(tester, expectedCount.and(expectedCountSparse, expectedCountSparseMap), 20, TimeUnit.SECONDS);
    assertTrue(expectedCount.valid());
    assertTrue(expectedCountSparse.valid());
    assertTrue(expectedCountSparseMap.valid());
    Tuple r1 = outTuples.getResult().get(0);
    // optional signed integers
    // 123, null
    assertEquals(r1.getObject("oi32v"), 123);
    assertNull(r1.getObject("oi32nv"));
    // ["a", "b"], null
    {
        @SuppressWarnings("unchecked") List<RString> lr = (List<RString>) r1.getObject("olrv");
        assertEquals(2, lr.size());
        assertEquals("a", lr.get(0).getString());
        assertEquals("b", lr.get(1).getString());
        assertNull(r1.getObject("olrnv"));
    }
    // Sparse tuple handling - source
    assertEquals(1, sparseTupleOut.getResult().size());
    Tuple st = sparseTupleOut.getResult().get(0);
    // set by op
    assertEquals(37, st.getObject("a"));
    // default as None in tuple
    assertEquals(null, st.getObject("b"));
    // set by op
    assertEquals(23, st.getObject("c"));
    // set by op
    assertEquals(-46, st.getObject("d"));
    // default as None in tuple
    assertEquals(null, st.getObject("e"));
    // set by op
    assertEquals(56, st.getObject("f"));
    // set by op
    assertEquals(67, st.getObject("g"));
    // set by op
    assertEquals(78, st.getObject("h"));
    // default as no value (short tuple)
    assertEquals(null, st.getObject("i"));
    // Sparse tuple handling - map
    assertEquals(1, sparseTupleMapOut.getResult().size());
    Tuple stm = sparseTupleMapOut.getResult().get(0);
    // set by op
    assertEquals(37 + 81, stm.getObject("a"));
    // match input as None in tuple
    assertEquals(null, stm.getObject("b"));
    // match input as None in tuple
    assertEquals(23, stm.getObject("c"));
    // default to matching input
    assertEquals(-46, stm.getObject("d"));
    // default to matching input
    assertEquals(null, stm.getObject("e"));
    // no match: opt to non-opt
    assertEquals(0, stm.getObject("f"));
    // match non-opt to non-opt
    assertEquals(67, stm.getObject("g"));
    // match non-opt to opt
    assertEquals(78, stm.getObject("h"));
    // default to matching input
    assertEquals(null, stm.getObject("i"));
    // default as no value (short tuple)
    assertEquals(null, stm.getObject("j"));
}