Examples with StreamSchema com.ibm.streams.operator.StreamSchema used on opensource projects

Example 46 with StreamSchema

use of com.ibm.streams.operator.StreamSchema in project streamsx.topology by IBMStreams.

the class PythonFunctionalOperatorsTest method testValues.

/**
 * Test that specific values in Python
 * make their way into SPL correctly
 * when returning as a tuple.
 * @throws Exception
 */
@Test
public void testValues() throws Exception {
    Topology topology = new Topology("testValues");
    addTestToolkit(topology);
    SPLStream pysrc = SPL.invokeSource(topology, "com.ibm.streamsx.topology.pytest.pysource::SpecificValues", null, ALL_PYTHON_TYPES_SCHEMA);
    StreamSchema sparseSchema = Type.Factory.getStreamSchema("tuple<int32 a, int32 b, int32 c, int32 d, int32 e>");
    Tester tester = topology.getTester();
    Condition<Long> expectedCount = tester.tupleCount(pysrc, 1);
    Condition<List<Tuple>> outTuples = tester.tupleContents(pysrc);
    SPLStream pysparse = SPL.invokeSource(topology, "com.ibm.streamsx.topology.pytest.pysource::SparseTuple", null, sparseSchema);
    SPLStream pysparsemap = SPL.invokeOperator("com.ibm.streamsx.topology.pytest.pymap::SparseTupleMap", pysparse, sparseSchema.extend("int32", "f"), null);
    Condition<Long> expectedCountSparse = tester.tupleCount(pysparse, 1);
    Condition<List<Tuple>> sparseTupleOut = tester.tupleContents(pysparse);
    Condition<Long> expectedCountSparseMap = tester.tupleCount(pysparsemap, 1);
    Condition<List<Tuple>> sparseTupleMapOut = tester.tupleContents(pysparsemap);
    // getConfig().put(ContextProperties.TRACING_LEVEL, TraceLevel.DEBUG);
    complete(tester, expectedCount.and(expectedCountSparse, expectedCountSparseMap), 20, TimeUnit.SECONDS);
    assertTrue(expectedCount.valid());
    assertTrue(expectedCountSparse.valid());
    assertTrue(expectedCountSparseMap.valid());
    Tuple r1 = outTuples.getResult().get(0);
    assertTrue(r1.getBoolean("b"));
    // signed integers
    // 23, -2525, 3252352, -2624565653,
    assertEquals(r1.getByte("i8"), 23);
    assertEquals(r1.getShort("i16"), -2525);
    assertEquals(r1.getInt("i32"), 3252352);
    assertEquals(r1.getLong("i64"), -2624565653L);
    // unsigned int
    // 72, 6873, 43665588, 357568872
    assertEquals(r1.getString("u8"), "72");
    assertEquals(r1.getString("u16"), "6873");
    assertEquals(r1.getString("u32"), "43665588");
    assertEquals(r1.getString("u64"), "357568872");
    // floats
    // 4367.34, -87657525334.22
    assertEquals(r1.getFloat("f32"), 4367.34f, 0.1);
    assertEquals(r1.getDouble("f64"), -87657525334.22d, 0.1);
    // rstring, Unicode data
    assertEquals("⡍⠔⠙⠖ ⡊ ⠙⠕⠝⠰⠞ ⠍⠑⠁⠝ ⠞⠕ ⠎⠁⠹ ⠹⠁⠞ ⡊ ⠅⠝⠪⠂ ⠕⠋ ⠍⠹", r1.getString("r"));
    // complex(-23.0, 325.38), complex(-35346.234, 952524.93)
    assertEquals(((Complex) r1.getObject("c32")).getReal(), -23.0, 0.1);
    assertEquals(((Complex) r1.getObject("c32")).getImaginary(), 325.38, 0.1);
    assertEquals(((Complex) r1.getObject("c64")).getReal(), -35346.234, 0.1);
    assertEquals(((Complex) r1.getObject("c64")).getImaginary(), 952524.93, 0.1);
    assertEquals(r1.getBigDecimal("d32"), new BigDecimal("3.459876E72"));
    assertEquals(r1.getBigDecimal("d64"), new BigDecimal("4.515716038731674E-307"));
    assertEquals(r1.getBigDecimal("d128"), new BigDecimal("1.085059319410602846995696978141388E+5922"));
    // Timestamp Timestamp(781959759, 9320, 76)
    assertEquals(781959759L, r1.getTimestamp("ts").getSeconds());
    assertEquals(9320, r1.getTimestamp("ts").getNanoseconds());
    assertEquals(76, r1.getTimestamp("ts").getMachineId());
    // ["a", "Streams!", "2H₂ + O₂ ⇌ 2H₂O, R = 4.7 kΩ, ⌀ 200 mm"]
    {
        @SuppressWarnings("unchecked") List<RString> lr = (List<RString>) r1.getObject("lr");
        assertEquals(3, lr.size());
        assertEquals("a", lr.get(0).getString());
        assertEquals("Streams!", lr.get(1).getString());
        assertEquals("2H₂ + O₂ ⇌ 2H₂O, R = 4.7 kΩ, ⌀ 200 mm", lr.get(2).getString());
    }
    // [345,-4578],
    {
        int[] li32 = (int[]) r1.getObject("li32");
        assertEquals(2, li32.length);
        assertEquals(345, li32[0]);
        assertEquals(-4578, li32[1]);
    }
    // [9983, -4647787587, 0]
    {
        long[] li64 = (long[]) r1.getObject("li64");
        assertEquals(3, li64.length);
        assertEquals(9983L, li64[0]);
        assertEquals(-4647787587L, li64[1]);
        assertEquals(0L, li64[2]);
    }
    {
        @SuppressWarnings("unchecked") List<Integer> lui32 = (List<Integer>) r1.getObject("lui32");
        assertEquals(1, lui32.size());
        assertEquals("87346", Integer.toUnsignedString(lui32.get(0)));
    }
    {
        @SuppressWarnings("unchecked") List<Long> lui64 = (List<Long>) r1.getObject("lui64");
        assertEquals(2, lui64.size());
        assertEquals("45433674", Long.toUnsignedString(lui64.get(0)));
        assertEquals("41876984848", Long.toUnsignedString(lui64.get(1)));
    }
    // 4.269986E+05, -8.072285E+02 -6.917091E-08 7.735085E8
    {
        float[] li32 = (float[]) r1.getObject("lf32");
        assertEquals(4, li32.length);
        assertEquals(4.269986E+05f, li32[0], 0.1);
        assertEquals(-8.072285E+02f, li32[1], 0.1);
        assertEquals(-6.917091E-08f, li32[2], 0.1);
        assertEquals(7.735085E8f, li32[3], 0.1);
    }
    {
        double[] lf64 = (double[]) r1.getObject("lf64");
        assertEquals(1, lf64.length);
        assertEquals(765.46477e19, lf64[0], 0.1);
    }
    {
        boolean[] lb = (boolean[]) r1.getObject("lb");
        assertEquals(3, lb.length);
        assertTrue(lb[0]);
        assertFalse(lb[1]);
        assertTrue(lb[2]);
    }
    assertTrue(r1.getMap("mi32r").isEmpty());
    assertTrue(r1.getMap("mru32").isEmpty());
    {
        Map<?, ?> mri32 = r1.getMap("mri32");
        assertEquals(2, mri32.size());
        System.out.println("mri32:" + mri32);
        assertTrue(mri32.containsKey(new RString("abc")));
        assertTrue(mri32.containsKey(new RString("многоязычных")));
        assertEquals(35320, mri32.get(new RString("abc")));
        assertEquals(-236325, mri32.get(new RString("многоязычных")));
    }
    assertTrue(r1.getMap("mu32r").isEmpty());
    assertTrue(r1.getMap("mi32i32").isEmpty());
    assertTrue(r1.getMap("mu32u32").isEmpty());
    assertTrue(r1.getMap("mrr").isEmpty());
    assertTrue(r1.getMap("mf64f64").isEmpty());
    assertTrue(r1.getMap("mf64i32").isEmpty());
    assertTrue(r1.getMap("mf64u32").isEmpty());
    assertTrue(r1.getMap("mf64r").isEmpty());
    assertTrue(r1.getMap("mrf64").isEmpty());
    // Sparse tuple handling - source
    assertEquals(1, sparseTupleOut.getResult().size());
    Tuple st = sparseTupleOut.getResult().get(0);
    // set by op
    assertEquals(37, st.getInt("a"));
    // default as None in tuple
    assertEquals(0, st.getInt("b"));
    // default as None in tuple
    assertEquals(0, st.getInt("c"));
    // set by op
    assertEquals(-46, st.getInt("d"));
    // default as no value (short tuple)
    assertEquals(0, st.getInt("e"));
    // Sparse tuple handling - map
    assertEquals(1, sparseTupleMapOut.getResult().size());
    Tuple stm = sparseTupleMapOut.getResult().get(0);
    // set by op
    assertEquals(37 + 81, stm.getInt("a"));
    // set by op
    assertEquals(23, stm.getInt("b"));
    // default as None in tuple
    assertEquals(0, stm.getInt("c"));
    // default to matching input
    assertEquals(-46, stm.getInt("d"));
    // set by op
    assertEquals(34, stm.getInt("e"));
    // default as no value (short tuple)
    assertEquals(0, stm.getInt("f"));
}

Example 47 with StreamSchema

use of com.ibm.streams.operator.StreamSchema in project streamsx.topology by IBMStreams.

the class SPLParallelTest method testBroadcast.

@Test
public void testBroadcast() throws Exception {
    Topology topology = newTopology();
    TStream<String> values = topology.strings("UDP-B", "@parallel-B", "Done-B");
    StreamSchema schema = com.ibm.streams.operator.Type.Factory.getStreamSchema("tuple<rstring x>");
    SPLStream vs = SPLStreams.convertStream(values, (s, o) -> {
        o.setString(0, s);
        return o;
    }, schema);
    vs = vs.parallel(() -> 3, Routing.BROADCAST);
    SPLStream r = vs.filter(new AllowAll<>()).endParallel();
    TStream<String> rs = SPLStreams.toStringStream(r);
    Tester tester = topology.getTester();
    Condition<Long> expectedCount = tester.tupleCount(r, 9);
    Condition<List<String>> content = tester.stringContentsUnordered(rs, "UDP-B", "@parallel-B", "Done-B", "UDP-B", "@parallel-B", "Done-B", "UDP-B", "@parallel-B", "Done-B");
    assertTrue(complete(tester, expectedCount, 20, TimeUnit.SECONDS));
    assertTrue(content.valid());
}

Example 48 with StreamSchema

use of com.ibm.streams.operator.StreamSchema in project streamsx.topology by IBMStreams.

the class ConditionTest method testSPLContentsBad.

@Test
public void testSPLContentsBad() throws Exception {
    final Topology topology = new Topology();
    TStream<String> source = topology.strings("A", "B", "C", "D");
    StreamSchema schema = SPLSchemas.STRING.extend("int32", "id");
    SPLStream tested = SPLStreams.convertStream(source, (s, t) -> {
        t.setString(0, s);
        t.setInt(1, s.charAt(0));
        return t;
    }, schema);
    ExpectedTuples expected = new ExpectedTuples(schema);
    int id = "A".charAt(0);
    expected.addAsTuple(new RString("A"), id);
    expected.addAsTuple(new RString("B"), ++id);
    expected.addAsTuple(new RString("C"), 1241241);
    expected.addAsTuple(new RString("D"), ++id);
    Condition<List<Tuple>> contents = expected.contents(tested);
    boolean passed = complete(topology.getTester(), contents, 10, TimeUnit.SECONDS);
    assertFalse(passed);
    assertFalse(contents.toString(), contents.valid());
}

Example 49 with StreamSchema

use of com.ibm.streams.operator.StreamSchema in project streamsx.topology by IBMStreams.

the class SPLOperatorsTest method testOpParamsOptionalTypes.

/**
 * Test we can invoke an SPL operator with various parameter types,
 * where the type is an optional type.
 */
private void testOpParamsOptionalTypes(String testName, OpParamAdder opParamAdder) throws Exception {
    Topology topology = new Topology(testName);
    opParamAdder.init(topology, getConfig());
    StreamSchema schema = Type.Factory.getStreamSchema("tuple<" + "rstring r" + ", optional<rstring> orv" + ", optional<rstring> ornv" + ", int32 i32" + ", optional<int32> oi32v" + ", optional<int32> oi32nv" + " >");
    Map<String, Object> expectedValues = new HashMap<>();
    Random rand = new Random();
    String r = "test    X\tY\"Lit\nerals\\nX\\tY " + rand.nextInt();
    opParamAdder.put("r", r);
    expectedValues.put("r", new RString(r));
    String orv = "test    X\tY\"Lit\nerals\\nX\\tY " + rand.nextInt();
    opParamAdder.put("orv", orv);
    // test setting optional type to null by using null in Map
    opParamAdder.put("ornv", null);
    expectedValues.put("orv", new RString(orv));
    expectedValues.put("ornv", null);
    int i32 = rand.nextInt();
    opParamAdder.put("i32", i32);
    int oi32v = rand.nextInt();
    opParamAdder.put("oi32v", oi32v);
    // test setting optional type to null by using createNullValue() in Map
    opParamAdder.put("oi32nv", SPL.createNullValue());
    expectedValues.put("i32", i32);
    expectedValues.put("oi32v", oi32v);
    expectedValues.put("oi32nv", null);
    SPL.addToolkit(topology, new File(getTestRoot(), "spl/testtkopt"));
    SPLStream paramTuple = SPL.invokeSource(topology, "testgen::TypeLiteralTester", opParamAdder.getParams(), schema);
    Tester tester = topology.getTester();
    Condition<Long> expectedCount = tester.tupleCount(paramTuple, 1);
    Condition<?> contents = tester.tupleContents(paramTuple, schema.getTuple(expectedValues));
    complete(tester, expectedCount.and(contents), 10, TimeUnit.SECONDS);
    assertTrue(contents.valid());
    assertTrue(expectedCount.valid());
}

Example 50 with StreamSchema

use of com.ibm.streams.operator.StreamSchema in project streamsx.topology by IBMStreams.

the class SPLOperatorsTest method testOpParams.

/**
 * Test we can invoke an SPL operator with various parameter types.
 */
private void testOpParams(String testName, OpParamAdder opParamAdder) throws Exception {
    Topology topology = new Topology(testName);
    opParamAdder.init(topology, getConfig());
    StreamSchema schema = Type.Factory.getStreamSchema("tuple<" + "rstring r" + ", ustring u" + ", boolean b" + ", int8 i8, int16 i16, int32 i32, int64 i64" + ", uint8 ui8, uint16 ui16, uint32 ui32, uint64 ui64" + ", float32 f32, float64 f64" + " >");
    Map<String, Object> expectedValues = new HashMap<>();
    Random rand = new Random();
    String r = "test    X\tY\"Lit\nerals\\nX\\tY " + rand.nextInt();
    opParamAdder.put("r", r);
    String u = "test    X\tY\"Lit\nerals\\nX\\tY " + rand.nextInt();
    opParamAdder.put("u", SPL.createValue(u, MetaType.USTRING));
    expectedValues.put("r", new RString(r));
    expectedValues.put("u", u);
    boolean b = rand.nextBoolean();
    opParamAdder.put("b", b);
    expectedValues.put("b", b);
    byte i8 = (byte) rand.nextInt();
    short i16 = (short) rand.nextInt();
    int i32 = rand.nextInt();
    long i64 = rand.nextLong();
    opParamAdder.put("i8", i8);
    opParamAdder.put("i16", i16);
    opParamAdder.put("i32", i32);
    opParamAdder.put("i64", i64);
    expectedValues.put("i8", i8);
    expectedValues.put("i16", i16);
    expectedValues.put("i32", i32);
    expectedValues.put("i64", i64);
    // 255 => -1
    byte ui8 = (byte) 0xFF;
    // 65534 => -2
    short ui16 = (short) 0xFFFE;
    // 4294967293 => -3
    int ui32 = 0xFFFFFFFD;
    // 18446744073709551612 => -4
    long ui64 = 0xFFFFFFFFFFFFFFFCL;
    opParamAdder.put("ui8", SPL.createValue(ui8, MetaType.UINT8));
    opParamAdder.put("ui16", SPL.createValue(ui16, MetaType.UINT16));
    opParamAdder.put("ui32", SPL.createValue(ui32, MetaType.UINT32));
    opParamAdder.put("ui64", SPL.createValue(ui64, MetaType.UINT64));
    expectedValues.put("ui8", ui8);
    expectedValues.put("ui16", ui16);
    expectedValues.put("ui32", ui32);
    expectedValues.put("ui64", ui64);
    float f32 = 4.0f;
    double f64 = 32.0;
    opParamAdder.put("f32", f32);
    opParamAdder.put("f64", f64);
    expectedValues.put("f32", f32);
    expectedValues.put("f64", f64);
    SPL.addToolkit(topology, new File(getTestRoot(), "spl/testtk"));
    SPLStream paramTuple = SPL.invokeSource(topology, "testgen::TypeLiteralTester", opParamAdder.getParams(), schema);
    Tuple expectedTuple = schema.getTuple(expectedValues);
    Tester tester = topology.getTester();
    Condition<Long> expectedCount = tester.tupleCount(paramTuple, 1);
    Condition<?> contents = tester.tupleContents(paramTuple, expectedTuple);
    ;
    complete(tester, expectedCount.and(contents), 10, TimeUnit.SECONDS);
    assertTrue(contents.valid());
    assertTrue(expectedCount.valid());
}