Examples with StreamSchema com.ibm.streams.operator.StreamSchema used on opensource projects

Example 21 with StreamSchema

use of com.ibm.streams.operator.StreamSchema in project streamsx.topology by IBMStreams.

the class JavaPrimitive method invokeJavaPrimitive.

/**
     * Invoke an SPL Java primitive operator with an arbitrary number
     * of input and output ports.
     * <BR>
     * Each input stream or window in {@code inputs} results in
     * a input port for the operator with the input port index
     * matching the position of the input in {@code inputs}.
     * If {@code inputs} is {@code null} or empty then the operator will not
     * have any input ports.
     * <BR>
     * Each SPL schema in {@code outputSchemas} an output port
     * for the operator with the output port index
     * matching the position of the schema in {@code outputSchemas}.
     * If {@code outputSchemas} is {@code null} or empty then the operator will not
     * have any output ports.
     * 
     * @param te Reference to Topology the operator will be in.
     * @param opClass Class of the operator to be invoked.
     * @param inputs Input streams to be connected to the operator. May be {@code null} if no input  streams are required.
     * @param outputSchemas Schemas of the output streams. May be {@code null} if no output streams are required.
     * @param params
     *            Parameters for the SPL Java Primitive operator, ignored if {@code null}.
     * @return List of {@code SPLStream} instances that represent the outputs of the operator.
     */
public static List<SPLStream> invokeJavaPrimitive(TopologyElement te, Class<? extends Operator> opClass, List<? extends SPLInput> inputs, List<StreamSchema> outputSchemas, Map<String, ? extends Object> params) {
    BOperatorInvocation op = te.builder().addOperator(getInvocationName(opClass), opClass, params);
    SourceInfo.setSourceInfo(op, JavaPrimitive.class);
    if (inputs != null && !inputs.isEmpty()) {
        for (SPLInput input : inputs) SPL.connectInputToOperator(input, op);
    }
    if (outputSchemas == null || outputSchemas.isEmpty())
        return Collections.emptyList();
    List<SPLStream> streams = new ArrayList<>(outputSchemas.size());
    for (StreamSchema outputSchema : outputSchemas) streams.add(new SPLStreamImpl(te, op.addOutput(outputSchema)));
    return streams;
}

Example 22 with StreamSchema

use of com.ibm.streams.operator.StreamSchema in project streamsx.topology by IBMStreams.

the class SPL method invokeOperator.

/**
     * Invoke an SPL operator with an arbitrary number
     * of input and output ports.
     * <BR>
     * Each input stream or window in {@code inputs} results in
     * a input port for the operator with the input port index
     * matching the position of the input in {@code inputs}.
     * If {@code inputs} is {@code null} or empty then the operator will not
     * have any input ports.
     * <BR>
     * Each SPL schema in {@code outputSchemas} an output port
     * for the operator with the output port index
     * matching the position of the schema in {@code outputSchemas}.
     * If {@code outputSchemas} is {@code null} or empty then the operator will not
     * have any output ports.
     * 
     * @param te Reference to Topology the operator will be in.
     * @param name Name for the operator invocation.
     * @param kind
     *            SPL kind of the operator to be invoked.
     * @param inputs Input streams to be connected to the operator. May be {@code null} if no input  streams are required.
     * @param outputSchemas Schemas of the output streams. May be {@code null} if no output streams are required.
     * @param params
     *            Parameters for the SPL Java Primitive operator, ignored if {@code null}.
     * @return List of {@code SPLStream} instances that represent the outputs of the operator.
     */
public static List<SPLStream> invokeOperator(TopologyElement te, String name, String kind, List<? extends SPLInput> inputs, List<StreamSchema> outputSchemas, Map<String, ? extends Object> params) {
    BOperatorInvocation op = te.builder().addSPLOperator(name, kind, params);
    SourceInfo.setSourceInfo(op, SPL.class);
    if (inputs != null && !inputs.isEmpty()) {
        for (SPLInput input : inputs) SPL.connectInputToOperator(input, op);
    }
    if (outputSchemas == null || outputSchemas.isEmpty())
        return Collections.emptyList();
    List<SPLStream> streams = new ArrayList<>(outputSchemas.size());
    for (StreamSchema outputSchema : outputSchemas) streams.add(new SPLStreamImpl(te, op.addOutput(outputSchema)));
    return streams;
}

Example 23 with StreamSchema

use of com.ibm.streams.operator.StreamSchema in project streamsx.topology by IBMStreams.

the class SPLOperatorsTest method testOpParams.

/**
     * Test we can invoke an SPL operator with various parameter types.
     */
private void testOpParams(String testName, OpParamAdder opParamAdder) throws Exception {
    Topology topology = new Topology(testName);
    opParamAdder.init(topology, getConfig());
    // getConfig().put(ContextProperties.KEEP_ARTIFACTS, true);
    StreamSchema schema = Type.Factory.getStreamSchema("tuple<" + "rstring r" + ", ustring u" + ", boolean b" + ", int8 i8, int16 i16, int32 i32, int64 i64" + ", uint8 ui8, uint16 ui16, uint32 ui32, uint64 ui64" + ", float32 f32, float64 f64" + " >");
    Random rand = new Random();
    String r = "test    X\tY\"Lit\nerals\\nX\\tY " + rand.nextInt();
    opParamAdder.put("r", r);
    String u = "test    X\tY\"Lit\nerals\\nX\\tY " + rand.nextInt();
    opParamAdder.put("u", SPL.createValue(u, MetaType.USTRING));
    boolean b = rand.nextBoolean();
    opParamAdder.put("b", b);
    byte i8 = (byte) rand.nextInt();
    short i16 = (short) rand.nextInt();
    int i32 = rand.nextInt();
    long i64 = rand.nextLong();
    opParamAdder.put("i8", i8);
    opParamAdder.put("i16", i16);
    opParamAdder.put("i32", i32);
    opParamAdder.put("i64", i64);
    // 255 => -1
    byte ui8 = (byte) 0xFF;
    // 65534 => -2 
    short ui16 = (short) 0xFFFE;
    // 4294967293 => -3
    int ui32 = 0xFFFFFFFD;
    // 18446744073709551612 => -4
    long ui64 = 0xFFFFFFFFFFFFFFFCL;
    opParamAdder.put("ui8", SPL.createValue(ui8, MetaType.UINT8));
    opParamAdder.put("ui16", SPL.createValue(ui16, MetaType.UINT16));
    opParamAdder.put("ui32", SPL.createValue(ui32, MetaType.UINT32));
    opParamAdder.put("ui64", SPL.createValue(ui64, MetaType.UINT64));
    float f32 = rand.nextFloat();
    double f64 = rand.nextDouble();
    opParamAdder.put("f32", f32);
    opParamAdder.put("f64", f64);
    SPL.addToolkit(topology, new File(getTestRoot(), "spl/testtk"));
    SPLStream paramTuple = SPL.invokeSource(topology, "testgen::TypeLiteralTester", opParamAdder.getParams(), schema);
    // paramTuple.print();
    // paramTuple.filter(new AllowAll<Tuple>());
    Tester tester = topology.getTester();
    Condition<Long> expectedCount = tester.tupleCount(paramTuple, 1);
    MostRecent<Tuple> mr = tester.splHandler(paramTuple, new MostRecent<Tuple>());
    // getConfig().put(ContextProperties.KEEP_ARTIFACTS, true);
    complete(tester, expectedCount, 10, TimeUnit.SECONDS);
    assertTrue(expectedCount.toString(), expectedCount.valid());
    Tuple tuple = mr.getMostRecentTuple();
    // System.out.println("tuple: " + tuple);
    assertEquals(r, tuple.getString("r"));
    assertEquals(u, tuple.getString("u"));
    assertEquals(i8, tuple.getByte("i8"));
    assertEquals(i16, tuple.getShort("i16"));
    assertEquals(i32, tuple.getInt("i32"));
    assertEquals(i64, tuple.getLong("i64"));
    assertEquals(ui8, tuple.getByte("ui8"));
    assertEquals(ui16, tuple.getShort("ui16"));
    assertEquals(ui32, tuple.getInt("ui32"));
    assertEquals(ui64, tuple.getLong("ui64"));
    assertEquals(f32, tuple.getFloat("f32"), 0.001);
    assertEquals(f64, tuple.getDouble("f64"), 0.001);
}

Example 24 with StreamSchema

use of com.ibm.streams.operator.StreamSchema in project streamsx.topology by IBMStreams.

the class StreamImpl method union.

@SuppressWarnings("unchecked")
@Override
public TStream<T> union(Set<TStream<T>> others) {
    if (others.isEmpty())
        return this;
    Set<TStream<T>> allStreams = new HashSet<>();
    allStreams.addAll(others);
    allStreams.add(this);
    // Check we don't have just a single stream.
    if (allStreams.size() == 1)
        return this;
    List<TStream<T>> sourceStreams = new ArrayList<>();
    sourceStreams.addAll(allStreams);
    StreamSchema schema = output().schema();
    Type tupleType = getTupleType();
    // Unwrap all streams so that we do not add the same stream twice
    // in multiple unions or explicitly and in a union.
    Set<BOutput> outputs = new HashSet<>();
    for (int i = 0; i < sourceStreams.size(); i++) {
        TStream<T> s = sourceStreams.get(i);
        // E..g TStream<String>.
        if (!schema.equals(s.output().schema())) {
            if (s.getTupleClass() != null) {
                // This stream has the direct schema!
                schema = s.output().schema();
                assert getTupleClass() == null;
                tupleType = s.getTupleClass();
                if (i != 0) {
                    // Didn't discover it first
                    // reset to go through the list
                    // again. Note this assumes there
                    // are just two options for the schema
                    // generic or direct
                    // to get back to 0.
                    i = -1;
                    outputs.clear();
                    continue;
                }
            } else {
                assert tupleType instanceof Class;
                s = s.asType((Class<T>) tupleType);
                assert s.output().schema().equals(schema);
                sourceStreams.set(i, s);
            }
        }
        outputs.add(s.output());
    }
    BOutput unionOutput = builder().addUnion(outputs);
    return new StreamImpl<T>(this, unionOutput, tupleType);
}

Example 25 with StreamSchema

use of com.ibm.streams.operator.StreamSchema in project streamsx.topology by IBMStreams.

the class StreamImpl method _parallel.

private TStream<T> _parallel(Supplier<Integer> width, Function<T, ?> keyer) {
    if (width == null)
        throw new IllegalArgumentException("width");
    Integer widthVal;
    if (width.get() != null)
        widthVal = width.get();
    else if (width instanceof SubmissionParameter<?>)
        widthVal = ((SubmissionParameter<Integer>) width).getDefaultValue();
    else
        throw new IllegalArgumentException("Illegal width Supplier: width.get() returns null.");
    if (widthVal != null && widthVal <= 0)
        throw new IllegalArgumentException("The parallel width must be greater than or equal to 1.");
    BOutput toBeParallelized = output();
    boolean isPartitioned = false;
    if (keyer != null) {
        final ToIntFunction<T> hasher = new KeyFunctionHasher<>(keyer);
        BOperatorInvocation hashAdder = JavaFunctional.addFunctionalOperator(this, "HashAdder", HashAdder.class, hasher);
        // hashAdder.json().put("routing", routing.toString());
        BInputPort ip = connectTo(hashAdder, true, null);
        StreamSchema hashSchema = ip.port().getStreamSchema().extend("int32", "__spl_hash");
        toBeParallelized = hashAdder.addOutput(hashSchema);
        isPartitioned = true;
    }
    BOutput parallelOutput = builder().parallel(toBeParallelized, width);
    if (isPartitioned) {
        parallelOutput.json().put("partitioned", true);
        JSONArray partitionKeys = new JSONArray();
        partitionKeys.add("__spl_hash");
        parallelOutput.json().put("partitionedKeys", partitionKeys);
        // Add hash remover
        StreamImpl<T> parallelStream = new StreamImpl<T>(this, parallelOutput, getTupleType());
        BOperatorInvocation hashRemover = builder().addOperator(HashRemover.class, null);
        BInputPort pip = parallelStream.connectTo(hashRemover, true, null);
        parallelOutput = hashRemover.addOutput(pip.port().getStreamSchema().remove("__spl_hash"));
    }
    return addMatchingStream(parallelOutput);
}