Examples with BOutput com.ibm.streamsx.topology.builder.BOutput used on opensource projects

Example 6 with BOutput

use of com.ibm.streamsx.topology.builder.BOutput in project streamsx.topology by IBMStreams.

the class StreamImpl method _parallel.

private TStream<T> _parallel(Supplier<Integer> width, Routing routing, Function<T, ?> keyer) {
    if (width == null)
        throw new IllegalArgumentException(PortProperties.WIDTH);
    Integer widthVal;
    if (width.get() != null)
        widthVal = width.get();
    else if (width instanceof SubmissionParameter<?>)
        widthVal = ((SubmissionParameter<Integer>) width).getDefaultValue();
    else
        throw new IllegalArgumentException(Messages.getString("CORE_ILLEGAL_WIDTH_NULL"));
    if (widthVal != null && widthVal <= 0)
        throw new IllegalArgumentException(Messages.getString("CORE_ILLEGAL_WIDTH_VALUE"));
    BOutput toBeParallelized = output();
    boolean isPartitioned = false;
    if (keyer != null) {
        final ToIntFunction<T> hasher = new KeyFunctionHasher<>(keyer);
        BOperatorInvocation hashAdder = JavaFunctional.addFunctionalOperator(this, "HashAdder", HASH_ADDER_KIND, hasher);
        hashAdder._json().addProperty(HASH_ADDER, true);
        if (serializer.isPresent()) {
            hashAdder.setParameter("inputSerializer", serializeLogic(serializer.get()));
            JavaFunctional.addDependency(this, hashAdder, serializer.get().getClass());
            // may have more than we need.
            if (getTupleType() != null)
                JavaFunctional.addDependency(this, hashAdder, getTupleType());
            else
                JavaFunctional.copyDependencies(this, operator(), hashAdder);
        }
        hashAdder.layout().addProperty("hidden", true);
        BInputPort ip = connectTo(hashAdder, true, null);
        String hashSchema = ObjectSchemas.schemaWithHash(ip._schema());
        toBeParallelized = hashAdder.addOutput(hashSchema);
        isPartitioned = true;
    }
    BOutput parallelOutput = builder().parallel(toBeParallelized, routing.name(), width);
    if (isPartitioned) {
        parallelOutput._json().addProperty(PortProperties.PARTITIONED, true);
        JsonArray partitionKeys = new JsonArray();
        partitionKeys.add(new JsonPrimitive("__spl_hash"));
        parallelOutput._json().add(PortProperties.PARTITION_KEYS, partitionKeys);
        // Add hash remover
        StreamImpl<T> parallelStream = new StreamImpl<T>(this, parallelOutput, getTupleType(), serializer);
        BOperatorInvocation hashRemover = builder().addOperator("HashRemover", HASH_REMOVER_KIND, null);
        hashRemover.setModel(MODEL_SPL, LANGUAGE_JAVA);
        hashRemover.layout().addProperty("hidden", true);
        @SuppressWarnings("unused") BInputPort pip = parallelStream.connectTo(hashRemover, true, null);
        parallelOutput = hashRemover.addOutput(output._type());
    }
    return addMatchingStream(parallelOutput);
}

Example 7 with BOutput

use of com.ibm.streamsx.topology.builder.BOutput in project streamsx.topology by IBMStreams.

the class StreamImpl method union.

@SuppressWarnings("unchecked")
@Override
public TStream<T> union(Set<TStream<T>> others) {
    if (others.isEmpty())
        return this;
    Set<TStream<T>> allStreams = new HashSet<>();
    allStreams.addAll(others);
    allStreams.add(this);
    // Check we don't have just a single stream.
    if (allStreams.size() == 1)
        return this;
    List<TStream<T>> sourceStreams = new ArrayList<>();
    sourceStreams.addAll(allStreams);
    String schema = output()._type();
    Type tupleType = getTupleType();
    // Unwrap all streams so that we do not add the same stream twice
    // in multiple unions or explicitly and in a union.
    Set<BOutput> outputs = new HashSet<>();
    for (int i = 0; i < sourceStreams.size(); i++) {
        TStream<T> s = sourceStreams.get(i);
        // E..g TStream<String>.
        if (!schema.equals(s.output()._type())) {
            if (s.getTupleClass() != null) {
                // This stream has the direct schema!
                schema = s.output()._type();
                assert getTupleClass() == null;
                tupleType = s.getTupleClass();
                if (i != 0) {
                    // Didn't discover it first
                    // reset to go through the list
                    // again. Note this assumes there
                    // are just two options for the schema
                    // generic or direct
                    // to get back to 0.
                    i = -1;
                    outputs.clear();
                    continue;
                }
            } else {
                assert tupleType instanceof Class;
                s = s.asType((Class<T>) tupleType);
                assert s.output()._type().equals(schema);
                sourceStreams.set(i, s);
            }
        }
        outputs.add(s.output());
    }
    BOutput unionOutput = builder().addUnion(outputs);
    return new StreamImpl<T>(this, unionOutput, tupleType, serializer);
}

Example 8 with BOutput

use of com.ibm.streamsx.topology.builder.BOutput in project streamsx.topology by IBMStreams.

the class EmbeddedTesterRuntime method setupEmbeddedTestHandlers.

private void setupEmbeddedTestHandlers(EmbeddedGraph eg) throws Exception {
    // Embedded does not support a StreamHandler against
    // a connected stream. In that case add a no-op intermediate stream.
    Set<TStream<?>> adds = new HashSet<>();
    // Find all that are connected.
    for (TStream<?> stream : handlers.keySet()) {
        if (stream.output().isConnected())
            adds.add(stream);
    }
    // Insert a filter for the handler.
    for (TStream<?> stream : adds) {
        TStream<?> filter = stream.filter(t -> true);
        Set<StreamHandler<Tuple>> handler = handlers.get(stream);
        handlers.remove(stream);
        handlers.put(filter, handler);
    }
    final JavaTestableGraph tg = eg.getExecutionGraph();
    for (TStream<?> stream : handlers.keySet()) {
        Set<StreamHandler<Tuple>> streamHandlers = handlers.get(stream);
        final BOutput output = stream.output();
        final OutputPortDeclaration portDecl = eg.getOutputPort(output.name());
        for (StreamHandler<Tuple> streamHandler : streamHandlers) {
            tg.registerStreamHandler(portDecl, streamHandler);
        }
    }
}