Examples with StreamCodec com.datatorrent.api.StreamCodec used on opensource projects

Example 6 with StreamCodec

use of com.datatorrent.api.StreamCodec in project apex-core by apache.

the class StreamingContainer method deployOutputStreams.

private HashMap<String, ComponentContextPair<Stream, StreamContext>> deployOutputStreams(List<OperatorDeployInfo> nodeList, HashMap<String, ArrayList<String>> groupedInputStreams) throws Exception {
    HashMap<String, ComponentContextPair<Stream, StreamContext>> newStreams = new HashMap<>();
    /*
     * We proceed to deploy all the output streams. At the end of this block, our streams collection
     * will contain all the streams which originate at the output port of the operators. The streams
     * are generally mapped against the "nodename.portname" string. But the BufferServerPublishers which
     * share the output port with other inline streams are mapped against the Buffer Server port to
     * avoid collision and at the same time keep track of these buffer streams.
     */
    for (OperatorDeployInfo ndi : nodeList) {
        Node<?> node = nodes.get(ndi.id);
        long checkpointWindowId = ndi.checkpoint.windowId;
        for (OperatorDeployInfo.OutputDeployInfo nodi : ndi.outputs) {
            String sourceIdentifier = Integer.toString(ndi.id).concat(Component.CONCAT_SEPARATOR).concat(nodi.portName);
            int queueCapacity = getValue(PortContext.QUEUE_CAPACITY, nodi, ndi);
            logger.debug("for stream {} the queue capacity is {}", sourceIdentifier, queueCapacity);
            ArrayList<String> collection = groupedInputStreams.get(sourceIdentifier);
            Map<Integer, StreamCodec<?>> streamCodecs = nodi.streamCodecs;
            if ((collection == null) && (streamCodecs.size() == 1)) {
                assert (nodi.bufferServerHost != null) : "resulting stream cannot be inline: " + nodi;
                /*
           * Let's create a stream to carry the data to the Buffer Server.
           * Nobody in this container is interested in the output placed on this stream, but
           * this stream exists. That means someone outside of this container must be interested.
           */
                Map.Entry<Integer, StreamCodec<?>> entry = streamCodecs.entrySet().iterator().next();
                StreamCodec<?> streamCodec = entry.getValue();
                Integer streamCodecIdentifier = entry.getKey();
                String connIdentifier = sourceIdentifier + Component.CONCAT_SEPARATOR + streamCodecIdentifier;
                SimpleEntry<String, ComponentContextPair<Stream, StreamContext>> deployBufferServerPublisher = deployBufferServerPublisher(connIdentifier, streamCodec, checkpointWindowId, queueCapacity, nodi);
                newStreams.put(sourceIdentifier, deployBufferServerPublisher.getValue());
                node.connectOutputPort(nodi.portName, deployBufferServerPublisher.getValue().component);
            } else {
                /*
           * In this case we have 2 possibilities, either we have 1 inline or multiple streams.
           * Since we cannot tell at this point, we assume that we will have multiple streams and
           * plan accordingly. we possibly will come to this code block multiple times. We create
           * the MuxStream only the first time and use it for subsequent calls of this block.
           *
           * There is also the possibility that we have a stream with multiple sinks having distinct codecs
           */
                ComponentContextPair<Stream, StreamContext> pair = newStreams.get(sourceIdentifier);
                if (pair == null) {
                    /**
             * Let's multiplex the output placed on this stream.
             * This container itself contains more than one parties interested.
             */
                    StreamContext context = new StreamContext(nodi.declaredStreamId);
                    context.setSourceId(sourceIdentifier);
                    context.setFinishedWindowId(checkpointWindowId);
                    Stream stream = new MuxStream();
                    newStreams.put(sourceIdentifier, pair = new ComponentContextPair<>(stream, context));
                    node.connectOutputPort(nodi.portName, stream);
                }
                if (nodi.bufferServerHost != null) {
                    /*
             * Although there is a node in this container interested in output placed on this stream, there
             * seems to at least one more party interested but placed in a container other than this one.
             */
                    for (Map.Entry<Integer, StreamCodec<?>> entry : streamCodecs.entrySet()) {
                        Integer streamCodecIdentifier = entry.getKey();
                        StreamCodec<?> streamCodec = entry.getValue();
                        String connIdentifier = sourceIdentifier + Component.CONCAT_SEPARATOR + streamCodecIdentifier;
                        SimpleEntry<String, ComponentContextPair<Stream, StreamContext>> deployBufferServerPublisher = deployBufferServerPublisher(connIdentifier, streamCodec, checkpointWindowId, queueCapacity, nodi);
                        newStreams.put(deployBufferServerPublisher.getKey(), deployBufferServerPublisher.getValue());
                        String sinkIdentifier = pair.context.getSinkId();
                        if (sinkIdentifier == null) {
                            pair.context.setSinkId(deployBufferServerPublisher.getKey());
                        } else {
                            pair.context.setSinkId(sinkIdentifier.concat(", ").concat(deployBufferServerPublisher.getKey()));
                        }
                        ((Stream.MultiSinkCapableStream) pair.component).setSink(deployBufferServerPublisher.getKey(), deployBufferServerPublisher.getValue().component);
                    }
                }
            }
        }
    }
    return newStreams;
}