Examples with StreamCodec com.datatorrent.api.StreamCodec used on opensource projects

Example 1 with StreamCodec

use of com.datatorrent.api.StreamCodec in project apex-core by apache.

the class BufferServerSubscriberTest method testEmergencySinks.

@Test
public void testEmergencySinks() throws InterruptedException {
    final List<Object> list = new ArrayList<>();
    final StreamCodec<Object> myserde = new StreamCodec<Object>() {

        @Override
        public Object fromByteArray(Slice fragment) {
            if (fragment.offset == 0 && fragment.length == fragment.buffer.length) {
                return fragment.buffer;
            } else {
                return Arrays.copyOfRange(fragment.buffer, fragment.offset, fragment.offset + fragment.length);
            }
        }

        @Override
        public Slice toByteArray(Object o) {
            return new Slice((byte[]) o, 0, ((byte[]) o).length);
        }

        @Override
        public int getPartition(Object o) {
            return 0;
        }
    };
    Sink<Object> unbufferedSink = new Sink<Object>() {

        @Override
        public void put(Object tuple) {
            list.add(tuple);
        }

        @Override
        public int getCount(boolean reset) {
            return 0;
        }
    };
    BufferServerSubscriber bss = new BufferServerSubscriber("subscriber", 5) {

        {
            serde = myserde;
        }

        @Override
        public void suspendRead() {
            logger.debug("read suspended");
        }

        @Override
        public void resumeRead() {
            logger.debug("read resumed");
        }
    };
    SweepableReservoir reservoir = bss.acquireReservoir("unbufferedSink", 3);
    reservoir.setSink(unbufferedSink);
    int i = 0;
    while (i++ < 10) {
        Slice fragment = myserde.toByteArray(new byte[] { (byte) i });
        byte[] buffer = PayloadTuple.getSerializedTuple(myserde.getPartition(i), fragment);
        bss.onMessage(buffer, 0, buffer.length);
    }
    reservoir.sweep();
    /* 4 make it to the reservoir */
    reservoir.sweep();
    /* we consume the 4; and 4 more make it to the reservoir */
    Assert.assertEquals("4 received", 4, list.size());
    reservoir.sweep();
    /* 8 consumed + 2 more make it to the reservoir */
    reservoir.sweep();
    /* consume 2 more */
    Assert.assertEquals("10  received", 10, list.size());
}

Example 2 with StreamCodec

use of com.datatorrent.api.StreamCodec in project apex-core by apache.

the class StreamPersistanceTests method testDynamicPartitioning.

@Test
public void testDynamicPartitioning() throws ClassNotFoundException, IOException {
    AscendingNumbersOperator ascend = dag.addOperator("ascend", new AscendingNumbersOperator());
    final TestReceiverOperator console = dag.addOperator("console", new TestReceiverOperator());
    dag.setOperatorAttribute(console, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<TestReceiverOperator>(2));
    dag.setOperatorAttribute(console, Context.OperatorContext.STATS_LISTENERS, Lists.newArrayList((StatsListener) new PartitioningTest.PartitionLoadWatch()));
    final PartitionedTestPersistanceOperator console1 = new PartitionedTestPersistanceOperator();
    StreamMeta s = dag.addStream("Stream1", ascend.outputPort, console.inport);
    dag.setInputPortAttribute(console.inport, PortContext.STREAM_CODEC, new TestPartitionCodec());
    s.persistUsing("persister", console1, console1.inport);
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, Integer.MAX_VALUE);
    StramTestSupport.MemoryStorageAgent msa = new StramTestSupport.MemoryStorageAgent();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, msa);
    StreamingContainerManager dnm = new StreamingContainerManager(dag);
    PhysicalPlan plan = dnm.getPhysicalPlan();
    List<PTContainer> containers = plan.getContainers();
    Assert.assertEquals("number containers", 4, containers.size());
    for (int i = 0; i < containers.size(); ++i) {
        StreamingContainerManagerTest.assignContainer(dnm, "container" + (i + 1));
    }
    LogicalPlan.OperatorMeta passThruMeta = dag.getMeta(console);
    List<PTOperator> ptos = plan.getOperators(passThruMeta);
    PTOperator persistOperatorContainer = null;
    for (PTContainer container : plan.getContainers()) {
        for (PTOperator operator : container.getOperators()) {
            operator.setState(PTOperator.State.ACTIVE);
            if (operator.getName().equals("persister")) {
                persistOperatorContainer = operator;
            }
        }
    }
    // Check that persist operator is part of dependents redeployed
    Set<PTOperator> operators = plan.getDependents(ptos);
    logger.debug("Operators to be re-deployed = {}", operators);
    // Validate that persist operator is part of dependents
    assertTrue("persist operator should be part of the operators to be redeployed", operators.contains(persistOperatorContainer));
    LogicalPlan.StreamMeta s1 = (LogicalPlan.StreamMeta) s;
    StreamCodec codec = s1.getPersistOperatorInputPort().getStreamCodec();
    assertEquals("Codec should be instance of StreamCodecWrapper", codec instanceof StreamCodecWrapperForPersistance, true);
    StreamCodecWrapperForPersistance wrapperCodec = (StreamCodecWrapperForPersistance) codec;
    Entry<InputPortMeta, Collection<PartitionKeys>> keys = (Entry<InputPortMeta, Collection<PartitionKeys>>) wrapperCodec.inputPortToPartitionMap.entrySet().iterator().next();
    logger.debug(keys.toString());
    assertEquals("Size of partitions should be 2", 2, keys.getValue().size());
    for (PTOperator ptOperator : ptos) {
        PartitioningTest.PartitionLoadWatch.put(ptOperator, -1);
        plan.onStatusUpdate(ptOperator);
    }
    dnm.processEvents();
    assertEquals("Input port map", wrapperCodec.inputPortToPartitionMap.size(), 1);
    keys = (Entry<InputPortMeta, Collection<PartitionKeys>>) wrapperCodec.inputPortToPartitionMap.entrySet().iterator().next();
    assertEquals("Size of partitions should be 1 after repartition", 1, keys.getValue().size());
    logger.debug(keys.toString());
}

Example 3 with StreamCodec

use of com.datatorrent.api.StreamCodec in project apex-core by apache.

the class StreamingContainer method deployInputStreams.

@SuppressWarnings("unchecked")
private void deployInputStreams(List<OperatorDeployInfo> operatorList, HashMap<String, ComponentContextPair<Stream, StreamContext>> newStreams) throws UnknownHostException {
    /*
     * collect any input operators along with their smallest window id,
     * those are subsequently used to setup the window generator
     */
    ArrayList<OperatorDeployInfo> inputNodes = new ArrayList<>();
    long smallestCheckpointedWindowId = Long.MAX_VALUE;
    // a simple map which maps the oio node to it's the node which owns the thread.
    Map<Integer, Integer> oioNodes = new ConcurrentHashMap<>();
    /*
     * Hook up all the downstream ports. There are 2 places where we deal with more than 1
     * downstream ports. The first one follows immediately for WindowGenerator. The second
     * case is when source for the input port of some node in this container is in another
     * container. So we need to create the stream. We need to track this stream along with
     * other streams,and many such streams may exist, we hash them against buffer server
     * info as we did for outputs but throw in the sinkid in the mix as well.
     */
    for (OperatorDeployInfo ndi : operatorList) {
        if (ndi.inputs == null || ndi.inputs.isEmpty()) {
            /*
         * This has to be InputNode, so let's hook the WindowGenerator to it.
         * A node which does not take any input cannot exist in the DAG since it would be completely
         * unaware of the windows. So for that reason, AbstractInputNode allows Component.INPUT port.
         */
            inputNodes.add(ndi);
            /*
         * When we activate the window Generator, we plan to activate it only from required windowId.
         */
            ndi.checkpoint = getFinishedCheckpoint(ndi);
            if (ndi.checkpoint.windowId < smallestCheckpointedWindowId) {
                smallestCheckpointedWindowId = ndi.checkpoint.windowId;
            }
        } else {
            Node<?> node = nodes.get(ndi.id);
            for (OperatorDeployInfo.InputDeployInfo nidi : ndi.inputs) {
                if (nidi.streamCodecs.size() != 1) {
                    throw new IllegalStateException("Only one input codec configuration should be present");
                }
                Map.Entry<Integer, StreamCodec<?>> entry = nidi.streamCodecs.entrySet().iterator().next();
                Integer streamCodecIdentifier = entry.getKey();
                StreamCodec<?> streamCodec = entry.getValue();
                String sourceIdentifier = Integer.toString(nidi.sourceNodeId).concat(Component.CONCAT_SEPARATOR).concat(nidi.sourcePortName);
                String sinkIdentifier = Integer.toString(ndi.id).concat(Component.CONCAT_SEPARATOR).concat(nidi.portName);
                int queueCapacity = getValue(PortContext.QUEUE_CAPACITY, nidi, ndi);
                Checkpoint checkpoint = getFinishedCheckpoint(ndi);
                ComponentContextPair<Stream, StreamContext> pair = streams.get(sourceIdentifier);
                if (pair == null) {
                    pair = newStreams.get(sourceIdentifier);
                }
                if (pair == null) {
                    /*
             * We connect to the buffer server for the input on this port.
             * We have already placed all the output streams for all the operators in this container.
             * Yet, there is no stream which can source this port so it has to come from the buffer
             * server, so let's make a connection to it.
             */
                    assert (nidi.locality != Locality.CONTAINER_LOCAL && nidi.locality != Locality.THREAD_LOCAL);
                    StreamContext context = new StreamContext(nidi.declaredStreamId);
                    context.setBufferServerAddress(InetSocketAddress.createUnresolved(nidi.bufferServerHost, nidi.bufferServerPort));
                    InetAddress inetAddress = context.getBufferServerAddress().getAddress();
                    if (inetAddress != null && NetUtils.isLocalAddress(inetAddress)) {
                        context.setBufferServerAddress(new InetSocketAddress(InetAddress.getByName(null), nidi.bufferServerPort));
                    }
                    context.put(StreamContext.BUFFER_SERVER_TOKEN, nidi.bufferServerToken);
                    String connIdentifier = sourceIdentifier + Component.CONCAT_SEPARATOR + streamCodecIdentifier;
                    context.setPortId(nidi.portName);
                    context.put(StreamContext.CODEC, streamCodec);
                    context.put(StreamContext.EVENT_LOOP, eventloop);
                    context.setPartitions(nidi.partitionMask, nidi.partitionKeys);
                    // context.setSourceId(sourceIdentifier);
                    context.setSourceId(connIdentifier);
                    context.setSinkId(sinkIdentifier);
                    context.setFinishedWindowId(checkpoint.windowId);
                    BufferServerSubscriber subscriber = fastPublisherSubscriber ? new FastSubscriber("tcp://".concat(nidi.bufferServerHost).concat(":").concat(String.valueOf(nidi.bufferServerPort)).concat("/").concat(connIdentifier), queueCapacity) : new BufferServerSubscriber("tcp://".concat(nidi.bufferServerHost).concat(":").concat(String.valueOf(nidi.bufferServerPort)).concat("/").concat(connIdentifier), queueCapacity);
                    if (streamCodec instanceof StreamCodecWrapperForPersistance) {
                        subscriber.acquireReservoirForPersistStream(sinkIdentifier, queueCapacity, streamCodec);
                    }
                    SweepableReservoir reservoir = subscriber.acquireReservoir(sinkIdentifier, queueCapacity);
                    if (checkpoint.windowId >= 0) {
                        node.connectInputPort(nidi.portName, new WindowIdActivatedReservoir(sinkIdentifier, reservoir, checkpoint.windowId));
                    }
                    node.connectInputPort(nidi.portName, reservoir);
                    newStreams.put(sinkIdentifier, new ComponentContextPair<Stream, StreamContext>(subscriber, context));
                    logger.debug("put input stream {} against key {}", subscriber, sinkIdentifier);
                } else {
                    assert (nidi.locality == Locality.CONTAINER_LOCAL || nidi.locality == Locality.THREAD_LOCAL);
                    /* we are still dealing with the MuxStream originating at the output of the source port */
                    StreamContext inlineContext = new StreamContext(nidi.declaredStreamId);
                    inlineContext.setSourceId(sourceIdentifier);
                    inlineContext.setSinkId(sinkIdentifier);
                    Stream stream;
                    SweepableReservoir reservoir;
                    switch(nidi.locality) {
                        case CONTAINER_LOCAL:
                            int outputQueueCapacity = getOutputQueueCapacity(operatorList, nidi.sourceNodeId, nidi.sourcePortName);
                            if (outputQueueCapacity > queueCapacity) {
                                queueCapacity = outputQueueCapacity;
                            }
                            stream = new InlineStream(queueCapacity);
                            reservoir = ((InlineStream) stream).getReservoir();
                            if (checkpoint.windowId >= 0) {
                                node.connectInputPort(nidi.portName, new WindowIdActivatedReservoir(sinkIdentifier, reservoir, checkpoint.windowId));
                            }
                            break;
                        case THREAD_LOCAL:
                            stream = new OiOStream();
                            reservoir = ((OiOStream) stream).getReservoir();
                            ((OiOStream.OiOReservoir) reservoir).setControlSink(((OiONode) node).getControlSink(reservoir));
                            oioNodes.put(ndi.id, nidi.sourceNodeId);
                            break;
                        default:
                            throw new IllegalStateException("Locality can be either ContainerLocal or ThreadLocal");
                    }
                    node.connectInputPort(nidi.portName, reservoir);
                    newStreams.put(sinkIdentifier, new ComponentContextPair<>(stream, inlineContext));
                    if (!(pair.component instanceof Stream.MultiSinkCapableStream)) {
                        String originalSinkId = pair.context.getSinkId();
                        /* we come here only if we are trying to augment the dag */
                        StreamContext muxContext = new StreamContext(nidi.declaredStreamId);
                        muxContext.setSourceId(sourceIdentifier);
                        muxContext.setFinishedWindowId(checkpoint.windowId);
                        muxContext.setSinkId(originalSinkId);
                        MuxStream muxStream = new MuxStream();
                        muxStream.setSink(originalSinkId, pair.component);
                        streams.put(originalSinkId, pair);
                        Node<?> sourceNode = nodes.get(nidi.sourceNodeId);
                        sourceNode.connectOutputPort(nidi.sourcePortName, muxStream);
                        newStreams.put(sourceIdentifier, pair = new ComponentContextPair<Stream, StreamContext>(muxStream, muxContext));
                    }
                    /* here everything should be multisink capable */
                    if (streamCodec instanceof StreamCodecWrapperForPersistance) {
                        PartitionAwareSinkForPersistence pas;
                        if (nidi.partitionKeys == null) {
                            pas = new PartitionAwareSinkForPersistence((StreamCodecWrapperForPersistance<Object>) streamCodec, nidi.partitionMask, stream);
                        } else {
                            pas = new PartitionAwareSinkForPersistence((StreamCodecWrapperForPersistance<Object>) streamCodec, nidi.partitionKeys, nidi.partitionMask, stream);
                        }
                        ((Stream.MultiSinkCapableStream) pair.component).setSink(sinkIdentifier, pas);
                    } else if (nidi.partitionKeys == null || nidi.partitionKeys.isEmpty()) {
                        ((Stream.MultiSinkCapableStream) pair.component).setSink(sinkIdentifier, stream);
                    } else {
                        /*
               * generally speaking we do not have partitions on the inline streams so the control should not
               * come here but if it comes, then we are ready to handle it using the partition aware streams.
               */
                        PartitionAwareSink<Object> pas = new PartitionAwareSink<>(streamCodec == null ? nonSerializingStreamCodec : (StreamCodec<Object>) streamCodec, nidi.partitionKeys, nidi.partitionMask, stream);
                        ((Stream.MultiSinkCapableStream) pair.component).setSink(sinkIdentifier, pas);
                    }
                    String streamSinkId = pair.context.getSinkId();
                    if (streamSinkId == null) {
                        pair.context.setSinkId(sinkIdentifier);
                    } else {
                        pair.context.setSinkId(streamSinkId.concat(", ").concat(sinkIdentifier));
                    }
                }
            }
        }
    }
    setupOiOGroups(oioNodes);
    if (!inputNodes.isEmpty()) {
        WindowGenerator windowGenerator = setupWindowGenerator(smallestCheckpointedWindowId);
        for (OperatorDeployInfo ndi : inputNodes) {
            generators.put(ndi.id, windowGenerator);
            Node<?> node = nodes.get(ndi.id);
            SweepableReservoir reservoir = windowGenerator.acquireReservoir(String.valueOf(ndi.id), 1024);
            if (ndi.checkpoint.windowId >= 0) {
                node.connectInputPort(Node.INPUT, new WindowIdActivatedReservoir(Integer.toString(ndi.id), reservoir, ndi.checkpoint.windowId));
            }
            node.connectInputPort(Node.INPUT, reservoir);
        }
    }
}

Example 4 with StreamCodec

use of com.datatorrent.api.StreamCodec in project apex-core by apache.

the class StreamMapping method redoMapping.

/**
 * rebuild the tree, which may cause more changes to execution layer than need be
 * TODO: investigate incremental logic
 */
private void redoMapping() {
    Set<Pair<PTOperator, InputPortMeta>> downstreamOpers = Sets.newHashSet();
    // figure out the downstream consumers
    for (InputPortMeta ipm : streamMeta.getSinks()) {
        // skipped for parallel partitions - those are handled elsewhere
        if (!ipm.getValue(PortContext.PARTITION_PARALLEL) && plan.hasMapping(ipm.getOperatorMeta())) {
            List<PTOperator> partitions = plan.getOperators(ipm.getOperatorMeta());
            for (PTOperator doper : partitions) {
                downstreamOpers.add(new Pair<>(doper, ipm));
            }
        }
    }
    if (!downstreamOpers.isEmpty()) {
        // unifiers are required
        for (PTOperator unifier : this.cascadingUnifiers) {
            detachUnifier(unifier);
        }
        if (this.finalUnifier != null) {
            detachUnifier(finalUnifier);
        }
        List<PTOperator> currentUnifiers = Lists.newArrayList(this.cascadingUnifiers);
        this.cascadingUnifiers.clear();
        plan.undeployOpers.addAll(currentUnifiers);
        addSlidingUnifiers();
        int limit = streamMeta.getSource().getValue(PortContext.UNIFIER_LIMIT);
        boolean separateUnifiers = false;
        Integer lastId = null;
        for (InputPortMeta ipm : streamMeta.getSinks()) {
            Integer id = plan.getStreamCodecIdentifier(ipm.getStreamCodec());
            if (lastId == null) {
                lastId = id;
            } else if (!id.equals(lastId)) {
                separateUnifiers = true;
                break;
            }
        }
        List<PTOutput> unifierSources = this.upstream;
        Map<StreamCodec<?>, List<PTOutput>> cascadeUnifierSourcesMap = Maps.newHashMap();
        if (limit > 1 && this.upstream.size() > limit) {
            // cascading unifier
            if (!separateUnifiers) {
                unifierSources = setupCascadingUnifiers(this.upstream, currentUnifiers, limit, 0);
            } else {
                for (InputPortMeta ipm : streamMeta.getSinks()) {
                    StreamCodec<?> streamCodec = ipm.getStreamCodec();
                    if (!cascadeUnifierSourcesMap.containsKey(streamCodec)) {
                        unifierSources = setupCascadingUnifiers(this.upstream, currentUnifiers, limit, 0);
                        cascadeUnifierSourcesMap.put(streamCodec, unifierSources);
                    }
                }
            }
        }
        // remove remaining unifiers
        for (PTOperator oper : currentUnifiers) {
            plan.removePTOperator(oper);
        }
        // Directly getting attribute from map to know if it is set or not as it can be overriden by the input
        Boolean sourceSingleFinal = streamMeta.getSource().getAttributes().get(PortContext.UNIFIER_SINGLE_FINAL);
        // link the downstream operators with the unifiers
        for (Pair<PTOperator, InputPortMeta> doperEntry : downstreamOpers) {
            Map<LogicalPlan.InputPortMeta, PartitionKeys> partKeys = doperEntry.first.partitionKeys;
            PartitionKeys pks = partKeys != null ? partKeys.get(doperEntry.second) : null;
            Boolean sinkSingleFinal = doperEntry.second.getAttributes().get(PortContext.UNIFIER_SINGLE_FINAL);
            boolean lastSingle = (sinkSingleFinal != null) ? sinkSingleFinal : (sourceSingleFinal != null ? sourceSingleFinal.booleanValue() : PortContext.UNIFIER_SINGLE_FINAL.defaultValue);
            if (upstream.size() > 1) {
                // detach downstream from upstream operator for the case where no unifier existed previously
                for (PTOutput source : upstream) {
                    Iterator<PTInput> sinks = source.sinks.iterator();
                    while (sinks.hasNext()) {
                        PTInput sink = sinks.next();
                        if (sink.target == doperEntry.first) {
                            doperEntry.first.inputs.remove(sink);
                            sinks.remove();
                        }
                    }
                }
                if (!separateUnifiers && lastSingle) {
                    if (finalUnifier == null) {
                        finalUnifier = createUnifier(streamMeta, plan);
                    }
                    setInput(doperEntry.first, doperEntry.second, finalUnifier, (pks == null) || (pks.mask == 0) ? null : pks);
                    if (finalUnifier.inputs.isEmpty()) {
                        // set unifier inputs once, regardless how many downstream operators there are
                        for (PTOutput out : unifierSources) {
                            addInput(this.finalUnifier, out, null);
                        }
                    }
                } else {
                    // MxN partitioning: unifier per downstream partition
                    LOG.debug("MxN unifier for {} {} {}", new Object[] { doperEntry.first, doperEntry.second.getPortName(), pks });
                    PTOperator unifier = doperEntry.first.upstreamMerge.get(doperEntry.second);
                    if (unifier == null) {
                        unifier = createUnifier(streamMeta, plan);
                        doperEntry.first.upstreamMerge.put(doperEntry.second, unifier);
                        setInput(doperEntry.first, doperEntry.second, unifier, null);
                    }
                    // sources may change dynamically, rebuild inputs (as for cascading unifiers)
                    for (PTInput in : unifier.inputs) {
                        in.source.sinks.remove(in);
                    }
                    unifier.inputs.clear();
                    List<PTOutput> doperUnifierSources = unifierSources;
                    if (separateUnifiers) {
                        List<PTOutput> cascadeSources = cascadeUnifierSourcesMap.get(doperEntry.second.getStreamCodec());
                        if (cascadeSources != null) {
                            doperUnifierSources = cascadeSources;
                        }
                    }
                    // add new inputs
                    for (PTOutput out : doperUnifierSources) {
                        addInput(unifier, out, (pks == null) || (pks.mask == 0) ? null : pks);
                    }
                }
            } else {
                // no partitioning
                PTOperator unifier = doperEntry.first.upstreamMerge.remove(doperEntry.second);
                if (unifier != null) {
                    plan.removePTOperator(unifier);
                }
                setInput(doperEntry.first, doperEntry.second, upstream.get(0).source, pks);
            }
        }
        // 2) Downstream operators partitions are scaled up from one to multiple. (replaced by merged unifier)
        if (finalUnifier != null && finalUnifier.inputs.isEmpty()) {
            plan.removePTOperator(finalUnifier);
            finalUnifier = null;
        }
    }
}

Example 5 with StreamCodec

use of com.datatorrent.api.StreamCodec in project apex-core by apache.

the class PhysicalPlan method updatePersistOperatorStreamCodec.

private void updatePersistOperatorStreamCodec(LogicalPlan dag) {
    HashMap<StreamMeta, StreamCodec<?>> streamMetaToCodecMap = new HashMap<>();
    try {
        for (OperatorMeta n : dag.getAllOperators()) {
            for (StreamMeta s : n.getOutputStreams().values()) {
                if (s.getPersistOperator() != null) {
                    Map<InputPortMeta, StreamCodec<?>> inputStreamCodecs = new HashMap<>();
                    // Logging is enabled for the stream
                    for (InputPortMeta portMeta : s.getSinksToPersist()) {
                        StreamCodec<?> inputStreamCodec = portMeta.getStreamCodec();
                        if (inputStreamCodec != null) {
                            boolean alreadyAdded = false;
                            for (StreamCodec<?> codec : inputStreamCodecs.values()) {
                                if (inputStreamCodec.equals(codec)) {
                                    alreadyAdded = true;
                                    break;
                                }
                            }
                            if (!alreadyAdded) {
                                inputStreamCodecs.put(portMeta, inputStreamCodec);
                            }
                        }
                    }
                    if (inputStreamCodecs.isEmpty()) {
                    // Stream codec not specified
                    // So everything out of Source should be captured without any
                    // StreamCodec
                    // Do nothing
                    } else {
                        // Create Wrapper codec for Stream persistence using all unique
                        // stream codecs
                        // Logger should write merged or union of all input stream codecs
                        StreamCodec<?> specifiedCodecForLogger = s.getPersistOperatorInputPort().getStreamCodec();
                        @SuppressWarnings({ "unchecked", "rawtypes" }) StreamCodecWrapperForPersistance<Object> codec = new StreamCodecWrapperForPersistance(inputStreamCodecs, specifiedCodecForLogger);
                        streamMetaToCodecMap.put(s, codec);
                    }
                }
            }
        }
        for (java.util.Map.Entry<StreamMeta, StreamCodec<?>> entry : streamMetaToCodecMap.entrySet()) {
            dag.setInputPortAttribute(entry.getKey().getPersistOperatorInputPort().getPort(), PortContext.STREAM_CODEC, entry.getValue());
        }
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}