Examples with Sink com.datatorrent.api.Sink used on opensource projects

Example 1 with Sink

use of com.datatorrent.api.Sink in project apex-core by apache.

the class BufferServerSubscriberTest method testEmergencySinks.

@Test
public void testEmergencySinks() throws InterruptedException {
    final List<Object> list = new ArrayList<>();
    final StreamCodec<Object> myserde = new StreamCodec<Object>() {

        @Override
        public Object fromByteArray(Slice fragment) {
            if (fragment.offset == 0 && fragment.length == fragment.buffer.length) {
                return fragment.buffer;
            } else {
                return Arrays.copyOfRange(fragment.buffer, fragment.offset, fragment.offset + fragment.length);
            }
        }

        @Override
        public Slice toByteArray(Object o) {
            return new Slice((byte[]) o, 0, ((byte[]) o).length);
        }

        @Override
        public int getPartition(Object o) {
            return 0;
        }
    };
    Sink<Object> unbufferedSink = new Sink<Object>() {

        @Override
        public void put(Object tuple) {
            list.add(tuple);
        }

        @Override
        public int getCount(boolean reset) {
            return 0;
        }
    };
    BufferServerSubscriber bss = new BufferServerSubscriber("subscriber", 5) {

        {
            serde = myserde;
        }

        @Override
        public void suspendRead() {
            logger.debug("read suspended");
        }

        @Override
        public void resumeRead() {
            logger.debug("read resumed");
        }
    };
    SweepableReservoir reservoir = bss.acquireReservoir("unbufferedSink", 3);
    reservoir.setSink(unbufferedSink);
    int i = 0;
    while (i++ < 10) {
        Slice fragment = myserde.toByteArray(new byte[] { (byte) i });
        byte[] buffer = PayloadTuple.getSerializedTuple(myserde.getPartition(i), fragment);
        bss.onMessage(buffer, 0, buffer.length);
    }
    reservoir.sweep();
    /* 4 make it to the reservoir */
    reservoir.sweep();
    /* we consume the 4; and 4 more make it to the reservoir */
    Assert.assertEquals("4 received", 4, list.size());
    reservoir.sweep();
    /* 8 consumed + 2 more make it to the reservoir */
    reservoir.sweep();
    /* consume 2 more */
    Assert.assertEquals("10  received", 10, list.size());
}

Example 2 with Sink

use of com.datatorrent.api.Sink in project apex-core by apache.

the class SocketStreamTest method init.

@Before
public void init() {
    final StreamCodec<Object> serde = new DefaultStatefulStreamCodec<>();
    messageCount = new AtomicInteger(0);
    Sink<Object> sink = new Sink<Object>() {

        @Override
        public void put(Object tuple) {
            logger.debug("received: " + tuple);
            messageCount.incrementAndGet();
        }

        @Override
        public int getCount(boolean reset) {
            throw new UnsupportedOperationException("Not supported yet.");
        }
    };
    issContext = new StreamContext(streamName);
    issContext.setSourceId(upstreamNodeId);
    issContext.setSinkId(downstreamNodeId);
    issContext.setFinishedWindowId(-1);
    issContext.setBufferServerAddress(InetSocketAddress.createUnresolved("localhost", bufferServerPort));
    issContext.put(StreamContext.CODEC, serde);
    issContext.put(StreamContext.EVENT_LOOP, eventloop);
    iss = new BufferServerSubscriber(downstreamNodeId, 1024);
    iss.setup(issContext);
    reservoir = iss.acquireReservoir("testReservoir", 1);
    reservoir.setSink(sink);
    ossContext = new StreamContext(streamName);
    ossContext.setSourceId(upstreamNodeId);
    ossContext.setSinkId(downstreamNodeId);
    ossContext.setBufferServerAddress(InetSocketAddress.createUnresolved("localhost", bufferServerPort));
    ossContext.put(StreamContext.CODEC, serde);
    ossContext.put(StreamContext.EVENT_LOOP, eventloop);
    oss = new BufferServerPublisher(upstreamNodeId, 1024);
    oss.setup(ossContext);
}

Example 3 with Sink

use of com.datatorrent.api.Sink in project apex-core by apache.

the class GenericNode method addSinks.

@Override
@SuppressWarnings("unchecked")
public void addSinks(Map<String, Sink<Object>> sinks) {
    for (Entry<String, Sink<Object>> e : sinks.entrySet()) {
        SweepableReservoir original = inputs.get(e.getKey());
        if (original instanceof TappedReservoir) {
            TappedReservoir tr = (TappedReservoir) original;
            tr.add(e.getValue());
        } else if (original != null) {
            TappedReservoir tr = new TappedReservoir(original, e.getValue());
            inputs.put(e.getKey(), tr);
        }
    }
    super.addSinks(sinks);
}

Example 4 with Sink

use of com.datatorrent.api.Sink in project apex-core by apache.

the class GenericNode method removeSinks.

@Override
public void removeSinks(Map<String, Sink<Object>> sinks) {
    for (Entry<String, Sink<Object>> e : sinks.entrySet()) {
        SweepableReservoir reservoir = inputs.get(e.getKey());
        if (reservoir instanceof TappedReservoir) {
            TappedReservoir tr = (TappedReservoir) reservoir;
            tr.remove(e.getValue());
            if (tr.getSinks().length == 0) {
                tr.reservoir.setSink(tr.setSink(null));
                inputs.put(e.getKey(), tr.reservoir);
            }
        }
    }
    super.removeSinks(sinks);
}

Example 5 with Sink

use of com.datatorrent.api.Sink in project apex-core by apache.

the class GenericNode method run.

/**
 * Originally this method was defined in an attempt to implement the interface Runnable.
 *
 * Note that activate does not return as long as there is useful workload for the node.
 */
@Override
@SuppressWarnings({ "SleepWhileInLoop", "UseSpecificCatch", "BroadCatchBlock", "TooBroadCatch" })
public final void run() {
    doCheckpoint = false;
    final long maxSpinMillis = context.getValue(OperatorContext.SPIN_MILLIS);
    long spinMillis = 0;
    final boolean handleIdleTime = operator instanceof IdleTimeHandler;
    int totalQueues = inputs.size();
    int regularQueues = totalQueues;
    // regularQueues is the number of queues that are not connected to a DelayOperator
    for (String portName : inputs.keySet()) {
        if (isInputPortConnectedToDelayOperator(portName)) {
            regularQueues--;
        }
    }
    ArrayList<Map.Entry<String, SweepableReservoir>> activeQueues = new ArrayList<>();
    activeQueues.addAll(inputs.entrySet());
    int expectingBeginWindow = activeQueues.size();
    int receivedEndWindow = 0;
    long firstWindowId = -1;
    calculateNextCheckpointWindow();
    TupleTracker tracker;
    LinkedList<TupleTracker> resetTupleTracker = new LinkedList<>();
    Map<SweepableReservoir, LinkedHashSet<CustomControlTuple>> immediateDeliveryTuples = Maps.newHashMap();
    Map<SweepableReservoir, LinkedHashSet<CustomControlTuple>> endWindowDeliveryTuples = Maps.newHashMap();
    try {
        do {
            Iterator<Map.Entry<String, SweepableReservoir>> buffers = activeQueues.iterator();
            activequeue: while (buffers.hasNext()) {
                Map.Entry<String, SweepableReservoir> activePortEntry = buffers.next();
                SweepableReservoir activePort = activePortEntry.getValue();
                Tuple t = activePort.sweep();
                if (t != null) {
                    spinMillis = 0;
                    boolean delay = (operator instanceof Operator.DelayOperator);
                    long windowAhead = 0;
                    if (delay) {
                        windowAhead = WindowGenerator.getAheadWindowId(t.getWindowId(), firstWindowMillis, windowWidthMillis, 1);
                    }
                    switch(t.getType()) {
                        case BEGIN_WINDOW:
                            if (expectingBeginWindow == totalQueues) {
                                // This is the first begin window tuple among all ports
                                if (isInputPortConnectedToDelayOperator(activePortEntry.getKey())) {
                                    // In the future, this condition will not be needed if we get rid of the CHECKPOINT tuple.
                                    continue;
                                }
                                activePort.remove();
                                expectingBeginWindow--;
                                receivedEndWindow = 0;
                                currentWindowId = t.getWindowId();
                                if (delay) {
                                    if (WindowGenerator.getBaseSecondsFromWindowId(windowAhead) > t.getBaseSeconds()) {
                                        // Buffer server code strips out the base seconds from BEGIN_WINDOW and END_WINDOW tuples for
                                        // serialization optimization.  That's why we need a reset window here to tell the buffer
                                        // server we are having a new baseSeconds now.
                                        Tuple resetWindowTuple = new ResetWindowTuple(windowAhead);
                                        for (int s = sinks.length; s-- > 0; ) {
                                            sinks[s].put(resetWindowTuple);
                                        }
                                    }
                                    controlTupleCount++;
                                    t.setWindowId(windowAhead);
                                }
                                for (int s = sinks.length; s-- > 0; ) {
                                    sinks[s].put(t);
                                }
                                controlTupleCount++;
                                context.setWindowsFromCheckpoint(nextCheckpointWindowCount--);
                                if (applicationWindowCount == 0) {
                                    insideWindow = true;
                                    operator.beginWindow(currentWindowId);
                                }
                            } else if (t.getWindowId() == currentWindowId) {
                                activePort.remove();
                                expectingBeginWindow--;
                            } else {
                                buffers.remove();
                                String port = activePortEntry.getKey();
                                if (PROCESSING_MODE == ProcessingMode.AT_MOST_ONCE) {
                                    if (t.getWindowId() < currentWindowId) {
                                        /*
                       * we need to fast forward this stream till we find the current
                       * window or the window which is bigger than the current window.
                       */
                                        /* lets move the current reservoir in the background */
                                        Sink<Object> sink = activePort.setSink(Sink.BLACKHOLE);
                                        deferredInputConnections.add(0, new DeferredInputConnection(port, activePort));
                                        /* replace it with the reservoir which blocks the tuples in the past */
                                        WindowIdActivatedReservoir wiar = new WindowIdActivatedReservoir(port, activePort, currentWindowId);
                                        wiar.setSink(sink);
                                        inputs.put(port, wiar);
                                        activeQueues.add(new AbstractMap.SimpleEntry<String, SweepableReservoir>(port, wiar));
                                        break activequeue;
                                    } else {
                                        expectingBeginWindow--;
                                        if (++receivedEndWindow == totalQueues) {
                                            processEndWindow(null);
                                            activeQueues.addAll(inputs.entrySet());
                                            expectingBeginWindow = activeQueues.size();
                                            break activequeue;
                                        }
                                    }
                                } else {
                                    logger.error("Catastrophic Error: Out of sequence {} tuple {} on port {} while expecting {}", t.getType(), Codec.getStringWindowId(t.getWindowId()), port, Codec.getStringWindowId(currentWindowId));
                                    System.exit(2);
                                }
                            }
                            break;
                        case END_WINDOW:
                            buffers.remove();
                            if (t.getWindowId() == currentWindowId) {
                                activePort.remove();
                                endWindowDequeueTimes.put(activePort, System.currentTimeMillis());
                                if (++receivedEndWindow == totalQueues) {
                                    assert (activeQueues.isEmpty());
                                    if (delay) {
                                        t.setWindowId(windowAhead);
                                    }
                                    /* Emit control tuples here */
                                    if (reservoirPortMap.isEmpty()) {
                                        populateReservoirInputPortMap();
                                    }
                                    for (Entry<SweepableReservoir, LinkedHashSet<CustomControlTuple>> portSet : endWindowDeliveryTuples.entrySet()) {
                                        Sink activeSink = reservoirPortMap.get(portSet.getKey());
                                        // activeSink may not be null
                                        if (activeSink instanceof ControlAwareDefaultInputPort) {
                                            ControlTupleEnabledSink sink = (ControlTupleEnabledSink) activeSink;
                                            for (CustomControlTuple cct : portSet.getValue()) {
                                                if (!sink.putControl((ControlTuple) cct.getUserObject())) {
                                                    // operator cannot handle control tuple; forward to sinks
                                                    forwardToSinks(delay, cct);
                                                }
                                            }
                                        } else {
                                            // Not a ControlAwarePort. Operator cannot handle a custom control tuple.
                                            for (CustomControlTuple cct : portSet.getValue()) {
                                                forwardToSinks(delay, cct);
                                            }
                                        }
                                    }
                                    immediateDeliveryTuples.clear();
                                    endWindowDeliveryTuples.clear();
                                    /* Now call endWindow() */
                                    processEndWindow(t);
                                    activeQueues.addAll(inputs.entrySet());
                                    expectingBeginWindow = activeQueues.size();
                                    break activequeue;
                                }
                            }
                            break;
                        case CUSTOM_CONTROL:
                            activePort.remove();
                            /* All custom control tuples are expected to be arriving in the current window only.*/
                            /* Buffer control tuples until end of the window */
                            CustomControlTuple cct = (CustomControlTuple) t;
                            ControlTuple udct = (ControlTuple) cct.getUserObject();
                            boolean forward = false;
                            // Handle Immediate Delivery Control Tuples
                            if (udct.getDeliveryType().equals(ControlTuple.DeliveryType.IMMEDIATE)) {
                                if (!isDuplicate(immediateDeliveryTuples.get(activePort), cct)) {
                                    // Forward immediately
                                    if (reservoirPortMap.isEmpty()) {
                                        populateReservoirInputPortMap();
                                    }
                                    Sink activeSink = reservoirPortMap.get(activePort);
                                    // activeSink may not be null
                                    if (activeSink instanceof ControlAwareDefaultInputPort) {
                                        ControlTupleEnabledSink sink = (ControlTupleEnabledSink) activeSink;
                                        if (!sink.putControl((ControlTuple) cct.getUserObject())) {
                                            forward = true;
                                        }
                                    } else {
                                        forward = true;
                                    }
                                    if (forward) {
                                        forwardToSinks(delay, cct);
                                    }
                                    // Add to set
                                    if (!immediateDeliveryTuples.containsKey(activePort)) {
                                        immediateDeliveryTuples.put(activePort, new LinkedHashSet<CustomControlTuple>());
                                    }
                                    immediateDeliveryTuples.get(activePort).add(cct);
                                }
                            } else {
                                // Buffer EndWindow Delivery Control Tuples
                                if (!endWindowDeliveryTuples.containsKey(activePort)) {
                                    endWindowDeliveryTuples.put(activePort, new LinkedHashSet<CustomControlTuple>());
                                }
                                if (!isDuplicate(endWindowDeliveryTuples.get(activePort), cct)) {
                                    endWindowDeliveryTuples.get(activePort).add(cct);
                                }
                            }
                            break;
                        case CHECKPOINT:
                            activePort.remove();
                            long checkpointWindow = t.getWindowId();
                            if (lastCheckpointWindowId < checkpointWindow) {
                                dagCheckpointOffsetCount = 0;
                                if (PROCESSING_MODE == ProcessingMode.EXACTLY_ONCE) {
                                    lastCheckpointWindowId = checkpointWindow;
                                } else if (!doCheckpoint) {
                                    if (checkpointWindowCount == 0) {
                                        checkpoint(checkpointWindow);
                                        lastCheckpointWindowId = checkpointWindow;
                                    } else {
                                        doCheckpoint = true;
                                    }
                                }
                                if (!delay) {
                                    for (int s = sinks.length; s-- > 0; ) {
                                        sinks[s].put(t);
                                    }
                                    controlTupleCount++;
                                }
                            }
                            break;
                        case RESET_WINDOW:
                            /**
                             * we will receive tuples which are equal to the number of input streams.
                             */
                            activePort.remove();
                            if (isInputPortConnectedToDelayOperator(activePortEntry.getKey())) {
                                // breaking out of the switch/case
                                break;
                            }
                            buffers.remove();
                            int baseSeconds = t.getBaseSeconds();
                            tracker = null;
                            for (Iterator<TupleTracker> trackerIterator = resetTupleTracker.iterator(); trackerIterator.hasNext(); ) {
                                tracker = trackerIterator.next();
                                if (tracker.tuple.getBaseSeconds() == baseSeconds) {
                                    break;
                                }
                            }
                            if (tracker == null) {
                                tracker = new TupleTracker(t, regularQueues);
                                resetTupleTracker.add(tracker);
                            }
                            int trackerIndex = 0;
                            while (trackerIndex < tracker.ports.length) {
                                if (tracker.ports[trackerIndex] == null) {
                                    tracker.ports[trackerIndex++] = activePort;
                                    break;
                                } else if (tracker.ports[trackerIndex] == activePort) {
                                    break;
                                }
                                trackerIndex++;
                            }
                            if (trackerIndex == regularQueues) {
                                Iterator<TupleTracker> trackerIterator = resetTupleTracker.iterator();
                                while (trackerIterator.hasNext()) {
                                    if (trackerIterator.next().tuple.getBaseSeconds() <= baseSeconds) {
                                        trackerIterator.remove();
                                    }
                                }
                                if (!delay) {
                                    for (int s = sinks.length; s-- > 0; ) {
                                        sinks[s].put(t);
                                    }
                                    controlTupleCount++;
                                }
                                if (!activeQueues.isEmpty()) {
                                    // make sure they are all queues from DelayOperator
                                    for (Map.Entry<String, SweepableReservoir> entry : activeQueues) {
                                        if (!isInputPortConnectedToDelayOperator(entry.getKey())) {
                                            assert (false);
                                        }
                                    }
                                    activeQueues.clear();
                                }
                                activeQueues.addAll(inputs.entrySet());
                                expectingBeginWindow = activeQueues.size();
                                if (firstWindowId == -1) {
                                    if (delay) {
                                        for (int s = sinks.length; s-- > 0; ) {
                                            sinks[s].put(t);
                                        }
                                        controlTupleCount++;
                                        // if it's a DelayOperator and this is the first RESET_WINDOW (start) or END_STREAM
                                        // (recovery), fabricate the first window
                                        fabricateFirstWindow((Operator.DelayOperator) operator, windowAhead);
                                    }
                                    firstWindowId = t.getWindowId();
                                }
                                break activequeue;
                            }
                            break;
                        case END_STREAM:
                            activePort.remove();
                            buffers.remove();
                            if (firstWindowId == -1) {
                                // this is for recovery from a checkpoint for DelayOperator
                                if (delay) {
                                    // if it's a DelayOperator and this is the first RESET_WINDOW (start) or END_STREAM (recovery),
                                    // fabricate the first window
                                    fabricateFirstWindow((Operator.DelayOperator) operator, windowAhead);
                                }
                                firstWindowId = t.getWindowId();
                            }
                            for (Iterator<Entry<String, SweepableReservoir>> it = inputs.entrySet().iterator(); it.hasNext(); ) {
                                Entry<String, SweepableReservoir> e = it.next();
                                if (e.getValue() == activePort) {
                                    if (!descriptor.inputPorts.isEmpty()) {
                                        descriptor.inputPorts.get(e.getKey()).component.setConnected(false);
                                    }
                                    it.remove();
                                    /* check the deferred connection list for any new port that should be connected here */
                                    Iterator<DeferredInputConnection> dici = deferredInputConnections.iterator();
                                    while (dici.hasNext()) {
                                        DeferredInputConnection dic = dici.next();
                                        if (e.getKey().equals(dic.portname)) {
                                            connectInputPort(dic.portname, dic.reservoir);
                                            dici.remove();
                                            activeQueues.add(new AbstractMap.SimpleEntry<>(dic.portname, dic.reservoir));
                                            break activequeue;
                                        }
                                    }
                                    break;
                                }
                            }
                            /**
                             * We are not going to receive begin window on this ever!
                             */
                            expectingBeginWindow--;
                            /**
                             * Since one of the operators we care about it gone, we should relook at our ports.
                             * We need to make sure that the END_STREAM comes outside of the window.
                             */
                            regularQueues--;
                            totalQueues--;
                            boolean break_activequeue = false;
                            if (regularQueues == 0) {
                                alive = false;
                                break_activequeue = true;
                            } else if (activeQueues.isEmpty()) {
                                assert (!inputs.isEmpty());
                                processEndWindow(null);
                                activeQueues.addAll(inputs.entrySet());
                                expectingBeginWindow = activeQueues.size();
                                break_activequeue = true;
                            }
                            /**
                             * also make sure that we update the reset tuple tracker if this stream had delivered any reset tuples.
                             * Check all the reset buffers to see if current input port has already delivered reset tuple. If it has
                             * then we are waiting for something else to deliver the reset tuple, so just clear current reservoir
                             * from the list of tracked reservoirs. If the current input port has not delivered the reset tuple, and
                             * it's the only one which has not, then we consider it delivered and release the reset tuple downstream.
                             */
                            Tuple tuple = null;
                            for (Iterator<TupleTracker> trackerIterator = resetTupleTracker.iterator(); trackerIterator.hasNext(); ) {
                                tracker = trackerIterator.next();
                                trackerIndex = 0;
                                while (trackerIndex < tracker.ports.length) {
                                    if (tracker.ports[trackerIndex] == activePort) {
                                        SweepableReservoir[] ports = new SweepableReservoir[regularQueues];
                                        System.arraycopy(tracker.ports, 0, ports, 0, trackerIndex);
                                        if (trackerIndex < regularQueues) {
                                            System.arraycopy(tracker.ports, trackerIndex + 1, ports, trackerIndex, tracker.ports.length - trackerIndex - 1);
                                        }
                                        tracker.ports = ports;
                                        break;
                                    } else if (tracker.ports[trackerIndex] == null) {
                                        if (trackerIndex == regularQueues) {
                                            /* regularQueues is already adjusted above */
                                            if (tuple == null || tuple.getBaseSeconds() < tracker.tuple.getBaseSeconds()) {
                                                tuple = tracker.tuple;
                                            }
                                            trackerIterator.remove();
                                        }
                                        break;
                                    } else {
                                        tracker.ports = Arrays.copyOf(tracker.ports, regularQueues);
                                    }
                                    trackerIndex++;
                                }
                            }
                            /*
                 * Since we were waiting for a reset tuple on this stream, we should not any longer.
                 */
                            if (tuple != null && !delay) {
                                for (int s = sinks.length; s-- > 0; ) {
                                    sinks[s].put(tuple);
                                }
                                controlTupleCount++;
                            }
                            if (break_activequeue) {
                                break activequeue;
                            }
                            break;
                        default:
                            throw new UnhandledException("Unrecognized Control Tuple", new IllegalArgumentException(t.toString()));
                    }
                }
            }
            if (activeQueues.isEmpty() && alive) {
                logger.error("Catastrophic Error: Invalid State - the operator blocked forever!");
                System.exit(2);
            } else {
                boolean need2sleep = true;
                for (Map.Entry<String, SweepableReservoir> cb : activeQueues) {
                    need2sleep = cb.getValue().isEmpty();
                    if (!need2sleep) {
                        spinMillis = 0;
                        break;
                    }
                }
                if (need2sleep) {
                    if (handleIdleTime && insideWindow) {
                        ((IdleTimeHandler) operator).handleIdleTime();
                    } else {
                        Thread.sleep(spinMillis);
                        spinMillis = Math.min(maxSpinMillis, spinMillis + 1);
                    }
                }
            }
        } while (alive);
    } catch (ShutdownException se) {
        logger.debug("Shutdown requested by the operator when alive = {}.", alive);
        alive = false;
    } catch (Throwable cause) {
        synchronized (this) {
            if (alive) {
                throw Throwables.propagate(cause);
            }
        }
        Throwable rootCause = cause;
        while (rootCause != null) {
            if (rootCause instanceof InterruptedException) {
                break;
            }
            rootCause = rootCause.getCause();
        }
        if (rootCause == null) {
            throw Throwables.propagate(cause);
        } else {
            logger.debug("Ignoring InterruptedException after shutdown", cause);
        }
    }
    /**
     * TODO: If shutdown and inside window provide alternate way of notifying the operator in such ways
     * TODO: as using a listener callback
     */
    if (insideWindow && !shutdown) {
        operator.endWindow();
        endWindowEmitTime = System.currentTimeMillis();
        if (++applicationWindowCount == APPLICATION_WINDOW_COUNT) {
            applicationWindowCount = 0;
        }
        if (++checkpointWindowCount == CHECKPOINT_WINDOW_COUNT) {
            checkpointWindowCount = 0;
            if (doCheckpoint || PROCESSING_MODE == ProcessingMode.EXACTLY_ONCE) {
                checkpoint(currentWindowId);
            }
        }
        ContainerStats.OperatorStats stats = new ContainerStats.OperatorStats();
        fixEndWindowDequeueTimesBeforeDeactivate();
        reportStats(stats, currentWindowId);
        stats.metrics = collectMetrics();
        handleRequests(currentWindowId);
    }
}