Examples with Tuple com.datatorrent.stram.tuple.Tuple used on opensource projects

Example 6 with Tuple

use of com.datatorrent.stram.tuple.Tuple in project apex-core by apache.

the class GenericNodeTest method testBufferServerSubscriberActivationBeforeOperator.

@Test
public void testBufferServerSubscriberActivationBeforeOperator() throws InterruptedException, IOException {
    final String streamName = "streamName";
    final String upstreamNodeId = "upstreamNodeId";
    final String downstreamNodeId = "downStreamNodeId";
    EventLoop eventloop = DefaultEventLoop.createEventLoop("StreamTestEventLoop");
    ((DefaultEventLoop) eventloop).start();
    // find random port
    final Server bufferServer = new Server(eventloop, 0);
    final int bufferServerPort = bufferServer.run().getPort();
    final StreamCodec<Object> serde = new DefaultStatefulStreamCodec<>();
    final BlockingQueue<Object> tuples = new ArrayBlockingQueue<>(10);
    GenericTestOperator go = new GenericTestOperator();
    final GenericNode gn = new GenericNode(go, new com.datatorrent.stram.engine.OperatorContext(0, "operator", new DefaultAttributeMap(), null));
    gn.setId(1);
    Sink<Object> output = new Sink<Object>() {

        @Override
        public void put(Object tuple) {
            tuples.add(tuple);
        }

        @Override
        public int getCount(boolean reset) {
            return 0;
        }
    };
    InetSocketAddress socketAddress = new InetSocketAddress("localhost", bufferServerPort);
    StreamContext issContext = new StreamContext(streamName);
    issContext.setSourceId(upstreamNodeId);
    issContext.setSinkId(downstreamNodeId);
    issContext.setFinishedWindowId(-1);
    issContext.setBufferServerAddress(socketAddress);
    issContext.put(StreamContext.CODEC, serde);
    issContext.put(StreamContext.EVENT_LOOP, eventloop);
    StreamContext ossContext = new StreamContext(streamName);
    ossContext.setSourceId(upstreamNodeId);
    ossContext.setSinkId(downstreamNodeId);
    ossContext.setBufferServerAddress(socketAddress);
    ossContext.put(StreamContext.CODEC, serde);
    ossContext.put(StreamContext.EVENT_LOOP, eventloop);
    BufferServerPublisher oss = new BufferServerPublisher(upstreamNodeId, 1024);
    oss.setup(ossContext);
    oss.activate(ossContext);
    oss.put(new Tuple(MessageType.BEGIN_WINDOW, 0x1L));
    byte[] buff = PayloadTuple.getSerializedTuple(0, 1);
    buff[buff.length - 1] = (byte) 1;
    oss.put(buff);
    oss.put(new EndWindowTuple(0x1L));
    oss.put(new Tuple(MessageType.BEGIN_WINDOW, 0x2L));
    buff = PayloadTuple.getSerializedTuple(0, 1);
    buff[buff.length - 1] = (byte) 2;
    oss.put(buff);
    oss.put(new EndWindowTuple(0x2L));
    oss.put(new Tuple(MessageType.BEGIN_WINDOW, 0x3L));
    buff = PayloadTuple.getSerializedTuple(0, 1);
    buff[buff.length - 1] = (byte) 3;
    oss.put(buff);
    oss.put(new EndWindowTuple(0x3L));
    oss.put(new EndStreamTuple(0L));
    BufferServerSubscriber iss = new BufferServerSubscriber(downstreamNodeId, 1024);
    iss.setup(issContext);
    gn.connectInputPort(GenericTestOperator.IPORT1, iss.acquireReservoir("testReservoir", 10));
    gn.connectOutputPort(GenericTestOperator.OPORT1, output);
    SweepableReservoir tupleWait = iss.acquireReservoir("testReservoir2", 10);
    iss.activate(issContext);
    while (tupleWait.sweep() == null) {
        Thread.sleep(100);
    }
    gn.firstWindowMillis = 0;
    gn.windowWidthMillis = 100;
    Thread t = new Thread() {

        @Override
        public void run() {
            gn.activate();
            gn.run();
            gn.deactivate();
        }
    };
    t.start();
    t.join();
    Assert.assertEquals(10, tuples.size());
    List<Object> list = new ArrayList<>(tuples);
    Assert.assertEquals("Payload Tuple 1", 1, ((byte[]) list.get(1))[5]);
    Assert.assertEquals("Payload Tuple 2", 2, ((byte[]) list.get(4))[5]);
    Assert.assertEquals("Payload Tuple 3", 3, ((byte[]) list.get(7))[5]);
    if (bufferServer != null) {
        bufferServer.stop();
    }
    ((DefaultEventLoop) eventloop).stop();
}

Example 7 with Tuple

use of com.datatorrent.stram.tuple.Tuple in project apex-core by apache.

the class BufferServerPublisher method put.

/**
   *
   * @param payload
   */
@Override
@SuppressWarnings("SleepWhileInLoop")
public void put(Object payload) {
    count++;
    byte[] array;
    if (payload instanceof Tuple) {
        final Tuple t = (Tuple) payload;
        switch(t.getType()) {
            case CHECKPOINT:
                if (statefulSerde != null) {
                    statefulSerde.resetState();
                }
                array = WindowIdTuple.getSerializedTuple((int) t.getWindowId());
                array[0] = MessageType.CHECKPOINT_VALUE;
                break;
            case BEGIN_WINDOW:
                array = BeginWindowTuple.getSerializedTuple((int) t.getWindowId());
                break;
            case END_WINDOW:
                array = EndWindowTuple.getSerializedTuple((int) t.getWindowId());
                break;
            case CUSTOM_CONTROL:
                if (statefulSerde == null) {
                    array = com.datatorrent.bufferserver.packet.CustomControlTuple.getSerializedTuple(MessageType.CUSTOM_CONTROL_VALUE, serde.toByteArray(payload));
                } else {
                    DataStatePair dsp = statefulSerde.toDataStatePair(payload);
                    if (dsp.state != null) {
                        array = com.datatorrent.bufferserver.packet.CustomControlTuple.getSerializedTuple(MessageType.CODEC_STATE_VALUE, dsp.state);
                        try {
                            while (!write(array)) {
                                sleep(5);
                            }
                        } catch (InterruptedException ie) {
                            throw new RuntimeException(ie);
                        }
                    }
                    array = com.datatorrent.bufferserver.packet.CustomControlTuple.getSerializedTuple(MessageType.CUSTOM_CONTROL_VALUE, dsp.data);
                }
                break;
            case END_STREAM:
                array = EndStreamTuple.getSerializedTuple((int) t.getWindowId());
                break;
            case RESET_WINDOW:
                com.datatorrent.stram.tuple.ResetWindowTuple rwt = (com.datatorrent.stram.tuple.ResetWindowTuple) t;
                array = ResetWindowTuple.getSerializedTuple(rwt.getBaseSeconds(), rwt.getIntervalMillis());
                break;
            default:
                throw new UnsupportedOperationException("this data type is not handled in the stream");
        }
    } else {
        if (statefulSerde == null) {
            array = PayloadTuple.getSerializedTuple(serde.getPartition(payload), serde.toByteArray(payload));
        } else {
            DataStatePair dsp = statefulSerde.toDataStatePair(payload);
            /*
         * if there is any state write that for the subscriber before we write the data.
         */
            if (dsp.state != null) {
                array = DataTuple.getSerializedTuple(MessageType.CODEC_STATE_VALUE, dsp.state);
                try {
                    while (!write(array)) {
                        sleep(5);
                    }
                } catch (InterruptedException ie) {
                    throw new RuntimeException(ie);
                }
            }
            /*
         * Now that the state if any has been sent, we can proceed with the actual data we want to send.
         */
            array = PayloadTuple.getSerializedTuple(statefulSerde.getPartition(payload), dsp.data);
        }
    }
    try {
        while (!write(array)) {
            sleep(5);
        }
        publishedByteCount.addAndGet(array.length);
    } catch (InterruptedException ie) {
        throw new RuntimeException(ie);
    }
}

Example 8 with Tuple

use of com.datatorrent.stram.tuple.Tuple in project apex-core by apache.

the class GenericNode method run.

/**
   * Originally this method was defined in an attempt to implement the interface Runnable.
   *
   * Note that activate does not return as long as there is useful workload for the node.
   */
@Override
@SuppressWarnings({ "SleepWhileInLoop", "UseSpecificCatch", "BroadCatchBlock", "TooBroadCatch" })
public final void run() {
    doCheckpoint = false;
    final long maxSpinMillis = context.getValue(OperatorContext.SPIN_MILLIS);
    long spinMillis = 0;
    final boolean handleIdleTime = operator instanceof IdleTimeHandler;
    int totalQueues = inputs.size();
    int regularQueues = totalQueues;
    // regularQueues is the number of queues that are not connected to a DelayOperator
    for (String portName : inputs.keySet()) {
        if (isInputPortConnectedToDelayOperator(portName)) {
            regularQueues--;
        }
    }
    ArrayList<Map.Entry<String, SweepableReservoir>> activeQueues = new ArrayList<>();
    activeQueues.addAll(inputs.entrySet());
    int expectingBeginWindow = activeQueues.size();
    int receivedEndWindow = 0;
    long firstWindowId = -1;
    calculateNextCheckpointWindow();
    TupleTracker tracker;
    LinkedList<TupleTracker> resetTupleTracker = new LinkedList<>();
    Map<SweepableReservoir, LinkedHashSet<CustomControlTuple>> immediateDeliveryTuples = Maps.newHashMap();
    Map<SweepableReservoir, LinkedHashSet<CustomControlTuple>> endWindowDeliveryTuples = Maps.newHashMap();
    try {
        do {
            Iterator<Map.Entry<String, SweepableReservoir>> buffers = activeQueues.iterator();
            activequeue: while (buffers.hasNext()) {
                Map.Entry<String, SweepableReservoir> activePortEntry = buffers.next();
                SweepableReservoir activePort = activePortEntry.getValue();
                Tuple t = activePort.sweep();
                if (t != null) {
                    spinMillis = 0;
                    boolean delay = (operator instanceof Operator.DelayOperator);
                    long windowAhead = 0;
                    if (delay) {
                        windowAhead = WindowGenerator.getAheadWindowId(t.getWindowId(), firstWindowMillis, windowWidthMillis, 1);
                    }
                    switch(t.getType()) {
                        case BEGIN_WINDOW:
                            if (expectingBeginWindow == totalQueues) {
                                // This is the first begin window tuple among all ports
                                if (isInputPortConnectedToDelayOperator(activePortEntry.getKey())) {
                                    // In the future, this condition will not be needed if we get rid of the CHECKPOINT tuple.
                                    continue;
                                }
                                activePort.remove();
                                expectingBeginWindow--;
                                receivedEndWindow = 0;
                                currentWindowId = t.getWindowId();
                                if (delay) {
                                    if (WindowGenerator.getBaseSecondsFromWindowId(windowAhead) > t.getBaseSeconds()) {
                                        // Buffer server code strips out the base seconds from BEGIN_WINDOW and END_WINDOW tuples for
                                        // serialization optimization.  That's why we need a reset window here to tell the buffer
                                        // server we are having a new baseSeconds now.
                                        Tuple resetWindowTuple = new ResetWindowTuple(windowAhead);
                                        for (int s = sinks.length; s-- > 0; ) {
                                            sinks[s].put(resetWindowTuple);
                                        }
                                    }
                                    controlTupleCount++;
                                    t.setWindowId(windowAhead);
                                }
                                for (int s = sinks.length; s-- > 0; ) {
                                    sinks[s].put(t);
                                }
                                controlTupleCount++;
                                context.setWindowsFromCheckpoint(nextCheckpointWindowCount--);
                                if (applicationWindowCount == 0) {
                                    insideWindow = true;
                                    operator.beginWindow(currentWindowId);
                                }
                            } else if (t.getWindowId() == currentWindowId) {
                                activePort.remove();
                                expectingBeginWindow--;
                            } else {
                                buffers.remove();
                                String port = activePortEntry.getKey();
                                if (PROCESSING_MODE == ProcessingMode.AT_MOST_ONCE) {
                                    if (t.getWindowId() < currentWindowId) {
                                        /*
                       * we need to fast forward this stream till we find the current
                       * window or the window which is bigger than the current window.
                       */
                                        /* lets move the current reservoir in the background */
                                        Sink<Object> sink = activePort.setSink(Sink.BLACKHOLE);
                                        deferredInputConnections.add(0, new DeferredInputConnection(port, activePort));
                                        /* replace it with the reservoir which blocks the tuples in the past */
                                        WindowIdActivatedReservoir wiar = new WindowIdActivatedReservoir(port, activePort, currentWindowId);
                                        wiar.setSink(sink);
                                        inputs.put(port, wiar);
                                        activeQueues.add(new AbstractMap.SimpleEntry<String, SweepableReservoir>(port, wiar));
                                        break activequeue;
                                    } else {
                                        expectingBeginWindow--;
                                        if (++receivedEndWindow == totalQueues) {
                                            processEndWindow(null);
                                            activeQueues.addAll(inputs.entrySet());
                                            expectingBeginWindow = activeQueues.size();
                                            break activequeue;
                                        }
                                    }
                                } else {
                                    logger.error("Catastrophic Error: Out of sequence {} tuple {} on port {} while expecting {}", t.getType(), Codec.getStringWindowId(t.getWindowId()), port, Codec.getStringWindowId(currentWindowId));
                                    System.exit(2);
                                }
                            }
                            break;
                        case END_WINDOW:
                            buffers.remove();
                            if (t.getWindowId() == currentWindowId) {
                                activePort.remove();
                                endWindowDequeueTimes.put(activePort, System.currentTimeMillis());
                                if (++receivedEndWindow == totalQueues) {
                                    assert (activeQueues.isEmpty());
                                    if (delay) {
                                        t.setWindowId(windowAhead);
                                    }
                                    /* Emit control tuples here */
                                    if (reservoirPortMap.isEmpty()) {
                                        populateReservoirInputPortMap();
                                    }
                                    for (Entry<SweepableReservoir, LinkedHashSet<CustomControlTuple>> portSet : endWindowDeliveryTuples.entrySet()) {
                                        Sink activeSink = reservoirPortMap.get(portSet.getKey());
                                        // activeSink may not be null
                                        if (activeSink instanceof ControlAwareDefaultInputPort) {
                                            ControlTupleEnabledSink sink = (ControlTupleEnabledSink) activeSink;
                                            for (CustomControlTuple cct : portSet.getValue()) {
                                                if (!sink.putControl((ControlTuple) cct.getUserObject())) {
                                                    // operator cannot handle control tuple; forward to sinks
                                                    forwardToSinks(delay, cct);
                                                }
                                            }
                                        } else {
                                            // Not a ControlAwarePort. Operator cannot handle a custom control tuple.
                                            for (CustomControlTuple cct : portSet.getValue()) {
                                                forwardToSinks(delay, cct);
                                            }
                                        }
                                    }
                                    immediateDeliveryTuples.clear();
                                    endWindowDeliveryTuples.clear();
                                    /* Now call endWindow() */
                                    processEndWindow(t);
                                    activeQueues.addAll(inputs.entrySet());
                                    expectingBeginWindow = activeQueues.size();
                                    break activequeue;
                                }
                            }
                            break;
                        case CUSTOM_CONTROL:
                            activePort.remove();
                            /* All custom control tuples are expected to be arriving in the current window only.*/
                            /* Buffer control tuples until end of the window */
                            CustomControlTuple cct = (CustomControlTuple) t;
                            ControlTuple udct = (ControlTuple) cct.getUserObject();
                            boolean forward = false;
                            // Handle Immediate Delivery Control Tuples
                            if (udct.getDeliveryType().equals(ControlTuple.DeliveryType.IMMEDIATE)) {
                                if (!isDuplicate(immediateDeliveryTuples.get(activePort), cct)) {
                                    // Forward immediately
                                    if (reservoirPortMap.isEmpty()) {
                                        populateReservoirInputPortMap();
                                    }
                                    Sink activeSink = reservoirPortMap.get(activePort);
                                    // activeSink may not be null
                                    if (activeSink instanceof ControlAwareDefaultInputPort) {
                                        ControlTupleEnabledSink sink = (ControlTupleEnabledSink) activeSink;
                                        if (!sink.putControl((ControlTuple) cct.getUserObject())) {
                                            forward = true;
                                        }
                                    } else {
                                        forward = true;
                                    }
                                    if (forward) {
                                        forwardToSinks(delay, cct);
                                    }
                                    // Add to set
                                    if (!immediateDeliveryTuples.containsKey(activePort)) {
                                        immediateDeliveryTuples.put(activePort, new LinkedHashSet<CustomControlTuple>());
                                    }
                                    immediateDeliveryTuples.get(activePort).add(cct);
                                }
                            } else {
                                // Buffer EndWindow Delivery Control Tuples
                                if (!endWindowDeliveryTuples.containsKey(activePort)) {
                                    endWindowDeliveryTuples.put(activePort, new LinkedHashSet<CustomControlTuple>());
                                }
                                if (!isDuplicate(endWindowDeliveryTuples.get(activePort), cct)) {
                                    endWindowDeliveryTuples.get(activePort).add(cct);
                                }
                            }
                            break;
                        case CHECKPOINT:
                            activePort.remove();
                            long checkpointWindow = t.getWindowId();
                            if (lastCheckpointWindowId < checkpointWindow) {
                                dagCheckpointOffsetCount = 0;
                                if (PROCESSING_MODE == ProcessingMode.EXACTLY_ONCE) {
                                    lastCheckpointWindowId = checkpointWindow;
                                } else if (!doCheckpoint) {
                                    if (checkpointWindowCount == 0) {
                                        checkpoint(checkpointWindow);
                                        lastCheckpointWindowId = checkpointWindow;
                                    } else {
                                        doCheckpoint = true;
                                    }
                                }
                                if (!delay) {
                                    for (int s = sinks.length; s-- > 0; ) {
                                        sinks[s].put(t);
                                    }
                                    controlTupleCount++;
                                }
                            }
                            break;
                        case RESET_WINDOW:
                            /**
                 * we will receive tuples which are equal to the number of input streams.
                 */
                            activePort.remove();
                            if (isInputPortConnectedToDelayOperator(activePortEntry.getKey())) {
                                // breaking out of the switch/case
                                break;
                            }
                            buffers.remove();
                            int baseSeconds = t.getBaseSeconds();
                            tracker = null;
                            for (Iterator<TupleTracker> trackerIterator = resetTupleTracker.iterator(); trackerIterator.hasNext(); ) {
                                tracker = trackerIterator.next();
                                if (tracker.tuple.getBaseSeconds() == baseSeconds) {
                                    break;
                                }
                            }
                            if (tracker == null) {
                                tracker = new TupleTracker(t, regularQueues);
                                resetTupleTracker.add(tracker);
                            }
                            int trackerIndex = 0;
                            while (trackerIndex < tracker.ports.length) {
                                if (tracker.ports[trackerIndex] == null) {
                                    tracker.ports[trackerIndex++] = activePort;
                                    break;
                                } else if (tracker.ports[trackerIndex] == activePort) {
                                    break;
                                }
                                trackerIndex++;
                            }
                            if (trackerIndex == regularQueues) {
                                Iterator<TupleTracker> trackerIterator = resetTupleTracker.iterator();
                                while (trackerIterator.hasNext()) {
                                    if (trackerIterator.next().tuple.getBaseSeconds() <= baseSeconds) {
                                        trackerIterator.remove();
                                    }
                                }
                                if (!delay) {
                                    for (int s = sinks.length; s-- > 0; ) {
                                        sinks[s].put(t);
                                    }
                                    controlTupleCount++;
                                }
                                if (!activeQueues.isEmpty()) {
                                    // make sure they are all queues from DelayOperator
                                    for (Map.Entry<String, SweepableReservoir> entry : activeQueues) {
                                        if (!isInputPortConnectedToDelayOperator(entry.getKey())) {
                                            assert (false);
                                        }
                                    }
                                    activeQueues.clear();
                                }
                                activeQueues.addAll(inputs.entrySet());
                                expectingBeginWindow = activeQueues.size();
                                if (firstWindowId == -1) {
                                    if (delay) {
                                        for (int s = sinks.length; s-- > 0; ) {
                                            sinks[s].put(t);
                                        }
                                        controlTupleCount++;
                                        // if it's a DelayOperator and this is the first RESET_WINDOW (start) or END_STREAM
                                        // (recovery), fabricate the first window
                                        fabricateFirstWindow((Operator.DelayOperator) operator, windowAhead);
                                    }
                                    firstWindowId = t.getWindowId();
                                }
                                break activequeue;
                            }
                            break;
                        case END_STREAM:
                            activePort.remove();
                            buffers.remove();
                            if (firstWindowId == -1) {
                                // this is for recovery from a checkpoint for DelayOperator
                                if (delay) {
                                    // if it's a DelayOperator and this is the first RESET_WINDOW (start) or END_STREAM (recovery),
                                    // fabricate the first window
                                    fabricateFirstWindow((Operator.DelayOperator) operator, windowAhead);
                                }
                                firstWindowId = t.getWindowId();
                            }
                            for (Iterator<Entry<String, SweepableReservoir>> it = inputs.entrySet().iterator(); it.hasNext(); ) {
                                Entry<String, SweepableReservoir> e = it.next();
                                if (e.getValue() == activePort) {
                                    if (!descriptor.inputPorts.isEmpty()) {
                                        descriptor.inputPorts.get(e.getKey()).component.setConnected(false);
                                    }
                                    it.remove();
                                    /* check the deferred connection list for any new port that should be connected here */
                                    Iterator<DeferredInputConnection> dici = deferredInputConnections.iterator();
                                    while (dici.hasNext()) {
                                        DeferredInputConnection dic = dici.next();
                                        if (e.getKey().equals(dic.portname)) {
                                            connectInputPort(dic.portname, dic.reservoir);
                                            dici.remove();
                                            activeQueues.add(new AbstractMap.SimpleEntry<>(dic.portname, dic.reservoir));
                                            break activequeue;
                                        }
                                    }
                                    break;
                                }
                            }
                            /**
                 * We are not going to receive begin window on this ever!
                 */
                            expectingBeginWindow--;
                            /**
                 * Since one of the operators we care about it gone, we should relook at our ports.
                 * We need to make sure that the END_STREAM comes outside of the window.
                 */
                            regularQueues--;
                            totalQueues--;
                            boolean break_activequeue = false;
                            if (regularQueues == 0) {
                                alive = false;
                                break_activequeue = true;
                            } else if (activeQueues.isEmpty()) {
                                assert (!inputs.isEmpty());
                                processEndWindow(null);
                                activeQueues.addAll(inputs.entrySet());
                                expectingBeginWindow = activeQueues.size();
                                break_activequeue = true;
                            }
                            /**
                 * also make sure that we update the reset tuple tracker if this stream had delivered any reset tuples.
                 * Check all the reset buffers to see if current input port has already delivered reset tuple. If it has
                 * then we are waiting for something else to deliver the reset tuple, so just clear current reservoir
                 * from the list of tracked reservoirs. If the current input port has not delivered the reset tuple, and
                 * it's the only one which has not, then we consider it delivered and release the reset tuple downstream.
                 */
                            Tuple tuple = null;
                            for (Iterator<TupleTracker> trackerIterator = resetTupleTracker.iterator(); trackerIterator.hasNext(); ) {
                                tracker = trackerIterator.next();
                                trackerIndex = 0;
                                while (trackerIndex < tracker.ports.length) {
                                    if (tracker.ports[trackerIndex] == activePort) {
                                        SweepableReservoir[] ports = new SweepableReservoir[regularQueues];
                                        System.arraycopy(tracker.ports, 0, ports, 0, trackerIndex);
                                        if (trackerIndex < regularQueues) {
                                            System.arraycopy(tracker.ports, trackerIndex + 1, ports, trackerIndex, tracker.ports.length - trackerIndex - 1);
                                        }
                                        tracker.ports = ports;
                                        break;
                                    } else if (tracker.ports[trackerIndex] == null) {
                                        if (trackerIndex == regularQueues) {
                                            /* regularQueues is already adjusted above */
                                            if (tuple == null || tuple.getBaseSeconds() < tracker.tuple.getBaseSeconds()) {
                                                tuple = tracker.tuple;
                                            }
                                            trackerIterator.remove();
                                        }
                                        break;
                                    } else {
                                        tracker.ports = Arrays.copyOf(tracker.ports, regularQueues);
                                    }
                                    trackerIndex++;
                                }
                            }
                            /*
                 * Since we were waiting for a reset tuple on this stream, we should not any longer.
                 */
                            if (tuple != null && !delay) {
                                for (int s = sinks.length; s-- > 0; ) {
                                    sinks[s].put(tuple);
                                }
                                controlTupleCount++;
                            }
                            if (break_activequeue) {
                                break activequeue;
                            }
                            break;
                        default:
                            throw new UnhandledException("Unrecognized Control Tuple", new IllegalArgumentException(t.toString()));
                    }
                }
            }
            if (activeQueues.isEmpty() && alive) {
                logger.error("Catastrophic Error: Invalid State - the operator blocked forever!");
                System.exit(2);
            } else {
                boolean need2sleep = true;
                for (Map.Entry<String, SweepableReservoir> cb : activeQueues) {
                    need2sleep = cb.getValue().isEmpty();
                    if (!need2sleep) {
                        spinMillis = 0;
                        break;
                    }
                }
                if (need2sleep) {
                    if (handleIdleTime && insideWindow) {
                        ((IdleTimeHandler) operator).handleIdleTime();
                    } else {
                        Thread.sleep(spinMillis);
                        spinMillis = Math.min(maxSpinMillis, spinMillis + 1);
                    }
                }
            }
        } while (alive);
    } catch (ShutdownException se) {
        logger.debug("Shutdown requested by the operator when alive = {}.", alive);
        alive = false;
    } catch (Throwable cause) {
        synchronized (this) {
            if (alive) {
                throw Throwables.propagate(cause);
            }
        }
        Throwable rootCause = cause;
        while (rootCause != null) {
            if (rootCause instanceof InterruptedException) {
                break;
            }
            rootCause = rootCause.getCause();
        }
        if (rootCause == null) {
            throw Throwables.propagate(cause);
        } else {
            logger.debug("Ignoring InterruptedException after shutdown", cause);
        }
    }
    /**
     * TODO: If shutdown and inside window provide alternate way of notifying the operator in such ways
     * TODO: as using a listener callback
     */
    if (insideWindow && !shutdown) {
        operator.endWindow();
        endWindowEmitTime = System.currentTimeMillis();
        if (++applicationWindowCount == APPLICATION_WINDOW_COUNT) {
            applicationWindowCount = 0;
        }
        if (++checkpointWindowCount == CHECKPOINT_WINDOW_COUNT) {
            checkpointWindowCount = 0;
            if (doCheckpoint || PROCESSING_MODE == ProcessingMode.EXACTLY_ONCE) {
                checkpoint(currentWindowId);
            }
        }
        ContainerStats.OperatorStats stats = new ContainerStats.OperatorStats();
        fixEndWindowDequeueTimesBeforeDeactivate();
        reportStats(stats, currentWindowId);
        stats.metrics = collectMetrics();
        handleRequests(currentWindowId);
    }
}

Example 9 with Tuple

use of com.datatorrent.stram.tuple.Tuple in project apex-core by apache.

the class GenericNode method fabricateFirstWindow.

private void fabricateFirstWindow(Operator.DelayOperator delayOperator, long windowAhead) {
    Tuple beginWindowTuple = new Tuple(MessageType.BEGIN_WINDOW, windowAhead);
    Tuple endWindowTuple = new Tuple(MessageType.END_WINDOW, windowAhead);
    for (Sink<Object> sink : outputs.values()) {
        sink.put(beginWindowTuple);
    }
    controlTupleCount++;
    delayOperator.firstWindow();
    for (Sink<Object> sink : outputs.values()) {
        sink.put(endWindowTuple);
    }
    controlTupleCount++;
}

Example 10 with Tuple

use of com.datatorrent.stram.tuple.Tuple in project apex-core by apache.

the class WindowGenerator method resetBeginNewWindow.

private void resetBeginNewWindow() throws InterruptedException {
    long timespanBetween2Resets = (long) MAX_WINDOW_ID * windowWidthMillis + windowWidthMillis;
    resetWindowMillis = currentWindowMillis - ((currentWindowMillis - resetWindowMillis) % timespanBetween2Resets);
    windowId = (int) ((currentWindowMillis - resetWindowMillis) / windowWidthMillis);
    baseSeconds = (resetWindowMillis / 1000) << 32;
    //logger.info("generating reset -> begin {}", Codec.getStringWindowId(baseSeconds));
    queue.put(new ResetWindowTuple(baseSeconds | windowWidthMillis));
    queue.put(new Tuple(MessageType.BEGIN_WINDOW, baseSeconds | windowId));
}