Examples with Tuple com.datatorrent.stram.tuple.Tuple used on opensource projects

Example 11 with Tuple

use of com.datatorrent.stram.tuple.Tuple in project apex-core by apache.

the class WindowGenerator method endCurrentBeginNewWindow.

/**
   * Updates window in a circular buffer on inputAdapters<p>
   * This code generates the windows
   */
private void endCurrentBeginNewWindow() throws InterruptedException {
    queue.put(new EndWindowTuple(baseSeconds | windowId));
    if (++checkPointWindowCount == checkpointCount) {
        queue.put(new Tuple(MessageType.CHECKPOINT, baseSeconds | windowId));
        checkPointWindowCount = 0;
    }
    if (windowId == MAX_WINDOW_ID) {
        advanceWindow();
        run();
    } else {
        advanceWindow();
        queue.put(new Tuple(MessageType.BEGIN_WINDOW, baseSeconds | windowId));
    }
}

Example 12 with Tuple

use of com.datatorrent.stram.tuple.Tuple in project apex-core by apache.

the class InputNode method run.

@Override
@SuppressWarnings(value = { "SleepWhileInLoop", "BroadCatchBlock", "TooBroadCatch" })
public final void run() {
    long maxSpinMillis = context.getValue(OperatorContext.SPIN_MILLIS);
    long spinMillis = 0;
    final boolean handleIdleTime = operator instanceof IdleTimeHandler;
    boolean insideApplicationWindow = applicationWindowCount != 0;
    boolean doCheckpoint = false;
    boolean insideStreamingWindow = false;
    calculateNextCheckpointWindow();
    try {
        while (alive) {
            Tuple t = controlTuples.sweep();
            if (t == null) {
                if (insideStreamingWindow) {
                    int generatedTuples = 0;
                    for (Sink<Object> cs : sinks) {
                        generatedTuples -= cs.getCount(false);
                    }
                    operator.emitTuples();
                    for (Sink<Object> cs : sinks) {
                        generatedTuples += cs.getCount(false);
                    }
                    if (generatedTuples == 0) {
                        if (handleIdleTime) {
                            ((IdleTimeHandler) operator).handleIdleTime();
                        } else {
                            Thread.sleep(spinMillis);
                            spinMillis = Math.min(spinMillis + 1, maxSpinMillis);
                        }
                    } else {
                        spinMillis = 0;
                    }
                } else {
                    Thread.sleep(0);
                }
            } else {
                controlTuples.remove();
                switch(t.getType()) {
                    case BEGIN_WINDOW:
                        for (int i = sinks.length; i-- > 0; ) {
                            sinks[i].put(t);
                        }
                        controlTupleCount++;
                        currentWindowId = t.getWindowId();
                        insideStreamingWindow = true;
                        if (applicationWindowCount == 0) {
                            insideApplicationWindow = true;
                            operator.beginWindow(currentWindowId);
                        }
                        operator.emitTuples();
                        break;
                    case END_WINDOW:
                        insideStreamingWindow = false;
                        if (++applicationWindowCount == APPLICATION_WINDOW_COUNT) {
                            insideApplicationWindow = false;
                            operator.endWindow();
                            applicationWindowCount = 0;
                        }
                        endWindowEmitTime = System.currentTimeMillis();
                        for (int i = sinks.length; i-- > 0; ) {
                            sinks[i].put(t);
                        }
                        controlTupleCount++;
                        if (doCheckpoint) {
                            dagCheckpointOffsetCount = (dagCheckpointOffsetCount + 1) % DAG_CHECKPOINT_WINDOW_COUNT;
                        }
                        if (++checkpointWindowCount == CHECKPOINT_WINDOW_COUNT) {
                            checkpointWindowCount = 0;
                            if (doCheckpoint) {
                                checkpoint(currentWindowId);
                                lastCheckpointWindowId = currentWindowId;
                                doCheckpoint = false;
                            } else if (PROCESSING_MODE == ProcessingMode.EXACTLY_ONCE) {
                                checkpoint(currentWindowId);
                                lastCheckpointWindowId = currentWindowId;
                            }
                        }
                        ContainerStats.OperatorStats stats = new ContainerStats.OperatorStats();
                        reportStats(stats, currentWindowId);
                        if (!insideApplicationWindow) {
                            stats.metrics = collectMetrics();
                        }
                        handleRequests(currentWindowId);
                        break;
                    case CHECKPOINT:
                        dagCheckpointOffsetCount = 0;
                        if (lastCheckpointWindowId < currentWindowId) {
                            if (checkpointWindowCount == 0 && PROCESSING_MODE != ProcessingMode.EXACTLY_ONCE) {
                                checkpoint(currentWindowId);
                                lastCheckpointWindowId = currentWindowId;
                            } else {
                                doCheckpoint = true;
                            }
                        }
                        for (int i = sinks.length; i-- > 0; ) {
                            sinks[i].put(t);
                        }
                        controlTupleCount++;
                        break;
                    case END_STREAM:
                        if (deferredInputConnections.isEmpty()) {
                            for (int i = sinks.length; i-- > 0; ) {
                                sinks[i].put(t);
                            }
                            controlTupleCount++;
                            alive = false;
                        } else {
                            controlTuples = deferredInputConnections.remove(0);
                        }
                        break;
                    default:
                        for (int i = sinks.length; i-- > 0; ) {
                            sinks[i].put(t);
                        }
                        controlTupleCount++;
                        break;
                }
            }
        }
    } catch (ShutdownException se) {
        logger.debug("Shutdown requested by the operator when alive = {}.", alive);
        alive = false;
    } catch (Throwable cause) {
        synchronized (this) {
            if (alive) {
                throw Throwables.propagate(cause);
            }
        }
        Throwable rootCause = cause;
        while (rootCause != null) {
            if (rootCause instanceof InterruptedException) {
                break;
            }
            rootCause = rootCause.getCause();
        }
        if (rootCause == null) {
            throw Throwables.propagate(cause);
        } else {
            logger.debug("Ignoring InterruptedException after shutdown", cause);
        }
    }
    if (insideApplicationWindow) {
        operator.endWindow();
        endWindowEmitTime = System.currentTimeMillis();
        if (++applicationWindowCount == APPLICATION_WINDOW_COUNT) {
            applicationWindowCount = 0;
        }
        if (lastCheckpointWindowId < currentWindowId) {
            //is 1
            if (++checkpointWindowCount == CHECKPOINT_WINDOW_COUNT) {
                checkpointWindowCount = 0;
                if (doCheckpoint || PROCESSING_MODE == ProcessingMode.EXACTLY_ONCE) {
                    checkpoint(currentWindowId);
                    lastCheckpointWindowId = currentWindowId;
                }
            }
        }
        ContainerStats.OperatorStats stats = new ContainerStats.OperatorStats();
        reportStats(stats, currentWindowId);
        stats.metrics = collectMetrics();
        handleRequests(currentWindowId);
    }
}

Example 13 with Tuple

use of com.datatorrent.stram.tuple.Tuple in project apex-core by apache.

the class ProcessingModeTests method testNonLinearOperatorRecovery.

@SuppressWarnings("SleepWhileInLoop")
public void testNonLinearOperatorRecovery() throws InterruptedException {
    final HashSet<Object> collection = new HashSet<>();
    com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap map = new com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap();
    map.put(OperatorContext.CHECKPOINT_WINDOW_COUNT, 0);
    map.put(OperatorContext.PROCESSING_MODE, processingMode);
    final GenericNode node = new GenericNode(new MultiInputOperator(), new com.datatorrent.stram.engine.OperatorContext(1, "operator", map, null));
    AbstractReservoir reservoir1 = AbstractReservoir.newReservoir("input1", 1024);
    AbstractReservoir reservoir2 = AbstractReservoir.newReservoir("input1", 1024);
    node.connectInputPort("input1", reservoir1);
    node.connectInputPort("input2", reservoir2);
    node.connectOutputPort("output", new Sink<Object>() {

        @Override
        public void put(Object t) {
            if (collection.contains(t)) {
                throw new RuntimeException("Duplicate Found!");
            }
            collection.add(t);
        }

        @Override
        public int getCount(boolean bln) {
            return 0;
        }
    });
    final AtomicBoolean active = new AtomicBoolean(false);
    Thread thread = new Thread() {

        @Override
        public void run() {
            active.set(true);
            node.activate();
            node.run();
            node.deactivate();
        }
    };
    thread.start();
    for (int i = 0; i < 100 && !active.get(); i++) {
        sleep(5);
    }
    reservoir1.add(new Tuple(MessageType.BEGIN_WINDOW, 1));
    reservoir1.add(1);
    reservoir2.add(new Tuple(MessageType.BEGIN_WINDOW, 1));
    reservoir1.add(new EndWindowTuple(1));
    reservoir2.add(1);
    reservoir2.add(new EndWindowTuple(1));
    for (int i = 0; i < 100 && collection.size() < 4; i++) {
        sleep(5);
    }
    reservoir1.add(new Tuple(MessageType.BEGIN_WINDOW, 2));
    reservoir1.add(2);
    reservoir1.add(new EndWindowTuple(2));
    for (int i = 0; i < 100 && collection.size() < 6; i++) {
        sleep(5);
    }
    reservoir2.add(new Tuple(MessageType.BEGIN_WINDOW, 4));
    reservoir2.add(4);
    reservoir2.add(new EndWindowTuple((4)));
    for (int i = 0; i < 100 && collection.size() < 9; i++) {
        sleep(5);
    }
    reservoir1.add(new Tuple(MessageType.BEGIN_WINDOW, 3));
    reservoir1.add(3);
    reservoir1.add(new EndWindowTuple(3));
    sleep(500);
    reservoir1.add(new Tuple(MessageType.BEGIN_WINDOW, 5));
    reservoir1.add(5);
    reservoir2.add(new Tuple(MessageType.BEGIN_WINDOW, 5));
    reservoir1.add(new EndWindowTuple(5));
    reservoir2.add(5);
    reservoir2.add(new EndWindowTuple(5));
    for (int i = 0; i < 100 && collection.size() < 14; i++) {
        sleep(5);
    }
    thread.interrupt();
    thread.join();
    /* lets make sure that we have all the tuples and nothing more */
    for (Object o : collection) {
        if (o instanceof Tuple) {
            Tuple t = (Tuple) o;
            long windowId = t.getWindowId();
            Assert.assertTrue("Valid Window Id", windowId == 1 || windowId == 2 || windowId == 4 || windowId == 5);
            Assert.assertTrue("Valid Tuple Type", t.getType() == MessageType.BEGIN_WINDOW || t.getType() == MessageType.END_WINDOW);
        } else {
            switch(((Integer) o).intValue()) {
                case 101:
                case 201:
                case 102:
                case 204:
                case 105:
                case 205:
                    break;
                default:
                    Assert.fail("Unexpected Data Tuple: " + o);
            }
        }
    }
}

Example 14 with Tuple

use of com.datatorrent.stram.tuple.Tuple in project apex-core by apache.

the class AbstractReservoirTest method testAddAndSweepTuple.

@Test
@Parameters(method = "defaultTestParameters")
public void testAddAndSweepTuple(final AbstractReservoir reservoir, final Class<? extends Throwable> type) {
    final Tuple t = new Tuple(BEGIN_WINDOW, 0L);
    assertTrue(reservoir.add(t));
    assertFalse(reservoir.isEmpty());
    assertEquals(1, reservoir.size());
    assertEquals(1, reservoir.size(false));
    assertEquals(t, reservoir.peek());
    assertEquals(t, reservoir.sweep());
    assertEquals(t, reservoir.sweep());
    assertEquals(0, reservoir.getCount(false));
    assertEquals(0, reservoir.getSink().getCount(false));
    assertFalse(reservoir.isEmpty());
    assertEquals(t, reservoir.remove());
    assertNull(reservoir.peek());
    assertNull(reservoir.poll());
    assertNull(reservoir.sweep());
    exception.expect(type);
    reservoir.remove();
}

Example 15 with Tuple

use of com.datatorrent.stram.tuple.Tuple in project apex-core by apache.

the class GenericNodeTest method testSynchingLogic.

@Test
@SuppressWarnings("SleepWhileInLoop")
public void testSynchingLogic() throws InterruptedException {
    long sleeptime = 25L;
    final ArrayList<Object> list = new ArrayList<>();
    GenericOperator go = new GenericOperator();
    final GenericNode gn = new GenericNode(go, new com.datatorrent.stram.engine.OperatorContext(0, "operator", new DefaultAttributeMap(), null));
    gn.setId(1);
    AbstractReservoir reservoir1 = AbstractReservoir.newReservoir("ip1Res", 1024);
    AbstractReservoir reservoir2 = AbstractReservoir.newReservoir("ip2Res", 1024);
    Sink<Object> output = new Sink<Object>() {

        @Override
        public void put(Object tuple) {
            list.add(tuple);
        }

        @Override
        public int getCount(boolean reset) {
            return 0;
        }
    };
    gn.connectInputPort("ip1", reservoir1);
    gn.connectInputPort("ip2", reservoir2);
    gn.connectOutputPort("op", output);
    gn.firstWindowMillis = 0;
    gn.windowWidthMillis = 100;
    final AtomicBoolean ab = new AtomicBoolean(false);
    Thread t = new Thread() {

        @Override
        public void run() {
            ab.set(true);
            gn.activate();
            gn.run();
            gn.deactivate();
        }
    };
    t.start();
    do {
        Thread.sleep(sleeptime);
    } while (ab.get() == false);
    Tuple beginWindow1 = new Tuple(MessageType.BEGIN_WINDOW, 0x1L);
    reservoir1.add(beginWindow1);
    Thread.sleep(sleeptime);
    Assert.assertEquals(1, list.size());
    reservoir2.add(beginWindow1);
    Thread.sleep(sleeptime);
    Assert.assertEquals(1, list.size());
    Tuple endWindow1 = new EndWindowTuple(0x1L);
    reservoir1.add(endWindow1);
    Thread.sleep(sleeptime);
    Assert.assertEquals(1, list.size());
    Tuple beginWindow2 = new Tuple(MessageType.BEGIN_WINDOW, 0x2L);
    reservoir1.add(beginWindow2);
    Thread.sleep(sleeptime);
    Assert.assertEquals(1, list.size());
    reservoir2.add(endWindow1);
    Thread.sleep(sleeptime);
    Assert.assertEquals(3, list.size());
    reservoir2.add(beginWindow2);
    Thread.sleep(sleeptime);
    Assert.assertEquals(3, list.size());
    Tuple endWindow2 = new EndWindowTuple(0x2L);
    reservoir2.add(endWindow2);
    Thread.sleep(sleeptime);
    Assert.assertEquals(3, list.size());
    reservoir1.add(endWindow2);
    Thread.sleep(sleeptime);
    Assert.assertEquals(4, list.size());
    EndStreamTuple est = new EndStreamTuple(0L);
    reservoir1.add(est);
    Thread.sleep(sleeptime);
    Assert.assertEquals(4, list.size());
    Tuple beginWindow3 = new Tuple(MessageType.BEGIN_WINDOW, 0x3L);
    reservoir2.add(beginWindow3);
    Thread.sleep(sleeptime);
    Assert.assertEquals(5, list.size());
    Tuple endWindow3 = new EndWindowTuple(0x3L);
    reservoir2.add(endWindow3);
    Thread.sleep(sleeptime);
    Assert.assertEquals(6, list.size());
    Assert.assertNotSame(Thread.State.TERMINATED, t.getState());
    reservoir2.add(est);
    Thread.sleep(sleeptime);
    Assert.assertEquals(7, list.size());
    Thread.sleep(sleeptime);
    Assert.assertEquals(Thread.State.TERMINATED, t.getState());
}