Examples with Tuple com.datatorrent.stram.tuple.Tuple used on opensource projects

Example 1 with Tuple

use of com.datatorrent.stram.tuple.Tuple in project apex-core by apache.

the class AtMostOnceTest method testNonLinearOperatorRecovery.

@Test
@SuppressWarnings("SleepWhileInLoop")
@Override
public void testNonLinearOperatorRecovery() throws InterruptedException {
    final HashSet<Object> collection = new HashSet<>();
    com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap map = new com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap();
    map.put(OperatorContext.CHECKPOINT_WINDOW_COUNT, 0);
    map.put(OperatorContext.PROCESSING_MODE, processingMode);
    final GenericNode node = new GenericNode(new MultiInputOperator(), new com.datatorrent.stram.engine.OperatorContext(1, "operator", map, null));
    AbstractReservoir reservoir1 = AbstractReservoir.newReservoir("input1", 1024);
    AbstractReservoir reservoir2 = AbstractReservoir.newReservoir("input1", 1024);
    node.connectInputPort("input1", reservoir1);
    node.connectInputPort("input2", reservoir2);
    node.connectOutputPort("output", new Sink<Object>() {

        @Override
        public void put(Object t) {
            if (collection.contains(t)) {
                throw new RuntimeException("Duplicate Found!");
            }
            collection.add(t);
        }

        @Override
        public int getCount(boolean bln) {
            return 0;
        }
    });
    final AtomicBoolean active = new AtomicBoolean(false);
    Thread thread = new Thread() {

        @Override
        public void run() {
            active.set(true);
            node.activate();
            node.run();
            node.deactivate();
        }
    };
    thread.start();
    for (int i = 0; i < 100 && !active.get(); i++) {
        sleep(5);
    }
    reservoir1.add(new Tuple(MessageType.BEGIN_WINDOW, 1));
    reservoir1.add(1);
    reservoir2.add(new Tuple(MessageType.BEGIN_WINDOW, 1));
    reservoir1.add(new EndWindowTuple(1));
    reservoir2.add(1);
    reservoir2.add(new EndWindowTuple(1));
    for (int i = 0; i < 100 && collection.size() < 4; i++) {
        sleep(5);
    }
    reservoir1.add(new Tuple(MessageType.BEGIN_WINDOW, 2));
    reservoir1.add(2);
    reservoir1.add(new EndWindowTuple(2));
    for (int i = 0; i < 100 && collection.size() < 6; i++) {
        sleep(5);
    }
    reservoir2.add(new Tuple(MessageType.BEGIN_WINDOW, 4));
    reservoir2.add(4);
    reservoir2.add(new EndWindowTuple((4)));
    for (int i = 0; i < 100 && collection.size() < 9; i++) {
        sleep(5);
    }
    reservoir1.add(new Tuple(MessageType.BEGIN_WINDOW, 3));
    reservoir1.add(3);
    reservoir1.add(new EndWindowTuple(3));
    sleep(500);
    reservoir1.add(new Tuple(MessageType.BEGIN_WINDOW, 5));
    reservoir1.add(5);
    reservoir2.add(new Tuple(MessageType.BEGIN_WINDOW, 5));
    reservoir1.add(new EndWindowTuple(5));
    reservoir2.add(5);
    reservoir2.add(new EndWindowTuple(5));
    for (int i = 0; i < 100 && collection.size() < 14; i++) {
        sleep(5);
    }
    thread.interrupt();
    thread.join();
    /* lets make sure that we have all the tuples and nothing more */
    for (Object o : collection) {
        if (o instanceof Tuple) {
            Tuple t = (Tuple) o;
            long windowId = t.getWindowId();
            Assert.assertTrue("Valid Window Id", windowId == 1 || windowId == 2 || windowId == 4 || windowId == 5);
            Assert.assertTrue("Valid Tuple Type", t.getType() == MessageType.BEGIN_WINDOW || t.getType() == MessageType.END_WINDOW || t.getType() == MessageType.END_STREAM);
        } else {
            switch(((Integer) o).intValue()) {
                case 101:
                case 201:
                case 102:
                case 204:
                case 105:
                case 205:
                    break;
                default:
                    Assert.fail("Unexpected Data Tuple: " + o);
            }
        }
    }
}

Example 2 with Tuple

use of com.datatorrent.stram.tuple.Tuple in project apex-core by apache.

the class InputNodeTest method emitTestHelper.

private void emitTestHelper(boolean trueEmitTuplesFalseHandleIdleTime) throws Exception {
    TestInputOperator tio = new TestInputOperator();
    tio.trueEmitTuplesFalseHandleIdleTime = trueEmitTuplesFalseHandleIdleTime;
    DefaultAttributeMap dam = new DefaultAttributeMap();
    dam.put(OperatorContext.APPLICATION_WINDOW_COUNT, 10);
    dam.put(OperatorContext.CHECKPOINT_WINDOW_COUNT, 10);
    final InputNode in = new InputNode(tio, new OperatorContext(0, "operator", dam, null));
    TestSink testSink = new TestSink();
    in.connectInputPort(Node.INPUT, new TestWindowGenerator());
    in.connectOutputPort("output", testSink);
    Thread t = new Thread() {

        @Override
        public void run() {
            in.activate();
            in.run();
            in.deactivate();
        }
    };
    t.start();
    Thread.sleep(3000);
    in.shutdown();
    t.join();
    Assert.assertTrue("Should have emitted some tuples", testSink.collectedTuples.size() > 0);
    boolean insideWindow = false;
    for (Object tuple : testSink.collectedTuples) {
        if (tuple instanceof Tuple) {
            Tuple controlTuple = (Tuple) tuple;
            MessageType tupleType = controlTuple.getType();
            if (tupleType == MessageType.RESET_WINDOW) {
                Assert.assertFalse(insideWindow);
            } else if (tupleType == MessageType.BEGIN_WINDOW) {
                Assert.assertFalse(insideWindow);
                insideWindow = true;
            } else if (tupleType == MessageType.END_WINDOW) {
                Assert.assertTrue(insideWindow);
                insideWindow = false;
            }
        } else {
            Assert.assertTrue(insideWindow);
        }
    }
}

Example 3 with Tuple

use of com.datatorrent.stram.tuple.Tuple in project apex-core by apache.

the class WindowGeneratorTest method test2ndResetWindow.

@Test
public void test2ndResetWindow() throws InterruptedException {
    logger.info("Testing 2nd Reset Window");
    ManualScheduledExecutorService msse = new ManualScheduledExecutorService(1);
    WindowGenerator generator = new WindowGenerator(msse, (WindowGenerator.MAX_WINDOW_ID << 1) + 1024);
    generator.setFirstWindow(0L);
    generator.setResetWindow(0L);
    generator.setWindowWidth(1);
    SweepableReservoir reservoir = generator.acquireReservoir(Node.OUTPUT, (WindowGenerator.MAX_WINDOW_ID << 1) + 1024);
    final AtomicBoolean loggingEnabled = new AtomicBoolean(true);
    reservoir.setSink(new Sink<Object>() {

        @Override
        public void put(Object payload) {
            if (loggingEnabled.get()) {
                logger.debug(payload.toString());
            }
        }

        @Override
        public int getCount(boolean reset) {
            return 0;
        }
    });
    generator.activate(null);
    msse.tick(1);
    /* reset window and begin window */
    msse.tick(1);
    /* end window and begin window */
    loggingEnabled.set(false);
    for (int i = 0; i < WindowGenerator.MAX_WINDOW_ID - 2; i++) {
        msse.tick(1);
    /* end window and begin window */
    }
    loggingEnabled.set(true);
    msse.tick(1);
    /* end window, reset window, begin window */
    final AtomicInteger beginWindowCount = new AtomicInteger(0);
    final AtomicInteger endWindowCount = new AtomicInteger(0);
    final AtomicInteger resetWindowCount = new AtomicInteger(0);
    Tuple t;
    reservoir.sweep();
    while ((t = reservoir.sweep()) != null) {
        reservoir.remove();
        switch(t.getType()) {
            case BEGIN_WINDOW:
                beginWindowCount.incrementAndGet();
                break;
            case END_WINDOW:
                endWindowCount.incrementAndGet();
                break;
            case RESET_WINDOW:
                resetWindowCount.incrementAndGet();
                break;
            default:
                break;
        }
    }
    Assert.assertEquals("begin windows", WindowGenerator.MAX_WINDOW_ID + 1 + 1, beginWindowCount.get());
    Assert.assertEquals("end windows", WindowGenerator.MAX_WINDOW_ID + 1, endWindowCount.get());
    Assert.assertEquals("reset windows", 2, resetWindowCount.get());
}

Example 4 with Tuple

use of com.datatorrent.stram.tuple.Tuple in project apex-core by apache.

the class WindowGeneratorTest method testResetWindow.

/**
   * Test of resetWindow functionality of WindowGenerator.
   */
@Test
public void testResetWindow() {
    ManualScheduledExecutorService msse = new ManualScheduledExecutorService(1);
    msse.setCurrentTimeMillis(0x0afebabe * 1000L);
    WindowGenerator generator = new WindowGenerator(msse, WindowGenerator.MAX_WINDOW_ID + 1024);
    final long currentTIme = msse.getCurrentTimeMillis();
    final int windowWidth = 0x1234abcd;
    generator.setFirstWindow(currentTIme);
    generator.setResetWindow(currentTIme);
    generator.setWindowWidth(windowWidth);
    SweepableReservoir reservoir = generator.acquireReservoir(Node.OUTPUT, 1024);
    reservoir.setSink(new Sink<Object>() {

        boolean firsttime = true;

        @Override
        public int getCount(boolean reset) {
            return 0;
        }

        @Override
        public void put(Object payload) {
            assert (false);
            if (firsttime) {
                assert (payload instanceof ResetWindowTuple);
                firsttime = false;
            } else {
                assert (payload instanceof Tuple);
            }
        }
    });
    generator.activate(null);
    msse.tick(1);
    Assert.assertNull(reservoir.sweep());
    ResetWindowTuple rwt = (ResetWindowTuple) reservoir.sweep();
    reservoir.remove();
    assert (rwt.getWindowId() == 0x0afebabe00000000L);
    assert (rwt.getBaseSeconds() * 1000L == currentTIme);
    assert (rwt.getIntervalMillis() == windowWidth);
    Tuple t = reservoir.sweep();
    reservoir.remove();
    assert (t.getType() == MessageType.BEGIN_WINDOW);
    assert (t.getWindowId() == 0x0afebabe00000000L);
    assert (reservoir.sweep() == null);
}

Example 5 with Tuple

use of com.datatorrent.stram.tuple.Tuple in project apex-core by apache.

the class GenericNodeTest method testControlTuplesDeliveryOiONode.

@Test
public void testControlTuplesDeliveryOiONode() throws InterruptedException {
    GenericOperator go = new GenericOperator();
    final OiONode oioNode = new OiONode(go, new com.datatorrent.stram.engine.OperatorContext(0, "operator", new DefaultAttributeMap(), null));
    oioNode.setId(1);
    OiOStream stream = new OiOStream();
    SweepableReservoir reservoir = stream.getReservoir();
    ((OiOStream.OiOReservoir) reservoir).setControlSink((oioNode).getControlSink(reservoir));
    oioNode.connectInputPort("ip1", reservoir);
    Sink controlSink = oioNode.getControlSink(reservoir);
    TestSink testSink = new TestSink();
    oioNode.connectOutputPort("op", testSink);
    oioNode.firstWindowMillis = 0;
    oioNode.windowWidthMillis = 100;
    oioNode.activate();
    Tuple beginWindow = new Tuple(MessageType.BEGIN_WINDOW, 0x1L);
    controlSink.put(beginWindow);
    Assert.assertTrue("Begin window", testSink.getResultCount() == 1);
    CustomControlTuple t1 = new CustomControlTuple(new CustomControlTupleTest.TestControlTuple(1, false));
    CustomControlTuple t2 = new CustomControlTuple(new CustomControlTupleTest.TestControlTuple(2, true));
    CustomControlTuple t3 = new CustomControlTuple(new CustomControlTupleTest.TestControlTuple(3, false));
    CustomControlTuple t4 = new CustomControlTuple(new CustomControlTupleTest.TestControlTuple(4, true));
    controlSink.put(t1);
    controlSink.put(t2);
    controlSink.put(t3);
    controlSink.put(t4);
    Assert.assertTrue("Custom control tuples emitted immediately", testSink.getResultCount() == 3);
    Tuple endWindow = new Tuple(MessageType.END_WINDOW, 0x1L);
    controlSink.put(endWindow);
    oioNode.deactivate();
    oioNode.shutdown();
    Assert.assertTrue("Total control tuples", testSink.getResultCount() == 6);
    long expected = 0;
    for (Object o : testSink.collectedTuples) {
        if (o instanceof CustomControlTuple) {
            expected++;
        }
    }
    Assert.assertTrue("Number of Custom control tuples", expected == 4);
}