Examples with Tuple com.datatorrent.stram.tuple.Tuple used on opensource projects

Example 16 with Tuple

use of com.datatorrent.stram.tuple.Tuple in project apex-core by apache.

the class GenericNodeTest method testControlTuplesDeliveryGenericNode.

@Test
public void testControlTuplesDeliveryGenericNode() throws InterruptedException {
    long maxSleep = 5000000;
    long sleeptime = 25L;
    GenericOperator go = new GenericOperator();
    final GenericNode gn = new GenericNode(go, new com.datatorrent.stram.engine.OperatorContext(0, "operator", new DefaultAttributeMap(), null));
    gn.setId(1);
    AbstractReservoir reservoir1 = AbstractReservoir.newReservoir("ip1Res", 1024);
    gn.connectInputPort("ip1", reservoir1);
    TestSink testSink = new TestSink();
    gn.connectOutputPort("op", testSink);
    gn.firstWindowMillis = 0;
    gn.windowWidthMillis = 100;
    final AtomicBoolean ab = new AtomicBoolean(false);
    Thread t = new Thread() {

        @Override
        public void run() {
            ab.set(true);
            gn.activate();
            gn.run();
            gn.deactivate();
        }
    };
    t.start();
    long interval = 0;
    do {
        Thread.sleep(sleeptime);
        interval += sleeptime;
    } while ((ab.get() == false) && (interval < maxSleep));
    int controlTupleCount = gn.controlTupleCount;
    Tuple beginWindow = new Tuple(MessageType.BEGIN_WINDOW, 0x1L);
    reservoir1.add(beginWindow);
    interval = 0;
    do {
        Thread.sleep(sleeptime);
        interval += sleeptime;
    } while ((gn.controlTupleCount == controlTupleCount) && (interval < maxSleep));
    controlTupleCount = gn.controlTupleCount;
    CustomControlTuple t1 = new CustomControlTuple(new CustomControlTupleTest.TestControlTuple(1, false));
    CustomControlTuple t2 = new CustomControlTuple(new CustomControlTupleTest.TestControlTuple(2, true));
    CustomControlTuple t3 = new CustomControlTuple(new CustomControlTupleTest.TestControlTuple(3, false));
    CustomControlTuple t4 = new CustomControlTuple(new CustomControlTupleTest.TestControlTuple(4, true));
    reservoir1.add(t1);
    reservoir1.add(t2);
    reservoir1.add(t3);
    reservoir1.add(t4);
    interval = 0;
    do {
        Thread.sleep(sleeptime);
        interval += sleeptime;
    } while ((gn.controlTupleCount == controlTupleCount) && (interval < maxSleep));
    Assert.assertTrue("Custom control tuples emitted immediately", testSink.getResultCount() == 3);
    controlTupleCount = gn.controlTupleCount;
    Tuple endWindow = new Tuple(MessageType.END_WINDOW, 0x1L);
    reservoir1.add(endWindow);
    interval = 0;
    do {
        Thread.sleep(sleeptime);
        interval += sleeptime;
    } while ((gn.controlTupleCount == controlTupleCount) && (interval < maxSleep));
    gn.shutdown();
    t.join();
    Assert.assertTrue("Total control tuples", testSink.getResultCount() == 6);
    long expected = 0;
    for (Object o : testSink.collectedTuples) {
        if (o instanceof CustomControlTuple) {
            expected++;
        }
    }
    Assert.assertTrue("Number of Custom control tuples", expected == 4);
}

Example 17 with Tuple

use of com.datatorrent.stram.tuple.Tuple in project apex-core by apache.

the class GenericNodeTest method testPrematureTermination.

@Test
public void testPrematureTermination() throws InterruptedException {
    long maxSleep = 5000;
    long sleeptime = 25L;
    GenericOperator go = new GenericOperator();
    final GenericNode gn = new GenericNode(go, new com.datatorrent.stram.engine.OperatorContext(0, "operator", new DefaultAttributeMap(), null));
    gn.setId(1);
    AbstractReservoir reservoir1 = AbstractReservoir.newReservoir("ip1Res", 1024);
    AbstractReservoir reservoir2 = AbstractReservoir.newReservoir("ip2Res", 1024);
    gn.connectInputPort("ip1", reservoir1);
    gn.connectInputPort("ip2", reservoir2);
    gn.connectOutputPort("op", Sink.BLACKHOLE);
    gn.firstWindowMillis = 0;
    gn.windowWidthMillis = 100;
    final AtomicBoolean ab = new AtomicBoolean(false);
    Thread t = new Thread() {

        @Override
        public void run() {
            ab.set(true);
            gn.activate();
            gn.run();
            gn.deactivate();
        }
    };
    t.start();
    long interval = 0;
    do {
        Thread.sleep(sleeptime);
        interval += sleeptime;
    } while ((ab.get() == false) && (interval < maxSleep));
    int controlTupleCount = gn.controlTupleCount;
    Tuple beginWindow1 = new Tuple(MessageType.BEGIN_WINDOW, 0x1L);
    reservoir1.add(beginWindow1);
    reservoir2.add(beginWindow1);
    interval = 0;
    do {
        Thread.sleep(sleeptime);
        interval += sleeptime;
    } while ((gn.controlTupleCount == controlTupleCount) && (interval < maxSleep));
    Assert.assertTrue("Begin window called", go.endWindowId != go.beginWindowId);
    controlTupleCount = gn.controlTupleCount;
    Tuple endWindow1 = new EndWindowTuple(0x1L);
    reservoir1.add(endWindow1);
    reservoir2.add(endWindow1);
    interval = 0;
    do {
        Thread.sleep(sleeptime);
        interval += sleeptime;
    } while ((gn.controlTupleCount == controlTupleCount) && (interval < maxSleep));
    Assert.assertTrue("End window called", go.endWindowId == go.beginWindowId);
    controlTupleCount = gn.controlTupleCount;
    Tuple beginWindow2 = new Tuple(MessageType.BEGIN_WINDOW, 0x2L);
    reservoir1.add(beginWindow2);
    reservoir2.add(beginWindow2);
    interval = 0;
    do {
        Thread.sleep(sleeptime);
        interval += sleeptime;
    } while ((gn.controlTupleCount == controlTupleCount) && (interval < maxSleep));
    gn.shutdown();
    t.join();
    Assert.assertTrue("End window not called", go.endWindowId != go.beginWindowId);
}

Example 18 with Tuple

use of com.datatorrent.stram.tuple.Tuple in project apex-core by apache.

the class WindowGeneratorTest method testWindowGen.

@Test
public void testWindowGen() throws Exception {
    final AtomicLong currentWindow = new AtomicLong();
    final AtomicInteger beginWindowCount = new AtomicInteger();
    final AtomicInteger endWindowCount = new AtomicInteger();
    final AtomicLong windowXor = new AtomicLong();
    Sink<Object> s = new Sink<Object>() {

        @Override
        public int getCount(boolean reset) {
            return 0;
        }

        @Override
        public void put(Object payload) {
            logger.debug("unexpected payload {}", payload);
        }
    };
    ScheduledThreadPoolExecutor stpe = new ScheduledThreadPoolExecutor(1, "WindowGenerator");
    int windowWidth = 200;
    long firstWindowMillis = stpe.getCurrentTimeMillis();
    firstWindowMillis -= firstWindowMillis % 1000L;
    WindowGenerator wg = new WindowGenerator(new ScheduledThreadPoolExecutor(1, "WindowGenerator"), WindowGenerator.MAX_WINDOW_ID + 1024);
    wg.setResetWindow(firstWindowMillis);
    wg.setFirstWindow(firstWindowMillis);
    wg.setWindowWidth(windowWidth);
    SweepableReservoir reservoir = wg.acquireReservoir("GeneratorTester", windowWidth);
    reservoir.setSink(s);
    wg.activate(null);
    Thread.sleep(200);
    wg.deactivate();
    reservoir.sweep();
    /* just transfer over all the control tuples */
    Tuple t;
    while ((t = reservoir.sweep()) != null) {
        reservoir.remove();
        long windowId = t.getWindowId();
        switch(t.getType()) {
            case BEGIN_WINDOW:
                currentWindow.set(windowId);
                beginWindowCount.incrementAndGet();
                windowXor.set(windowXor.get() ^ windowId);
                break;
            case END_WINDOW:
                endWindowCount.incrementAndGet();
                windowXor.set(windowXor.get() ^ windowId);
                break;
            case RESET_WINDOW:
                break;
            default:
                currentWindow.set(0);
                break;
        }
    }
    long lastWindowMillis = System.currentTimeMillis();
    Assert.assertEquals("only last window open", currentWindow.get(), windowXor.get());
    long expectedCnt = (lastWindowMillis - firstWindowMillis) / windowWidth;
    Assert.assertTrue("Minimum begin window count", expectedCnt + 1 <= beginWindowCount.get());
    Assert.assertEquals("end window count", beginWindowCount.get() - 1, endWindowCount.get());
}

Example 19 with Tuple

use of com.datatorrent.stram.tuple.Tuple in project apex-core by apache.

the class FastStreamTest method testBufferServerStream.

/**
   * Test buffer server stream by sending
   * tuple on outputstream and receive same tuple from inputstream
   *
   * @throws Exception
   */
@Test
@SuppressWarnings({ "SleepWhileInLoop" })
public void testBufferServerStream() throws Exception {
    final StreamCodec<Object> serde = new DefaultStatefulStreamCodec<>();
    final AtomicInteger messageCount = new AtomicInteger();
    Sink<Object> sink = new Sink<Object>() {

        @Override
        public void put(Object tuple) {
            logger.debug("received: " + tuple);
            messageCount.incrementAndGet();
        }

        @Override
        public int getCount(boolean reset) {
            throw new UnsupportedOperationException("Not supported yet.");
        }
    };
    String streamName = "streamName";
    String upstreamNodeId = "upstreamNodeId";
    String downstreamNodeId = "downStreamNodeId";
    StreamContext issContext = new StreamContext(streamName);
    issContext.setSourceId(upstreamNodeId);
    issContext.setSinkId(downstreamNodeId);
    issContext.setFinishedWindowId(-1);
    issContext.setBufferServerAddress(InetSocketAddress.createUnresolved("localhost", bufferServerPort));
    issContext.put(StreamContext.CODEC, serde);
    issContext.put(StreamContext.EVENT_LOOP, eventloop);
    FastSubscriber subscriber = new FastSubscriber(downstreamNodeId, 1024);
    subscriber.setup(issContext);
    SweepableReservoir reservoir = subscriber.acquireReservoir("testReservoir", 1);
    reservoir.setSink(sink);
    StreamContext ossContext = new StreamContext(streamName);
    ossContext.setSourceId(upstreamNodeId);
    ossContext.setSinkId(downstreamNodeId);
    ossContext.setFinishedWindowId(-1);
    ossContext.setBufferServerAddress(InetSocketAddress.createUnresolved("localhost", bufferServerPort));
    ossContext.put(StreamContext.CODEC, serde);
    ossContext.put(StreamContext.EVENT_LOOP, eventloop);
    FastPublisher publisher = new FastPublisher(upstreamNodeId, 8);
    StreamContext publisherContext = new StreamContext(streamName);
    publisherContext.setSourceId(upstreamNodeId);
    publisherContext.setSinkId(downstreamNodeId);
    publisherContext.setBufferServerAddress(InetSocketAddress.createUnresolved("localhost", bufferServerPort));
    publisherContext.put(StreamContext.CODEC, serde);
    publisherContext.put(StreamContext.EVENT_LOOP, eventloop);
    publisher.setup(publisherContext);
    subscriber.activate(issContext);
    LOG.debug("input stream activated");
    publisher.activate(publisherContext);
    LOG.debug("output stream activated");
    LOG.debug("Sending hello message");
    publisher.put(StramTestSupport.generateBeginWindowTuple(upstreamNodeId, 0));
    publisher.put(StramTestSupport.generateTuple("hello", 0));
    publisher.put(StramTestSupport.generateEndWindowTuple(upstreamNodeId, 0));
    // it's a spurious tuple, presence of it should not affect the outcome of the test.
    publisher.put(StramTestSupport.generateBeginWindowTuple(upstreamNodeId, 1));
    for (int i = 0; i < 100; i++) {
        Tuple t = reservoir.sweep();
        if (t == null) {
            sleep(5);
            continue;
        }
        reservoir.remove();
        if (t instanceof EndWindowTuple) {
            break;
        }
    }
    eventloop.disconnect(publisher);
    eventloop.disconnect(subscriber);
    Assert.assertEquals("Received messages", 1, messageCount.get());
}

Example 20 with Tuple

use of com.datatorrent.stram.tuple.Tuple in project apex-core by apache.

the class SocketStreamTest method testBufferServerStreamWithLateActivationForSubscriber.

/**
   * Test buffer server stream by sending
   * tuple on outputstream and receive same tuple from inputstream with following changes
   *
   * 1. Sink is sweeped befere the BufferServerSubscriber is activated.
   * 2. BufferServerSubscriber is activated after the messages are sent from BufferServerPublisher
   *
   * @throws Exception
   */
@Test
@SuppressWarnings({ "SleepWhileInLoop" })
public void testBufferServerStreamWithLateActivationForSubscriber() throws Exception {
    for (int i = 0; i < 50; i++) {
        Tuple t = reservoir.sweep();
        if (t == null) {
            sleep(5);
            continue;
        }
        throw new Exception("Unexpected control tuple.");
    }
    oss.activate(ossContext);
    LOG.debug("output stream activated");
    sendMessage();
    iss.activate(issContext);
    LOG.debug("input stream activated");
}