Examples with EndWindowTuple com.datatorrent.stram.tuple.EndWindowTuple used on opensource projects

Example 1 with EndWindowTuple

use of com.datatorrent.stram.tuple.EndWindowTuple in project apex-core by apache.

the class AtMostOnceTest method testNonLinearOperatorRecovery.

@Test
@SuppressWarnings("SleepWhileInLoop")
@Override
public void testNonLinearOperatorRecovery() throws InterruptedException {
    final HashSet<Object> collection = new HashSet<>();
    com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap map = new com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap();
    map.put(OperatorContext.CHECKPOINT_WINDOW_COUNT, 0);
    map.put(OperatorContext.PROCESSING_MODE, processingMode);
    final GenericNode node = new GenericNode(new MultiInputOperator(), new com.datatorrent.stram.engine.OperatorContext(1, "operator", map, null));
    AbstractReservoir reservoir1 = AbstractReservoir.newReservoir("input1", 1024);
    AbstractReservoir reservoir2 = AbstractReservoir.newReservoir("input1", 1024);
    node.connectInputPort("input1", reservoir1);
    node.connectInputPort("input2", reservoir2);
    node.connectOutputPort("output", new Sink<Object>() {

        @Override
        public void put(Object t) {
            if (collection.contains(t)) {
                throw new RuntimeException("Duplicate Found!");
            }
            collection.add(t);
        }

        @Override
        public int getCount(boolean bln) {
            return 0;
        }
    });
    final AtomicBoolean active = new AtomicBoolean(false);
    Thread thread = new Thread() {

        @Override
        public void run() {
            active.set(true);
            node.activate();
            node.run();
            node.deactivate();
        }
    };
    thread.start();
    for (int i = 0; i < 100 && !active.get(); i++) {
        sleep(5);
    }
    reservoir1.add(new Tuple(MessageType.BEGIN_WINDOW, 1));
    reservoir1.add(1);
    reservoir2.add(new Tuple(MessageType.BEGIN_WINDOW, 1));
    reservoir1.add(new EndWindowTuple(1));
    reservoir2.add(1);
    reservoir2.add(new EndWindowTuple(1));
    for (int i = 0; i < 100 && collection.size() < 4; i++) {
        sleep(5);
    }
    reservoir1.add(new Tuple(MessageType.BEGIN_WINDOW, 2));
    reservoir1.add(2);
    reservoir1.add(new EndWindowTuple(2));
    for (int i = 0; i < 100 && collection.size() < 6; i++) {
        sleep(5);
    }
    reservoir2.add(new Tuple(MessageType.BEGIN_WINDOW, 4));
    reservoir2.add(4);
    reservoir2.add(new EndWindowTuple((4)));
    for (int i = 0; i < 100 && collection.size() < 9; i++) {
        sleep(5);
    }
    reservoir1.add(new Tuple(MessageType.BEGIN_WINDOW, 3));
    reservoir1.add(3);
    reservoir1.add(new EndWindowTuple(3));
    sleep(500);
    reservoir1.add(new Tuple(MessageType.BEGIN_WINDOW, 5));
    reservoir1.add(5);
    reservoir2.add(new Tuple(MessageType.BEGIN_WINDOW, 5));
    reservoir1.add(new EndWindowTuple(5));
    reservoir2.add(5);
    reservoir2.add(new EndWindowTuple(5));
    for (int i = 0; i < 100 && collection.size() < 14; i++) {
        sleep(5);
    }
    thread.interrupt();
    thread.join();
    /* lets make sure that we have all the tuples and nothing more */
    for (Object o : collection) {
        if (o instanceof Tuple) {
            Tuple t = (Tuple) o;
            long windowId = t.getWindowId();
            Assert.assertTrue("Valid Window Id", windowId == 1 || windowId == 2 || windowId == 4 || windowId == 5);
            Assert.assertTrue("Valid Tuple Type", t.getType() == MessageType.BEGIN_WINDOW || t.getType() == MessageType.END_WINDOW || t.getType() == MessageType.END_STREAM);
        } else {
            switch(((Integer) o).intValue()) {
                case 101:
                case 201:
                case 102:
                case 204:
                case 105:
                case 205:
                    break;
                default:
                    Assert.fail("Unexpected Data Tuple: " + o);
            }
        }
    }
}

Example 2 with EndWindowTuple

use of com.datatorrent.stram.tuple.EndWindowTuple in project apex-core by apache.

the class GenericNodeTest method testBufferServerSubscriberActivationBeforeOperator.

@Test
public void testBufferServerSubscriberActivationBeforeOperator() throws InterruptedException, IOException {
    final String streamName = "streamName";
    final String upstreamNodeId = "upstreamNodeId";
    final String downstreamNodeId = "downStreamNodeId";
    EventLoop eventloop = DefaultEventLoop.createEventLoop("StreamTestEventLoop");
    ((DefaultEventLoop) eventloop).start();
    // find random port
    final Server bufferServer = new Server(eventloop, 0);
    final int bufferServerPort = bufferServer.run().getPort();
    final StreamCodec<Object> serde = new DefaultStatefulStreamCodec<>();
    final BlockingQueue<Object> tuples = new ArrayBlockingQueue<>(10);
    GenericTestOperator go = new GenericTestOperator();
    final GenericNode gn = new GenericNode(go, new com.datatorrent.stram.engine.OperatorContext(0, "operator", new DefaultAttributeMap(), null));
    gn.setId(1);
    Sink<Object> output = new Sink<Object>() {

        @Override
        public void put(Object tuple) {
            tuples.add(tuple);
        }

        @Override
        public int getCount(boolean reset) {
            return 0;
        }
    };
    InetSocketAddress socketAddress = new InetSocketAddress("localhost", bufferServerPort);
    StreamContext issContext = new StreamContext(streamName);
    issContext.setSourceId(upstreamNodeId);
    issContext.setSinkId(downstreamNodeId);
    issContext.setFinishedWindowId(-1);
    issContext.setBufferServerAddress(socketAddress);
    issContext.put(StreamContext.CODEC, serde);
    issContext.put(StreamContext.EVENT_LOOP, eventloop);
    StreamContext ossContext = new StreamContext(streamName);
    ossContext.setSourceId(upstreamNodeId);
    ossContext.setSinkId(downstreamNodeId);
    ossContext.setBufferServerAddress(socketAddress);
    ossContext.put(StreamContext.CODEC, serde);
    ossContext.put(StreamContext.EVENT_LOOP, eventloop);
    BufferServerPublisher oss = new BufferServerPublisher(upstreamNodeId, 1024);
    oss.setup(ossContext);
    oss.activate(ossContext);
    oss.put(new Tuple(MessageType.BEGIN_WINDOW, 0x1L));
    byte[] buff = PayloadTuple.getSerializedTuple(0, 1);
    buff[buff.length - 1] = (byte) 1;
    oss.put(buff);
    oss.put(new EndWindowTuple(0x1L));
    oss.put(new Tuple(MessageType.BEGIN_WINDOW, 0x2L));
    buff = PayloadTuple.getSerializedTuple(0, 1);
    buff[buff.length - 1] = (byte) 2;
    oss.put(buff);
    oss.put(new EndWindowTuple(0x2L));
    oss.put(new Tuple(MessageType.BEGIN_WINDOW, 0x3L));
    buff = PayloadTuple.getSerializedTuple(0, 1);
    buff[buff.length - 1] = (byte) 3;
    oss.put(buff);
    oss.put(new EndWindowTuple(0x3L));
    oss.put(new EndStreamTuple(0L));
    BufferServerSubscriber iss = new BufferServerSubscriber(downstreamNodeId, 1024);
    iss.setup(issContext);
    gn.connectInputPort(GenericTestOperator.IPORT1, iss.acquireReservoir("testReservoir", 10));
    gn.connectOutputPort(GenericTestOperator.OPORT1, output);
    SweepableReservoir tupleWait = iss.acquireReservoir("testReservoir2", 10);
    iss.activate(issContext);
    while (tupleWait.sweep() == null) {
        Thread.sleep(100);
    }
    gn.firstWindowMillis = 0;
    gn.windowWidthMillis = 100;
    Thread t = new Thread() {

        @Override
        public void run() {
            gn.activate();
            gn.run();
            gn.deactivate();
        }
    };
    t.start();
    t.join();
    Assert.assertEquals(10, tuples.size());
    List<Object> list = new ArrayList<>(tuples);
    Assert.assertEquals("Payload Tuple 1", 1, ((byte[]) list.get(1))[5]);
    Assert.assertEquals("Payload Tuple 2", 2, ((byte[]) list.get(4))[5]);
    Assert.assertEquals("Payload Tuple 3", 3, ((byte[]) list.get(7))[5]);
    if (bufferServer != null) {
        bufferServer.stop();
    }
    ((DefaultEventLoop) eventloop).stop();
}

Example 3 with EndWindowTuple

use of com.datatorrent.stram.tuple.EndWindowTuple in project apex-core by apache.

the class Node method emitEndWindow.

protected void emitEndWindow() {
    long windowId = (operator instanceof Operator.DelayOperator) ? WindowGenerator.getAheadWindowId(currentWindowId, firstWindowMillis, windowWidthMillis, 1) : currentWindowId;
    EndWindowTuple ewt = new EndWindowTuple(windowId);
    for (int s = sinks.length; s-- > 0; ) {
        sinks[s].put(ewt);
    }
    controlTupleCount++;
}

Example 4 with EndWindowTuple

use of com.datatorrent.stram.tuple.EndWindowTuple in project apex-core by apache.

the class WindowGenerator method endCurrentBeginNewWindow.

/**
   * Updates window in a circular buffer on inputAdapters<p>
   * This code generates the windows
   */
private void endCurrentBeginNewWindow() throws InterruptedException {
    queue.put(new EndWindowTuple(baseSeconds | windowId));
    if (++checkPointWindowCount == checkpointCount) {
        queue.put(new Tuple(MessageType.CHECKPOINT, baseSeconds | windowId));
        checkPointWindowCount = 0;
    }
    if (windowId == MAX_WINDOW_ID) {
        advanceWindow();
        run();
    } else {
        advanceWindow();
        queue.put(new Tuple(MessageType.BEGIN_WINDOW, baseSeconds | windowId));
    }
}

Example 5 with EndWindowTuple

use of com.datatorrent.stram.tuple.EndWindowTuple in project apex-core by apache.

the class ProcessingModeTests method testNonLinearOperatorRecovery.

@SuppressWarnings("SleepWhileInLoop")
public void testNonLinearOperatorRecovery() throws InterruptedException {
    final HashSet<Object> collection = new HashSet<>();
    com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap map = new com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap();
    map.put(OperatorContext.CHECKPOINT_WINDOW_COUNT, 0);
    map.put(OperatorContext.PROCESSING_MODE, processingMode);
    final GenericNode node = new GenericNode(new MultiInputOperator(), new com.datatorrent.stram.engine.OperatorContext(1, "operator", map, null));
    AbstractReservoir reservoir1 = AbstractReservoir.newReservoir("input1", 1024);
    AbstractReservoir reservoir2 = AbstractReservoir.newReservoir("input1", 1024);
    node.connectInputPort("input1", reservoir1);
    node.connectInputPort("input2", reservoir2);
    node.connectOutputPort("output", new Sink<Object>() {

        @Override
        public void put(Object t) {
            if (collection.contains(t)) {
                throw new RuntimeException("Duplicate Found!");
            }
            collection.add(t);
        }

        @Override
        public int getCount(boolean bln) {
            return 0;
        }
    });
    final AtomicBoolean active = new AtomicBoolean(false);
    Thread thread = new Thread() {

        @Override
        public void run() {
            active.set(true);
            node.activate();
            node.run();
            node.deactivate();
        }
    };
    thread.start();
    for (int i = 0; i < 100 && !active.get(); i++) {
        sleep(5);
    }
    reservoir1.add(new Tuple(MessageType.BEGIN_WINDOW, 1));
    reservoir1.add(1);
    reservoir2.add(new Tuple(MessageType.BEGIN_WINDOW, 1));
    reservoir1.add(new EndWindowTuple(1));
    reservoir2.add(1);
    reservoir2.add(new EndWindowTuple(1));
    for (int i = 0; i < 100 && collection.size() < 4; i++) {
        sleep(5);
    }
    reservoir1.add(new Tuple(MessageType.BEGIN_WINDOW, 2));
    reservoir1.add(2);
    reservoir1.add(new EndWindowTuple(2));
    for (int i = 0; i < 100 && collection.size() < 6; i++) {
        sleep(5);
    }
    reservoir2.add(new Tuple(MessageType.BEGIN_WINDOW, 4));
    reservoir2.add(4);
    reservoir2.add(new EndWindowTuple((4)));
    for (int i = 0; i < 100 && collection.size() < 9; i++) {
        sleep(5);
    }
    reservoir1.add(new Tuple(MessageType.BEGIN_WINDOW, 3));
    reservoir1.add(3);
    reservoir1.add(new EndWindowTuple(3));
    sleep(500);
    reservoir1.add(new Tuple(MessageType.BEGIN_WINDOW, 5));
    reservoir1.add(5);
    reservoir2.add(new Tuple(MessageType.BEGIN_WINDOW, 5));
    reservoir1.add(new EndWindowTuple(5));
    reservoir2.add(5);
    reservoir2.add(new EndWindowTuple(5));
    for (int i = 0; i < 100 && collection.size() < 14; i++) {
        sleep(5);
    }
    thread.interrupt();
    thread.join();
    /* lets make sure that we have all the tuples and nothing more */
    for (Object o : collection) {
        if (o instanceof Tuple) {
            Tuple t = (Tuple) o;
            long windowId = t.getWindowId();
            Assert.assertTrue("Valid Window Id", windowId == 1 || windowId == 2 || windowId == 4 || windowId == 5);
            Assert.assertTrue("Valid Tuple Type", t.getType() == MessageType.BEGIN_WINDOW || t.getType() == MessageType.END_WINDOW);
        } else {
            switch(((Integer) o).intValue()) {
                case 101:
                case 201:
                case 102:
                case 204:
                case 105:
                case 205:
                    break;
                default:
                    Assert.fail("Unexpected Data Tuple: " + o);
            }
        }
    }
}