Examples with SweepableReservoir com.datatorrent.stram.engine.SweepableReservoir used on opensource projects

Example 1 with SweepableReservoir

use of com.datatorrent.stram.engine.SweepableReservoir in project apex-core by apache.

the class BufferServerSubscriberTest method testEmergencySinks.

@Test
public void testEmergencySinks() throws InterruptedException {
    final List<Object> list = new ArrayList<>();
    final StreamCodec<Object> myserde = new StreamCodec<Object>() {

        @Override
        public Object fromByteArray(Slice fragment) {
            if (fragment.offset == 0 && fragment.length == fragment.buffer.length) {
                return fragment.buffer;
            } else {
                return Arrays.copyOfRange(fragment.buffer, fragment.offset, fragment.offset + fragment.length);
            }
        }

        @Override
        public Slice toByteArray(Object o) {
            return new Slice((byte[]) o, 0, ((byte[]) o).length);
        }

        @Override
        public int getPartition(Object o) {
            return 0;
        }
    };
    Sink<Object> unbufferedSink = new Sink<Object>() {

        @Override
        public void put(Object tuple) {
            list.add(tuple);
        }

        @Override
        public int getCount(boolean reset) {
            return 0;
        }
    };
    BufferServerSubscriber bss = new BufferServerSubscriber("subscriber", 5) {

        {
            serde = myserde;
        }

        @Override
        public void suspendRead() {
            logger.debug("read suspended");
        }

        @Override
        public void resumeRead() {
            logger.debug("read resumed");
        }
    };
    SweepableReservoir reservoir = bss.acquireReservoir("unbufferedSink", 3);
    reservoir.setSink(unbufferedSink);
    int i = 0;
    while (i++ < 10) {
        Slice fragment = myserde.toByteArray(new byte[] { (byte) i });
        byte[] buffer = PayloadTuple.getSerializedTuple(myserde.getPartition(i), fragment);
        bss.onMessage(buffer, 0, buffer.length);
    }
    reservoir.sweep();
    /* 4 make it to the reservoir */
    reservoir.sweep();
    /* we consume the 4; and 4 more make it to the reservoir */
    Assert.assertEquals("4 received", 4, list.size());
    reservoir.sweep();
    /* 8 consumed + 2 more make it to the reservoir */
    reservoir.sweep();
    /* consume 2 more */
    Assert.assertEquals("10  received", 10, list.size());
}

Example 2 with SweepableReservoir

use of com.datatorrent.stram.engine.SweepableReservoir in project apex-core by apache.

the class FastPublisherTest method testSerialization.

@Test
public void testSerialization() throws Exception {
    FastPublisherImpl publisher = new FastPublisherImpl(24 * 1024);
    final String message = "hello!";
    publisher.put(message);
    byte[] buffer = publisher.consume();
    FastSubscriber subscriber = new FastSubscriber("subscriber", 1024);
    subscriber.serde = subscriber.statefulSerde = new DefaultStatefulStreamCodec<>();
    SweepableReservoir sr = subscriber.acquireReservoir("res", 1024);
    sr.setSink(new Sink<Object>() {

        @Override
        public void put(Object tuple) {
            assert (tuple.equals(message));
        }

        @Override
        public int getCount(boolean reset) {
            throw new UnsupportedOperationException("Not supported yet.");
        }
    });
    int size = buffer[0];
    size |= buffer[1] << 8;
    Assert.assertEquals("size", buffer.length - 2, size);
    subscriber.onMessage(buffer, 2, buffer.length - 2);
    sr.sweep();
    sr.sweep();
    for (int i = 0; i < 1024; i++) {
        publisher.put(message);
    }
    buffer = publisher.consume();
    assert (buffer.length == (size + 2) * 1024);
    int index = 0;
    for (int i = 0; i < 1024; i++) {
        size = buffer[index++];
        size |= buffer[index++] << 8;
        subscriber.onMessage(buffer, index, size);
        index += size;
    }
    sr.sweep();
    sr.sweep();
    for (int i = 0; i < 1024; i++) {
        publisher.put(message);
    }
    buffer = publisher.consume();
    assert (buffer.length == (size + 2) * 1024);
    index = 0;
    for (int i = 0; i < 1024; i++) {
        size = buffer[index++];
        size |= buffer[index++] << 8;
        subscriber.onMessage(buffer, index, size);
        index += size;
    }
    sr.sweep();
    sr.sweep();
}

Example 3 with SweepableReservoir

use of com.datatorrent.stram.engine.SweepableReservoir in project apex-core by apache.

the class FastStreamTest method testBufferServerStream.

/**
 * Test buffer server stream by sending
 * tuple on outputstream and receive same tuple from inputstream
 *
 * @throws Exception
 */
@Test
@SuppressWarnings({ "SleepWhileInLoop" })
public void testBufferServerStream() throws Exception {
    final StreamCodec<Object> serde = new DefaultStatefulStreamCodec<>();
    final AtomicInteger messageCount = new AtomicInteger();
    Sink<Object> sink = new Sink<Object>() {

        @Override
        public void put(Object tuple) {
            logger.debug("received: " + tuple);
            messageCount.incrementAndGet();
        }

        @Override
        public int getCount(boolean reset) {
            throw new UnsupportedOperationException("Not supported yet.");
        }
    };
    String streamName = "streamName";
    String upstreamNodeId = "upstreamNodeId";
    String downstreamNodeId = "downStreamNodeId";
    StreamContext issContext = new StreamContext(streamName);
    issContext.setSourceId(upstreamNodeId);
    issContext.setSinkId(downstreamNodeId);
    issContext.setFinishedWindowId(-1);
    issContext.setBufferServerAddress(InetSocketAddress.createUnresolved("localhost", bufferServerPort));
    issContext.put(StreamContext.CODEC, serde);
    issContext.put(StreamContext.EVENT_LOOP, eventloop);
    FastSubscriber subscriber = new FastSubscriber(downstreamNodeId, 1024);
    subscriber.setup(issContext);
    SweepableReservoir reservoir = subscriber.acquireReservoir("testReservoir", 1);
    reservoir.setSink(sink);
    StreamContext ossContext = new StreamContext(streamName);
    ossContext.setSourceId(upstreamNodeId);
    ossContext.setSinkId(downstreamNodeId);
    ossContext.setFinishedWindowId(-1);
    ossContext.setBufferServerAddress(InetSocketAddress.createUnresolved("localhost", bufferServerPort));
    ossContext.put(StreamContext.CODEC, serde);
    ossContext.put(StreamContext.EVENT_LOOP, eventloop);
    FastPublisher publisher = new FastPublisher(upstreamNodeId, 8);
    StreamContext publisherContext = new StreamContext(streamName);
    publisherContext.setSourceId(upstreamNodeId);
    publisherContext.setSinkId(downstreamNodeId);
    publisherContext.setBufferServerAddress(InetSocketAddress.createUnresolved("localhost", bufferServerPort));
    publisherContext.put(StreamContext.CODEC, serde);
    publisherContext.put(StreamContext.EVENT_LOOP, eventloop);
    publisher.setup(publisherContext);
    subscriber.activate(issContext);
    LOG.debug("input stream activated");
    publisher.activate(publisherContext);
    LOG.debug("output stream activated");
    LOG.debug("Sending hello message");
    publisher.put(StramTestSupport.generateBeginWindowTuple(upstreamNodeId, 0));
    publisher.put(StramTestSupport.generateTuple("hello", 0));
    publisher.put(StramTestSupport.generateEndWindowTuple(upstreamNodeId, 0));
    // it's a spurious tuple, presence of it should not affect the outcome of the test.
    publisher.put(StramTestSupport.generateBeginWindowTuple(upstreamNodeId, 1));
    for (int i = 0; i < 100; i++) {
        Tuple t = reservoir.sweep();
        if (t == null) {
            sleep(5);
            continue;
        }
        reservoir.remove();
        if (t instanceof EndWindowTuple) {
            break;
        }
    }
    eventloop.disconnect(publisher);
    eventloop.disconnect(subscriber);
    Assert.assertEquals("Received messages", 1, messageCount.get());
}

Example 4 with SweepableReservoir

use of com.datatorrent.stram.engine.SweepableReservoir in project apex-core by apache.

the class InlineStreamTest method test.

@Test
@SuppressWarnings("SleepWhileInLoop")
public void test() throws Exception {
    final int totalTupleCount = 5000;
    final PassThroughNode<Object> operator1 = new PassThroughNode<>();
    final GenericNode node1 = new GenericNode(operator1, new OperatorContext(1, "operator1", new DefaultAttributeMap(), null));
    node1.setId(1);
    operator1.setup(node1.context);
    final PassThroughNode<Object> operator2 = new PassThroughNode<>();
    final GenericNode node2 = new GenericNode(operator2, new OperatorContext(2, "operator2", new DefaultAttributeMap(), null));
    node2.setId(2);
    operator2.setup(node2.context);
    StreamContext streamContext = new StreamContext("node1->node2");
    final InlineStream stream = new InlineStream(1024);
    stream.setup(streamContext);
    node1.connectOutputPort("output", stream);
    node2.connectInputPort("input", stream.getReservoir());
    prev = null;
    Sink<Object> sink = new Sink<Object>() {

        @Override
        public void put(Object payload) {
            if (payload instanceof Tuple) {
                return;
            }
            if (prev == null) {
                prev = payload;
            } else {
                if (Integer.valueOf(payload.toString()) - Integer.valueOf(prev.toString()) != 1) {
                    synchronized (InlineStreamTest.this) {
                        InlineStreamTest.this.notify();
                    }
                }
                prev = payload;
            }
            if (Integer.valueOf(prev.toString()) == totalTupleCount - 1) {
                synchronized (InlineStreamTest.this) {
                    InlineStreamTest.this.notify();
                }
            }
        }

        @Override
        public int getCount(boolean reset) {
            return 0;
        }
    };
    node2.connectOutputPort("output", sink);
    AbstractReservoir reservoir1 = AbstractReservoir.newReservoir("input", 1024 * 5);
    node1.connectInputPort("input", reservoir1);
    Map<Integer, Node<?>> activeNodes = new ConcurrentHashMap<>();
    launchNodeThread(node1, activeNodes);
    launchNodeThread(node2, activeNodes);
    stream.activate(streamContext);
    reservoir1.put(StramTestSupport.generateBeginWindowTuple("irrelevant", 0));
    for (int i = 0; i < totalTupleCount; i++) {
        reservoir1.put(i);
    }
    reservoir1.put(StramTestSupport.generateEndWindowTuple("irrelevant", 0));
    synchronized (this) {
        this.wait(200);
    }
    Assert.assertNotNull(prev);
    Assert.assertEquals("processing complete", totalTupleCount, Integer.valueOf(prev.toString()) + 1);
    Assert.assertEquals("active operators", 2, activeNodes.size());
    WaitCondition c = new WaitCondition() {

        @Override
        public boolean isComplete() {
            final SweepableReservoir reservoir = stream.getReservoir();
            logger.debug("stream {} empty {}, size {}", stream, reservoir.isEmpty(), reservoir.size(false));
            return reservoir.isEmpty();
        }
    };
    Assert.assertTrue("operator should finish processing all events within 1 second", StramTestSupport.awaitCompletion(c, 1000));
    stream.deactivate();
    for (Node<?> node : activeNodes.values()) {
        node.shutdown();
    }
    for (int i = 0; i < 10; i++) {
        Thread.sleep(20);
        if (activeNodes.isEmpty()) {
            break;
        }
    }
    stream.teardown();
    operator2.teardown();
    operator1.teardown();
    Assert.assertEquals("active operators", 0, activeNodes.size());
}