Examples with DefaultStatefulStreamCodec com.datatorrent.stram.codec.DefaultStatefulStreamCodec used on opensource projects

Example 1 with DefaultStatefulStreamCodec

use of com.datatorrent.stram.codec.DefaultStatefulStreamCodec in project apex-core by apache.

the class GenericNodeTest method testBufferServerSubscriberActivationBeforeOperator.

@Test
public void testBufferServerSubscriberActivationBeforeOperator() throws InterruptedException, IOException {
    final String streamName = "streamName";
    final String upstreamNodeId = "upstreamNodeId";
    final String downstreamNodeId = "downStreamNodeId";
    EventLoop eventloop = DefaultEventLoop.createEventLoop("StreamTestEventLoop");
    ((DefaultEventLoop) eventloop).start();
    // find random port
    final Server bufferServer = new Server(eventloop, 0);
    final int bufferServerPort = bufferServer.run().getPort();
    final StreamCodec<Object> serde = new DefaultStatefulStreamCodec<>();
    final BlockingQueue<Object> tuples = new ArrayBlockingQueue<>(10);
    GenericTestOperator go = new GenericTestOperator();
    final GenericNode gn = new GenericNode(go, new com.datatorrent.stram.engine.OperatorContext(0, "operator", new DefaultAttributeMap(), null));
    gn.setId(1);
    Sink<Object> output = new Sink<Object>() {

        @Override
        public void put(Object tuple) {
            tuples.add(tuple);
        }

        @Override
        public int getCount(boolean reset) {
            return 0;
        }
    };
    InetSocketAddress socketAddress = new InetSocketAddress("localhost", bufferServerPort);
    StreamContext issContext = new StreamContext(streamName);
    issContext.setSourceId(upstreamNodeId);
    issContext.setSinkId(downstreamNodeId);
    issContext.setFinishedWindowId(-1);
    issContext.setBufferServerAddress(socketAddress);
    issContext.put(StreamContext.CODEC, serde);
    issContext.put(StreamContext.EVENT_LOOP, eventloop);
    StreamContext ossContext = new StreamContext(streamName);
    ossContext.setSourceId(upstreamNodeId);
    ossContext.setSinkId(downstreamNodeId);
    ossContext.setBufferServerAddress(socketAddress);
    ossContext.put(StreamContext.CODEC, serde);
    ossContext.put(StreamContext.EVENT_LOOP, eventloop);
    BufferServerPublisher oss = new BufferServerPublisher(upstreamNodeId, 1024);
    oss.setup(ossContext);
    oss.activate(ossContext);
    oss.put(new Tuple(MessageType.BEGIN_WINDOW, 0x1L));
    byte[] buff = PayloadTuple.getSerializedTuple(0, 1);
    buff[buff.length - 1] = (byte) 1;
    oss.put(buff);
    oss.put(new EndWindowTuple(0x1L));
    oss.put(new Tuple(MessageType.BEGIN_WINDOW, 0x2L));
    buff = PayloadTuple.getSerializedTuple(0, 1);
    buff[buff.length - 1] = (byte) 2;
    oss.put(buff);
    oss.put(new EndWindowTuple(0x2L));
    oss.put(new Tuple(MessageType.BEGIN_WINDOW, 0x3L));
    buff = PayloadTuple.getSerializedTuple(0, 1);
    buff[buff.length - 1] = (byte) 3;
    oss.put(buff);
    oss.put(new EndWindowTuple(0x3L));
    oss.put(new EndStreamTuple(0L));
    BufferServerSubscriber iss = new BufferServerSubscriber(downstreamNodeId, 1024);
    iss.setup(issContext);
    gn.connectInputPort(GenericTestOperator.IPORT1, iss.acquireReservoir("testReservoir", 10));
    gn.connectOutputPort(GenericTestOperator.OPORT1, output);
    SweepableReservoir tupleWait = iss.acquireReservoir("testReservoir2", 10);
    iss.activate(issContext);
    while (tupleWait.sweep() == null) {
        Thread.sleep(100);
    }
    gn.firstWindowMillis = 0;
    gn.windowWidthMillis = 100;
    Thread t = new Thread() {

        @Override
        public void run() {
            gn.activate();
            gn.run();
            gn.deactivate();
        }
    };
    t.start();
    t.join();
    Assert.assertEquals(10, tuples.size());
    List<Object> list = new ArrayList<>(tuples);
    Assert.assertEquals("Payload Tuple 1", 1, ((byte[]) list.get(1))[5]);
    Assert.assertEquals("Payload Tuple 2", 2, ((byte[]) list.get(4))[5]);
    Assert.assertEquals("Payload Tuple 3", 3, ((byte[]) list.get(7))[5]);
    if (bufferServer != null) {
        bufferServer.stop();
    }
    ((DefaultEventLoop) eventloop).stop();
}

Example 2 with DefaultStatefulStreamCodec

use of com.datatorrent.stram.codec.DefaultStatefulStreamCodec in project apex-core by apache.

the class SocketStreamTest method init.

@Before
public void init() {
    final StreamCodec<Object> serde = new DefaultStatefulStreamCodec<>();
    messageCount = new AtomicInteger(0);
    Sink<Object> sink = new Sink<Object>() {

        @Override
        public void put(Object tuple) {
            logger.debug("received: " + tuple);
            messageCount.incrementAndGet();
        }

        @Override
        public int getCount(boolean reset) {
            throw new UnsupportedOperationException("Not supported yet.");
        }
    };
    issContext = new StreamContext(streamName);
    issContext.setSourceId(upstreamNodeId);
    issContext.setSinkId(downstreamNodeId);
    issContext.setFinishedWindowId(-1);
    issContext.setBufferServerAddress(InetSocketAddress.createUnresolved("localhost", bufferServerPort));
    issContext.put(StreamContext.CODEC, serde);
    issContext.put(StreamContext.EVENT_LOOP, eventloop);
    iss = new BufferServerSubscriber(downstreamNodeId, 1024);
    iss.setup(issContext);
    reservoir = iss.acquireReservoir("testReservoir", 1);
    reservoir.setSink(sink);
    ossContext = new StreamContext(streamName);
    ossContext.setSourceId(upstreamNodeId);
    ossContext.setSinkId(downstreamNodeId);
    ossContext.setBufferServerAddress(InetSocketAddress.createUnresolved("localhost", bufferServerPort));
    ossContext.put(StreamContext.CODEC, serde);
    ossContext.put(StreamContext.EVENT_LOOP, eventloop);
    oss = new BufferServerPublisher(upstreamNodeId, 1024);
    oss.setup(ossContext);
}

Example 3 with DefaultStatefulStreamCodec

use of com.datatorrent.stram.codec.DefaultStatefulStreamCodec in project apex-core by apache.

the class FastPublisherTest method testSerialization.

@Test
public void testSerialization() throws Exception {
    FastPublisherImpl publisher = new FastPublisherImpl(24 * 1024);
    final String message = "hello!";
    publisher.put(message);
    byte[] buffer = publisher.consume();
    FastSubscriber subscriber = new FastSubscriber("subscriber", 1024);
    subscriber.serde = subscriber.statefulSerde = new DefaultStatefulStreamCodec<>();
    SweepableReservoir sr = subscriber.acquireReservoir("res", 1024);
    sr.setSink(new Sink<Object>() {

        @Override
        public void put(Object tuple) {
            assert (tuple.equals(message));
        }

        @Override
        public int getCount(boolean reset) {
            throw new UnsupportedOperationException("Not supported yet.");
        }
    });
    int size = buffer[0];
    size |= buffer[1] << 8;
    Assert.assertEquals("size", buffer.length - 2, size);
    subscriber.onMessage(buffer, 2, buffer.length - 2);
    sr.sweep();
    sr.sweep();
    for (int i = 0; i < 1024; i++) {
        publisher.put(message);
    }
    buffer = publisher.consume();
    assert (buffer.length == (size + 2) * 1024);
    int index = 0;
    for (int i = 0; i < 1024; i++) {
        size = buffer[index++];
        size |= buffer[index++] << 8;
        subscriber.onMessage(buffer, index, size);
        index += size;
    }
    sr.sweep();
    sr.sweep();
    for (int i = 0; i < 1024; i++) {
        publisher.put(message);
    }
    buffer = publisher.consume();
    assert (buffer.length == (size + 2) * 1024);
    index = 0;
    for (int i = 0; i < 1024; i++) {
        size = buffer[index++];
        size |= buffer[index++] << 8;
        subscriber.onMessage(buffer, index, size);
        index += size;
    }
    sr.sweep();
    sr.sweep();
}

Example 4 with DefaultStatefulStreamCodec

use of com.datatorrent.stram.codec.DefaultStatefulStreamCodec in project apex-core by apache.

the class FastStreamTest method testBufferServerStream.

/**
   * Test buffer server stream by sending
   * tuple on outputstream and receive same tuple from inputstream
   *
   * @throws Exception
   */
@Test
@SuppressWarnings({ "SleepWhileInLoop" })
public void testBufferServerStream() throws Exception {
    final StreamCodec<Object> serde = new DefaultStatefulStreamCodec<>();
    final AtomicInteger messageCount = new AtomicInteger();
    Sink<Object> sink = new Sink<Object>() {

        @Override
        public void put(Object tuple) {
            logger.debug("received: " + tuple);
            messageCount.incrementAndGet();
        }

        @Override
        public int getCount(boolean reset) {
            throw new UnsupportedOperationException("Not supported yet.");
        }
    };
    String streamName = "streamName";
    String upstreamNodeId = "upstreamNodeId";
    String downstreamNodeId = "downStreamNodeId";
    StreamContext issContext = new StreamContext(streamName);
    issContext.setSourceId(upstreamNodeId);
    issContext.setSinkId(downstreamNodeId);
    issContext.setFinishedWindowId(-1);
    issContext.setBufferServerAddress(InetSocketAddress.createUnresolved("localhost", bufferServerPort));
    issContext.put(StreamContext.CODEC, serde);
    issContext.put(StreamContext.EVENT_LOOP, eventloop);
    FastSubscriber subscriber = new FastSubscriber(downstreamNodeId, 1024);
    subscriber.setup(issContext);
    SweepableReservoir reservoir = subscriber.acquireReservoir("testReservoir", 1);
    reservoir.setSink(sink);
    StreamContext ossContext = new StreamContext(streamName);
    ossContext.setSourceId(upstreamNodeId);
    ossContext.setSinkId(downstreamNodeId);
    ossContext.setFinishedWindowId(-1);
    ossContext.setBufferServerAddress(InetSocketAddress.createUnresolved("localhost", bufferServerPort));
    ossContext.put(StreamContext.CODEC, serde);
    ossContext.put(StreamContext.EVENT_LOOP, eventloop);
    FastPublisher publisher = new FastPublisher(upstreamNodeId, 8);
    StreamContext publisherContext = new StreamContext(streamName);
    publisherContext.setSourceId(upstreamNodeId);
    publisherContext.setSinkId(downstreamNodeId);
    publisherContext.setBufferServerAddress(InetSocketAddress.createUnresolved("localhost", bufferServerPort));
    publisherContext.put(StreamContext.CODEC, serde);
    publisherContext.put(StreamContext.EVENT_LOOP, eventloop);
    publisher.setup(publisherContext);
    subscriber.activate(issContext);
    LOG.debug("input stream activated");
    publisher.activate(publisherContext);
    LOG.debug("output stream activated");
    LOG.debug("Sending hello message");
    publisher.put(StramTestSupport.generateBeginWindowTuple(upstreamNodeId, 0));
    publisher.put(StramTestSupport.generateTuple("hello", 0));
    publisher.put(StramTestSupport.generateEndWindowTuple(upstreamNodeId, 0));
    // it's a spurious tuple, presence of it should not affect the outcome of the test.
    publisher.put(StramTestSupport.generateBeginWindowTuple(upstreamNodeId, 1));
    for (int i = 0; i < 100; i++) {
        Tuple t = reservoir.sweep();
        if (t == null) {
            sleep(5);
            continue;
        }
        reservoir.remove();
        if (t instanceof EndWindowTuple) {
            break;
        }
    }
    eventloop.disconnect(publisher);
    eventloop.disconnect(subscriber);
    Assert.assertEquals("Received messages", 1, messageCount.get());
}