Examples with Server com.datatorrent.bufferserver.server.Server used on opensource projects

Example 1 with Server

use of com.datatorrent.bufferserver.server.Server in project apex-core by apache.

the class StramLocalCluster method run.

@Override
@SuppressWarnings({ "SleepWhileInLoop", "ResultOfObjectAllocationIgnored" })
public void run(long runMillis) {
    Thread eventLoopThread = null;
    List<Thread> containerThreads = new LinkedList<>();
    try {
        if (!perContainerBufferServer) {
            eventLoopThread = StreamingContainer.eventloop.start();
            bufferServer = new Server(StreamingContainer.eventloop, 0, 1024 * 1024, 8);
            try {
                bufferServer.setSpoolStorage(new DiskStorage());
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
            bufferServerAddress = InetSocketAddress.createUnresolved(LOCALHOST, bufferServer.run().getPort());
            LOG.info("Buffer server started: {}", bufferServerAddress);
        }
        long endMillis = System.currentTimeMillis() + runMillis;
        while (!appDone) {
            for (String containerIdStr : dnmgr.containerStopRequests.values()) {
                // teardown child thread
                StreamingContainer c = childContainers.get(containerIdStr);
                if (c != null) {
                    ContainerHeartbeatResponse r = new ContainerHeartbeatResponse();
                    r.shutdown = StreamingContainerUmbilicalProtocol.ShutdownType.ABORT;
                    c.processHeartbeatResponse(r);
                }
                dnmgr.containerStopRequests.remove(containerIdStr);
                LOG.info("Container {} restart.", containerIdStr);
                dnmgr.scheduleContainerRestart(containerIdStr);
            // dnmgr.removeContainerAgent(containerIdStr);
            }
            // start containers
            while (!dnmgr.containerStartRequests.isEmpty()) {
                ContainerStartRequest cdr = dnmgr.containerStartRequests.poll();
                if (cdr != null) {
                    new LocalStreamingContainerLauncher(cdr, containerThreads);
                }
            }
            if (heartbeatMonitoringEnabled) {
                // monitor child containers
                dnmgr.monitorHeartbeat(false);
            }
            if (childContainers.isEmpty() && dnmgr.containerStartRequests.isEmpty()) {
                appDone = true;
            }
            if (runMillis > 0 && System.currentTimeMillis() > endMillis) {
                appDone = true;
            }
            try {
                if (exitCondition != null && exitCondition.call()) {
                    LOG.info("Stopping on exit condition");
                    appDone = true;
                }
            } catch (Exception ex) {
                break;
            }
            if (Thread.interrupted()) {
                break;
            }
            if (!appDone) {
                try {
                    Thread.sleep(1000);
                } catch (InterruptedException e) {
                    LOG.debug("Sleep interrupted", e);
                    break;
                }
            }
        }
    } finally {
        for (LocalStreamingContainer lsc : childContainers.values()) {
            injectShutdown.put(lsc.getContainerId(), lsc);
            lsc.triggerHeartbeat();
        }
        for (Thread thread : containerThreads) {
            try {
                thread.join(1000);
            } catch (InterruptedException e) {
                LOG.debug("Wait for {} to terminate interrupted", thread, e);
            }
            if (thread.isAlive()) {
                LOG.warn("Container thread {} is still alive", thread.getName());
            }
        }
        try {
            dnmgr.teardown();
        } catch (RuntimeException e) {
            LOG.warn("Exception during StreamingContainerManager teardown", e);
        }
        if (bufferServerAddress != null) {
            try {
                bufferServer.stop();
            } catch (RuntimeException e) {
                LOG.warn("Exception during BufferServer stop", e);
            }
        }
        if (eventLoopThread != null) {
            try {
                StreamingContainer.eventloop.stop();
                eventLoopThread.join(1000);
            } catch (InterruptedException ie) {
                LOG.debug("Wait for {} to terminate interrupted", eventLoopThread.getName(), ie);
            } catch (RuntimeException e) {
                LOG.warn("Exception during {} stop", StreamingContainer.eventloop, e);
            }
            if (StreamingContainer.eventloop.isActive()) {
                LOG.warn("Event loop {} is still active", StreamingContainer.eventloop);
            }
        }
    }
    LOG.info("Application finished.");
}

Example 2 with Server

use of com.datatorrent.bufferserver.server.Server in project apex-core by apache.

the class SubscriberTest method setupServerAndClients.

@BeforeClass
public static void setupServerAndClients() throws Exception {
    try {
        eventloopServer = DefaultEventLoop.createEventLoop("server");
        eventloopClient = DefaultEventLoop.createEventLoop("client");
    } catch (IOException ioe) {
        throw new RuntimeException(ioe);
    }
    eventloopServer.start();
    eventloopClient.start();
    instance = new Server(eventloopServer, 0, 64, 2);
    address = instance.run();
    assertTrue(address instanceof InetSocketAddress);
    assertFalse(address.isUnresolved());
}

Example 3 with Server

use of com.datatorrent.bufferserver.server.Server in project apex-core by apache.

the class GenericNodeTest method testBufferServerSubscriberActivationBeforeOperator.

@Test
public void testBufferServerSubscriberActivationBeforeOperator() throws InterruptedException, IOException {
    final String streamName = "streamName";
    final String upstreamNodeId = "upstreamNodeId";
    final String downstreamNodeId = "downStreamNodeId";
    EventLoop eventloop = DefaultEventLoop.createEventLoop("StreamTestEventLoop");
    ((DefaultEventLoop) eventloop).start();
    // find random port
    final Server bufferServer = new Server(eventloop, 0);
    final int bufferServerPort = bufferServer.run().getPort();
    final StreamCodec<Object> serde = new DefaultStatefulStreamCodec<>();
    final BlockingQueue<Object> tuples = new ArrayBlockingQueue<>(10);
    GenericTestOperator go = new GenericTestOperator();
    final GenericNode gn = new GenericNode(go, new com.datatorrent.stram.engine.OperatorContext(0, "operator", new DefaultAttributeMap(), null));
    gn.setId(1);
    Sink<Object> output = new Sink<Object>() {

        @Override
        public void put(Object tuple) {
            tuples.add(tuple);
        }

        @Override
        public int getCount(boolean reset) {
            return 0;
        }
    };
    InetSocketAddress socketAddress = new InetSocketAddress("localhost", bufferServerPort);
    StreamContext issContext = new StreamContext(streamName);
    issContext.setSourceId(upstreamNodeId);
    issContext.setSinkId(downstreamNodeId);
    issContext.setFinishedWindowId(-1);
    issContext.setBufferServerAddress(socketAddress);
    issContext.put(StreamContext.CODEC, serde);
    issContext.put(StreamContext.EVENT_LOOP, eventloop);
    StreamContext ossContext = new StreamContext(streamName);
    ossContext.setSourceId(upstreamNodeId);
    ossContext.setSinkId(downstreamNodeId);
    ossContext.setBufferServerAddress(socketAddress);
    ossContext.put(StreamContext.CODEC, serde);
    ossContext.put(StreamContext.EVENT_LOOP, eventloop);
    BufferServerPublisher oss = new BufferServerPublisher(upstreamNodeId, 1024);
    oss.setup(ossContext);
    oss.activate(ossContext);
    oss.put(new Tuple(MessageType.BEGIN_WINDOW, 0x1L));
    byte[] buff = PayloadTuple.getSerializedTuple(0, 1);
    buff[buff.length - 1] = (byte) 1;
    oss.put(buff);
    oss.put(new EndWindowTuple(0x1L));
    oss.put(new Tuple(MessageType.BEGIN_WINDOW, 0x2L));
    buff = PayloadTuple.getSerializedTuple(0, 1);
    buff[buff.length - 1] = (byte) 2;
    oss.put(buff);
    oss.put(new EndWindowTuple(0x2L));
    oss.put(new Tuple(MessageType.BEGIN_WINDOW, 0x3L));
    buff = PayloadTuple.getSerializedTuple(0, 1);
    buff[buff.length - 1] = (byte) 3;
    oss.put(buff);
    oss.put(new EndWindowTuple(0x3L));
    oss.put(new EndStreamTuple(0L));
    BufferServerSubscriber iss = new BufferServerSubscriber(downstreamNodeId, 1024);
    iss.setup(issContext);
    gn.connectInputPort(GenericTestOperator.IPORT1, iss.acquireReservoir("testReservoir", 10));
    gn.connectOutputPort(GenericTestOperator.OPORT1, output);
    SweepableReservoir tupleWait = iss.acquireReservoir("testReservoir2", 10);
    iss.activate(issContext);
    while (tupleWait.sweep() == null) {
        Thread.sleep(100);
    }
    gn.firstWindowMillis = 0;
    gn.windowWidthMillis = 100;
    Thread t = new Thread() {

        @Override
        public void run() {
            gn.activate();
            gn.run();
            gn.deactivate();
        }
    };
    t.start();
    t.join();
    Assert.assertEquals(10, tuples.size());
    List<Object> list = new ArrayList<>(tuples);
    Assert.assertEquals("Payload Tuple 1", 1, ((byte[]) list.get(1))[5]);
    Assert.assertEquals("Payload Tuple 2", 2, ((byte[]) list.get(4))[5]);
    Assert.assertEquals("Payload Tuple 3", 3, ((byte[]) list.get(7))[5]);
    if (bufferServer != null) {
        bufferServer.stop();
    }
    ((DefaultEventLoop) eventloop).stop();
}

Example 4 with Server

use of com.datatorrent.bufferserver.server.Server in project apex-core by apache.

the class FastStreamTest method setup.

@BeforeClass
public static void setup() throws InterruptedException, IOException, Exception {
    ((DefaultEventLoop) eventloop).start();
    // find random port
    bufferServer = new Server(eventloop, 0);
    InetSocketAddress bindAddr = bufferServer.run();
    bufferServerPort = bindAddr.getPort();
}

Example 5 with Server

use of com.datatorrent.bufferserver.server.Server in project apex-core by apache.

the class SocketStreamTest method setup.

@BeforeClass
public static void setup() throws InterruptedException, IOException, Exception {
    ((DefaultEventLoop) eventloop).start();
    // find random port
    bufferServer = new Server(eventloop, 0);
    InetSocketAddress bindAddr = bufferServer.run();
    bufferServerPort = bindAddr.getPort();
}