Search in sources :

Example 1 with DiskStorage

use of com.datatorrent.bufferserver.storage.DiskStorage in project apex-core by apache.

the class StramLocalCluster method run.

@Override
@SuppressWarnings({ "SleepWhileInLoop", "ResultOfObjectAllocationIgnored" })
public void run(long runMillis) {
    Thread eventLoopThread = null;
    List<Thread> containerThreads = new LinkedList<>();
    try {
        if (!perContainerBufferServer) {
            eventLoopThread = StreamingContainer.eventloop.start();
            bufferServer = new Server(StreamingContainer.eventloop, 0, 1024 * 1024, 8);
            try {
                bufferServer.setSpoolStorage(new DiskStorage());
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
            bufferServerAddress = InetSocketAddress.createUnresolved(LOCALHOST, bufferServer.run().getPort());
            LOG.info("Buffer server started: {}", bufferServerAddress);
        }
        long endMillis = System.currentTimeMillis() + runMillis;
        while (!appDone) {
            for (String containerIdStr : dnmgr.containerStopRequests.values()) {
                // teardown child thread
                StreamingContainer c = childContainers.get(containerIdStr);
                if (c != null) {
                    ContainerHeartbeatResponse r = new ContainerHeartbeatResponse();
                    r.shutdown = StreamingContainerUmbilicalProtocol.ShutdownType.ABORT;
                    c.processHeartbeatResponse(r);
                }
                dnmgr.containerStopRequests.remove(containerIdStr);
                LOG.info("Container {} restart.", containerIdStr);
                dnmgr.scheduleContainerRestart(containerIdStr);
            // dnmgr.removeContainerAgent(containerIdStr);
            }
            // start containers
            while (!dnmgr.containerStartRequests.isEmpty()) {
                ContainerStartRequest cdr = dnmgr.containerStartRequests.poll();
                if (cdr != null) {
                    new LocalStreamingContainerLauncher(cdr, containerThreads);
                }
            }
            if (heartbeatMonitoringEnabled) {
                // monitor child containers
                dnmgr.monitorHeartbeat(false);
            }
            if (childContainers.isEmpty() && dnmgr.containerStartRequests.isEmpty()) {
                appDone = true;
            }
            if (runMillis > 0 && System.currentTimeMillis() > endMillis) {
                appDone = true;
            }
            try {
                if (exitCondition != null && exitCondition.call()) {
                    LOG.info("Stopping on exit condition");
                    appDone = true;
                }
            } catch (Exception ex) {
                break;
            }
            if (Thread.interrupted()) {
                break;
            }
            if (!appDone) {
                try {
                    Thread.sleep(1000);
                } catch (InterruptedException e) {
                    LOG.debug("Sleep interrupted", e);
                    break;
                }
            }
        }
    } finally {
        for (LocalStreamingContainer lsc : childContainers.values()) {
            injectShutdown.put(lsc.getContainerId(), lsc);
            lsc.triggerHeartbeat();
        }
        for (Thread thread : containerThreads) {
            try {
                thread.join(1000);
            } catch (InterruptedException e) {
                LOG.debug("Wait for {} to terminate interrupted", thread, e);
            }
            if (thread.isAlive()) {
                LOG.warn("Container thread {} is still alive", thread.getName());
            }
        }
        try {
            dnmgr.teardown();
        } catch (RuntimeException e) {
            LOG.warn("Exception during StreamingContainerManager teardown", e);
        }
        if (bufferServerAddress != null) {
            try {
                bufferServer.stop();
            } catch (RuntimeException e) {
                LOG.warn("Exception during BufferServer stop", e);
            }
        }
        if (eventLoopThread != null) {
            try {
                StreamingContainer.eventloop.stop();
                eventLoopThread.join(1000);
            } catch (InterruptedException ie) {
                LOG.debug("Wait for {} to terminate interrupted", eventLoopThread.getName(), ie);
            } catch (RuntimeException e) {
                LOG.warn("Exception during {} stop", StreamingContainer.eventloop, e);
            }
            if (StreamingContainer.eventloop.isActive()) {
                LOG.warn("Event loop {} is still active", StreamingContainer.eventloop);
            }
        }
    }
    LOG.info("Application finished.");
}
Also used : StreamingContainer(com.datatorrent.stram.engine.StreamingContainer) ContainerStartRequest(com.datatorrent.stram.StreamingContainerAgent.ContainerStartRequest) Server(com.datatorrent.bufferserver.server.Server) ContainerHeartbeatResponse(com.datatorrent.stram.api.StreamingContainerUmbilicalProtocol.ContainerHeartbeatResponse) IOException(java.io.IOException) LinkedList(java.util.LinkedList) IOException(java.io.IOException) DiskStorage(com.datatorrent.bufferserver.storage.DiskStorage)

Example 2 with DiskStorage

use of com.datatorrent.bufferserver.storage.DiskStorage in project apex-core by apache.

the class StreamingContainer method setup.

@SuppressWarnings("unchecked")
public void setup(StreamingContainerContext ctx) {
    containerContext = ctx;
    /* add a request factory local to this container */
    this.requestFactory = new RequestFactory();
    ctx.attributes.put(ContainerContext.REQUEST_FACTORY, requestFactory);
    heartbeatIntervalMillis = ctx.getValue(Context.DAGContext.HEARTBEAT_INTERVAL_MILLIS);
    firstWindowMillis = ctx.startWindowMillis;
    windowWidthMillis = ctx.getValue(Context.DAGContext.STREAMING_WINDOW_SIZE_MILLIS);
    checkpointWindowCount = ctx.getValue(Context.DAGContext.CHECKPOINT_WINDOW_COUNT);
    fastPublisherSubscriber = ctx.getValue(LogicalPlan.FAST_PUBLISHER_SUBSCRIBER);
    Map<Class<?>, Class<? extends StringCodec<?>>> codecs = ctx.getValue(Context.DAGContext.STRING_CODECS);
    StringCodecs.loadConverters(codecs);
    try {
        if (ctx.deployBufferServer) {
            eventloop.start();
            int bufferServerRAM = ctx.getValue(ContainerContext.BUFFER_SERVER_MB);
            logger.debug("buffer server memory {}", bufferServerRAM);
            int blockCount;
            int blocksize;
            if (bufferServerRAM < ContainerContext.BUFFER_SERVER_MB.defaultValue) {
                blockCount = 8;
                blocksize = bufferServerRAM / blockCount;
                if (blocksize < 1) {
                    blocksize = 1;
                }
            } else {
                blocksize = 64;
                blockCount = bufferServerRAM / blocksize;
            }
            // start buffer server, if it was not set externally
            bufferServer = new Server(eventloop, 0, blocksize * 1024 * 1024, blockCount);
            bufferServer.setAuthToken(ctx.getValue(StreamingContainerContext.BUFFER_SERVER_TOKEN));
            if (ctx.getValue(Context.DAGContext.BUFFER_SPOOLING)) {
                bufferServer.setSpoolStorage(new DiskStorage());
            }
            bufferServerAddress = NetUtils.getConnectAddress(bufferServer.run());
            logger.debug("Buffer server started: {}", bufferServerAddress);
        }
    } catch (IOException ex) {
        logger.warn("deploy request failed due to {}", ex);
        throw new IllegalStateException("Failed to deploy buffer server", ex);
    }
    for (Class<?> clazz : ContainerEvent.CONTAINER_EVENTS_LISTENERS) {
        try {
            Object newInstance = clazz.newInstance();
            singletons.put(clazz.getName(), newInstance);
            if (newInstance instanceof Component) {
                components.add((Component<ContainerContext>) newInstance);
            }
            eventBus.subscribe(newInstance);
        } catch (InstantiationException ex) {
            logger.warn("Container Event Listener Instantiation", ex);
        } catch (IllegalAccessException ex) {
            logger.warn("Container Event Listener Instantiation", ex);
        }
    }
    operateListeners(ctx, true);
}
Also used : Server(com.datatorrent.bufferserver.server.Server) IOException(java.io.IOException) Checkpoint(com.datatorrent.stram.api.Checkpoint) ContainerContext(com.datatorrent.stram.api.ContainerContext) StreamingContainerContext(com.datatorrent.stram.api.StreamingContainerUmbilicalProtocol.StreamingContainerContext) StringCodec(com.datatorrent.api.StringCodec) RequestFactory(com.datatorrent.stram.api.RequestFactory) Component(com.datatorrent.api.Component) DiskStorage(com.datatorrent.bufferserver.storage.DiskStorage)

Aggregations

Server (com.datatorrent.bufferserver.server.Server)2 DiskStorage (com.datatorrent.bufferserver.storage.DiskStorage)2 IOException (java.io.IOException)2 Component (com.datatorrent.api.Component)1 StringCodec (com.datatorrent.api.StringCodec)1 ContainerStartRequest (com.datatorrent.stram.StreamingContainerAgent.ContainerStartRequest)1 Checkpoint (com.datatorrent.stram.api.Checkpoint)1 ContainerContext (com.datatorrent.stram.api.ContainerContext)1 RequestFactory (com.datatorrent.stram.api.RequestFactory)1 ContainerHeartbeatResponse (com.datatorrent.stram.api.StreamingContainerUmbilicalProtocol.ContainerHeartbeatResponse)1 StreamingContainerContext (com.datatorrent.stram.api.StreamingContainerUmbilicalProtocol.StreamingContainerContext)1 StreamingContainer (com.datatorrent.stram.engine.StreamingContainer)1 LinkedList (java.util.LinkedList)1