use of com.datatorrent.bufferserver.storage.DiskStorage in project apex-core by apache.
the class StramLocalCluster method run.
@Override
@SuppressWarnings({ "SleepWhileInLoop", "ResultOfObjectAllocationIgnored" })
public void run(long runMillis) {
Thread eventLoopThread = null;
List<Thread> containerThreads = new LinkedList<>();
try {
if (!perContainerBufferServer) {
eventLoopThread = StreamingContainer.eventloop.start();
bufferServer = new Server(StreamingContainer.eventloop, 0, 1024 * 1024, 8);
try {
bufferServer.setSpoolStorage(new DiskStorage());
} catch (IOException e) {
throw new RuntimeException(e);
}
bufferServerAddress = InetSocketAddress.createUnresolved(LOCALHOST, bufferServer.run().getPort());
LOG.info("Buffer server started: {}", bufferServerAddress);
}
long endMillis = System.currentTimeMillis() + runMillis;
while (!appDone) {
for (String containerIdStr : dnmgr.containerStopRequests.values()) {
// teardown child thread
StreamingContainer c = childContainers.get(containerIdStr);
if (c != null) {
ContainerHeartbeatResponse r = new ContainerHeartbeatResponse();
r.shutdown = StreamingContainerUmbilicalProtocol.ShutdownType.ABORT;
c.processHeartbeatResponse(r);
}
dnmgr.containerStopRequests.remove(containerIdStr);
LOG.info("Container {} restart.", containerIdStr);
dnmgr.scheduleContainerRestart(containerIdStr);
// dnmgr.removeContainerAgent(containerIdStr);
}
// start containers
while (!dnmgr.containerStartRequests.isEmpty()) {
ContainerStartRequest cdr = dnmgr.containerStartRequests.poll();
if (cdr != null) {
new LocalStreamingContainerLauncher(cdr, containerThreads);
}
}
if (heartbeatMonitoringEnabled) {
// monitor child containers
dnmgr.monitorHeartbeat(false);
}
if (childContainers.isEmpty() && dnmgr.containerStartRequests.isEmpty()) {
appDone = true;
}
if (runMillis > 0 && System.currentTimeMillis() > endMillis) {
appDone = true;
}
try {
if (exitCondition != null && exitCondition.call()) {
LOG.info("Stopping on exit condition");
appDone = true;
}
} catch (Exception ex) {
break;
}
if (Thread.interrupted()) {
break;
}
if (!appDone) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
LOG.debug("Sleep interrupted", e);
break;
}
}
}
} finally {
for (LocalStreamingContainer lsc : childContainers.values()) {
injectShutdown.put(lsc.getContainerId(), lsc);
lsc.triggerHeartbeat();
}
for (Thread thread : containerThreads) {
try {
thread.join(1000);
} catch (InterruptedException e) {
LOG.debug("Wait for {} to terminate interrupted", thread, e);
}
if (thread.isAlive()) {
LOG.warn("Container thread {} is still alive", thread.getName());
}
}
try {
dnmgr.teardown();
} catch (RuntimeException e) {
LOG.warn("Exception during StreamingContainerManager teardown", e);
}
if (bufferServerAddress != null) {
try {
bufferServer.stop();
} catch (RuntimeException e) {
LOG.warn("Exception during BufferServer stop", e);
}
}
if (eventLoopThread != null) {
try {
StreamingContainer.eventloop.stop();
eventLoopThread.join(1000);
} catch (InterruptedException ie) {
LOG.debug("Wait for {} to terminate interrupted", eventLoopThread.getName(), ie);
} catch (RuntimeException e) {
LOG.warn("Exception during {} stop", StreamingContainer.eventloop, e);
}
if (StreamingContainer.eventloop.isActive()) {
LOG.warn("Event loop {} is still active", StreamingContainer.eventloop);
}
}
}
LOG.info("Application finished.");
}
use of com.datatorrent.bufferserver.storage.DiskStorage in project apex-core by apache.
the class StreamingContainer method setup.
@SuppressWarnings("unchecked")
public void setup(StreamingContainerContext ctx) {
containerContext = ctx;
/* add a request factory local to this container */
this.requestFactory = new RequestFactory();
ctx.attributes.put(ContainerContext.REQUEST_FACTORY, requestFactory);
heartbeatIntervalMillis = ctx.getValue(Context.DAGContext.HEARTBEAT_INTERVAL_MILLIS);
firstWindowMillis = ctx.startWindowMillis;
windowWidthMillis = ctx.getValue(Context.DAGContext.STREAMING_WINDOW_SIZE_MILLIS);
checkpointWindowCount = ctx.getValue(Context.DAGContext.CHECKPOINT_WINDOW_COUNT);
fastPublisherSubscriber = ctx.getValue(LogicalPlan.FAST_PUBLISHER_SUBSCRIBER);
Map<Class<?>, Class<? extends StringCodec<?>>> codecs = ctx.getValue(Context.DAGContext.STRING_CODECS);
StringCodecs.loadConverters(codecs);
try {
if (ctx.deployBufferServer) {
eventloop.start();
int bufferServerRAM = ctx.getValue(ContainerContext.BUFFER_SERVER_MB);
logger.debug("buffer server memory {}", bufferServerRAM);
int blockCount;
int blocksize;
if (bufferServerRAM < ContainerContext.BUFFER_SERVER_MB.defaultValue) {
blockCount = 8;
blocksize = bufferServerRAM / blockCount;
if (blocksize < 1) {
blocksize = 1;
}
} else {
blocksize = 64;
blockCount = bufferServerRAM / blocksize;
}
// start buffer server, if it was not set externally
bufferServer = new Server(eventloop, 0, blocksize * 1024 * 1024, blockCount);
bufferServer.setAuthToken(ctx.getValue(StreamingContainerContext.BUFFER_SERVER_TOKEN));
if (ctx.getValue(Context.DAGContext.BUFFER_SPOOLING)) {
bufferServer.setSpoolStorage(new DiskStorage());
}
bufferServerAddress = NetUtils.getConnectAddress(bufferServer.run());
logger.debug("Buffer server started: {}", bufferServerAddress);
}
} catch (IOException ex) {
logger.warn("deploy request failed due to {}", ex);
throw new IllegalStateException("Failed to deploy buffer server", ex);
}
for (Class<?> clazz : ContainerEvent.CONTAINER_EVENTS_LISTENERS) {
try {
Object newInstance = clazz.newInstance();
singletons.put(clazz.getName(), newInstance);
if (newInstance instanceof Component) {
components.add((Component<ContainerContext>) newInstance);
}
eventBus.subscribe(newInstance);
} catch (InstantiationException ex) {
logger.warn("Container Event Listener Instantiation", ex);
} catch (IllegalAccessException ex) {
logger.warn("Container Event Listener Instantiation", ex);
}
}
operateListeners(ctx, true);
}
Aggregations