Examples with StreamingContainerContext com.datatorrent.stram.api.StreamingContainerUmbilicalProtocol.StreamingContainerContext used on opensource projects

Example 1 with StreamingContainerContext

use of com.datatorrent.stram.api.StreamingContainerUmbilicalProtocol.StreamingContainerContext in project apex-core by apache.

the class StreamingContainer method main.

/**
 * Initialize container. Establishes heartbeat connection to the master
 * distribute through the callback address provided on the command line. Deploys
 * initial modules, then enters the heartbeat loop, which will only terminate
 * once container receives shutdown request from the master. On shutdown,
 * after exiting heartbeat loop, shutdown all modules and terminate
 * processing threads.
 *
 * @param args
 * @throws Throwable
 */
public static void main(String[] args) throws Throwable {
    LoggerUtil.setupMDC("worker");
    StdOutErrLog.tieSystemOutAndErrToLog();
    logger.debug("PID: " + System.getenv().get("JVM_PID"));
    logger.info("Child starting with classpath: {}", System.getProperty("java.class.path"));
    String appPath = System.getProperty(PROP_APP_PATH);
    if (appPath == null) {
        logger.error("{} not set in container environment.", PROP_APP_PATH);
        System.exit(1);
    }
    // interpreted as unrecoverable container failure
    int exitStatus = 1;
    RecoverableRpcProxy rpcProxy = null;
    StreamingContainerUmbilicalProtocol umbilical = null;
    final String childId = System.getProperty(StreamingApplication.DT_PREFIX + "cid");
    try {
        rpcProxy = new RecoverableRpcProxy(appPath, new Configuration());
        umbilical = rpcProxy.getProxy();
        StreamingContainerContext ctx = umbilical.getInitContext(childId);
        StreamingContainer stramChild = new StreamingContainer(childId, umbilical);
        logger.debug("Container Context = {}", ctx);
        stramChild.setup(ctx);
        try {
            /* main thread enters heartbeat loop */
            stramChild.heartbeatLoop();
            exitStatus = 0;
        } finally {
            stramChild.teardown();
        }
    } catch (Error | Exception e) {
        LogFileInformation logFileInfo = LoggerUtil.getLogFileInformation();
        logger.error("Fatal {} in container!", (e instanceof Error) ? "Error" : "Exception", e);
        /* Report back any failures, for diagnostic purposes */
        try {
            umbilical.reportError(childId, null, ExceptionUtils.getStackTrace(e), logFileInfo);
        } catch (Exception ex) {
            logger.debug("Fail to log", ex);
        }
    } finally {
        if (rpcProxy != null) {
            rpcProxy.close();
        }
        DefaultMetricsSystem.shutdown();
        logger.info("Exit status for container: {}", exitStatus);
        LogManager.shutdown();
        if (exitStatus != 0) {
            System.exit(exitStatus);
        }
    }
}

Example 2 with StreamingContainerContext

use of com.datatorrent.stram.api.StreamingContainerUmbilicalProtocol.StreamingContainerContext in project apex-core by apache.

the class StreamingContainerManager method newStreamingContainerContext.

private StreamingContainerContext newStreamingContainerContext(PTContainer container) {
    try {
        int bufferServerMemory = 0;
        Iterator<PTOperator> operatorIterator = container.getOperators().iterator();
        while (operatorIterator.hasNext()) {
            bufferServerMemory += operatorIterator.next().getBufferServerMemory();
        }
        LOG.debug("Buffer Server Memory {}", bufferServerMemory);
        // the logical plan is not to be serialized via RPC, clone attributes only
        StreamingContainerContext scc = new StreamingContainerContext(plan.getLogicalPlan().getAttributes().clone(), null);
        scc.attributes.put(ContainerContext.IDENTIFIER, container.getExternalId());
        scc.attributes.put(ContainerContext.BUFFER_SERVER_MB, bufferServerMemory);
        scc.attributes.put(ContainerContext.BUFFER_SERVER_TOKEN, container.getBufferServerToken());
        scc.startWindowMillis = this.vars.windowStartMillis;
        return scc;
    } catch (CloneNotSupportedException ex) {
        throw new RuntimeException("Cannot clone DAG attributes", ex);
    }
}

Example 3 with StreamingContainerContext

use of com.datatorrent.stram.api.StreamingContainerUmbilicalProtocol.StreamingContainerContext in project apex-core by apache.

the class StreamingContainer method setup.

@SuppressWarnings("unchecked")
public void setup(StreamingContainerContext ctx) {
    containerContext = ctx;
    /* add a request factory local to this container */
    this.requestFactory = new RequestFactory();
    ctx.attributes.put(ContainerContext.REQUEST_FACTORY, requestFactory);
    heartbeatIntervalMillis = ctx.getValue(Context.DAGContext.HEARTBEAT_INTERVAL_MILLIS);
    firstWindowMillis = ctx.startWindowMillis;
    windowWidthMillis = ctx.getValue(Context.DAGContext.STREAMING_WINDOW_SIZE_MILLIS);
    checkpointWindowCount = ctx.getValue(Context.DAGContext.CHECKPOINT_WINDOW_COUNT);
    fastPublisherSubscriber = ctx.getValue(LogicalPlan.FAST_PUBLISHER_SUBSCRIBER);
    Map<Class<?>, Class<? extends StringCodec<?>>> codecs = ctx.getValue(Context.DAGContext.STRING_CODECS);
    StringCodecs.loadConverters(codecs);
    try {
        if (ctx.deployBufferServer) {
            eventloop.start();
            int bufferServerRAM = ctx.getValue(ContainerContext.BUFFER_SERVER_MB);
            logger.debug("buffer server memory {}", bufferServerRAM);
            int blockCount;
            int blocksize;
            if (bufferServerRAM < ContainerContext.BUFFER_SERVER_MB.defaultValue) {
                blockCount = 8;
                blocksize = bufferServerRAM / blockCount;
                if (blocksize < 1) {
                    blocksize = 1;
                }
            } else {
                blocksize = 64;
                blockCount = bufferServerRAM / blocksize;
            }
            // start buffer server, if it was not set externally
            bufferServer = new Server(eventloop, 0, blocksize * 1024 * 1024, blockCount);
            bufferServer.setAuthToken(ctx.getValue(StreamingContainerContext.BUFFER_SERVER_TOKEN));
            if (ctx.getValue(Context.DAGContext.BUFFER_SPOOLING)) {
                bufferServer.setSpoolStorage(new DiskStorage());
            }
            bufferServerAddress = NetUtils.getConnectAddress(bufferServer.run());
            logger.debug("Buffer server started: {}", bufferServerAddress);
        }
    } catch (IOException ex) {
        logger.warn("deploy request failed due to {}", ex);
        throw new IllegalStateException("Failed to deploy buffer server", ex);
    }
    for (Class<?> clazz : ContainerEvent.CONTAINER_EVENTS_LISTENERS) {
        try {
            Object newInstance = clazz.newInstance();
            singletons.put(clazz.getName(), newInstance);
            if (newInstance instanceof Component) {
                components.add((Component<ContainerContext>) newInstance);
            }
            eventBus.subscribe(newInstance);
        } catch (InstantiationException ex) {
            logger.warn("Container Event Listener Instantiation", ex);
        } catch (IllegalAccessException ex) {
            logger.warn("Container Event Listener Instantiation", ex);
        }
    }
    operateListeners(ctx, true);
}