Examples with StreamingContainerUmbilicalProtocol com.datatorrent.stram.api.StreamingContainerUmbilicalProtocol used on opensource projects

Example 1 with StreamingContainerUmbilicalProtocol

use of com.datatorrent.stram.api.StreamingContainerUmbilicalProtocol in project apex-core by apache.

the class StreamingContainer method main.

/**
   * Initialize container. Establishes heartbeat connection to the master
   * distribute through the callback address provided on the command line. Deploys
   * initial modules, then enters the heartbeat loop, which will only terminate
   * once container receives shutdown request from the master. On shutdown,
   * after exiting heartbeat loop, shutdown all modules and terminate
   * processing threads.
   *
   * @param args
   * @throws Throwable
   */
public static void main(String[] args) throws Throwable {
    StdOutErrLog.tieSystemOutAndErrToLog();
    logger.debug("PID: " + System.getenv().get("JVM_PID"));
    logger.info("Child starting with classpath: {}", System.getProperty("java.class.path"));
    String appPath = System.getProperty(PROP_APP_PATH);
    if (appPath == null) {
        logger.error("{} not set in container environment.", PROP_APP_PATH);
        System.exit(1);
    }
    // interpreted as unrecoverable container failure
    int exitStatus = 1;
    RecoverableRpcProxy rpcProxy = null;
    StreamingContainerUmbilicalProtocol umbilical = null;
    final String childId = System.getProperty(StreamingApplication.DT_PREFIX + "cid");
    try {
        rpcProxy = new RecoverableRpcProxy(appPath, new Configuration());
        umbilical = rpcProxy.getProxy();
        StreamingContainerContext ctx = umbilical.getInitContext(childId);
        StreamingContainer stramChild = new StreamingContainer(childId, umbilical);
        logger.debug("Container Context = {}", ctx);
        stramChild.setup(ctx);
        try {
            /* main thread enters heartbeat loop */
            stramChild.heartbeatLoop();
            exitStatus = 0;
        } finally {
            stramChild.teardown();
        }
    } catch (Error | Exception e) {
        LogFileInformation logFileInfo = LoggerUtil.getLogFileInformation();
        logger.error("Fatal {} in container!", (e instanceof Error) ? "Error" : "Exception", e);
        /* Report back any failures, for diagnostic purposes */
        try {
            umbilical.reportError(childId, null, ExceptionUtils.getStackTrace(e), logFileInfo);
        } catch (Exception ex) {
            logger.debug("Fail to log", ex);
        }
    } finally {
        if (rpcProxy != null) {
            rpcProxy.close();
        }
        DefaultMetricsSystem.shutdown();
        logger.info("Exit status for container: {}", exitStatus);
        LogManager.shutdown();
        if (exitStatus != 0) {
            System.exit(exitStatus);
        }
    }
}

Example 2 with StreamingContainerUmbilicalProtocol

use of com.datatorrent.stram.api.StreamingContainerUmbilicalProtocol in project apex-core by apache.

the class StramRecoveryTest method testRpcFailover.

@Test
public void testRpcFailover() throws Exception {
    String appPath = testMeta.getPath();
    Configuration conf = new Configuration(false);
    final AtomicBoolean timedout = new AtomicBoolean();
    StreamingContainerUmbilicalProtocol impl = Mockito.mock(StreamingContainerUmbilicalProtocol.class, Mockito.withSettings().extraInterfaces(Closeable.class));
    final Answer<Void> answer = new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) {
            LOG.debug("got call: " + invocation.getMethod());
            if (!timedout.get()) {
                try {
                    timedout.set(true);
                    Thread.sleep(1000);
                } catch (Exception e) {
                // ignore
                }
            //throw new RuntimeException("fail");
            }
            return null;
        }
    };
    Mockito.doAnswer(answer).when(impl).log("containerId", "timeout");
    Mockito.doAnswer(answer).when(impl).reportError("containerId", null, "timeout", null);
    Server server = new RPC.Builder(conf).setProtocol(StreamingContainerUmbilicalProtocol.class).setInstance(impl).setBindAddress("0.0.0.0").setPort(0).setNumHandlers(1).setVerbose(false).build();
    server.start();
    InetSocketAddress address = NetUtils.getConnectAddress(server);
    LOG.info("Mock server listening at " + address);
    int rpcTimeoutMillis = 500;
    int retryDelayMillis = 100;
    int retryTimeoutMillis = 500;
    FSRecoveryHandler recoveryHandler = new FSRecoveryHandler(appPath, conf);
    URI uri = RecoverableRpcProxy.toConnectURI(address, rpcTimeoutMillis, retryDelayMillis, retryTimeoutMillis);
    recoveryHandler.writeConnectUri(uri.toString());
    RecoverableRpcProxy rp = new RecoverableRpcProxy(appPath, conf);
    StreamingContainerUmbilicalProtocol protocolProxy = rp.getProxy();
    protocolProxy.log("containerId", "msg");
    // simulate socket read timeout
    try {
        protocolProxy.log("containerId", "timeout");
        Assert.fail("expected socket timeout");
    } catch (java.net.SocketTimeoutException e) {
    // expected
    }
    Assert.assertTrue("timedout", timedout.get());
    rp.close();
    // test success on retry
    timedout.set(false);
    retryTimeoutMillis = 1500;
    uri = RecoverableRpcProxy.toConnectURI(address, rpcTimeoutMillis, retryDelayMillis, retryTimeoutMillis);
    recoveryHandler.writeConnectUri(uri.toString());
    protocolProxy.log("containerId", "timeout");
    Assert.assertTrue("timedout", timedout.get());
    rp.close();
    String rpcTimeout = System.getProperty(RecoverableRpcProxy.RPC_TIMEOUT);
    String rpcRetryDelay = System.getProperty(RecoverableRpcProxy.RETRY_DELAY);
    String rpcRetryTimeout = System.getProperty(RecoverableRpcProxy.RETRY_TIMEOUT);
    System.setProperty(RecoverableRpcProxy.RPC_TIMEOUT, Integer.toString(500));
    System.setProperty(RecoverableRpcProxy.RETRY_DELAY, Long.toString(100));
    System.setProperty(RecoverableRpcProxy.RETRY_TIMEOUT, Long.toString(500));
    timedout.set(false);
    uri = RecoverableRpcProxy.toConnectURI(address);
    recoveryHandler.writeConnectUri(uri.toString());
    rp = new RecoverableRpcProxy(appPath, conf);
    protocolProxy = rp.getProxy();
    protocolProxy.reportError("containerId", null, "msg", null);
    try {
        protocolProxy.log("containerId", "timeout");
        Assert.fail("expected socket timeout");
    } catch (java.net.SocketTimeoutException e) {
    // expected
    }
    Assert.assertTrue("timedout", timedout.get());
    rp.close();
    timedout.set(false);
    System.setProperty(RecoverableRpcProxy.RETRY_TIMEOUT, Long.toString(1500));
    uri = RecoverableRpcProxy.toConnectURI(address);
    recoveryHandler.writeConnectUri(uri.toString());
    protocolProxy.reportError("containerId", null, "timeout", null);
    Assert.assertTrue("timedout", timedout.get());
    restoreSystemProperty(RecoverableRpcProxy.RPC_TIMEOUT, rpcTimeout);
    restoreSystemProperty(RecoverableRpcProxy.RETRY_DELAY, rpcRetryDelay);
    restoreSystemProperty(RecoverableRpcProxy.RETRY_TIMEOUT, rpcRetryTimeout);
    server.stop();
}