Examples with AppDataPushAgent com.datatorrent.stram.appdata.AppDataPushAgent used on opensource projects

Example 1 with AppDataPushAgent

use of com.datatorrent.stram.appdata.AppDataPushAgent in project apex-core by apache.

the class StreamingContainerManagerTest method testAppDataPush.

@Test
public void testAppDataPush() throws Exception {
    if (StramTestSupport.isInTravis()) {
        // disable this test in travis because of an intermittent problem similar to this:
        // http://stackoverflow.com/questions/32172925/travis-ci-sporadic-timeouts-to-localhost
        // We should remove this when we find a solution to this.
        LOG.info("Test testAppDataPush is disabled in Travis");
        return;
    }
    final String topic = "xyz";
    final List<String> messages = new ArrayList<>();
    EmbeddedWebSocketServer server = new EmbeddedWebSocketServer(0);
    server.setWebSocket(new WebSocket.OnTextMessage() {

        @Override
        public void onMessage(String data) {
            messages.add(data);
        }

        @Override
        public void onOpen(WebSocket.Connection connection) {
        }

        @Override
        public void onClose(int closeCode, String message) {
        }
    });
    try {
        server.start();
        int port = server.getPort();
        TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
        GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
        dag.addStream("o1.outport", o1.outport, o2.inport1);
        dag.setAttribute(LogicalPlan.METRICS_TRANSPORT, new AutoMetricBuiltInTransport(topic));
        dag.setAttribute(LogicalPlan.GATEWAY_CONNECT_ADDRESS, "localhost:" + port);
        dag.setAttribute(LogicalPlan.PUBSUB_CONNECT_TIMEOUT_MILLIS, 2000);
        StramLocalCluster lc = new StramLocalCluster(dag);
        StreamingContainerManager dnmgr = lc.dnmgr;
        StramAppContext appContext = new StramTestSupport.TestAppContext(dag.getAttributes());
        AppDataPushAgent pushAgent = new AppDataPushAgent(dnmgr, appContext);
        pushAgent.init();
        pushAgent.pushData();
        Thread.sleep(1000);
        Assert.assertTrue(messages.size() > 0);
        pushAgent.close();
        JSONObject message = new JSONObject(messages.get(0));
        Assert.assertEquals(topic, message.getString("topic"));
        Assert.assertEquals("publish", message.getString("type"));
        JSONObject data = message.getJSONObject("data");
        Assert.assertTrue(StringUtils.isNotBlank(data.getString("appId")));
        Assert.assertTrue(StringUtils.isNotBlank(data.getString("appUser")));
        Assert.assertTrue(StringUtils.isNotBlank(data.getString("appName")));
        JSONObject logicalOperators = data.getJSONObject("logicalOperators");
        for (String opName : new String[] { "o1", "o2" }) {
            JSONObject opObj = logicalOperators.getJSONObject(opName);
            Assert.assertTrue(opObj.has("totalTuplesProcessed"));
            Assert.assertTrue(opObj.has("totalTuplesEmitted"));
            Assert.assertTrue(opObj.has("tuplesProcessedPSMA"));
            Assert.assertTrue(opObj.has("tuplesEmittedPSMA"));
            Assert.assertTrue(opObj.has("latencyMA"));
        }
    } finally {
        server.stop();
    }
}

Example 2 with AppDataPushAgent

use of com.datatorrent.stram.appdata.AppDataPushAgent in project apex-core by apache.

the class StreamingContainerManagerTest method testCustomMetricsTransport.

@Test
public void testCustomMetricsTransport() throws Exception {
    TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    dag.addStream("o1.outport", o1.outport, o2.inport1);
    dag.setAttribute(LogicalPlan.METRICS_TRANSPORT, new TestMetricTransport("xyz"));
    StramLocalCluster lc = new StramLocalCluster(dag);
    StreamingContainerManager dnmgr = lc.dnmgr;
    StramAppContext appContext = new StramTestSupport.TestAppContext(dag.getAttributes());
    AppDataPushAgent pushAgent = new AppDataPushAgent(dnmgr, appContext);
    pushAgent.init();
    pushAgent.pushData();
    Assert.assertTrue(TestMetricTransport.messages.size() > 0);
    pushAgent.close();
    String msg = TestMetricTransport.messages.get(0);
    Assert.assertTrue(msg.startsWith("xyz:"));
}

Example 3 with AppDataPushAgent

use of com.datatorrent.stram.appdata.AppDataPushAgent in project apex-core by apache.

the class StreamingAppMasterService method serviceInit.

@Override
protected void serviceInit(Configuration conf) throws Exception {
    LOG.info("Application master" + ", appId=" + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId=" + appAttemptID.getAttemptId());
    FileInputStream fis = new FileInputStream("./" + LogicalPlan.SER_FILE_NAME);
    try {
        this.dag = LogicalPlan.read(fis);
    } finally {
        fis.close();
    }
    // "debug" simply dumps all data using LOG.info
    if (dag.isDebug()) {
        dumpOutDebugInfo();
    }
    dag.setAttribute(LogicalPlan.APPLICATION_ATTEMPT_ID, appAttemptID.getAttemptId());
    FSRecoveryHandler recoveryHandler = new FSRecoveryHandler(dag.assertAppPath(), conf);
    this.dnmgr = StreamingContainerManager.getInstance(recoveryHandler, dag, true);
    dag = this.dnmgr.getLogicalPlan();
    this.appContext = new ClusterAppContextImpl(dag.getAttributes());
    Map<Class<?>, Class<? extends StringCodec<?>>> codecs = dag.getAttributes().get(DAG.STRING_CODECS);
    StringCodecs.loadConverters(codecs);
    LOG.info("Starting application with {} operators in {} containers", dnmgr.getPhysicalPlan().getAllOperators().size(), dnmgr.getPhysicalPlan().getContainers().size());
    // Setup security configuration such as that for web security
    SecurityUtils.init(conf, dag.getValue(LogicalPlan.STRAM_HTTP_AUTHENTICATION));
    if (UserGroupInformation.isSecurityEnabled()) {
        // TODO :- Need to perform token renewal
        delegationTokenManager = new StramDelegationTokenManager(DELEGATION_KEY_UPDATE_INTERVAL, DELEGATION_TOKEN_MAX_LIFETIME, DELEGATION_TOKEN_RENEW_INTERVAL, DELEGATION_TOKEN_REMOVER_SCAN_INTERVAL);
    }
    this.nmClient = new NMClientAsyncImpl(new NMCallbackHandler());
    addService(nmClient);
    this.amRmClient = AMRMClient.createAMRMClient();
    addService(amRmClient);
    // start RPC server
    int rpcListenerCount = dag.getValue(DAGContext.HEARTBEAT_LISTENER_THREAD_COUNT);
    this.heartbeatListener = new StreamingContainerParent(this.getClass().getName(), dnmgr, delegationTokenManager, rpcListenerCount);
    addService(heartbeatListener);
    AutoMetric.Transport appDataPushTransport = dag.getValue(LogicalPlan.METRICS_TRANSPORT);
    if (appDataPushTransport != null) {
        this.appDataPushAgent = new AppDataPushAgent(dnmgr, appContext);
        addService(this.appDataPushAgent);
    }
    initApexPluginDispatcher();
    // Initialize all services added above
    super.serviceInit(conf);
}