Examples with LogicalPlanConfiguration com.datatorrent.stram.plan.logical.LogicalPlanConfiguration used on opensource projects

Example 11 with LogicalPlanConfiguration

use of com.datatorrent.stram.plan.logical.LogicalPlanConfiguration in project apex-core by apache.

the class StramMiniClusterTest method testSetupShutdown.

@Test
public void testSetupShutdown() throws Exception {
    GetClusterNodesRequest request = Records.newRecord(GetClusterNodesRequest.class);
    ClientRMService clientRMService = yarnCluster.getResourceManager().getClientRMService();
    GetClusterNodesResponse response = clientRMService.getClusterNodes(request);
    List<NodeReport> nodeReports = response.getNodeReports();
    LOG.info("{}", nodeReports);
    for (NodeReport nr : nodeReports) {
        LOG.info("Node: {}", nr.getNodeId());
        LOG.info("Total memory: {}", nr.getCapability());
        LOG.info("Used memory: {}", nr.getUsed());
        LOG.info("Number containers: {}", nr.getNumContainers());
    }
    JarHelper jarHelper = new JarHelper();
    LOG.info("engine jar: {}", jarHelper.getJar(StreamingAppMaster.class));
    LOG.info("engine test jar: {}", jarHelper.getJar(StramMiniClusterTest.class));
    // create test application
    Properties dagProps = new Properties();
    // input module (ensure shutdown works while windows are generated)
    dagProps.put(StreamingApplication.APEX_PREFIX + "operator.numGen.classname", TestGeneratorInputOperator.class.getName());
    dagProps.put(StreamingApplication.APEX_PREFIX + "operator.numGen.maxTuples", "1");
    dagProps.put(StreamingApplication.APEX_PREFIX + "operator.module1.classname", GenericTestOperator.class.getName());
    dagProps.put(StreamingApplication.APEX_PREFIX + "operator.module2.classname", GenericTestOperator.class.getName());
    dagProps.put(StreamingApplication.APEX_PREFIX + "stream.fromNumGen.source", "numGen.outport");
    dagProps.put(StreamingApplication.APEX_PREFIX + "stream.fromNumGen.sinks", "module1.inport1");
    dagProps.put(StreamingApplication.APEX_PREFIX + "stream.n1n2.source", "module1.outport1");
    dagProps.put(StreamingApplication.APEX_PREFIX + "stream.n1n2.sinks", "module2.inport1");
    dagProps.setProperty(StreamingApplication.APEX_PREFIX + LogicalPlan.MASTER_MEMORY_MB.getName(), "128");
    dagProps.setProperty(StreamingApplication.APEX_PREFIX + LogicalPlan.CONTAINER_JVM_OPTIONS.getName(), "-Dlog4j.properties=custom_log4j.properties");
    dagProps.setProperty(StreamingApplication.APEX_PREFIX + "operator.*." + OperatorContext.MEMORY_MB.getName(), "64");
    dagProps.setProperty(StreamingApplication.APEX_PREFIX + "operator.*." + OperatorContext.VCORES.getName(), "1");
    dagProps.setProperty(StreamingApplication.APEX_PREFIX + "operator.*.port.*." + Context.PortContext.BUFFER_MEMORY_MB.getName(), "32");
    dagProps.setProperty(StreamingApplication.APEX_PREFIX + LogicalPlan.DEBUG.getName(), "true");
    LOG.info("dag properties: {}", dagProps);
    LOG.info("Initializing Client");
    LogicalPlanConfiguration tb = new LogicalPlanConfiguration(conf);
    tb.addFromProperties(dagProps, null);
    LogicalPlan dag = createDAG(tb);
    Configuration yarnConf = new Configuration(yarnCluster.getConfig());
    StramClient client = new StramClient(yarnConf, dag);
    try {
        client.start();
        if (StringUtils.isBlank(System.getenv("JAVA_HOME"))) {
            // JAVA_HOME not set in the yarn mini cluster
            client.javaCmd = "java";
        }
        LOG.info("Running client");
        client.startApplication();
        boolean result = client.monitorApplication();
        LOG.info("Client run completed. Result=" + result);
        Assert.assertTrue(result);
    } finally {
        client.stop();
    }
}

Example 12 with LogicalPlanConfiguration

use of com.datatorrent.stram.plan.logical.LogicalPlanConfiguration in project apex-core by apache.

the class StramMiniClusterTest method testWebService.

/**
   * Verify the web service deployment and lifecycle functionality
   *
   * @throws Exception
   */
//disabled due to web service init delay issue
@Ignore
@Test
public void testWebService() throws Exception {
    // single container topology of inline input and module
    Properties props = new Properties();
    props.put(StreamingApplication.APEX_PREFIX + "stream.input.classname", TestGeneratorInputOperator.class.getName());
    props.put(StreamingApplication.APEX_PREFIX + "stream.input.outputNode", "module1");
    props.put(StreamingApplication.APEX_PREFIX + "module.module1.classname", GenericTestOperator.class.getName());
    LOG.info("Initializing Client");
    LogicalPlanConfiguration tb = new LogicalPlanConfiguration(new Configuration(false));
    tb.addFromProperties(props, null);
    StramClient client = new StramClient(new Configuration(yarnCluster.getConfig()), createDAG(tb));
    if (StringUtils.isBlank(System.getenv("JAVA_HOME"))) {
        // JAVA_HOME not set in the yarn mini cluster
        client.javaCmd = "java";
    }
    try {
        client.start();
        client.startApplication();
        // attempt web service connection
        ApplicationReport appReport = client.getApplicationReport();
        // delay to give web service time to fully initialize
        Thread.sleep(5000);
        Client wsClient = Client.create();
        wsClient.setFollowRedirects(true);
        WebResource r = wsClient.resource("http://" + appReport.getTrackingUrl()).path(StramWebServices.PATH).path(StramWebServices.PATH_INFO);
        LOG.info("Requesting: " + r.getURI());
        ClientResponse response = r.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
        JSONObject json = response.getEntity(JSONObject.class);
        LOG.info("Got response: " + json.toString());
        assertEquals("incorrect number of elements", 1, json.length());
        assertEquals("appId", appReport.getApplicationId().toString(), json.get("id"));
        r = wsClient.resource("http://" + appReport.getTrackingUrl()).path(StramWebServices.PATH).path(StramWebServices.PATH_PHYSICAL_PLAN_OPERATORS);
        LOG.info("Requesting: " + r.getURI());
        response = r.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
        json = response.getEntity(JSONObject.class);
        LOG.info("Got response: " + json.toString());
    } finally {
        //LOG.info("waiting...");
        //synchronized (this) {
        //  this.wait();
        //}
        //boolean result = client.monitorApplication();
        client.killApplication();
        client.stop();
    }
}

Example 13 with LogicalPlanConfiguration

use of com.datatorrent.stram.plan.logical.LogicalPlanConfiguration in project apex-core by apache.

the class EmbeddedAppLauncherImpl method prepareDAG.

@Override
public DAG prepareDAG(StreamingApplication app, Configuration conf) throws Exception {
    if (app == null && conf == null) {
        throw new IllegalArgumentException("Require app or configuration to populate logical plan.");
    }
    if (conf == null) {
        conf = new Configuration(false);
    }
    LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
    String appName = app != null ? app.getClass().getName() : "unknown";
    lpc.prepareDAG(lp, app, appName);
    return lp;
}

Example 14 with LogicalPlanConfiguration

use of com.datatorrent.stram.plan.logical.LogicalPlanConfiguration in project apex-core by apache.

the class YarnAppLauncherImpl method launchApp.

@Override
public YarnAppHandleImpl launchApp(final StreamingApplication app, Configuration conf, Attribute.AttributeMap launchParameters) throws LauncherException {
    if (launchParameters != null) {
        for (Map.Entry<Attribute<?>, Object> entry : launchParameters.entrySet()) {
            String property = propMapping.get(entry.getKey());
            if (property != null) {
                setConfiguration(conf, property, entry.getValue());
            }
        }
    }
    try {
        String name = app.getClass().getName();
        StramAppLauncher appLauncher = new StramAppLauncher(name, conf);
        appLauncher.loadDependencies();
        StreamingAppFactory appFactory = new StreamingAppFactory(name, app.getClass()) {

            @Override
            public LogicalPlan createApp(LogicalPlanConfiguration planConfig) {
                return super.createApp(app, planConfig);
            }
        };
        ApplicationId appId = appLauncher.launchApp(appFactory);
        return new YarnAppHandleImpl(appId);
    } catch (Exception ex) {
        throw new LauncherException(ex);
    }
}

Example 15 with LogicalPlanConfiguration

use of com.datatorrent.stram.plan.logical.LogicalPlanConfiguration in project apex-core by apache.

the class AutoMetricTest method testInjectionOfDefaultMetricsAggregator.

@Test
public void testInjectionOfDefaultMetricsAggregator() throws Exception {
    LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(new Configuration());
    TestGeneratorInputOperator inputOperator = dag.addOperator("input", TestGeneratorInputOperator.class);
    OperatorWithMetricMethod o1 = dag.addOperator("o1", OperatorWithMetricMethod.class);
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
    dag.addStream("TestTuples", inputOperator.outport, o1.inport1);
    lpc.prepareDAG(dag, null, "AutoMetricTest");
    LogicalPlan.OperatorMeta o1meta = dag.getOperatorMeta("o1");
    Assert.assertNotNull("default aggregator injected", o1meta.getMetricAggregatorMeta().getAggregator());
}