Examples with LogicalPlan com.datatorrent.stram.plan.logical.LogicalPlan used on opensource projects

Example 46 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.

the class StramMiniClusterTest method testSetupShutdown.

@Test
public void testSetupShutdown() throws Exception {
    GetClusterNodesRequest request = Records.newRecord(GetClusterNodesRequest.class);
    ClientRMService clientRMService = yarnCluster.getResourceManager().getClientRMService();
    GetClusterNodesResponse response = clientRMService.getClusterNodes(request);
    List<NodeReport> nodeReports = response.getNodeReports();
    LOG.info("{}", nodeReports);
    for (NodeReport nr : nodeReports) {
        LOG.info("Node: {}", nr.getNodeId());
        LOG.info("Total memory: {}", nr.getCapability());
        LOG.info("Used memory: {}", nr.getUsed());
        LOG.info("Number containers: {}", nr.getNumContainers());
    }
    JarHelper jarHelper = new JarHelper();
    LOG.info("engine jar: {}", jarHelper.getJar(StreamingAppMaster.class));
    LOG.info("engine test jar: {}", jarHelper.getJar(StramMiniClusterTest.class));
    // create test application
    Properties dagProps = new Properties();
    // input module (ensure shutdown works while windows are generated)
    dagProps.put(StreamingApplication.APEX_PREFIX + "operator.numGen.classname", TestGeneratorInputOperator.class.getName());
    dagProps.put(StreamingApplication.APEX_PREFIX + "operator.numGen.maxTuples", "1");
    dagProps.put(StreamingApplication.APEX_PREFIX + "operator.module1.classname", GenericTestOperator.class.getName());
    dagProps.put(StreamingApplication.APEX_PREFIX + "operator.module2.classname", GenericTestOperator.class.getName());
    dagProps.put(StreamingApplication.APEX_PREFIX + "stream.fromNumGen.source", "numGen.outport");
    dagProps.put(StreamingApplication.APEX_PREFIX + "stream.fromNumGen.sinks", "module1.inport1");
    dagProps.put(StreamingApplication.APEX_PREFIX + "stream.n1n2.source", "module1.outport1");
    dagProps.put(StreamingApplication.APEX_PREFIX + "stream.n1n2.sinks", "module2.inport1");
    dagProps.setProperty(StreamingApplication.APEX_PREFIX + LogicalPlan.MASTER_MEMORY_MB.getName(), "128");
    dagProps.setProperty(StreamingApplication.APEX_PREFIX + LogicalPlan.CONTAINER_JVM_OPTIONS.getName(), "-Dlog4j.properties=custom_log4j.properties");
    dagProps.setProperty(StreamingApplication.APEX_PREFIX + "operator.*." + OperatorContext.MEMORY_MB.getName(), "64");
    dagProps.setProperty(StreamingApplication.APEX_PREFIX + "operator.*." + OperatorContext.VCORES.getName(), "1");
    dagProps.setProperty(StreamingApplication.APEX_PREFIX + "operator.*.port.*." + Context.PortContext.BUFFER_MEMORY_MB.getName(), "32");
    dagProps.setProperty(StreamingApplication.APEX_PREFIX + LogicalPlan.DEBUG.getName(), "true");
    LOG.info("dag properties: {}", dagProps);
    LOG.info("Initializing Client");
    LogicalPlanConfiguration tb = new LogicalPlanConfiguration(conf);
    tb.addFromProperties(dagProps, null);
    LogicalPlan dag = createDAG(tb);
    Configuration yarnConf = new Configuration(yarnCluster.getConfig());
    StramClient client = new StramClient(yarnConf, dag);
    try {
        client.start();
        if (StringUtils.isBlank(System.getenv("JAVA_HOME"))) {
            // JAVA_HOME not set in the yarn mini cluster
            client.javaCmd = "java";
        }
        LOG.info("Running client");
        client.startApplication();
        boolean result = client.monitorApplication();
        LOG.info("Client run completed. Result=" + result);
        Assert.assertTrue(result);
    } finally {
        client.stop();
    }
}

Example 47 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.

the class StreamCodecTest method getSingleOperatorDeployInfo.

private OperatorDeployInfo getSingleOperatorDeployInfo(Operator oper, StreamingContainerManager scm) {
    LogicalPlan dag = scm.getLogicalPlan();
    String id = dag.getMeta(oper).toString();
    PhysicalPlan plan = scm.getPhysicalPlan();
    List<PTOperator> operators = plan.getOperators(dag.getMeta(oper));
    Assert.assertEquals("number of operators " + id, 1, operators.size());
    PTOperator operator = operators.get(0);
    return getOperatorDeployInfo(operator, id, scm);
}

Example 48 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-malhar by apache.

the class SerDeTest method testSQLWithApexFactory.

@Test
public void testSQLWithApexFactory() throws IOException, ClassNotFoundException {
    File modelFile = new File("src/test/resources/model/model_file_csv.json");
    String model = FileUtils.readFileToString(modelFile);
    LogicalPlan dag = new LogicalPlan();
    SQLExecEnvironment.getEnvironment().withModel(model).executeSQL(dag, "SELECT STREAM ROWTIME, PRODUCT FROM ORDERS");
    dag.validate();
}

Example 49 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-malhar by apache.

the class SerDeTest method testSQLWithAPI.

@Test
public void testSQLWithAPI() throws ClassNotFoundException, IOException {
    LogicalPlan dag = new LogicalPlan();
    String schema = "{\"separator\":\",\",\"quoteChar\":\"\\\"\",\"fields\":[" + "{\"name\":\"RowTime\",\"type\":\"Date\",\"constraints\":{\"format\":\"dd/MM/yyyy hh:mm:ss Z\"}}," + "{\"name\":\"id\",\"type\":\"Integer\"},{\"name\":\"Product\",\"type\":\"String\"}," + "{\"name\":\"units\",\"type\":\"Integer\"}]}";
    Endpoint endpoint = new FileEndpoint("dummyFilePath", new CSVMessageFormat(schema));
    SQLExecEnvironment.getEnvironment().registerTable("ORDERS", endpoint).executeSQL(dag, "SELECT STREAM FLOOR(ROWTIME TO HOUR), SUBSTRING(PRODUCT, 0, 5) FROM ORDERS WHERE id > 3");
    dag.validate();
}

Example 50 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-malhar by apache.

the class ApexStreamImplTest method testAddOperator.

@Test
public void testAddOperator() {
    LogicalPlan dag = new LogicalPlan();
    TestOperator<String, Integer> firstOperator = new TestOperator<>();
    TestOperator<Integer, Date> secondOperator = new TestOperator<>();
    new ApexStreamImpl<String>().addOperator(firstOperator, null, firstOperator.output, name("first")).endWith(secondOperator, secondOperator.input, name("second")).with(DAG.Locality.THREAD_LOCAL).with(Context.OperatorContext.AUTO_RECORD, true).with("prop", "TestProp").populateDag(dag);
    Assert.assertTrue(dag.getAllOperators().size() == 2);
    Set<String> opNames = new HashSet<>();
    opNames.add("first");
    opNames.add("second");
    for (LogicalPlan.OperatorMeta operatorMeta : dag.getAllOperators()) {
        Assert.assertTrue(operatorMeta.getOperator() instanceof TestOperator);
        Assert.assertTrue(opNames.contains(operatorMeta.getName()));
        if (operatorMeta.getName().equals("second")) {
            // Assert the Context.OperatorContext.AUTO_RECORD attribute has been set to true for second operator
            Assert.assertTrue(operatorMeta.getAttributes().get(Context.OperatorContext.AUTO_RECORD));
            // Assert the prop has been set to TestProp for second operator
            Assert.assertEquals("TestProp", ((TestOperator) operatorMeta.getOperator()).prop);
        } else {
            // Assert the Context.OperatorContext.AUTO_RECORD attribute keeps as default false for first operator
            Assert.assertNull(operatorMeta.getAttributes().get(Context.OperatorContext.AUTO_RECORD));
            // Assert the prop has not been set for first operator
            Assert.assertNull(((TestOperator) operatorMeta.getOperator()).prop);
        }
    }
    Collection<LogicalPlan.StreamMeta> streams = dag.getAllStreams();
    // Assert there is only one stream
    Assert.assertTrue(streams.size() == 1);
    for (LogicalPlan.StreamMeta stream : streams) {
        // Assert the stream is from first operator to second operator
        Assert.assertEquals("first", stream.getSource().getOperatorMeta().getName());
        Collection<InputPortMeta> portMetaCollection = stream.getSinks();
        Assert.assertTrue(1 == portMetaCollection.size());
        for (InputPortMeta inputPortMeta : portMetaCollection) {
            Assert.assertEquals("second", inputPortMeta.getOperatorMeta().getName());
        }
        // Assert the stream is thread local
        Assert.assertTrue(stream.getLocality() == DAG.Locality.THREAD_LOCAL);
    }
}