Examples with LogicalPlan com.datatorrent.stram.plan.logical.LogicalPlan used on opensource projects

Example 56 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.

the class ModuleAppTest method validateTestApplication.

@Test
public void validateTestApplication() {
    Configuration conf = new Configuration(false);
    LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
    LogicalPlan dag = new LogicalPlan();
    lpc.prepareDAG(dag, new Application(), "TestApp");
    Assert.assertEquals(2, dag.getAllModules().size(), 2);
    Assert.assertEquals(5, dag.getAllOperators().size());
    Assert.assertEquals(4, dag.getAllStreams().size());
    dag.validate();
}

Example 57 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.

the class TestModuleExpansion method testLoadFromPropertiesFile.

@Test
public void testLoadFromPropertiesFile() throws IOException {
    Properties props = new Properties();
    String resourcePath = "/testModuleTopology.properties";
    InputStream is = this.getClass().getResourceAsStream(resourcePath);
    if (is == null) {
        throw new RuntimeException("Could not load " + resourcePath);
    }
    props.load(is);
    LogicalPlanConfiguration pb = new LogicalPlanConfiguration(new Configuration(false)).addFromProperties(props, null);
    LogicalPlan dag = new LogicalPlan();
    pb.populateDAG(dag);
    pb.prepareDAG(dag, null, "testApplication");
    dag.validate();
    validateTopLevelOperators(dag);
    validateTopLevelStreams(dag);
    validatePublicMethods(dag);
}

Example 58 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.

the class TestModuleExpansion method testLoadFromJson.

@Test
public void testLoadFromJson() throws Exception {
    String resourcePath = "/testModuleTopology.json";
    InputStream is = this.getClass().getResourceAsStream(resourcePath);
    if (is == null) {
        throw new RuntimeException("Could not load " + resourcePath);
    }
    StringWriter writer = new StringWriter();
    IOUtils.copy(is, writer);
    JSONObject json = new JSONObject(writer.toString());
    Configuration conf = new Configuration(false);
    conf.set(StreamingApplication.APEX_PREFIX + "operator.operator3.prop.myStringProperty", "o3StringFromConf");
    LogicalPlanConfiguration planConf = new LogicalPlanConfiguration(conf);
    LogicalPlan dag = planConf.createFromJson(json, "testLoadFromJson");
    planConf.prepareDAG(dag, null, "testApplication");
    dag.validate();
    validateTopLevelOperators(dag);
    validateTopLevelStreams(dag);
    validatePublicMethods(dag);
}

Example 59 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.

the class PluginTests method testDispatch.

@Test
public void testDispatch() throws InterruptedException {
    DebugPlugin debugPlugin = new DebugPlugin();
    StaticPluginLocator<? extends DAGExecutionPlugin> locator = new StaticPluginLocator<>(debugPlugin);
    ApexPluginDispatcher pluginManager = new DefaultApexPluginDispatcher(locator, new StramTestSupport.TestAppContext(new Attribute.AttributeMap.DefaultAttributeMap()), null, null);
    pluginManager.init(new Configuration());
    pluginManager.dispatch(new DAGExecutionEvent.StramExecutionEvent(new StramEvent(StramEvent.LogLevel.DEBUG) {

        @Override
        public String getType() {
            return "TestEvent";
        }
    }));
    pluginManager.dispatch(new DAGExecutionEvent.CommitExecutionEvent(1234));
    pluginManager.dispatch(new DAGExecutionEvent.HeartbeatExecutionEvent(new StreamingContainerUmbilicalProtocol.ContainerHeartbeat()));
    LogicalPlan plan = new LogicalPlan();
    pluginManager.dispatch(new ApexPluginDispatcher.DAGChangeEvent(plan));
    debugPlugin.waitForEventDelivery(10);
    pluginManager.stop();
    Assert.assertEquals(1, debugPlugin.getEventCount());
    Assert.assertEquals(1, debugPlugin.getHeartbeatCount());
    Assert.assertEquals(1, debugPlugin.getCommitCount());
    Assert.assertEquals(plan, debugPlugin.getLogicalPlan());
}

Example 60 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.

the class StreamingContainerManager method getCheckpointGroups.

protected Map<OperatorMeta, Set<OperatorMeta>> getCheckpointGroups() {
    if (this.checkpointGroups == null) {
        this.checkpointGroups = new HashMap<>();
        LogicalPlan dag = this.plan.getLogicalPlan();
        dag.resetNIndex();
        LogicalPlan.ValidationContext vc = new LogicalPlan.ValidationContext();
        for (OperatorMeta om : dag.getRootOperators()) {
            this.plan.getLogicalPlan().findStronglyConnected(om, vc);
        }
        for (Set<OperatorMeta> checkpointGroup : vc.stronglyConnected) {
            for (OperatorMeta om : checkpointGroup) {
                this.checkpointGroups.put(om, checkpointGroup);
            }
        }
    }
    return checkpointGroups;
}