Examples with LogicalPlanConfiguration com.datatorrent.stram.plan.logical.LogicalPlanConfiguration used on opensource projects

Example 1 with LogicalPlanConfiguration

use of com.datatorrent.stram.plan.logical.LogicalPlanConfiguration in project apex-core by apache.

the class AutoMetricTest method testMetricsAggregations.

@Test
@Ignore
public void testMetricsAggregations() throws Exception {
    CountDownLatch latch = new CountDownLatch(2);
    LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(new Configuration());
    TestGeneratorInputOperator inputOperator = dag.addOperator("input", TestGeneratorInputOperator.class);
    OperatorWithMetrics o1 = dag.addOperator("o1", OperatorWithMetrics.class);
    MockAggregator aggregator = new MockAggregator(latch);
    dag.setOperatorAttribute(o1, Context.OperatorContext.METRICS_AGGREGATOR, aggregator);
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
    dag.setOperatorAttribute(o1, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<TestGeneratorInputOperator>(2));
    dag.addStream("TestTuples", inputOperator.outport, o1.inport1);
    lpc.prepareDAG(dag, null, "AutoMetricTest");
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.runAsync();
    latch.await();
    Assert.assertEquals("progress", 2L, ((Long) aggregator.result.get("progress")).longValue());
    lc.shutdown();
}

Example 2 with LogicalPlanConfiguration

use of com.datatorrent.stram.plan.logical.LogicalPlanConfiguration in project apex-core by apache.

the class AutoMetricTest method testMetrics.

@Test
public void testMetrics() throws Exception {
    CountDownLatch latch = new CountDownLatch(1);
    LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(new Configuration());
    TestGeneratorInputOperator inputOperator = dag.addOperator("input", TestGeneratorInputOperator.class);
    OperatorWithMetrics o1 = dag.addOperator("o1", OperatorWithMetrics.class);
    MockAggregator aggregator = new MockAggregator(latch);
    dag.setOperatorAttribute(o1, Context.OperatorContext.METRICS_AGGREGATOR, aggregator);
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
    dag.addStream("TestTuples", inputOperator.outport, o1.inport1);
    lpc.prepareDAG(dag, null, "AutoMetricTest");
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.runAsync();
    latch.await();
    Assert.assertEquals("progress", 1L, ((Long) aggregator.result.get("progress")).longValue());
    lc.shutdown();
}

Example 3 with LogicalPlanConfiguration

use of com.datatorrent.stram.plan.logical.LogicalPlanConfiguration in project apex-core by apache.

the class AutoMetricTest method testMetricsAnnotatedMethod.

@Test
public void testMetricsAnnotatedMethod() throws Exception {
    CountDownLatch latch = new CountDownLatch(1);
    LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(new Configuration());
    TestGeneratorInputOperator inputOperator = dag.addOperator("input", TestGeneratorInputOperator.class);
    OperatorWithMetricMethod o1 = dag.addOperator("o1", OperatorWithMetricMethod.class);
    MockAggregator aggregator = new MockAggregator(latch);
    dag.setOperatorAttribute(o1, Context.OperatorContext.METRICS_AGGREGATOR, aggregator);
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
    dag.addStream("TestTuples", inputOperator.outport, o1.inport1);
    lpc.prepareDAG(dag, null, "AutoMetricTest");
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.runAsync();
    latch.await();
    Assert.assertEquals("myMetric", 3, ((Integer) aggregator.result.get("myMetric")).intValue());
    lc.shutdown();
}

Example 4 with LogicalPlanConfiguration

use of com.datatorrent.stram.plan.logical.LogicalPlanConfiguration in project apex-core by apache.

the class OperatorDiscoveryTest method testLogicalPlanConfiguration.

@Test
public void testLogicalPlanConfiguration() throws Exception {
    TestOperator<String, Map<String, Number>> bean = new InputTestOperator<>();
    bean.map.put("key1", new Structured());
    bean.stringArray = new String[] { "one", "two", "three" };
    bean.stringList = Lists.newArrayList("four", "five");
    bean.props = new Properties();
    bean.props.setProperty("key1", "value1");
    bean.structuredArray = new Structured[] { new Structured() };
    bean.genericArray = new String[] { "s1" };
    bean.structuredArray[0].name = "s1";
    bean.color = Color.BLUE;
    bean.booleanProp = true;
    bean.realName = "abc";
    ObjectMapper mapper = ObjectMapperFactory.getOperatorValueSerializer();
    String s = mapper.writeValueAsString(bean);
    //    LOG.debug(new JSONObject(s).toString(2));
    //
    Assert.assertTrue("Shouldn't contain field 'realName' !", !s.contains("realName"));
    Assert.assertTrue("Should contain property 'alias' !", s.contains("alias"));
    Assert.assertTrue("Shouldn't contain property 'getterOnly' !", !s.contains("getterOnly"));
    JSONObject jsonObj = new JSONObject(s);
    // create the json dag representation
    JSONObject jsonPlan = new JSONObject();
    jsonPlan.put("streams", new JSONArray());
    JSONObject jsonOper = new JSONObject();
    jsonOper.put("name", "Test Operator");
    jsonOper.put("class", InputTestOperator.class.getName());
    jsonOper.put("properties", jsonObj);
    jsonPlan.put("operators", new JSONArray(Lists.newArrayList(jsonOper)));
    Configuration conf = new Configuration(false);
    LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
    // create logical plan from the json
    LogicalPlan lp = lpc.createFromJson(jsonPlan, "jsontest");
    OperatorMeta om = lp.getOperatorMeta("Test Operator");
    Assert.assertTrue(om.getOperator() instanceof InputTestOperator);
    @SuppressWarnings("rawtypes") TestOperator beanBack = (InputTestOperator) om.getOperator();
    // The operator deserialized back from json should be same as original operator
    Assert.assertEquals(bean.map, beanBack.map);
    Assert.assertArrayEquals(bean.stringArray, beanBack.stringArray);
    Assert.assertEquals(bean.stringList, beanBack.stringList);
    Assert.assertEquals(bean.props, beanBack.props);
    Assert.assertArrayEquals(bean.structuredArray, beanBack.structuredArray);
    Assert.assertArrayEquals(bean.genericArray, beanBack.genericArray);
    Assert.assertEquals(bean.color, beanBack.color);
    Assert.assertEquals(bean.booleanProp, beanBack.booleanProp);
    Assert.assertEquals(bean.realName, beanBack.realName);
    Assert.assertEquals(bean.getterOnly, beanBack.getterOnly);
}

Example 5 with LogicalPlanConfiguration

use of com.datatorrent.stram.plan.logical.LogicalPlanConfiguration in project apex-core by apache.

the class TestModuleProperties method testModuleProperties.

@Test
public void testModuleProperties() {
    Configuration conf = new Configuration(false);
    conf.set(StreamingApplication.APEX_PREFIX + "operator.o1.prop.myStringProperty", "myStringPropertyValue");
    conf.set(StreamingApplication.APEX_PREFIX + "operator.o2.prop.stringArrayField", "a,b,c");
    conf.set(StreamingApplication.APEX_PREFIX + "operator.o2.prop.mapProperty.key1", "key1Val");
    conf.set(StreamingApplication.APEX_PREFIX + "operator.o2.prop.mapProperty(key1.dot)", "key1dotVal");
    conf.set(StreamingApplication.APEX_PREFIX + "operator.o2.prop.mapProperty(key2.dot)", "key2dotVal");
    LogicalPlan dag = new LogicalPlan();
    TestModules.GenericModule o1 = dag.addModule("o1", new TestModules.GenericModule());
    TestModules.ValidationTestModule o2 = dag.addModule("o2", new TestModules.ValidationTestModule());
    LogicalPlanConfiguration pb = new LogicalPlanConfiguration(conf);
    pb.setModuleProperties(dag, "testSetOperatorProperties");
    Assert.assertEquals("o1.myStringProperty", "myStringPropertyValue", o1.getMyStringProperty());
    Assert.assertArrayEquals("o2.stringArrayField", new String[] { "a", "b", "c" }, o2.getStringArrayField());
    Assert.assertEquals("o2.mapProperty.key1", "key1Val", o2.getMapProperty().get("key1"));
    Assert.assertEquals("o2.mapProperty(key1.dot)", "key1dotVal", o2.getMapProperty().get("key1.dot"));
    Assert.assertEquals("o2.mapProperty(key2.dot)", "key2dotVal", o2.getMapProperty().get("key2.dot"));
}