Examples with GenericTestOperator com.datatorrent.stram.engine.GenericTestOperator used on opensource projects

Example 76 with GenericTestOperator

use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.

the class StreamingContainerManagerTest method testPhysicalPropertyUpdate.

@Test
public void testPhysicalPropertyUpdate() throws Exception {
    TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    dag.addStream("o1.outport", o1.outport, o2.inport1);
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.runAsync();
    StreamingContainerManager dnmgr = lc.dnmgr;
    Map<Integer, PTOperator> operatorMap = dnmgr.getPhysicalPlan().getAllOperators();
    for (PTOperator p : operatorMap.values()) {
        StramTestSupport.waitForActivation(lc, p);
    }
    dnmgr.setPhysicalOperatorProperty(lc.getPlanOperators(dag.getMeta(o1)).get(0).getId(), "maxTuples", "2");
    Future<?> future = dnmgr.getPhysicalOperatorProperty(lc.getPlanOperators(dag.getMeta(o1)).get(0).getId(), "maxTuples", 10000);
    Object object = future.get(10000, TimeUnit.MILLISECONDS);
    Assert.assertNotNull(object);
    @SuppressWarnings("unchecked") Map<String, Object> propertyValue = (Map<String, Object>) object;
    Assert.assertEquals(2, propertyValue.get("maxTuples"));
    lc.shutdown();
}

Example 77 with GenericTestOperator

use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.

the class StreamingContainerManagerTest method testShutdownOperatorTimeout.

@Test
public void testShutdownOperatorTimeout() throws Exception {
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    dag.addStream("s1", o1.outport1, o2.inport1);
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    dag.setAttribute(Context.DAGContext.STREAMING_WINDOW_SIZE_MILLIS, 50);
    dag.setAttribute(OperatorContext.TIMEOUT_WINDOW_COUNT, 1);
    StreamingContainerManager scm = new StreamingContainerManager(dag);
    PhysicalPlan plan = scm.getPhysicalPlan();
    PTOperator p1 = plan.getOperators(dag.getMeta(o1)).get(0);
    PTOperator p2 = plan.getOperators(dag.getMeta(o2)).get(0);
    shutdownOperator(scm, p1, p2);
    scm.monitorHeartbeat(false);
    Assert.assertTrue(scm.containerStopRequests.isEmpty());
    Thread.sleep(100);
    scm.monitorHeartbeat(false);
    Assert.assertFalse(scm.containerStopRequests.containsKey(p1.getContainer().getExternalId()));
    Assert.assertTrue(scm.containerStopRequests.containsKey(p2.getContainer().getExternalId()));
}

Example 78 with GenericTestOperator

use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.

the class StramLocalClusterTest method testLocalClusterInitShutdown.

/**
   * Verify test configuration launches and stops after input terminates.
   * Test validates expected output end to end.
   *
   * @throws Exception
   */
@Test
public void testLocalClusterInitShutdown() throws Exception {
    TestGeneratorInputOperator genNode = dag.addOperator("genNode", TestGeneratorInputOperator.class);
    genNode.setMaxTuples(2);
    GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
    node1.setEmitFormat("%s >> node1");
    File outFile = new File("./target/" + StramLocalClusterTest.class.getName() + "-testLocalClusterInitShutdown.out");
    outFile.delete();
    TestOutputOperator outNode = dag.addOperator("outNode", TestOutputOperator.class);
    outNode.pathSpec = outFile.toURI().toString();
    dag.addStream("fromGenNode", genNode.outport, node1.inport1);
    dag.addStream("fromNode1", node1.outport1, outNode.inport);
    dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 2);
    StramLocalCluster localCluster = new StramLocalCluster(dag);
    localCluster.setHeartbeatMonitoringEnabled(false);
    localCluster.run();
    Assert.assertTrue(outFile + " exists", outFile.exists());
    LineNumberReader lnr = new LineNumberReader(new FileReader(outFile));
    String line;
    while ((line = lnr.readLine()) != null) {
        Assert.assertTrue("line match " + line, line.matches("" + lnr.getLineNumber() + " >> node1"));
    }
    Assert.assertEquals("number lines", 2, lnr.getLineNumber());
    lnr.close();
}

Example 79 with GenericTestOperator

use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.

the class LogicalPlanConfigurationTest method testLoadFromPropertiesFileWithLegacyPrefix.

@Test
public void testLoadFromPropertiesFileWithLegacyPrefix() throws IOException {
    Properties props = new Properties();
    String resourcePath = "/testTopologyLegacyPrefix.properties";
    InputStream is = this.getClass().getResourceAsStream(resourcePath);
    if (is == null) {
        fail("Could not load " + resourcePath);
    }
    props.load(is);
    LogicalPlanConfiguration pb = new LogicalPlanConfiguration(new Configuration(false)).addFromProperties(props, null);
    LogicalPlan dag = new LogicalPlan();
    pb.populateDAG(dag);
    dag.validate();
    assertEquals("number of operators", 2, dag.getAllOperators().size());
    assertEquals("number of root operators", 1, dag.getRootOperators().size());
    StreamMeta s1 = dag.getStream("s1");
    assertNotNull(s1);
    assertTrue("s1 locality", DAG.Locality.CONTAINER_LOCAL == s1.getLocality());
    OperatorMeta o2m = dag.getOperatorMeta("o2");
    assertEquals(GenericTestOperator.class, o2m.getOperator().getClass());
    GenericTestOperator o2 = (GenericTestOperator) o2m.getOperator();
    assertEquals("myStringProperty " + o2, "myStringPropertyValue", o2.getMyStringProperty());
}

Example 80 with GenericTestOperator

use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.

the class LogicalPlanConfigurationTest method testOperatorLevelProperties.

@Test
public void testOperatorLevelProperties() {
    String appName = "app1";
    final GenericTestOperator operator1 = new GenericTestOperator();
    final GenericTestOperator operator2 = new GenericTestOperator();
    StreamingApplication app = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
            dag.addOperator("operator1", operator1);
            dag.addOperator("operator2", operator2);
        }
    };
    Properties props = new Properties();
    props.put(StreamingApplication.APEX_PREFIX + "application." + appName + ".class", app.getClass().getName());
    props.put(StreamingApplication.APEX_PREFIX + "operator.*.myStringProperty", "pv1");
    props.put(StreamingApplication.APEX_PREFIX + "operator.*.booleanProperty", Boolean.TRUE.toString());
    props.put(StreamingApplication.APEX_PREFIX + "application." + appName + ".operator.operator1.myStringProperty", "apv1");
    LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
    dagBuilder.addFromProperties(props, null);
    String appPath = app.getClass().getName().replace(".", "/") + ".class";
    LogicalPlan dag = new LogicalPlan();
    dagBuilder.prepareDAG(dag, app, appPath);
    Assert.assertEquals("apv1", operator1.getMyStringProperty());
    Assert.assertEquals("pv1", operator2.getMyStringProperty());
    Assert.assertEquals(true, operator2.isBooleanProperty());
}