Examples with GenericTestOperator com.datatorrent.stram.engine.GenericTestOperator used on opensource projects

Example 71 with GenericTestOperator

use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.

the class StreamingContainerManagerTest method testShutdownOperatorRecovery.

@Test
public void testShutdownOperatorRecovery() throws Exception {
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    dag.addStream("s1", o1.outport1, o2.inport1);
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    StreamingContainerManager scm = new StreamingContainerManager(dag);
    scm.containerStartRequests.poll();
    scm.containerStartRequests.poll();
    PhysicalPlan plan = scm.getPhysicalPlan();
    PTOperator p1 = plan.getOperators(dag.getMeta(o1)).get(0);
    PTOperator p2 = plan.getOperators(dag.getMeta(o2)).get(0);
    shutdownOperator(scm, p1, p2);
    scm.scheduleContainerRestart(p1.getContainer().getExternalId());
    ContainerStartRequest dr = scm.containerStartRequests.poll();
    Assert.assertTrue(dr.container.getOperators().contains(p1));
}

Example 72 with GenericTestOperator

use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.

the class StreamingContainerManagerTest method testCustomMetricsTransport.

@Test
public void testCustomMetricsTransport() throws Exception {
    TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    dag.addStream("o1.outport", o1.outport, o2.inport1);
    dag.setAttribute(LogicalPlan.METRICS_TRANSPORT, new TestMetricTransport("xyz"));
    StramLocalCluster lc = new StramLocalCluster(dag);
    StreamingContainerManager dnmgr = lc.dnmgr;
    StramAppContext appContext = new StramTestSupport.TestAppContext(dag.getAttributes());
    AppDataPushAgent pushAgent = new AppDataPushAgent(dnmgr, appContext);
    pushAgent.init();
    pushAgent.pushData();
    Assert.assertTrue(TestMetricTransport.messages.size() > 0);
    pushAgent.close();
    String msg = TestMetricTransport.messages.get(0);
    Assert.assertTrue(msg.startsWith("xyz:"));
}

Example 73 with GenericTestOperator

use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.

the class StreamingContainerManagerTest method testValidInputOperatorDeployInfoType.

@Test
public void testValidInputOperatorDeployInfoType() {
    TestGeneratorInputOperator.ValidInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.ValidInputOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    dag.addStream("stream1", o1.outport, o2.inport1);
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    StreamingContainerManager scm = new StreamingContainerManager(dag);
    PhysicalPlan physicalPlan = scm.getPhysicalPlan();
    List<PTContainer> containers = physicalPlan.getContainers();
    for (int i = 0; i < containers.size(); ++i) {
        assignContainer(scm, "container" + (i + 1));
    }
    OperatorMeta o1Meta = dag.getMeta(o1);
    PTOperator o1Physical = physicalPlan.getOperators(o1Meta).get(0);
    String containerId = o1Physical.getContainer().getExternalId();
    OperatorDeployInfo o1DeployInfo = getDeployInfo(scm.getContainerAgent(containerId)).get(0);
    Assert.assertEquals("type " + o1DeployInfo, OperatorDeployInfo.OperatorType.INPUT, o1DeployInfo.type);
}

Example 74 with GenericTestOperator

use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.

the class OutputUnifiedTest method testParallelPartition.

@Test
public void testParallelPartition() throws Exception {
    TestInputOperator i1 = new TestInputOperator();
    dag.addOperator("i1", i1);
    dag.setOperatorAttribute(i1, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2));
    GenericTestOperator op1 = new GenericTestOperator();
    dag.addOperator("op1", op1);
    dag.setInputPortAttribute(op1.inport1, PortContext.PARTITION_PARALLEL, true);
    TestOutputOperator op2 = new TestOutputOperator();
    dag.addOperator("op2", op2);
    dag.addStream("s1", i1.output, op1.inport1);
    dag.addStream("s2", op1.outport1, op2.inport);
    StreamingContainerManager scm = new StreamingContainerManager(dag);
    PhysicalPlan physicalPlan = scm.getPhysicalPlan();
    List<PTContainer> containers = physicalPlan.getContainers();
    Assert.assertEquals("Number of containers", 5, containers.size());
    assignContainers(scm, containers);
    testOutputAttribute(dag, i1, scm, physicalPlan, false);
    testOutputAttribute(dag, op1, scm, physicalPlan, true);
}

Example 75 with GenericTestOperator

use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.

the class StreamingContainerManagerTest method testRecoveryUpstreamInline.

@Test
public void testRecoveryUpstreamInline() throws Exception {
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
    dag.addStream("o1o3", o1.outport1, o3.inport1);
    dag.addStream("o2o3", o2.outport1, o3.inport2);
    dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 2);
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    StreamingContainerManager scm = new StreamingContainerManager(dag);
    PhysicalPlan plan = scm.getPhysicalPlan();
    Assert.assertEquals(2, plan.getContainers().size());
    plan.getOperators(dag.getMeta(o1)).get(0);
    Assert.assertEquals(2, plan.getContainers().size());
    PTContainer c1 = plan.getContainers().get(0);
    Assert.assertEquals(Sets.newHashSet(plan.getOperators(dag.getMeta(o1)).get(0), plan.getOperators(dag.getMeta(o3)).get(0)), Sets.newHashSet(c1.getOperators()));
    PTContainer c2 = plan.getContainers().get(1);
    assignContainer(scm, "c1");
    assignContainer(scm, "c2");
    for (PTOperator oper : c1.getOperators()) {
        Assert.assertEquals("state " + oper, PTOperator.State.PENDING_DEPLOY, oper.getState());
    }
    scm.scheduleContainerRestart(c2.getExternalId());
    for (PTOperator oper : c1.getOperators()) {
        Assert.assertEquals("state " + oper, PTOperator.State.PENDING_UNDEPLOY, oper.getState());
    }
}