Examples with GenericTestOperator com.datatorrent.stram.engine.GenericTestOperator used on opensource projects

Example 16 with GenericTestOperator

use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.

the class PhysicalPlanTest method testInputOperatorPartitioning.

/**
 * Test partitioning of an input operator (no input port).
 * Cover aspects that are not part of generic operator test.
 * Test scaling from one to multiple partitions with unifier when one partition remains unmodified.
 */
@Test
public void testInputOperatorPartitioning() {
    LogicalPlan dag = new LogicalPlan();
    final TestInputOperator<Object> o1 = dag.addOperator("o1", new TestInputOperator<>());
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    dag.addStream("o1.outport1", o1.output, o2.inport1);
    OperatorMeta o1Meta = dag.getMeta(o1);
    dag.setOperatorAttribute(o1, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { new PartitioningTest.PartitionLoadWatch() }));
    TestPartitioner<TestInputOperator<Object>> partitioner = new TestPartitioner<>();
    dag.setOperatorAttribute(o1, OperatorContext.PARTITIONER, partitioner);
    TestPlanContext ctx = new TestPlanContext();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
    PhysicalPlan plan = new PhysicalPlan(dag, ctx);
    Assert.assertEquals("number of containers", 2, plan.getContainers().size());
    List<PTOperator> o1Partitions = plan.getOperators(o1Meta);
    Assert.assertEquals("partitions " + o1Partitions, 1, o1Partitions.size());
    PTOperator o1p1 = o1Partitions.get(0);
    // verify load update generates expected events per configuration
    Assert.assertEquals("stats handlers " + o1p1, 1, o1p1.statsListeners.size());
    StatsListener l = o1p1.statsListeners.get(0);
    Assert.assertTrue("stats handlers " + o1p1.statsListeners, l instanceof PartitioningTest.PartitionLoadWatch);
    PartitioningTest.PartitionLoadWatch.put(o1p1, 1);
    plan.onStatusUpdate(o1p1);
    Assert.assertEquals("scale up triggered", 1, ctx.events.size());
    // add another partition, keep existing as is
    partitioner.extraPartitions.add(new DefaultPartition<>(o1));
    Runnable r = ctx.events.remove(0);
    r.run();
    partitioner.extraPartitions.clear();
    o1Partitions = plan.getOperators(o1Meta);
    Assert.assertEquals("operators after scale up", 2, o1Partitions.size());
    Assert.assertEquals("first partition unmodified", o1p1, o1Partitions.get(0));
    Assert.assertEquals("single output", 1, o1p1.getOutputs().size());
    Assert.assertEquals("output to unifier", 1, o1p1.getOutputs().get(0).sinks.size());
    Set<PTOperator> expUndeploy = Sets.newHashSet(plan.getOperators(dag.getMeta(o2)));
    Set<PTOperator> expDeploy = Sets.newHashSet(o1Partitions.get(1));
    expDeploy.addAll(plan.getMergeOperators(dag.getMeta(o1)));
    expDeploy.addAll(expUndeploy);
    expDeploy.add(o1p1.getOutputs().get(0).sinks.get(0).target);
    Assert.assertEquals("undeploy", expUndeploy, ctx.undeploy);
    Assert.assertEquals("deploy", expDeploy, ctx.deploy);
    for (PTOperator p : o1Partitions) {
        Assert.assertEquals("activation window id " + p, Checkpoint.INITIAL_CHECKPOINT, p.recoveryCheckpoint);
        Assert.assertEquals("checkpoints " + p + " " + p.checkpoints, Lists.newArrayList(), p.checkpoints);
        PartitioningTest.PartitionLoadWatch.put(p, -1);
        plan.onStatusUpdate(p);
    }
    ctx.events.remove(0).run();
    Assert.assertEquals("operators after scale down", 1, plan.getOperators(o1Meta).size());
}

Example 17 with GenericTestOperator

use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.

the class StramRecoveryTest method testPhysicalPlanSerialization.

private void testPhysicalPlanSerialization(StorageAgent agent) throws Exception {
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    PartitioningTestOperator o2 = dag.addOperator("o2", PartitioningTestOperator.class);
    o2.setPartitionCount(3);
    GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
    dag.addStream("o1.outport1", o1.outport1, o2.inport1, o2.inportWithCodec);
    dag.addStream("mergeStream", o2.outport1, o3.inport1);
    dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 2);
    TestPlanContext ctx = new TestPlanContext();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, agent);
    PhysicalPlan plan = new PhysicalPlan(dag, ctx);
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    LogicalPlan.write(dag, bos);
    LOG.debug("logicalPlan size: " + bos.toByteArray().length);
    bos = new ByteArrayOutputStream();
    ObjectOutputStream oos = new ObjectOutputStream(bos);
    oos.writeObject(plan);
    LOG.debug("physicalPlan size: " + bos.toByteArray().length);
    ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
    plan = (PhysicalPlan) new ObjectInputStream(bis).readObject();
    dag = plan.getLogicalPlan();
    Field f = PhysicalPlan.class.getDeclaredField("ctx");
    f.setAccessible(true);
    f.set(plan, ctx);
    f.setAccessible(false);
    OperatorMeta o2Meta = dag.getOperatorMeta("o2");
    List<PTOperator> o2Partitions = plan.getOperators(o2Meta);
    assertEquals(3, o2Partitions.size());
    for (PTOperator o : o2Partitions) {
        Assert.assertNotNull("partition null " + o, o.getPartitionKeys());
        assertEquals("partition keys " + o + " " + o.getPartitionKeys(), 2, o.getPartitionKeys().size());
        PartitioningTestOperator partitionedInstance = (PartitioningTestOperator) plan.loadOperator(o);
        assertEquals("instance per partition", o.getPartitionKeys().values().toString(), partitionedInstance.pks);
        Assert.assertNotNull("partition stats null " + o, o.stats);
    }
}

Example 18 with GenericTestOperator

use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.

the class StreamingContainerManagerTest method testAppDataPush.

@Test
public void testAppDataPush() throws Exception {
    if (StramTestSupport.isInTravis()) {
        // disable this test in travis because of an intermittent problem similar to this:
        // http://stackoverflow.com/questions/32172925/travis-ci-sporadic-timeouts-to-localhost
        // We should remove this when we find a solution to this.
        LOG.info("Test testAppDataPush is disabled in Travis");
        return;
    }
    final String topic = "xyz";
    final List<String> messages = new ArrayList<>();
    EmbeddedWebSocketServer server = new EmbeddedWebSocketServer(0);
    server.setWebSocket(new WebSocket.OnTextMessage() {

        @Override
        public void onMessage(String data) {
            messages.add(data);
        }

        @Override
        public void onOpen(WebSocket.Connection connection) {
        }

        @Override
        public void onClose(int closeCode, String message) {
        }
    });
    try {
        server.start();
        int port = server.getPort();
        TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
        GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
        dag.addStream("o1.outport", o1.outport, o2.inport1);
        dag.setAttribute(LogicalPlan.METRICS_TRANSPORT, new AutoMetricBuiltInTransport(topic));
        dag.setAttribute(LogicalPlan.GATEWAY_CONNECT_ADDRESS, "localhost:" + port);
        dag.setAttribute(LogicalPlan.PUBSUB_CONNECT_TIMEOUT_MILLIS, 2000);
        StramLocalCluster lc = new StramLocalCluster(dag);
        StreamingContainerManager dnmgr = lc.dnmgr;
        StramAppContext appContext = new StramTestSupport.TestAppContext(dag.getAttributes());
        AppDataPushAgent pushAgent = new AppDataPushAgent(dnmgr, appContext);
        pushAgent.init();
        pushAgent.pushData();
        Thread.sleep(1000);
        Assert.assertTrue(messages.size() > 0);
        pushAgent.close();
        JSONObject message = new JSONObject(messages.get(0));
        Assert.assertEquals(topic, message.getString("topic"));
        Assert.assertEquals("publish", message.getString("type"));
        JSONObject data = message.getJSONObject("data");
        Assert.assertTrue(StringUtils.isNotBlank(data.getString("appId")));
        Assert.assertTrue(StringUtils.isNotBlank(data.getString("appUser")));
        Assert.assertTrue(StringUtils.isNotBlank(data.getString("appName")));
        JSONObject logicalOperators = data.getJSONObject("logicalOperators");
        for (String opName : new String[] { "o1", "o2" }) {
            JSONObject opObj = logicalOperators.getJSONObject(opName);
            Assert.assertTrue(opObj.has("totalTuplesProcessed"));
            Assert.assertTrue(opObj.has("totalTuplesEmitted"));
            Assert.assertTrue(opObj.has("tuplesProcessedPSMA"));
            Assert.assertTrue(opObj.has("tuplesEmittedPSMA"));
            Assert.assertTrue(opObj.has("latencyMA"));
        }
    } finally {
        server.stop();
    }
}

Example 19 with GenericTestOperator

use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.

the class StreamingContainerManagerTest method testValidGenericOperatorDeployInfoType.

@Test
public void testValidGenericOperatorDeployInfoType() {
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    TestGeneratorInputOperator.ValidGenericOperator o2 = dag.addOperator("o2", TestGeneratorInputOperator.ValidGenericOperator.class);
    dag.addStream("stream1", o1.outport1, o2.input);
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    StreamingContainerManager scm = new StreamingContainerManager(dag);
    PhysicalPlan physicalPlan = scm.getPhysicalPlan();
    List<PTContainer> containers = physicalPlan.getContainers();
    for (int i = 0; i < containers.size(); ++i) {
        assignContainer(scm, "container" + (i + 1));
    }
    OperatorMeta o2Meta = dag.getMeta(o2);
    PTOperator o2Physical = physicalPlan.getOperators(o2Meta).get(0);
    String containerId = o2Physical.getContainer().getExternalId();
    OperatorDeployInfo o1DeployInfo = getDeployInfo(scm.getContainerAgent(containerId)).get(0);
    Assert.assertEquals("type " + o1DeployInfo, OperatorDeployInfo.OperatorType.GENERIC, o1DeployInfo.type);
}

Example 20 with GenericTestOperator

use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.

the class StreamingContainerManagerTest method testRecoveryOrder.

@Test
public void testRecoveryOrder() throws Exception {
    GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
    GenericTestOperator node2 = dag.addOperator("node2", GenericTestOperator.class);
    GenericTestOperator node3 = dag.addOperator("node3", GenericTestOperator.class);
    dag.addStream("n1n2", node1.outport1, node2.inport1);
    dag.addStream("n2n3", node2.outport1, node3.inport1);
    dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 2);
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    StreamingContainerManager scm = new StreamingContainerManager(dag);
    Assert.assertEquals("" + scm.containerStartRequests, 2, scm.containerStartRequests.size());
    scm.containerStartRequests.clear();
    PhysicalPlan plan = scm.getPhysicalPlan();
    List<PTContainer> containers = plan.getContainers();
    Assert.assertEquals("" + containers, 2, plan.getContainers().size());
    PTContainer c1 = containers.get(0);
    Assert.assertEquals("c1.operators " + c1.getOperators(), 2, c1.getOperators().size());
    PTContainer c2 = containers.get(1);
    Assert.assertEquals("c2.operators " + c2.getOperators(), 1, c2.getOperators().size());
    assignContainer(scm, "container1");
    assignContainer(scm, "container2");
    StreamingContainerAgent sca1 = scm.getContainerAgent(c1.getExternalId());
    StreamingContainerAgent sca2 = scm.getContainerAgent(c2.getExternalId());
    Assert.assertEquals("", 0, countState(sca1.container, PTOperator.State.PENDING_UNDEPLOY));
    Assert.assertEquals("", 2, countState(sca1.container, PTOperator.State.PENDING_DEPLOY));
    scm.scheduleContainerRestart(c1.getExternalId());
    Assert.assertEquals("", 0, countState(sca1.container, PTOperator.State.PENDING_UNDEPLOY));
    Assert.assertEquals("", 2, countState(sca1.container, PTOperator.State.PENDING_DEPLOY));
    Assert.assertEquals("" + scm.containerStartRequests, 1, scm.containerStartRequests.size());
    ContainerStartRequest dr = scm.containerStartRequests.peek();
    Assert.assertNotNull(dr);
    Assert.assertEquals("" + sca2.container, 1, countState(sca2.container, PTOperator.State.PENDING_UNDEPLOY));
    Assert.assertEquals("" + sca2.container, 0, countState(sca2.container, PTOperator.State.PENDING_DEPLOY));
}