Examples with LogicalPlan com.datatorrent.stram.plan.logical.LogicalPlan used on opensource projects

Example 16 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.

the class StatsTest method testPortStatsPropagation.

/**
   * Verify buffer server bytes and tuple count.
   *
   * @throws Exception
   */
@Test
@SuppressWarnings("SleepWhileInLoop")
public void testPortStatsPropagation() throws Exception {
    int tupleCount = 10;
    LogicalPlan dag = new LogicalPlan();
    String workingDir = new File("target").getAbsolutePath();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
    TestOperator testOper = dag.addOperator("TestOperator", TestOperator.class);
    TestInputStatsListener testInputStatsListener = new TestInputStatsListener();
    dag.setOperatorAttribute(testOper, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { testInputStatsListener }));
    testOper.setMaxTuples(tupleCount);
    testOper.setEmitInterval(0);
    TestCollector collector = dag.addOperator("Collector", new TestCollector());
    TestCollectorStatsListener testCollectorStatsListener = new TestCollectorStatsListener();
    dag.setOperatorAttribute(collector, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { testCollectorStatsListener }));
    dag.addStream("TestTuples", testOper.outport, collector.inport1).setLocality(null);
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.run();
    Assert.assertFalse("input operator stats", testInputStatsListener.inputOperatorStats.isEmpty());
    Assert.assertFalse("collector operator stats", testCollectorStatsListener.collectorOperatorStats.isEmpty());
    try {
        int outputPortTupleCount = 0;
        long outputPortBufferServerBytes = 0L;
        for (Iterator<OperatorStats> it = testInputStatsListener.inputOperatorStats.iterator(); it.hasNext(); ) {
            OperatorStats operatorStats = it.next();
            for (PortStats outputPortStats : operatorStats.outputPorts) {
                outputPortTupleCount += outputPortStats.tupleCount;
                outputPortBufferServerBytes += outputPortStats.bufferServerBytes;
            }
        }
        int inputPortTupleCount = 0;
        long inputPortBufferServerBytes = 0L;
        for (Iterator<OperatorStats> it = testCollectorStatsListener.collectorOperatorStats.iterator(); it.hasNext(); ) {
            OperatorStats operatorStats = it.next();
            for (PortStats inputPortStats : operatorStats.inputPorts) {
                inputPortTupleCount += inputPortStats.tupleCount;
                inputPortBufferServerBytes += inputPortStats.bufferServerBytes;
            }
        }
        Assert.assertEquals("Tuple Count emitted", tupleCount, outputPortTupleCount);
        Assert.assertTrue("Buffer server bytes", inputPortBufferServerBytes > 0);
        Assert.assertEquals("Tuple Count processed", tupleCount, inputPortTupleCount);
        Assert.assertTrue("Buffer server bytes", outputPortBufferServerBytes > 0);
    } finally {
        lc.shutdown();
    }
}

Example 17 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.

the class StreamingContainerTest method testOiOCommitted.

@Test
public void testOiOCommitted() throws IOException, ClassNotFoundException {
    LogicalPlan lp = new LogicalPlan();
    String workingDir = new File("target/testCommitted").getAbsolutePath();
    lp.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
    lp.setAttribute(DAGContext.CHECKPOINT_WINDOW_COUNT, 1);
    String op1Name = "CommitAwareOperatorTestOioCommit1";
    String op2Name = "CommitAwareOperatorTestOioCommit2";
    CommitAwareOperator operator1 = lp.addOperator(op1Name, new CommitAwareOperator());
    CommitAwareOperator operator2 = lp.addOperator(op2Name, new CommitAwareOperator());
    lp.addStream("local", operator1.output, operator2.input).setLocality(Locality.THREAD_LOCAL);
    StramLocalCluster lc = new StramLocalCluster(lp);
    lc.run(5000);
    /* this is not foolproof but some insurance is better than nothing */
    Assert.assertTrue("No Committed Windows", committedWindowIds.contains(op1Name));
    Assert.assertTrue("No Committed Windows", committedWindowIds.contains(op2Name));
}

Example 18 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.

the class StreamingContainerTest method testCommitted.

@Test
public void testCommitted() throws IOException, ClassNotFoundException {
    LogicalPlan lp = new LogicalPlan();
    String workingDir = new File("target/testCommitted").getAbsolutePath();
    lp.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
    lp.setAttribute(DAGContext.CHECKPOINT_WINDOW_COUNT, 1);
    String opName = "CommitAwareOperatorTestCommit";
    lp.addOperator(opName, new CommitAwareOperator());
    StramLocalCluster lc = new StramLocalCluster(lp);
    lc.run(5000);
    /* this is not foolproof but some insurance is better than nothing */
    Assert.assertTrue("No Committed Windows", committedWindowIds.contains(opName));
}

Example 19 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.

the class WindowGeneratorTest method testOutofSequenceError.

@Test
public void testOutofSequenceError() throws Exception {
    logger.info("Testing Out of Sequence Error");
    LogicalPlan dag = new LogicalPlan();
    String workingDir = new File("target/testOutofSequenceError").getAbsolutePath();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
    RandomNumberGenerator rng = dag.addOperator("random", new RandomNumberGenerator());
    MyLogger ml = dag.addOperator("logger", new MyLogger());
    dag.addStream("stream", rng.output, ml.input);
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.run(10000);
}

Example 20 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.

the class HostLocalTest method testUnavailableResources.

@Test
public void testUnavailableResources() {
    LogicalPlan dag = new LogicalPlan();
    dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, new File("target", HostLocalTest.class.getName()).getAbsolutePath());
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    dag.getMeta(o1).getAttributes().put(OperatorContext.LOCALITY_HOST, "host2");
    GenericTestOperator partitioned = dag.addOperator("partitioned", GenericTestOperator.class);
    dag.addStream("o1_outport1", o1.outport1, partitioned.inport1).setLocality(Locality.CONTAINER_LOCAL);
    dag.setOperatorAttribute(o1, OperatorContext.MEMORY_MB, 256);
    dag.setOperatorAttribute(o1, OperatorContext.VCORES, 2);
    dag.setOperatorAttribute(partitioned, OperatorContext.VCORES, 1);
    StreamingContainerManager scm = new StreamingContainerManager(dag);
    ResourceRequestHandler rr = new ResourceRequestHandler();
    int containerMem = 1000;
    Map<String, NodeReport> nodeReports = Maps.newHashMap();
    NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
    nodeReports.put(nr.getNodeId().getHost(), nr);
    nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
    nodeReports.put(nr.getNodeId().getHost(), nr);
    // set resources
    rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));
    Assert.assertEquals("number of containers is 1", 1, scm.containerStartRequests.size());
    for (ContainerStartRequest csr : scm.containerStartRequests) {
        String host = rr.getHost(csr, true);
        Assert.assertEquals("number of vcores", 3, csr.container.getRequiredVCores());
        Assert.assertNull("Host is null", host);
    }
}