Examples with TestCollectorStatsListener com.datatorrent.stram.engine.StatsTest.TestCollector.TestCollectorStatsListener used on opensource projects

Example 1 with TestCollectorStatsListener

use of com.datatorrent.stram.engine.StatsTest.TestCollector.TestCollectorStatsListener in project apex-core by apache.

the class StatsTest method testPortStatsPropagation.

/**
 * Verify buffer server bytes and tuple count.
 *
 * @throws Exception
 */
@Test
@SuppressWarnings("SleepWhileInLoop")
public void testPortStatsPropagation() throws Exception {
    int tupleCount = 10;
    LogicalPlan dag = new LogicalPlan();
    String workingDir = new File("target").getAbsolutePath();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
    TestOperator testOper = dag.addOperator("TestOperator", TestOperator.class);
    TestInputStatsListener testInputStatsListener = new TestInputStatsListener();
    dag.setOperatorAttribute(testOper, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { testInputStatsListener }));
    testOper.setMaxTuples(tupleCount);
    testOper.setEmitInterval(0);
    TestCollector collector = dag.addOperator("Collector", new TestCollector());
    TestCollectorStatsListener testCollectorStatsListener = new TestCollectorStatsListener();
    dag.setOperatorAttribute(collector, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { testCollectorStatsListener }));
    dag.addStream("TestTuples", testOper.outport, collector.inport1).setLocality(null);
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.run();
    Assert.assertFalse("input operator stats", testInputStatsListener.inputOperatorStats.isEmpty());
    Assert.assertFalse("collector operator stats", testCollectorStatsListener.collectorOperatorStats.isEmpty());
    try {
        int outputPortTupleCount = 0;
        long outputPortBufferServerBytes = 0L;
        for (Iterator<OperatorStats> it = testInputStatsListener.inputOperatorStats.iterator(); it.hasNext(); ) {
            OperatorStats operatorStats = it.next();
            for (PortStats outputPortStats : operatorStats.outputPorts) {
                outputPortTupleCount += outputPortStats.tupleCount;
                outputPortBufferServerBytes += outputPortStats.bufferServerBytes;
            }
        }
        int inputPortTupleCount = 0;
        long inputPortBufferServerBytes = 0L;
        for (Iterator<OperatorStats> it = testCollectorStatsListener.collectorOperatorStats.iterator(); it.hasNext(); ) {
            OperatorStats operatorStats = it.next();
            for (PortStats inputPortStats : operatorStats.inputPorts) {
                inputPortTupleCount += inputPortStats.tupleCount;
                inputPortBufferServerBytes += inputPortStats.bufferServerBytes;
            }
        }
        Assert.assertEquals("Tuple Count emitted", tupleCount, outputPortTupleCount);
        Assert.assertTrue("Buffer server bytes", inputPortBufferServerBytes > 0);
        Assert.assertEquals("Tuple Count processed", tupleCount, inputPortTupleCount);
        Assert.assertTrue("Buffer server bytes", outputPortBufferServerBytes > 0);
    } finally {
        lc.shutdown();
    }
}

Example 2 with TestCollectorStatsListener

use of com.datatorrent.stram.engine.StatsTest.TestCollector.TestCollectorStatsListener in project apex-core by apache.

the class StatsTest method baseTestForQueueSize.

@SuppressWarnings("SleepWhileInLoop")
private void baseTestForQueueSize(int maxTuples, TestCollectorStatsListener statsListener, DAG.Locality locality) throws Exception {
    LogicalPlan dag = new LogicalPlan();
    String workingDir = new File("target/baseTestForQueueSize").getAbsolutePath();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
    dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 200);
    TestOperator testOper = dag.addOperator("TestOperator", TestOperator.class);
    testOper.setMaxTuples(maxTuples);
    TestCollector collector = dag.addOperator("Collector", new TestCollector());
    if (statsListener != null) {
        dag.setOperatorAttribute(collector, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { statsListener }));
    }
    dag.addStream("TestTuples", testOper.outport, collector.inport1).setLocality(locality);
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.runAsync();
    StreamingContainerManager dnmgr = lc.getStreamingContainerManager();
    Map<Integer, PTOperator> operatorMap = dnmgr.getPhysicalPlan().getAllOperators();
    for (PTOperator p : operatorMap.values()) {
        StramTestSupport.waitForActivation(lc, p);
    }
    long startTms = System.currentTimeMillis();
    if (statsListener != null) {
        while (statsListener.collectorOperatorStats.isEmpty() && (StramTestSupport.DEFAULT_TIMEOUT_MILLIS > System.currentTimeMillis() - startTms)) {
            Thread.sleep(300);
            LOG.debug("Waiting for stats");
        }
    } else {
        while (collector.collectorOperatorStats.isEmpty() && (StramTestSupport.DEFAULT_TIMEOUT_MILLIS > System.currentTimeMillis() - startTms)) {
            Thread.sleep(300);
            LOG.debug("Waiting for stats");
        }
    }
    if (statsListener != null) {
        statsListener.validateStats();
    } else {
        collector.validateStats();
    }
    lc.shutdown();
}