Examples with SummaryStatistics org.apache.commons.math.stat.descriptive.SummaryStatistics used on opensource projects

Example 1 with SummaryStatistics

use of org.apache.commons.math.stat.descriptive.SummaryStatistics in project ACS by ACS-Community.

the class BlobData method calculateStatistics.

/**
	 * Calculates the statistics and stores it in {@link #statistics}
	 * if our monitor point data is represented as Number objects;
	 * otherwise this call is ignored.
	 * 
	 * @param inDataList
	 */
void calculateStatistics() {
    if (getDataSize() > 0) {
        // We trust that the data is homogeneous and check only the first MonitorPointValue
        MonitorPointValue sampleMonitorPointValue = mpTs.getDataList().get(0);
        if (sampleMonitorPointValue.getData().isEmpty()) {
            logger.finer("Ignoring calculateStatistics() call for a time series of MonitorPointValue objects that hold no data.");
            return;
        }
        // and so far we keep this behavior. 
        if (sampleMonitorPointValue.isMultiValued()) {
            logger.finer("Ignoring calculateStatistics() call for a time series of multi-valued MonitorPointValue objects.");
            return;
        }
        // After the above checks, there should be a single data item in our sampleMonitorPointValue
        // We now verify that it has one of the expected numeric types.
        Object sampleData = sampleMonitorPointValue.getData().get(0);
        if (!(sampleData instanceof Integer || sampleData instanceof Long || sampleData instanceof Float || sampleData instanceof Double)) {
            logger.finer("Ignoring calculateStatistics() call for data type " + sampleData.getClass().getName());
            return;
        }
        // Now we calculate the statistics, 
        // using apache math lib that works only with 'double' type
        SummaryStatistics stat = new SummaryStatistics();
        for (MonitorPointValue blobData : mpTs.getDataList()) {
            Number value = (Number) blobData.getData().get(0);
            stat.addValue(value.doubleValue());
        }
        statistics = new ComponentStatistics();
        // converting to original data types where it makes sense
        if (sampleData instanceof Integer) {
            statistics.min = new Integer((int) Math.round(stat.getMin()));
            statistics.max = new Integer((int) Math.round(stat.getMax()));
            // or Float, to indicate lower precision?
            statistics.mean = new Double(stat.getMean());
            // or Float, to indicate lower precision?
            statistics.stdDev = new Double(stat.getStandardDeviation());
        } else if (sampleData instanceof Long) {
            statistics.min = new Long(Math.round(stat.getMin()));
            statistics.max = new Long(Math.round(stat.getMax()));
            statistics.mean = new Double(stat.getMean());
            statistics.stdDev = new Double(stat.getStandardDeviation());
        } else if (sampleData instanceof Float) {
            statistics.min = new Float(stat.getMin());
            statistics.max = new Float(stat.getMax());
            statistics.mean = new Float(stat.getMean());
            statistics.stdDev = new Float(stat.getStandardDeviation());
        } else if (sampleData instanceof Double) {
            statistics.min = new Double(stat.getMin());
            statistics.max = new Double(stat.getMax());
            statistics.mean = new Double(stat.getMean());
            statistics.stdDev = new Double(stat.getStandardDeviation());
        }
    }
}

Example 2 with SummaryStatistics

use of org.apache.commons.math.stat.descriptive.SummaryStatistics in project sling by apache.

the class RequestProcessorMBeanImplTest method test_statistics.

/**
     * Asserts that the simple standard deviation algorithm used by the
     * RequestProcessorMBeanImpl is equivalent to the Commons Math
     * SummaryStatistics implementation.
     * 
     * It also tests that resetStatistics method, actually resets all the statistics
     *
     * @throws NotCompliantMBeanException not expected
     */
@Test
public void test_statistics() throws NotCompliantMBeanException {
    final SummaryStatistics durationStats = new SummaryStatistics();
    final SummaryStatistics servletCallCountStats = new SummaryStatistics();
    final SummaryStatistics peakRecursionDepthStats = new SummaryStatistics();
    final RequestProcessorMBeanImpl bean = new RequestProcessorMBeanImpl();
    assertEquals(0l, bean.getRequestsCount());
    assertEquals(Long.MAX_VALUE, bean.getMinRequestDurationMsec());
    assertEquals(0l, bean.getMaxRequestDurationMsec());
    assertEquals(0.0, bean.getMeanRequestDurationMsec(), 0);
    assertEquals(0.0, bean.getStandardDeviationDurationMsec(), 0);
    assertEquals(Integer.MAX_VALUE, bean.getMinServletCallCount());
    assertEquals(0l, bean.getMaxServletCallCount());
    assertEquals(0.0, bean.getMeanServletCallCount(), 0);
    assertEquals(0.0, bean.getStandardDeviationServletCallCount(), 0);
    assertEquals(Integer.MAX_VALUE, bean.getMinPeakRecursionDepth());
    assertEquals(0l, bean.getMaxPeakRecursionDepth());
    assertEquals(0.0, bean.getMeanPeakRecursionDepth(), 0);
    assertEquals(0.0, bean.getStandardDeviationPeakRecursionDepth(), 0);
    final Random random = new Random(System.currentTimeMillis() / 17);
    final int num = 10000;
    final int min = 85;
    final int max = 250;
    for (int i = 0; i < num; i++) {
        final long durationValue = min + random.nextInt(max - min);
        final int callCountValue = min + random.nextInt(max - min);
        final int peakRecursionDepthValue = min + random.nextInt(max - min);
        durationStats.addValue(durationValue);
        servletCallCountStats.addValue(callCountValue);
        peakRecursionDepthStats.addValue(peakRecursionDepthValue);
        final RequestData requestData = context.mock(RequestData.class, "requestData" + i);
        context.checking(new Expectations() {

            {
                one(requestData).getElapsedTimeMsec();
                will(returnValue(durationValue));
                one(requestData).getServletCallCount();
                will(returnValue(callCountValue));
                one(requestData).getPeakRecusionDepth();
                will(returnValue(peakRecursionDepthValue));
            }
        });
        bean.addRequestData(requestData);
    }
    assertEquals("Number of points must be the same", durationStats.getN(), bean.getRequestsCount());
    assertEquals("Min Duration must be equal", (long) durationStats.getMin(), bean.getMinRequestDurationMsec());
    assertEquals("Max Duration must be equal", (long) durationStats.getMax(), bean.getMaxRequestDurationMsec());
    assertAlmostEqual("Mean Duration", durationStats.getMean(), bean.getMeanRequestDurationMsec(), num);
    assertAlmostEqual("Standard Deviation Duration", durationStats.getStandardDeviation(), bean.getStandardDeviationDurationMsec(), num);
    assertEquals("Min Servlet Call Count must be equal", (long) servletCallCountStats.getMin(), bean.getMinServletCallCount());
    assertEquals("Max Servlet Call Count must be equal", (long) servletCallCountStats.getMax(), bean.getMaxServletCallCount());
    assertAlmostEqual("Mean Servlet Call Count", servletCallCountStats.getMean(), bean.getMeanServletCallCount(), num);
    assertAlmostEqual("Standard Deviation Servlet Call Count", servletCallCountStats.getStandardDeviation(), bean.getStandardDeviationServletCallCount(), num);
    assertEquals("Min Peak Recursion Depth must be equal", (long) peakRecursionDepthStats.getMin(), bean.getMinPeakRecursionDepth());
    assertEquals("Max Peak Recursion Depth must be equal", (long) peakRecursionDepthStats.getMax(), bean.getMaxPeakRecursionDepth());
    assertAlmostEqual("Mean Peak Recursion Depth", peakRecursionDepthStats.getMean(), bean.getMeanPeakRecursionDepth(), num);
    assertAlmostEqual("Standard Deviation Peak Recursion Depth", peakRecursionDepthStats.getStandardDeviation(), bean.getStandardDeviationPeakRecursionDepth(), num);
    //check method resetStatistics 
    //In the previous test, some requests have been processed, now we reset the statistics so everything statistic is reinitialized
    bean.resetStatistics();
    //Simulate a single request 
    final long durationValue = min + random.nextInt(max - min);
    final int callCountValue = min + random.nextInt(max - min);
    final int peakRecursionDepthValue = min + random.nextInt(max - min);
    final RequestData requestData = context.mock(RequestData.class, "requestDataAfterReset");
    context.checking(new Expectations() {

        {
            one(requestData).getElapsedTimeMsec();
            will(returnValue(durationValue));
            one(requestData).getServletCallCount();
            will(returnValue(callCountValue));
            one(requestData).getPeakRecusionDepth();
            will(returnValue(peakRecursionDepthValue));
        }
    });
    bean.addRequestData(requestData);
    //As only one request has been simulated since resetStatiscts: min, max and mean statistics should be equals to the request data
    assertEquals("After resetStatistics Number of requests must be one", 1, bean.getRequestsCount());
    assertEquals("After resetStatistics Min Duration must be equal", bean.getMinRequestDurationMsec(), (long) durationValue);
    assertEquals("After resetStatistics Max Duration must be equal", bean.getMaxRequestDurationMsec(), (long) durationValue);
    assertEquals("After resetStatistics Mean Duration must be equal", bean.getMeanRequestDurationMsec(), (double) durationValue, 0d);
    assertEquals("After resetStatistics Min Servlet Call Count must be equal", bean.getMinServletCallCount(), callCountValue);
    assertEquals("After resetStatistics Max Servlet Call Count must be equal", bean.getMaxServletCallCount(), callCountValue);
    assertEquals("After resetStatistics Mean Servlet Call Count", bean.getMeanServletCallCount(), (double) callCountValue, 0d);
    assertEquals("After resetStatistics Min Peak Recursion Depth must be equal", bean.getMinPeakRecursionDepth(), peakRecursionDepthValue);
    assertEquals("After resetStatistics Max Peak Recursion Depth must be equal", bean.getMinPeakRecursionDepth(), peakRecursionDepthValue);
    assertEquals("After resetStatistics Mean Peak Recursion Depth", bean.getMeanPeakRecursionDepth(), (double) peakRecursionDepthValue, 0d);
}

Example 3 with SummaryStatistics

use of org.apache.commons.math.stat.descriptive.SummaryStatistics in project compiler by boalang.

the class StDevAggregator method finish.

/** {@inheritDoc} */
@Override
public void finish() throws IOException, InterruptedException {
    if (this.isCombining()) {
        String s = "";
        for (final Long key : map.keySet()) s += key + ":" + map.get(key) + ";";
        this.collect(s, null);
        return;
    }
    final SummaryStatistics summaryStatistics = new SummaryStatistics();
    for (final Long key : map.keySet()) {
        final long count = map.get(key);
        for (long i = 0; i < count; i++) summaryStatistics.addValue(key);
    }
    this.collect(summaryStatistics.getStandardDeviation());
}

Example 4 with SummaryStatistics

use of org.apache.commons.math.stat.descriptive.SummaryStatistics in project compiler by boalang.

the class StatisticsAggregator method finish.

/** {@inheritDoc} */
@Override
public void finish() throws IOException, InterruptedException {
    if (this.isCombining()) {
        String s = "";
        for (final Long key : map.keySet()) s += key + ":" + map.get(key) + ";";
        this.collect(s, null);
        return;
    }
    float median = 0;
    long medianPos = count / 2L;
    long curPos = 0;
    long prevPos = 0;
    long prevKey = 0;
    for (final Long key : map.keySet()) {
        curPos = prevPos + map.get(key);
        if (prevPos <= medianPos && medianPos < curPos) {
            if (curPos % 2 == 0 && prevPos == medianPos)
                median = (float) (key + prevKey) / 2.0f;
            else
                median = key;
            break;
        }
        prevKey = key;
        prevPos = curPos;
    }
    double s1 = 0;
    double s2 = 0;
    double s3 = 0;
    double s4 = 0;
    final SummaryStatistics summaryStatistics = new SummaryStatistics();
    for (final Long key : map.keySet()) {
        s1 += key * map.get(key);
        s2 += key * key * map.get(key);
        s3 += key * key * key * map.get(key);
        s4 += key * key * key * key * map.get(key);
        for (int i = 0; i < map.get(key); i++) summaryStatistics.addValue(key);
    }
    final double mean = s1 / (double) count;
    final double var = s2 / (double) (count - 1) - s1 * s1 / (double) (count * (count - 1));
    final double stdev = Math.sqrt(var);
    final double skewness = (s3 - 3 * s1 * s2 / (double) count + s1 * s1 * s1 * 2 / (count * count)) / (count * stdev * var);
    final double kurtosis = (s4 - s3 * s1 * 4 / count + s2 * s1 * s1 * 6 / (double) (count * count) - s1 * s1 * s1 * s1 * 3 / (double) (count * count * count)) / (count * var * var);
    double ci = 0.0;
    try {
        final TDistributionImpl tDist = new TDistributionImpl(summaryStatistics.getN() - 1);
        final double a = tDist.inverseCumulativeProbability(1.0 - 0.025);
        ci = a * summaryStatistics.getStandardDeviation() / Math.sqrt(summaryStatistics.getN());
    } catch (final MathException e) {
    }
    this.collect(s1 + ", " + mean + ", " + median + ", " + stdev + ", " + var + ", " + kurtosis + ", " + skewness + ", " + ci);
}

Example 5 with SummaryStatistics

use of org.apache.commons.math.stat.descriptive.SummaryStatistics in project titan by thinkaurelius.

the class TitanGraphPerformanceMemoryTest method testMemoryLeakage.

@Test
public void testMemoryLeakage() {
    long memoryBaseline = 0;
    SummaryStatistics stats = new SummaryStatistics();
    int numRuns = 25;
    for (int r = 0; r < numRuns; r++) {
        if (r == 1 || r == (numRuns - 1)) {
            memoryBaseline = MemoryAssess.getMemoryUse();
            stats.addValue(memoryBaseline);
        // System.out.println("Memory before run "+(r+1)+": " + memoryBaseline / 1024 + " KB");
        }
        for (int t = 0; t < 1000; t++) {
            graph.addVertex(null);
            graph.rollback();
            TitanTransaction tx = graph.newTransaction();
            tx.addVertex();
            tx.rollback();
        }
        if (r == 1 || r == (numRuns - 1)) {
            memoryBaseline = MemoryAssess.getMemoryUse();
            stats.addValue(memoryBaseline);
        // System.out.println("Memory after run " + (r + 1) + ": " + memoryBaseline / 1024 + " KB");
        }
        clopen();
    }
    System.out.println("Average: " + stats.getMean() + " Std. Dev: " + stats.getStandardDeviation());
    assertTrue(stats.getStandardDeviation() < stats.getMin());
}