Examples with DescriptiveStatistics org.apache.commons.math3.stat.descriptive.DescriptiveStatistics used on opensource projects

Example 76 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project jenetics by jenetics.

the class LongMomentStatisticsTest method summary.

@Test(dataProvider = "sampleCounts")
public void summary(final Integer sampleCounts, final Double epsilon) {
    final List<Long> numbers = numbers(sampleCounts);
    final DescriptiveStatistics expected = new DescriptiveStatistics();
    numbers.forEach(expected::addValue);
    final LongMomentStatistics summary = numbers.stream().collect(toLongMomentStatistics(Long::longValue));
    Assert.assertEquals(summary.count(), numbers.size());
    assertEqualsDouble(min(summary.min()), expected.getMin(), 0.0);
    assertEqualsDouble(max(summary.max()), expected.getMax(), 0.0);
    assertEqualsDouble(summary.sum(), expected.getSum(), epsilon);
    assertEqualsDouble(summary.mean(), expected.getMean(), epsilon);
    assertEqualsDouble(summary.variance(), expected.getVariance(), epsilon);
    assertEqualsDouble(summary.skewness(), expected.getSkewness(), epsilon);
    assertEqualsDouble(summary.kurtosis(), expected.getKurtosis(), epsilon);
}

Example 77 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project vsp-playgrounds by matsim-vsp.

the class AR1ConvergenceCriterion method process.

/**
 * @param data
 *            A data array, sorted in <b>decreasing</b> time order: [x(k),
 *            x(k-1), ..., x(0)].
 */
private void process(final double[] data) {
    this.converged = false;
    this.convergedMean = null;
    this.convergedMeanStddev = null;
    this.convergedBinSize = null;
    this.convergedSinceIteration = null;
    final int minBinCnt = 2 * this.minSampleCntPerParameter + 1;
    final int maxBinSize = data.length / minBinCnt;
    final int minBinSize = Math.max(1, (int) Math.round(0.1 * maxBinSize));
    // The smaller the bins, the better.
    for (int binSize = minBinSize; binSize <= maxBinSize; binSize++) {
        final int maxBinCnt = data.length / binSize;
        final double[] allBinData = new double[maxBinCnt];
        for (int bin = 0; bin < maxBinCnt; bin++) {
            allBinData[bin] = this.binAvg(data, bin, binSize);
        }
        // The more bins, the better.
        for (int usedBinCnt = maxBinCnt; usedBinCnt >= minBinCnt; usedBinCnt--) {
            final double[] usedBinData = new double[usedBinCnt];
            System.arraycopy(allBinData, 0, usedBinData, 0, usedBinCnt);
            final DescriptiveStatistics stats = new DescriptiveStatistics(usedBinData);
            double[] y = new double[usedBinCnt];
            double[][] x = new double[usedBinCnt][1];
            for (int bin = 0; bin < usedBinCnt - 1; bin++) {
                y[bin] = allBinData[bin] - stats.getMean();
                x[bin][0] = allBinData[bin + 1] - stats.getMean();
            }
            y[usedBinCnt - 1] = 0.0;
            x[usedBinCnt - 1][0] = 1e-8;
            final OLSMultipleLinearRegression regr = new OLSMultipleLinearRegression();
            regr.setNoIntercept(false);
            regr.newSampleData(y, x);
            final double[] beta = regr.estimateRegressionParameters();
            final double[] betaSigma = regr.estimateRegressionParametersStandardErrors();
            final double meanStddev = stats.getStandardDeviation() / Math.sqrt(usedBinCnt - 1);
            if ((meanStddev <= this.maxConvergedMeanStddev) && this.statisticallyZero(beta[0], betaSigma[0]) && this.statisticallyZero(beta[1], betaSigma[1])) {
                this.converged = true;
                this.convergedMean = stats.getMean();
                this.convergedMeanStddev = meanStddev;
                this.convergedBinSize = binSize;
                this.convergedSinceIteration = data.length - usedBinCnt * binSize;
            }
        }
    }
}

Example 78 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project vsp-playgrounds by matsim-vsp.

the class ArticleFigureCreator method createDetourFlows.

static void createDetourFlows() {
    /*
		 * WITHOUT TOLL, MORNING PEAK
		 */
    // final String[] results = new String[] {
    // "0	0	0	0	0	5.2	13.4	588.2	744.2	686.2	298.2	9.2	0.4	0	0	0	0	0	0	0	0	0	0	0",
    // "0	0	0	0	0	7	15.8	583.8	769	640.6	294.4	7.6	0.6	0	0	0	0	0	0	0	0	0	0	0",
    // "0	0	0	0	0	4.8	16.6	580.6	793.4	628.6	297.2	6.8	0.8	0	0	0	0	0	0	0	0	0	0	0",
    // "0	0	0	0	0	7.4	11.6	605	750.6	689.6	280.6	6.4	0.4	0	0	0	0	0	0	0	0	0	0	0",
    // "0	0	0	0	0	5	16	610.8	766.4	688	283.4	6.6	0.2	0	0	0	0	0	0	0	0	0	0	0",
    // "0	0	0	0	0	5.2	17.4	612.4	773.8	651.8	278.8	8.8	1.4	0	0	0	0	0	0	0	0	0	0	0",
    // "0	0	0	0	0	2.4	16.6	615	763.6	666.8	309.2	7.2	0.4	0	0	0	0	0	0	0	0	0	0	0",
    // "0	0	0	0	0	6.2	16.2	591	760.2	638.8	292	5	1	0	0	0	0	0	0	0	0	0	0	0",
    // "0	0	0	0	0	4	15.2	595.8	764.8	642.4	276.6	6.4	1	0	0	0	0	0	0	0	0	0	0	0",
    // "0	0	0	0	0	6	15.6	604	788.2	642.6	289.4	7	0.6	0	0	0	0	0	0	0	0	0	0	0" };
    /*
		 * WITHOUT TOLL, EVENING PEAK
		 */
    // final String[] results = new String[] {
    // "0	0	0	0	0	0	0	0	0	0	0	0	0.6	8.4	179.4	627.2	659.2	740.2	131.4	4.6	0	0	0	0",
    // "0	0	0	0	0	0	0	0	0	0	0	0	0	7.4	165.2	611	627.4	674	98.8	2.2	0	0	0	0",
    // "0	0	0	0	0	0	0	0	0	0	0	0	0	6.4	170.8	637.6	607.8	709.4	133.8	3.4	0	0	0	0",
    // "0	0	0	0	0	0	0	0	0	0	0	0	0.2	7.4	185.2	623.8	646.4	629.8	107.8	3.4	0	0	0	0",
    // "0	0	0	0	0	0	0	0	0	0	0	0	0.4	10.2	199	605	658.6	659.4	114.6	4.2	0	0	0	0",
    // "0	0	0	0	0	0	0	0	0	0	0	0	0.6	7.4	189.2	653.2	626.6	738.8	140	3	0	0	0	0",
    // "0	0	0	0	0	0	0	0	0	0	0	0	0.4	7.8	176.8	614.2	629.4	616	103.2	4.6	0	0	0	0",
    // "0	0	0	0	0	0	0	0	0	0	0	0	0	5.8	185.6	614	659.4	681.6	157	3.2	0	0	0	0",
    // "0	0	0	0	0	0	0	0	0	0	0	0	0.4	4.6	178.4	619	655.8	699.6	139.6	4.4	0	0	0	0",
    // "0	0	0	0	0	0	0	0	0	0	0	0	0.2	5.6	185.6	608.6	665	648.2	164.6	3.4	0	0	0	0"
    // };
    /*
		 * WITH TOLL, MORNING PEAK
		 */
    // final String[] results = new String[] {
    // "0	0	0	0	0	10.4	25.2	1056.2	1151.8	1132.8	970.2	221.4	11.2	0.8	0	0	0	0	0	0	0	0	0	0",
    // "0	0	0	0	0	6.8	20.4	993.4	1143.8	1129.8	994	380.2	22	1.6	0	0	0	0	0	0	0	0	0	0",
    // "0	0	0	0	0	7.4	23	996.6	1147	1123	995.6	398.2	20.4	1.4	0	0	0	0	0	0	0	0	0	0",
    // "0	0	0	0	0	8.2	24.8	1078	1144.8	1128.8	916.4	251.2	41.4	4.6	0.4	0	0	0	0	0	0	0	0	0",
    // "0	0	0	0	0	10.4	22.8	1006.6	1152	1131	1002.6	373	20.4	2.2	0	0	0	0	0	0	0	0	0	0",
    // "0	0	0	0	0	5.8	23.6	977	1147.2	1144.6	640.4	17.6	2.4	0	0	0	0	0	0	0	0	0	0	0",
    // "0	0	0	0	0	8.6	20.2	1054.4	1148.6	1119.2	1020.2	377.6	18.8	1.8	0	0	0	0	0	0	0	0	0	0",
    // "0	0	0	0	0	8.6	22	999.6	1143	1140.2	1004.8	355.6	24.2	1.4	0	0	0	0	0	0	0	0	0	0",
    // "0	0	0	0	0	9.4	22.8	1012	1140	1126.4	1022.6	357.6	24.2	1.2	0	0	0	0	0	0	0	0	0	0",
    // "0	0	0	0	0	10	22.6	1057.6	1145.4	1137.6	987.4	383.8	25.6	2.8	0	0	0	0	0	0	0	0	0	0"
    // };
    /*
		 * WITH TOLL, EVENING PEAK
		 */
    final String[] results = new String[] { "0	0	0	0	0	0	0	0	0	0	0	0.8	15.2	192.2	780	1113.4	1147.2	1146.6	109.2	8	0	0	0	0", "0	0	0	0	0	0	0	0	0	0	0	1.4	16.2	307.8	929.4	1135.2	1148.2	1145.2	86.6	6.8	0	0	0	0", "0	0	0	0	0	0	0	0	0	0	0	0.4	15.2	316.6	933.2	1129.6	1149	1152	86.8	5.4	0	0	0	0", "0	0	0	0	0	0	0	0	0	0	0	3.6	17.8	241.6	752.4	1135.6	1142.6	1150	109.8	4	0	0	0	0", "0	0	0	0	0	0	0	0	0	0	0	1.2	21.4	325.2	905	1123	1150	1152	98	5.6	0	0	0	0", "0	0	0	0	0	0	0	0	0	0	0	0	0.2	11.8	277.6	1138.2	1152	1151.8	121.8	6	0	0	0	0", "0	0	0	0	0	0	0	0	0	0	0	0.4	11.6	307.8	927.8	1127	1147.4	1146.6	98	4.4	0.2	0	0	0", "0	0	0	0	0	0	0	0	0	0	0	0.8	18	301	942.6	1130.2	1139	1149.4	86.4	6.4	0	0	0	0", "0	0	0	0	0	0	0	0	0	0	0	0.6	14.2	296.2	939.6	1127.6	1140	1147.2	94.2	4.8	0	0	0	0", "0	0	0	0	0	0	0	0	0	0	0	1	19	307.6	911.8	1132	1142	1148.6	104.4	8.6	0	0	0	0" };
    final List<List<Double>> allDataHW = new ArrayList<>();
    for (final String result : results) {
        final List<Double> data = new ArrayList<>();
        allDataHW.add(data);
        for (String element : result.split("\\s")) {
            element = element.trim();
            if (element.length() > 0) {
                final double val = Double.parseDouble(element);
                data.add(val);
            }
        }
        System.out.println();
        System.out.println();
    }
    List<Double> modes = new ArrayList<>();
    for (int k = 0; k < allDataHW.get(0).size(); k++) {
        final DescriptiveStatistics stat = new DescriptiveStatistics();
        for (List<Double> data : allDataHW) {
            stat.addValue(data.get(k));
        }
        final double x = 0.5 + k;
        final double dx = 0.4;
        final double fact = 1.0;
        final double min = fact * stat.getMin();
        final double max = fact * stat.getMax();
        final double p25 = fact * stat.getPercentile(25);
        final double p50 = fact * stat.getPercentile(50);
        final double p75 = fact * stat.getPercentile(75);
        modes.add(p50);
        // the box
        System.out.println("\\psframe*[linecolor=white](" + (x - dx) + "," + p25 + ")(" + (x + dx) + "," + p75 + ")\n");
        System.out.println("\\psframe(" + (x - dx) + "," + p25 + ")(" + (x + dx) + "," + p75 + ")\n");
        System.out.println("\\psline(" + (x - dx) + "," + p50 + ")(" + (x + dx) + "," + p50 + ")\n");
        // upper vertical line
        System.out.println("\\psline(" + x + "," + p75 + ")(" + x + "," + max + ")\n");
        // upper horizontal line
        System.out.println("\\psline(" + (x - 0.5 * dx) + "," + max + ")(" + (x + 0.5 * dx) + "," + max + ")\n");
        // lower vertical line
        System.out.println("\\psline(" + x + "," + min + ")(" + x + "," + p25 + ")\n");
        // upper horizontal line
        System.out.println("\\psline(" + (x - 0.5 * dx) + "," + min + ")(" + (x + 0.5 * dx) + "," + min + ")\n");
        System.out.println();
    }
    double x = 0.5;
    for (Double mode : modes) {
        System.out.print("(" + x + "," + mode + ")");
        x += 1.0;
    }
}

Example 79 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project hive by apache.

the class ReplStatsTracker method addEntry.

/**
 * Adds an entry for tracking.
 * @param eventType the type of event.
 * @param eventId the event id.
 * @param timeTaken time taken to process the event.
 */
public synchronized void addEntry(String eventType, String eventId, long timeTaken) {
    // Store the last EventId for the JMX.
    lastEventId = eventId;
    // Update the entry in the descriptive statistics.
    DescriptiveStatistics descStatistics = descMap.get(eventType);
    if (descStatistics == null) {
        descStatistics = new DescriptiveStatistics();
        descStatistics.addValue(timeTaken);
        descMap.put(eventType, descStatistics);
    } else {
        descStatistics.addValue(timeTaken);
    }
    // Tracking for top K events, Maintain the list in descending order.
    ListOrderedMap<Long, Long> topKEntries = topKEvents.get(eventType);
    if (topKEntries == null) {
        topKEntries = new ListOrderedMap<>();
        topKEntries.put(Long.parseLong(eventId), timeTaken);
        topKEvents.put(eventType, topKEntries);
    } else {
        // Get the index of insertion, by descending order.
        int index = Collections.binarySearch(new ArrayList(topKEntries.values()), timeTaken, Collections.reverseOrder());
        // If the element comes as top K add it to the topEntries.
        // The index returned from the binary search, is either the index where the element already exist, else
        // (-insertionIndex) -1, so convert it to actual insertion index
        int insertionIndex = index < 0 ? -1 * (index) - 1 : index;
        if (insertionIndex < k && k >= 0) {
            topKEntries.put(insertionIndex, Long.parseLong(eventId), timeTaken);
        }
        // entries, since the list is sorted in descending order.
        if (topKEntries.size() > k) {
            topKEntries.remove(k);
        }
    }
}

Example 80 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project hive by apache.

the class HMSBenchmarks method benchmarkRenameTable.

static DescriptiveStatistics benchmarkRenameTable(@NotNull MicroBenchmark bench, @NotNull BenchData data, int count) {
    final HMSClient client = data.getClient();
    String dbName = data.dbName;
    String tableName = data.tableName;
    BenchmarkUtils.createPartitionedTable(client, dbName, tableName);
    try {
        addManyPartitionsNoException(client, dbName, tableName, null, Collections.singletonList("d"), count);
        Table oldTable = client.getTable(dbName, tableName);
        oldTable.getSd().setLocation("");
        Table newTable = oldTable.deepCopy();
        newTable.setTableName(tableName + "_renamed");
        return bench.measure(() -> {
            // Measuring 2 renames, so the tests are idempotent
            throwingSupplierWrapper(() -> client.alterTable(oldTable.getDbName(), oldTable.getTableName(), newTable));
            throwingSupplierWrapper(() -> client.alterTable(newTable.getDbName(), newTable.getTableName(), oldTable));
        });
    } catch (TException e) {
        e.printStackTrace();
        return new DescriptiveStatistics();
    } finally {
        throwingSupplierWrapper(() -> client.dropTable(dbName, tableName));
    }
}