Examples with DescriptiveStatistics org.apache.commons.math3.stat.descriptive.DescriptiveStatistics used on opensource projects

Example 26 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project GDSC-SMLM by aherbert.

the class Fire method calculatePrecisionHistogram.

/**
 * Calculate a histogram of the precision. The precision can be either stored in the results or
 * calculated using the Mortensen formula. If the precision method for Q estimation is not fixed
 * then the histogram is fitted with a Gaussian to create an initial estimate.
 *
 * @return The precision histogram
 */
private PrecisionHistogram calculatePrecisionHistogram() {
    final boolean logFitParameters = false;
    final String title = results.getName() + " Precision Histogram";
    // Check if the results has the precision already or if it can be computed.
    final boolean canUseStored = canUseStoredPrecision(results);
    final boolean canCalculatePrecision = canCalculatePrecision(results);
    // Set the method to compute a histogram. Default to the user selected option.
    PrecisionMethod method = null;
    if ((canUseStored && precisionMethod == PrecisionMethod.STORED) || (canCalculatePrecision && precisionMethod == PrecisionMethod.CALCULATE)) {
        method = precisionMethod;
    }
    if (method == null) {
        // We only have two choices so if one is available then select it.
        if (canUseStored) {
            method = PrecisionMethod.STORED;
        } else if (canCalculatePrecision) {
            method = PrecisionMethod.CALCULATE;
        }
        // If the user selected a method not available then log a warning
        if (method != null && precisionMethod != PrecisionMethod.FIXED) {
            IJ.log(String.format("%s : Selected precision method '%s' not available, switching to '%s'", pluginTitle, precisionMethod, method.getName()));
        }
        if (method == null) {
            // This does not matter if the user has provide a fixed input.
            if (precisionMethod == PrecisionMethod.FIXED) {
                final PrecisionHistogram histogram = new PrecisionHistogram(title);
                histogram.mean = settings.mean;
                histogram.sigma = settings.sigma;
                return histogram;
            }
            // No precision
            return null;
        }
    }
    // We get here if we can compute precision.
    // Build the histogram
    StoredDataStatistics precision = new StoredDataStatistics(results.size());
    if (method == PrecisionMethod.STORED) {
        final StoredDataStatistics p = precision;
        results.forEach((PeakResultProcedure) result -> p.add(result.getPrecision()));
    } else {
        precision.add(pp.precisions);
    }
    double yMin = Double.NEGATIVE_INFINITY;
    double yMax = 0;
    // Set the min and max y-values using 1.5 x IQR
    final DescriptiveStatistics stats = precision.getStatistics();
    final double lower = stats.getPercentile(25);
    final double upper = stats.getPercentile(75);
    if (Double.isNaN(lower) || Double.isNaN(upper)) {
        if (logFitParameters) {
            ImageJUtils.log("Error computing IQR: %f - %f", lower, upper);
        }
    } else {
        final double iqr = upper - lower;
        yMin = Math.max(lower - iqr, stats.getMin());
        yMax = Math.min(upper + iqr, stats.getMax());
        if (logFitParameters) {
            ImageJUtils.log("  Data range: %f - %f. Plotting 1.5x IQR: %f - %f", stats.getMin(), stats.getMax(), yMin, yMax);
        }
    }
    if (yMin == Double.NEGATIVE_INFINITY) {
        final int n = 5;
        yMin = Math.max(stats.getMin(), stats.getMean() - n * stats.getStandardDeviation());
        yMax = Math.min(stats.getMax(), stats.getMean() + n * stats.getStandardDeviation());
        if (logFitParameters) {
            ImageJUtils.log("  Data range: %f - %f. Plotting mean +/- %dxSD: %f - %f", stats.getMin(), stats.getMax(), n, yMin, yMax);
        }
    }
    // Get the data within the range
    final double[] data = precision.getValues();
    precision = new StoredDataStatistics(data.length);
    for (final double d : data) {
        if (d < yMin || d > yMax) {
            continue;
        }
        precision.add(d);
    }
    final int histogramBins = HistogramPlot.getBins(precision, HistogramPlot.BinMethod.SCOTT);
    final float[][] hist = HistogramPlot.calcHistogram(precision.getFloatValues(), yMin, yMax, histogramBins);
    final PrecisionHistogram histogram = new PrecisionHistogram(hist, precision, title);
    if (precisionMethod == PrecisionMethod.FIXED) {
        histogram.mean = settings.mean;
        histogram.sigma = settings.sigma;
        return histogram;
    }
    // Fitting of the histogram to produce the initial estimate
    // Extract non-zero data
    float[] x = Arrays.copyOf(hist[0], hist[0].length);
    float[] y = Arrays.copyOf(hist[1], hist[1].length);
    int count = 0;
    for (int i = 0; i < y.length; i++) {
        if (y[i] > 0) {
            x[count] = x[i];
            y[count] = y[i];
            count++;
        }
    }
    x = Arrays.copyOf(x, count);
    y = Arrays.copyOf(y, count);
    // Sense check to fitted data. Get mean and SD of histogram
    final double[] stats2 = HistogramPlot.getHistogramStatistics(x, y);
    if (logFitParameters) {
        ImageJUtils.log("  Initial Statistics: %f +/- %f", stats2[0], stats2[1]);
    }
    histogram.mean = stats2[0];
    histogram.sigma = stats2[1];
    // Standard Gaussian fit
    final double[] parameters = fitGaussian(x, y);
    if (parameters == null) {
        ImageJUtils.log("  Failed to fit initial Gaussian");
        return histogram;
    }
    final double newMean = parameters[1];
    final double error = Math.abs(stats2[0] - newMean) / stats2[1];
    if (error > 3) {
        ImageJUtils.log("  Failed to fit Gaussian: %f standard deviations from histogram mean", error);
        return histogram;
    }
    if (newMean < yMin || newMean > yMax) {
        ImageJUtils.log("  Failed to fit Gaussian: %f outside data range %f - %f", newMean, yMin, yMax);
        return histogram;
    }
    if (logFitParameters) {
        ImageJUtils.log("  Initial Gaussian: %f @ %f +/- %f", parameters[0], parameters[1], parameters[2]);
    }
    histogram.mean = parameters[1];
    histogram.sigma = parameters[2];
    return histogram;
}

Example 27 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project android-beacon-library by AltBeacon.

the class NotifierSetCopyBenchmarksTest method logStats.

private void logStats(Set<?> set, double[] raw) {
    DescriptiveStatistics descStats = new DescriptiveStatistics(raw);
    Log.i(TAG, String.format(Locale.US, STAT_FORMAT, set.getClass().getSimpleName(), set.size(), descStats.getN(), Math.round(descStats.getMin()), Math.round(descStats.getMax()), Math.round(descStats.getMean()), descStats.getStandardDeviation(), descStats.getVariance()));
}

Example 28 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project ontrack by nemerosa.

the class JobScatteringTest method scatteringInSameType.

@Test
public void scatteringInSameType() {
    // Scheduler
    DefaultJobScheduler scheduler = new DefaultJobScheduler(NOPJobDecorator.INSTANCE, new SynchronousScheduledExecutorService(), NOPJobListener.INSTANCE, false, true, 1.0);
    // Creates a list of jobs with a weak key
    List<TestJob> jobs = TestUtils.range(1, 100).stream().map(i -> TestJob.of(String.format("%d", i))).collect(Collectors.toList());
    // Orchestration of all those jobs every 6 hours
    Collection<JobOrchestratorSupplier> jobOrchestratorSupplier = Collections.singletonList(() -> jobs.stream().map(j -> JobRegistration.of(j).everyMinutes(6 * 60)));
    // Orchestrator
    JobOrchestrator orchestrator = new JobOrchestrator(scheduler, "Orchestrator", jobOrchestratorSupplier);
    // Scheduling the orchestrator (manual mode)
    scheduler.schedule(orchestrator, Schedule.NONE);
    // Launching the orchestrator (manually)
    orchestrator.orchestrate(JobRunListener.out());
    // Getting the actual schedules of the jobs
    List<Schedule> actualSchedules = jobs.stream().map(job -> scheduler.getJobStatus(job.getKey())).filter(Optional::isPresent).map(Optional::get).map(JobStatus::getActualSchedule).collect(Collectors.toList());
    List<Long> initialPeriods = actualSchedules.stream().map(Schedule::getInitialPeriod).collect(Collectors.toList());
    initialPeriods.forEach(l -> System.out.format("--> %d%n", l));
    // Checks that all jobs have been scheduled
    assertEquals("All jobs have been scheduled", jobs.size(), initialPeriods.size());
    // Checks that all schedules more or less different
    DescriptiveStatistics stats = new DescriptiveStatistics();
    initialPeriods.forEach(stats::addValue);
    // Gets the std deviation
    double standardDeviation = stats.getStandardDeviation();
    double max = stats.getMax();
    // Gets this in minutes (this was returned in ms)
    double stdDevMinutes = TimeUnit.MINUTES.convert((long) standardDeviation, TimeUnit.MILLISECONDS);
    double maxMinutes = TimeUnit.MINUTES.convert((long) max, TimeUnit.MILLISECONDS);
    // It must be >> 0
    assertTrue("Std deviation must be >> 0", stdDevMinutes > 60.0);
    System.out.println("Max = " + maxMinutes);
    assertTrue("Max is <= period", maxMinutes <= 6 * 60.0);
}

Example 29 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project hive by apache.

the class ReplStatsTracker method toString.

@Override
public String toString() {
    StringBuilder sb = new StringBuilder();
    DecimalFormat dFormat = new DecimalFormat("#.##");
    dFormat.setRoundingMode(RoundingMode.HALF_UP);
    sb.append("Replication Stats{");
    for (Map.Entry<String, DescriptiveStatistics> event : descMap.entrySet()) {
        DescriptiveStatistics statistics = event.getValue();
        sb.append("[[Event Name: ").append(event.getKey()).append("; ");
        sb.append("Total Number: ").append(statistics.getN()).append("; ");
        sb.append("Total Time: ").append(dFormat.format(statistics.getSum())).append("; ");
        sb.append("Mean: ").append(formatDouble(dFormat, statistics.getMean())).append("; ");
        sb.append("Median: ").append(formatDouble(dFormat, statistics.getPercentile(50))).append("; ");
        sb.append("Standard Deviation: ").append(formatDouble(dFormat, statistics.getStandardDeviation())).append("; ");
        sb.append("Variance: ").append(formatDouble(dFormat, statistics.getVariance())).append("; ");
        sb.append("Kurtosis: ").append(formatDouble(dFormat, statistics.getKurtosis())).append("; ");
        sb.append("Skewness: ").append(formatDouble(dFormat, statistics.getSkewness())).append("; ");
        sb.append("25th Percentile: ").append(formatDouble(dFormat, statistics.getPercentile(25))).append("; ");
        sb.append("50th Percentile: ").append(formatDouble(dFormat, statistics.getPercentile(50))).append("; ");
        sb.append("75th Percentile: ").append(formatDouble(dFormat, statistics.getPercentile(75))).append("; ");
        sb.append("90th Percentile: ").append(formatDouble(dFormat, statistics.getPercentile(90))).append("; ");
        sb.append("Top ").append(k).append(" EventIds(EventId=Time) ").append(topKEvents.get(event.getKey())).append(";" + "]]");
    }
    sb.append("}");
    return sb.toString();
}

Example 30 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project hive by apache.

the class HMSBenchmarks method benchmarkCreatePartition.

static DescriptiveStatistics benchmarkCreatePartition(@NotNull MicroBenchmark bench, @NotNull BenchData data) {
    final HMSClient client = data.getClient();
    String dbName = data.dbName;
    String tableName = data.tableName;
    BenchmarkUtils.createPartitionedTable(client, dbName, tableName);
    final List<String> values = Collections.singletonList("d1");
    try {
        Table t = client.getTable(dbName, tableName);
        Partition partition = new Util.PartitionBuilder(t).withValues(values).build();
        return bench.measure(null, () -> throwingSupplierWrapper(() -> client.addPartition(partition)), () -> throwingSupplierWrapper(() -> client.dropPartition(dbName, tableName, values)));
    } catch (TException e) {
        e.printStackTrace();
        return new DescriptiveStatistics();
    } finally {
        throwingSupplierWrapper(() -> client.dropTable(dbName, tableName));
    }
}