Examples with SummaryStatistics org.apache.commons.math3.stat.descriptive.SummaryStatistics used on opensource projects

Example 11 with SummaryStatistics

use of org.apache.commons.math3.stat.descriptive.SummaryStatistics in project cassandra by apache.

the class TokenAllocation method replicatedOwnershipStats.

public static SummaryStatistics replicatedOwnershipStats(TokenMetadata tokenMetadata, AbstractReplicationStrategy rs, InetAddress endpoint) {
    SummaryStatistics stat = new SummaryStatistics();
    StrategyAdapter strategy = getStrategy(tokenMetadata, rs, endpoint);
    for (Map.Entry<InetAddress, Double> en : evaluateReplicatedOwnership(tokenMetadata, rs).entrySet()) {
        // Filter only in the same datacentre.
        if (strategy.inAllocationRing(en.getKey()))
            stat.addValue(en.getValue() / tokenMetadata.getTokens(en.getKey()).size());
    }
    return stat;
}

Example 12 with SummaryStatistics

use of org.apache.commons.math3.stat.descriptive.SummaryStatistics in project cassandra by apache.

the class NoReplicationTokenAllocatorTest method updateSummary.

private void updateSummary(NoReplicationTokenAllocator<Unit> t, Summary su, Summary st, boolean print) {
    int size = t.sortedTokens.size();
    SummaryStatistics unitStat = new SummaryStatistics();
    for (TokenAllocatorBase.Weighted<TokenAllocatorBase.UnitInfo> wu : t.sortedUnits) {
        unitStat.addValue(wu.weight * size / t.tokensInUnits.get(wu.value.unit).size());
    }
    su.update(unitStat);
    SummaryStatistics tokenStat = new SummaryStatistics();
    for (PriorityQueue<TokenAllocatorBase.Weighted<TokenAllocatorBase.TokenInfo>> tokens : t.tokensInUnits.values()) {
        for (TokenAllocatorBase.Weighted<TokenAllocatorBase.TokenInfo> token : tokens) {
            tokenStat.addValue(token.weight);
        }
    }
    st.update(tokenStat);
    if (print) {
        System.out.format("Size %d(%d)   \tunit %s  token %s   %s\n", t.sortedUnits.size(), size, mms(unitStat), mms(tokenStat), t.strategy);
        System.out.format("Worst intermediate unit\t%s  token %s\n", su, st);
    }
}

Example 13 with SummaryStatistics

use of org.apache.commons.math3.stat.descriptive.SummaryStatistics in project GDSC-SMLM by aherbert.

the class DensityImage method logDensityResults.

/**
	 * Output a log message of the results including the average density for localisations and the expected average.
	 * 
	 * @param results
	 * @param density
	 * @param radius
	 * @param filtered
	 * @return
	 */
private SummaryStatistics logDensityResults(MemoryPeakResults results, int[] density, float radius, int filtered) {
    float region = (float) (radius * radius * ((useSquareApproximation) ? 4 : Math.PI));
    Rectangle bounds = results.getBounds();
    float area = bounds.width * bounds.height;
    float expected = results.size() * region / area;
    SummaryStatistics summary = new SummaryStatistics();
    for (int i = 0; i < results.size(); i++) {
        summary.addValue(density[i]);
    }
    DensityManager dm = createDensityManager(results);
    // Compute this using the input density scores since the radius is the same.
    final double l = (useSquareApproximation) ? dm.ripleysLFunction(radius) : dm.ripleysLFunction(density, radius);
    String msg = String.format("Density %s : N=%d, %.0fpx : Radius=%s : L(r) - r = %s : E = %s, Obs = %s (%sx)", results.getName(), summary.getN(), area, rounded(radius), rounded(l - radius), rounded(expected), rounded(summary.getMean()), rounded(summary.getMean() / expected));
    if (filterLocalisations)
        msg += String.format(" : Filtered=%d (%s%%)", filtered, rounded(filtered * 100.0 / density.length));
    IJ.log(msg);
    return summary;
}

Example 14 with SummaryStatistics

use of org.apache.commons.math3.stat.descriptive.SummaryStatistics in project GDSC-SMLM by aherbert.

the class PSFEstimator method checkAngleSignificance.

private boolean checkAngleSignificance() {
    boolean tryAgain = false;
    if (ignore[ANGLE])
        return tryAgain;
    // The angle is relative to the major axis (X). 
    // It could be close to 0, 90 or 180 to allow it to be ignored in favour of a free circular function.
    final double[] angles = sampleNew[ANGLE].getValues();
    for (double testAngle : new double[] { 90, 0, 180 }) {
        // The angle will be in the 0-180 domain.
        // We need to compute the Statistical summary around the testAngle.
        StatisticalSummary sampleStats;
        if (testAngle == 0 || testAngle == 180) {
            SummaryStatistics stats = new SummaryStatistics();
            boolean zeroAngle = (testAngle == 0);
            for (double a : angles) {
                if (zeroAngle) {
                    // Convert to -90-90 domain
                    if (a > 90)
                        a -= 180;
                } else {
                    // Convert to 90-270 domain
                    if (a < 90)
                        a += 180;
                }
                stats.addValue(a);
            }
            sampleStats = stats;
        } else {
            // Already in the 0-180 domain around the angle 90
            sampleStats = sampleNew[ANGLE];
        }
        final double p = TestUtils.tTest(testAngle, sampleStats);
        if (p > settings.pValue) {
            log("NOTE: Angle is not significant: %g ~ %g (p=%g) => Re-run with fixed zero angle", sampleStats.getMean(), testAngle, p);
            ignore[ANGLE] = true;
            config.getFitConfiguration().setFitFunction(FitFunction.FREE_CIRCULAR);
            tryAgain = true;
            break;
        } else
            debug("  NOTE: Angle is significant: %g !~ %g (p=%g)", sampleNew[ANGLE].getMean(), testAngle, p);
    }
    return tryAgain;
}

Example 15 with SummaryStatistics

use of org.apache.commons.math3.stat.descriptive.SummaryStatistics in project GDSC-SMLM by aherbert.

the class PCPALMFitting method fitRandomModel.

/**
	 * Fits the correlation curve with r>0 to the random model using the estimated density and precision. Parameters
	 * must be fit within a tolerance of the starting values.
	 * 
	 * @param gr
	 * @param sigmaS
	 *            The estimated precision
	 * @param proteinDensity
	 *            The estimate protein density
	 * @return The fitted parameters [precision, density]
	 */
private double[] fitRandomModel(double[][] gr, double sigmaS, double proteinDensity, String resultColour) {
    final RandomModelFunction function = new RandomModelFunction();
    randomModel = function;
    log("Fitting %s: Estimated precision = %f nm, estimated protein density = %g um^-2", randomModel.getName(), sigmaS, proteinDensity * 1e6);
    randomModel.setLogging(true);
    for (int i = offset; i < gr[0].length; i++) {
        // Only fit the curve above the estimated resolution (points below it will be subject to error)
        if (gr[0][i] > sigmaS * fitAboveEstimatedPrecision)
            randomModel.addPoint(gr[0][i], gr[1][i]);
    }
    LevenbergMarquardtOptimizer optimizer = new LevenbergMarquardtOptimizer();
    Optimum optimum;
    try {
        //@formatter:off
        LeastSquaresProblem problem = new LeastSquaresBuilder().maxEvaluations(Integer.MAX_VALUE).maxIterations(3000).start(new double[] { sigmaS, proteinDensity }).target(function.getY()).weight(new DiagonalMatrix(function.getWeights())).model(function, new MultivariateMatrixFunction() {

            public double[][] value(double[] point) throws IllegalArgumentException {
                return function.jacobian(point);
            }
        }).build();
        //@formatter:on
        optimum = optimizer.optimize(problem);
    } catch (TooManyIterationsException e) {
        log("Failed to fit %s: Too many iterations (%s)", randomModel.getName(), e.getMessage());
        return null;
    } catch (ConvergenceException e) {
        log("Failed to fit %s: %s", randomModel.getName(), e.getMessage());
        return null;
    }
    randomModel.setLogging(false);
    double[] parameters = optimum.getPoint().toArray();
    // Ensure the width is positive
    parameters[0] = Math.abs(parameters[0]);
    double ss = optimum.getResiduals().dotProduct(optimum.getResiduals());
    ic1 = Maths.getAkaikeInformationCriterionFromResiduals(ss, randomModel.size(), parameters.length);
    final double fitSigmaS = parameters[0];
    final double fitProteinDensity = parameters[1];
    // Check the fitted parameters are within tolerance of the initial estimates
    double e1 = parameterDrift(sigmaS, fitSigmaS);
    double e2 = parameterDrift(proteinDensity, fitProteinDensity);
    log("  %s fit: SS = %f. cAIC = %f. %d evaluations", randomModel.getName(), ss, ic1, optimum.getEvaluations());
    log("  %s parameters:", randomModel.getName());
    log("    Average precision = %s nm (%s%%)", Utils.rounded(fitSigmaS, 4), Utils.rounded(e1, 4));
    log("    Average protein density = %s um^-2 (%s%%)", Utils.rounded(fitProteinDensity * 1e6, 4), Utils.rounded(e2, 4));
    valid1 = true;
    if (fittingTolerance > 0 && (Math.abs(e1) > fittingTolerance || Math.abs(e2) > fittingTolerance)) {
        log("  Failed to fit %s within tolerance (%s%%): Average precision = %f nm (%s%%), average protein density = %g um^-2 (%s%%)", randomModel.getName(), Utils.rounded(fittingTolerance, 4), fitSigmaS, Utils.rounded(e1, 4), fitProteinDensity * 1e6, Utils.rounded(e2, 4));
        valid1 = false;
    }
    if (valid1) {
        // ---------
        // TODO - My data does not comply with this criteria. 
        // This could be due to the PC-PALM Molecule code limiting the nmPerPixel to fit the images in memory 
        // thus removing correlations at small r.
        // It could also be due to the nature of the random simulations being 3D not 2D membranes 
        // as per the PC-PALM paper. 
        // ---------
        // Evaluate g(r)protein where:
        // g(r)peaks = g(r)protein + g(r)stoch
        // g(r)peaks ~ 1           + g(r)stoch
        // Verify g(r)protein should be <1.5 for all r>0
        double[] gr_stoch = randomModel.value(parameters);
        double[] gr_peaks = randomModel.getY();
        double[] gr_ = randomModel.getX();
        //SummaryStatistics stats = new SummaryStatistics();
        for (int i = 0; i < gr_peaks.length; i++) {
            // Only evaluate above the fitted average precision 
            if (gr_[i] < fitSigmaS)
                continue;
            // Note the RandomModelFunction evaluates g(r)stoch + 1;
            double gr_protein_i = gr_peaks[i] - (gr_stoch[i] - 1);
            if (gr_protein_i > gr_protein_threshold) {
                // Failed fit
                log("  Failed to fit %s: g(r)protein %s > %s @ r=%s", randomModel.getName(), Utils.rounded(gr_protein_i, 4), Utils.rounded(gr_protein_threshold, 4), Utils.rounded(gr_[i], 4));
                valid1 = false;
            }
        //stats.addValue(gr_i);
        //System.out.printf("g(r)protein @ %f = %f\n", gr[0][i], gr_protein_i);
        }
    }
    addResult(randomModel.getName(), resultColour, valid1, fitSigmaS, fitProteinDensity, 0, 0, 0, 0, ic1);
    return parameters;
}