Examples with FractionClassificationResult uk.ac.sussex.gdsc.core.match.FractionClassificationResult used on opensource projects

Example 1 with FractionClassificationResult

use of uk.ac.sussex.gdsc.core.match.FractionClassificationResult in project GDSC-SMLM by aherbert.

the class BenchmarkSpotFilter method summariseResults.

private BenchmarkSpotFilterResult summariseResults(TIntObjectHashMap<FilterResult> filterResults, FitEngineConfiguration config, MaximaSpotFilter spotFilter, boolean batchSummary) {
    final BenchmarkSpotFilterResult filterResult = new BenchmarkSpotFilterResult(simulationParameters.id, filterResults, config, spotFilter);
    // Note:
    // Although we can compute the TP/FP score as each additional spot is added
    // using the RankedScoreCalculator this is not applicable to the PeakFit method.
    // The method relies on all spot candidates being present in order to make a
    // decision to fit the candidate as a multiple. So scoring the filter candidates using
    // for example the top 10 may get a better score than if all candidates were scored
    // and the scores accumulated for the top 10, it is not how the algorithm will use the
    // candidate set. I.e. It does not use the top 10, then top 20 to refine the fit, etc.
    // (the method is not iterative) .
    // We require an assessment of how a subset of the scored candidates
    // in ranked order contributes to the overall score, i.e. are the candidates ranked
    // in the correct order, those most contributing to the match to the underlying data
    // should be higher up and those least contributing will be at the end.
    // TODO We could add some smart filtering of candidates before ranking. This would
    // allow assessment of the candidate set handed to PeakFit. E.g. Threshold the image
    // and only use candidates that are in the foreground region.
    final double[][] cumul = histogramFailures(filterResult);
    // Create the overall match score
    final double[] total = new double[3];
    final ArrayList<ScoredSpot> allSpots = new ArrayList<>();
    filterResults.forEachValue(result -> {
        total[0] += result.result.getTruePositives();
        total[1] += result.result.getFalsePositives();
        total[2] += result.result.getFalseNegatives();
        allSpots.addAll(Arrays.asList(result.spots));
        return true;
    });
    double tp = total[0];
    double fp = total[1];
    final double fn = total[2];
    final FractionClassificationResult allResult = new FractionClassificationResult(tp, fp, 0, fn);
    // The number of actual results
    final double numberOfResults = (tp + fn);
    final StringBuilder sb = new StringBuilder();
    final double signal = (simulationParameters.minSignal + simulationParameters.maxSignal) * 0.5;
    // Create the benchmark settings and the fitting settings
    sb.append(imp.getStackSize()).append('\t');
    final int w = border.width;
    final int h = border.height;
    sb.append(w).append('\t');
    sb.append(h).append('\t');
    sb.append(MathUtils.rounded(numberOfResults)).append('\t');
    final double density = (numberOfResults / imp.getStackSize()) / (w * h) / (simulationParameters.pixelPitch * simulationParameters.pixelPitch / 1e6);
    sb.append(MathUtils.rounded(density)).append('\t');
    sb.append(MathUtils.rounded(signal)).append('\t');
    sb.append(MathUtils.rounded(simulationParameters.sd)).append('\t');
    sb.append(MathUtils.rounded(simulationParameters.pixelPitch)).append('\t');
    sb.append(MathUtils.rounded(simulationParameters.depth)).append('\t');
    sb.append(simulationParameters.fixedDepth).append('\t');
    // Camera specific
    CreateData.addCameraDescription(sb, simulationParameters).append('\t');
    sb.append(MathUtils.rounded(simulationParameters.background)).append('\t');
    sb.append(MathUtils.rounded(simulationParameters.noise)).append('\t');
    sb.append(MathUtils.rounded(signal / simulationParameters.noise)).append('\t');
    sb.append(MathUtils.rounded(simulationParameters.sd / simulationParameters.pixelPitch)).append('\t');
    sb.append(config.getDataFilterType()).append('\t');
    sb.append(spotFilter.getSearch()).append('\t');
    sb.append(spotFilter.getBorder()).append('\t');
    sb.append(MathUtils.rounded(spotFilter.getSpread())).append('\t');
    sb.append(config.getDataFilterMethod(0)).append('\t');
    final double param = config.getDataFilterParameterValue(0);
    final boolean absolute = config.getDataFilterParameterAbsolute(0);
    final double hwhmMin = config.getHwhmMin();
    if (absolute) {
        sb.append(MathUtils.rounded(param)).append('\t');
        sb.append(MathUtils.roundUsingDecimalPlacesToBigDecimal(param / hwhmMin, 3)).append('\t');
    } else {
        sb.append(MathUtils.roundUsingDecimalPlacesToBigDecimal(param * hwhmMin, 3)).append('\t');
        sb.append(MathUtils.rounded(param)).append('\t');
    }
    sb.append(spotFilter.getDescription()).append('\t');
    sb.append(border.x).append('\t');
    sb.append(Settings.MATCHING_METHOD[settings.matchingMethod]).append('\t');
    sb.append(MathUtils.rounded(lowerMatchDistance)).append('\t');
    sb.append(MathUtils.rounded(matchDistance)).append('\t');
    sb.append(MathUtils.rounded(settings.lowerSignalFactor)).append('\t');
    sb.append(MathUtils.rounded(settings.upperSignalFactor));
    filterResult.resultPrefix = sb.toString();
    // Add the results
    sb.append('\t');
    // Rank the scored spots by intensity
    Collections.sort(allSpots, ScoredSpot::compare);
    // Produce Recall, Precision, Jaccard for each cut of the spot candidates
    final double[] recall = new double[allSpots.size() + 1];
    final double[] precision = new double[recall.length];
    final double[] jaccard = new double[recall.length];
    final double[] correlation = new double[recall.length];
    final double[] truePositives = new double[recall.length];
    final double[] falsePositives = new double[recall.length];
    final double[] intensity = new double[recall.length];
    // Note: fn = n - tp
    tp = fp = 0;
    int index = 1;
    precision[0] = 1;
    final FastCorrelator corr = new FastCorrelator();
    double lastCorrelation = 0;
    double[] i1 = new double[recall.length];
    double[] i2 = new double[recall.length];
    int ci = 0;
    final SimpleRegression regression = new SimpleRegression(false);
    for (final ScoredSpot s : allSpots) {
        if (s.match) {
            // Score partial matches as part true-positive and part false-positive.
            // TP can be above 1 if we are allowing multiple matches.
            tp += s.getScore();
            fp += s.antiScore();
            // Just use a rounded intensity for now
            final double spotIntensity = s.getIntensity();
            final long v1 = Math.round(spotIntensity);
            final long v2 = Math.round(s.intensity);
            regression.addData(spotIntensity, s.intensity);
            i1[ci] = spotIntensity;
            i2[ci] = s.intensity;
            ci++;
            corr.add(v1, v2);
            lastCorrelation = corr.getCorrelation();
        } else {
            fp++;
        }
        recall[index] = tp / numberOfResults;
        precision[index] = tp / (tp + fp);
        // (tp+fp+fn) == (fp+n) since tp+fn=n
        jaccard[index] = tp / (fp + numberOfResults);
        correlation[index] = lastCorrelation;
        truePositives[index] = tp;
        falsePositives[index] = fp;
        intensity[index] = s.getIntensity();
        index++;
    }
    i1 = Arrays.copyOf(i1, ci);
    i2 = Arrays.copyOf(i2, ci);
    final double slope = regression.getSlope();
    sb.append(MathUtils.rounded(slope)).append('\t');
    addResult(sb, allResult, correlation[correlation.length - 1]);
    // Output the match results when the recall achieves the fraction of the maximum.
    double target = recall[recall.length - 1];
    if (settings.recallFraction < 100) {
        target *= settings.recallFraction / 100.0;
    }
    int fractionIndex = 0;
    while (fractionIndex < recall.length && recall[fractionIndex] < target) {
        fractionIndex++;
    }
    if (fractionIndex == recall.length) {
        fractionIndex--;
    }
    sb.append('\t');
    addResult(sb, new FractionClassificationResult(truePositives[fractionIndex], falsePositives[fractionIndex], 0, numberOfResults - truePositives[fractionIndex]), correlation[fractionIndex]);
    // Output the match results at the maximum jaccard score
    int maxIndex = 0;
    for (int ii = 1; ii < recall.length; ii++) {
        if (jaccard[maxIndex] < jaccard[ii]) {
            maxIndex = ii;
        }
    }
    sb.append('\t');
    addResult(sb, new FractionClassificationResult(truePositives[maxIndex], falsePositives[maxIndex], 0, numberOfResults - truePositives[maxIndex]), correlation[maxIndex]);
    sb.append(MathUtils.rounded(time / 1e6));
    // Calculate AUC (Average precision == Area Under Precision-Recall curve)
    final double auc = AucCalculator.auc(precision, recall);
    // Compute the AUC using the adjusted precision curve
    // which uses the maximum precision for recall >= r
    final double[] maxp = new double[precision.length];
    double max = 0;
    for (int k = maxp.length; k-- > 0; ) {
        if (max < precision[k]) {
            max = precision[k];
        }
        maxp[k] = max;
    }
    final double auc2 = AucCalculator.auc(maxp, recall);
    sb.append('\t').append(MathUtils.rounded(auc));
    sb.append('\t').append(MathUtils.rounded(auc2));
    // positives
    if (cumul[0].length != 0) {
        sb.append('\t').append(MathUtils.rounded(getFailures(cumul, 0.80)));
        sb.append('\t').append(MathUtils.rounded(getFailures(cumul, 0.90)));
        sb.append('\t').append(MathUtils.rounded(getFailures(cumul, 0.95)));
        sb.append('\t').append(MathUtils.rounded(getFailures(cumul, 0.99)));
        sb.append('\t').append(MathUtils.rounded(cumul[0][cumul[0].length - 1]));
    } else {
        sb.append("\t\t\t\t\t");
    }
    getTable(batchSummary).append(sb.toString());
    // Store results
    filterResult.auc = auc;
    filterResult.auc2 = auc2;
    filterResult.recall = recall;
    filterResult.precision = precision;
    filterResult.jaccard = jaccard;
    filterResult.correlation = correlation;
    filterResult.maxIndex = maxIndex;
    filterResult.fractionIndex = fractionIndex;
    filterResult.cumul = cumul;
    filterResult.slope = slope;
    filterResult.i1 = i1;
    filterResult.i2 = i2;
    filterResult.intensity = intensity;
    filterResult.time = time;
    filterResult.analysisBorder = this.border;
    return filterResult;
}

Example 2 with FractionClassificationResult

use of uk.ac.sussex.gdsc.core.match.FractionClassificationResult in project GDSC-SMLM by aherbert.

the class BenchmarkFilterAnalysis method scoreFilter.

private FilterScoreResult scoreFilter(DirectFilter filter, DirectFilter minFilter, boolean createTextResult, CoordinateStore coordinateStore) {
    final FractionClassificationResult r = scoreFilter(filter, minFilter, gaResultsListToScore, coordinateStore);
    final double score = getScore(r);
    final double criteria = getCriteria(r);
    // Show the result if it achieves the criteria limit
    final String text = (createTextResult && criteria >= minCriteria) ? createResult(filter, r).toString() : null;
    return new FilterScoreResult(score, criteria, filter, text);
}

Example 3 with FractionClassificationResult

use of uk.ac.sussex.gdsc.core.match.FractionClassificationResult in project GDSC-SMLM by aherbert.

the class BenchmarkFilterAnalysis method score.

@Nullable
@Override
public SearchResult<FilterScore>[] score(double[][] points) {
    gaIteration++;
    SimpleFilterScore max = filterScoreOptimum;
    final FilterScoreResult[] scoreResults = scoreFilters(setStrength(new FilterSet(searchSpaceToFilters(points))), false);
    if (scoreResults == null) {
        return null;
    }
    @SuppressWarnings("unchecked") final SearchResult<FilterScore>[] scores = new SearchResult[scoreResults.length];
    for (int index = 0; index < scoreResults.length; index++) {
        final FilterScoreResult scoreResult = scoreResults[index];
        final SimpleFilterScore result = new SimpleFilterScore(scoreResult, true, scoreResult.criteria >= minCriteria);
        if (result.compareTo(max) < 0) {
            max = result;
        }
        scores[index] = new SearchResult<>(result.result.filter.getParameters(), result);
    }
    filterScoreOptimum = max;
    // Add the best filter to the table
    // This filter may not have been part of the scored subset so use the entire results set for
    // reporting
    final DirectFilter filter = max.result.filter;
    final FractionClassificationResult r = scoreFilter(filter, defaultMinimalFilter, gaResultsList, coordinateStore);
    final StringBuilder text = createResult(filter, r);
    add(text, gaIteration);
    gaWindow.accept(text.toString());
    return scores;
}

Example 4 with FractionClassificationResult

use of uk.ac.sussex.gdsc.core.match.FractionClassificationResult in project GDSC-SMLM by aherbert.

the class BenchmarkFilterAnalysis method calculateSensitivity.

private void calculateSensitivity() {
    if (!settings.calculateSensitivity) {
        return;
    }
    if (!filterAnalysisResult.bestFilter.isEmpty()) {
        final Consumer<String> output = createSensitivityWindow();
        final Ticker ticker = ImageJUtils.createTicker(filterAnalysisResult.bestFilter.size(), 0, "Calculating sensitivity ...");
        for (final String type : filterAnalysisResult.bestFilter.keySet()) {
            final DirectFilter filter = filterAnalysisResult.bestFilter.get(type).getFilter();
            FractionClassificationResult score = scoreFilter(filter, defaultMinimalFilter, fitResultData.resultsList, coordinateStore);
            score = getOriginalScore(score);
            final String message = type + "\t\t\t" + MathUtils.rounded(score.getJaccard(), 4) + "\t\t" + MathUtils.rounded(score.getPrecision(), 4) + "\t\t" + MathUtils.rounded(score.getRecall(), 4);
            output.accept(message);
            // List all the parameters that can be adjusted.
            final int parameters = filter.getNumberOfParameters();
            for (int index = 0; index < parameters; index++) {
                // For each parameter compute as upward + downward delta and get the average gradient
                final DirectFilter higher = (DirectFilter) filter.adjustParameter(index, settings.delta);
                final DirectFilter lower = (DirectFilter) filter.adjustParameter(index, -settings.delta);
                FractionClassificationResult scoreHigher = scoreFilter(higher, defaultMinimalFilter, fitResultData.resultsList, coordinateStore);
                scoreHigher = getOriginalScore(scoreHigher);
                FractionClassificationResult scoreLower = scoreFilter(lower, defaultMinimalFilter, fitResultData.resultsList, coordinateStore);
                scoreLower = getOriginalScore(scoreLower);
                final StringBuilder sb = new StringBuilder();
                sb.append('\t').append(filter.getParameterName(index)).append('\t');
                sb.append(MathUtils.rounded(filter.getParameterValue(index), 4)).append('\t');
                final double dx1 = higher.getParameterValue(index) - filter.getParameterValue(index);
                final double dx2 = filter.getParameterValue(index) - lower.getParameterValue(index);
                addSensitivityScore(sb, score.getJaccard(), scoreHigher.getJaccard(), scoreLower.getJaccard(), dx1, dx2);
                addSensitivityScore(sb, score.getPrecision(), scoreHigher.getPrecision(), scoreLower.getPrecision(), dx1, dx2);
                addSensitivityScore(sb, score.getRecall(), scoreHigher.getRecall(), scoreLower.getRecall(), dx1, dx2);
                output.accept(sb.toString());
            }
            ticker.tick();
        }
        final String message = "-=-=-=-";
        output.accept(message);
        ImageJUtils.finished();
    }
}

Example 5 with FractionClassificationResult

use of uk.ac.sussex.gdsc.core.match.FractionClassificationResult in project GDSC-SMLM by aherbert.

the class BenchmarkFilterAnalysis method createResult.

private StringBuilder createResult(DirectFilter filter, FractionClassificationResult result, String resultsPrefix2) {
    final StringBuilder sb = new StringBuilder(resultsPrefix);
    sb.append(filter.getName()).append(resultsPrefix2).append(fitResultData.resultsPrefix3);
    int index = 0;
    // Integer results
    if (settings.requireIntegerResults) {
        final ClassificationResult r2 = createIntegerResult(result);
        add(sb, r2.getTruePositives(), index++);
        add(sb, r2.getFalsePositives(), index++);
        add(sb, r2.getFalseNegatives(), index++);
        add(sb, r2.getPrecision(), index++);
        add(sb, r2.getRecall(), index++);
        add(sb, r2.getF1Score(), index++);
        add(sb, r2.getJaccard(), index++);
    } else {
        index += 7;
    }
    addCount(sb, result.getTruePositives(), index++);
    addCount(sb, result.getFalsePositives(), index++);
    addCount(sb, result.getFalseNegatives(), index++);
    add(sb, result.getPrecision(), index++);
    add(sb, result.getRecall(), index++);
    add(sb, result.getF1Score(), index++);
    add(sb, result.getJaccard(), index);
    return sb;
}