Examples with FractionClassificationResult uk.ac.sussex.gdsc.core.match.FractionClassificationResult used on opensource projects

Example 11 with FractionClassificationResult

use of uk.ac.sussex.gdsc.core.match.FractionClassificationResult in project GDSC-SMLM by aherbert.

the class BenchmarkSmartSpotRanking method summariseResults.

private void summariseResults(TIntObjectHashMap<RankResults> rankResults, CandidateData candidateData) {
    // Summarise the ranking results.
    final StringBuilder sb = new StringBuilder(filterResult.resultPrefix);
    // countPositive and countNegative is the fractional score of the spot candidates
    addCount(sb, (double) candidateData.countPositive + candidateData.countNegative);
    addCount(sb, candidateData.countPositive);
    addCount(sb, candidateData.countNegative);
    addCount(sb, candidateData.fractionPositive);
    addCount(sb, candidateData.fractionNegative);
    final double[] counter1 = new double[2];
    final int[] counter2 = new int[2];
    candidateData.filterCandidates.forEachValue(result -> {
        counter1[0] += result.np;
        counter1[1] += result.nn;
        counter2[0] += result.pos;
        counter2[1] += result.neg;
        return true;
    });
    final int countTp = counter2[0];
    final int countFp = counter2[1];
    // The fraction of positive and negative candidates that were included
    add(sb, (100.0 * countTp) / candidateData.countPositive);
    add(sb, (100.0 * countFp) / candidateData.countNegative);
    // Add counts of the the candidates
    add(sb, countTp + countFp);
    add(sb, countTp);
    add(sb, countFp);
    // Add fractional counts of the the candidates
    double tp = counter1[0];
    double fp = counter1[1];
    add(sb, tp + fp);
    add(sb, tp);
    add(sb, fp);
    // Materialise rankResults
    final int[] frames = new int[rankResults.size()];
    final RankResults[] rankResultsArray = new RankResults[rankResults.size()];
    final int[] counter = new int[1];
    rankResults.forEachEntry((frame, result) -> {
        frames[counter[0]] = frame;
        rankResultsArray[counter[0]] = result;
        counter[0]++;
        return true;
    });
    // Summarise actual and candidate spots per frame
    final Statistics actual = new Statistics();
    final Statistics candidates = new Statistics();
    for (final RankResults rr : rankResultsArray) {
        actual.add(rr.zPosition.length);
        candidates.add(rr.spots.length);
    }
    add(sb, actual.getMean());
    add(sb, actual.getStandardDeviation());
    add(sb, candidates.getMean());
    add(sb, candidates.getStandardDeviation());
    final String resultPrefix = sb.toString();
    // ---
    // TODO: Further explore pre-filtering of spot candidates.
    // Add good label to spot candidates and have the benchmark spot filter respect this before
    // applying the fail count limit.
    // Correlation between intensity and SNR ...
    // SNR is very good at low density
    // SNR fails at high density. The SNR estimate is probably wrong for high intensity spots.
    // Triangle is very good when there are a large number of good spots in a region of the image
    // (e.g. a mask is used).
    // Triangle is poor when there are few good spots in an image.
    // Perhaps we can estimate the density of the spots and choose the correct thresholding method?
    // ---
    // Do a full benchmark through different Spot SNR, image sizes, densities and mask structures
    // and see if there are patterns for a good threshold method.
    // ---
    // Allow using the fitted results from benchmark spot fit. Will it make a difference if we fit
    // the candidates (some will fail if weak).
    // Can this be done by allowing the user to select the input (spot candidates or fitted
    // positions)?
    // Perhaps I need to produce a precision estimate for all simulated spots and then only use
    // those that achieve a certain precision, i.e. are reasonably in focus. Can this be done?
    // Does the image PSF have a width estimate for the entire stack?
    // Perhaps I should filter, fit and then filter all spots using no fail count. These then become
    // the spots to work with for creating a smart fail count filter.
    // ---
    // Pre-compute the results and have optional sort
    final ArrayList<ScoredResult> list = new ArrayList<>(methodNames.length);
    for (int i = 0; i < methodNames.length; i++) {
        tp = 0;
        fp = 0;
        double tn = 0;
        int itp = 0;
        int ifp = 0;
        int itn = 0;
        final Statistics s = new Statistics();
        long time = 0;
        for (final RankResults rr : rankResultsArray) {
            final RankResult r = rr.results.get(i);
            // Some results will not have a threshold
            if (!Float.isInfinite(r.threshold)) {
                s.add(r.threshold);
            }
            time += r.time;
            tp += r.fresult.getTruePositives();
            fp += r.fresult.getFalsePositives();
            tn += r.fresult.getTrueNegatives();
            itp += r.cresult.getTruePositives();
            ifp += r.cresult.getFalsePositives();
            itn += r.cresult.getTrueNegatives();
        }
        sb.setLength(0);
        sb.append(resultPrefix);
        add(sb, methodNames[i]);
        if (methodNames[i].startsWith("SNR")) {
            sb.append('\t');
        } else {
            add(sb, settings.compactBins);
        }
        add(sb, s.getMean());
        add(sb, s.getStandardDeviation());
        add(sb, TextUtils.nanosToString(time));
        // TP are all accepted candidates that can be matched to a spot
        // FP are all accepted candidates that cannot be matched to a spot
        // TN are all accepted candidates that cannot be matched to a spot
        // FN = The number of missed spots
        // Raw counts of match or no-match
        final FractionClassificationResult f1 = new FractionClassificationResult(itp, ifp, itn, simulationParameters.molecules - itp);
        final double s1 = addScores(sb, f1);
        // Fractional scoring
        final FractionClassificationResult f2 = new FractionClassificationResult(tp, fp, tn, simulationParameters.molecules - tp);
        final double s2 = addScores(sb, f2);
        // Store for sorting
        list.add(new ScoredResult(i, (settings.useFractionScores) ? s2 : s1, sb.toString()));
    }
    if (list.isEmpty()) {
        return;
    }
    Collections.sort(list, ScoredResult::compare);
    final TextWindow summaryTable = createTable();
    if (summaryTable.getTextPanel().getLineCount() > 0) {
        summaryTable.append("");
    }
    try (BufferedTextWindow tw = new BufferedTextWindow(summaryTable)) {
        tw.setIncrement(0);
        for (final ScoredResult r : list) {
            tw.append(r.result);
        }
    }
    if (settings.showOverlay) {
        final int bestMethod = list.get(0).index;
        final Overlay o = new Overlay();
        for (int j = 0; j < rankResultsArray.length; j++) {
            final int frame = frames[j];
            final RankResults rr = rankResultsArray[j];
            final RankResult r = rr.results.get(bestMethod);
            final int[] x1 = new int[r.good.length];
            final int[] y1 = new int[r.good.length];
            int c1 = 0;
            final int[] x2 = new int[r.good.length];
            final int[] y2 = new int[r.good.length];
            int c2 = 0;
            final int[] x3 = new int[r.good.length];
            final int[] y3 = new int[r.good.length];
            int c3 = 0;
            final int[] x4 = new int[r.good.length];
            final int[] y4 = new int[r.good.length];
            int c4 = 0;
            for (int i = 0; i < x1.length; i++) {
                if (r.good[i] == TP) {
                    x1[c1] = rr.spots[i].spot.x;
                    y1[c1] = rr.spots[i].spot.y;
                    c1++;
                } else if (r.good[i] == FP) {
                    x2[c2] = rr.spots[i].spot.x;
                    y2[c2] = rr.spots[i].spot.y;
                    c2++;
                } else if (r.good[i] == TN) {
                    x3[c3] = rr.spots[i].spot.x;
                    y3[c3] = rr.spots[i].spot.y;
                    c3++;
                } else if (r.good[i] == FN) {
                    x4[c4] = rr.spots[i].spot.x;
                    y4[c4] = rr.spots[i].spot.y;
                    c4++;
                }
            }
            addToOverlay(o, frame, x1, y1, c1, Color.green);
            addToOverlay(o, frame, x2, y2, c2, Color.red);
            if (IJ.debugMode) {
                // light green
                addToOverlay(o, frame, x3, y3, c3, new Color(153, 255, 153));
            }
            // light red
            addToOverlay(o, frame, x4, y4, c4, new Color(255, 153, 153));
        }
        imp.setOverlay(o);
    }
}

Example 12 with FractionClassificationResult

use of uk.ac.sussex.gdsc.core.match.FractionClassificationResult in project GDSC-SMLM by aherbert.

the class BenchmarkFilterAnalysis method shutdown.

@Override
public void shutdown() {
    // Report the score for the best filter
    final List<? extends Chromosome<FilterScore>> individuals = gaPopulation.getIndividuals();
    FilterScore max = null;
    for (final Chromosome<FilterScore> c : individuals) {
        final FilterScore f = c.getFitness();
        if (f != null && f.compareTo(max) < 0) {
            max = f;
        }
    }
    if (max == null) {
        return;
    }
    final DirectFilter filter = (DirectFilter) ((SimpleFilterScore) max).filter;
    // This filter may not have been part of the scored subset so use the entire results set for
    // reporting
    final FractionClassificationResult r = scoreFilter(filter, defaultMinimalFilter, gaResultsList, coordinateStore);
    final StringBuilder text = createResult(filter, r);
    add(text, gaIteration);
    gaWindow.accept(text.toString());
}

Example 13 with FractionClassificationResult

use of uk.ac.sussex.gdsc.core.match.FractionClassificationResult in project GDSC-SMLM by aherbert.

the class BenchmarkFilterAnalysis method scoreFilter.

private FilterScoreResult scoreFilter(DirectFilter filter) {
    final FractionClassificationResult r = scoreFilter(filter, defaultMinimalFilter, fitResultData.resultsList, coordinateStore);
    final double score = getScore(r);
    final double criteria = getCriteria(r);
    // Create the score output
    final String text = createResult(filter, r).toString();
    return new FilterScoreResult(score, criteria, filter, text);
}

Example 14 with FractionClassificationResult

use of uk.ac.sussex.gdsc.core.match.FractionClassificationResult in project GDSC-SMLM by aherbert.

the class BenchmarkFilterAnalysis method findOptimum.

@Nullable
@Override
public SearchResult<FilterScore> findOptimum(double[][] points) {
    gaIteration++;
    SimpleFilterScore max = filterScoreOptimum;
    final FilterScoreResult[] scoreResults = scoreFilters(setStrength(new FilterSet(searchSpaceToFilters(points))), false);
    if (scoreResults == null) {
        return null;
    }
    for (int index = 0; index < scoreResults.length; index++) {
        final FilterScoreResult scoreResult = scoreResults[index];
        final SimpleFilterScore result = new SimpleFilterScore(scoreResult, true, scoreResult.criteria >= minCriteria);
        if (result.compareTo(max) < 0) {
            max = result;
        }
    }
    filterScoreOptimum = max;
    // Add the best filter to the table
    // This filter may not have been part of the scored subset so use the entire results set for
    // reporting
    final DirectFilter filter = max.result.filter;
    final FractionClassificationResult r = scoreFilter(filter, defaultMinimalFilter, gaResultsList, coordinateStore);
    final StringBuilder text = createResult(filter, r);
    add(text, gaIteration);
    gaWindow.accept(text.toString());
    return new SearchResult<>(filter.getParameters(), max);
}

Example 15 with FractionClassificationResult

use of uk.ac.sussex.gdsc.core.match.FractionClassificationResult in project GDSC-SMLM by aherbert.

the class Filter method fractionScore2.

/**
 * Filter the results and return the performance score. Allows benchmarking the filter by marking
 * the results as true or false.
 *
 * <p>Input PeakResults must be allocated a score for true positive, false positive, true negative
 * and false negative (accessed via the object property get methods). The filter is run and
 * results that pass accumulate scores for true positive and false positive, otherwise the scores
 * are accumulated for true negative and false negative. The simplest scoring scheme is to mark
 * valid results as tp=fn=1 and fp=tn=0 and invalid results the opposite.
 *
 * <p>The number of consecutive rejections are counted per frame. When the configured number of
 * failures is reached all remaining results for the frame are rejected. This assumes the results
 * are ordered by the frame.
 *
 * <p>Note that this method is to be used to score a set of results that may have been extracted
 * from a larger set since the number of consecutive failures before each peak are expected to be
 * stored in the origY property. Set this to zero and the results should be identical to
 * {@link #fractionScore(List, int)}.
 *
 * @param resultsList a list of results to analyse
 * @param failures the number of failures to allow per frame before all peaks are rejected
 * @return the score
 */
public FractionClassificationResult fractionScore2(List<MemoryPeakResults> resultsList, final int failures) {
    final double[] s = new double[4];
    final Counter p = new Counter();
    int negatives = 0;
    for (final MemoryPeakResults peakResults : resultsList) {
        setup(peakResults);
        final FrameCounter counter = new FrameCounter();
        peakResults.forEach((PeakResultProcedure) peak -> {
            counter.advanceAndReset(peak.getFrame());
            counter.increment(peak.getOrigY());
            final boolean isPositive;
            if (counter.getCount() > failures) {
                isPositive = false;
            } else {
                isPositive = accept(peak);
            }
            if (isPositive) {
                counter.reset();
            } else {
                counter.increment();
            }
            if (isPositive) {
                p.increment();
                s[TP] += peak.getTruePositiveScore();
                s[FP] += peak.getFalsePositiveScore();
            } else {
                s[FN] += peak.getFalseNegativeScore();
                s[TN] += peak.getTrueNegativeScore();
            }
        });
        negatives += peakResults.size();
        end();
    }
    negatives -= p.getCount();
    return new FractionClassificationResult(s[TP], s[FP], s[TN], s[FN], p.getCount(), negatives);
}