Examples with RampedScore gdsc.core.utils.RampedScore used on opensource projects

Example 1 with RampedScore

use of gdsc.core.utils.RampedScore in project GDSC-SMLM by aherbert.

the class BenchmarkFilterAnalysis method readResults.

private MultiPathFitResults[] readResults() {
    // XXX set to true when debugging
    boolean update = resultsList == null;
    if (lastId != BenchmarkSpotFit.fitResultsId) {
        if (lastId == 0) {
            // Copy the settings from the fitter if this is the first run
            failCount = BenchmarkSpotFit.config.getFailuresLimit();
            duplicateDistance = BenchmarkSpotFit.fitConfig.getDuplicateDistance();
            sResidualsThreshold = (BenchmarkSpotFit.computeDoublets) ? BenchmarkSpotFit.multiFilter.residualsThreshold : 1;
        }
        lastId = BenchmarkSpotFit.fitResultsId;
        update = true;
        actualCoordinates = getCoordinates(results.getResults());
    }
    Settings settings = new Settings(partialMatchDistance, upperMatchDistance, partialSignalFactor, upperSignalFactor);
    boolean equalScoreSettings = settings.equals(lastReadResultsSettings);
    if (update || !equalScoreSettings || lastDuplicateDistance != duplicateDistance) {
        IJ.showStatus("Reading results ...");
        // This functionality is for choosing the optimum filter for the given scoring metric.
        if (!equalScoreSettings)
            scores.clear();
        lastReadResultsSettings = settings;
        lastDuplicateDistance = duplicateDistance;
        depthStats = null;
        depthFitStats = null;
        signalFactorStats = null;
        distanceStats = null;
        matches = 0;
        fittedResults = 0;
        totalResults = 0;
        notDuplicateCount = 0;
        newResultCount = 0;
        maxUniqueId = 0;
        nActual = 0;
        // -=-=-=-
        // The scoring is designed to find the best fitter+filter combination for the given spot candidates.
        // The ideal combination would correctly fit+pick all the candidate positions that are close to a
        // localisation.
        //
        // Use the following scoring scheme for all candidates:
        // 
        //  Candidates
        // +----------------------------------------+  
        // |   Actual matches                       | 
        // |  +-----------+                TN       |
        // |  |  FN       |                         |
        // |  |      +----------                    |
        // |  |      | TP |    | Fitted             |
        // |  +-----------+    | spots              |
        // |         |     FP  |                    |
        // |         +---------+                    |
        // +----------------------------------------+
        //
        // Candidates     = All the spot candidates
        // Actual matches = Any spot candidate or fitted spot candidate that matches a localisation
        // Fitted spots   = Any spot candidate that was successfully fitted
        //
        // TP = A spot candidate that was fitted and matches a localisation and is accepted
        // FP = A spot candidate that was fitted but does not match a localisation and is accepted
        // FN = A spot candidate that failed to be fitted but matches a localisation
        //    = A spot candidate that was fitted and matches a localisation and is rejected
        // TN = A spot candidate that failed to be fitted and does not match a localisation
        //    = A spot candidate that was fitted and does not match a localisation and is rejected
        //
        // When fitting only produces one result it is possible to compute the TN score.
        // Since unfitted candidates can only be TN or FN we could accumulate these scores and cache them.
        // This was the old method of benchmarking single spot fitting and allowed more scores to be 
        // computed.
        //
        // When fitting produces multiple results then we have to score each fit result against all possible
        // actual results and keep a record of the scores. These can then be assessed when the specific 
        // results have been chosen by result filtering.
        //
        // Using a distance ramped scoring function the degree of match can be varied from 0 to 1.
        // Using a signal-factor ramped scoring function the degree of fitted can be varied from 0 to 1.
        // When using ramped scoring functions the fractional allocation of scores using the above scheme 
        // is performed, i.e. candidates are treated as if they both match and unmatch. This results in 
        // an equivalent to multiple analysis using different thresholds and averaging of the scores.
        //
        // The totals TP+FP+TN+FN must equal the number of spot candidates. This allows different fitting 
        // methods to be compared since the total number of candidates is the same.
        //
        // Precision = TP / (TP+FP)    : This is always valid as a minimum criteria score
        // Recall    = TP / (TP+FN)    : This is valid between different fitting methods since a method that 
        //                               fits more spots will have a potentially lower FN
        // Jaccard   = TP / (TP+FN+FP) : This is valid between fitting methods
        //
        // -=-=-=-
        // As an alternative scoring system, different fitting methods can be compared using the same TP 
        // value but calculating FN = localisations - TP and FP as Positives - TP. This creates a score 
        // against the original number of simulated molecules using everything that was passed through the 
        // filter (Positives). This score is comparable when a different spot candidate filter has been used 
        // and the total number of candidates is different, e.g. Mean filtering vs. Gaussian filtering
        // -=-=-=-
        final RampedScore distanceScore = new RampedScore(BenchmarkSpotFit.distanceInPixels * partialMatchDistance / 100.0, BenchmarkSpotFit.distanceInPixels * upperMatchDistance / 100.0);
        lowerDistanceInPixels = distanceScore.lower;
        distanceInPixels = distanceScore.upper;
        final double matchDistance = distanceInPixels * distanceInPixels;
        resultsPrefix3 = "\t" + Utils.rounded(distanceScore.lower * simulationParameters.a) + "\t" + Utils.rounded(distanceScore.upper * simulationParameters.a);
        limitRange = ", d=" + Utils.rounded(distanceScore.lower * simulationParameters.a) + "-" + Utils.rounded(distanceScore.upper * simulationParameters.a);
        // Signal factor must be greater than 1
        final RampedScore signalScore;
        if (BenchmarkSpotFit.signalFactor > 0 && upperSignalFactor > 0) {
            signalScore = new RampedScore(BenchmarkSpotFit.signalFactor * partialSignalFactor / 100.0, BenchmarkSpotFit.signalFactor * upperSignalFactor / 100.0);
            lowerSignalFactor = signalScore.lower;
            signalFactor = signalScore.upper;
            resultsPrefix3 += "\t" + Utils.rounded(signalScore.lower) + "\t" + Utils.rounded(signalScore.upper);
            limitRange += ", s=" + Utils.rounded(signalScore.lower) + "-" + Utils.rounded(signalScore.upper);
        } else {
            signalScore = null;
            resultsPrefix3 += "\t0\t0";
            lowerSignalFactor = signalFactor = 0;
        }
        // Store all the results
        final ArrayList<MultiPathFitResults> results = new ArrayList<MultiPathFitResults>(BenchmarkSpotFit.fitResults.size());
        final List<MultiPathFitResults> syncResults = Collections.synchronizedList(results);
        // This could be multi-threaded ...
        final int nThreads = getThreads(BenchmarkSpotFit.fitResults.size());
        final BlockingQueue<Job> jobs = new ArrayBlockingQueue<Job>(nThreads * 2);
        final List<FitResultsWorker> workers = new LinkedList<FitResultsWorker>();
        final List<Thread> threads = new LinkedList<Thread>();
        final AtomicInteger uniqueId = new AtomicInteger();
        CoordinateStore coordinateStore = createCoordinateStore();
        for (int i = 0; i < nThreads; i++) {
            final FitResultsWorker worker = new FitResultsWorker(jobs, syncResults, matchDistance, distanceScore, signalScore, uniqueId, coordinateStore.newInstance());
            final Thread t = new Thread(worker);
            workers.add(worker);
            threads.add(t);
            t.start();
        }
        totalProgress = BenchmarkSpotFit.fitResults.size();
        stepProgress = Utils.getProgressInterval(totalProgress);
        progress = 0;
        BenchmarkSpotFit.fitResults.forEachEntry(new TIntObjectProcedure<FilterCandidates>() {

            public boolean execute(int a, FilterCandidates b) {
                put(jobs, new Job(a, b));
                return true;
            }
        });
        // Finish all the worker threads by passing in a null job
        for (int i = 0; i < threads.size(); i++) {
            put(jobs, new Job(0, null));
        }
        // Wait for all to finish
        for (int i = 0; i < threads.size(); i++) {
            try {
                threads.get(i).join();
                FitResultsWorker worker = workers.get(i);
                matches += worker.matches;
                fittedResults += worker.included;
                totalResults += worker.total;
                notDuplicateCount += worker.notDuplicateCount;
                newResultCount += worker.newResultCount;
                nActual += worker.includedActual;
                if (i == 0) {
                    depthStats = worker.depthStats;
                    depthFitStats = worker.depthFitStats;
                    signalFactorStats = worker.signalFactorStats;
                    distanceStats = worker.distanceStats;
                } else {
                    depthStats.add(worker.depthStats);
                    depthFitStats.add(worker.depthFitStats);
                    signalFactorStats.add(worker.signalFactorStats);
                    distanceStats.add(worker.distanceStats);
                }
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
        threads.clear();
        IJ.showProgress(1);
        IJ.showStatus("");
        maxUniqueId = uniqueId.get();
        resultsList = results.toArray(new MultiPathFitResults[results.size()]);
        Arrays.sort(resultsList, new Comparator<MultiPathFitResults>() {

            public int compare(MultiPathFitResults o1, MultiPathFitResults o2) {
                return o1.frame - o2.frame;
            }
        });
    }
    // In case a previous run was interrupted
    if (resultsList != null) {
        MultiPathFilter.resetValidationFlag(resultsList);
    }
    return resultsList;
}