use of gdsc.smlm.ij.plugins.BenchmarkSpotFit.FilterCandidates in project GDSC-SMLM by aherbert.
the class BenchmarkFilterAnalysis method readResults.
private MultiPathFitResults[] readResults() {
// XXX set to true when debugging
boolean update = resultsList == null;
if (lastId != BenchmarkSpotFit.fitResultsId) {
if (lastId == 0) {
// Copy the settings from the fitter if this is the first run
failCount = BenchmarkSpotFit.config.getFailuresLimit();
duplicateDistance = BenchmarkSpotFit.fitConfig.getDuplicateDistance();
sResidualsThreshold = (BenchmarkSpotFit.computeDoublets) ? BenchmarkSpotFit.multiFilter.residualsThreshold : 1;
}
lastId = BenchmarkSpotFit.fitResultsId;
update = true;
actualCoordinates = getCoordinates(results.getResults());
}
Settings settings = new Settings(partialMatchDistance, upperMatchDistance, partialSignalFactor, upperSignalFactor);
boolean equalScoreSettings = settings.equals(lastReadResultsSettings);
if (update || !equalScoreSettings || lastDuplicateDistance != duplicateDistance) {
IJ.showStatus("Reading results ...");
// This functionality is for choosing the optimum filter for the given scoring metric.
if (!equalScoreSettings)
scores.clear();
lastReadResultsSettings = settings;
lastDuplicateDistance = duplicateDistance;
depthStats = null;
depthFitStats = null;
signalFactorStats = null;
distanceStats = null;
matches = 0;
fittedResults = 0;
totalResults = 0;
notDuplicateCount = 0;
newResultCount = 0;
maxUniqueId = 0;
nActual = 0;
// -=-=-=-
// The scoring is designed to find the best fitter+filter combination for the given spot candidates.
// The ideal combination would correctly fit+pick all the candidate positions that are close to a
// localisation.
//
// Use the following scoring scheme for all candidates:
//
// Candidates
// +----------------------------------------+
// | Actual matches |
// | +-----------+ TN |
// | | FN | |
// | | +---------- |
// | | | TP | | Fitted |
// | +-----------+ | spots |
// | | FP | |
// | +---------+ |
// +----------------------------------------+
//
// Candidates = All the spot candidates
// Actual matches = Any spot candidate or fitted spot candidate that matches a localisation
// Fitted spots = Any spot candidate that was successfully fitted
//
// TP = A spot candidate that was fitted and matches a localisation and is accepted
// FP = A spot candidate that was fitted but does not match a localisation and is accepted
// FN = A spot candidate that failed to be fitted but matches a localisation
// = A spot candidate that was fitted and matches a localisation and is rejected
// TN = A spot candidate that failed to be fitted and does not match a localisation
// = A spot candidate that was fitted and does not match a localisation and is rejected
//
// When fitting only produces one result it is possible to compute the TN score.
// Since unfitted candidates can only be TN or FN we could accumulate these scores and cache them.
// This was the old method of benchmarking single spot fitting and allowed more scores to be
// computed.
//
// When fitting produces multiple results then we have to score each fit result against all possible
// actual results and keep a record of the scores. These can then be assessed when the specific
// results have been chosen by result filtering.
//
// Using a distance ramped scoring function the degree of match can be varied from 0 to 1.
// Using a signal-factor ramped scoring function the degree of fitted can be varied from 0 to 1.
// When using ramped scoring functions the fractional allocation of scores using the above scheme
// is performed, i.e. candidates are treated as if they both match and unmatch. This results in
// an equivalent to multiple analysis using different thresholds and averaging of the scores.
//
// The totals TP+FP+TN+FN must equal the number of spot candidates. This allows different fitting
// methods to be compared since the total number of candidates is the same.
//
// Precision = TP / (TP+FP) : This is always valid as a minimum criteria score
// Recall = TP / (TP+FN) : This is valid between different fitting methods since a method that
// fits more spots will have a potentially lower FN
// Jaccard = TP / (TP+FN+FP) : This is valid between fitting methods
//
// -=-=-=-
// As an alternative scoring system, different fitting methods can be compared using the same TP
// value but calculating FN = localisations - TP and FP as Positives - TP. This creates a score
// against the original number of simulated molecules using everything that was passed through the
// filter (Positives). This score is comparable when a different spot candidate filter has been used
// and the total number of candidates is different, e.g. Mean filtering vs. Gaussian filtering
// -=-=-=-
final RampedScore distanceScore = new RampedScore(BenchmarkSpotFit.distanceInPixels * partialMatchDistance / 100.0, BenchmarkSpotFit.distanceInPixels * upperMatchDistance / 100.0);
lowerDistanceInPixels = distanceScore.lower;
distanceInPixels = distanceScore.upper;
final double matchDistance = distanceInPixels * distanceInPixels;
resultsPrefix3 = "\t" + Utils.rounded(distanceScore.lower * simulationParameters.a) + "\t" + Utils.rounded(distanceScore.upper * simulationParameters.a);
limitRange = ", d=" + Utils.rounded(distanceScore.lower * simulationParameters.a) + "-" + Utils.rounded(distanceScore.upper * simulationParameters.a);
// Signal factor must be greater than 1
final RampedScore signalScore;
if (BenchmarkSpotFit.signalFactor > 0 && upperSignalFactor > 0) {
signalScore = new RampedScore(BenchmarkSpotFit.signalFactor * partialSignalFactor / 100.0, BenchmarkSpotFit.signalFactor * upperSignalFactor / 100.0);
lowerSignalFactor = signalScore.lower;
signalFactor = signalScore.upper;
resultsPrefix3 += "\t" + Utils.rounded(signalScore.lower) + "\t" + Utils.rounded(signalScore.upper);
limitRange += ", s=" + Utils.rounded(signalScore.lower) + "-" + Utils.rounded(signalScore.upper);
} else {
signalScore = null;
resultsPrefix3 += "\t0\t0";
lowerSignalFactor = signalFactor = 0;
}
// Store all the results
final ArrayList<MultiPathFitResults> results = new ArrayList<MultiPathFitResults>(BenchmarkSpotFit.fitResults.size());
final List<MultiPathFitResults> syncResults = Collections.synchronizedList(results);
// This could be multi-threaded ...
final int nThreads = getThreads(BenchmarkSpotFit.fitResults.size());
final BlockingQueue<Job> jobs = new ArrayBlockingQueue<Job>(nThreads * 2);
final List<FitResultsWorker> workers = new LinkedList<FitResultsWorker>();
final List<Thread> threads = new LinkedList<Thread>();
final AtomicInteger uniqueId = new AtomicInteger();
CoordinateStore coordinateStore = createCoordinateStore();
for (int i = 0; i < nThreads; i++) {
final FitResultsWorker worker = new FitResultsWorker(jobs, syncResults, matchDistance, distanceScore, signalScore, uniqueId, coordinateStore.newInstance());
final Thread t = new Thread(worker);
workers.add(worker);
threads.add(t);
t.start();
}
totalProgress = BenchmarkSpotFit.fitResults.size();
stepProgress = Utils.getProgressInterval(totalProgress);
progress = 0;
BenchmarkSpotFit.fitResults.forEachEntry(new TIntObjectProcedure<FilterCandidates>() {
public boolean execute(int a, FilterCandidates b) {
put(jobs, new Job(a, b));
return true;
}
});
// Finish all the worker threads by passing in a null job
for (int i = 0; i < threads.size(); i++) {
put(jobs, new Job(0, null));
}
// Wait for all to finish
for (int i = 0; i < threads.size(); i++) {
try {
threads.get(i).join();
FitResultsWorker worker = workers.get(i);
matches += worker.matches;
fittedResults += worker.included;
totalResults += worker.total;
notDuplicateCount += worker.notDuplicateCount;
newResultCount += worker.newResultCount;
nActual += worker.includedActual;
if (i == 0) {
depthStats = worker.depthStats;
depthFitStats = worker.depthFitStats;
signalFactorStats = worker.signalFactorStats;
distanceStats = worker.distanceStats;
} else {
depthStats.add(worker.depthStats);
depthFitStats.add(worker.depthFitStats);
signalFactorStats.add(worker.signalFactorStats);
distanceStats.add(worker.distanceStats);
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
threads.clear();
IJ.showProgress(1);
IJ.showStatus("");
maxUniqueId = uniqueId.get();
resultsList = results.toArray(new MultiPathFitResults[results.size()]);
Arrays.sort(resultsList, new Comparator<MultiPathFitResults>() {
public int compare(MultiPathFitResults o1, MultiPathFitResults o2) {
return o1.frame - o2.frame;
}
});
}
// In case a previous run was interrupted
if (resultsList != null) {
MultiPathFilter.resetValidationFlag(resultsList);
}
return resultsList;
}
Aggregations