use of gdsc.core.utils.Settings in project GDSC-SMLM by aherbert.
the class BenchmarkFilterAnalysis method readResults.
private MultiPathFitResults[] readResults() {
// XXX set to true when debugging
boolean update = resultsList == null;
if (lastId != BenchmarkSpotFit.fitResultsId) {
if (lastId == 0) {
// Copy the settings from the fitter if this is the first run
failCount = BenchmarkSpotFit.config.getFailuresLimit();
duplicateDistance = BenchmarkSpotFit.fitConfig.getDuplicateDistance();
sResidualsThreshold = (BenchmarkSpotFit.computeDoublets) ? BenchmarkSpotFit.multiFilter.residualsThreshold : 1;
}
lastId = BenchmarkSpotFit.fitResultsId;
update = true;
actualCoordinates = getCoordinates(results.getResults());
}
Settings settings = new Settings(partialMatchDistance, upperMatchDistance, partialSignalFactor, upperSignalFactor);
boolean equalScoreSettings = settings.equals(lastReadResultsSettings);
if (update || !equalScoreSettings || lastDuplicateDistance != duplicateDistance) {
IJ.showStatus("Reading results ...");
// This functionality is for choosing the optimum filter for the given scoring metric.
if (!equalScoreSettings)
scores.clear();
lastReadResultsSettings = settings;
lastDuplicateDistance = duplicateDistance;
depthStats = null;
depthFitStats = null;
signalFactorStats = null;
distanceStats = null;
matches = 0;
fittedResults = 0;
totalResults = 0;
notDuplicateCount = 0;
newResultCount = 0;
maxUniqueId = 0;
nActual = 0;
// -=-=-=-
// The scoring is designed to find the best fitter+filter combination for the given spot candidates.
// The ideal combination would correctly fit+pick all the candidate positions that are close to a
// localisation.
//
// Use the following scoring scheme for all candidates:
//
// Candidates
// +----------------------------------------+
// | Actual matches |
// | +-----------+ TN |
// | | FN | |
// | | +---------- |
// | | | TP | | Fitted |
// | +-----------+ | spots |
// | | FP | |
// | +---------+ |
// +----------------------------------------+
//
// Candidates = All the spot candidates
// Actual matches = Any spot candidate or fitted spot candidate that matches a localisation
// Fitted spots = Any spot candidate that was successfully fitted
//
// TP = A spot candidate that was fitted and matches a localisation and is accepted
// FP = A spot candidate that was fitted but does not match a localisation and is accepted
// FN = A spot candidate that failed to be fitted but matches a localisation
// = A spot candidate that was fitted and matches a localisation and is rejected
// TN = A spot candidate that failed to be fitted and does not match a localisation
// = A spot candidate that was fitted and does not match a localisation and is rejected
//
// When fitting only produces one result it is possible to compute the TN score.
// Since unfitted candidates can only be TN or FN we could accumulate these scores and cache them.
// This was the old method of benchmarking single spot fitting and allowed more scores to be
// computed.
//
// When fitting produces multiple results then we have to score each fit result against all possible
// actual results and keep a record of the scores. These can then be assessed when the specific
// results have been chosen by result filtering.
//
// Using a distance ramped scoring function the degree of match can be varied from 0 to 1.
// Using a signal-factor ramped scoring function the degree of fitted can be varied from 0 to 1.
// When using ramped scoring functions the fractional allocation of scores using the above scheme
// is performed, i.e. candidates are treated as if they both match and unmatch. This results in
// an equivalent to multiple analysis using different thresholds and averaging of the scores.
//
// The totals TP+FP+TN+FN must equal the number of spot candidates. This allows different fitting
// methods to be compared since the total number of candidates is the same.
//
// Precision = TP / (TP+FP) : This is always valid as a minimum criteria score
// Recall = TP / (TP+FN) : This is valid between different fitting methods since a method that
// fits more spots will have a potentially lower FN
// Jaccard = TP / (TP+FN+FP) : This is valid between fitting methods
//
// -=-=-=-
// As an alternative scoring system, different fitting methods can be compared using the same TP
// value but calculating FN = localisations - TP and FP as Positives - TP. This creates a score
// against the original number of simulated molecules using everything that was passed through the
// filter (Positives). This score is comparable when a different spot candidate filter has been used
// and the total number of candidates is different, e.g. Mean filtering vs. Gaussian filtering
// -=-=-=-
final RampedScore distanceScore = new RampedScore(BenchmarkSpotFit.distanceInPixels * partialMatchDistance / 100.0, BenchmarkSpotFit.distanceInPixels * upperMatchDistance / 100.0);
lowerDistanceInPixels = distanceScore.lower;
distanceInPixels = distanceScore.upper;
final double matchDistance = distanceInPixels * distanceInPixels;
resultsPrefix3 = "\t" + Utils.rounded(distanceScore.lower * simulationParameters.a) + "\t" + Utils.rounded(distanceScore.upper * simulationParameters.a);
limitRange = ", d=" + Utils.rounded(distanceScore.lower * simulationParameters.a) + "-" + Utils.rounded(distanceScore.upper * simulationParameters.a);
// Signal factor must be greater than 1
final RampedScore signalScore;
if (BenchmarkSpotFit.signalFactor > 0 && upperSignalFactor > 0) {
signalScore = new RampedScore(BenchmarkSpotFit.signalFactor * partialSignalFactor / 100.0, BenchmarkSpotFit.signalFactor * upperSignalFactor / 100.0);
lowerSignalFactor = signalScore.lower;
signalFactor = signalScore.upper;
resultsPrefix3 += "\t" + Utils.rounded(signalScore.lower) + "\t" + Utils.rounded(signalScore.upper);
limitRange += ", s=" + Utils.rounded(signalScore.lower) + "-" + Utils.rounded(signalScore.upper);
} else {
signalScore = null;
resultsPrefix3 += "\t0\t0";
lowerSignalFactor = signalFactor = 0;
}
// Store all the results
final ArrayList<MultiPathFitResults> results = new ArrayList<MultiPathFitResults>(BenchmarkSpotFit.fitResults.size());
final List<MultiPathFitResults> syncResults = Collections.synchronizedList(results);
// This could be multi-threaded ...
final int nThreads = getThreads(BenchmarkSpotFit.fitResults.size());
final BlockingQueue<Job> jobs = new ArrayBlockingQueue<Job>(nThreads * 2);
final List<FitResultsWorker> workers = new LinkedList<FitResultsWorker>();
final List<Thread> threads = new LinkedList<Thread>();
final AtomicInteger uniqueId = new AtomicInteger();
CoordinateStore coordinateStore = createCoordinateStore();
for (int i = 0; i < nThreads; i++) {
final FitResultsWorker worker = new FitResultsWorker(jobs, syncResults, matchDistance, distanceScore, signalScore, uniqueId, coordinateStore.newInstance());
final Thread t = new Thread(worker);
workers.add(worker);
threads.add(t);
t.start();
}
totalProgress = BenchmarkSpotFit.fitResults.size();
stepProgress = Utils.getProgressInterval(totalProgress);
progress = 0;
BenchmarkSpotFit.fitResults.forEachEntry(new TIntObjectProcedure<FilterCandidates>() {
public boolean execute(int a, FilterCandidates b) {
put(jobs, new Job(a, b));
return true;
}
});
// Finish all the worker threads by passing in a null job
for (int i = 0; i < threads.size(); i++) {
put(jobs, new Job(0, null));
}
// Wait for all to finish
for (int i = 0; i < threads.size(); i++) {
try {
threads.get(i).join();
FitResultsWorker worker = workers.get(i);
matches += worker.matches;
fittedResults += worker.included;
totalResults += worker.total;
notDuplicateCount += worker.notDuplicateCount;
newResultCount += worker.newResultCount;
nActual += worker.includedActual;
if (i == 0) {
depthStats = worker.depthStats;
depthFitStats = worker.depthFitStats;
signalFactorStats = worker.signalFactorStats;
distanceStats = worker.distanceStats;
} else {
depthStats.add(worker.depthStats);
depthFitStats.add(worker.depthFitStats);
signalFactorStats.add(worker.signalFactorStats);
distanceStats.add(worker.distanceStats);
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
threads.clear();
IJ.showProgress(1);
IJ.showStatus("");
maxUniqueId = uniqueId.get();
resultsList = results.toArray(new MultiPathFitResults[results.size()]);
Arrays.sort(resultsList, new Comparator<MultiPathFitResults>() {
public int compare(MultiPathFitResults o1, MultiPathFitResults o2) {
return o1.frame - o2.frame;
}
});
}
// In case a previous run was interrupted
if (resultsList != null) {
MultiPathFilter.resetValidationFlag(resultsList);
}
return resultsList;
}
use of gdsc.core.utils.Settings in project GDSC-SMLM by aherbert.
the class BenchmarkFilterAnalysis method analyseParameters.
/**
* Run the optimum filter on a set of labelled peak results using various parameter settings outputting performance
* statistics on the success of the filter to an ImageJ table.
*
* @param iterative
* the iterative
* @param optimum
* the optimum
* @param rangeReduction
* the range reduction
* @return the best filter
*/
private ComplexFilterScore analyseParameters(boolean iterative, ComplexFilterScore optimum, double rangeReduction) {
// Non-zero modes are used for the iterative optimisation which require new results
boolean newResults = iterative;
if (!iterative) {
// Interactive run, this may be the first run during iterative optimisation
//if (reset)
// scores.clear();
createResultsWindow();
// Only repeat analysis if necessary
double min = minResidualsThreshold;
double max = maxResidualsThreshold;
if (BenchmarkSpotFit.computeDoublets) {
min = max = 0;
}
Settings settings = new Settings(optimum, resultsList, failCount, minFailCount, maxFailCount, residualsThreshold, min, max, duplicateDistance, minDuplicateDistance, maxDuplicateDistance, summaryDepth, criteriaIndex, criteriaLimit, scoreIndex, searchParam);
if (repeatSearch || !settings.equals(lastAnalyseParametersSettings)) {
newResults = true;
lastAnalyseParametersSettings = settings;
}
}
if (newResults) {
optimum = runParameterAnalysis(iterative, optimum, rangeReduction);
if (optimum == null || Utils.isInterrupted())
return null;
}
return reportResults(newResults, optimum);
}
use of gdsc.core.utils.Settings in project GDSC-SMLM by aherbert.
the class BenchmarkFilterAnalysis method analyse.
/**
* Run different filtering methods on a set of labelled peak results outputting performance statistics on the
* success of the filter to an ImageJ table.
* <p>
* For each filter set a plot is shown of the score verses the filter value, thus filters should be provided in
* ascending numerical order otherwise they are sorted.
*
* @param filterSets
* the filter sets
* @param iterative
* the iterative
* @param optimum
* the optimum
* @param rangeReduction
* the range reduction
* @return the best filter
*/
private ComplexFilterScore analyse(List<FilterSet> filterSets, boolean iterative, ComplexFilterScore optimum, double rangeReduction) {
// Non-zero modes are used for the iterative optimisation which require new results
boolean newResults = iterative;
if (optimum != null) {
// Non-interactive re-run when iterating
scores.clear();
runAnalysis(filterSets, optimum, rangeReduction);
if (Utils.isInterrupted())
return null;
} else {
// Interactive run, this may be the first run during iterative optimisation
if (reset)
scores.clear();
createResultsWindow();
boolean debugSpeed = false;
// Only repeat analysis if necessary
double evolveSetting = evolve;
if (evolve == 1)
// The delta effects the step size for the Genetic Algorithm
evolveSetting *= delta;
Settings settings = new Settings(filterSets, resultsList, failCount, residualsThreshold, duplicateDistance, plotTopN, summaryDepth, criteriaIndex, criteriaLimit, scoreIndex, evolveSetting);
boolean equalSettings = settings.equals(lastAnalyseSettings);
if (debugSpeed || !equalSettings || (evolve != 0 && repeatEvolve)) {
newResults = true;
lastAnalyseSettings = settings;
runAnalysis(filterSets);
if (Utils.isInterrupted())
return null;
}
}
return reportResults(newResults);
}
use of gdsc.core.utils.Settings in project GDSC-SMLM by aherbert.
the class BenchmarkSpotFilter method showDialog.
private boolean showDialog() {
GenericDialog gd = new GenericDialog(TITLE);
gd.addHelp(About.HELP_URL);
StringBuilder sb = new StringBuilder();
sb.append("Finds spots in the benchmark image created by CreateData plugin.\n");
final double sa = getSa() / simulationParameters.a;
sb.append("PSF width = ").append(Utils.rounded(simulationParameters.s / simulationParameters.a)).append(" px (sa = ").append(Utils.rounded(sa)).append(" px). HWHM = ").append(Utils.rounded(sa * Gaussian2DFunction.SD_TO_HWHM_FACTOR)).append(" px\n");
sb.append("Simulation depth = ").append(Utils.rounded(simulationParameters.depth)).append(" nm");
if (simulationParameters.fixedDepth)
sb.append(" (fixed)");
sb.append("\n \nConfigure the spot filter:");
gd.addMessage(sb.toString());
if (batchMode) {
// Support enumeration of single spot filters
gd.addCheckbox("Mean", batchMean);
gd.addCheckbox("Gaussian", batchGaussian);
gd.addCheckbox("Circular", batchCircular);
gd.addCheckbox("Median", batchMedian);
gd.addSlider("Min_search_width", 1, 4, minSearch);
gd.addSlider("Max_search_width", 1, 4, maxSearch);
gd.addCheckbox("Filter_relative_distances (to HWHM)", filterRelativeDistances);
} else {
String[] filterTypes = SettingsManager.getNames((Object[]) DataFilterType.values());
gd.addChoice("Spot_filter_type", filterTypes, filterTypes[config.getDataFilterType().ordinal()]);
String[] filterNames = SettingsManager.getNames((Object[]) DataFilter.values());
gd.addChoice("Spot_filter", filterNames, filterNames[config.getDataFilter(0).ordinal()]);
gd.addCheckbox("Filter_relative_distances (to HWHM)", filterRelativeDistances);
gd.addSlider("Smoothing", 0, 2.5, config.getSmooth(0));
gd.addSlider("Search_width", 1, 4, search);
}
gd.addSlider("Border", 0, 5, border);
gd.addCheckbox("Hard_border", hardBorder);
gd.addMessage("Scoring options:");
gd.addCheckbox("Score_relative_distances (to HWHM)", scoreRelativeDistances);
gd.addSlider("Analysis_border", 0, 5, sAnalysisBorder);
gd.addChoice("Matching_method", MATCHING_METHOD, MATCHING_METHOD[matchingMethod]);
gd.addSlider("Match_distance", 0.5, 3.5, upperDistance);
gd.addSlider("Lower_distance", 0, 3.5, lowerDistance);
gd.addSlider("Signal_factor", 0, 3.5, upperSignalFactor);
gd.addSlider("Lower_factor", 0, 3.5, lowerSignalFactor);
gd.addSlider("Recall_fraction", 50, 100, recallFraction);
if (!batchMode) {
gd.addCheckbox("Show_plots", showPlot);
gd.addCheckbox("Plot_rank_by_intensity", rankByIntensity);
gd.addCheckbox("Show_failures_plots", showFailuresPlot);
gd.addCheckbox("Show_TP", showTP);
gd.addCheckbox("Show_FP", showFP);
gd.addCheckbox("Show_FN", showFN);
}
if (extraOptions)
gd.addCheckbox("Debug", sDebug);
gd.showDialog();
if (gd.wasCanceled())
return false;
fitConfig.setInitialPeakStdDev(Maths.round(sa));
if (batchMode) {
batchMean = gd.getNextBoolean();
batchGaussian = gd.getNextBoolean();
batchCircular = gd.getNextBoolean();
batchMedian = gd.getNextBoolean();
if (!(batchMean || batchGaussian || batchCircular || batchMedian))
return false;
minSearch = gd.getNextNumber();
maxSearch = gd.getNextNumber();
filterRelativeDistances = gd.getNextBoolean();
} else {
config.setDataFilterType(gd.getNextChoiceIndex());
config.setDataFilter(gd.getNextChoiceIndex(), Maths.round(Math.abs(gd.getNextNumber()), 0.001), 0);
filterRelativeDistances = gd.getNextBoolean();
search = gd.getNextNumber();
}
border = gd.getNextNumber();
hardBorder = gd.getNextBoolean();
scoreRelativeDistances = gd.getNextBoolean();
sAnalysisBorder = Math.abs(gd.getNextNumber());
matchingMethod = gd.getNextChoiceIndex();
upperDistance = Math.abs(gd.getNextNumber());
lowerDistance = Math.abs(gd.getNextNumber());
upperSignalFactor = Math.abs(gd.getNextNumber());
lowerSignalFactor = Math.abs(gd.getNextNumber());
recallFraction = Math.abs(gd.getNextNumber());
if (!batchMode) {
showPlot = gd.getNextBoolean();
rankByIntensity = gd.getNextBoolean();
showFailuresPlot = gd.getNextBoolean();
showTP = gd.getNextBoolean();
showFP = gd.getNextBoolean();
showFN = gd.getNextBoolean();
}
if (extraOptions)
debug = sDebug = gd.getNextBoolean();
if (gd.invalidNumber())
return false;
if (lowerDistance > upperDistance)
lowerDistance = upperDistance;
if (lowerSignalFactor > upperSignalFactor)
lowerSignalFactor = upperSignalFactor;
if (batchMode) {
// Clear the cached results if the setting changed
Settings settings = new Settings(simulationParameters.id, filterRelativeDistances, //search, maxSearch, // Ignore search distance for smart caching
border, scoreRelativeDistances, sAnalysisBorder, hardBorder, matchingMethod, upperDistance, lowerDistance, upperSignalFactor, lowerSignalFactor, recallFraction);
if (!settings.equals(batchSettings)) {
cachedBatchResults.clear();
}
batchSettings = settings;
// relative (if requested) so that the results are consistent with single-filter mode.
if (filterRelativeDistances) {
final double hwhmMax = config.getHWHMMax();
config.setBorder(Maths.round(border * hwhmMax, 0.001));
} else {
config.setBorder(Maths.round(border, 0.001));
}
} else {
config.setSearch(Maths.round(search, 0.001));
config.setBorder(Maths.round(border, 0.001));
// Single filter ...
// Allow more complicated filters to be configured
GlobalSettings settings = new GlobalSettings();
settings.setFitEngineConfiguration(config);
if (!PeakFit.configureDataFilter(settings, null, false))
return false;
}
int analysisBorder;
if (scoreRelativeDistances) {
// Convert distance to PSF standard deviation units
final double hwhmMax = config.getHWHMMax();
matchDistance = upperDistance * hwhmMax;
lowerMatchDistance = lowerDistance * hwhmMax;
analysisBorder = (int) (sAnalysisBorder * hwhmMax);
} else {
matchDistance = upperDistance;
lowerMatchDistance = lowerDistance;
analysisBorder = (int) (sAnalysisBorder);
}
if (analysisBorder > 0) {
lastAnalysisBorder = new Rectangle(analysisBorder, analysisBorder, imp.getWidth() - 2 * analysisBorder, imp.getHeight() - 2 * analysisBorder);
} else {
lastAnalysisBorder = new Rectangle(imp.getWidth(), imp.getHeight());
}
return true;
}
use of gdsc.core.utils.Settings in project GDSC-SMLM by aherbert.
the class BenchmarkSpotFit method run.
private void run() {
// Extract all the results in memory into a list per frame. This can be cached
boolean refresh = false;
if (lastId != simulationParameters.id) {
// Do not get integer coordinates
// The Coordinate objects will be PeakResultPoint objects that store the original PeakResult
// from the MemoryPeakResults
actualCoordinates = ResultsMatchCalculator.getCoordinates(results.getResults(), false);
lastId = simulationParameters.id;
refresh = true;
}
// Extract all the candidates into a list per frame. This can be cached if the settings have not changed
final int width = (config.isIncludeNeighbours()) ? config.getRelativeFitting() : 0;
final Settings settings = new Settings(BenchmarkSpotFilter.filterResult.id, fractionPositives, fractionNegativesAfterAllPositives, negativesAfterAllPositives, width);
if (refresh || !settings.equals(lastSettings)) {
filterCandidates = subsetFilterResults(BenchmarkSpotFilter.filterResult.filterResults, width);
lastSettings = settings;
lastFilterId = BenchmarkSpotFilter.filterResult.id;
}
stopWatch = StopWatch.createStarted();
final ImageStack stack = imp.getImageStack();
clearFitResults();
// Save results to memory
MemoryPeakResults peakResults = new MemoryPeakResults();
peakResults.copySettings(this.results);
peakResults.setName(TITLE);
MemoryPeakResults.addResults(peakResults);
// Create a pool of workers
final int nThreads = Prefs.getThreads();
BlockingQueue<Integer> jobs = new ArrayBlockingQueue<Integer>(nThreads * 2);
List<Worker> workers = new LinkedList<Worker>();
List<Thread> threads = new LinkedList<Thread>();
for (int i = 0; i < nThreads; i++) {
Worker worker = new Worker(jobs, stack, actualCoordinates, filterCandidates, peakResults);
Thread t = new Thread(worker);
workers.add(worker);
threads.add(t);
t.start();
}
// Fit the frames
long runTime = System.nanoTime();
totalProgress = stack.getSize();
stepProgress = Utils.getProgressInterval(totalProgress);
progress = 0;
for (int i = 1; i <= totalProgress; i++) {
put(jobs, i);
}
// Finish all the worker threads by passing in a null job
for (int i = 0; i < threads.size(); i++) {
put(jobs, -1);
}
// Wait for all to finish
for (int i = 0; i < threads.size(); i++) {
try {
threads.get(i).join();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
threads.clear();
IJ.showProgress(1);
runTime = System.nanoTime() - runTime;
if (Utils.isInterrupted()) {
return;
}
stopWatch.stop();
final String timeString = stopWatch.toString();
IJ.log("Spot fit time : " + timeString);
IJ.showStatus("Collecting results ...");
fitResultsId++;
fitResults = new TIntObjectHashMap<FilterCandidates>();
for (Worker w : workers) {
fitResults.putAll(w.results);
}
// Assign a unique ID to each result
int count = 0;
// Materialise into an array since we use it twice
FilterCandidates[] candidates = fitResults.values(new FilterCandidates[fitResults.size()]);
for (FilterCandidates result : candidates) {
for (int i = 0; i < result.fitResult.length; i++) {
final MultiPathFitResult fitResult = result.fitResult[i];
count += count(fitResult.getSingleFitResult());
count += count(fitResult.getMultiFitResult());
count += count(fitResult.getDoubletFitResult());
count += count(fitResult.getMultiDoubletFitResult());
}
}
PreprocessedPeakResult[] preprocessedPeakResults = new PreprocessedPeakResult[count];
count = 0;
for (FilterCandidates result : candidates) {
for (int i = 0; i < result.fitResult.length; i++) {
final MultiPathFitResult fitResult = result.fitResult[i];
count = store(fitResult.getSingleFitResult(), count, preprocessedPeakResults);
count = store(fitResult.getMultiFitResult(), count, preprocessedPeakResults);
count = store(fitResult.getDoubletFitResult(), count, preprocessedPeakResults);
count = store(fitResult.getMultiDoubletFitResult(), count, preprocessedPeakResults);
}
}
summariseResults(fitResults, runTime, preprocessedPeakResults, count);
IJ.showStatus("");
}
Aggregations