Example 1 with Well44497b
use of org.apache.commons.math3.random.Well44497b in project GDSC-SMLM by aherbert.
the class BenchmarkFilterAnalysis method filterAnalysis.
private int filterAnalysis(FilterSet filterSet, int setNumber, DirectFilter currentOptimum, double rangeReduction) {
// Check if the filters are the same so allowing optimisation
final boolean allSameType = filterSet.allSameType();
this.ga_resultsList = resultsList;
Chromosome<FilterScore> best = null;
String algorithm = "";
// All the search algorithms use search dimensions.
// Create search dimensions if needed (these are used for testing if the optimum is at the limit).
ss_filter = null;
ss_lower = null;
ss_upper = null;
FixedDimension[] originalDimensions = null;
boolean rangeInput = false;
boolean[] disabled = null;
double[][] seed = null;
boolean nonInteractive = false;
if (allSameType) {
// There should always be 1 filter
ss_filter = (DirectFilter) filterSet.getFilters().get(0);
int n = ss_filter.getNumberOfParameters();
// Option to configure a range
rangeInput = filterSet.getName().contains("Range");
double[] range = new double[n];
if (rangeInput && filterSet.size() == 4) {
originalDimensions = new FixedDimension[n];
// This is used as min/lower/upper/max
final Filter minF = ss_filter;
final Filter lowerF = filterSet.getFilters().get(1);
final Filter upperF = filterSet.getFilters().get(2);
final Filter maxF = filterSet.getFilters().get(3);
for (int i = 0; i < n; i++) {
double min = minF.getParameterValue(i);
double lower = lowerF.getParameterValue(i);
double upper = upperF.getParameterValue(i);
range[i] = upper - lower;
double max = maxF.getParameterValue(i);
double minIncrement = ss_filter.getParameterIncrement(i);
try {
originalDimensions[i] = new FixedDimension(min, max, minIncrement, lower, upper);
} catch (IllegalArgumentException e) {
Utils.log(TITLE + " : Unable to configure dimension [%d] %s: " + e.getMessage(), i, ss_filter.getParameterName(i));
originalDimensions = null;
rangeInput = false;
break;
}
}
}
if (rangeInput && (filterSet.size() == 3 || filterSet.size() == 2)) {
originalDimensions = new FixedDimension[n];
// This is used as lower/upper[/increment]
final Filter lowerF = ss_filter;
final Filter upperF = filterSet.getFilters().get(1);
for (int i = 0; i < n; i++) {
// Do not disable if the increment is not set. This is left to the user to decide.
// if (incF.getParameterValue(i) == incF.getDisabledParameterValue(i) ||
// Double.isInfinite(incF.getParameterValue(i)))
// {
// // Not enabled
// dimensions[i] = new SearchDimension(incF.getDisabledParameterValue(i));
// continue;
// }
double lower = lowerF.getParameterValue(i);
double upper = upperF.getParameterValue(i);
range[i] = upper - lower;
ParameterType type = ss_filter.getParameterType(i);
double min = BenchmarkSpotFit.getMin(type);
double max = BenchmarkSpotFit.getMax(type);
double minIncrement = ss_filter.getParameterIncrement(i);
try {
originalDimensions[i] = new FixedDimension(min, max, minIncrement, lower, upper);
} catch (IllegalArgumentException e) {
Utils.log(TITLE + " : Unable to configure dimension [%d] %s: " + e.getMessage(), i, ss_filter.getParameterName(i));
originalDimensions = null;
rangeInput = false;
break;
}
}
}
// Get the dimensions from the filters
if (originalDimensions == null) {
originalDimensions = new FixedDimension[n];
// Allow inputing a filter set (e.g. saved from previous optimisation)
// Find the limits in the current scores
final double[] lower = ss_filter.getParameters().clone();
final double[] upper = lower.clone();
// Allow the SearchSpace algorithms to be seeded with an initial population
// for the first evaluation of the optimum. This is done when the input filter
// set is not a range.
seed = new double[filterSet.size()][];
int c = 0;
for (Filter f : filterSet.getFilters()) {
final double[] point = f.getParameters();
seed[c++] = point;
for (int j = 0; j < lower.length; j++) {
if (lower[j] > point[j])
lower[j] = point[j];
if (upper[j] < point[j])
upper[j] = point[j];
}
}
// Min/max must be set using values from BenchmarkSpotFit.
for (int i = 0; i < n; i++) {
if (lower[i] == upper[i]) {
// Not enabled
originalDimensions[i] = new FixedDimension(lower[i]);
continue;
}
ParameterType type = ss_filter.getParameterType(i);
double min = BenchmarkSpotFit.getMin(type);
double max = BenchmarkSpotFit.getMax(type);
double minIncrement = ss_filter.getParameterIncrement(i);
if (min > lower[i])
min = lower[i];
if (max < upper[i])
max = upper[i];
try {
originalDimensions[i] = new FixedDimension(min, max, minIncrement, lower[i], upper[i]);
} catch (IllegalArgumentException e) {
Utils.log(TITLE + " : Unable to configure dimension [%d] %s: " + e.getMessage(), i, ss_filter.getParameterName(i));
originalDimensions = null;
break;
}
}
if (originalDimensions == null) {
// Failed to work out the dimensions. No optimisation will be possible.
// Sort so that the filters are in a nice order for reporting
filterSet.sort();
// This will not be used when the dimensions are null
seed = null;
}
}
if (originalDimensions != null) {
// Use the current optimum if we are doing a range optimisation
if (currentOptimum != null && rangeInput && currentOptimum.getType().equals(ss_filter.getType()) && evolve != 0) {
// Suppress dialogs and use the current settings
nonInteractive = true;
double[] p = currentOptimum.getParameters();
// Range search uses SearchDimension and we must centre on the optimum after creation.
for (int i = 0; i < originalDimensions.length; i++) {
double centre = p[i];
double r = 0;
if (originalDimensions[i].isActive()) {
// Set the range around the centre.
// This uses the range for each param when we read the filters.
r = range[i];
// Optionally reduce the width of the dimensions.
if (rangeReduction > 0 && rangeReduction < 1)
r *= rangeReduction;
}
double lower = centre - r * 0.5;
double upper = centre + r * 0.5;
originalDimensions[i] = originalDimensions[i].create(lower, upper);
}
}
// Store the dimensions so we can do an 'at limit' check
disabled = new boolean[originalDimensions.length];
ss_lower = new double[originalDimensions.length];
ss_upper = new double[originalDimensions.length];
for (int i = 0; i < disabled.length; i++) {
disabled[i] = !originalDimensions[i].isActive();
ss_lower[i] = originalDimensions[i].lower;
ss_upper[i] = originalDimensions[i].upper;
}
}
} else {
// Sort so that the filters are in a nice order for reporting
filterSet.sort();
}
analysisStopWatch = StopWatch.createStarted();
if (evolve == 1 && originalDimensions != null) {
// Collect parameters for the genetic algorithm
pauseFilterTimer();
// Remember the step size settings
double[] stepSize = stepSizeMap.get(setNumber);
if (stepSize == null || stepSize.length != ss_filter.length()) {
stepSize = ss_filter.mutationStepRange().clone();
for (int j = 0; j < stepSize.length; j++) stepSize[j] *= delta;
// See if the same number of parameters have been optimised in other algorithms
boolean[] enabled = searchRangeMap.get(setNumber);
if (enabled != null && enabled.length == stepSize.length) {
for (int j = 0; j < stepSize.length; j++) if (!enabled[j])
stepSize[j] *= -1;
}
}
GenericDialog gd = null;
int[] indices = ss_filter.getChromosomeParameters();
boolean runAlgorithm = nonInteractive;
if (!nonInteractive) {
// Ask the user for the mutation step parameters.
gd = new GenericDialog(TITLE);
String prefix = setNumber + "_";
gd.addMessage("Configure the genetic algorithm for [" + setNumber + "] " + filterSet.getName());
gd.addNumericField(prefix + "Population_size", populationSize, 0);
gd.addNumericField(prefix + "Failure_limit", failureLimit, 0);
gd.addNumericField(prefix + "Tolerance", tolerance, -1);
gd.addNumericField(prefix + "Converged_count", convergedCount, 0);
gd.addSlider(prefix + "Mutation_rate", 0.05, 1, mutationRate);
gd.addSlider(prefix + "Crossover_rate", 0.05, 1, crossoverRate);
gd.addSlider(prefix + "Mean_children", 0.05, 3, meanChildren);
gd.addSlider(prefix + "Selection_fraction", 0.05, 0.5, selectionFraction);
gd.addCheckbox(prefix + "Ramped_selection", rampedSelection);
gd.addCheckbox(prefix + "Save_option", saveOption);
gd.addMessage("Configure the step size for each parameter");
for (int j = 0; j < indices.length; j++) {
// Do not mutate parameters that were not expanded, i.e. the input did not vary them.
final double step = (originalDimensions[indices[j]].isActive()) ? stepSize[j] : 0;
gd.addNumericField(getDialogName(prefix, ss_filter, indices[j]), step, 2);
}
gd.showDialog();
runAlgorithm = !gd.wasCanceled();
}
if (runAlgorithm) {
// Used to create random sample
FixedDimension[] dimensions = Arrays.copyOf(originalDimensions, originalDimensions.length);
if (!nonInteractive) {
populationSize = (int) Math.abs(gd.getNextNumber());
if (populationSize < 10)
populationSize = 10;
failureLimit = (int) Math.abs(gd.getNextNumber());
tolerance = gd.getNextNumber();
// Allow negatives
convergedCount = (int) gd.getNextNumber();
mutationRate = Math.abs(gd.getNextNumber());
crossoverRate = Math.abs(gd.getNextNumber());
meanChildren = Math.abs(gd.getNextNumber());
selectionFraction = Math.abs(gd.getNextNumber());
rampedSelection = gd.getNextBoolean();
saveOption = gd.getNextBoolean();
for (int j = 0; j < indices.length; j++) {
stepSize[j] = gd.getNextNumber();
}
// Store for repeat analysis
stepSizeMap.put(setNumber, stepSize);
}
for (int j = 0; j < indices.length; j++) {
// A zero step size will keep the parameter but prevent range mutation.
if (stepSize[j] < 0) {
dimensions[indices[j]] = new FixedDimension(ss_filter.getDisabledParameterValue(indices[j]));
disabled[indices[j]] = true;
}
}
// // Reset negatives to zero
// stepSize = stepSize.clone();
// for (int j = 0; j < stepSize.length; j++)
// if (stepSize[j] < 0)
// stepSize[j] = 0;
// Create the genetic algorithm
RandomDataGenerator random = new RandomDataGenerator(new Well44497b());
SimpleMutator<FilterScore> mutator = new SimpleMutator<FilterScore>(random, mutationRate);
// Override the settings with the step length, a min of zero and the configured upper
double[] upper = ss_filter.upperLimit();
mutator.overrideChromosomeSettings(stepSize, new double[stepSize.length], upper);
Recombiner<FilterScore> recombiner = new SimpleRecombiner<FilterScore>(random, crossoverRate, meanChildren);
SelectionStrategy<FilterScore> selectionStrategy;
// If the initial population is huge ensure that the first selection culls to the correct size
final int selectionMax = (int) (selectionFraction * populationSize);
if (rampedSelection)
selectionStrategy = new RampedSelectionStrategy<FilterScore>(random, selectionFraction, selectionMax);
else
selectionStrategy = new SimpleSelectionStrategy<FilterScore>(random, selectionFraction, selectionMax);
ToleranceChecker<FilterScore> ga_checker = new InterruptChecker(tolerance, tolerance * 1e-3, convergedCount);
// Create new random filters if the population is initially below the population size
List<Filter> filters = filterSet.getFilters();
if (filterSet.size() < populationSize) {
filters = new ArrayList<Filter>(populationSize);
// Add the existing filters if they are not a range input file
if (!rangeInput)
filters.addAll(filterSet.getFilters());
// Add current optimum to seed
if (nonInteractive)
filters.add(currentOptimum);
// The GA does not use the min interval grid so sample without rounding
double[][] sample = SearchSpace.sampleWithoutRounding(dimensions, populationSize - filters.size(), null);
filters.addAll(searchSpaceToFilters(sample));
}
ga_population = new Population<FilterScore>(filters);
ga_population.setPopulationSize(populationSize);
ga_population.setFailureLimit(failureLimit);
selectionStrategy.setTracker(this);
// Evolve
algorithm = EVOLVE[evolve];
ga_statusPrefix = algorithm + " [" + setNumber + "] " + filterSet.getName() + " ... ";
ga_iteration = 0;
ga_population.setTracker(this);
createGAWindow();
resumeFilterTimer();
best = ga_population.evolve(mutator, recombiner, this, selectionStrategy, ga_checker);
if (best != null) {
// In case optimisation was stopped
IJ.resetEscape();
// The GA may produce coordinates off the min interval grid
best = enumerateMinInterval(best, stepSize, indices);
// Now update the filter set for final assessment
filterSet = new FilterSet(filterSet.getName(), populationToFilters(ga_population.getIndividuals()));
// Option to save the filters
if (saveOption)
saveFilterSet(filterSet, setNumber, !nonInteractive);
}
} else
resumeFilterTimer();
}
if ((evolve == 2 || evolve == 4) && originalDimensions != null) {
// Collect parameters for the range search algorithm
pauseFilterTimer();
boolean isStepSearch = evolve == 4;
// The step search should use a multi-dimension refinement and no range reduction
SearchSpace.RefinementMode myRefinementMode = SearchSpace.RefinementMode.MULTI_DIMENSION;
// Remember the enabled settings
boolean[] enabled = searchRangeMap.get(setNumber);
int n = ss_filter.getNumberOfParameters();
if (enabled == null || enabled.length != n) {
enabled = new boolean[n];
Arrays.fill(enabled, true);
// See if the same number of parameters have been optimised in other algorithms
double[] stepSize = stepSizeMap.get(setNumber);
if (stepSize != null && enabled.length == stepSize.length) {
for (int j = 0; j < stepSize.length; j++) if (stepSize[j] < 0)
enabled[j] = false;
}
}
GenericDialog gd = null;
boolean runAlgorithm = nonInteractive;
if (!nonInteractive) {
// Ask the user for the search parameters.
gd = new GenericDialog(TITLE);
String prefix = setNumber + "_";
gd.addMessage("Configure the " + EVOLVE[evolve] + " algorithm for [" + setNumber + "] " + filterSet.getName());
gd.addSlider(prefix + "Width", 1, 5, rangeSearchWidth);
if (!isStepSearch) {
gd.addCheckbox(prefix + "Save_option", saveOption);
gd.addNumericField(prefix + "Max_iterations", maxIterations, 0);
String[] modes = SettingsManager.getNames((Object[]) SearchSpace.RefinementMode.values());
gd.addSlider(prefix + "Reduce", 0.01, 0.99, rangeSearchReduce);
gd.addChoice("Refinement", modes, modes[refinementMode]);
}
gd.addNumericField(prefix + "Seed_size", seedSize, 0);
// Add choice of fields to optimise
for (int i = 0; i < n; i++) gd.addCheckbox(getDialogName(prefix, ss_filter, i), enabled[i]);
gd.showDialog();
runAlgorithm = !gd.wasCanceled();
}
if (runAlgorithm) {
SearchDimension[] dimensions = new SearchDimension[n];
if (!nonInteractive) {
rangeSearchWidth = (int) gd.getNextNumber();
if (!isStepSearch) {
saveOption = gd.getNextBoolean();
maxIterations = (int) gd.getNextNumber();
refinementMode = gd.getNextChoiceIndex();
rangeSearchReduce = gd.getNextNumber();
}
seedSize = (int) gd.getNextNumber();
for (int i = 0; i < n; i++) enabled[i] = gd.getNextBoolean();
searchRangeMap.put(setNumber, enabled);
}
if (!isStepSearch)
myRefinementMode = SearchSpace.RefinementMode.values()[refinementMode];
for (int i = 0; i < n; i++) {
if (enabled[i]) {
try {
dimensions[i] = originalDimensions[i].create(rangeSearchWidth);
dimensions[i].setPad(true);
// Prevent range reduction so that the step search just does a single refinement step
dimensions[i].setReduceFactor((isStepSearch) ? 1 : rangeSearchReduce);
// Centre on current optimum
if (nonInteractive)
dimensions[i].setCentre(currentOptimum.getParameterValue(i));
} catch (IllegalArgumentException e) {
IJ.error(TITLE, String.format("Unable to configure dimension [%d] %s: " + e.getMessage(), i, ss_filter.getParameterName(i)));
return -1;
}
} else {
dimensions[i] = new SearchDimension(ss_filter.getDisabledParameterValue(i));
}
}
for (int i = 0; i < disabled.length; i++) disabled[i] = !dimensions[i].active;
// Check the number of combinations is OK
long combinations = SearchSpace.countCombinations(dimensions);
if (!nonInteractive && combinations > 10000) {
gd = new GenericDialog(TITLE);
gd.addMessage(String.format("%d combinations for the configured dimensions.\n \nClick 'Yes' to optimise.", combinations));
gd.enableYesNoCancel();
gd.hideCancelButton();
gd.showDialog();
if (!gd.wasOKed()) {
combinations = 0;
}
}
if (combinations == 0) {
resumeFilterTimer();
} else {
algorithm = EVOLVE[evolve] + " " + rangeSearchWidth;
ga_statusPrefix = algorithm + " [" + setNumber + "] " + filterSet.getName() + " ... ";
ga_iteration = 0;
es_optimum = null;
SearchSpace ss = new SearchSpace();
ss.setTracker(this);
if (seedSize > 0) {
double[][] sample;
// Add current optimum to seed
if (nonInteractive) {
sample = new double[1][];
sample[0] = currentOptimum.getParameters();
seed = merge(seed, sample);
}
int size = (seed == null) ? 0 : seed.length;
// Sample without rounding as the seed will be rounded
sample = SearchSpace.sampleWithoutRounding(dimensions, seedSize - size, null);
seed = merge(seed, sample);
}
// Note: If we have an optimum and we are not seeding this should not matter as the dimensions
// have been centred on the current optimum
ss.seed(seed);
ConvergenceChecker<FilterScore> checker = new InterruptConvergenceChecker(0, 0, maxIterations);
createGAWindow();
resumeFilterTimer();
SearchResult<FilterScore> optimum = ss.search(dimensions, this, checker, myRefinementMode);
if (optimum != null) {
// In case optimisation was stopped
IJ.resetEscape();
best = ((SimpleFilterScore) optimum.score).r.filter;
if (seedSize > 0) {
// Not required as the search now respects the min interval
// The optimum may be off grid if it was from the seed
//best = enumerateMinInterval(best, enabled);
}
// Now update the filter set for final assessment
filterSet = new FilterSet(filterSet.getName(), searchSpaceToFilters((DirectFilter) best, ss.getSearchSpace()));
// Option to save the filters
if (saveOption)
saveFilterSet(filterSet, setNumber, !nonInteractive);
}
}
} else
resumeFilterTimer();
}
if (evolve == 3 && originalDimensions != null) {
// Collect parameters for the enrichment search algorithm
pauseFilterTimer();
boolean[] enabled = searchRangeMap.get(setNumber);
int n = ss_filter.getNumberOfParameters();
if (enabled == null || enabled.length != n) {
enabled = new boolean[n];
Arrays.fill(enabled, true);
// See if the same number of parameters have been optimised in other algorithms
double[] stepSize = stepSizeMap.get(setNumber);
if (stepSize != null && enabled.length == stepSize.length) {
for (int j = 0; j < stepSize.length; j++) if (stepSize[j] < 0)
enabled[j] = false;
}
}
GenericDialog gd = null;
boolean runAlgorithm = nonInteractive;
if (!nonInteractive) {
// Ask the user for the search parameters.
gd = new GenericDialog(TITLE);
String prefix = setNumber + "_";
gd.addMessage("Configure the enrichment search algorithm for [" + setNumber + "] " + filterSet.getName());
gd.addCheckbox(prefix + "Save_option", saveOption);
gd.addNumericField(prefix + "Max_iterations", maxIterations, 0);
gd.addNumericField(prefix + "Converged_count", convergedCount, 0);
gd.addNumericField(prefix + "Samples", enrichmentSamples, 0);
gd.addSlider(prefix + "Fraction", 0.01, 0.99, enrichmentFraction);
gd.addSlider(prefix + "Padding", 0, 0.99, enrichmentPadding);
// Add choice of fields to optimise
for (int i = 0; i < n; i++) gd.addCheckbox(getDialogName(prefix, ss_filter, i), enabled[i]);
gd.showDialog();
runAlgorithm = !gd.wasCanceled();
}
if (runAlgorithm) {
FixedDimension[] dimensions = Arrays.copyOf(originalDimensions, originalDimensions.length);
if (!nonInteractive) {
saveOption = gd.getNextBoolean();
maxIterations = (int) gd.getNextNumber();
convergedCount = (int) gd.getNextNumber();
enrichmentSamples = (int) gd.getNextNumber();
enrichmentFraction = gd.getNextNumber();
enrichmentPadding = gd.getNextNumber();
for (int i = 0; i < n; i++) enabled[i] = gd.getNextBoolean();
searchRangeMap.put(setNumber, enabled);
}
for (int i = 0; i < n; i++) {
if (!enabled[i])
dimensions[i] = new FixedDimension(ss_filter.getDisabledParameterValue(i));
}
for (int i = 0; i < disabled.length; i++) disabled[i] = !dimensions[i].active;
algorithm = EVOLVE[evolve];
ga_statusPrefix = algorithm + " [" + setNumber + "] " + filterSet.getName() + " ... ";
ga_iteration = 0;
es_optimum = null;
SearchSpace ss = new SearchSpace();
ss.setTracker(this);
// Add current optimum to seed
if (nonInteractive) {
double[][] sample = new double[1][];
sample[0] = currentOptimum.getParameters();
seed = merge(seed, sample);
}
ss.seed(seed);
ConvergenceChecker<FilterScore> checker = new InterruptConvergenceChecker(0, 0, maxIterations, convergedCount);
createGAWindow();
resumeFilterTimer();
SearchResult<FilterScore> optimum = ss.enrichmentSearch(dimensions, this, checker, enrichmentSamples, enrichmentFraction, enrichmentPadding);
if (optimum != null) {
// In case optimisation was stopped
IJ.resetEscape();
best = ((SimpleFilterScore) optimum.score).r.filter;
// Not required as the search now respects the min interval
// Enumerate on the min interval to produce the final filter
//best = enumerateMinInterval(best, enabled);
// Now update the filter set for final assessment
filterSet = new FilterSet(filterSet.getName(), searchSpaceToFilters((DirectFilter) best, ss.getSearchSpace()));
// Option to save the filters
if (saveOption)
saveFilterSet(filterSet, setNumber, !nonInteractive);
}
} else
resumeFilterTimer();
}
IJ.showStatus("Analysing [" + setNumber + "] " + filterSet.getName() + " ...");
// Do not support plotting if we used optimisation
double[] xValues = (best != null || isHeadless || (plotTopN == 0)) ? null : new double[filterSet.size()];
double[] yValues = (xValues == null) ? null : new double[xValues.length];
SimpleFilterScore max = null;
// It can just assess the top 1 required for the summary.
if (best != null) {
// Only assess the top 1 filter for the summary
List<Filter> list = new ArrayList<Filter>();
list.add((DirectFilter) best);
filterSet = new FilterSet(filterSet.getName(), list);
}
// Score the filters and report the results if configured.
FilterScoreResult[] scoreResults = scoreFilters(setUncomputedStrength(filterSet), showResultsTable);
if (scoreResults == null)
return -1;
analysisStopWatch.stop();
for (int index = 0; index < scoreResults.length; index++) {
final FilterScoreResult scoreResult = scoreResults[index];
if (xValues != null) {
xValues[index] = scoreResult.filter.getNumericalValue();
yValues[index] = scoreResult.score;
}
final SimpleFilterScore result = new SimpleFilterScore(scoreResult, allSameType, scoreResult.criteria >= minCriteria);
if (result.compareTo(max) < 0) {
max = result;
}
}
if (showResultsTable) {
BufferedTextWindow tw = null;
if (resultsWindow != null) {
tw = new BufferedTextWindow(resultsWindow);
tw.setIncrement(Integer.MAX_VALUE);
}
for (int index = 0; index < scoreResults.length; index++) addToResultsWindow(tw, scoreResults[index].text);
if (resultsWindow != null)
resultsWindow.getTextPanel().updateDisplay();
}
// Check the top filter against the limits of the original dimensions
char[] atLimit = null;
if (allSameType && originalDimensions != null) {
DirectFilter filter = max.r.filter;
int[] indices = filter.getChromosomeParameters();
atLimit = new char[indices.length];
StringBuilder sb = new StringBuilder(200);
for (int j = 0; j < indices.length; j++) {
atLimit[j] = ComplexFilterScore.WITHIN;
final int p = indices[j];
if (disabled[p])
continue;
final double value = filter.getParameterValue(p);
double lowerLimit = originalDimensions[p].getLower();
double upperLimit = originalDimensions[p].getUpper();
int c1 = Double.compare(value, lowerLimit);
if (c1 <= 0) {
atLimit[j] = ComplexFilterScore.FLOOR;
sb.append(" : ").append(filter.getParameterName(p)).append(' ').append(atLimit[j]).append('[').append(Utils.rounded(value));
if (c1 == -1) {
atLimit[j] = ComplexFilterScore.BELOW;
sb.append("<").append(Utils.rounded(lowerLimit));
}
sb.append("]");
} else {
int c2 = Double.compare(value, upperLimit);
if (c2 >= 0) {
atLimit[j] = ComplexFilterScore.CEIL;
sb.append(" : ").append(filter.getParameterName(p)).append(' ').append(atLimit[j]).append('[').append(Utils.rounded(value));
if (c2 == 1) {
atLimit[j] = ComplexFilterScore.ABOVE;
sb.append(">").append(Utils.rounded(upperLimit));
}
sb.append("]");
}
}
}
if (sb.length() > 0) {
if (max.criteriaPassed) {
Utils.log("Warning: Top filter (%s @ %s|%s) [%s] at the limit of the expanded range%s", filter.getName(), Utils.rounded((invertScore) ? -max.score : max.score), Utils.rounded((invertCriteria) ? -minCriteria : minCriteria), limitFailCount + limitRange, sb.toString());
} else {
Utils.log("Warning: Top filter (%s @ -|%s) [%s] at the limit of the expanded range%s", filter.getName(), Utils.rounded((invertCriteria) ? -max.criteria : max.criteria), limitFailCount + limitRange, sb.toString());
}
}
}
// Note that max should never be null since this method is not run with an empty filter set
// We may have no filters that pass the criteria
String type = max.r.filter.getType();
if (!max.criteriaPassed) {
Utils.log("Warning: Filter does not pass the criteria: %s : Best = %s using %s", type, Utils.rounded((invertCriteria) ? -max.criteria : max.criteria), max.r.filter.getName());
return 0;
}
// This could be an option?
boolean allowDuplicates = true;
// XXX - Commented out the requirement to be the same type to store for later analysis.
// This may break the code, however I think that all filter sets should be able to have a best filter
// irrespective of whether they were the same type or not.
//if (allSameType)
//{
ComplexFilterScore newFilterScore = new ComplexFilterScore(max.r, atLimit, algorithm, analysisStopWatch.getTime(), "", 0);
addBestFilter(type, allowDuplicates, newFilterScore);
// Add spacer at end of each result set
if (isHeadless) {
if (showResultsTable && filterSet.size() > 1)
IJ.log("");
} else {
if (showResultsTable && filterSet.size() > 1)
resultsWindow.append("");
if (plotTopN > 0 && xValues != null) {
// Check the xValues are unique. Since the filters have been sorted by their
// numeric value we only need to compare adjacent entries.
boolean unique = true;
for (int ii = 0; ii < xValues.length - 1; ii++) {
if (xValues[ii] == xValues[ii + 1]) {
unique = false;
break;
}
}
String xAxisName = filterSet.getValueName();
if (unique) {
// Check the values all refer to the same property
for (Filter filter : filterSet.getFilters()) {
if (!xAxisName.equals(filter.getNumericalValueName())) {
unique = false;
break;
}
}
}
if (!unique) {
// If not unique then renumber them and use an arbitrary label
xAxisName = "Filter";
for (int ii = 0; ii < xValues.length; ii++) xValues[ii] = ii + 1;
}
String title = filterSet.getName();
// Check if a previous filter set had the same name, update if necessary
NamedPlot p = getNamedPlot(title);
if (p == null)
plots.add(new NamedPlot(title, xAxisName, xValues, yValues));
else
p.updateValues(xAxisName, xValues, yValues);
if (plots.size() > plotTopN) {
Collections.sort(plots);
p = plots.remove(plots.size() - 1);
}
}
}
return 0;
}
Also used :
SimpleRecombiner(gdsc.smlm.ga.SimpleRecombiner)
SearchSpace(gdsc.smlm.search.SearchSpace)
ArrayList(java.util.ArrayList)
RandomDataGenerator(org.apache.commons.math3.random.RandomDataGenerator)
BufferedTextWindow(gdsc.core.ij.BufferedTextWindow)
SearchDimension(gdsc.smlm.search.SearchDimension)
FixedDimension(gdsc.smlm.search.FixedDimension)
FilterSet(gdsc.smlm.results.filter.FilterSet)
RampedSelectionStrategy(gdsc.smlm.ga.RampedSelectionStrategy)
GenericDialog(ij.gui.GenericDialog)
NonBlockingGenericDialog(ij.gui.NonBlockingGenericDialog)
ParameterType(gdsc.smlm.results.filter.ParameterType)
IDirectFilter(gdsc.smlm.results.filter.IDirectFilter)
DirectFilter(gdsc.smlm.results.filter.DirectFilter)
SimpleSelectionStrategy(gdsc.smlm.ga.SimpleSelectionStrategy)
IDirectFilter(gdsc.smlm.results.filter.IDirectFilter)
DirectFilter(gdsc.smlm.results.filter.DirectFilter)
Filter(gdsc.smlm.results.filter.Filter)
MultiPathFilter(gdsc.smlm.results.filter.MultiPathFilter)
MaximaSpotFilter(gdsc.smlm.filters.MaximaSpotFilter)
SimpleMutator(gdsc.smlm.ga.SimpleMutator)
FilterScore(gdsc.smlm.results.filter.FilterScore)
Well44497b(org.apache.commons.math3.random.Well44497b)
Example 2 with Well44497b
use of org.apache.commons.math3.random.Well44497b in project mixcr by milaboratory.
the class TargetBuilderTest method testGenerate1.
@Test
public void testGenerate1() throws Exception {
VDJCLibrary library = VDJCLibraryRegistry.getDefault().getLibrary("default", "hs");
TargetBuilder.VDJCGenes genes = new TargetBuilder.VDJCGenes(library, "TRBV12-1*00", "TRBD1*00", "TRBJ1-3*00", "TRBC2*00");
String model = "{CDR3Begin(-20)}VVVVVVVVVVVvVVVvVVVVVVVVVVVVVNNNNNNN{DBegin(0)}DDDDDDDDDDNN{CDR3End(-10)}JJJJJJJJJJJJJJJJJJJJ";
NucleotideSequence nucleotideSequence = TargetBuilder.generateSequence(genes, model, new Well44497b());
Assert.assertEquals(36 + 12 + 20, nucleotideSequence.size());
}
Also used :
NucleotideSequence(com.milaboratory.core.sequence.NucleotideSequence)
Well44497b(org.apache.commons.math3.random.Well44497b)
VDJCLibrary(io.repseq.core.VDJCLibrary)
Test(org.junit.Test)
Example 3 with Well44497b
use of org.apache.commons.math3.random.Well44497b in project mixcr by milaboratory.
the class VDJCObjectExtenderTest method testTripleRead.
@Test
public void testTripleRead() throws Exception {
final boolean print = true;
Well44497b rg = new Well44497b(12312);
final VDJCAlignerParameters rnaSeqParams = VDJCParametersPresets.getByName("rna-seq");
final PartialAlignmentsAssemblerAligner aligner = new PartialAlignmentsAssemblerAligner(rnaSeqParams);
final VDJCLibrary lib = VDJCLibraryRegistry.getDefault().getLibrary("default", "hs");
for (VDJCGene gene : VDJCLibraryRegistry.getDefault().getLibrary("default", "hs").getGenes()) if (gene.isFunctional())
aligner.addGene(gene);
TargetBuilder.VDJCGenes genes = new TargetBuilder.VDJCGenes(lib, "TRBV12-3*00", "TRBD1*00", "TRBJ1-3*00", "TRBC2*00");
// | 310 | 338 | 438
// 250V + 60CDR3 (20V 7N 10D 3N 20J) + 28J + 100C + 100N
final NucleotideSequence baseSeq = TargetBuilder.generateSequence(genes, "{CDR3Begin(-250)}V*270 NNNNNNN {DBegin(0)}D*10 NNN {CDR3End(-20):FR4End} {CBegin}C*100 N*100", rg);
F4 goAssert = new F4() {
@Override
public VDJCAlignments go(int len, int offset1, int offset2, int offset3) {
NucleotideSequence seq1 = baseSeq.getRange(offset1, offset1 + len);
NucleotideSequence seq2 = baseSeq.getRange(offset2, offset2 + len);
NucleotideSequence seq3 = offset3 == -1 ? null : baseSeq.getRange(offset3, offset3 + len);
VDJCAlignmentResult<VDJCMultiRead> alignment = offset3 == -1 ? aligner.process(MiXCRTestUtils.createMultiRead(seq1, seq2)) : aligner.process(MiXCRTestUtils.createMultiRead(seq1, seq2, seq3));
VDJCAlignments al = alignment.alignment;
Assert.assertNotNull(al);
VDJCObjectExtender<VDJCAlignments> extender = new VDJCObjectExtender<>(Chains.TCR, (byte) 35, rnaSeqParams.getVAlignerParameters().getParameters().getScoring(), rnaSeqParams.getJAlignerParameters().getParameters().getScoring(), 100, 70, ReferencePoint.CDR3Begin, ReferencePoint.CDR3End);
MiXCRTestUtils.assertAlignments(al);
if (print) {
MiXCRTestUtils.printAlignment(al);
System.out.println();
System.out.println("-------------------------------------------");
System.out.println();
}
VDJCAlignments processed = extender.process(al);
if (print) {
MiXCRTestUtils.printAlignment(processed);
System.out.println();
System.out.println("===========================================");
System.out.println();
System.out.println();
}
MiXCRTestUtils.assertAlignments(processed);
Assert.assertEquals(al.getFeature(GeneFeature.VDJunction), processed.getFeature(GeneFeature.VDJunction));
Assert.assertEquals(al.getFeature(GeneFeature.DJJunction), processed.getFeature(GeneFeature.DJJunction));
Assert.assertEquals(al.getFeature(GeneFeature.VJJunction), processed.getFeature(GeneFeature.VJJunction));
return processed;
}
};
VDJCAlignments a1 = goAssert.go(60, 245, 307, 450);
Assert.assertEquals(2, a1.numberOfTargets());
VDJCAlignments a2 = goAssert.go(60, 245, 315, 450);
Assert.assertEquals(3, a2.numberOfTargets());
VDJCAlignments a3 = goAssert.go(60, 245, 315, -1);
Assert.assertEquals(2, a3.numberOfTargets());
VDJCAlignments a4 = goAssert.go(60, 245, 307, -1);
Assert.assertEquals(1, a4.numberOfTargets());
VDJCAlignments a5 = goAssert.go(53, 252, 307, -1);
Assert.assertEquals(1, a5.numberOfTargets());
VDJCAlignments a6 = goAssert.go(53, 252, 307, 450);
Assert.assertEquals(2, a6.numberOfTargets());
}
Also used :
VDJCMultiRead(com.milaboratory.mixcr.partialassembler.VDJCMultiRead)
VDJCAlignerParameters(com.milaboratory.mixcr.vdjaligners.VDJCAlignerParameters)
PartialAlignmentsAssemblerAligner(com.milaboratory.mixcr.partialassembler.PartialAlignmentsAssemblerAligner)
NucleotideSequence(com.milaboratory.core.sequence.NucleotideSequence)
Well44497b(org.apache.commons.math3.random.Well44497b)
TargetBuilder(com.milaboratory.mixcr.tests.TargetBuilder)
VDJCAlignments(com.milaboratory.mixcr.basictypes.VDJCAlignments)
Test(org.junit.Test)
Example 4 with Well44497b
use of org.apache.commons.math3.random.Well44497b in project mixcr by milaboratory.
the class FieldExtractorsTest method testAnchorPoints1.
@Test
public void testAnchorPoints1() throws Exception {
final boolean print = false;
final Well44497b rg = new Well44497b(12312);
final VDJCAlignerParameters rnaSeqParams = VDJCParametersPresets.getByName("rna-seq");
final PartialAlignmentsAssemblerAligner aligner = new PartialAlignmentsAssemblerAligner(rnaSeqParams);
final VDJCLibrary lib = VDJCLibraryRegistry.getDefault().getLibrary("default", "hs");
for (VDJCGene gene : VDJCLibraryRegistry.getDefault().getLibrary("default", "hs").getGenes()) if (gene.isFunctional())
aligner.addGene(gene);
final TargetBuilder.VDJCGenes genes = new TargetBuilder.VDJCGenes(lib, "TRBV12-3*00", "TRBD1*00", "TRBJ1-3*00", "TRBC2*00");
// | 310 | 338 | 438
// 250V + 60CDR3 (20V 7N 10D 3N 20J) + 28J + 100C + 100N
// "{CDR3Begin(-250)}V*270 NNNNNNN {DBegin(0)}D*10 NNN {CDR3End(-20):FR4End} {CBegin}C*100 N*100"
final FieldExtractors.ExtractDefaultReferencePointsPositions extractor = new FieldExtractors.ExtractDefaultReferencePointsPositions();
F6 goAssert = new F6() {
@Override
public Integer[][] go(String seq, int len, int offset1, int offset2, int offset3, String expected) {
final NucleotideSequence baseSeq = TargetBuilder.generateSequence(genes, seq, rg);
NucleotideSequence seq1 = baseSeq.getRange(offset1, Math.min(baseSeq.size(), offset1 + len));
NucleotideSequence seq2 = offset2 == -1 ? null : baseSeq.getRange(offset2, Math.min(baseSeq.size(), offset2 + len));
NucleotideSequence seq3 = offset3 == -1 ? null : baseSeq.getRange(offset3, Math.min(baseSeq.size(), offset3 + len));
VDJCAlignmentResult<VDJCMultiRead> alignment = offset3 == -1 ? offset2 == -1 ? aligner.process(MiXCRTestUtils.createMultiRead(seq1)) : aligner.process(MiXCRTestUtils.createMultiRead(seq1, seq2)) : aligner.process(MiXCRTestUtils.createMultiRead(seq1, seq2, seq3));
VDJCAlignments al = alignment.alignment;
Assert.assertNotNull(al);
if (print) {
MiXCRTestUtils.printAlignment(al);
System.out.println();
System.out.println("-------------------------------------------");
System.out.println();
}
String val = extractor.extract(al);
if (print)
System.out.println(val);
String[] spl = val.split(",");
Integer[][] result = new Integer[spl.length][ReferencePoint.DefaultReferencePoints.length];
for (int i = 0; i < spl.length; i++) {
String[] spl1 = spl[i].split(":");
for (int j = 0; j < spl1.length; j++) {
try {
result[i][j] = Integer.decode(spl1[j]);
} catch (NumberFormatException e) {
}
}
}
return result;
}
};
// No PSegments, just deletions
Integer[][] r = goAssert.go("{CDR3Begin(-250):VEnd(-3)} 'CCAAA' {DBegin(0):DEnd(0)} 'AAA' {JBegin(2):FR4End} " + "{CBegin}C*100 N*100", 100, 240, 307, 450, "");
assertExportPoint(r[0], ReferencePoint.VEnd, -3);
assertExportPoint(r[0], ReferencePoint.DBegin, 0);
assertExportPoint(r[0], ReferencePoint.DEnd, 0);
assertExportPoint(r[0], ReferencePoint.JBegin, -2);
r = goAssert.go("{CDR3Begin(-250):VEnd(0)} 'CCAAA' {DBegin(0):DEnd(-2)} 'AAA' {JBegin:FR4End} {CBegin}C*100 N*100", 100, 240, 307, 450, "");
assertExportPoint(r[0], ReferencePoint.VEnd, 0);
assertExportPoint(r[0], ReferencePoint.DBegin, 0);
assertExportPoint(r[0], ReferencePoint.DEnd, -2);
assertExportPoint(r[0], ReferencePoint.JBegin, 0);
// With PSegments
r = goAssert.go("{CDR3Begin(-250):VEnd(0)} {VEnd:VEnd(-3)} 'CCAAA' {DBegin(3):DBegin} {DBegin:DEnd(-2)} 'AAA' " + "{JBegin(2):JBegin} {JBegin:FR4End} {CBegin}C*100 N*100", 100, 240, 307, 450, "");
assertExportPoint(r[0], ReferencePoint.VEnd, 3);
assertExportPoint(r[0], ReferencePoint.DBegin, 3);
assertExportPoint(r[0], ReferencePoint.DEnd, -2);
assertExportPoint(r[0], ReferencePoint.JBegin, 2);
}
Also used :
VDJCMultiRead(com.milaboratory.mixcr.partialassembler.VDJCMultiRead)
VDJCAlignerParameters(com.milaboratory.mixcr.vdjaligners.VDJCAlignerParameters)
PartialAlignmentsAssemblerAligner(com.milaboratory.mixcr.partialassembler.PartialAlignmentsAssemblerAligner)
NucleotideSequence(com.milaboratory.core.sequence.NucleotideSequence)
Well44497b(org.apache.commons.math3.random.Well44497b)
TargetBuilder(com.milaboratory.mixcr.tests.TargetBuilder)
VDJCAlignments(com.milaboratory.mixcr.basictypes.VDJCAlignments)
Test(org.junit.Test)
Example 5 with Well44497b
use of org.apache.commons.math3.random.Well44497b in project GDSC-SMLM by aherbert.
the class PCPALMFitting method runBoundedOptimiser.
private PointValuePair runBoundedOptimiser(double[][] gr, double[] initialSolution, double[] lB, double[] uB, SumOfSquaresModelFunction function) {
// Create the functions to optimise
ObjectiveFunction objective = new ObjectiveFunction(new SumOfSquaresMultivariateFunction(function));
ObjectiveFunctionGradient gradient = new ObjectiveFunctionGradient(new SumOfSquaresMultivariateVectorFunction(function));
final boolean debug = false;
// Try a BFGS optimiser since this will produce a deterministic solution and can respect bounds.
PointValuePair optimum = null;
boundedEvaluations = 0;
final MaxEval maxEvaluations = new MaxEval(2000);
MultivariateOptimizer opt = null;
for (int iteration = 0; iteration <= fitRestarts; iteration++) {
try {
opt = new BFGSOptimizer();
final double relativeThreshold = 1e-6;
// Configure maximum step length for each dimension using the bounds
double[] stepLength = new double[lB.length];
for (int i = 0; i < stepLength.length; i++) stepLength[i] = (uB[i] - lB[i]) * 0.3333333;
// The GoalType is always minimise so no need to pass this in
optimum = opt.optimize(maxEvaluations, gradient, objective, new InitialGuess((optimum == null) ? initialSolution : optimum.getPointRef()), new SimpleBounds(lB, uB), new BFGSOptimizer.GradientTolerance(relativeThreshold), new BFGSOptimizer.StepLength(stepLength));
if (debug)
System.out.printf("BFGS Iter %d = %g (%d)\n", iteration, optimum.getValue(), opt.getEvaluations());
} catch (TooManyEvaluationsException e) {
// No need to restart
break;
} catch (RuntimeException e) {
// No need to restart
break;
} finally {
boundedEvaluations += opt.getEvaluations();
}
}
// Try a CMAES optimiser which is non-deterministic. To overcome this we perform restarts.
// CMAESOptimiser based on Matlab code:
// https://www.lri.fr/~hansen/cmaes.m
// Take the defaults from the Matlab documentation
//Double.NEGATIVE_INFINITY;
double stopFitness = 0;
boolean isActiveCMA = true;
int diagonalOnly = 0;
int checkFeasableCount = 1;
//Well19937c();
RandomGenerator random = new Well44497b();
boolean generateStatistics = false;
ConvergenceChecker<PointValuePair> checker = new SimpleValueChecker(1e-6, 1e-10);
// The sigma determines the search range for the variables. It should be 1/3 of the initial search region.
double[] range = new double[lB.length];
for (int i = 0; i < lB.length; i++) range[i] = (uB[i] - lB[i]) / 3;
OptimizationData sigma = new CMAESOptimizer.Sigma(range);
OptimizationData popSize = new CMAESOptimizer.PopulationSize((int) (4 + Math.floor(3 * Math.log(initialSolution.length))));
SimpleBounds bounds = new SimpleBounds(lB, uB);
opt = new CMAESOptimizer(maxEvaluations.getMaxEval(), stopFitness, isActiveCMA, diagonalOnly, checkFeasableCount, random, generateStatistics, checker);
// Restart the optimiser several times and take the best answer.
for (int iteration = 0; iteration <= fitRestarts; iteration++) {
try {
// Start from the initial solution
PointValuePair constrainedSolution = opt.optimize(new InitialGuess(initialSolution), objective, GoalType.MINIMIZE, bounds, sigma, popSize, maxEvaluations);
if (debug)
System.out.printf("CMAES Iter %d initial = %g (%d)\n", iteration, constrainedSolution.getValue(), opt.getEvaluations());
boundedEvaluations += opt.getEvaluations();
if (optimum == null || constrainedSolution.getValue() < optimum.getValue()) {
optimum = constrainedSolution;
}
} catch (TooManyEvaluationsException e) {
} catch (TooManyIterationsException e) {
} finally {
boundedEvaluations += maxEvaluations.getMaxEval();
}
if (optimum == null)
continue;
try {
// Also restart from the current optimum
PointValuePair constrainedSolution = opt.optimize(new InitialGuess(optimum.getPointRef()), objective, GoalType.MINIMIZE, bounds, sigma, popSize, maxEvaluations);
if (debug)
System.out.printf("CMAES Iter %d restart = %g (%d)\n", iteration, constrainedSolution.getValue(), opt.getEvaluations());
if (constrainedSolution.getValue() < optimum.getValue()) {
optimum = constrainedSolution;
}
} catch (TooManyEvaluationsException e) {
} catch (TooManyIterationsException e) {
} finally {
boundedEvaluations += maxEvaluations.getMaxEval();
}
}
return optimum;
}
Also used :
MultivariateOptimizer(org.apache.commons.math3.optim.nonlinear.scalar.MultivariateOptimizer)
MaxEval(org.apache.commons.math3.optim.MaxEval)
InitialGuess(org.apache.commons.math3.optim.InitialGuess)
SimpleBounds(org.apache.commons.math3.optim.SimpleBounds)
ObjectiveFunction(org.apache.commons.math3.optim.nonlinear.scalar.ObjectiveFunction)
SimpleValueChecker(org.apache.commons.math3.optim.SimpleValueChecker)
BFGSOptimizer(org.apache.commons.math3.optim.nonlinear.scalar.gradient.BFGSOptimizer)
RandomGenerator(org.apache.commons.math3.random.RandomGenerator)
PointValuePair(org.apache.commons.math3.optim.PointValuePair)
TooManyEvaluationsException(org.apache.commons.math3.exception.TooManyEvaluationsException)
TooManyIterationsException(org.apache.commons.math3.exception.TooManyIterationsException)
CMAESOptimizer(org.apache.commons.math3.optim.nonlinear.scalar.noderiv.CMAESOptimizer)
ObjectiveFunctionGradient(org.apache.commons.math3.optim.nonlinear.scalar.ObjectiveFunctionGradient)
Well44497b(org.apache.commons.math3.random.Well44497b)
OptimizationData(org.apache.commons.math3.optim.OptimizationData)
Aggregations
Well44497b (org.apache.commons.math3.random.Well44497b)11
NucleotideSequence (com.milaboratory.core.sequence.NucleotideSequence)5
Test (org.junit.Test)5
VDJCAlignments (com.milaboratory.mixcr.basictypes.VDJCAlignments)4
TargetBuilder (com.milaboratory.mixcr.tests.TargetBuilder)3
VDJCAlignerParameters (com.milaboratory.mixcr.vdjaligners.VDJCAlignerParameters)3
RandomGenerator (org.apache.commons.math3.random.RandomGenerator)3
PartialAlignmentsAssemblerAligner (com.milaboratory.mixcr.partialassembler.PartialAlignmentsAssemblerAligner)2
VDJCMultiRead (com.milaboratory.mixcr.partialassembler.VDJCMultiRead)2
VDJCLibrary (io.repseq.core.VDJCLibrary)2
Point (java.awt.Point)2
CUtils (cc.redberry.pipe.CUtils)1
Range (com.milaboratory.core.Range)1
PairedRead (com.milaboratory.core.io.sequence.PairedRead)1
SequenceRead (com.milaboratory.core.io.sequence.SequenceRead)1
SequenceReaderCloseable (com.milaboratory.core.io.sequence.SequenceReaderCloseable)1
SingleReadImpl (com.milaboratory.core.io.sequence.SingleReadImpl)1
NSequenceWithQuality (com.milaboratory.core.sequence.NSequenceWithQuality)1
SequenceQuality (com.milaboratory.core.sequence.SequenceQuality)1
CachedSequenceProvider (com.milaboratory.core.sequence.provider.CachedSequenceProvider)1
