Example 31 with Project
use of org.openstack4j.model.identity.v3.Project in project TrakEM2 by trakem2.
the class Compare method reliabilityAnalysis.
public static final Bureaucrat reliabilityAnalysis(final String[] ignore, final boolean output_arff, final boolean weka_classify, final boolean show_dialog, final double delta, final double wi, final double wd, final double wm) {
// gather all open projects
final Project[] p = Project.getProjects().toArray(new Project[0]);
final Worker worker = new Worker("Reliability by name") {
@Override
public void run() {
startedWorking();
try {
final CATAParameters cp = new CATAParameters();
cp.delta = delta;
if (show_dialog && !cp.setup(false, null, false, false)) {
finishedWorking();
return;
}
Object[] ob = gatherChains(p, cp, ignore);
final ArrayList<Chain> chains = (ArrayList<Chain>) ob[0];
// to keep track of each project's chains
final ArrayList[] p_chains = (ArrayList[]) ob[1];
ob = null;
if (null == chains) {
finishedWorking();
return;
}
// For each pipe in a brain:
// - score against all other brains in which that pipe name exists,
// - record the score position within that brain.
//
final ExecutorService exec = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
// for each individual lineage:
final TreeMap<String, ArrayList<Integer>> indices = new TreeMap<String, ArrayList<Integer>>();
final ArrayList<CITuple> cin = new ArrayList<CITuple>();
// for each family:
final TreeMap<String, ArrayList<Integer>> indices_f = new TreeMap<String, ArrayList<Integer>>();
final ArrayList<CITuple> cin_f = new ArrayList<CITuple>();
final ArrayList<Future> fus = new ArrayList<Future>();
// For neural network analysis:
final StringBuilder arff = output_arff ? new StringBuilder("@RELATION Lineages\n\n") : null;
if (output_arff) {
arff.append("@ATTRIBUTE APD NUMERIC\n");
arff.append("@ATTRIBUTE CPD NUMERIC\n");
arff.append("@ATTRIBUTE STD NUMERIC\n");
arff.append("@ATTRIBUTE MPD NUMERIC\n");
arff.append("@ATTRIBUTE PM NUMERIC\n");
arff.append("@ATTRIBUTE LEV NUMERIC\n");
arff.append("@ATTRIBUTE SIM NUMERIC\n");
arff.append("@ATTRIBUTE PRX NUMERIC\n");
arff.append("@ATTRIBUTE PRM NUMERIC\n");
// length ratio: len(query) / len(ref)
arff.append("@ATTRIBUTE LR NUMERIC\n");
arff.append("@ATTRIBUTE TR NUMERIC\n");
arff.append("@ATTRIBUTE CLASS {false,true}\n");
arff.append("\n@DATA\n");
}
// Count number of times when decision tree says it's good, versus number of times when it should be good
// observed
final AtomicInteger obs_good = new AtomicInteger(0);
// observed wrong
final AtomicInteger obs_wrong = new AtomicInteger(0);
// expected
final AtomicInteger exp_good = new AtomicInteger(0);
final AtomicInteger exp_bad = new AtomicInteger(0);
final AtomicInteger obs_bad_classified_good_ones = new AtomicInteger(0);
final AtomicInteger obs_well_classified_bad_ones = new AtomicInteger(0);
// inc by one when a lineage to compare is not found at all in the brain that works as reference
final AtomicInteger not_found = new AtomicInteger(0);
final AtomicInteger already_classified = new AtomicInteger(0);
Method classify_ = null;
if (weka_classify) {
try {
classify_ = Class.forName("lineage.LineageClassifier").getDeclaredMethod("classify", new Class[] { double[].class });
} catch (final Exception e) {
IJError.print(e);
}
}
final Method classify = classify_;
// All possible pairs of projects, with repetition (it's not the same, although the pipe pairwise comparison itself will be.)
for (int _i = 0; _i < p_chains.length; _i++) {
final int i = _i;
Utils.log2("Project " + p[i] + " has " + p_chains[i].size() + " chains.");
for (int _j = 0; _j < p_chains.length; _j++) {
final int j = _j;
// skip same project (would have a score of zero, identical.)
if (i == j)
continue;
final String[] titles_j = new String[p_chains[j].size()];
int next = 0;
for (final Chain cj : (ArrayList<Chain>) p_chains[j]) {
final String t = cj.getCellTitle();
titles_j[next++] = t.substring(0, t.indexOf(' '));
}
// families:
final TreeSet<String> ts_families = new TreeSet<String>();
for (int f = 0; f < titles_j.length; f++) {
// extract family name from title: read the first continuous string of capital letters
final String title = titles_j[f];
int u = 0;
for (; u < title.length(); u++) {
if (!Character.isUpperCase(title.charAt(u)))
break;
}
ts_families.add(title.substring(0, u));
}
final ArrayList<String> families = new ArrayList<String>(ts_families);
fus.add(exec.submit(new Callable() {
@Override
public Object call() {
// All chains of one project to all chains of the other:
for (final Chain chain : (ArrayList<Chain>) p_chains[i]) {
final VectorString3D vs1 = chain.vs;
// Prepare title
String title = chain.getCellTitle();
title = title.substring(0, title.indexOf(' '));
// check if the other project j contains a chain of name chain.getCellTitle() up to the space.
int title_index = -1;
for (int k = 0; k < titles_j.length; k++) {
if (title.equals(titles_j[k])) {
title_index = k;
break;
}
}
if (-1 == title_index) {
Utils.log2(title + " not found in project " + p[j]);
if (weka_classify)
not_found.incrementAndGet();
continue;
}
// should be there:
if (weka_classify) {
exp_good.incrementAndGet();
exp_bad.addAndGet(titles_j.length - 1);
}
final ArrayList<ChainMatch> list = new ArrayList<ChainMatch>();
// extract family name from title: read the first continuous string of capital letters
int u = 0;
for (; u < title.length(); u++) {
if (!Character.isUpperCase(title.charAt(u)))
break;
}
final String family_name = title.substring(0, u);
String last_classify = null;
int g = 0;
for (final Chain cj : (ArrayList<Chain>) p_chains[j]) {
final VectorString3D vs2 = cj.vs;
final Object[] ob = findBestMatch(vs1, vs2, cp.delta, cp.skip_ends, cp.max_mut, cp.min_chunk, cp.distance_type, cp.direct, cp.substring_matching, wi, wd, wm);
final Editions ed = (Editions) ob[0];
final double[] stats = ed.getStatistics(cp.skip_ends, cp.max_mut, cp.min_chunk, cp.score_mut_only);
final ChainMatch cm = new ChainMatch(cj, null, ed, stats, score(ed.getSimilarity(), ed.getDistance(), stats[3], Compare.W));
cm.title = titles_j[g];
list.add(cm);
g++;
if (weka_classify) {
// from decision tree: is it good?
final double[] param = new double[11];
for (int p = 0; p < stats.length; p++) param[p] = stats[p];
try {
if (((Boolean) classify.invoke(null, param)).booleanValue()) {
if (null != last_classify) {
Utils.log2("ALREADY CLASSIFIED " + title + " as " + last_classify + " (now: " + cm.title + " )");
already_classified.incrementAndGet();
}
last_classify = cm.title;
if (title.equals(cm.title)) {
obs_good.incrementAndGet();
} else {
Utils.log2("WRONG CLASSIFICATION of " + title + " as " + cm.title);
obs_wrong.incrementAndGet();
}
} else {
if (title.equals(cm.title)) {
obs_bad_classified_good_ones.incrementAndGet();
} else {
obs_well_classified_bad_ones.incrementAndGet();
}
}
} catch (final Exception ee) {
IJError.print(ee);
}
}
}
// sort scores:
Compare.sortMatches(list, cp.distance_type, cp.distance_type_2, cp.min_matches);
if (output_arff) {
// Take top 8 and put them into training set for WEKA in arff format
for (int h = 0; h < 8; h++) {
final ChainMatch cm = list.get(h);
final StringBuilder sb = new StringBuilder();
sb.append(cm.phys_dist).append(',').append(cm.cum_phys_dist).append(',').append(cm.stdDev).append(',').append(cm.median).append(',').append(cm.prop_mut).append(',').append(cm.ed.getDistance()).append(',').append(cm.seq_sim).append(',').append(cm.proximity).append(',').append(cm.proximity_mut).append(',').append(cm.prop_len).append(',').append(cm.tortuosity_ratio).append(',').append(title.equals(cm.title)).append(// append('-').append(cm.title.startsWith(family_name)).append('\n');
'\n');
synchronized (arff) {
arff.append(sb);
}
}
}
// record scoring index
int f = 0;
boolean found_specific = false;
boolean found_family = false;
for (final ChainMatch cm : list) {
// Exact match: for each individual lineage
if (!found_specific && title.equals(cm.title)) {
synchronized (indices) {
ArrayList<Integer> al = indices.get(title);
if (null == al) {
al = new ArrayList<Integer>();
indices.put(title, al);
// so I can keep a list of chains sorted by name
cin.add(new CITuple(title, chain, al));
}
al.add(f);
}
found_specific = true;
}
if (!found_family && cm.title.startsWith(family_name)) {
synchronized (indices_f) {
ArrayList<Integer> al = indices_f.get(family_name);
if (null == al) {
al = new ArrayList<Integer>();
indices_f.put(family_name, al);
cin_f.add(new CITuple(family_name, chain, al));
}
al.add(f);
}
found_family = true;
}
if (found_specific && found_family) {
break;
}
//
f++;
}
if (!found_specific) {
Utils.log2("NOT FOUND any match for " + title + " within a list of size " + list.size() + ", in project " + chain.getRoot().getProject());
}
}
return null;
}
}));
}
}
for (final Future fu : fus) {
try {
fu.get();
} catch (final Exception e) {
IJError.print(e);
}
}
exec.shutdownNow();
if (weka_classify) {
// so stateful ... it's a sin.
try {
Class.forName("lineage.LineageClassifier").getDeclaredMethod("flush", new Class[] {}).invoke(null, new Object[] {});
} catch (final Exception e) {
IJError.print(e);
}
}
// export ARFF for neural network training
if (output_arff) {
Utils.saveToFile(new File(System.getProperty("user.dir") + "/lineages.arff"), arff.toString());
}
// Show the results from indices map
final StringBuilder sb = new StringBuilder();
// scoring index vs count of occurrences
final TreeMap<Integer, Integer> sum = new TreeMap<Integer, Integer>();
// best scoring index of best family member vs count of ocurrences
final TreeMap<Integer, Integer> sum_f = new TreeMap<Integer, Integer>();
// scoring index vs count of ocurrences, within each family
final TreeMap<String, TreeMap<Integer, Integer>> sum_fw = new TreeMap<String, TreeMap<Integer, Integer>>();
// From collected data, several kinds of results:
// - a list of how well each chain scores: its index position in the sorted list of scores of one to many.
// - a list of how well each chain scores relative to family: the lowest (best) index position of a lineage of the same family in the sorted list of scores.
sb.append("List of scoring indices for each (starting at index 1, aka best possible score):\n");
for (final CITuple ci : cin) {
// sort indices in place
Collections.sort(ci.list);
// count occurrences of each scoring index
// lowest possible index
int last = 0;
int count = 1;
for (final int i : ci.list) {
if (last == i)
count++;
else {
sb.append(ci.title).append(' ').append(last + 1).append(' ').append(count).append('\n');
// reset
last = i;
count = 1;
}
// global count of occurrences
final Integer oi = new Integer(i);
sum.put(oi, (sum.containsKey(oi) ? sum.get(oi) : 0) + 1);
// Same thing but not for all lineages, but only for lineages within a family:
// extract family name from title: read the first continuous string of capital letters
int u = 0;
for (; u < ci.title.length(); u++) {
if (!Character.isUpperCase(ci.title.charAt(u)))
break;
}
final String family_name = ci.title.substring(0, u);
TreeMap<Integer, Integer> sfw = sum_fw.get(family_name);
if (null == sfw) {
sfw = new TreeMap<Integer, Integer>();
sum_fw.put(family_name, sfw);
}
sfw.put(oi, (sfw.containsKey(oi) ? sfw.get(oi) : 0) + 1);
}
if (0 != count)
sb.append(ci.title).append(' ').append(last + 1).append(' ').append(count).append('\n');
// find the very-off ones:
if (last > 6) {
Utils.log2("BAD index " + last + " for chain " + ci.title + " " + ci.chain.getRoot() + " of project " + ci.chain.getRoot().getProject());
}
}
sb.append("===============================\n");
// / family score:
for (final CITuple ci : cin_f) {
// sort indices in place
Collections.sort(ci.list);
// count occurrences of each scoring index
// lowest possible index
int last = 0;
int count = 1;
for (final int i : ci.list) {
if (last == i)
count++;
else {
// reset
last = i;
count = 1;
}
// global count of occurrences
final Integer oi = new Integer(i);
sum_f.put(oi, (sum_f.containsKey(oi) ? sum_f.get(oi) : 0) + 1);
}
}
sb.append("===============================\n");
// - a summarizing histogram that collects how many 1st, how many 2nd, etc. in total, normalized to total number of one-to-many matches performed (i.e. the number of scoring indices recorded.)
//
{
sb.append("Global count of index ocurrences:\n");
int total = 0;
int top2 = 0;
int top5 = 0;
for (final Map.Entry<Integer, Integer> e : sum.entrySet()) {
sb.append(e.getKey()).append(' ').append(e.getValue()).append('\n');
total += e.getValue();
if (e.getKey() < 2)
top2 += e.getValue();
if (e.getKey() < 5)
top5 += e.getValue();
}
sb.append("total: ").append(total).append('\n');
sb.append("top1: ").append(sum.get(sum.firstKey()) / (float) total).append('\n');
sb.append("top2: ").append(top2 / (float) total).append('\n');
sb.append("top5: ").append(top5 / (float) total).append('\n');
sb.append("===============================\n");
}
sb.append("Family-wise count of index ocurrences:\n");
for (final Map.Entry<String, TreeMap<Integer, Integer>> fe : sum_fw.entrySet()) {
int total = 0;
int top5 = 0;
for (final Map.Entry<Integer, Integer> e : fe.getValue().entrySet()) {
sb.append(fe.getKey()).append(' ').append(e.getKey()).append(' ').append(e.getValue()).append('\n');
total += e.getValue();
if (e.getKey() < 5)
top5 += e.getValue();
}
sb.append("total: ").append(total).append('\n');
sb.append("top1: ").append(fe.getValue().get(fe.getValue().firstKey()) / (float) total).append('\n');
sb.append("top5: ").append(top5 / (float) total).append('\n');
}
sb.append("===============================\n");
// - the percent of first score being the correct one:
double first = 0;
double first_5 = 0;
double all = 0;
for (final Map.Entry<Integer, Integer> e : sum.entrySet()) {
final int k = e.getKey();
final int a = e.getValue();
all += a;
if (0 == k)
first = a;
if (k < 5)
first_5 += a;
}
// STORE
this.result = new double[] { // Top one ratio
first / all, // Top 5 ratio
first_5 / all };
sb.append("Global count of index occurrences family-wise:\n");
for (final Map.Entry<Integer, Integer> e : sum_f.entrySet()) {
sb.append(e.getKey()).append(' ').append(e.getValue()).append('\n');
}
sb.append("===============================\n");
// - a summarizing histogram of how well each chain scores (4/4, 3/4, 2/4, 1/4, 0/4 only for those that have 4 homologous members.)
// Must consider that there are 5 projects taken in pairs with repetition.
sb.append("A summarizing histogram of how well each chain scores, for those that have 4 homologous members. It's the number of 1st scores (zeroes) versus the total number of scores:\n");
// First, classify them in having 4, 3, 2, 1
// For 5 brains: 5! / (5-2)! = 5 * 4 = 20 --- 5 elements taken in groups of 2, where order matters
// For 4 brains: 4! / (4-2)! = 4 * 3 = 12
// For 3 brains: 3! / (3-2)! = 3 * 2 = 6;
final TreeMap<Integer, ArrayList<String>> hsc = new TreeMap<Integer, ArrayList<String>>();
for (final CITuple ci : cin) {
final int size = ci.list.size();
ArrayList<String> al = hsc.get(size);
if (null == al) {
al = new ArrayList<String>();
hsc.put(size, al);
}
// Count the number of 0s -- top scoring
int count = 0;
for (final Integer i : ci.list) {
if (0 == i)
count++;
else
break;
}
al.add(new StringBuffer(ci.title).append(" =").append(count).append('/').append(ci.list.size()).append('\n').toString());
}
// Then just print:
for (final Map.Entry<Integer, ArrayList<String>> e : hsc.entrySet()) {
sb.append("For ").append(e.getKey()).append(" matches:\n");
for (final String s : e.getValue()) sb.append(s);
}
sb.append("=========================\n");
// Family-wise, count the number of zeros per family:
sb.append("Number of top scoring per family:\n");
final TreeMap<String, String> family_scores = new TreeMap<String, String>();
for (final CITuple ci : cin_f) {
int count = 0;
for (final Integer i : ci.list) {
if (0 == i)
count++;
else
// ci.list is sorted
break;
}
family_scores.put(ci.title, new StringBuilder().append(ci.title).append(" =").append(count).append('/').append(ci.list.size()).append('\n').toString());
}
// Now print sorted by family name:
for (final String s : family_scores.values()) {
sb.append(s);
}
sb.append("=========================\n");
if (weka_classify) {
sb.append("Decision tree:\n");
sb.append("Expected good matches: " + exp_good.get() + "\n");
sb.append("Expected bad matches: " + exp_bad.get() + "\n");
sb.append("Observed good matches: " + obs_good.get() + "\n");
sb.append("Observed bad matches: " + obs_wrong.get() + "\n");
sb.append("Observed well classified bad ones: " + obs_well_classified_bad_ones.get() + "\n");
sb.append("Observed bad classified good ones: " + obs_bad_classified_good_ones.get() + "\n");
sb.append("Not found, so skipped: " + not_found.get() + "\n");
sb.append("Already classified: " + already_classified.get() + "\n");
sb.append("=========================\n");
}
if (output_arff) {
Utils.log(sb.toString());
} else {
Utils.log2(sb.toString());
}
} catch (final Exception e) {
e.printStackTrace();
} finally {
finishedWorking();
}
}
};
return Bureaucrat.createAndStart(worker, p);
}
Also used :
ArrayList(java.util.ArrayList)
TreeSet(java.util.TreeSet)
Method(java.lang.reflect.Method)
Project(ini.trakem2.Project)
AtomicInteger(java.util.concurrent.atomic.AtomicInteger)
File(java.io.File)
Map(java.util.Map)
HashMap(java.util.HashMap)
TreeMap(java.util.TreeMap)
CATAParameters(ini.trakem2.analysis.Compare.CATAParameters)
Callable(java.util.concurrent.Callable)
Worker(ini.trakem2.utils.Worker)
TreeMap(java.util.TreeMap)
AtomicInteger(java.util.concurrent.atomic.AtomicInteger)
Editions(ini.trakem2.vector.Editions)
VectorString3D(ini.trakem2.vector.VectorString3D)
ExecutorService(java.util.concurrent.ExecutorService)
Future(java.util.concurrent.Future)
Example 32 with Project
use of org.openstack4j.model.identity.v3.Project in project TrakEM2 by trakem2.
the class ProjectTiler method createRetiledSibling.
/**
* Take a {@link Project}, a size for the image tiles, and a target directory,
* and create a new copy of the current project in that folder but with the underlying
* images converted to tiles with a translation-only transform (saved as zipped TIFFs,
* with extension ".tif.zip").
* The new, returned {@link Project} represents the given project but with much
* simpler transformations (just translation) for the images and a defined size for
* the latter, which helps a lot regarding storage space of the XML (and parsing and
* saving time) and performance when browsing layers (keep in mind that, for a 32k x 32k image,
* at 100% zoom one would have to load a 32k x 32k image and render just a tiny bit
* of it). The copied Project preserves the ID of the {@link Layer}s of the original
* {@link Project}, as well as the dimensions; this means the copy is a sibling of
* the original, and it is possible to send segmentations from one to the other "as is"
* (directly, without having to transform along with the images which would not be possible).
*
* Image files are stored as
*
* The non-image objects of the given project are copied into the new project as well.
*
* @param srcProject The project to create a sibling of.
* @param targetDirectory The directory in which to create all the necessary data and mipmap folders for the new Project.
* @param tileWidth The width of the tiles to create for the data of the new project.
* @param tileHeight The height of the tiles.
* @param exportImageType Any of {@link ImagePlus#GRAY8}, {@link ImagePlus#GRAY16} or {@link ImagePlus#COLOR_RGB}, otherwise an {@link IllegalArgumentException} is thrown.
* @param onlyVisibleImages Whether to consider visible images only.
* @param nExportThreads Number of layers to export in parallel. Use a small number when original images are huge (such as larger than 4096 x 4096 pixels).
* @param createMipMaps Whether to generate the mipmaps when done or not.
*
* @throws Exception IllegalArgumentException When {@code exportImageType} is not {@link ImagePlus#GRAY16} or {@link ImagePlus#COLOR_RGB}, or when the directory exists and cannot be written to.
*/
public static final Project createRetiledSibling(final Project srcProject, final String targetDirectory, final int tileWidth, final int tileHeight, final int exportImageType, final boolean onlyVisibleImages, final int nExportThreads, final boolean createMipMaps) throws Exception {
// Validate exportImageType
switch(exportImageType) {
case ImagePlus.GRAY8:
case ImagePlus.GRAY16:
case ImagePlus.COLOR_RGB:
break;
default:
throw new IllegalArgumentException("Can only accept GRAY8, GRAY16 or COLOR_RGB as values for 'exportImageType'!");
}
// Validate targetDirectory
final File fdir = new File(targetDirectory);
if (fdir.exists()) {
if (!fdir.isDirectory() || !fdir.canWrite())
throw new IllegalArgumentException("Invalid directory: not a directory or cannot write to: " + targetDirectory);
} else {
if (!fdir.mkdirs()) {
throw new IllegalArgumentException("Cannot create directory at: " + targetDirectory);
}
}
final String targetDir = Utils.fixDir(targetDirectory);
// Create "data" directory
final String dataDir = new StringBuilder(targetDir).append("data/").toString();
final File fDataDir = new File(dataDir);
if (fDataDir.exists() && (!fDataDir.isDirectory() || !fDataDir.canWrite())) {
throw new IllegalArgumentException("Cannot create or write to 'data' directory in the targetDirectory at: " + targetDir);
} else {
fDataDir.mkdir();
}
// Create new Project, plain, without any automatic creation of a Layer or a Display
final Project newProject = Project.newFSProject("blank", null, targetDir, false);
final LayerSet newLayerSet = newProject.getRootLayerSet();
newLayerSet.setCalibration(srcProject.getRootLayerSet().getCalibrationCopy());
if (!createMipMaps) {
Utils.log("MipMaps are DISABLED:\n --> When done, right-click and choose 'Display - Properties...' and enable mipmaps,\n and then run 'Project - Regenerate all mipmaps'\n");
newProject.getLoader().setMipMapsRegeneration(false);
Utils.log("mipmaps enabled? " + newProject.getLoader().isMipMapsRegenerationEnabled());
}
// Copy the Template Tree of types
newProject.resetRootTemplateThing(srcProject.getRootTemplateThing().clone(newProject, true), null);
for (final TemplateThing tt : newProject.getRootTemplateThing().getUniqueTypes(new HashMap<String, TemplateThing>()).values()) {
newProject.addUniqueType(tt);
}
// Clone layers with the exact same IDs, so that the two projects are siblings at the layer-level:
// (Being siblings allows for treelines, arealists, etc. to be transferred from one to another "as is").
final List<Layer> srcLayers = srcProject.getRootLayerSet().getLayers();
final List<Layer> newLayers = new ArrayList<Layer>();
for (final Layer srcLayer : srcLayers) {
final Layer newLayer = new Layer(newProject, srcLayer.getId(), srcLayer.getZ(), srcLayer.getThickness());
// to update the ID generator in FSLoader
newLayer.addToDatabase();
newLayerSet.add(newLayer);
newLayers.add(newLayer);
newProject.getRootLayerThing().addChild(new LayerThing(newProject.getRootLayerThing().getChildTemplate("layer"), newProject, newLayer));
}
newProject.getLayerTree().rebuild();
// Update the LayerSet
newLayerSet.setDimensions(srcProject.getRootLayerSet().getLayerWidth(), srcProject.getRootLayerSet().getLayerHeight(), LayerSet.NORTHWEST);
Display.updateLayerScroller(newLayerSet);
Display.update(newLayerSet);
// Copy template from the src Project
// (It's done after creating layers so the IDs will not collide with those of the Layers)
newProject.resetRootTemplateThing(srcProject.getRootTemplateThing().clone(newProject, false), null);
// Export tiles as new Patch instances, creating new image files in disk
final int numThreads = Math.max(1, Math.min(nExportThreads, Runtime.getRuntime().availableProcessors()));
int i = 0;
for (final Layer srcLayer : srcLayers) {
Utils.log("Processing layer " + (i + 1) + "/" + srcLayers.size() + " -- " + new Date());
final int layerIndex = i++;
// Create subDirectory
final String dir = dataDir + "/" + layerIndex + "/";
new File(dir).mkdir();
// Create a new Layer with the same Z and thickness
final Layer newLayer = newLayers.get(layerIndex);
// Export layer tiles
final ArrayList<Patch> patches = new ArrayList<Patch>();
if (ImagePlus.GRAY16 == exportImageType) {
Process.progressive(ExportUnsignedShort.exportTiles(srcLayer, tileWidth, tileHeight, onlyVisibleImages), new CountingTaskFactory<Callable<ExportedTile>, Patch>() {
public Patch process(final Callable<ExportedTile> c, final int index) {
try {
// Create the tile
final ExportedTile t = c.call();
// Store the file
final String title = layerIndex + "-" + index;
final String path = dir + title + ".tif.zip";
final ImagePlus imp = new ImagePlus(title, t.sp);
if (!new FileSaver(imp).saveAsZip(path)) {
throw new Exception("Could not save tile: " + path);
}
// Create a Patch
final Patch patch = new Patch(newProject, title, t.x, t.y, imp);
patch.setLocked(true);
newProject.getLoader().addedPatchFrom(path, patch);
return patch;
} catch (Exception e) {
IJError.print(e);
return null;
}
}
}, patches, numThreads);
} else {
// GRAY8 or COLOR_RGB: created from mipmaps
Process.progressive(tileSequence(srcLayer, tileWidth, tileHeight, onlyVisibleImages), new CountingTaskFactory<Rectangle, Patch>() {
@Override
public Patch process(final Rectangle bounds, final int index) {
try {
// Create the tile
final ImagePlus imp = srcLayer.getProject().getLoader().getFlatImage(srcLayer, bounds, 1.0, -1, exportImageType, Patch.class, null, false, Color.black);
final String title = layerIndex + "-" + index;
imp.setTitle(title);
final String path = dir + title + ".tif.zip";
if (!new FileSaver(imp).saveAsZip(path)) {
throw new Exception("Could not save tile: " + path);
}
// Create a Patch
final Patch patch = new Patch(newProject, title, bounds.x, bounds.y, imp);
patch.setLocked(true);
newProject.getLoader().addedPatchFrom(path, patch);
return patch;
} catch (Exception e) {
IJError.print(e);
return null;
}
}
}, patches, numThreads);
}
// Add all Patches to the new Layer
for (final Patch p : patches) {
newLayer.add(p);
}
}
// Copy all segmentations "As is"
final ProjectThing root = srcProject.getRootProjectThing();
if (null != root.getChildren() && !root.getChildren().isEmpty()) {
final ProjectThing source_pt = srcProject.getRootProjectThing().getChildren().get(0);
// "As is"
final int transfer_mode = 0;
final ProjectThing landing_parent = newProject.getRootProjectThing();
srcProject.getProjectTree().rawSendToSiblingProject(source_pt, transfer_mode, newProject, landing_parent);
}
// Copy all floating text labels
i = 0;
for (final Layer srcLayer : srcLayers) {
for (final DLabel srcLabel : srcLayer.getAll(DLabel.class)) {
newLayers.get(i++).add(srcLabel.clone(newProject, false));
}
}
if (createMipMaps) {
final LinkedList<Future<?>> fus = new LinkedList<Future<?>>();
final int batch = Runtime.getRuntime().availableProcessors();
for (final Layer newLayer : newLayers) {
for (final Patch p : newLayer.getAll(Patch.class)) {
fus.add(p.updateMipMaps());
// Don't build-up too much
if (fus.size() > batch * 3) {
while (fus.size() > batch) {
try {
fus.removeFirst().get();
} catch (Exception e) {
IJError.print(e);
}
}
}
}
}
Utils.wait(fus);
}
// Save:
newProject.saveAs(targetDir + "exported.xml", false);
return newProject;
}
Also used :
HashMap(java.util.HashMap)
ArrayList(java.util.ArrayList)
Rectangle(java.awt.Rectangle)
Callable(java.util.concurrent.Callable)
FileSaver(ij.io.FileSaver)
ProjectThing(ini.trakem2.tree.ProjectThing)
LayerSet(ini.trakem2.display.LayerSet)
LayerThing(ini.trakem2.tree.LayerThing)
Layer(ini.trakem2.display.Layer)
ImagePlus(ij.ImagePlus)
Date(java.util.Date)
LinkedList(java.util.LinkedList)
Project(ini.trakem2.Project)
DLabel(ini.trakem2.display.DLabel)
TemplateThing(ini.trakem2.tree.TemplateThing)
Future(java.util.concurrent.Future)
File(java.io.File)
Patch(ini.trakem2.display.Patch)
ExportedTile(mpicbg.trakem2.transform.ExportedTile)
Example 33 with Project
use of org.openstack4j.model.identity.v3.Project in project TrakEM2 by trakem2.
the class TMLHandler method startElement.
public void startElement(String namespace_URI, String local_name, String qualified_name, Attributes attributes) throws SAXException {
if (null == loader)
return;
// Utils.log2("startElement: " + qualified_name);
this.counter++;
if (0 == counter % 100) {
// davi-experimenting: don't talk so much when you have > 600,000 patches to load!
Utils.showStatus("Loading " + counter, false);
}
try {
// failsafe:
qualified_name = qualified_name.toLowerCase();
final HashMap<String, String> ht_attributes = new HashMap<String, String>();
for (int i = attributes.getLength() - 1; i > -1; i--) {
ht_attributes.put(attributes.getQName(i).toLowerCase(), attributes.getValue(i));
}
// get the id, which whenever possible it's the id of the encapsulating Thing object. The encapsulated object id is the oid
// The type is specified by the qualified_name
Thing thing = null;
if (0 == qualified_name.indexOf("t2_")) {
if (qualified_name.equals("t2_display")) {
// store for later, until the layers exist
if (open_displays)
al_displays.add(ht_attributes);
} else {
// a Layer, LayerSet or Displayable object
thing = makeLayerThing(qualified_name, ht_attributes);
if (null != thing) {
if (null == root_lt && thing.getObject() instanceof LayerSet) {
root_lt = (LayerThing) thing;
}
}
}
} else if (qualified_name.equals("project")) {
if (null != this.root_pt) {
Utils.log("WARNING: more than one project definitions.");
return;
}
// Create the project
this.project = new Project(Long.parseLong(ht_attributes.remove("id")), ht_attributes.remove("title"));
// temp, but will be the same anyway
this.project.setTempLoader(this.loader);
this.project.parseXMLOptions(ht_attributes);
// register id
this.project.addToDatabase();
String title = ht_attributes.get("title");
if (null != title)
this.project.setTitle(title);
// Add all unique TemplateThing types to the project
for (Iterator<TemplateThing> it = root_tt.getUniqueTypes(new HashMap<String, TemplateThing>()).values().iterator(); it.hasNext(); ) {
this.project.addUniqueType(it.next());
}
this.project.addUniqueType(this.project_tt);
this.root_pt = new ProjectThing(this.project_tt, this.project, this.project);
// Add a project pointer to all template things
this.root_tt.addToDatabase(this.project);
thing = root_pt;
} else if (qualified_name.startsWith("ict_transform") || qualified_name.startsWith("iict_transform")) {
makeCoordinateTransform(qualified_name, ht_attributes);
} else if (!qualified_name.equals("trakem2")) {
// Any abstract object
thing = makeProjectThing(qualified_name, ht_attributes);
}
if (null != thing) {
// get the previously open thing and add this new_thing to it as a child
int size = al_open.size();
if (size > 0) {
Thing parent = al_open.get(size - 1);
parent.addChild(thing);
// Utils.log2("Adding child " + thing + " to parent " + parent);
}
// add the new thing as open
al_open.add(thing);
}
} catch (Exception e) {
IJError.print(e);
skip = true;
}
}
Also used :
Project(ini.trakem2.Project)
HashMap(java.util.HashMap)
LayerSet(ini.trakem2.display.LayerSet)
Iterator(java.util.Iterator)
LayerThing(ini.trakem2.tree.LayerThing)
TemplateThing(ini.trakem2.tree.TemplateThing)
ProjectThing(ini.trakem2.tree.ProjectThing)
Thing(ini.trakem2.tree.Thing)
ProjectThing(ini.trakem2.tree.ProjectThing)
SAXException(org.xml.sax.SAXException)
SAXParseException(org.xml.sax.SAXParseException)
Example 34 with Project
use of org.openstack4j.model.identity.v3.Project in project TrakEM2 by trakem2.
the class ProjectTree method sendToSiblingProject.
/**
* When two or more people work on the same XML file, images may be the same but segmentations and the transformations of the images may diverge.
* This function provides the means to send VectorData instances, wrapped in tree nodes, from one project to another,
* transforming the VectorData as appropriate to fall onto the same locations on the images.
* The ids of the copied objects will be new and unique for the target project.
* A dialog opens asking for options.
*/
@SuppressWarnings("unchecked")
public boolean sendToSiblingProject(final DefaultMutableTreeNode node) {
ArrayList<Project> ps = Project.getProjects();
if (1 == ps.size()) {
Utils.log("There aren't any other projects open!");
return false;
}
final ProjectThing pt = (ProjectThing) node.getUserObject();
if (pt.getTemplate().getType().equals("project")) {
Utils.log("Cannot transfer the project node.");
return false;
}
final ArrayList<Project> psother = new ArrayList<Project>(ps);
psother.remove(this.project);
ps = null;
// Find all potential landing nodes for this node: those with a TemplateThing type like the parent of node:
final String parent_type = ((ProjectThing) pt.getParent()).getTemplate().getType();
final List<ProjectThing> landing_pt = new ArrayList<ProjectThing>(psother.get(0).getRootProjectThing().findChildrenOfTypeR(parent_type));
final Comparator<ProjectThing> comparator = new Comparator<ProjectThing>() {
public int compare(ProjectThing t1, ProjectThing t2) {
return t1.toString().compareTo(t2.toString());
}
public boolean equals(Object o) {
return this == o;
}
};
Collections.sort(landing_pt, comparator);
String[] landing = new String[landing_pt.size()];
int next = 0;
if (landing_pt.isEmpty()) {
landing = new String[] { "-- NONE --" };
} else
for (ProjectThing t : landing_pt) landing[next++] = t.toString();
// Suggest the first potential landing node that has the same title
String parentTitle = pt.getParent().toString();
int k = 0;
boolean matched = false;
// First search for exact match
for (final String candidate : landing) {
if (candidate.equals(parentTitle)) {
matched = true;
break;
}
k += 1;
}
// If not matched, find one that contains the string
if (!matched) {
k = 0;
for (final String candidate : landing) {
if (-1 != candidate.indexOf(parentTitle)) {
matched = true;
break;
}
k += 1;
}
}
if (!matched) {
k = 0;
}
// Ask:
GenericDialog gd = new GenericDialog("Send to sibling project");
gd.addMessage("Transfering node: " + pt);
final String[] trmode = new String[] { "As is", "Transformed as the images" };
gd.addChoice("Transfer:", trmode, trmode[0]);
String[] ptitles = new String[psother.size()];
for (int i = 0; i < ptitles.length; i++) ptitles[i] = psother.get(i).toString();
gd.addChoice("Target project:", ptitles, ptitles[0]);
gd.addChoice("Landing node:", landing, landing[k]);
final Vector<Choice> vc = (Vector<Choice>) gd.getChoices();
final Choice choice_project = vc.get(vc.size() - 2);
final Choice choice_landing = vc.get(vc.size() - 1);
choice_project.addItemListener(new ItemListener() {
public void itemStateChanged(ItemEvent ie) {
landing_pt.clear();
landing_pt.addAll(psother.get(choice_project.getSelectedIndex()).getRootProjectThing().findChildrenOfTypeR(parent_type));
Collections.sort(landing_pt, comparator);
choice_landing.removeAll();
if (landing_pt.isEmpty()) {
choice_landing.add("-- NONE --");
} else
for (ProjectThing t : landing_pt) choice_landing.add(t.toString());
}
});
gd.showDialog();
if (gd.wasCanceled())
return false;
if (choice_landing.getSelectedItem().equals("-- NONE --")) {
Utils.log("No valid landing nodes!");
return false;
}
final int transfer_mode = gd.getNextChoiceIndex();
final Project target_project = psother.get(gd.getNextChoiceIndex());
final ProjectThing landing_parent = landing_pt.get(gd.getNextChoiceIndex());
return rawSendToSiblingProject(pt, transfer_mode, target_project, landing_parent);
}
Also used :
ItemEvent(java.awt.event.ItemEvent)
Choice(java.awt.Choice)
ArrayList(java.util.ArrayList)
Comparator(java.util.Comparator)
Project(ini.trakem2.Project)
GenericDialog(ij.gui.GenericDialog)
DBObject(ini.trakem2.persistence.DBObject)
ItemListener(java.awt.event.ItemListener)
Vector(java.util.Vector)
Example 35 with Project
use of org.openstack4j.model.identity.v3.Project in project TrakEM2 by trakem2.
the class Search method makeGUI.
private void makeGUI() {
// create GUI if not there
if (null == search_frame) {
search_frame = ControlWindow.createJFrame("Search Regular Expressions");
search_frame.addWindowListener(new WindowAdapter() {
public void windowClosing(WindowEvent we) {
instance.destroy();
}
});
search_tabs = new JTabbedPane(JTabbedPane.TOP);
kl = new KeyAdapter() {
public void keyPressed(KeyEvent ke) {
tryCloseTab(ke);
}
};
search_tabs.addKeyListener(kl);
search_field = new JTextField(14);
search_field.addKeyListener(new VKEnterListener());
GridBagLayout gb = new GridBagLayout();
GridBagConstraints c = new GridBagConstraints();
JPanel all = new JPanel();
all.setLayout(gb);
all.setPreferredSize(new Dimension(600, 400));
JButton b = new JButton("Search");
b.addActionListener(new ButtonListener());
pulldown = new JComboBox(new String[] { "All", "All displayables", "Labels", "Images", "Area Lists", "Profiles", "Pipes", "Balls", "Layers", "Dissectors", "Polylines", "Treelines", "AreaTrees", "Connectors" });
List<Project> ps = Project.getProjects();
String[] sps = new String[ps.size()];
int k = 0;
for (final Project p : ps) sps[k++] = p.getTitle();
this.projects = new JComboBox(sps);
Display front = Display.getFront();
if (null != front)
this.projects.setSelectedIndex(ps.indexOf(front.getProject()));
c.gridx = 0;
c.gridy = 0;
c.fill = GridBagConstraints.HORIZONTAL;
c.weightx = 1;
c.insets = new Insets(4, 10, 5, 2);
gb.setConstraints(search_field, c);
all.add(search_field);
c.gridx = 1;
c.weightx = 0;
c.insets = new Insets(4, 2, 5, 10);
gb.setConstraints(b, c);
all.add(b);
c.gridx = 2;
gb.setConstraints(pulldown, c);
all.add(pulldown);
c.gridx = 3;
gb.setConstraints(projects, c);
all.add(projects);
c.gridx = 0;
c.gridy = 1;
c.gridwidth = 4;
c.weightx = 1;
c.weighty = 1;
c.fill = GridBagConstraints.BOTH;
c.insets = new Insets(0, 0, 0, 0);
gb.setConstraints(search_tabs, c);
all.add(search_tabs);
search_frame.getContentPane().add(all);
search_frame.pack();
javax.swing.SwingUtilities.invokeLater(new Runnable() {
public void run() {
ij.gui.GUI.center(search_frame);
search_frame.setVisible(true);
}
});
} else {
search_frame.toFront();
}
}
Also used :
JPanel(javax.swing.JPanel)
GridBagConstraints(java.awt.GridBagConstraints)
Insets(java.awt.Insets)
GridBagLayout(java.awt.GridBagLayout)
JComboBox(javax.swing.JComboBox)
JTabbedPane(javax.swing.JTabbedPane)
KeyAdapter(java.awt.event.KeyAdapter)
JButton(javax.swing.JButton)
WindowAdapter(java.awt.event.WindowAdapter)
Dimension(java.awt.Dimension)
JTextField(javax.swing.JTextField)
KeyEvent(java.awt.event.KeyEvent)
Project(ini.trakem2.Project)
WindowEvent(java.awt.event.WindowEvent)
Display(ini.trakem2.display.Display)
Aggregations
Project (ini.trakem2.Project)26
ArrayList (java.util.ArrayList)13
Project (org.openstack4j.model.identity.v3.Project)9
File (java.io.File)7
HashMap (java.util.HashMap)7
ImagePlus (ij.ImagePlus)6
Worker (ini.trakem2.utils.Worker)6
Map (java.util.Map)6
LayerSet (ini.trakem2.display.LayerSet)5
ProjectThing (ini.trakem2.tree.ProjectThing)5
HashSet (java.util.HashSet)5
DBObject (ini.trakem2.persistence.DBObject)4
VectorString3D (ini.trakem2.vector.VectorString3D)4
GenericDialog (ij.gui.GenericDialog)3
Patch (ini.trakem2.display.Patch)3
Rectangle (java.awt.Rectangle)3
ImageJ (ij.ImageJ)2
Roi (ij.gui.Roi)2
YesNoCancelDialog (ij.gui.YesNoCancelDialog)2
DirectoryChooser (ij.io.DirectoryChooser)2
