Search in sources :

Example 16 with FileStitcher

use of loci.formats.FileStitcher in project bioformats by openmicroscopy.

the class FormatReaderTestFactory method createInstances.

// -- TestNG factory methods --
@Factory
public Object[] createInstances() {
    List<String> files = new ArrayList<String>();
    // parse explicit filename, if any
    final String nameProp = "testng.filename";
    String filename = getProperty(nameProp);
    if (filename != null && !new File(filename).exists()) {
        LOGGER.error("Invalid filename: {}", filename);
        return new Object[0];
    }
    String baseDir = null;
    String[] validSubdirs = null;
    if (filename == null) {
        // parse base directory
        final String baseDirProp = "testng.directory";
        baseDir = getProperty(baseDirProp);
        if (baseDir == null) {
            baseDir = getProperty("testng.directory-prefix");
            String dirList = System.getProperty("testng.directory-list");
            try {
                validSubdirs = DataTools.readFile(dirList).split("\n");
            } catch (IOException e) {
                LOGGER.debug("", e);
            }
        }
        // Return early if no base directory is supplied
        if (baseDir == null) {
            LOGGER.error("No base directory specified.");
            LOGGER.error("Please specify a directory containing files to test:");
            LOGGER.error("   ant -D{}=\"/path/to/data\" test-all", baseDirProp);
            return new Object[0];
        }
        // Test base directory validity
        File baseDirFile = new File(baseDir);
        if (!baseDirFile.isDirectory()) {
            LOGGER.info("Directory: {}", baseDir);
            LOGGER.info("  exists?: {}", baseDirFile.exists());
            LOGGER.info("  readable?: {}", baseDirFile.canRead());
            LOGGER.info("  is a directory?: {}", baseDirFile.isDirectory());
            LOGGER.error("Please specify a directory containing files to test:");
            LOGGER.error("   ant -D{}=\"/path/to/data\" test-all", baseDirProp);
            return new Object[0];
        }
        // check for an alternate configuration directory
        final String configDirProperty = "testng.configDirectory";
        String configDir = getProperty(configDirProperty);
        LOGGER.info("testng.directory = {}", baseDir);
        if (configDir != null) {
            LOGGER.info("testng.configDirectory = {}", configDir);
        }
        FormatReaderTest.configTree = new ConfigurationTree(baseDir, configDir);
    }
    // parse multiplier
    final String multProp = "testng.multiplier";
    String mult = getProperty(multProp);
    float multiplier = 1;
    if (mult != null) {
        try {
            multiplier = Float.parseFloat(mult);
        } catch (NumberFormatException exc) {
            LOGGER.warn("Invalid multiplier: {}", mult);
        }
    }
    LOGGER.info("testng.multiplier = {}", multiplier);
    // detect whether or not the map the files into memory
    final String inMemoryProp = "testng.in-memory";
    String inMemoryValue = getProperty(inMemoryProp);
    boolean inMemory = Boolean.parseBoolean(inMemoryValue);
    LOGGER.info("testng.in-memory = {}", inMemory);
    // check for an alternate top level configuration file
    final String toplevelConfig = "testng.toplevel-config";
    String configFile = getProperty(toplevelConfig);
    if (configFile != null) {
        LOGGER.info("testng.toplevel-config = {}", configFile);
    }
    // check for a configuration file suffix
    final String configSuffixProperty = "testng.configSuffix";
    String configSuffix = getProperty(configSuffixProperty);
    if (configSuffix == null) {
        configSuffix = "";
    }
    // display local information
    LOGGER.info("user.language = {}", System.getProperty("user.language"));
    LOGGER.info("user.country = {}", System.getProperty("user.country"));
    // detect maximum heap size
    long maxMemory = Runtime.getRuntime().maxMemory() >> 20;
    LOGGER.info("Maximum heap size = {} MB", maxMemory);
    if (filename == null) {
        // scan for files
        System.out.println("Scanning for files...");
        long start = System.currentTimeMillis();
        try {
            TestTools.getFiles(baseDir, files, FormatReaderTest.configTree, configFile, validSubdirs, configSuffix);
        } catch (Exception e) {
            LOGGER.info("Failed to retrieve complete list of files", e);
        }
        long end = System.currentTimeMillis();
        double time = (end - start) / 1000.0;
        LOGGER.info(TestTools.DIVIDER);
        LOGGER.info("Total files: {}", files.size());
        long avg = (end - start);
        if (files.size() > 0)
            avg /= files.size();
        LOGGER.info("Scan time: {} s ({} ms/file)", time, avg);
        LOGGER.info(TestTools.DIVIDER);
    } else {
        files.add(filename);
    }
    // remove duplicates
    Set<String> fileSet = new LinkedHashSet<String>();
    Map<String, String> originalPath = new HashMap<String, String>();
    for (String s : files) {
        String canonicalPath;
        try {
            canonicalPath = (new File(s)).getCanonicalPath();
        } catch (IOException e) {
            LOGGER.warn("Could not get canonical path for {}", s);
            canonicalPath = s;
        }
        fileSet.add(canonicalPath);
        originalPath.put(canonicalPath, s);
    }
    Set<String> minimalFiles = new LinkedHashSet<String>();
    FileStitcher reader = new FileStitcher();
    Set<String> failingIds = new LinkedHashSet<String>();
    while (!fileSet.isEmpty()) {
        String file = fileSet.iterator().next();
        try {
            reader.setId(file);
        } catch (Exception e) {
            LOGGER.error("setId(\"{}\") failed", file, e);
            failingIds.add(file);
            fileSet.remove(file);
            continue;
        }
        try {
            String[] usedFiles = reader.getUsedFiles();
            Set<String> auxFiles = new LinkedHashSet<String>();
            for (String s : usedFiles) {
                auxFiles.add((new File(s)).getCanonicalPath());
            }
            fileSet.removeAll(auxFiles);
            String masterFile = reader.getCurrentFile();
            auxFiles.remove(masterFile);
            minimalFiles.removeAll(auxFiles);
            minimalFiles.add(masterFile);
        } catch (Exception e) {
            LOGGER.warn("Could not determine duplicate status for {}", file, e);
            minimalFiles.add(file);
        } finally {
            fileSet.remove(file);
            try {
                reader.close();
            } catch (IOException e) {
            }
        }
    }
    if (!failingIds.isEmpty()) {
        String msg = String.format("setId failed on %s", failingIds);
        LOGGER.error(msg);
        throw new RuntimeException(msg);
    }
    files = new ArrayList<String>();
    for (String s : minimalFiles) {
        if (!originalPath.containsKey(s)) {
            String msg = "No match found for " + s;
            LOGGER.error(msg);
            throw new RuntimeException(msg);
        }
        files.add(originalPath.get(s));
    }
    // create test class instances
    System.out.println("Building list of tests...");
    Object[] tests = new Object[files.size()];
    for (int i = 0; i < tests.length; i++) {
        String id = (String) files.get(i);
        try {
            if (FormatReaderTest.configTree.get(id) == null) {
                LOGGER.error("{} not configured.", id);
            }
        } catch (Exception e) {
            LOGGER.warn("", e);
        }
        tests[i] = new FormatReaderTest(id, multiplier, inMemory);
    }
    if (tests.length == 1)
        System.out.println("Ready to test " + files.get(0));
    else
        System.out.println("Ready to test " + tests.length + " files");
    return tests;
}
Also used : LinkedHashSet(java.util.LinkedHashSet) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) IOException(java.io.IOException) IOException(java.io.IOException) FileStitcher(loci.formats.FileStitcher) File(java.io.File) Factory(org.testng.annotations.Factory) LoggerFactory(org.slf4j.LoggerFactory)

Example 17 with FileStitcher

use of loci.formats.FileStitcher in project bioformats by openmicroscopy.

the class FormatReaderTest method testUnflattenedPixelsHashes.

@Test(groups = { "all", "pixels", "automated" })
public void testUnflattenedPixelsHashes() {
    if (config == null)
        throw new SkipException("No config tree");
    String testName = "testUnflattenedPixelsHashes";
    if (!initFile())
        result(testName, false, "initFile");
    boolean success = true;
    String msg = null;
    try {
        IFormatReader resolutionReader = new BufferedImageReader(new FileStitcher());
        resolutionReader.setFlattenedResolutions(false);
        resolutionReader.setNormalized(true);
        resolutionReader.setOriginalMetadataPopulated(false);
        resolutionReader.setMetadataFiltered(true);
        resolutionReader.setId(id);
        // check the MD5 of the first plane in each resolution
        for (int i = 0; i < resolutionReader.getSeriesCount() && success; i++) {
            resolutionReader.setSeries(i);
            for (int r = 0; r < resolutionReader.getResolutionCount() && success; r++) {
                resolutionReader.setResolution(r);
                config.setSeries(resolutionReader.getCoreIndex());
                long planeSize = -1;
                try {
                    planeSize = DataTools.safeMultiply32(resolutionReader.getSizeX(), resolutionReader.getSizeY(), resolutionReader.getRGBChannelCount(), FormatTools.getBytesPerPixel(resolutionReader.getPixelType()));
                } catch (IllegalArgumentException e) {
                    continue;
                }
                if (planeSize < 0 || !TestTools.canFitInMemory(planeSize)) {
                    continue;
                }
                String md5 = TestTools.md5(resolutionReader.openBytes(0));
                String expected1 = config.getMD5();
                String expected2 = config.getAlternateMD5();
                if (expected1 == null && expected2 == null) {
                    continue;
                }
                if (!md5.equals(expected1) && !md5.equals(expected2)) {
                    success = false;
                    msg = "series " + i + ", resolution " + r;
                }
            }
        }
        resolutionReader.close();
    } catch (Throwable t) {
        if (TestTools.isOutOfMemory(t)) {
            result(testName, true, "Image too large");
            return;
        }
        LOGGER.info("", t);
        success = false;
    }
    result(testName, success, msg);
}
Also used : IFormatReader(loci.formats.IFormatReader) FileStitcher(loci.formats.FileStitcher) SkipException(org.testng.SkipException) BufferedImageReader(loci.formats.gui.BufferedImageReader) Test(org.testng.annotations.Test)

Aggregations

FileStitcher (loci.formats.FileStitcher)17 IFormatReader (loci.formats.IFormatReader)10 Test (org.testng.annotations.Test)9 ImageReader (loci.formats.ImageReader)6 BufferedImageReader (loci.formats.gui.BufferedImageReader)5 Location (loci.common.Location)4 ServiceException (loci.common.services.ServiceException)4 ChannelFiller (loci.formats.ChannelFiller)4 ChannelSeparator (loci.formats.ChannelSeparator)4 FormatException (loci.formats.FormatException)4 MinMaxCalculator (loci.formats.MinMaxCalculator)4 File (java.io.File)3 IOException (java.io.IOException)3 DependencyException (loci.common.services.DependencyException)3 ChannelMerger (loci.formats.ChannelMerger)3 DimensionSwapper (loci.formats.DimensionSwapper)3 Memoizer (loci.formats.Memoizer)3 ReaderWrapper (loci.formats.ReaderWrapper)3 SkipException (org.testng.SkipException)3 ServiceFactory (loci.common.services.ServiceFactory)2