Examples with FileStitcher loci.formats.FileStitcher used on opensource projects

Example 16 with FileStitcher

use of loci.formats.FileStitcher in project bioformats by openmicroscopy.

the class FormatReaderTestFactory method createInstances.

// -- TestNG factory methods --
@Factory
public Object[] createInstances() {
    List<String> files = new ArrayList<String>();
    // parse explicit filename, if any
    final String nameProp = "testng.filename";
    String filename = getProperty(nameProp);
    if (filename != null && !new File(filename).exists()) {
        LOGGER.error("Invalid filename: {}", filename);
        return new Object[0];
    }
    String baseDir = null;
    String[] validSubdirs = null;
    if (filename == null) {
        // parse base directory
        final String baseDirProp = "testng.directory";
        baseDir = getProperty(baseDirProp);
        if (baseDir == null) {
            baseDir = getProperty("testng.directory-prefix");
            String dirList = System.getProperty("testng.directory-list");
            try {
                validSubdirs = DataTools.readFile(dirList).split("\n");
            } catch (IOException e) {
                LOGGER.debug("", e);
            }
        }
        // Return early if no base directory is supplied
        if (baseDir == null) {
            LOGGER.error("No base directory specified.");
            LOGGER.error("Please specify a directory containing files to test:");
            LOGGER.error("   ant -D{}=\"/path/to/data\" test-all", baseDirProp);
            return new Object[0];
        }
        // Test base directory validity
        File baseDirFile = new File(baseDir);
        if (!baseDirFile.isDirectory()) {
            LOGGER.info("Directory: {}", baseDir);
            LOGGER.info("  exists?: {}", baseDirFile.exists());
            LOGGER.info("  readable?: {}", baseDirFile.canRead());
            LOGGER.info("  is a directory?: {}", baseDirFile.isDirectory());
            LOGGER.error("Please specify a directory containing files to test:");
            LOGGER.error("   ant -D{}=\"/path/to/data\" test-all", baseDirProp);
            return new Object[0];
        }
        // check for an alternate configuration directory
        final String configDirProperty = "testng.configDirectory";
        String configDir = getProperty(configDirProperty);
        LOGGER.info("testng.directory = {}", baseDir);
        if (configDir != null) {
            LOGGER.info("testng.configDirectory = {}", configDir);
        }
        FormatReaderTest.configTree = new ConfigurationTree(baseDir, configDir);
    }
    // parse multiplier
    final String multProp = "testng.multiplier";
    String mult = getProperty(multProp);
    float multiplier = 1;
    if (mult != null) {
        try {
            multiplier = Float.parseFloat(mult);
        } catch (NumberFormatException exc) {
            LOGGER.warn("Invalid multiplier: {}", mult);
        }
    }
    LOGGER.info("testng.multiplier = {}", multiplier);
    // detect whether or not the map the files into memory
    final String inMemoryProp = "testng.in-memory";
    String inMemoryValue = getProperty(inMemoryProp);
    boolean inMemory = Boolean.parseBoolean(inMemoryValue);
    LOGGER.info("testng.in-memory = {}", inMemory);
    // check for an alternate top level configuration file
    final String toplevelConfig = "testng.toplevel-config";
    String configFile = getProperty(toplevelConfig);
    if (configFile != null) {
        LOGGER.info("testng.toplevel-config = {}", configFile);
    }
    // check for a configuration file suffix
    final String configSuffixProperty = "testng.configSuffix";
    String configSuffix = getProperty(configSuffixProperty);
    if (configSuffix == null) {
        configSuffix = "";
    }
    // display local information
    LOGGER.info("user.language = {}", System.getProperty("user.language"));
    LOGGER.info("user.country = {}", System.getProperty("user.country"));
    // detect maximum heap size
    long maxMemory = Runtime.getRuntime().maxMemory() >> 20;
    LOGGER.info("Maximum heap size = {} MB", maxMemory);
    if (filename == null) {
        // scan for files
        System.out.println("Scanning for files...");
        long start = System.currentTimeMillis();
        try {
            TestTools.getFiles(baseDir, files, FormatReaderTest.configTree, configFile, validSubdirs, configSuffix);
        } catch (Exception e) {
            LOGGER.info("Failed to retrieve complete list of files", e);
        }
        long end = System.currentTimeMillis();
        double time = (end - start) / 1000.0;
        LOGGER.info(TestTools.DIVIDER);
        LOGGER.info("Total files: {}", files.size());
        long avg = (end - start);
        if (files.size() > 0)
            avg /= files.size();
        LOGGER.info("Scan time: {} s ({} ms/file)", time, avg);
        LOGGER.info(TestTools.DIVIDER);
    } else {
        files.add(filename);
    }
    // remove duplicates
    Set<String> fileSet = new LinkedHashSet<String>();
    Map<String, String> originalPath = new HashMap<String, String>();
    for (String s : files) {
        String canonicalPath;
        try {
            canonicalPath = (new File(s)).getCanonicalPath();
        } catch (IOException e) {
            LOGGER.warn("Could not get canonical path for {}", s);
            canonicalPath = s;
        }
        fileSet.add(canonicalPath);
        originalPath.put(canonicalPath, s);
    }
    Set<String> minimalFiles = new LinkedHashSet<String>();
    FileStitcher reader = new FileStitcher();
    Set<String> failingIds = new LinkedHashSet<String>();
    while (!fileSet.isEmpty()) {
        String file = fileSet.iterator().next();
        try {
            reader.setId(file);
        } catch (Exception e) {
            LOGGER.error("setId(\"{}\") failed", file, e);
            failingIds.add(file);
            fileSet.remove(file);
            continue;
        }
        try {
            String[] usedFiles = reader.getUsedFiles();
            Set<String> auxFiles = new LinkedHashSet<String>();
            for (String s : usedFiles) {
                auxFiles.add((new File(s)).getCanonicalPath());
            }
            fileSet.removeAll(auxFiles);
            String masterFile = reader.getCurrentFile();
            auxFiles.remove(masterFile);
            minimalFiles.removeAll(auxFiles);
            minimalFiles.add(masterFile);
        } catch (Exception e) {
            LOGGER.warn("Could not determine duplicate status for {}", file, e);
            minimalFiles.add(file);
        } finally {
            fileSet.remove(file);
            try {
                reader.close();
            } catch (IOException e) {
            }
        }
    }
    if (!failingIds.isEmpty()) {
        String msg = String.format("setId failed on %s", failingIds);
        LOGGER.error(msg);
        throw new RuntimeException(msg);
    }
    files = new ArrayList<String>();
    for (String s : minimalFiles) {
        if (!originalPath.containsKey(s)) {
            String msg = "No match found for " + s;
            LOGGER.error(msg);
            throw new RuntimeException(msg);
        }
        files.add(originalPath.get(s));
    }
    // create test class instances
    System.out.println("Building list of tests...");
    Object[] tests = new Object[files.size()];
    for (int i = 0; i < tests.length; i++) {
        String id = (String) files.get(i);
        try {
            if (FormatReaderTest.configTree.get(id) == null) {
                LOGGER.error("{} not configured.", id);
            }
        } catch (Exception e) {
            LOGGER.warn("", e);
        }
        tests[i] = new FormatReaderTest(id, multiplier, inMemory);
    }
    if (tests.length == 1)
        System.out.println("Ready to test " + files.get(0));
    else
        System.out.println("Ready to test " + tests.length + " files");
    return tests;
}

Example 17 with FileStitcher

use of loci.formats.FileStitcher in project bioformats by openmicroscopy.

the class FormatReaderTest method testUnflattenedPixelsHashes.

@Test(groups = { "all", "pixels", "automated" })
public void testUnflattenedPixelsHashes() {
    if (config == null)
        throw new SkipException("No config tree");
    String testName = "testUnflattenedPixelsHashes";
    if (!initFile())
        result(testName, false, "initFile");
    boolean success = true;
    String msg = null;
    try {
        IFormatReader resolutionReader = new BufferedImageReader(new FileStitcher());
        resolutionReader.setFlattenedResolutions(false);
        resolutionReader.setNormalized(true);
        resolutionReader.setOriginalMetadataPopulated(false);
        resolutionReader.setMetadataFiltered(true);
        resolutionReader.setId(id);
        // check the MD5 of the first plane in each resolution
        for (int i = 0; i < resolutionReader.getSeriesCount() && success; i++) {
            resolutionReader.setSeries(i);
            for (int r = 0; r < resolutionReader.getResolutionCount() && success; r++) {
                resolutionReader.setResolution(r);
                config.setSeries(resolutionReader.getCoreIndex());
                long planeSize = -1;
                try {
                    planeSize = DataTools.safeMultiply32(resolutionReader.getSizeX(), resolutionReader.getSizeY(), resolutionReader.getRGBChannelCount(), FormatTools.getBytesPerPixel(resolutionReader.getPixelType()));
                } catch (IllegalArgumentException e) {
                    continue;
                }
                if (planeSize < 0 || !TestTools.canFitInMemory(planeSize)) {
                    continue;
                }
                String md5 = TestTools.md5(resolutionReader.openBytes(0));
                String expected1 = config.getMD5();
                String expected2 = config.getAlternateMD5();
                if (expected1 == null && expected2 == null) {
                    continue;
                }
                if (!md5.equals(expected1) && !md5.equals(expected2)) {
                    success = false;
                    msg = "series " + i + ", resolution " + r;
                }
            }
        }
        resolutionReader.close();
    } catch (Throwable t) {
        if (TestTools.isOutOfMemory(t)) {
            result(testName, true, "Image too large");
            return;
        }
        LOGGER.info("", t);
        success = false;
    }
    result(testName, success, msg);
}