Examples with CorpusSelectionCriteria com.joliciel.jochre.graphics.CorpusSelectionCriteria used on opensource projects

Example 1 with CorpusSelectionCriteria

use of com.joliciel.jochre.graphics.CorpusSelectionCriteria in project jochre by urieli.

the class Jochre method doCommandEvaluate.

/**
 * Evaluate a given letter guessing model.
 *  @param criteria
 *          the criteria used to select the evaluation corpus
 */
public void doCommandEvaluate(CorpusSelectionCriteria criteria, File outputDir, MostLikelyWordChooser wordChooser, boolean reconstructLetters, boolean save, String suffix, boolean includeBeam, List<DocumentObserver> observers) throws IOException {
    ClassificationModel letterModel = jochreSession.getLetterModel();
    List<String> letterFeatureDescriptors = letterModel.getFeatureDescriptors();
    LetterFeatureParser letterFeatureParser = new LetterFeatureParser();
    Set<LetterFeature<?>> letterFeatures = letterFeatureParser.getLetterFeatureSet(letterFeatureDescriptors);
    LetterGuesser letterGuesser = new LetterGuesser(letterFeatures, letterModel.getDecisionMaker());
    String baseName = jochreSession.getLetterModelPath().substring(0, jochreSession.getLetterModelPath().indexOf("."));
    if (baseName.lastIndexOf("/") > 0)
        baseName = baseName.substring(baseName.lastIndexOf("/") + 1);
    baseName += suffix;
    BoundaryDetector boundaryDetector = null;
    if (reconstructLetters) {
        ShapeSplitter splitter = new TrainingCorpusShapeSplitter(jochreSession);
        ShapeMerger merger = new TrainingCorpusShapeMerger();
        boundaryDetector = new LetterByLetterBoundaryDetector(splitter, merger, jochreSession);
    } else {
        boundaryDetector = new OriginalBoundaryDetector();
    }
    ImageAnalyser evaluator = new BeamSearchImageAnalyser(boundaryDetector, letterGuesser, wordChooser, jochreSession);
    FScoreObserver fScoreObserver = null;
    LetterValidator letterValidator = new ComponentCharacterValidator(jochreSession);
    if (reconstructLetters) {
        OriginalShapeLetterAssigner originalShapeLetterAssigner = new OriginalShapeLetterAssigner();
        originalShapeLetterAssigner.setEvaluate(true);
        originalShapeLetterAssigner.setSave(save);
        originalShapeLetterAssigner.setLetterValidator(letterValidator);
        fScoreObserver = originalShapeLetterAssigner;
    } else {
        LetterAssigner letterAssigner = new LetterAssigner();
        letterAssigner.setSave(save);
        evaluator.addObserver(letterAssigner);
        fScoreObserver = new SimpleLetterFScoreObserver(letterValidator, jochreSession);
    }
    evaluator.addObserver(fScoreObserver);
    ErrorLogger errorLogger = new ErrorLogger(jochreSession);
    Writer errorWriter = null;
    File errorFile = new File(outputDir, baseName + "_errors.txt");
    errorFile.delete();
    errorWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(errorFile, true), "UTF8"));
    errorLogger.setErrorWriter(errorWriter);
    evaluator.addObserver(errorLogger);
    LexiconErrorWriter lexiconErrorWriter = new LexiconErrorWriter(outputDir, baseName, wordChooser, jochreSession);
    if (documentGroups != null)
        lexiconErrorWriter.setDocumentGroups(documentGroups);
    lexiconErrorWriter.setIncludeBeam(includeBeam);
    // find all document names (alphabetical ordering)
    Set<String> documentNameSet = new TreeSet<>();
    JochreCorpusImageReader imageReader1 = new JochreCorpusImageReader(jochreSession);
    CorpusSelectionCriteria docCriteria = new CorpusSelectionCriteria();
    docCriteria.setImageStatusesToInclude(criteria.getImageStatusesToInclude());
    docCriteria.setImageId(criteria.getImageId());
    docCriteria.setDocumentId(criteria.getDocumentId());
    docCriteria.setDocumentIds(criteria.getDocumentIds());
    imageReader1.setSelectionCriteria(docCriteria);
    JochreDocument currentDoc = null;
    while (imageReader1.hasNext()) {
        JochreImage image = imageReader1.next();
        if (!image.getPage().getDocument().equals(currentDoc)) {
            currentDoc = image.getPage().getDocument();
            documentNameSet.add(currentDoc.getName());
        }
    }
    List<String> documentNames = new ArrayList<>(documentNameSet);
    lexiconErrorWriter.setDocumentNames(documentNames);
    evaluator.addObserver(lexiconErrorWriter);
    JochreCorpusImageProcessor imageProcessor = new JochreCorpusImageProcessor(criteria, jochreSession);
    imageProcessor.addObserver(evaluator);
    for (DocumentObserver observer : observers) imageProcessor.addObserver(observer);
    try {
        imageProcessor.process();
    } finally {
        if (errorWriter != null)
            errorWriter.close();
    }
    LOG.debug("F-score for " + jochreSession.getLetterModelPath() + ": " + fScoreObserver.getFScoreCalculator().getTotalFScore());
    String modelFileName = baseName;
    if (reconstructLetters)
        modelFileName += "_Reconstruct";
    File fscoreFile = new File(outputDir, modelFileName + "_fscores.csv");
    Writer fscoreWriter = errorWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fscoreFile, true), jochreSession.getCsvEncoding()));
    fScoreObserver.getFScoreCalculator().writeScoresToCSV(fscoreWriter);
}

Example 2 with CorpusSelectionCriteria

use of com.joliciel.jochre.graphics.CorpusSelectionCriteria in project jochre by urieli.

the class Jochre method execute.

/**
 * Usage (* indicates optional):<br/>
 * Jochre load [filename] [isoLanguageCode] [firstPage]* [lastPage]*<br/>
 * Loads a file (pdf or image) and segments it into letters. The analysed
 * version is stored in the persistent store. Writes [filename].xml to the same
 * location, to enable the user to indicate the text to associate with this
 * file.<br/>
 * Jochre extract [filename] [outputDirectory] [firstPage]* [lastPage]*<br/>
 * Extracts images form a pdf file.<br/>
 */
public void execute(Map<String, String> argMap) throws Exception {
    if (argMap.size() == 0) {
        System.out.println("See jochre wiki for usage");
        return;
    }
    String logConfigPath = argMap.get("logConfigFile");
    if (logConfigPath != null) {
        argMap.remove("logConfigFile");
        JochreLogUtils.configureLogging(logConfigPath);
    }
    String command = "";
    String inFilePath = "";
    String inDirPath = null;
    String userFriendlyName = "";
    String outputDirPath = null;
    String outputFilePath = null;
    int firstPage = -1;
    int lastPage = -1;
    Set<Integer> pages = Collections.emptySet();
    int shapeId = -1;
    int docId = -1;
    int imageId = 0;
    int userId = -1;
    int imageCount = 0;
    int multiplier = 0;
    boolean save = false;
    ImageStatus[] imageSet = null;
    boolean reconstructLetters = false;
    int excludeImageId = 0;
    int crossValidationSize = -1;
    int includeIndex = -1;
    int excludeIndex = -1;
    Set<Integer> documentSet = null;
    String suffix = "";
    String docGroupPath = null;
    boolean includeBeam = false;
    List<OutputFormat> outputFormats = new ArrayList<>();
    String docSelectionPath = null;
    List<String> featureDescriptors = null;
    boolean includeDate = false;
    for (Entry<String, String> argMapEntry : argMap.entrySet()) {
        String argName = argMapEntry.getKey();
        String argValue = argMapEntry.getValue();
        if (argName.equals("command"))
            command = argValue;
        else if (argName.equals("file"))
            inFilePath = argValue;
        else if (argName.equals("name"))
            userFriendlyName = argValue;
        else if (argName.equals("first"))
            firstPage = Integer.parseInt(argValue);
        else if (argName.equals("last"))
            lastPage = Integer.parseInt(argValue);
        else if (argName.equals("pages")) {
            final String WITH_DELIMITER = "((?<=%1$s)|(?=%1$s))";
            final Pattern numberPattern = Pattern.compile("\\d+");
            final String[] parts = argValue.split(String.format(WITH_DELIMITER, "[\\-,]"));
            int number = -1;
            boolean inRange = false;
            final Set<Integer> myPages = new HashSet<>();
            for (String part : parts) {
                if (numberPattern.matcher(part).matches()) {
                    int lowerBound = number;
                    number = Integer.parseInt(part);
                    if (inRange) {
                        if (lowerBound > number)
                            throw new IllegalArgumentException("Lower bound (" + lowerBound + ") greater than upper bound (" + number + "): " + argValue);
                        IntStream.rangeClosed(lowerBound, number).forEach(i -> myPages.add(i));
                        number = -1;
                        inRange = false;
                    }
                } else if (part.equals(",")) {
                    if (number >= 0)
                        myPages.add(number);
                    number = -1;
                } else if (part.equals("-")) {
                    if (inRange)
                        throw new IllegalArgumentException("Unable to parse pages (unclosed range): " + argValue);
                    if (number < 0)
                        throw new IllegalArgumentException("Range without lower bound: " + argValue);
                    inRange = true;
                } else {
                    throw new IllegalArgumentException("Unable to parse pages - unexpected character '" + part + "': " + argValue);
                }
            }
            if (inRange) {
                throw new IllegalArgumentException("Unable to parse pages (unclosed range): " + argValue);
            }
            if (number >= 0)
                myPages.add(number);
            pages = myPages;
        } else if (argName.equals("inDir"))
            inDirPath = argValue;
        else if (argName.equals("outDir"))
            outputDirPath = argValue;
        else if (argName.equals("outputFile"))
            outputFilePath = argValue;
        else if (argName.equals("save"))
            save = (argValue.equals("true"));
        else if (argName.equals("shapeId"))
            shapeId = Integer.parseInt(argValue);
        else if (argName.equals("imageId"))
            imageId = Integer.parseInt(argValue);
        else if (argName.equals("docId"))
            docId = Integer.parseInt(argValue);
        else if (argName.equals("userId"))
            userId = Integer.parseInt(argValue);
        else if (argName.equals("imageCount"))
            imageCount = Integer.parseInt(argValue);
        else if (argName.equals("multiplier"))
            multiplier = Integer.parseInt(argValue);
        else if (argName.equals("imageStatus")) {
            String[] statusCodes = argValue.split(",");
            Set<ImageStatus> imageStasuses = new HashSet<>();
            for (String statusCode : statusCodes) {
                if (statusCode.equals("heldOut"))
                    imageStasuses.add(ImageStatus.TRAINING_HELD_OUT);
                else if (statusCode.equals("test"))
                    imageStasuses.add(ImageStatus.TRAINING_TEST);
                else if (statusCode.equals("training"))
                    imageStasuses.add(ImageStatus.TRAINING_VALIDATED);
                else if (statusCode.equals("all")) {
                    imageStasuses.add(ImageStatus.TRAINING_VALIDATED);
                    imageStasuses.add(ImageStatus.TRAINING_HELD_OUT);
                    imageStasuses.add(ImageStatus.TRAINING_TEST);
                } else
                    throw new RuntimeException("Unknown imageSet: " + statusCode);
            }
            imageSet = new ImageStatus[imageStasuses.size()];
            int i = 0;
            for (ImageStatus imageStatus : imageStasuses) {
                imageSet[i++] = imageStatus;
            }
        } else if (argName.equals("reconstructLetters"))
            reconstructLetters = (argValue.equals("true"));
        else if (argName.equals("excludeImageId"))
            excludeImageId = Integer.parseInt(argValue);
        else if (argName.equals("crossValidationSize"))
            crossValidationSize = Integer.parseInt(argValue);
        else if (argName.equals("includeIndex"))
            includeIndex = Integer.parseInt(argValue);
        else if (argName.equals("excludeIndex"))
            excludeIndex = Integer.parseInt(argValue);
        else if (argName.equals("docSet")) {
            String[] docIdArray = argValue.split(",");
            documentSet = new HashSet<>();
            for (String docIdString : docIdArray) {
                int oneId = Integer.parseInt(docIdString);
                documentSet.add(oneId);
            }
        } else if (argName.equals("docSelection")) {
            docSelectionPath = argValue;
        } else if (argName.equals("docGroupFile"))
            docGroupPath = argValue;
        else if (argName.equals("suffix"))
            suffix = argValue;
        else if (argName.equals("includeBeam"))
            includeBeam = argValue.equalsIgnoreCase("true");
        else if (argName.equals("outputFormat")) {
            outputFormats = new ArrayList<>();
            String[] outputFormatStrings = argValue.split(",");
            for (String outputFormatString : outputFormatStrings) {
                outputFormats.add(OutputFormat.valueOf(outputFormatString));
            }
            if (outputFormats.size() == 0)
                throw new JochreException("At least one outputFormat required.");
        } else if (argName.equals("features")) {
            featureDescriptors = new ArrayList<>();
            InputStream featureFile = new FileInputStream(new File(argValue));
            try (Scanner scanner = new Scanner(new BufferedReader(new InputStreamReader(featureFile, "UTF-8")))) {
                while (scanner.hasNextLine()) {
                    String descriptor = scanner.nextLine();
                    featureDescriptors.add(descriptor);
                    LOG.debug(descriptor);
                }
            }
        } else if (argName.equals("includeDate")) {
            includeDate = argValue.equalsIgnoreCase("true");
        } else {
            throw new RuntimeException("Unknown argument: " + argName);
        }
    }
    if (pages.isEmpty() && (firstPage >= 0 || lastPage >= 0)) {
        if (firstPage < 0)
            firstPage = 0;
        if (lastPage < 0)
            lastPage = config.getInt("jochre.pdf.max-page");
        pages = IntStream.rangeClosed(firstPage, lastPage).boxed().collect(Collectors.toSet());
    }
    long startTime = System.currentTimeMillis();
    try {
        this.setUserId(userId);
        CorpusSelectionCriteria criteria = new CorpusSelectionCriteria();
        if (docSelectionPath != null) {
            File docSelectionFile = new File(docSelectionPath);
            Scanner scanner = new Scanner(new BufferedReader(new InputStreamReader(new FileInputStream(docSelectionFile), jochreSession.getEncoding())));
            criteria.loadSelection(scanner);
            scanner.close();
        } else {
            criteria.setImageId(imageId);
            criteria.setImageCount(imageCount);
            if (imageSet != null)
                criteria.setImageStatusesToInclude(imageSet);
            criteria.setExcludeImageId(excludeImageId);
            criteria.setCrossValidationSize(crossValidationSize);
            criteria.setIncludeIndex(includeIndex);
            criteria.setExcludeIndex(excludeIndex);
            criteria.setDocumentId(docId);
            criteria.setDocumentIds(documentSet);
        }
        if (LOG.isDebugEnabled())
            LOG.debug(criteria.getAttributes().toString());
        if (docGroupPath != null) {
            File docGroupFile = new File(docGroupPath);
            Scanner scanner = new Scanner(new BufferedReader(new InputStreamReader(new FileInputStream(docGroupFile), jochreSession.getEncoding())));
            while (scanner.hasNextLine()) {
                String line = scanner.nextLine();
                int equalsPos = line.indexOf('=');
                String groupName = line.substring(0, equalsPos);
                String[] ids = line.substring(equalsPos + 1).split(",");
                Set<Integer> idSet = new HashSet<>();
                for (String idString : ids) {
                    idSet.add(Integer.parseInt(idString));
                }
                documentGroups.put(groupName, idSet);
            }
            scanner.close();
        }
        MostLikelyWordChooser wordChooser = new MostLikelyWordChooser(jochreSession);
        File outputDir = null;
        File outputFile = null;
        if (outputDirPath != null) {
            outputDir = new File(outputDirPath);
        } else if (outputFilePath != null) {
            outputFile = new File(outputFilePath);
            outputDir = outputFile.getParentFile();
        }
        if (outputDir != null)
            outputDir.mkdirs();
        List<DocumentObserver> observers = null;
        List<PdfImageObserver> imageObservers = null;
        if (outputFormats.size() > 0 && !command.equals("analyseFolder")) {
            if (outputDir == null) {
                throw new JochreException("Either outputDir our outputFile are required with outputFormats");
            }
            String baseName = null;
            if (userFriendlyName != null && userFriendlyName.length() > 0) {
                baseName = userFriendlyName;
            } else if (inFilePath != null && inFilePath.length() > 0) {
                File inFile = new File(inFilePath);
                baseName = this.getBaseName(inFile);
            }
            observers = this.getObservers(outputFormats, baseName, outputDir, includeDate);
            imageObservers = this.getImageObservers(outputFormats, baseName, outputDir);
        }
        if (userFriendlyName.length() == 0)
            userFriendlyName = inFilePath;
        if (command.equals("segment")) {
            this.doCommandSegment(inFilePath, userFriendlyName, outputDir, save, pages);
        } else if (command.equals("extract")) {
            this.doCommandExtractImages(inFilePath, outputDir, pages);
        } else if (command.equals("updateImages")) {
            this.doCommandUpdateImages(inFilePath, docId, pages);
        } else if (command.equals("applyFeatures")) {
            this.doCommandApplyFeatures(imageId, shapeId, featureDescriptors);
        } else if (command.equals("train")) {
            this.doCommandTrain(featureDescriptors, criteria, reconstructLetters);
        } else if (command.equals("evaluate") || command.equals("evaluateComplex")) {
            this.doCommandEvaluate(criteria, outputDir, wordChooser, reconstructLetters, save, suffix, includeBeam, observers);
        } else if (command.equals("evaluateFull")) {
            this.doCommandEvaluateFull(criteria, save, outputDir, wordChooser, suffix, observers);
        } else if (command.equals("analyse")) {
            this.doCommandAnalyse(criteria, wordChooser, observers);
        } else if (command.equals("transform")) {
            this.doCommandTransform(criteria, observers, imageObservers);
        } else if (command.equals("trainSplits")) {
            this.doCommandTrainSplits(featureDescriptors, criteria);
        } else if (command.equals("evaluateSplits")) {
            this.doCommandEvaluateSplits(criteria);
        } else if (command.equals("trainMerge")) {
            this.doCommandTrainMerge(featureDescriptors, multiplier, criteria);
        } else if (command.equals("evaluateMerge")) {
            this.doCommandEvaluateMerge(criteria);
        } else if (command.equals("logImage")) {
            this.doCommandLogImage(shapeId);
        } else if (command.equals("testFeature")) {
            this.doCommandTestFeature(shapeId);
        } else if (command.equals("serializeLexicon")) {
            if (outputDir == null) {
                throw new JochreException("Either outputDir our outputFile are required for " + command);
            }
            File inputFile = new File(inFilePath);
            if (inputFile.isDirectory()) {
                File[] lexiconFiles = inputFile.listFiles();
                for (File oneLexFile : lexiconFiles) {
                    LOG.debug(oneLexFile.getName() + ": " + ", size: " + oneLexFile.length());
                    TextFileLexicon lexicon = new TextFileLexicon(oneLexFile, jochreSession.getEncoding());
                    String baseName = oneLexFile.getName().substring(0, oneLexFile.getName().indexOf("."));
                    if (baseName.lastIndexOf("/") > 0)
                        baseName = baseName.substring(baseName.lastIndexOf("/") + 1);
                    File lexiconFile = new File(outputDir, baseName + ".obj");
                    lexicon.serialize(lexiconFile);
                }
            } else {
                LOG.debug(inFilePath + ": " + inputFile.exists() + ", size: " + inputFile.length());
                TextFileLexicon lexicon = new TextFileLexicon(inputFile, jochreSession.getEncoding());
                String baseName = inFilePath.substring(0, inFilePath.indexOf("."));
                if (baseName.lastIndexOf("/") > 0)
                    baseName = baseName.substring(baseName.lastIndexOf("/") + 1);
                File lexiconFile = outputFile;
                if (lexiconFile == null)
                    lexiconFile = new File(outputDir, baseName + ".obj");
                lexicon.serialize(lexiconFile);
            }
        } else if (command.equals("analyseFolder")) {
            File inDir = new File(inDirPath);
            File[] pdfFiles = inDir.listFiles(new FilenameFilter() {

                @Override
                public boolean accept(File dir, String name) {
                    return (name.toLowerCase().endsWith(".pdf"));
                }
            });
            Arrays.sort(pdfFiles);
            for (File pdfFile : pdfFiles) {
                LOG.info("Analysing file: " + pdfFile.getAbsolutePath());
                try {
                    String baseName = this.getBaseName(pdfFile);
                    File analysisDir = new File(inDir, baseName);
                    analysisDir.mkdirs();
                    List<DocumentObserver> pdfObservers = this.getObservers(outputFormats, baseName, analysisDir, includeDate);
                    List<PdfImageObserver> pdfImageObservers = this.getImageObservers(outputFormats, baseName, analysisDir);
                    this.doCommandAnalyse(pdfFile, wordChooser, pages, pdfObservers, pdfImageObservers);
                    File pdfOutputDir = new File(outputDir, baseName);
                    pdfOutputDir.mkdirs();
                    File targetFile = new File(pdfOutputDir, pdfFile.getName());
                    Files.move(pdfFile.toPath(), targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
                    File[] analysisFiles = analysisDir.listFiles();
                    for (File analysisFile : analysisFiles) {
                        targetFile = new File(pdfOutputDir, analysisFile.getName());
                        Files.move(analysisFile.toPath(), targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
                    }
                    Files.delete(analysisDir.toPath());
                } catch (Exception e) {
                    // log errors, but continue processing
                    LOG.error("Error processing file: " + pdfFile.getAbsolutePath(), e);
                }
            }
        } else if (command.equals("analyseFile")) {
            File pdfFile = new File(inFilePath);
            this.doCommandAnalyse(pdfFile, wordChooser, pages, observers, imageObservers);
        } else if (command.equals("findSplits")) {
            GraphicsDao graphicsDao = GraphicsDao.getInstance(jochreSession);
            List<Shape> shapesToSplit = graphicsDao.findShapesToSplit(jochreSession.getLocale());
            for (Shape shape : shapesToSplit) {
                LOG.info(shape.toString());
            }
        } else {
            throw new RuntimeException("Unknown command: " + command);
        }
    } catch (Exception e) {
        LOG.error("An error occurred while running Jochre", e);
        throw e;
    } finally {
        long duration = System.currentTimeMillis() - startTime;
        LOG.info("Duration (ms):" + duration);
    }
    LOG.info("#### finished #####");
}