Examples with ClassificationModel com.joliciel.talismane.machineLearning.ClassificationModel used on opensource projects

Example 16 with ClassificationModel

use of com.joliciel.talismane.machineLearning.ClassificationModel in project jochre by urieli.

the class Jochre method doCommandTrainSplits.

/**
 * Train the letter splitting model.
 *
 * @param featureDescriptors
 *          the feature descriptors for training this model
 * @param criteria
 *          the criteria used to select the training corpus
 */
public void doCommandTrainSplits(List<String> featureDescriptors, CorpusSelectionCriteria criteria) {
    if (jochreSession.getSplitModelPath() == null)
        throw new RuntimeException("Missing argument: splitModel");
    if (featureDescriptors == null)
        throw new JochreException("features is required");
    File splitModelFile = new File(jochreSession.getSplitModelPath());
    splitModelFile.getParentFile().mkdirs();
    SplitFeatureParser splitFeatureParser = new SplitFeatureParser();
    Set<SplitFeature<?>> splitFeatures = splitFeatureParser.getSplitFeatureSet(featureDescriptors);
    ClassificationEventStream corpusEventStream = new JochreSplitEventStream(criteria, splitFeatures, jochreSession);
    ModelTrainerFactory modelTrainerFactory = new ModelTrainerFactory();
    ClassificationModelTrainer trainer = modelTrainerFactory.constructTrainer(jochreSession.getConfig());
    ClassificationModel splitModel = trainer.trainModel(corpusEventStream, featureDescriptors);
    splitModel.persist(splitModelFile);
}

Example 17 with ClassificationModel

use of com.joliciel.talismane.machineLearning.ClassificationModel in project jochre by urieli.

the class Jochre method doCommandAnalyse.

/**
 * Full analysis, including merge, split and letter guessing.
 *
 * @param pages
 *          the pages to process, empty means all
 */
public void doCommandAnalyse(File sourceFile, MostLikelyWordChooser wordChooser, Set<Integer> pages, List<DocumentObserver> observers, List<PdfImageObserver> imageObservers) throws IOException {
    ClassificationModel letterModel = jochreSession.getLetterModel();
    List<String> letterFeatureDescriptors = letterModel.getFeatureDescriptors();
    LetterFeatureParser letterFeatureParser = new LetterFeatureParser();
    Set<LetterFeature<?>> letterFeatures = letterFeatureParser.getLetterFeatureSet(letterFeatureDescriptors);
    LetterGuesser letterGuesser = new LetterGuesser(letterFeatures, letterModel.getDecisionMaker());
    BoundaryDetector boundaryDetector = null;
    LetterGuessObserver letterGuessObserver = null;
    if (jochreSession.getSplitModel() != null && jochreSession.getMergeModel() != null) {
        boundaryDetector = new DeterministicBoundaryDetector(jochreSession.getSplitModel(), jochreSession.getMergeModel(), jochreSession);
        OriginalShapeLetterAssigner shapeLetterAssigner = new OriginalShapeLetterAssigner();
        shapeLetterAssigner.setEvaluate(false);
        shapeLetterAssigner.setSingleLetterMethod(false);
        letterGuessObserver = shapeLetterAssigner;
    } else {
        boundaryDetector = new OriginalBoundaryDetector();
        LetterAssigner letterAssigner = new LetterAssigner();
        letterGuessObserver = letterAssigner;
    }
    ImageAnalyser analyser = new BeamSearchImageAnalyser(boundaryDetector, letterGuesser, wordChooser, jochreSession);
    analyser.addObserver(letterGuessObserver);
    JochreDocumentGenerator documentGenerator = new JochreDocumentGenerator(sourceFile.getName(), "", jochreSession);
    documentGenerator.addDocumentObserver(analyser);
    for (DocumentObserver observer : observers) documentGenerator.addDocumentObserver(observer);
    if (!sourceFile.exists())
        throw new JochreException("The file " + sourceFile.getPath() + " does not exist");
    if (sourceFile.getName().toLowerCase().endsWith(".pdf")) {
        PdfDocumentProcessor pdfDocumentProcessor = new PdfDocumentProcessor(sourceFile, pages, documentGenerator);
        for (PdfImageObserver imageObserver : imageObservers) {
            pdfDocumentProcessor.addImageObserver(imageObserver);
        }
        pdfDocumentProcessor.process();
    } else if (sourceFile.getName().toLowerCase().endsWith(".png") || sourceFile.getName().toLowerCase().endsWith(".jpg") || sourceFile.getName().toLowerCase().endsWith(".jpeg") || sourceFile.getName().toLowerCase().endsWith(".gif")) {
        ImageDocumentExtractor extractor = new ImageDocumentExtractor(sourceFile, documentGenerator);
        extractor.extractDocument();
    } else if (sourceFile.isDirectory()) {
        ImageDocumentExtractor extractor = new ImageDocumentExtractor(sourceFile, documentGenerator);
        extractor.extractDocument();
    } else {
        throw new RuntimeException("Unrecognised file extension");
    }
}

Example 18 with ClassificationModel

use of com.joliciel.talismane.machineLearning.ClassificationModel in project jochre by urieli.

the class Jochre method doCommandEvaluate.

/**
 * Evaluate a given letter guessing model.
 *  @param criteria
 *          the criteria used to select the evaluation corpus
 */
public void doCommandEvaluate(CorpusSelectionCriteria criteria, File outputDir, MostLikelyWordChooser wordChooser, boolean reconstructLetters, boolean save, String suffix, boolean includeBeam, List<DocumentObserver> observers) throws IOException {
    ClassificationModel letterModel = jochreSession.getLetterModel();
    List<String> letterFeatureDescriptors = letterModel.getFeatureDescriptors();
    LetterFeatureParser letterFeatureParser = new LetterFeatureParser();
    Set<LetterFeature<?>> letterFeatures = letterFeatureParser.getLetterFeatureSet(letterFeatureDescriptors);
    LetterGuesser letterGuesser = new LetterGuesser(letterFeatures, letterModel.getDecisionMaker());
    String baseName = jochreSession.getLetterModelPath().substring(0, jochreSession.getLetterModelPath().indexOf("."));
    if (baseName.lastIndexOf("/") > 0)
        baseName = baseName.substring(baseName.lastIndexOf("/") + 1);
    baseName += suffix;
    BoundaryDetector boundaryDetector = null;
    if (reconstructLetters) {
        ShapeSplitter splitter = new TrainingCorpusShapeSplitter(jochreSession);
        ShapeMerger merger = new TrainingCorpusShapeMerger();
        boundaryDetector = new LetterByLetterBoundaryDetector(splitter, merger, jochreSession);
    } else {
        boundaryDetector = new OriginalBoundaryDetector();
    }
    ImageAnalyser evaluator = new BeamSearchImageAnalyser(boundaryDetector, letterGuesser, wordChooser, jochreSession);
    FScoreObserver fScoreObserver = null;
    LetterValidator letterValidator = new ComponentCharacterValidator(jochreSession);
    if (reconstructLetters) {
        OriginalShapeLetterAssigner originalShapeLetterAssigner = new OriginalShapeLetterAssigner();
        originalShapeLetterAssigner.setEvaluate(true);
        originalShapeLetterAssigner.setSave(save);
        originalShapeLetterAssigner.setLetterValidator(letterValidator);
        fScoreObserver = originalShapeLetterAssigner;
    } else {
        LetterAssigner letterAssigner = new LetterAssigner();
        letterAssigner.setSave(save);
        evaluator.addObserver(letterAssigner);
        fScoreObserver = new SimpleLetterFScoreObserver(letterValidator, jochreSession);
    }
    evaluator.addObserver(fScoreObserver);
    ErrorLogger errorLogger = new ErrorLogger(jochreSession);
    Writer errorWriter = null;
    File errorFile = new File(outputDir, baseName + "_errors.txt");
    errorFile.delete();
    errorWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(errorFile, true), "UTF8"));
    errorLogger.setErrorWriter(errorWriter);
    evaluator.addObserver(errorLogger);
    LexiconErrorWriter lexiconErrorWriter = new LexiconErrorWriter(outputDir, baseName, wordChooser, jochreSession);
    if (documentGroups != null)
        lexiconErrorWriter.setDocumentGroups(documentGroups);
    lexiconErrorWriter.setIncludeBeam(includeBeam);
    // find all document names (alphabetical ordering)
    Set<String> documentNameSet = new TreeSet<>();
    JochreCorpusImageReader imageReader1 = new JochreCorpusImageReader(jochreSession);
    CorpusSelectionCriteria docCriteria = new CorpusSelectionCriteria();
    docCriteria.setImageStatusesToInclude(criteria.getImageStatusesToInclude());
    docCriteria.setImageId(criteria.getImageId());
    docCriteria.setDocumentId(criteria.getDocumentId());
    docCriteria.setDocumentIds(criteria.getDocumentIds());
    imageReader1.setSelectionCriteria(docCriteria);
    JochreDocument currentDoc = null;
    while (imageReader1.hasNext()) {
        JochreImage image = imageReader1.next();
        if (!image.getPage().getDocument().equals(currentDoc)) {
            currentDoc = image.getPage().getDocument();
            documentNameSet.add(currentDoc.getName());
        }
    }
    List<String> documentNames = new ArrayList<>(documentNameSet);
    lexiconErrorWriter.setDocumentNames(documentNames);
    evaluator.addObserver(lexiconErrorWriter);
    JochreCorpusImageProcessor imageProcessor = new JochreCorpusImageProcessor(criteria, jochreSession);
    imageProcessor.addObserver(evaluator);
    for (DocumentObserver observer : observers) imageProcessor.addObserver(observer);
    try {
        imageProcessor.process();
    } finally {
        if (errorWriter != null)
            errorWriter.close();
    }
    LOG.debug("F-score for " + jochreSession.getLetterModelPath() + ": " + fScoreObserver.getFScoreCalculator().getTotalFScore());
    String modelFileName = baseName;
    if (reconstructLetters)
        modelFileName += "_Reconstruct";
    File fscoreFile = new File(outputDir, modelFileName + "_fscores.csv");
    Writer fscoreWriter = errorWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fscoreFile, true), jochreSession.getCsvEncoding()));
    fScoreObserver.getFScoreCalculator().writeScoresToCSV(fscoreWriter);
}

Example 19 with ClassificationModel

use of com.joliciel.talismane.machineLearning.ClassificationModel in project jochre by urieli.

the class Jochre method doCommandEvaluateSplits.

/**
 * Evaluate the letter splitting model on its own.
 *
 * @param criteria
 *          the criteria used to select the evaluation corpus
 */
public void doCommandEvaluateSplits(CorpusSelectionCriteria criteria) throws IOException {
    ClassificationModel splitModel = jochreSession.getSplitModel();
    if (splitModel == null)
        throw new IllegalArgumentException("Missing parameter: jochre.image-analyser.split-model");
    List<String> splitFeatureDescriptors = splitModel.getFeatureDescriptors();
    SplitFeatureParser splitFeatureParser = new SplitFeatureParser();
    Set<SplitFeature<?>> splitFeatures = splitFeatureParser.getSplitFeatureSet(splitFeatureDescriptors);
    SplitCandidateFinder splitCandidateFinder = new SplitCandidateFinder(jochreSession);
    splitCandidateFinder.setMinDistanceBetweenSplits(5);
    ShapeSplitter shapeSplitter = new RecursiveShapeSplitter(splitCandidateFinder, splitFeatures, splitModel.getDecisionMaker(), jochreSession);
    JochreCorpusShapeReader shapeReader = new JochreCorpusShapeReader(jochreSession);
    shapeReader.setSelectionCriteria(criteria);
    SplitEvaluator splitEvaluator = new SplitEvaluator(jochreSession);
    FScoreCalculator<String> fScoreCalculator = splitEvaluator.evaluate(shapeReader, shapeSplitter);
    LOG.debug("" + fScoreCalculator.getTotalFScore());
}

Example 20 with ClassificationModel

use of com.joliciel.talismane.machineLearning.ClassificationModel in project jochre by urieli.

the class JochreDocumentGenerator method requestAnalysis.

/**
 * Call if this document should be analysed for letters, after applying
 * split/merge models.
 */
public void requestAnalysis(MostLikelyWordChooser wordChooser) {
    try {
        ClassificationModel letterModel = jochreSession.getLetterModel();
        List<String> letterFeatureDescriptors = letterModel.getFeatureDescriptors();
        LetterFeatureParser letterFeatureParser = new LetterFeatureParser();
        Set<LetterFeature<?>> letterFeatures = letterFeatureParser.getLetterFeatureSet(letterFeatureDescriptors);
        LetterGuesser letterGuesser = new LetterGuesser(letterFeatures, letterModel.getDecisionMaker());
        BoundaryDetector boundaryDetector = null;
        LetterGuessObserver observer = null;
        if (jochreSession.getSplitModel() != null && jochreSession.getMergeModel() != null) {
            boundaryDetector = new DeterministicBoundaryDetector(jochreSession.getSplitModel(), jochreSession.getMergeModel(), jochreSession);
            OriginalShapeLetterAssigner shapeLetterAssigner = new OriginalShapeLetterAssigner();
            shapeLetterAssigner.setEvaluate(false);
            shapeLetterAssigner.setSave(save);
            shapeLetterAssigner.setSingleLetterMethod(false);
            observer = shapeLetterAssigner;
        } else {
            boundaryDetector = new OriginalBoundaryDetector();
            LetterAssigner letterAssigner = new LetterAssigner();
            letterAssigner.setSave(save);
            observer = letterAssigner;
        }
        ImageAnalyser analyser = new BeamSearchImageAnalyser(boundaryDetector, letterGuesser, wordChooser, jochreSession);
        analyser.addObserver(observer);
        this.documentObservers.add(0, analyser);
    } catch (Exception e) {
        LOG.error("Failed to load models", e);
        throw new RuntimeException(e);
    }
}