Examples with TextFileLexicon com.joliciel.jochre.lexicon.TextFileLexicon used on opensource projects

Example 6 with TextFileLexicon

use of com.joliciel.jochre.lexicon.TextFileLexicon in project jochre by urieli.

the class NameListReader method read.

public TextFileLexicon read(File file) throws IOException {
    Scanner scanner = new Scanner(file);
    try {
        while (scanner.hasNextLine()) {
            String line = scanner.nextLine();
            if (!line.startsWith("#")) {
                String name = line;
                NameLexicalEntry entry = new NameLexicalEntry(name, "np", name, "s", defaultAttribute);
                allVariants.add(entry);
                entries.add(entry.text);
                String radical = YiddishTextUtils.removeEndForm(entry.text);
                String possessiveForm = radical;
                if (radical.endsWith("ס") || radical.endsWith("ש") || radical.endsWith("צ") || radical.endsWith("ת")) {
                    possessiveForm += "עס";
                } else {
                    possessiveForm += "ס";
                }
                if (!exceptions.contains(possessiveForm)) {
                    NameLexicalEntry possessiveEntry = new NameLexicalEntry(possessiveForm, "np", name, "s", defaultAttribute + ",@poss");
                    allVariants.add(possessiveEntry);
                    entries.add(possessiveEntry.text);
                }
                String accusativeForm = radical;
                if (radical.endsWith("ל") || radical.endsWith("מ") || radical.endsWith("נ")) {
                    accusativeForm += "ען";
                } else {
                    accusativeForm += "ן";
                }
                if (!exceptions.contains(accusativeForm)) {
                    NameLexicalEntry accusativeEntry = new NameLexicalEntry(accusativeForm, "np", name, "s", defaultAttribute + ",@acc,@dat");
                    allVariants.add(accusativeEntry);
                    entries.add(accusativeEntry.text);
                }
            }
        }
        if (this.variantWriter != null) {
            for (NameLexicalEntry variant : allVariants) {
                variantWriter.write(variant.toString() + "\n");
                variantWriter.flush();
            }
        }
        TextFileLexicon lexicon = new TextFileLexicon();
        for (String word : entries) lexicon.setEntry(word, 1);
        return lexicon;
    } finally {
        scanner.close();
    }
}

Example 7 with TextFileLexicon

use of com.joliciel.jochre.lexicon.TextFileLexicon in project jochre by urieli.

the class NiborskiLexiconReader method main.

public static void main(String[] args) throws Exception {
    long startTime = (new Date()).getTime();
    try {
        String command = args[0];
        if (command.equals("load")) {
            NiborskiLexiconReader reader = new NiborskiLexiconReader();
            File file = new File(args[1]);
            Writer variantWriter = null;
            if (args.length > 2) {
                File variantFile = new File(args[2]);
                variantFile.getParentFile().mkdirs();
                variantFile.delete();
                variantWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(variantFile, true), "UTF8"));
            }
            reader.setVariantWriter(variantWriter);
            try {
                TextFileLexicon lexicon = reader.read(file);
                if (args.length > 3) {
                    File lexiconFile = new File(args[3]);
                    lexicon.serialize(lexiconFile);
                }
            } finally {
                if (variantWriter != null)
                    variantWriter.close();
            }
        } else if (command.equals("deserialise")) {
            File memoryBaseFile = new File(args[1]);
            Lexicon lexicon = TextFileLexicon.deserialize(memoryBaseFile);
            String[] words = new String[] { "אײגל", "אױגל" };
            for (String word : words) LOG.debug("Have entry " + word + ": " + lexicon.getFrequency(word));
        } else {
            throw new RuntimeException("Unknown command: " + command);
        }
    } finally {
        long endTime = (new Date()).getTime() - startTime;
        LOG.debug("Total runtime: " + ((double) endTime / 1000) + " seconds");
    }
}

Example 8 with TextFileLexicon

use of com.joliciel.jochre.lexicon.TextFileLexicon in project jochre by urieli.

the class PlaceListReader method main.

public static void main(String[] args) throws Exception {
    long startTime = (new Date()).getTime();
    try {
        String command = args[0];
        if (command.equals("load")) {
            PlaceListReader reader = new PlaceListReader();
            File file = new File(args[1]);
            Writer variantWriter = null;
            if (args.length > 2) {
                File variantFile = new File(args[2]);
                variantFile.delete();
                variantWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(variantFile, true), "UTF8"));
            }
            reader.setVariantWriter(variantWriter);
            try {
                TextFileLexicon lexicon = reader.read(file);
                if (args.length > 3) {
                    File lexiconFile = new File(args[3]);
                    lexicon.serialize(lexiconFile);
                }
            } finally {
                if (variantWriter != null)
                    variantWriter.close();
            }
        } else if (command.equals("deserialise")) {
            File memoryBaseFile = new File(args[1]);
            Lexicon lexicon = TextFileLexicon.deserialize(memoryBaseFile);
            String[] words = new String[] { "חײמס", "חױמס" };
            for (String word : words) LOG.debug("Have entry " + word + ": " + lexicon.getFrequency(word));
        } else {
            throw new RuntimeException("Unknown command: " + command);
        }
    } finally {
        long endTime = (new Date()).getTime() - startTime;
        LOG.debug("Total runtime: " + ((double) endTime / 1000) + " seconds");
    }
}

Example 9 with TextFileLexicon

use of com.joliciel.jochre.lexicon.TextFileLexicon in project jochre by urieli.

the class Jochre method doCommandBuildLexicon.

/**
 * Rebuild the training corpus lexicon.
 */
public void doCommandBuildLexicon(File outputDir, CorpusSelectionCriteria criteria) {
    try {
        CorpusLexiconBuilder builder = new CorpusLexiconBuilder(criteria, jochreSession);
        TextFileLexicon lexicon = builder.buildLexicon();
        File textFile = new File(outputDir, "jochreCorpusLexicon.txt");
        textFile.delete();
        Writer textFileWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(textFile, true), "UTF8"));
        try {
            lexicon.writeFile(textFileWriter);
        } finally {
            textFileWriter.flush();
            textFileWriter.close();
        }
        File lexiconFile = new File(outputDir, "jochreCorpusLexicon.zip");
        lexicon.serialize(lexiconFile);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

Example 10 with TextFileLexicon

use of com.joliciel.jochre.lexicon.TextFileLexicon in project jochre by urieli.

the class Jochre method execute.

/**
 * Usage (* indicates optional):<br/>
 * Jochre load [filename] [isoLanguageCode] [firstPage]* [lastPage]*<br/>
 * Loads a file (pdf or image) and segments it into letters. The analysed
 * version is stored in the persistent store. Writes [filename].xml to the same
 * location, to enable the user to indicate the text to associate with this
 * file.<br/>
 * Jochre extract [filename] [outputDirectory] [firstPage]* [lastPage]*<br/>
 * Extracts images form a pdf file.<br/>
 */
public void execute(Map<String, String> argMap) throws Exception {
    if (argMap.size() == 0) {
        System.out.println("See jochre wiki for usage");
        return;
    }
    String logConfigPath = argMap.get("logConfigFile");
    if (logConfigPath != null) {
        argMap.remove("logConfigFile");
        JochreLogUtils.configureLogging(logConfigPath);
    }
    String command = "";
    String inFilePath = "";
    String inDirPath = null;
    String userFriendlyName = "";
    String outputDirPath = null;
    String outputFilePath = null;
    int firstPage = -1;
    int lastPage = -1;
    Set<Integer> pages = Collections.emptySet();
    int shapeId = -1;
    int docId = -1;
    int imageId = 0;
    int userId = -1;
    int imageCount = 0;
    int multiplier = 0;
    boolean save = false;
    ImageStatus[] imageSet = null;
    boolean reconstructLetters = false;
    int excludeImageId = 0;
    int crossValidationSize = -1;
    int includeIndex = -1;
    int excludeIndex = -1;
    Set<Integer> documentSet = null;
    String suffix = "";
    String docGroupPath = null;
    boolean includeBeam = false;
    List<OutputFormat> outputFormats = new ArrayList<>();
    String docSelectionPath = null;
    List<String> featureDescriptors = null;
    boolean includeDate = false;
    for (Entry<String, String> argMapEntry : argMap.entrySet()) {
        String argName = argMapEntry.getKey();
        String argValue = argMapEntry.getValue();
        if (argName.equals("command"))
            command = argValue;
        else if (argName.equals("file"))
            inFilePath = argValue;
        else if (argName.equals("name"))
            userFriendlyName = argValue;
        else if (argName.equals("first"))
            firstPage = Integer.parseInt(argValue);
        else if (argName.equals("last"))
            lastPage = Integer.parseInt(argValue);
        else if (argName.equals("pages")) {
            final String WITH_DELIMITER = "((?<=%1$s)|(?=%1$s))";
            final Pattern numberPattern = Pattern.compile("\\d+");
            final String[] parts = argValue.split(String.format(WITH_DELIMITER, "[\\-,]"));
            int number = -1;
            boolean inRange = false;
            final Set<Integer> myPages = new HashSet<>();
            for (String part : parts) {
                if (numberPattern.matcher(part).matches()) {
                    int lowerBound = number;
                    number = Integer.parseInt(part);
                    if (inRange) {
                        if (lowerBound > number)
                            throw new IllegalArgumentException("Lower bound (" + lowerBound + ") greater than upper bound (" + number + "): " + argValue);
                        IntStream.rangeClosed(lowerBound, number).forEach(i -> myPages.add(i));
                        number = -1;
                        inRange = false;
                    }
                } else if (part.equals(",")) {
                    if (number >= 0)
                        myPages.add(number);
                    number = -1;
                } else if (part.equals("-")) {
                    if (inRange)
                        throw new IllegalArgumentException("Unable to parse pages (unclosed range): " + argValue);
                    if (number < 0)
                        throw new IllegalArgumentException("Range without lower bound: " + argValue);
                    inRange = true;
                } else {
                    throw new IllegalArgumentException("Unable to parse pages - unexpected character '" + part + "': " + argValue);
                }
            }
            if (inRange) {
                throw new IllegalArgumentException("Unable to parse pages (unclosed range): " + argValue);
            }
            if (number >= 0)
                myPages.add(number);
            pages = myPages;
        } else if (argName.equals("inDir"))
            inDirPath = argValue;
        else if (argName.equals("outDir"))
            outputDirPath = argValue;
        else if (argName.equals("outputFile"))
            outputFilePath = argValue;
        else if (argName.equals("save"))
            save = (argValue.equals("true"));
        else if (argName.equals("shapeId"))
            shapeId = Integer.parseInt(argValue);
        else if (argName.equals("imageId"))
            imageId = Integer.parseInt(argValue);
        else if (argName.equals("docId"))
            docId = Integer.parseInt(argValue);
        else if (argName.equals("userId"))
            userId = Integer.parseInt(argValue);
        else if (argName.equals("imageCount"))
            imageCount = Integer.parseInt(argValue);
        else if (argName.equals("multiplier"))
            multiplier = Integer.parseInt(argValue);
        else if (argName.equals("imageStatus")) {
            String[] statusCodes = argValue.split(",");
            Set<ImageStatus> imageStasuses = new HashSet<>();
            for (String statusCode : statusCodes) {
                if (statusCode.equals("heldOut"))
                    imageStasuses.add(ImageStatus.TRAINING_HELD_OUT);
                else if (statusCode.equals("test"))
                    imageStasuses.add(ImageStatus.TRAINING_TEST);
                else if (statusCode.equals("training"))
                    imageStasuses.add(ImageStatus.TRAINING_VALIDATED);
                else if (statusCode.equals("all")) {
                    imageStasuses.add(ImageStatus.TRAINING_VALIDATED);
                    imageStasuses.add(ImageStatus.TRAINING_HELD_OUT);
                    imageStasuses.add(ImageStatus.TRAINING_TEST);
                } else
                    throw new RuntimeException("Unknown imageSet: " + statusCode);
            }
            imageSet = new ImageStatus[imageStasuses.size()];
            int i = 0;
            for (ImageStatus imageStatus : imageStasuses) {
                imageSet[i++] = imageStatus;
            }
        } else if (argName.equals("reconstructLetters"))
            reconstructLetters = (argValue.equals("true"));
        else if (argName.equals("excludeImageId"))
            excludeImageId = Integer.parseInt(argValue);
        else if (argName.equals("crossValidationSize"))
            crossValidationSize = Integer.parseInt(argValue);
        else if (argName.equals("includeIndex"))
            includeIndex = Integer.parseInt(argValue);
        else if (argName.equals("excludeIndex"))
            excludeIndex = Integer.parseInt(argValue);
        else if (argName.equals("docSet")) {
            String[] docIdArray = argValue.split(",");
            documentSet = new HashSet<>();
            for (String docIdString : docIdArray) {
                int oneId = Integer.parseInt(docIdString);
                documentSet.add(oneId);
            }
        } else if (argName.equals("docSelection")) {
            docSelectionPath = argValue;
        } else if (argName.equals("docGroupFile"))
            docGroupPath = argValue;
        else if (argName.equals("suffix"))
            suffix = argValue;
        else if (argName.equals("includeBeam"))
            includeBeam = argValue.equalsIgnoreCase("true");
        else if (argName.equals("outputFormat")) {
            outputFormats = new ArrayList<>();
            String[] outputFormatStrings = argValue.split(",");
            for (String outputFormatString : outputFormatStrings) {
                outputFormats.add(OutputFormat.valueOf(outputFormatString));
            }
            if (outputFormats.size() == 0)
                throw new JochreException("At least one outputFormat required.");
        } else if (argName.equals("features")) {
            featureDescriptors = new ArrayList<>();
            InputStream featureFile = new FileInputStream(new File(argValue));
            try (Scanner scanner = new Scanner(new BufferedReader(new InputStreamReader(featureFile, "UTF-8")))) {
                while (scanner.hasNextLine()) {
                    String descriptor = scanner.nextLine();
                    featureDescriptors.add(descriptor);
                    LOG.debug(descriptor);
                }
            }
        } else if (argName.equals("includeDate")) {
            includeDate = argValue.equalsIgnoreCase("true");
        } else {
            throw new RuntimeException("Unknown argument: " + argName);
        }
    }
    if (pages.isEmpty() && (firstPage >= 0 || lastPage >= 0)) {
        if (firstPage < 0)
            firstPage = 0;
        if (lastPage < 0)
            lastPage = config.getInt("jochre.pdf.max-page");
        pages = IntStream.rangeClosed(firstPage, lastPage).boxed().collect(Collectors.toSet());
    }
    long startTime = System.currentTimeMillis();
    try {
        this.setUserId(userId);
        CorpusSelectionCriteria criteria = new CorpusSelectionCriteria();
        if (docSelectionPath != null) {
            File docSelectionFile = new File(docSelectionPath);
            Scanner scanner = new Scanner(new BufferedReader(new InputStreamReader(new FileInputStream(docSelectionFile), jochreSession.getEncoding())));
            criteria.loadSelection(scanner);
            scanner.close();
        } else {
            criteria.setImageId(imageId);
            criteria.setImageCount(imageCount);
            if (imageSet != null)
                criteria.setImageStatusesToInclude(imageSet);
            criteria.setExcludeImageId(excludeImageId);
            criteria.setCrossValidationSize(crossValidationSize);
            criteria.setIncludeIndex(includeIndex);
            criteria.setExcludeIndex(excludeIndex);
            criteria.setDocumentId(docId);
            criteria.setDocumentIds(documentSet);
        }
        if (LOG.isDebugEnabled())
            LOG.debug(criteria.getAttributes().toString());
        if (docGroupPath != null) {
            File docGroupFile = new File(docGroupPath);
            Scanner scanner = new Scanner(new BufferedReader(new InputStreamReader(new FileInputStream(docGroupFile), jochreSession.getEncoding())));
            while (scanner.hasNextLine()) {
                String line = scanner.nextLine();
                int equalsPos = line.indexOf('=');
                String groupName = line.substring(0, equalsPos);
                String[] ids = line.substring(equalsPos + 1).split(",");
                Set<Integer> idSet = new HashSet<>();
                for (String idString : ids) {
                    idSet.add(Integer.parseInt(idString));
                }
                documentGroups.put(groupName, idSet);
            }
            scanner.close();
        }
        MostLikelyWordChooser wordChooser = new MostLikelyWordChooser(jochreSession);
        File outputDir = null;
        File outputFile = null;
        if (outputDirPath != null) {
            outputDir = new File(outputDirPath);
        } else if (outputFilePath != null) {
            outputFile = new File(outputFilePath);
            outputDir = outputFile.getParentFile();
        }
        if (outputDir != null)
            outputDir.mkdirs();
        List<DocumentObserver> observers = null;
        List<PdfImageObserver> imageObservers = null;
        if (outputFormats.size() > 0 && !command.equals("analyseFolder")) {
            if (outputDir == null) {
                throw new JochreException("Either outputDir our outputFile are required with outputFormats");
            }
            String baseName = null;
            if (userFriendlyName != null && userFriendlyName.length() > 0) {
                baseName = userFriendlyName;
            } else if (inFilePath != null && inFilePath.length() > 0) {
                File inFile = new File(inFilePath);
                baseName = this.getBaseName(inFile);
            }
            observers = this.getObservers(outputFormats, baseName, outputDir, includeDate);
            imageObservers = this.getImageObservers(outputFormats, baseName, outputDir);
        }
        if (userFriendlyName.length() == 0)
            userFriendlyName = inFilePath;
        if (command.equals("segment")) {
            this.doCommandSegment(inFilePath, userFriendlyName, outputDir, save, pages);
        } else if (command.equals("extract")) {
            this.doCommandExtractImages(inFilePath, outputDir, pages);
        } else if (command.equals("updateImages")) {
            this.doCommandUpdateImages(inFilePath, docId, pages);
        } else if (command.equals("applyFeatures")) {
            this.doCommandApplyFeatures(imageId, shapeId, featureDescriptors);
        } else if (command.equals("train")) {
            this.doCommandTrain(featureDescriptors, criteria, reconstructLetters);
        } else if (command.equals("evaluate") || command.equals("evaluateComplex")) {
            this.doCommandEvaluate(criteria, outputDir, wordChooser, reconstructLetters, save, suffix, includeBeam, observers);
        } else if (command.equals("evaluateFull")) {
            this.doCommandEvaluateFull(criteria, save, outputDir, wordChooser, suffix, observers);
        } else if (command.equals("analyse")) {
            this.doCommandAnalyse(criteria, wordChooser, observers);
        } else if (command.equals("transform")) {
            this.doCommandTransform(criteria, observers, imageObservers);
        } else if (command.equals("trainSplits")) {
            this.doCommandTrainSplits(featureDescriptors, criteria);
        } else if (command.equals("evaluateSplits")) {
            this.doCommandEvaluateSplits(criteria);
        } else if (command.equals("trainMerge")) {
            this.doCommandTrainMerge(featureDescriptors, multiplier, criteria);
        } else if (command.equals("evaluateMerge")) {
            this.doCommandEvaluateMerge(criteria);
        } else if (command.equals("logImage")) {
            this.doCommandLogImage(shapeId);
        } else if (command.equals("testFeature")) {
            this.doCommandTestFeature(shapeId);
        } else if (command.equals("serializeLexicon")) {
            if (outputDir == null) {
                throw new JochreException("Either outputDir our outputFile are required for " + command);
            }
            File inputFile = new File(inFilePath);
            if (inputFile.isDirectory()) {
                File[] lexiconFiles = inputFile.listFiles();
                for (File oneLexFile : lexiconFiles) {
                    LOG.debug(oneLexFile.getName() + ": " + ", size: " + oneLexFile.length());
                    TextFileLexicon lexicon = new TextFileLexicon(oneLexFile, jochreSession.getEncoding());
                    String baseName = oneLexFile.getName().substring(0, oneLexFile.getName().indexOf("."));
                    if (baseName.lastIndexOf("/") > 0)
                        baseName = baseName.substring(baseName.lastIndexOf("/") + 1);
                    File lexiconFile = new File(outputDir, baseName + ".obj");
                    lexicon.serialize(lexiconFile);
                }
            } else {
                LOG.debug(inFilePath + ": " + inputFile.exists() + ", size: " + inputFile.length());
                TextFileLexicon lexicon = new TextFileLexicon(inputFile, jochreSession.getEncoding());
                String baseName = inFilePath.substring(0, inFilePath.indexOf("."));
                if (baseName.lastIndexOf("/") > 0)
                    baseName = baseName.substring(baseName.lastIndexOf("/") + 1);
                File lexiconFile = outputFile;
                if (lexiconFile == null)
                    lexiconFile = new File(outputDir, baseName + ".obj");
                lexicon.serialize(lexiconFile);
            }
        } else if (command.equals("analyseFolder")) {
            File inDir = new File(inDirPath);
            File[] pdfFiles = inDir.listFiles(new FilenameFilter() {

                @Override
                public boolean accept(File dir, String name) {
                    return (name.toLowerCase().endsWith(".pdf"));
                }
            });
            Arrays.sort(pdfFiles);
            for (File pdfFile : pdfFiles) {
                LOG.info("Analysing file: " + pdfFile.getAbsolutePath());
                try {
                    String baseName = this.getBaseName(pdfFile);
                    File analysisDir = new File(inDir, baseName);
                    analysisDir.mkdirs();
                    List<DocumentObserver> pdfObservers = this.getObservers(outputFormats, baseName, analysisDir, includeDate);
                    List<PdfImageObserver> pdfImageObservers = this.getImageObservers(outputFormats, baseName, analysisDir);
                    this.doCommandAnalyse(pdfFile, wordChooser, pages, pdfObservers, pdfImageObservers);
                    File pdfOutputDir = new File(outputDir, baseName);
                    pdfOutputDir.mkdirs();
                    File targetFile = new File(pdfOutputDir, pdfFile.getName());
                    Files.move(pdfFile.toPath(), targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
                    File[] analysisFiles = analysisDir.listFiles();
                    for (File analysisFile : analysisFiles) {
                        targetFile = new File(pdfOutputDir, analysisFile.getName());
                        Files.move(analysisFile.toPath(), targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
                    }
                    Files.delete(analysisDir.toPath());
                } catch (Exception e) {
                    // log errors, but continue processing
                    LOG.error("Error processing file: " + pdfFile.getAbsolutePath(), e);
                }
            }
        } else if (command.equals("analyseFile")) {
            File pdfFile = new File(inFilePath);
            this.doCommandAnalyse(pdfFile, wordChooser, pages, observers, imageObservers);
        } else if (command.equals("findSplits")) {
            GraphicsDao graphicsDao = GraphicsDao.getInstance(jochreSession);
            List<Shape> shapesToSplit = graphicsDao.findShapesToSplit(jochreSession.getLocale());
            for (Shape shape : shapesToSplit) {
                LOG.info(shape.toString());
            }
        } else {
            throw new RuntimeException("Unknown command: " + command);
        }
    } catch (Exception e) {
        LOG.error("An error occurred while running Jochre", e);
        throw e;
    } finally {
        long duration = System.currentTimeMillis() - startTime;
        LOG.info("Duration (ms):" + duration);
    }
    LOG.info("#### finished #####");
}