Examples with AnalysisDecision zemberek.morphology.ambiguity.RuleBasedDisambiguator.AnalysisDecision used on opensource projects

Example 1 with AnalysisDecision

use of zemberek.morphology.ambiguity.RuleBasedDisambiguator.AnalysisDecision in project zemberek-nlp by ahmetaa.

the class GenerateDataWithRules method extractData.

private void extractData(Path p, Path outRoot, int resultLimit, int maxAmbigiousWordCount) throws IOException {
    List<Path> files = Files.walk(p, 1).filter(s -> s.toFile().isFile()).collect(Collectors.toList());
    BatchResult result = new BatchResult();
    int i = 0;
    for (Path file : files) {
        Log.info("Processing %s", file);
        LinkedHashSet<String> sentences = getSentences(file);
        collect(result, sentences, maxAmbigiousWordCount, resultLimit);
        i++;
        Log.info("%d of %d", i, files.size());
        if (resultLimit > 0 && result.results.size() > resultLimit) {
            break;
        }
    }
    String s = p.toFile().getName();
    Log.info("Saving.");
    Path out = outRoot.resolve(s + "-rule-result.txt");
    Path amb = outRoot.resolve(s + "-rule-result-amb.txt");
    try (PrintWriter pwu = new PrintWriter(out.toFile(), "utf-8");
        PrintWriter pwa = new PrintWriter(amb.toFile(), "utf-8")) {
        for (ResultSentence sentence : result.results) {
            pwu.println("S:" + sentence.sentence);
            pwa.println("S:" + sentence.sentence);
            for (AmbiguityAnalysis analysis : sentence.results) {
                List<String> forTrain = analysis.getForTrainingOutput();
                forTrain.forEach(pwu::println);
                pwa.println(analysis.token);
                for (AnalysisDecision r : analysis.choices) {
                    pwa.println(r.analysis.formatLong());
                }
            }
            pwu.println();
            pwa.println();
        }
    }
}