Examples with SimpleTextWriter zemberek.core.io.SimpleTextWriter used on opensource projects

Example 11 with SimpleTextWriter

use of zemberek.core.io.SimpleTextWriter in project zemberek-nlp by ahmetaa.

the class OflazerAnalyzerRunner method prepareForAnalysis.

private void prepareForAnalysis(File f, List<String> sentences) throws IOException {
    SimpleTextWriter stw = SimpleTextWriter.keepOpenUTF8Writer(f);
    for (String sentence : sentences) {
        for (String s : Splitter.on(p).omitEmptyStrings().split(sentence)) {
            stw.writeLine(s);
        }
        stw.writeLine("#");
    }
    stw.close();
}

Example 12 with SimpleTextWriter

use of zemberek.core.io.SimpleTextWriter in project zemberek-nlp by ahmetaa.

the class DisambiguatorPreprocessor method processOflazerAnalyzerOutputYuret.

public void processOflazerAnalyzerOutputYuret(File oflazerResult, File out) throws IOException {
    SimpleTextWriter yuretFileWriter = SimpleTextWriter.keepOpenWriter(new FileOutputStream(out), "ISO-8859-9");
    yuretFileWriter.writeLine("<DOC>\t<DOC>");
    yuretFileWriter.writeLine();
    LineIterator li = new SimpleTextReader(oflazerResult, "UTF-8").getLineIterator();
    boolean sentenceStarted = false;
    List<String> parses = new ArrayList<>();
    while (li.hasNext()) {
        String line = li.next().trim().replaceAll("AorPart", "PresPart");
        String word = Strings.subStringUntilFirst(line, "\t").trim();
        if (line.length() == 0 && !sentenceStarted) {
            continue;
        }
        if (line.length() == 0 && parses.size() > 0) {
            yuretFileWriter.writeLines(parses);
            yuretFileWriter.writeLine();
            yuretFileWriter.writeLine();
            parses = Lists.newArrayList();
        }
        if (line.length() > 0) {
            if (parses.size() == 0) {
                if (!sentenceStarted) {
                    yuretFileWriter.writeLine("<S>\t<S>");
                    yuretFileWriter.writeLine();
                }
                sentenceStarted = true;
            }
            if (punctuations.contains(word)) {
                // because analyser i use does not parse punctuations. i do it myself.
                parses.add(word + "\t" + word + "\t+Punc");
            } else if (!line.endsWith("?")) {
                parses.add(line);
            } else if (!word.equals("#")) {
                String inferred = inferUnknownWordParse(word);
                System.out.println("Bad word: [" + line + "] inferred to [" + inferred + "]");
                parses.add(inferred);
            }
        }
        if (word.equals("#")) {
            sentenceStarted = false;
            yuretFileWriter.writeLine("</S>\t</S>\n");
            parses = new ArrayList<>();
        }
    }
    yuretFileWriter.writeLine("</DOC>\t</DOC>");
    yuretFileWriter.close();
}