Examples with NumberRangeFileFilter edu.stanford.nlp.io.NumberRangeFileFilter used on opensource projects

Example 6 with NumberRangeFileFilter

use of edu.stanford.nlp.io.NumberRangeFileFilter in project CoreNLP by stanfordnlp.

the class ChineseLexiconAndWordSegmenter method main.

/**
 * This method lets you train and test a segmenter relative to a
 *  Treebank.
 *  <p>
 *  <i>Implementation note:</i> This method is largely cloned from
 *  LexicalizedParser's main method.  Should we try to have it be able
 *  to train segmenters to stop things going out of sync?
 */
public static void main(String[] args) {
    boolean train = false;
    boolean saveToSerializedFile = false;
    boolean saveToTextFile = false;
    String serializedInputFileOrUrl = null;
    String textInputFileOrUrl = null;
    String serializedOutputFileOrUrl = null;
    String textOutputFileOrUrl = null;
    String treebankPath = null;
    Treebank testTreebank = null;
    // Treebank tuneTreebank = null;
    String testPath = null;
    FileFilter testFilter = null;
    FileFilter trainFilter = null;
    String encoding = null;
    // variables needed to process the files to be parsed
    TokenizerFactory<Word> tokenizerFactory = null;
    // DocumentPreprocessor documentPreprocessor = new DocumentPreprocessor();
    // whether or not the input file has already been tokenized
    boolean tokenized = false;
    Function<List<HasWord>, List<HasWord>> escaper = new ChineseEscaper();
    // int tagDelimiter = -1;
    // String sentenceDelimiter = "\n";
    // boolean fromXML = false;
    int argIndex = 0;
    if (args.length < 1) {
        log.info("usage: java edu.stanford.nlp.parser.lexparser." + "LexicalizedParser parserFileOrUrl filename*");
        return;
    }
    Options op = new Options();
    op.tlpParams = new ChineseTreebankParserParams();
    // while loop through option arguments
    while (argIndex < args.length && args[argIndex].charAt(0) == '-') {
        if (args[argIndex].equalsIgnoreCase("-train")) {
            train = true;
            saveToSerializedFile = true;
            int numSubArgs = numSubArgs(args, argIndex);
            argIndex++;
            if (numSubArgs > 1) {
                treebankPath = args[argIndex];
                argIndex++;
            } else {
                throw new RuntimeException("Error: -train option must have treebankPath as first argument.");
            }
            if (numSubArgs == 2) {
                trainFilter = new NumberRangesFileFilter(args[argIndex++], true);
            } else if (numSubArgs >= 3) {
                try {
                    int low = Integer.parseInt(args[argIndex]);
                    int high = Integer.parseInt(args[argIndex + 1]);
                    trainFilter = new NumberRangeFileFilter(low, high, true);
                    argIndex += 2;
                } catch (NumberFormatException e) {
                    // maybe it's a ranges expression?
                    trainFilter = new NumberRangesFileFilter(args[argIndex], true);
                    argIndex++;
                }
            }
        } else if (args[argIndex].equalsIgnoreCase("-encoding")) {
            // sets encoding for TreebankLangParserParams
            encoding = args[argIndex + 1];
            op.tlpParams.setInputEncoding(encoding);
            op.tlpParams.setOutputEncoding(encoding);
            argIndex += 2;
        } else if (args[argIndex].equalsIgnoreCase("-loadFromSerializedFile")) {
            // load the parser from a binary serialized file
            // the next argument must be the path to the parser file
            serializedInputFileOrUrl = args[argIndex + 1];
            argIndex += 2;
        // doesn't make sense to load from TextFile -pichuan
        // } else if (args[argIndex].equalsIgnoreCase("-loadFromTextFile")) {
        // // load the parser from declarative text file
        // // the next argument must be the path to the parser file
        // textInputFileOrUrl = args[argIndex + 1];
        // argIndex += 2;
        } else if (args[argIndex].equalsIgnoreCase("-saveToSerializedFile")) {
            saveToSerializedFile = true;
            serializedOutputFileOrUrl = args[argIndex + 1];
            argIndex += 2;
        } else if (args[argIndex].equalsIgnoreCase("-saveToTextFile")) {
            // save the parser to declarative text file
            saveToTextFile = true;
            textOutputFileOrUrl = args[argIndex + 1];
            argIndex += 2;
        } else if (args[argIndex].equalsIgnoreCase("-treebank")) {
            // the next argument is the treebank path and range for testing
            int numSubArgs = numSubArgs(args, argIndex);
            argIndex++;
            if (numSubArgs == 1) {
                testFilter = new NumberRangesFileFilter(args[argIndex++], true);
            } else if (numSubArgs > 1) {
                testPath = args[argIndex++];
                if (numSubArgs == 2) {
                    testFilter = new NumberRangesFileFilter(args[argIndex++], true);
                } else if (numSubArgs >= 3) {
                    try {
                        int low = Integer.parseInt(args[argIndex]);
                        int high = Integer.parseInt(args[argIndex + 1]);
                        testFilter = new NumberRangeFileFilter(low, high, true);
                        argIndex += 2;
                    } catch (NumberFormatException e) {
                        // maybe it's a ranges expression?
                        testFilter = new NumberRangesFileFilter(args[argIndex++], true);
                    }
                }
            }
        } else {
            int j = op.tlpParams.setOptionFlag(args, argIndex);
            if (j == argIndex) {
                log.info("Unknown option ignored: " + args[argIndex]);
                j++;
            }
            argIndex = j;
        }
    }
    // end while loop through arguments
    TreebankLangParserParams tlpParams = op.tlpParams;
    // all other arguments are order dependent and
    // are processed in order below
    ChineseLexiconAndWordSegmenter cs = null;
    if (!train && op.testOptions.verbose) {
        System.out.println("Currently " + new Date());
        printArgs(args, System.out);
    }
    if (train) {
        printArgs(args, System.out);
        // so we train a parser using the treebank
        if (treebankPath == null) {
            // the next arg must be the treebank path, since it wasn't give earlier
            treebankPath = args[argIndex];
            argIndex++;
            if (args.length > argIndex + 1) {
                try {
                    // the next two args might be the range
                    int low = Integer.parseInt(args[argIndex]);
                    int high = Integer.parseInt(args[argIndex + 1]);
                    trainFilter = new NumberRangeFileFilter(low, high, true);
                    argIndex += 2;
                } catch (NumberFormatException e) {
                    // maybe it's a ranges expression?
                    trainFilter = new NumberRangesFileFilter(args[argIndex], true);
                    argIndex++;
                }
            }
        }
        Treebank trainTreebank = makeTreebank(treebankPath, op, trainFilter);
        Index<String> wordIndex = new HashIndex<>();
        Index<String> tagIndex = new HashIndex<>();
        cs = new ChineseLexiconAndWordSegmenter(trainTreebank, op, wordIndex, tagIndex);
    } else if (textInputFileOrUrl != null) {
    // so we load the segmenter from a text grammar file
    // XXXXX fix later -pichuan
    // cs = new LexicalizedParser(textInputFileOrUrl, true, op);
    } else {
        // so we load a serialized segmenter
        if (serializedInputFileOrUrl == null) {
            // the next argument must be the path to the serialized parser
            serializedInputFileOrUrl = args[argIndex];
            argIndex++;
        }
        try {
            cs = new ChineseLexiconAndWordSegmenter(serializedInputFileOrUrl, op);
        } catch (IllegalArgumentException e) {
            log.info("Error loading segmenter, exiting...");
            System.exit(0);
        }
    }
    // the following has to go after reading parser to make sure
    // op and tlpParams are the same for train and test
    TreePrint treePrint = op.testOptions.treePrint(tlpParams);
    if (testFilter != null) {
        if (testPath == null) {
            if (treebankPath == null) {
                throw new RuntimeException("No test treebank path specified...");
            } else {
                log.info("No test treebank path specified.  Using train path: \"" + treebankPath + "\"");
                testPath = treebankPath;
            }
        }
        testTreebank = tlpParams.testMemoryTreebank();
        testTreebank.loadPath(testPath, testFilter);
    }
    op.trainOptions.sisterSplitters = Generics.newHashSet(Arrays.asList(tlpParams.sisterSplitters()));
    // -- Roger
    if (op.testOptions.verbose) {
        log.info("Lexicon is " + cs.getClass().getName());
    }
    PrintWriter pwOut = tlpParams.pw();
    PrintWriter pwErr = tlpParams.pw(System.err);
    // Now what do we do with the parser we've made
    if (saveToTextFile) {
        // save the parser to textGrammar format
        if (textOutputFileOrUrl != null) {
            saveSegmenterDataToText(cs, textOutputFileOrUrl);
        } else {
            log.info("Usage: must specify a text segmenter data output path");
        }
    }
    if (saveToSerializedFile) {
        if (serializedOutputFileOrUrl == null && argIndex < args.length) {
            // the next argument must be the path to serialize to
            serializedOutputFileOrUrl = args[argIndex];
            argIndex++;
        }
        if (serializedOutputFileOrUrl != null) {
            saveSegmenterDataToSerialized(cs, serializedOutputFileOrUrl);
        } else if (textOutputFileOrUrl == null && testTreebank == null) {
            // no saving/parsing request has been specified
            log.info("usage: " + "java edu.stanford.nlp.parser.lexparser.ChineseLexiconAndWordSegmenter" + "-train trainFilesPath [start stop] serializedParserFilename");
        }
    }
    /* --------------------- Testing part!!!! ----------------------- */
    if (op.testOptions.verbose) {
    // printOptions(false, op);
    }
    if (testTreebank != null || (argIndex < args.length && args[argIndex].equalsIgnoreCase("-treebank"))) {
        // test parser on treebank
        if (testTreebank == null) {
            // the next argument is the treebank path and range for testing
            testTreebank = tlpParams.testMemoryTreebank();
            if (args.length < argIndex + 4) {
                testTreebank.loadPath(args[argIndex + 1]);
            } else {
                int testlow = Integer.parseInt(args[argIndex + 2]);
                int testhigh = Integer.parseInt(args[argIndex + 3]);
                testTreebank.loadPath(args[argIndex + 1], new NumberRangeFileFilter(testlow, testhigh, true));
            }
        }
    /* TODO - test segmenting on treebank. -pichuan */
    // lp.testOnTreebank(testTreebank);
    // } else if (argIndex >= args.length) {
    // // no more arguments, so we just parse our own test sentence
    // if (lp.parse(op.tlpParams.defaultTestSentence())) {
    // treePrint.printTree(lp.getBestParse(), pwOut);
    // } else {
    // pwErr.println("Error. Can't parse test sentence: " +
    // lp.parse(op.tlpParams.defaultTestSentence()));
    // }
    }
// wsg2010: This code block doesn't actually do anything. It appears to read and tokenize a file, and then just print it.
// There are easier ways to do that. This code was copied from an old version of LexicalizedParser.
// else {
// // We parse filenames given by the remaining arguments
// int numWords = 0;
// Timing timer = new Timing();
// // set the tokenizer
// if (tokenized) {
// tokenizerFactory = WhitespaceTokenizer.factory();
// }
// TreebankLanguagePack tlp = tlpParams.treebankLanguagePack();
// if (tokenizerFactory == null) {
// tokenizerFactory = (TokenizerFactory<Word>) tlp.getTokenizerFactory();
// }
// documentPreprocessor.setTokenizerFactory(tokenizerFactory);
// documentPreprocessor.setSentenceFinalPuncWords(tlp.sentenceFinalPunctuationWords());
// if (encoding != null) {
// documentPreprocessor.setEncoding(encoding);
// }
// timer.start();
// for (int i = argIndex; i < args.length; i++) {
// String filename = args[i];
// try {
// List document = null;
// if (fromXML) {
// document = documentPreprocessor.getSentencesFromXML(filename, sentenceDelimiter, tokenized);
// } else {
// document = documentPreprocessor.getSentencesFromText(filename, escaper, sentenceDelimiter, tagDelimiter);
// }
// log.info("Segmenting file: " + filename + " with " + document.size() + " sentences.");
// PrintWriter pwo = pwOut;
// if (op.testOptions.writeOutputFiles) {
// try {
// pwo = tlpParams.pw(new FileOutputStream(filename + ".stp"));
// } catch (IOException ioe) {
// ioe.printStackTrace();
// }
// }
// int num = 0;
// treePrint.printHeader(pwo, tlp.getEncoding());
// for (Iterator it = document.iterator(); it.hasNext();) {
// num++;
// List sentence = (List) it.next();
// int len = sentence.size();
// numWords += len;
// //            pwErr.println("Parsing [sent. " + num + " len. " + len + "]: " + sentence);
// pwo.println(Sentence.listToString(sentence));
// }
// treePrint.printFooter(pwo);
// if (op.testOptions.writeOutputFiles) {
// pwo.close();
// }
// } catch (IOException e) {
// pwErr.println("Couldn't find file: " + filename);
// }
// 
// } // end for each file
// long millis = timer.stop();
// double wordspersec = numWords / (((double) millis) / 1000);
// NumberFormat nf = new DecimalFormat("0.00"); // easier way!
// pwErr.println("Segmented " + numWords + " words at " + nf.format(wordspersec) + " words per second.");
// }
}

Example 7 with NumberRangeFileFilter

use of edu.stanford.nlp.io.NumberRangeFileFilter in project CoreNLP by stanfordnlp.

the class GrammarCoverageChecker method runTest.

public void runTest(String[] args) {
    // get a parser from file
    LexicalizedParser pd = LexicalizedParser.loadModel(args[0]);
    // in case a serialized options was read in
    op = pd.getOp();
    Treebank testTreebank = op.tlpParams.memoryTreebank();
    int testlow = Integer.parseInt(args[2]);
    int testhigh = Integer.parseInt(args[3]);
    testTreebank.loadPath(args[1], new NumberRangeFileFilter(testlow, testhigh, true));
    op.setOptionsOrWarn(args, 4, args.length);
    testOnTreebank(pd, new EnglishTreebankParserParams(), testTreebank, args[1], pd.stateIndex);
}

Example 8 with NumberRangeFileFilter

use of edu.stanford.nlp.io.NumberRangeFileFilter in project CoreNLP by stanfordnlp.

the class ParentAnnotationStats method getEnglishSplitCategories.

/**
 * This is hardwired to calculate the split categories from English
 * Penn Treebank sections 2-21 with a default cutoff of 300 (as used
 * in ACL03PCFG).  It was added to upgrading of code in cases where no
 * Treebank was available, and the pre-stored list was being used).
 */
public static Set<String> getEnglishSplitCategories(String treebankRoot) {
    TreebankLangParserParams tlpParams = new EnglishTreebankParserParams();
    Treebank trees = tlpParams.memoryTreebank();
    trees.loadPath(treebankRoot, new NumberRangeFileFilter(200, 2199, true));
    return getSplitCategories(trees, 300.0, tlpParams.treebankLanguagePack());
}

Example 9 with NumberRangeFileFilter

use of edu.stanford.nlp.io.NumberRangeFileFilter in project CoreNLP by stanfordnlp.

the class ArgUtils method getWeightedTreebankDescription.

public static Triple<String, FileFilter, Double> getWeightedTreebankDescription(String[] args, int argIndex, String flag) {
    String path = null;
    FileFilter filter = null;
    Double weight = 1.0;
    // the next arguments are the treebank path and maybe the range for testing
    int numSubArgs = numSubArgs(args, argIndex);
    if (numSubArgs > 0 && numSubArgs < 4) {
        argIndex++;
        path = args[argIndex++];
        boolean hasWeight = false;
        if (numSubArgs > 1 && DOUBLE_PATTERN.matcher(args[argIndex + numSubArgs - 2]).matches()) {
            weight = Double.parseDouble(args[argIndex + numSubArgs - 2]);
            hasWeight = true;
            numSubArgs--;
        }
        if (numSubArgs == 2) {
            if (args[argIndex].equals("train")) {
                filter = new RegExFileFilter(".*train.*");
            } else if (args[argIndex].equals("dev")) {
                filter = new RegExFileFilter(".*dev.*");
            } else if (args[argIndex].equals("test")) {
                filter = new RegExFileFilter(".*test.*");
            } else {
                filter = new NumberRangesFileFilter(args[argIndex], true);
            }
            argIndex++;
        } else if (numSubArgs == 3) {
            try {
                int low = Integer.parseInt(args[argIndex]);
                int high = Integer.parseInt(args[argIndex + 1]);
                filter = new NumberRangeFileFilter(low, high, true);
                argIndex += 2;
            } catch (NumberFormatException e) {
                // maybe it's a ranges expression?
                filter = new NumberRangesFileFilter(args[argIndex++], true);
            }
        }
        if (hasWeight) {
            argIndex++;
        }
    } else {
        throw new IllegalArgumentException("Bad arguments after " + flag);
    }
    return Triple.makeTriple(path, filter, weight);
}