Examples with DVModelReranker edu.stanford.nlp.parser.dvparser.DVModelReranker used on opensource projects

Example 1 with DVModelReranker

use of edu.stanford.nlp.parser.dvparser.DVModelReranker in project CoreNLP by stanfordnlp.

the class ConvertModels method readParser.

public static LexicalizedParser readParser(ObjectInputStream in) throws IOException, ClassNotFoundException {
    LexicalizedParser model = ErasureUtils.uncheckedCast(in.readObject());
    Function<List<List<Double>>, SimpleMatrix> f = (x) -> toMatrix(x);
    TwoDimensionalMap<String, String, List<List<Double>>> map2dSM = ErasureUtils.uncheckedCast(in.readObject());
    TwoDimensionalMap<String, String, SimpleMatrix> binaryTransform = transform2DMap(map2dSM, f);
    Map<String, List<List<Double>>> map = ErasureUtils.uncheckedCast(in.readObject());
    Map<String, SimpleMatrix> unaryTransform = transformMap(map, f);
    map2dSM = ErasureUtils.uncheckedCast(in.readObject());
    TwoDimensionalMap<String, String, SimpleMatrix> binaryScore = transform2DMap(map2dSM, f);
    map = ErasureUtils.uncheckedCast(in.readObject());
    Map<String, SimpleMatrix> unaryScore = transformMap(map, f);
    map = ErasureUtils.uncheckedCast(in.readObject());
    Map<String, SimpleMatrix> wordVectors = transformMap(map, f);
    DVModel dvModel = new DVModel(binaryTransform, unaryTransform, binaryScore, unaryScore, wordVectors, model.getOp());
    DVModelReranker reranker = new DVModelReranker(dvModel);
    model.reranker = reranker;
    return model;
}

Example 2 with DVModelReranker

use of edu.stanford.nlp.parser.dvparser.DVModelReranker in project CoreNLP by stanfordnlp.

the class ConvertModels method main.

/**
 * This program converts a sentiment model or an RNN parser model
 * from EJML v23, used by CoreNLP 3.9.2, to a more recent version of
 * EJML, such as v38.  The reason for this is that the EJML v31
 * update changed the serialization of SimpleMatrix in a way that
 * broke all the old models, so for years we never bit the bullet of
 * upgrading.  This script handles the upgrade of our models which
 * use SimpleMatrix.  It needs to be done in two steps.
 * <br>
 * The first conversion turns a model into a file with lists of
 * doubles in place of the SimpleMatrix objects used in ejml.
 * <br>
 * The second conversion turns the lists of doubles back into a new
 * SentimentModel or RNN parser.
 * <br>
 * In between the two steps you should replace the EJML version you
 * are using with 0.38, although no one is judging you if you
 * aimlessly convert back and forth using the same EJML version.
 * <br>
 * Concrete steps:
 * <br>
 * <code> java edu.stanford.nlp.neural.ConvertModels -stage OLD -model SENTIMENT -input edu/stanford/nlp/models/sentiment/sentiment.ser.gz -output sentiment.INT.ser.gz</code>
 * <br>
 * ... update EJML library to v38 or a later
 * <br>
 * <code> java edu.stanford.nlp.neural.ConvertModels -stage NEW -model SENTIMENT -input sentiment.INT.ser.gz -output sentiment.38.ser.gz</code>
 * <br>
 * Congratulations, your old model will now work with EJML v38.
 * <br>
 * To upgrade an RNN model (did anyone train this themselves?) use <code>-model DVPARSER</code>
 * <br>
 * <code> java edu.stanford.nlp.neural.ConvertModels -stage OLD -model DVPARSER -input /u/nlp/data/lexparser/chineseRNN.e21.ser.gz -output /u/nlp/data/lexparser/chineseRNN.INT.ser.gz</code>
 * <br>
 * <code> java edu.stanford.nlp.neural.ConvertModels -stage NEW -model DVPARSER -input /u/nlp/data/lexparser/chineseRNN.INT.ser.gz -output /u/nlp/data/lexparser/chineseRNN.e38.ser.gz</code>
 * <br>
 * To upgrade a neural coref model, use <code>-model COREF</code>
 * <br>
 * <code> java edu.stanford.nlp.neural.ConvertModels -stage OLD -model COREF -input /scr/nlp/data/coref/models/neural/english/english-model-default.ser.gz -output /scr/nlp/data/coref/models/neural/english/english-model-default.INT.ser.gz</code>
 * <br>
 * <code> java edu.stanford.nlp.neural.ConvertModels -stage NEW -model COREF -input /scr/nlp/data/coref/models/neural/english/english-model-default.INT.ser.gz -output /scr/nlp/data/coref/models/neural/english/english-model-default.e39.ser.gz</code>
 * <br>
 * Neural coref models ship with a separate embedding file, although these are also embedded in the model itself.  To upgrade this, use <code>-model EMBEDDING</code>
 * <br>
 * <code> java edu.stanford.nlp.neural.ConvertModels -stage OLD -model EMBEDDING -input /scr/nlp/data/coref/models/neural/english/english-embeddings.e38.ser.gz -output /scr/nlp/data/coref/models/neural/english/english-embeddings.INT.ser.gz</code>
 * <br>
 * <code> java edu.stanford.nlp.neural.ConvertModels -stage NEW -model EMBEDDING -input /scr/nlp/data/coref/models/neural/english/english-embeddings.INT.ser.gz -output /scr/nlp/data/coref/models/neural/english/english-embeddings.e39.ser.gz</code>
 * <br>
 * There is another coref model which isn't used in corenlp, but it might be in the future.  To upgrade this, use <code>-model FASTCOREF</code>
 * <br>
 * <code> java edu.stanford.nlp.neural.ConvertModels -stage OLD -model FASTCOREF -input /scr/nlp/data/coref/models/fastneural/fast-english-model.e38.ser.gz -output /scr/nlp/data/coref/models/fastneural/fast-english-model.INT.ser.gz</code>
 * <br>
 * <code> java edu.stanford.nlp.neural.ConvertModels -stage NEW -model FASTCOREF -input /scr/nlp/data/coref/models/fastneural/fast-english-model.INT.ser.gz -output /scr/nlp/data/coref/models/fastneural/fast-english-model.e39.ser.gz</code>
 * <br>
 *
 * @author <a href=horatio@gmail.com>John Bauer</a>
 */
public static void main(String[] args) throws IOException, ClassNotFoundException, InstantiationException, NoSuchMethodException {
    Properties props = StringUtils.argsToProperties(args);
    final Stage stage;
    try {
        stage = Stage.valueOf(props.getProperty("stage").toUpperCase(Locale.ROOT));
    } catch (IllegalArgumentException | NullPointerException e) {
        throw new IllegalArgumentException("Please specify -stage, either OLD or NEW");
    }
    final Model modelType;
    try {
        modelType = Model.valueOf(props.getProperty("model").toUpperCase(Locale.ROOT));
    } catch (IllegalArgumentException | NullPointerException e) {
        throw new IllegalArgumentException("Please specify -model, either SENTIMENT, DVPARSER, EMBEDDING, COREF, FASTCOREF");
    }
    if (!props.containsKey("input")) {
        throw new IllegalArgumentException("Please specify -input");
    }
    if (!props.containsKey("output")) {
        throw new IllegalArgumentException("Please specify -output");
    }
    String inputPath = props.getProperty("input");
    String outputPath = props.getProperty("output");
    if (modelType == Model.SENTIMENT) {
        if (stage == Stage.OLD) {
            SentimentModel model = SentimentModel.loadSerialized(inputPath);
            ObjectOutputStream out = IOUtils.writeStreamFromString(outputPath);
            writeSentiment(model, out);
            out.close();
        } else {
            ObjectInputStream in = IOUtils.readStreamFromString(inputPath);
            SentimentModel model = readSentiment(in);
            in.close();
            model.saveSerialized(outputPath);
        }
    } else if (modelType == Model.DVPARSER) {
        if (stage == Stage.OLD) {
            LexicalizedParser model = LexicalizedParser.loadModel(inputPath);
            if (model.reranker == null) {
                System.out.println("Nothing to do for " + inputPath);
            } else {
                // will barf if not successful
                DVModelReranker reranker = (DVModelReranker) model.reranker;
                model.reranker = null;
                ObjectOutputStream out = IOUtils.writeStreamFromString(outputPath);
                writeParser(model, reranker, out);
                out.close();
            }
        } else {
            ObjectInputStream in = IOUtils.readStreamFromString(inputPath);
            LexicalizedParser model = readParser(in);
            in.close();
            model.saveParserToSerialized(outputPath);
        }
    } else if (modelType == Model.EMBEDDING) {
        if (stage == Stage.OLD) {
            Embedding embedding = ErasureUtils.uncheckedCast(IOUtils.readObjectFromURLOrClasspathOrFileSystem(inputPath));
            ObjectOutputStream out = IOUtils.writeStreamFromString(outputPath);
            writeEmbedding(embedding, out);
            out.close();
        } else {
            ObjectInputStream in = IOUtils.readStreamFromString(inputPath);
            Embedding embedding = readEmbedding(in);
            in.close();
            IOUtils.writeObjectToFile(embedding, outputPath);
        }
    } else if (modelType == Model.COREF) {
        if (stage == Stage.OLD) {
            NeuralCorefModel model = ErasureUtils.uncheckedCast(IOUtils.readObjectFromURLOrClasspathOrFileSystem(inputPath));
            ObjectOutputStream out = IOUtils.writeStreamFromString(outputPath);
            writeCoref(model, out);
            out.close();
        } else {
            ObjectInputStream in = IOUtils.readStreamFromString(inputPath);
            NeuralCorefModel model = readCoref(in);
            in.close();
            IOUtils.writeObjectToFile(model, outputPath);
        }
    } else if (modelType == Model.FASTCOREF) {
        if (stage == Stage.OLD) {
            FastNeuralCorefModel model = ErasureUtils.uncheckedCast(IOUtils.readObjectFromURLOrClasspathOrFileSystem(inputPath));
            ObjectOutputStream out = IOUtils.writeStreamFromString(outputPath);
            writeFastCoref(model, out);
            out.close();
        } else {
            ObjectInputStream in = IOUtils.readStreamFromString(inputPath);
            FastNeuralCorefModel model = readFastCoref(in);
            in.close();
            IOUtils.writeObjectToFile(model, outputPath);
        }
    } else {
        throw new IllegalArgumentException("Unknown model type " + modelType);
    }
}