Search in sources :

Example 1 with SceneGraphImageRegion

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageRegion in project CoreNLP by stanfordnlp.

the class EntityClassifier method train.

private static void train(List<SceneGraphImage> images, String modelPath, Embedding embeddings) throws IOException {
    RVFDataset<String, String> dataset = new RVFDataset<String, String>();
    SceneGraphSentenceMatcher sentenceMatcher = new SceneGraphSentenceMatcher(embeddings);
    for (SceneGraphImage img : images) {
        for (SceneGraphImageRegion region : img.regions) {
            SemanticGraph sg = region.getEnhancedSemanticGraph();
            SemanticGraphEnhancer.enhance(sg);
            List<Triple<IndexedWord, IndexedWord, String>> relationTriples = sentenceMatcher.getRelationTriples(region);
            for (Triple<IndexedWord, IndexedWord, String> relation : relationTriples) {
                IndexedWord w1 = sg.getNodeByIndexSafe(relation.first.index());
                if (w1 != null) {
                    dataset.add(getDatum(w1, relation.first.get(SceneGraphCoreAnnotations.GoldEntityAnnotation.class), embeddings));
                }
            }
        }
    }
    LinearClassifierFactory<String, String> classifierFactory = new LinearClassifierFactory<String, String>(new QNMinimizer(15), 1e-4, false, REG_STRENGTH);
    Classifier<String, String> classifier = classifierFactory.trainClassifier(dataset);
    IOUtils.writeObjectToFile(classifier, modelPath);
    System.err.println(classifier.evaluateAccuracy(dataset));
}
Also used : RVFDataset(edu.stanford.nlp.classify.RVFDataset) SceneGraphImage(edu.stanford.nlp.scenegraph.image.SceneGraphImage) QNMinimizer(edu.stanford.nlp.optimization.QNMinimizer) Triple(edu.stanford.nlp.util.Triple) LinearClassifierFactory(edu.stanford.nlp.classify.LinearClassifierFactory) SemanticGraph(edu.stanford.nlp.semgraph.SemanticGraph) IndexedWord(edu.stanford.nlp.ling.IndexedWord) SceneGraphImageRegion(edu.stanford.nlp.scenegraph.image.SceneGraphImageRegion)

Example 2 with SceneGraphImageRegion

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageRegion in project CoreNLP by stanfordnlp.

the class GenerateAlignmentData method main.

public static void main(String[] args) throws IOException {
    Properties props = new Properties();
    props.put("annotators", "tokenize,ssplit");
    props.put("ssplit.eolonly", "true");
    StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
    String filename = args[0];
    String sentences = args[1];
    String graphs = args[2];
    BufferedReader reader = IOUtils.readerFromString(filename);
    PrintWriter sentencesFile = IOUtils.getPrintWriter(sentences);
    PrintWriter graphsFile = IOUtils.getPrintWriter(graphs);
    for (String line = reader.readLine(); line != null; line = reader.readLine()) {
        SceneGraphImage img = SceneGraphImage.readFromJSON(line);
        if (img == null) {
            continue;
        }
        for (SceneGraphImageRegion region : img.regions) {
            Annotation doc = new Annotation(region.phrase);
            pipeline.annotate(doc);
            CoreMap sentence = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
            List<CoreLabel> tokens = sentence.get(CoreAnnotations.TokensAnnotation.class);
            String tokenizedSentence = StringUtils.join(tokens.stream().map(CoreLabel::word), " ");
            for (SceneGraphImageAttribute attr : region.attributes) {
                sentencesFile.printf("%s%n", tokenizedSentence);
                graphsFile.printf("%s%n", StringUtils.join(attr.text));
            }
            for (SceneGraphImageRelationship reln : region.relationships) {
                sentencesFile.printf("%s%n", tokenizedSentence);
                graphsFile.printf("%s%n", StringUtils.join(reln.text));
            }
        }
    }
}
Also used : SceneGraphImage(edu.stanford.nlp.scenegraph.image.SceneGraphImage) Properties(java.util.Properties) StanfordCoreNLP(edu.stanford.nlp.pipeline.StanfordCoreNLP) Annotation(edu.stanford.nlp.pipeline.Annotation) SceneGraphImageRelationship(edu.stanford.nlp.scenegraph.image.SceneGraphImageRelationship) CoreLabel(edu.stanford.nlp.ling.CoreLabel) BufferedReader(java.io.BufferedReader) CoreAnnotations(edu.stanford.nlp.ling.CoreAnnotations) SceneGraphImageRegion(edu.stanford.nlp.scenegraph.image.SceneGraphImageRegion) CoreMap(edu.stanford.nlp.util.CoreMap) SceneGraphImageAttribute(edu.stanford.nlp.scenegraph.image.SceneGraphImageAttribute) PrintWriter(java.io.PrintWriter)

Example 3 with SceneGraphImageRegion

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageRegion in project CoreNLP by stanfordnlp.

the class SceneGraphImageDependencyParser method main.

public static void main(String[] args) throws IOException {
    DependencyParser parser = DependencyParser.loadFromModelFile(DependencyParser.DEFAULT_MODEL);
    String filename = args[0];
    BufferedReader reader = IOUtils.readerFromString(filename);
    for (String line = reader.readLine(); line != null; line = reader.readLine()) {
        SceneGraphImage img = SceneGraphImage.readFromJSON(line);
        if (img == null) {
            continue;
        }
        for (SceneGraphImageRegion region : img.regions) {
            if (region.tokens != null) {
                region.gs = parser.predict(region.tokens);
            }
        }
        System.out.println(img.toJSON());
    }
}
Also used : SceneGraphImage(edu.stanford.nlp.scenegraph.image.SceneGraphImage) DependencyParser(edu.stanford.nlp.parser.nndep.DependencyParser) BufferedReader(java.io.BufferedReader) SceneGraphImageRegion(edu.stanford.nlp.scenegraph.image.SceneGraphImageRegion)

Example 4 with SceneGraphImageRegion

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageRegion in project CoreNLP by stanfordnlp.

the class SceneGraphImagePCFGParser method main.

public static void main(String[] args) throws IOException {
    LexicalizedParser parser = LexicalizedParser.getParserFromSerializedFile(PCFG_MODEL);
    TreebankLanguagePack tlp = new PennTreebankLanguagePack();
    GrammaticalStructureFactory gsf = tlp.grammaticalStructureFactory();
    String filename = args[0];
    BufferedReader reader = IOUtils.readerFromString(filename);
    for (String line = reader.readLine(); line != null; line = reader.readLine()) {
        SceneGraphImage img = SceneGraphImage.readFromJSON(line);
        if (img == null) {
            continue;
        }
        for (SceneGraphImageRegion region : img.regions) {
            if (region.tokens != null) {
                for (CoreLabel token : region.tokens) {
                    token.remove(CoreAnnotations.PartOfSpeechAnnotation.class);
                }
                Tree t = parser.apply(region.tokens);
                region.gs = gsf.newGrammaticalStructure(t);
            }
        }
        System.out.println(img.toJSON());
    }
}
Also used : CoreLabel(edu.stanford.nlp.ling.CoreLabel) SceneGraphImage(edu.stanford.nlp.scenegraph.image.SceneGraphImage) LexicalizedParser(edu.stanford.nlp.parser.lexparser.LexicalizedParser) GrammaticalStructureFactory(edu.stanford.nlp.trees.GrammaticalStructureFactory) BufferedReader(java.io.BufferedReader) CoreAnnotations(edu.stanford.nlp.ling.CoreAnnotations) Tree(edu.stanford.nlp.trees.Tree) PennTreebankLanguagePack(edu.stanford.nlp.trees.PennTreebankLanguagePack) TreebankLanguagePack(edu.stanford.nlp.trees.TreebankLanguagePack) SceneGraphImageRegion(edu.stanford.nlp.scenegraph.image.SceneGraphImageRegion) PennTreebankLanguagePack(edu.stanford.nlp.trees.PennTreebankLanguagePack)

Example 5 with SceneGraphImageRegion

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageRegion in project CoreNLP by stanfordnlp.

the class SceneGraphImageCleaner method lemmatize.

public void lemmatize(SceneGraphImage img) {
    StanfordCoreNLP pipeline = getPipeline();
    /* attributes */
    for (SceneGraphImageAttribute attr : img.attributes) {
        String attribute = removeDeterminersAndNumbers(removeFinalPunctuation(attr.attribute));
        String sentence = String.format("She is %s .\n", attribute);
        Annotation doc = new Annotation(sentence);
        pipeline.annotate(doc);
        CoreMap sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
        List<CoreLabel> tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
        attr.attributeGloss = tokens.subList(2, tokens.size() - 1);
        String subject = removeDeterminersAndNumbers(removeFinalPunctuation(attr.text[0]));
        sentence = String.format("The %s is tall .", subject);
        doc = new Annotation(sentence);
        pipeline.annotate(doc);
        sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
        tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
        attr.subjectGloss = tokens.subList(1, tokens.size() - 3);
        attr.subject.labels.add(attr.subjectGloss);
    }
    /* relations */
    for (SceneGraphImageRelationship reln : img.relationships) {
        String object = removeDeterminersAndNumbers(removeFinalPunctuation(reln.text[2]));
        String sentence = String.format("She is the %s .\n", object);
        Annotation doc = new Annotation(sentence);
        pipeline.annotate(doc);
        CoreMap sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
        List<CoreLabel> tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
        reln.objectGloss = tokens.subList(3, tokens.size() - 1);
        reln.object.labels.add(reln.objectGloss);
        String subject = removeDeterminersAndNumbers(removeFinalPunctuation(reln.text[0]));
        sentence = String.format("The %s is tall .", subject);
        doc = new Annotation(sentence);
        pipeline.annotate(doc);
        sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
        tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
        reln.subjectGloss = tokens.subList(1, tokens.size() - 3);
        reln.subject.labels.add(reln.subjectGloss);
        String predicate = removeDeterminersAndNumbers(removeFinalPunctuation(reln.predicate));
        sentence = String.format("A horse %s an apple .", predicate);
        doc = new Annotation(sentence);
        pipeline.annotate(doc);
        sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
        tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
        reln.predicateGloss = tokens.subList(2, tokens.size() - 3);
    }
    for (SceneGraphImageObject object : img.objects) {
        if (object.names.size() > object.labels.size()) {
            for (String name : object.names) {
                String x = removeDeterminersAndNumbers(removeFinalPunctuation(name));
                String sentence = String.format("The %s is tall .", x);
                Annotation doc = new Annotation(sentence);
                pipeline.annotate(doc);
                CoreMap sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
                List<CoreLabel> tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
                object.labels.add(tokens.subList(1, tokens.size() - 3));
            }
        }
    }
    StanfordCoreNLP tokenizerPipeline = getTokenizerPipeline();
    for (SceneGraphImageRegion region : img.regions) {
        Annotation doc = new Annotation(region.phrase.toLowerCase());
        tokenizerPipeline.annotate(doc);
        CoreMap sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
        region.tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
    }
}
Also used : StanfordCoreNLP(edu.stanford.nlp.pipeline.StanfordCoreNLP) Annotation(edu.stanford.nlp.pipeline.Annotation) SceneGraphImageRelationship(edu.stanford.nlp.scenegraph.image.SceneGraphImageRelationship) SceneGraphImageObject(edu.stanford.nlp.scenegraph.image.SceneGraphImageObject) CoreLabel(edu.stanford.nlp.ling.CoreLabel) CoreAnnotations(edu.stanford.nlp.ling.CoreAnnotations) CoreMap(edu.stanford.nlp.util.CoreMap) SceneGraphImageRegion(edu.stanford.nlp.scenegraph.image.SceneGraphImageRegion) SceneGraphImageAttribute(edu.stanford.nlp.scenegraph.image.SceneGraphImageAttribute)

Aggregations

SceneGraphImageRegion (edu.stanford.nlp.scenegraph.image.SceneGraphImageRegion)14 SceneGraphImage (edu.stanford.nlp.scenegraph.image.SceneGraphImage)13 BufferedReader (java.io.BufferedReader)8 SceneGraphImageAttribute (edu.stanford.nlp.scenegraph.image.SceneGraphImageAttribute)5 SceneGraphImageRelationship (edu.stanford.nlp.scenegraph.image.SceneGraphImageRelationship)5 PrintWriter (java.io.PrintWriter)5 CoreLabel (edu.stanford.nlp.ling.CoreLabel)4 SemanticGraph (edu.stanford.nlp.semgraph.SemanticGraph)4 CoreAnnotations (edu.stanford.nlp.ling.CoreAnnotations)3 IndexedWord (edu.stanford.nlp.ling.IndexedWord)2 Embedding (edu.stanford.nlp.neural.Embedding)2 Annotation (edu.stanford.nlp.pipeline.Annotation)2 StanfordCoreNLP (edu.stanford.nlp.pipeline.StanfordCoreNLP)2 CoreMap (edu.stanford.nlp.util.CoreMap)2 Triple (edu.stanford.nlp.util.Triple)2 Dataset (edu.stanford.nlp.classify.Dataset)1 KNNClassifierFactory (edu.stanford.nlp.classify.KNNClassifierFactory)1 LinearClassifierFactory (edu.stanford.nlp.classify.LinearClassifierFactory)1 RVFDataset (edu.stanford.nlp.classify.RVFDataset)1 RVFDatum (edu.stanford.nlp.ling.RVFDatum)1