Examples with SceneGraphImageObject edu.stanford.nlp.scenegraph.image.SceneGraphImageObject used on opensource projects

Example 1 with SceneGraphImageObject

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageObject in project CoreNLP by stanfordnlp.

the class SceneGraphImageCleaner method lemmatize.

public void lemmatize(SceneGraphImage img) {
    StanfordCoreNLP pipeline = getPipeline();
    /* attributes */
    for (SceneGraphImageAttribute attr : img.attributes) {
        String attribute = removeDeterminersAndNumbers(removeFinalPunctuation(attr.attribute));
        String sentence = String.format("She is %s .\n", attribute);
        Annotation doc = new Annotation(sentence);
        pipeline.annotate(doc);
        CoreMap sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
        List<CoreLabel> tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
        attr.attributeGloss = tokens.subList(2, tokens.size() - 1);
        String subject = removeDeterminersAndNumbers(removeFinalPunctuation(attr.text[0]));
        sentence = String.format("The %s is tall .", subject);
        doc = new Annotation(sentence);
        pipeline.annotate(doc);
        sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
        tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
        attr.subjectGloss = tokens.subList(1, tokens.size() - 3);
        attr.subject.labels.add(attr.subjectGloss);
    }
    /* relations */
    for (SceneGraphImageRelationship reln : img.relationships) {
        String object = removeDeterminersAndNumbers(removeFinalPunctuation(reln.text[2]));
        String sentence = String.format("She is the %s .\n", object);
        Annotation doc = new Annotation(sentence);
        pipeline.annotate(doc);
        CoreMap sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
        List<CoreLabel> tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
        reln.objectGloss = tokens.subList(3, tokens.size() - 1);
        reln.object.labels.add(reln.objectGloss);
        String subject = removeDeterminersAndNumbers(removeFinalPunctuation(reln.text[0]));
        sentence = String.format("The %s is tall .", subject);
        doc = new Annotation(sentence);
        pipeline.annotate(doc);
        sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
        tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
        reln.subjectGloss = tokens.subList(1, tokens.size() - 3);
        reln.subject.labels.add(reln.subjectGloss);
        String predicate = removeDeterminersAndNumbers(removeFinalPunctuation(reln.predicate));
        sentence = String.format("A horse %s an apple .", predicate);
        doc = new Annotation(sentence);
        pipeline.annotate(doc);
        sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
        tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
        reln.predicateGloss = tokens.subList(2, tokens.size() - 3);
    }
    for (SceneGraphImageObject object : img.objects) {
        if (object.names.size() > object.labels.size()) {
            for (String name : object.names) {
                String x = removeDeterminersAndNumbers(removeFinalPunctuation(name));
                String sentence = String.format("The %s is tall .", x);
                Annotation doc = new Annotation(sentence);
                pipeline.annotate(doc);
                CoreMap sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
                List<CoreLabel> tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
                object.labels.add(tokens.subList(1, tokens.size() - 3));
            }
        }
    }
    StanfordCoreNLP tokenizerPipeline = getTokenizerPipeline();
    for (SceneGraphImageRegion region : img.regions) {
        Annotation doc = new Annotation(region.phrase.toLowerCase());
        tokenizerPipeline.annotate(doc);
        CoreMap sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
        region.tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
    }
}

Example 2 with SceneGraphImageObject

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageObject in project CoreNLP by stanfordnlp.

the class SceneGraphEvaluation method toSmatchString.

public void toSmatchString(SceneGraph scene, SceneGraphImageRegion region, PrintWriter predWriter, PrintWriter goldWriter) throws IOException {
    StringBuffer predString = new StringBuffer();
    StringBuffer goldString = new StringBuffer();
    List<SceneGraphNode> nodes = scene.nodeListSorted();
    boolean first = true;
    for (int i = 0, sz = nodes.size(); i < sz; i++) {
        SceneGraphNode node = nodes.get(i);
        if (!first) {
            predString.append("|||");
        } else {
            first = false;
        }
        predString.append(String.format("instance###a%d###%s", i, node.toJSONString()));
        for (SceneGraphAttribute attr : node.getAttributes()) {
            predString.append("|||");
            predString.append(String.format("is###a%d###%s", i, attr.toString()));
        }
    }
    for (SceneGraphRelation reln : scene.relationListSorted()) {
        int node1Idx = nodes.indexOf(reln.getSource());
        int node2Idx = nodes.indexOf(reln.getTarget());
        predString.append("|||");
        predString.append(String.format("%s###a%d###%s", reln.getRelation(), node1Idx, node2Idx));
    }
    if (first) {
        predString.append("-");
    }
    predWriter.println(predString.toString());
    Map<SceneGraphImageObject, Integer> objects = Generics.newHashMap();
    first = true;
    int i = 0;
    for (SceneGraphImageAttribute attr : region.attributes) {
        if (!objects.containsKey(attr.subject)) {
            if (!first) {
                goldString.append("|||");
            } else {
                first = false;
            }
            goldString.append(String.format("instance###b%d###%s", i, attr.subjectLemmaGloss()));
            objects.put(attr.subject, i++);
        }
    }
    for (SceneGraphImageRelationship reln : region.relationships) {
        if (!objects.containsKey(reln.subject)) {
            if (!first) {
                goldString.append("|||");
            } else {
                first = false;
            }
            goldString.append(String.format("instance###b%d###%s", i, reln.subjectLemmaGloss()));
            objects.put(reln.subject, i++);
        }
        if (!objects.containsKey(reln.object)) {
            goldString.append("|||");
            goldString.append(String.format("instance###b%d###%s", i, reln.objectLemmaGloss()));
            objects.put(reln.object, i++);
        }
    }
    for (SceneGraphImageAttribute attr : region.attributes) {
        goldString.append("|||");
        goldString.append(String.format("is###b%d###%s", objects.get(attr.subject), attr.attributeLemmaGloss()));
    }
    for (SceneGraphImageRelationship reln : region.relationships) {
        goldString.append("|||");
        goldString.append(String.format("%s###b%d###b%d", reln.predicateLemmaGloss(), objects.get(reln.subject), objects.get(reln.object)));
    }
    goldWriter.println(goldString.toString());
}

Example 3 with SceneGraphImageObject

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageObject in project CoreNLP by stanfordnlp.

the class SceneGraphEvaluation method toSmatchString.

public void toSmatchString(SceneGraphImageRegion predicted, SceneGraphImageRegion region, PrintWriter predWriter, PrintWriter goldWriter) throws IOException {
    StringBuffer predString = new StringBuffer();
    Map<SceneGraphImageObject, Integer> predictedObjects = Generics.newHashMap();
    boolean first = true;
    int i = 0;
    for (SceneGraphImageAttribute attr : predicted.attributes) {
        if (!predictedObjects.containsKey(attr.subject)) {
            if (!first) {
                predString.append("|||");
            } else {
                first = false;
            }
            predString.append(String.format("instance###a%d###%s", i, attr.subjectLemmaGloss()));
            predictedObjects.put(attr.subject, i++);
        }
    }
    for (SceneGraphImageRelationship reln : predicted.relationships) {
        if (!predictedObjects.containsKey(reln.subject)) {
            if (!first) {
                predString.append("|||");
            } else {
                first = false;
            }
            predString.append(String.format("instance###a%d###%s", i, reln.subjectLemmaGloss()));
            predictedObjects.put(reln.subject, i++);
        }
        if (!predictedObjects.containsKey(reln.object)) {
            predString.append("|||");
            predString.append(String.format("instance###a%d###%s", i, reln.objectLemmaGloss()));
            predictedObjects.put(reln.object, i++);
        }
    }
    for (SceneGraphImageAttribute attr : predicted.attributes) {
        predString.append("|||");
        predString.append(String.format("is###a%d###%s", predictedObjects.get(attr.subject), attr.attributeLemmaGloss()));
    }
    for (SceneGraphImageRelationship reln : predicted.relationships) {
        predString.append("|||");
        predString.append(String.format("%s###a%d###a%d", reln.predicateLemmaGloss(), predictedObjects.get(reln.subject), predictedObjects.get(reln.object)));
    }
    predWriter.println(predString.toString());
    StringBuffer goldString = new StringBuffer();
    Map<SceneGraphImageObject, Integer> objects = Generics.newHashMap();
    first = true;
    i = 0;
    for (SceneGraphImageAttribute attr : region.attributes) {
        if (!objects.containsKey(attr.subject)) {
            if (!first) {
                goldString.append("|||");
            } else {
                first = false;
            }
            goldString.append(String.format("instance###b%d###%s", i, attr.subjectLemmaGloss()));
            objects.put(attr.subject, i++);
        }
    }
    for (SceneGraphImageRelationship reln : region.relationships) {
        if (!objects.containsKey(reln.subject)) {
            if (!first) {
                goldString.append("|||");
            } else {
                first = false;
            }
            goldString.append(String.format("instance###b%d###%s", i, reln.subjectLemmaGloss()));
            objects.put(reln.subject, i++);
        }
        if (!objects.containsKey(reln.object)) {
            goldString.append("|||");
            goldString.append(String.format("instance###b%d###%s", i, reln.objectLemmaGloss()));
            objects.put(reln.object, i++);
        }
    }
    for (SceneGraphImageAttribute attr : region.attributes) {
        goldString.append("|||");
        goldString.append(String.format("is###b%d###%s", objects.get(attr.subject), attr.attributeLemmaGloss()));
    }
    for (SceneGraphImageRelationship reln : region.relationships) {
        goldString.append("|||");
        goldString.append(String.format("%s###b%d###b%d", reln.predicateLemmaGloss(), objects.get(reln.subject), objects.get(reln.object)));
    }
    goldWriter.println(goldString.toString());
}