Examples with SceneGraphImageAttribute edu.stanford.nlp.scenegraph.image.SceneGraphImageAttribute used on opensource projects

Example 6 with SceneGraphImageAttribute

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageAttribute in project CoreNLP by stanfordnlp.

the class GroundTruthConverter method main.

public static void main(String[] args) throws IOException {
    BufferedReader reader = IOUtils.readerFromString(args[0]);
    for (String line = reader.readLine(); line != null; line = reader.readLine()) {
        SceneGraphImage img = SceneGraphImage.readFromJSON(line);
        if (img == null) {
            continue;
        }
        for (SceneGraphImageRegion region : img.regions) {
            SceneGraphImage predictedImg = new SceneGraphImage();
            predictedImg.id = img.id;
            predictedImg.url = img.url;
            predictedImg.height = img.height;
            predictedImg.width = img.width;
            Set<Integer> objectIds = Generics.newHashSet();
            for (SceneGraphImageAttribute attr : region.attributes) {
                objectIds.add(img.objects.indexOf(attr.subject));
            }
            for (SceneGraphImageRelationship reln : region.relationships) {
                objectIds.add(img.objects.indexOf(reln.subject));
                objectIds.add(img.objects.indexOf(reln.object));
            }
            predictedImg.objects = Generics.newArrayList();
            for (Integer objectId : objectIds) {
                predictedImg.objects.add(img.objects.get(objectId));
            }
            SceneGraphImageRegion newRegion = new SceneGraphImageRegion();
            newRegion.phrase = region.phrase;
            newRegion.x = region.x;
            newRegion.y = region.y;
            newRegion.h = region.h;
            newRegion.w = region.w;
            newRegion.attributes = Generics.newHashSet();
            newRegion.relationships = Generics.newHashSet();
            predictedImg.regions = Generics.newArrayList();
            predictedImg.regions.add(newRegion);
            predictedImg.attributes = Generics.newLinkedList();
            for (SceneGraphImageAttribute attr : region.attributes) {
                SceneGraphImageAttribute attrCopy = attr.clone();
                attrCopy.region = newRegion;
                attrCopy.image = predictedImg;
                predictedImg.addAttribute(attrCopy);
            }
            predictedImg.relationships = Generics.newLinkedList();
            for (SceneGraphImageRelationship reln : region.relationships) {
                SceneGraphImageRelationship relnCopy = reln.clone();
                relnCopy.image = predictedImg;
                relnCopy.region = newRegion;
                predictedImg.addRelationship(relnCopy);
            }
            System.out.println(predictedImg.toJSON());
        }
    }
}

Example 7 with SceneGraphImageAttribute

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageAttribute in project CoreNLP by stanfordnlp.

the class SceneGraphEvaluation method toSmatchString.

public void toSmatchString(SceneGraph scene, SceneGraphImageRegion region, PrintWriter predWriter, PrintWriter goldWriter) throws IOException {
    StringBuffer predString = new StringBuffer();
    StringBuffer goldString = new StringBuffer();
    List<SceneGraphNode> nodes = scene.nodeListSorted();
    boolean first = true;
    for (int i = 0, sz = nodes.size(); i < sz; i++) {
        SceneGraphNode node = nodes.get(i);
        if (!first) {
            predString.append("|||");
        } else {
            first = false;
        }
        predString.append(String.format("instance###a%d###%s", i, node.toJSONString()));
        for (SceneGraphAttribute attr : node.getAttributes()) {
            predString.append("|||");
            predString.append(String.format("is###a%d###%s", i, attr.toString()));
        }
    }
    for (SceneGraphRelation reln : scene.relationListSorted()) {
        int node1Idx = nodes.indexOf(reln.getSource());
        int node2Idx = nodes.indexOf(reln.getTarget());
        predString.append("|||");
        predString.append(String.format("%s###a%d###%s", reln.getRelation(), node1Idx, node2Idx));
    }
    if (first) {
        predString.append("-");
    }
    predWriter.println(predString.toString());
    Map<SceneGraphImageObject, Integer> objects = Generics.newHashMap();
    first = true;
    int i = 0;
    for (SceneGraphImageAttribute attr : region.attributes) {
        if (!objects.containsKey(attr.subject)) {
            if (!first) {
                goldString.append("|||");
            } else {
                first = false;
            }
            goldString.append(String.format("instance###b%d###%s", i, attr.subjectLemmaGloss()));
            objects.put(attr.subject, i++);
        }
    }
    for (SceneGraphImageRelationship reln : region.relationships) {
        if (!objects.containsKey(reln.subject)) {
            if (!first) {
                goldString.append("|||");
            } else {
                first = false;
            }
            goldString.append(String.format("instance###b%d###%s", i, reln.subjectLemmaGloss()));
            objects.put(reln.subject, i++);
        }
        if (!objects.containsKey(reln.object)) {
            goldString.append("|||");
            goldString.append(String.format("instance###b%d###%s", i, reln.objectLemmaGloss()));
            objects.put(reln.object, i++);
        }
    }
    for (SceneGraphImageAttribute attr : region.attributes) {
        goldString.append("|||");
        goldString.append(String.format("is###b%d###%s", objects.get(attr.subject), attr.attributeLemmaGloss()));
    }
    for (SceneGraphImageRelationship reln : region.relationships) {
        goldString.append("|||");
        goldString.append(String.format("%s###b%d###b%d", reln.predicateLemmaGloss(), objects.get(reln.subject), objects.get(reln.object)));
    }
    goldWriter.println(goldString.toString());
}

Example 8 with SceneGraphImageAttribute

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageAttribute in project CoreNLP by stanfordnlp.

the class SceneGraphEvaluation method evaluate.

public Triple<Double, Double, Double> evaluate(SceneGraphImageRegion predicted, SceneGraphImageRegion region) {
    Counter<SceneGraphRelationTriplet> goldTriplets = new ClassicCounter<SceneGraphRelationTriplet>();
    Counter<SceneGraphRelationTriplet> predictedTriplets = new ClassicCounter<SceneGraphRelationTriplet>();
    for (SceneGraphImageAttribute attr : predicted.attributes) {
        SceneGraphRelationTriplet t = new SceneGraphRelationTriplet(attr);
        predictedTriplets.incrementCount(t);
    }
    for (SceneGraphImageRelationship reln : predicted.relationships) {
        SceneGraphRelationTriplet t = new SceneGraphRelationTriplet(reln);
        predictedTriplets.incrementCount(t);
    }
    for (SceneGraphImageAttribute attr : region.attributes) {
        SceneGraphRelationTriplet t = new SceneGraphRelationTriplet(attr);
        goldTriplets.incrementCount(t);
    }
    for (SceneGraphImageRelationship reln : region.relationships) {
        SceneGraphRelationTriplet t = new SceneGraphRelationTriplet(reln);
        goldTriplets.incrementCount(t);
    }
    double predictedCount = predictedTriplets.totalCount();
    double goldCount = goldTriplets.totalCount();
    double numerator = 0.0;
    for (SceneGraphRelationTriplet t : goldTriplets.keySet()) {
        double gold = goldTriplets.getCount(t);
        double pred = predictedTriplets.getCount(t);
        numerator += Math.min(gold, pred);
    }
    double precision = predictedCount > 0 ? numerator / predictedCount : 1.0;
    double recall = goldCount > 0 ? numerator / goldCount : 1.0;
    double f1 = (precision + recall > 0) ? 2 * precision * recall / (precision + recall) : 0.0;
    Triple<Double, Double, Double> scores = new Triple<Double, Double, Double>(precision, recall, f1);
    return scores;
}

Example 9 with SceneGraphImageAttribute

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageAttribute in project CoreNLP by stanfordnlp.

the class SceneGraphEvaluation method toSmatchString.

public void toSmatchString(SceneGraphImageRegion predicted, SceneGraphImageRegion region, PrintWriter predWriter, PrintWriter goldWriter) throws IOException {
    StringBuffer predString = new StringBuffer();
    Map<SceneGraphImageObject, Integer> predictedObjects = Generics.newHashMap();
    boolean first = true;
    int i = 0;
    for (SceneGraphImageAttribute attr : predicted.attributes) {
        if (!predictedObjects.containsKey(attr.subject)) {
            if (!first) {
                predString.append("|||");
            } else {
                first = false;
            }
            predString.append(String.format("instance###a%d###%s", i, attr.subjectLemmaGloss()));
            predictedObjects.put(attr.subject, i++);
        }
    }
    for (SceneGraphImageRelationship reln : predicted.relationships) {
        if (!predictedObjects.containsKey(reln.subject)) {
            if (!first) {
                predString.append("|||");
            } else {
                first = false;
            }
            predString.append(String.format("instance###a%d###%s", i, reln.subjectLemmaGloss()));
            predictedObjects.put(reln.subject, i++);
        }
        if (!predictedObjects.containsKey(reln.object)) {
            predString.append("|||");
            predString.append(String.format("instance###a%d###%s", i, reln.objectLemmaGloss()));
            predictedObjects.put(reln.object, i++);
        }
    }
    for (SceneGraphImageAttribute attr : predicted.attributes) {
        predString.append("|||");
        predString.append(String.format("is###a%d###%s", predictedObjects.get(attr.subject), attr.attributeLemmaGloss()));
    }
    for (SceneGraphImageRelationship reln : predicted.relationships) {
        predString.append("|||");
        predString.append(String.format("%s###a%d###a%d", reln.predicateLemmaGloss(), predictedObjects.get(reln.subject), predictedObjects.get(reln.object)));
    }
    predWriter.println(predString.toString());
    StringBuffer goldString = new StringBuffer();
    Map<SceneGraphImageObject, Integer> objects = Generics.newHashMap();
    first = true;
    i = 0;
    for (SceneGraphImageAttribute attr : region.attributes) {
        if (!objects.containsKey(attr.subject)) {
            if (!first) {
                goldString.append("|||");
            } else {
                first = false;
            }
            goldString.append(String.format("instance###b%d###%s", i, attr.subjectLemmaGloss()));
            objects.put(attr.subject, i++);
        }
    }
    for (SceneGraphImageRelationship reln : region.relationships) {
        if (!objects.containsKey(reln.subject)) {
            if (!first) {
                goldString.append("|||");
            } else {
                first = false;
            }
            goldString.append(String.format("instance###b%d###%s", i, reln.subjectLemmaGloss()));
            objects.put(reln.subject, i++);
        }
        if (!objects.containsKey(reln.object)) {
            goldString.append("|||");
            goldString.append(String.format("instance###b%d###%s", i, reln.objectLemmaGloss()));
            objects.put(reln.object, i++);
        }
    }
    for (SceneGraphImageAttribute attr : region.attributes) {
        goldString.append("|||");
        goldString.append(String.format("is###b%d###%s", objects.get(attr.subject), attr.attributeLemmaGloss()));
    }
    for (SceneGraphImageRelationship reln : region.relationships) {
        goldString.append("|||");
        goldString.append(String.format("%s###b%d###b%d", reln.predicateLemmaGloss(), objects.get(reln.subject), objects.get(reln.object)));
    }
    goldWriter.println(goldString.toString());
}

Example 10 with SceneGraphImageAttribute

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageAttribute in project CoreNLP by stanfordnlp.

the class SceneGraphSentenceMatcher method getRelationTriples.

/**
 * Aligns nodes in the scene graph to words in the corresponding sentence.
 * Returns a list of {@link Triple} of the IndexedWord corresponding to the
 * subject, the IndexedWord corresponding to the object or attribute and the
 * relation between them.
 *
 * @param region SceneGraphRegion with sentence and corresponding scene graph.
 * @return List of relation triples.
 */
public List<Triple<IndexedWord, IndexedWord, String>> getRelationTriples(SceneGraphImageRegion region) {
    List<Triple<IndexedWord, IndexedWord, String>> triples = Generics.newLinkedList();
    /* Perform some of the enhancements.
     * We don't resolve plurals because we wouldn't know
     * to which copy the nodes in the scene graph should be aligned
     * and we don't collapse particles and compounds because we should
     * be able to align them just using the head word. */
    SemanticGraph sg = region.getBasicSemanticGraph();
    SemanticGraphEnhancer.processQuanftificationModifiers(sg);
    SemanticGraphEnhancer.resolvePronouns(sg);
    for (SceneGraphImageAttribute attr : region.attributes) {
        boolean attrContainsNull = false;
        List<IndexedWord> subjResult = getMatch(region, attr.subjectGloss, attr.subjectLemmaGloss(), sg, false);
        attrContainsNull = attrContainsNull || containsNull(subjResult);
        List<IndexedWord> attrResult = getMatch(region, attr.attributeGloss, attr.attributeLemmaGloss(), sg, true);
        attrContainsNull = attrContainsNull || containsNull(attrResult);
        if (!attrContainsNull) {
            /* Construct attribute triple. */
            Pair<IndexedWord, IndexedWord> res = findClosestPair(subjResult, attrResult);
            res.first.set(SceneGraphCoreAnnotations.SceneGraphEntitiyAnnotation.class, attr.subjectLemmaGloss());
            res.second.set(SceneGraphCoreAnnotations.SceneGraphEntitiyAnnotation.class, attr.attributeLemmaGloss());
            triples.add(new Triple<IndexedWord, IndexedWord, String>(res.first, res.second, "is"));
        }
    }
    for (SceneGraphImageRelationship reln : region.relationships) {
        boolean relnContainsNull = false;
        List<IndexedWord> subjResult = getMatch(region, reln.subjectGloss, reln.subjectLemmaGloss(), sg, false);
        relnContainsNull = relnContainsNull || containsNull(subjResult);
        List<IndexedWord> objResult = getMatch(region, reln.objectGloss, reln.objectLemmaGloss(), sg, false);
        relnContainsNull = relnContainsNull || containsNull(objResult);
        if (!relnContainsNull) {
            /* Construct relation triple. */
            Pair<IndexedWord, IndexedWord> res = findClosestPair(subjResult, objResult);
            res.first.set(SceneGraphCoreAnnotations.SceneGraphEntitiyAnnotation.class, reln.subjectLemmaGloss());
            res.second.set(SceneGraphCoreAnnotations.SceneGraphEntitiyAnnotation.class, reln.objectLemmaGloss());
            triples.add(new Triple<IndexedWord, IndexedWord, String>(res.first, res.second, reln.predicateLemmaGloss()));
        }
    }
    return triples;
}