Examples with SceneGraphImageRelationship edu.stanford.nlp.scenegraph.image.SceneGraphImageRelationship used on opensource projects

Example 6 with SceneGraphImageRelationship

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageRelationship in project CoreNLP by stanfordnlp.

the class SceneGraphEvaluation method toSmatchString.

public void toSmatchString(SceneGraph scene, SceneGraphImageRegion region, PrintWriter predWriter, PrintWriter goldWriter) throws IOException {
    StringBuffer predString = new StringBuffer();
    StringBuffer goldString = new StringBuffer();
    List<SceneGraphNode> nodes = scene.nodeListSorted();
    boolean first = true;
    for (int i = 0, sz = nodes.size(); i < sz; i++) {
        SceneGraphNode node = nodes.get(i);
        if (!first) {
            predString.append("|||");
        } else {
            first = false;
        }
        predString.append(String.format("instance###a%d###%s", i, node.toJSONString()));
        for (SceneGraphAttribute attr : node.getAttributes()) {
            predString.append("|||");
            predString.append(String.format("is###a%d###%s", i, attr.toString()));
        }
    }
    for (SceneGraphRelation reln : scene.relationListSorted()) {
        int node1Idx = nodes.indexOf(reln.getSource());
        int node2Idx = nodes.indexOf(reln.getTarget());
        predString.append("|||");
        predString.append(String.format("%s###a%d###%s", reln.getRelation(), node1Idx, node2Idx));
    }
    if (first) {
        predString.append("-");
    }
    predWriter.println(predString.toString());
    Map<SceneGraphImageObject, Integer> objects = Generics.newHashMap();
    first = true;
    int i = 0;
    for (SceneGraphImageAttribute attr : region.attributes) {
        if (!objects.containsKey(attr.subject)) {
            if (!first) {
                goldString.append("|||");
            } else {
                first = false;
            }
            goldString.append(String.format("instance###b%d###%s", i, attr.subjectLemmaGloss()));
            objects.put(attr.subject, i++);
        }
    }
    for (SceneGraphImageRelationship reln : region.relationships) {
        if (!objects.containsKey(reln.subject)) {
            if (!first) {
                goldString.append("|||");
            } else {
                first = false;
            }
            goldString.append(String.format("instance###b%d###%s", i, reln.subjectLemmaGloss()));
            objects.put(reln.subject, i++);
        }
        if (!objects.containsKey(reln.object)) {
            goldString.append("|||");
            goldString.append(String.format("instance###b%d###%s", i, reln.objectLemmaGloss()));
            objects.put(reln.object, i++);
        }
    }
    for (SceneGraphImageAttribute attr : region.attributes) {
        goldString.append("|||");
        goldString.append(String.format("is###b%d###%s", objects.get(attr.subject), attr.attributeLemmaGloss()));
    }
    for (SceneGraphImageRelationship reln : region.relationships) {
        goldString.append("|||");
        goldString.append(String.format("%s###b%d###b%d", reln.predicateLemmaGloss(), objects.get(reln.subject), objects.get(reln.object)));
    }
    goldWriter.println(goldString.toString());
}

Example 7 with SceneGraphImageRelationship

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageRelationship in project CoreNLP by stanfordnlp.

the class SceneGraphEvaluation method evaluate.

public Triple<Double, Double, Double> evaluate(SceneGraphImageRegion predicted, SceneGraphImageRegion region) {
    Counter<SceneGraphRelationTriplet> goldTriplets = new ClassicCounter<SceneGraphRelationTriplet>();
    Counter<SceneGraphRelationTriplet> predictedTriplets = new ClassicCounter<SceneGraphRelationTriplet>();
    for (SceneGraphImageAttribute attr : predicted.attributes) {
        SceneGraphRelationTriplet t = new SceneGraphRelationTriplet(attr);
        predictedTriplets.incrementCount(t);
    }
    for (SceneGraphImageRelationship reln : predicted.relationships) {
        SceneGraphRelationTriplet t = new SceneGraphRelationTriplet(reln);
        predictedTriplets.incrementCount(t);
    }
    for (SceneGraphImageAttribute attr : region.attributes) {
        SceneGraphRelationTriplet t = new SceneGraphRelationTriplet(attr);
        goldTriplets.incrementCount(t);
    }
    for (SceneGraphImageRelationship reln : region.relationships) {
        SceneGraphRelationTriplet t = new SceneGraphRelationTriplet(reln);
        goldTriplets.incrementCount(t);
    }
    double predictedCount = predictedTriplets.totalCount();
    double goldCount = goldTriplets.totalCount();
    double numerator = 0.0;
    for (SceneGraphRelationTriplet t : goldTriplets.keySet()) {
        double gold = goldTriplets.getCount(t);
        double pred = predictedTriplets.getCount(t);
        numerator += Math.min(gold, pred);
    }
    double precision = predictedCount > 0 ? numerator / predictedCount : 1.0;
    double recall = goldCount > 0 ? numerator / goldCount : 1.0;
    double f1 = (precision + recall > 0) ? 2 * precision * recall / (precision + recall) : 0.0;
    Triple<Double, Double, Double> scores = new Triple<Double, Double, Double>(precision, recall, f1);
    return scores;
}

Example 8 with SceneGraphImageRelationship

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageRelationship in project CoreNLP by stanfordnlp.

the class SceneGraphEvaluation method toSmatchString.

public void toSmatchString(SceneGraphImageRegion predicted, SceneGraphImageRegion region, PrintWriter predWriter, PrintWriter goldWriter) throws IOException {
    StringBuffer predString = new StringBuffer();
    Map<SceneGraphImageObject, Integer> predictedObjects = Generics.newHashMap();
    boolean first = true;
    int i = 0;
    for (SceneGraphImageAttribute attr : predicted.attributes) {
        if (!predictedObjects.containsKey(attr.subject)) {
            if (!first) {
                predString.append("|||");
            } else {
                first = false;
            }
            predString.append(String.format("instance###a%d###%s", i, attr.subjectLemmaGloss()));
            predictedObjects.put(attr.subject, i++);
        }
    }
    for (SceneGraphImageRelationship reln : predicted.relationships) {
        if (!predictedObjects.containsKey(reln.subject)) {
            if (!first) {
                predString.append("|||");
            } else {
                first = false;
            }
            predString.append(String.format("instance###a%d###%s", i, reln.subjectLemmaGloss()));
            predictedObjects.put(reln.subject, i++);
        }
        if (!predictedObjects.containsKey(reln.object)) {
            predString.append("|||");
            predString.append(String.format("instance###a%d###%s", i, reln.objectLemmaGloss()));
            predictedObjects.put(reln.object, i++);
        }
    }
    for (SceneGraphImageAttribute attr : predicted.attributes) {
        predString.append("|||");
        predString.append(String.format("is###a%d###%s", predictedObjects.get(attr.subject), attr.attributeLemmaGloss()));
    }
    for (SceneGraphImageRelationship reln : predicted.relationships) {
        predString.append("|||");
        predString.append(String.format("%s###a%d###a%d", reln.predicateLemmaGloss(), predictedObjects.get(reln.subject), predictedObjects.get(reln.object)));
    }
    predWriter.println(predString.toString());
    StringBuffer goldString = new StringBuffer();
    Map<SceneGraphImageObject, Integer> objects = Generics.newHashMap();
    first = true;
    i = 0;
    for (SceneGraphImageAttribute attr : region.attributes) {
        if (!objects.containsKey(attr.subject)) {
            if (!first) {
                goldString.append("|||");
            } else {
                first = false;
            }
            goldString.append(String.format("instance###b%d###%s", i, attr.subjectLemmaGloss()));
            objects.put(attr.subject, i++);
        }
    }
    for (SceneGraphImageRelationship reln : region.relationships) {
        if (!objects.containsKey(reln.subject)) {
            if (!first) {
                goldString.append("|||");
            } else {
                first = false;
            }
            goldString.append(String.format("instance###b%d###%s", i, reln.subjectLemmaGloss()));
            objects.put(reln.subject, i++);
        }
        if (!objects.containsKey(reln.object)) {
            goldString.append("|||");
            goldString.append(String.format("instance###b%d###%s", i, reln.objectLemmaGloss()));
            objects.put(reln.object, i++);
        }
    }
    for (SceneGraphImageAttribute attr : region.attributes) {
        goldString.append("|||");
        goldString.append(String.format("is###b%d###%s", objects.get(attr.subject), attr.attributeLemmaGloss()));
    }
    for (SceneGraphImageRelationship reln : region.relationships) {
        goldString.append("|||");
        goldString.append(String.format("%s###b%d###b%d", reln.predicateLemmaGloss(), objects.get(reln.subject), objects.get(reln.object)));
    }
    goldWriter.println(goldString.toString());
}

Example 9 with SceneGraphImageRelationship

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageRelationship in project CoreNLP by stanfordnlp.

the class SceneGraphSentenceMatcher method getRelationTriples.

/**
 * Aligns nodes in the scene graph to words in the corresponding sentence.
 * Returns a list of {@link Triple} of the IndexedWord corresponding to the
 * subject, the IndexedWord corresponding to the object or attribute and the
 * relation between them.
 *
 * @param region SceneGraphRegion with sentence and corresponding scene graph.
 * @return List of relation triples.
 */
public List<Triple<IndexedWord, IndexedWord, String>> getRelationTriples(SceneGraphImageRegion region) {
    List<Triple<IndexedWord, IndexedWord, String>> triples = Generics.newLinkedList();
    /* Perform some of the enhancements.
     * We don't resolve plurals because we wouldn't know
     * to which copy the nodes in the scene graph should be aligned
     * and we don't collapse particles and compounds because we should
     * be able to align them just using the head word. */
    SemanticGraph sg = region.getBasicSemanticGraph();
    SemanticGraphEnhancer.processQuanftificationModifiers(sg);
    SemanticGraphEnhancer.resolvePronouns(sg);
    for (SceneGraphImageAttribute attr : region.attributes) {
        boolean attrContainsNull = false;
        List<IndexedWord> subjResult = getMatch(region, attr.subjectGloss, attr.subjectLemmaGloss(), sg, false);
        attrContainsNull = attrContainsNull || containsNull(subjResult);
        List<IndexedWord> attrResult = getMatch(region, attr.attributeGloss, attr.attributeLemmaGloss(), sg, true);
        attrContainsNull = attrContainsNull || containsNull(attrResult);
        if (!attrContainsNull) {
            /* Construct attribute triple. */
            Pair<IndexedWord, IndexedWord> res = findClosestPair(subjResult, attrResult);
            res.first.set(SceneGraphCoreAnnotations.SceneGraphEntitiyAnnotation.class, attr.subjectLemmaGloss());
            res.second.set(SceneGraphCoreAnnotations.SceneGraphEntitiyAnnotation.class, attr.attributeLemmaGloss());
            triples.add(new Triple<IndexedWord, IndexedWord, String>(res.first, res.second, "is"));
        }
    }
    for (SceneGraphImageRelationship reln : region.relationships) {
        boolean relnContainsNull = false;
        List<IndexedWord> subjResult = getMatch(region, reln.subjectGloss, reln.subjectLemmaGloss(), sg, false);
        relnContainsNull = relnContainsNull || containsNull(subjResult);
        List<IndexedWord> objResult = getMatch(region, reln.objectGloss, reln.objectLemmaGloss(), sg, false);
        relnContainsNull = relnContainsNull || containsNull(objResult);
        if (!relnContainsNull) {
            /* Construct relation triple. */
            Pair<IndexedWord, IndexedWord> res = findClosestPair(subjResult, objResult);
            res.first.set(SceneGraphCoreAnnotations.SceneGraphEntitiyAnnotation.class, reln.subjectLemmaGloss());
            res.second.set(SceneGraphCoreAnnotations.SceneGraphEntitiyAnnotation.class, reln.objectLemmaGloss());
            triples.add(new Triple<IndexedWord, IndexedWord, String>(res.first, res.second, reln.predicateLemmaGloss()));
        }
    }
    return triples;
}

Example 10 with SceneGraphImageRelationship

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageRelationship in project CoreNLP by stanfordnlp.

the class KNNSceneGraphParser method main.

public static void main(String[] args) throws IOException {
    if (args.length < 3 || !args[2].equals("-train")) {
        KNNSceneGraphParser parser = new KNNSceneGraphParser(args[1]);
        Map<Integer, SceneGraphImage> trainImages = parser.loadImages(args[2]);
        BufferedReader reader = IOUtils.readerFromString(args[0]);
        PrintWriter predWriter = IOUtils.getPrintWriter(args[3]);
        PrintWriter goldWriter = IOUtils.getPrintWriter(args[4]);
        SceneGraphEvaluation evaluation = new SceneGraphEvaluation();
        double count = 0.0;
        double f1Sum = 0.0;
        for (String line = reader.readLine(); line != null; line = reader.readLine()) {
            SceneGraphImage img = SceneGraphImage.readFromJSON(line);
            for (SceneGraphImageRegion region : img.regions) {
                count += 1.0;
                SceneGraphImageRegion predicted = parser.parse(region.tokens, trainImages);
                Triple<Double, Double, Double> scores = evaluation.evaluate(predicted, region);
                evaluation.toSmatchString(predicted, region, predWriter, goldWriter);
                SceneGraphImage predictedImg = new SceneGraphImage();
                predictedImg.id = img.id;
                predictedImg.url = img.url;
                predictedImg.height = img.height;
                predictedImg.width = img.width;
                Set<Integer> objectIds = Generics.newHashSet();
                for (SceneGraphImageAttribute attr : region.attributes) {
                    objectIds.add(img.objects.indexOf(attr.subject));
                }
                for (SceneGraphImageRelationship reln : region.relationships) {
                    objectIds.add(img.objects.indexOf(reln.subject));
                    objectIds.add(img.objects.indexOf(reln.object));
                }
                predictedImg.objects = Generics.newArrayList();
                for (Integer objectId : objectIds) {
                    predictedImg.objects.add(img.objects.get(objectId));
                }
                SceneGraphImageRegion newRegion = new SceneGraphImageRegion();
                newRegion.phrase = region.phrase;
                newRegion.x = region.x;
                newRegion.y = region.y;
                newRegion.h = region.h;
                newRegion.w = region.w;
                newRegion.attributes = Generics.newHashSet();
                newRegion.relationships = Generics.newHashSet();
                predictedImg.regions = Generics.newArrayList();
                predictedImg.regions.add(newRegion);
                predictedImg.attributes = Generics.newLinkedList();
                for (SceneGraphImageAttribute attr : region.attributes) {
                    SceneGraphImageAttribute attrCopy = attr.clone();
                    attrCopy.region = newRegion;
                    attrCopy.image = predictedImg;
                    predictedImg.addAttribute(attrCopy);
                }
                predictedImg.relationships = Generics.newLinkedList();
                for (SceneGraphImageRelationship reln : region.relationships) {
                    SceneGraphImageRelationship relnCopy = reln.clone();
                    relnCopy.image = predictedImg;
                    relnCopy.region = newRegion;
                    predictedImg.addRelationship(relnCopy);
                }
                System.out.println(predictedImg.toJSON());
                System.err.printf("Prec: %f, Recall: %f, F1: %f%n", scores.first, scores.second, scores.third);
                f1Sum += scores.third;
            }
        }
        System.err.println("#########################################################");
        System.err.printf("Macro-averaged F1: %f%n", f1Sum / count);
        System.err.println("#########################################################");
    } else {
        KNNSceneGraphParser parser = new KNNSceneGraphParser(null);
        parser.train(args[0], args[1]);
    }
}