Examples with SceneGraphImageRelationship edu.stanford.nlp.scenegraph.image.SceneGraphImageRelationship used on opensource projects

Example 1 with SceneGraphImageRelationship

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageRelationship in project CoreNLP by stanfordnlp.

the class GenerateAlignmentData method main.

public static void main(String[] args) throws IOException {
    Properties props = new Properties();
    props.put("annotators", "tokenize,ssplit");
    props.put("ssplit.eolonly", "true");
    StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
    String filename = args[0];
    String sentences = args[1];
    String graphs = args[2];
    BufferedReader reader = IOUtils.readerFromString(filename);
    PrintWriter sentencesFile = IOUtils.getPrintWriter(sentences);
    PrintWriter graphsFile = IOUtils.getPrintWriter(graphs);
    for (String line = reader.readLine(); line != null; line = reader.readLine()) {
        SceneGraphImage img = SceneGraphImage.readFromJSON(line);
        if (img == null) {
            continue;
        }
        for (SceneGraphImageRegion region : img.regions) {
            Annotation doc = new Annotation(region.phrase);
            pipeline.annotate(doc);
            CoreMap sentence = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
            List<CoreLabel> tokens = sentence.get(CoreAnnotations.TokensAnnotation.class);
            String tokenizedSentence = StringUtils.join(tokens.stream().map(CoreLabel::word), " ");
            for (SceneGraphImageAttribute attr : region.attributes) {
                sentencesFile.printf("%s%n", tokenizedSentence);
                graphsFile.printf("%s%n", StringUtils.join(attr.text));
            }
            for (SceneGraphImageRelationship reln : region.relationships) {
                sentencesFile.printf("%s%n", tokenizedSentence);
                graphsFile.printf("%s%n", StringUtils.join(reln.text));
            }
        }
    }
}

Example 2 with SceneGraphImageRelationship

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageRelationship in project CoreNLP by stanfordnlp.

the class SceneGraphEvaluation method evaluate.

public Triple<Double, Double, Double> evaluate(SceneGraph scene, SceneGraphImageRegion region) {
    Counter<SceneGraphRelationTriplet> goldTriplets = new ClassicCounter<SceneGraphRelationTriplet>();
    Counter<SceneGraphRelationTriplet> predictedTriplets = new ClassicCounter<SceneGraphRelationTriplet>();
    for (SceneGraphNode node : scene.nodeListSorted()) {
        for (SceneGraphAttribute attr : node.getAttributes()) {
            SceneGraphRelationTriplet t = new SceneGraphRelationTriplet(node.value().backingLabel(), attr.value().backingLabel(), "is");
            predictedTriplets.incrementCount(t);
        }
    }
    for (SceneGraphRelation reln : scene.relationListSorted()) {
        SceneGraphRelationTriplet t = new SceneGraphRelationTriplet(reln.getSource().value().backingLabel(), reln.getTarget().value().backingLabel(), reln.getRelation());
        predictedTriplets.incrementCount(t);
    }
    for (SceneGraphImageAttribute attr : region.attributes) {
        SceneGraphRelationTriplet t = new SceneGraphRelationTriplet(attr);
        goldTriplets.incrementCount(t);
    }
    for (SceneGraphImageRelationship reln : region.relationships) {
        SceneGraphRelationTriplet t = new SceneGraphRelationTriplet(reln);
        goldTriplets.incrementCount(t);
    }
    double predictedCount = predictedTriplets.totalCount();
    double goldCount = goldTriplets.totalCount();
    double numerator = 0.0;
    for (SceneGraphRelationTriplet t : goldTriplets.keySet()) {
        double gold = goldTriplets.getCount(t);
        double pred = predictedTriplets.getCount(t);
        numerator += Math.min(gold, pred);
    }
    double precision = predictedCount > 0 ? numerator / predictedCount : 1.0;
    double recall = goldCount > 0 ? numerator / goldCount : 1.0;
    double f1 = (precision + recall > 0) ? 2 * precision * recall / (precision + recall) : 0.0;
    Triple<Double, Double, Double> scores = new Triple<Double, Double, Double>(precision, recall, f1);
    return scores;
}

Example 3 with SceneGraphImageRelationship

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageRelationship in project CoreNLP by stanfordnlp.

the class SceneGraphImageCleaner method lemmatize.

public void lemmatize(SceneGraphImage img) {
    StanfordCoreNLP pipeline = getPipeline();
    /* attributes */
    for (SceneGraphImageAttribute attr : img.attributes) {
        String attribute = removeDeterminersAndNumbers(removeFinalPunctuation(attr.attribute));
        String sentence = String.format("She is %s .\n", attribute);
        Annotation doc = new Annotation(sentence);
        pipeline.annotate(doc);
        CoreMap sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
        List<CoreLabel> tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
        attr.attributeGloss = tokens.subList(2, tokens.size() - 1);
        String subject = removeDeterminersAndNumbers(removeFinalPunctuation(attr.text[0]));
        sentence = String.format("The %s is tall .", subject);
        doc = new Annotation(sentence);
        pipeline.annotate(doc);
        sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
        tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
        attr.subjectGloss = tokens.subList(1, tokens.size() - 3);
        attr.subject.labels.add(attr.subjectGloss);
    }
    /* relations */
    for (SceneGraphImageRelationship reln : img.relationships) {
        String object = removeDeterminersAndNumbers(removeFinalPunctuation(reln.text[2]));
        String sentence = String.format("She is the %s .\n", object);
        Annotation doc = new Annotation(sentence);
        pipeline.annotate(doc);
        CoreMap sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
        List<CoreLabel> tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
        reln.objectGloss = tokens.subList(3, tokens.size() - 1);
        reln.object.labels.add(reln.objectGloss);
        String subject = removeDeterminersAndNumbers(removeFinalPunctuation(reln.text[0]));
        sentence = String.format("The %s is tall .", subject);
        doc = new Annotation(sentence);
        pipeline.annotate(doc);
        sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
        tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
        reln.subjectGloss = tokens.subList(1, tokens.size() - 3);
        reln.subject.labels.add(reln.subjectGloss);
        String predicate = removeDeterminersAndNumbers(removeFinalPunctuation(reln.predicate));
        sentence = String.format("A horse %s an apple .", predicate);
        doc = new Annotation(sentence);
        pipeline.annotate(doc);
        sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
        tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
        reln.predicateGloss = tokens.subList(2, tokens.size() - 3);
    }
    for (SceneGraphImageObject object : img.objects) {
        if (object.names.size() > object.labels.size()) {
            for (String name : object.names) {
                String x = removeDeterminersAndNumbers(removeFinalPunctuation(name));
                String sentence = String.format("The %s is tall .", x);
                Annotation doc = new Annotation(sentence);
                pipeline.annotate(doc);
                CoreMap sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
                List<CoreLabel> tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
                object.labels.add(tokens.subList(1, tokens.size() - 3));
            }
        }
    }
    StanfordCoreNLP tokenizerPipeline = getTokenizerPipeline();
    for (SceneGraphImageRegion region : img.regions) {
        Annotation doc = new Annotation(region.phrase.toLowerCase());
        tokenizerPipeline.annotate(doc);
        CoreMap sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
        region.tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
    }
}

Example 4 with SceneGraphImageRelationship

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageRelationship in project CoreNLP by stanfordnlp.

the class SceneGraphImageFilter method filterRegions.

private static void filterRegions(List<SceneGraphImage> images, int threshold) {
    int regionCount = 0;
    int filterCount = 0;
    int imgCount = 0;
    int removedEntireImgCount = 0;
    int removedPartialImgCount = 0;
    for (SceneGraphImage img : images) {
        imgCount++;
        List<SceneGraphImageRegion> toDelete = Generics.newLinkedList();
        for (SceneGraphImageRegion region : img.regions) {
            regionCount++;
            boolean delete = false;
            for (SceneGraphImageAttribute attr : region.attributes) {
                if (attributeCounter.getCount(attr.attributeLemmaGloss()) < threshold || entityCounter.getCount(attr.subjectLemmaGloss()) < threshold) {
                    delete = true;
                    break;
                }
            }
            if (delete) {
                toDelete.add(region);
                continue;
            }
            for (SceneGraphImageRelationship reln : region.relationships) {
                if (entityCounter.getCount(reln.objectLemmaGloss()) < threshold || entityCounter.getCount(reln.subjectLemmaGloss()) < threshold || relationCounter.getCount(reln.predicateLemmaGloss()) < threshold) {
                    delete = true;
                    break;
                }
            }
            if (delete) {
                toDelete.add(region);
                continue;
            }
        }
        for (SceneGraphImageRegion region : toDelete) {
            img.removeRegion(region);
            filterCount++;
        }
        if (!toDelete.isEmpty()) {
            removedPartialImgCount++;
        }
        if (img.regions.isEmpty()) {
            removedEntireImgCount++;
        }
    }
    System.err.printf("%d\t%f\t%f\t%f %n", filterCount, filterCount * 100.0 / regionCount, removedPartialImgCount * 100.0 / imgCount, removedEntireImgCount * 100.0 / imgCount);
}

Example 5 with SceneGraphImageRelationship

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageRelationship in project CoreNLP by stanfordnlp.

the class GroundTruthConverter method main.

public static void main(String[] args) throws IOException {
    BufferedReader reader = IOUtils.readerFromString(args[0]);
    for (String line = reader.readLine(); line != null; line = reader.readLine()) {
        SceneGraphImage img = SceneGraphImage.readFromJSON(line);
        if (img == null) {
            continue;
        }
        for (SceneGraphImageRegion region : img.regions) {
            SceneGraphImage predictedImg = new SceneGraphImage();
            predictedImg.id = img.id;
            predictedImg.url = img.url;
            predictedImg.height = img.height;
            predictedImg.width = img.width;
            Set<Integer> objectIds = Generics.newHashSet();
            for (SceneGraphImageAttribute attr : region.attributes) {
                objectIds.add(img.objects.indexOf(attr.subject));
            }
            for (SceneGraphImageRelationship reln : region.relationships) {
                objectIds.add(img.objects.indexOf(reln.subject));
                objectIds.add(img.objects.indexOf(reln.object));
            }
            predictedImg.objects = Generics.newArrayList();
            for (Integer objectId : objectIds) {
                predictedImg.objects.add(img.objects.get(objectId));
            }
            SceneGraphImageRegion newRegion = new SceneGraphImageRegion();
            newRegion.phrase = region.phrase;
            newRegion.x = region.x;
            newRegion.y = region.y;
            newRegion.h = region.h;
            newRegion.w = region.w;
            newRegion.attributes = Generics.newHashSet();
            newRegion.relationships = Generics.newHashSet();
            predictedImg.regions = Generics.newArrayList();
            predictedImg.regions.add(newRegion);
            predictedImg.attributes = Generics.newLinkedList();
            for (SceneGraphImageAttribute attr : region.attributes) {
                SceneGraphImageAttribute attrCopy = attr.clone();
                attrCopy.region = newRegion;
                attrCopy.image = predictedImg;
                predictedImg.addAttribute(attrCopy);
            }
            predictedImg.relationships = Generics.newLinkedList();
            for (SceneGraphImageRelationship reln : region.relationships) {
                SceneGraphImageRelationship relnCopy = reln.clone();
                relnCopy.image = predictedImg;
                relnCopy.region = newRegion;
                predictedImg.addRelationship(relnCopy);
            }
            System.out.println(predictedImg.toJSON());
        }
    }
}