Examples with SceneGraphImageAttribute edu.stanford.nlp.scenegraph.image.SceneGraphImageAttribute used on opensource projects

Example 1 with SceneGraphImageAttribute

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageAttribute in project CoreNLP by stanfordnlp.

the class GenerateAlignmentData method main.

public static void main(String[] args) throws IOException {
    Properties props = new Properties();
    props.put("annotators", "tokenize,ssplit");
    props.put("ssplit.eolonly", "true");
    StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
    String filename = args[0];
    String sentences = args[1];
    String graphs = args[2];
    BufferedReader reader = IOUtils.readerFromString(filename);
    PrintWriter sentencesFile = IOUtils.getPrintWriter(sentences);
    PrintWriter graphsFile = IOUtils.getPrintWriter(graphs);
    for (String line = reader.readLine(); line != null; line = reader.readLine()) {
        SceneGraphImage img = SceneGraphImage.readFromJSON(line);
        if (img == null) {
            continue;
        }
        for (SceneGraphImageRegion region : img.regions) {
            Annotation doc = new Annotation(region.phrase);
            pipeline.annotate(doc);
            CoreMap sentence = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
            List<CoreLabel> tokens = sentence.get(CoreAnnotations.TokensAnnotation.class);
            String tokenizedSentence = StringUtils.join(tokens.stream().map(CoreLabel::word), " ");
            for (SceneGraphImageAttribute attr : region.attributes) {
                sentencesFile.printf("%s%n", tokenizedSentence);
                graphsFile.printf("%s%n", StringUtils.join(attr.text));
            }
            for (SceneGraphImageRelationship reln : region.relationships) {
                sentencesFile.printf("%s%n", tokenizedSentence);
                graphsFile.printf("%s%n", StringUtils.join(reln.text));
            }
        }
    }
}

Example 2 with SceneGraphImageAttribute

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageAttribute in project CoreNLP by stanfordnlp.

the class SceneGraphEvaluation method evaluate.

public Triple<Double, Double, Double> evaluate(SceneGraph scene, SceneGraphImageRegion region) {
    Counter<SceneGraphRelationTriplet> goldTriplets = new ClassicCounter<SceneGraphRelationTriplet>();
    Counter<SceneGraphRelationTriplet> predictedTriplets = new ClassicCounter<SceneGraphRelationTriplet>();
    for (SceneGraphNode node : scene.nodeListSorted()) {
        for (SceneGraphAttribute attr : node.getAttributes()) {
            SceneGraphRelationTriplet t = new SceneGraphRelationTriplet(node.value().backingLabel(), attr.value().backingLabel(), "is");
            predictedTriplets.incrementCount(t);
        }
    }
    for (SceneGraphRelation reln : scene.relationListSorted()) {
        SceneGraphRelationTriplet t = new SceneGraphRelationTriplet(reln.getSource().value().backingLabel(), reln.getTarget().value().backingLabel(), reln.getRelation());
        predictedTriplets.incrementCount(t);
    }
    for (SceneGraphImageAttribute attr : region.attributes) {
        SceneGraphRelationTriplet t = new SceneGraphRelationTriplet(attr);
        goldTriplets.incrementCount(t);
    }
    for (SceneGraphImageRelationship reln : region.relationships) {
        SceneGraphRelationTriplet t = new SceneGraphRelationTriplet(reln);
        goldTriplets.incrementCount(t);
    }
    double predictedCount = predictedTriplets.totalCount();
    double goldCount = goldTriplets.totalCount();
    double numerator = 0.0;
    for (SceneGraphRelationTriplet t : goldTriplets.keySet()) {
        double gold = goldTriplets.getCount(t);
        double pred = predictedTriplets.getCount(t);
        numerator += Math.min(gold, pred);
    }
    double precision = predictedCount > 0 ? numerator / predictedCount : 1.0;
    double recall = goldCount > 0 ? numerator / goldCount : 1.0;
    double f1 = (precision + recall > 0) ? 2 * precision * recall / (precision + recall) : 0.0;
    Triple<Double, Double, Double> scores = new Triple<Double, Double, Double>(precision, recall, f1);
    return scores;
}

Example 3 with SceneGraphImageAttribute

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageAttribute in project CoreNLP by stanfordnlp.

the class SceneGraphImageCleaner method lemmatize.

public void lemmatize(SceneGraphImage img) {
    StanfordCoreNLP pipeline = getPipeline();
    /* attributes */
    for (SceneGraphImageAttribute attr : img.attributes) {
        String attribute = removeDeterminersAndNumbers(removeFinalPunctuation(attr.attribute));
        String sentence = String.format("She is %s .\n", attribute);
        Annotation doc = new Annotation(sentence);
        pipeline.annotate(doc);
        CoreMap sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
        List<CoreLabel> tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
        attr.attributeGloss = tokens.subList(2, tokens.size() - 1);
        String subject = removeDeterminersAndNumbers(removeFinalPunctuation(attr.text[0]));
        sentence = String.format("The %s is tall .", subject);
        doc = new Annotation(sentence);
        pipeline.annotate(doc);
        sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
        tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
        attr.subjectGloss = tokens.subList(1, tokens.size() - 3);
        attr.subject.labels.add(attr.subjectGloss);
    }
    /* relations */
    for (SceneGraphImageRelationship reln : img.relationships) {
        String object = removeDeterminersAndNumbers(removeFinalPunctuation(reln.text[2]));
        String sentence = String.format("She is the %s .\n", object);
        Annotation doc = new Annotation(sentence);
        pipeline.annotate(doc);
        CoreMap sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
        List<CoreLabel> tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
        reln.objectGloss = tokens.subList(3, tokens.size() - 1);
        reln.object.labels.add(reln.objectGloss);
        String subject = removeDeterminersAndNumbers(removeFinalPunctuation(reln.text[0]));
        sentence = String.format("The %s is tall .", subject);
        doc = new Annotation(sentence);
        pipeline.annotate(doc);
        sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
        tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
        reln.subjectGloss = tokens.subList(1, tokens.size() - 3);
        reln.subject.labels.add(reln.subjectGloss);
        String predicate = removeDeterminersAndNumbers(removeFinalPunctuation(reln.predicate));
        sentence = String.format("A horse %s an apple .", predicate);
        doc = new Annotation(sentence);
        pipeline.annotate(doc);
        sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
        tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
        reln.predicateGloss = tokens.subList(2, tokens.size() - 3);
    }
    for (SceneGraphImageObject object : img.objects) {
        if (object.names.size() > object.labels.size()) {
            for (String name : object.names) {
                String x = removeDeterminersAndNumbers(removeFinalPunctuation(name));
                String sentence = String.format("The %s is tall .", x);
                Annotation doc = new Annotation(sentence);
                pipeline.annotate(doc);
                CoreMap sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
                List<CoreLabel> tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
                object.labels.add(tokens.subList(1, tokens.size() - 3));
            }
        }
    }
    StanfordCoreNLP tokenizerPipeline = getTokenizerPipeline();
    for (SceneGraphImageRegion region : img.regions) {
        Annotation doc = new Annotation(region.phrase.toLowerCase());
        tokenizerPipeline.annotate(doc);
        CoreMap sentenceAnn = doc.get(CoreAnnotations.SentencesAnnotation.class).get(0);
        region.tokens = sentenceAnn.get(CoreAnnotations.TokensAnnotation.class);
    }
}

Example 4 with SceneGraphImageAttribute

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageAttribute in project CoreNLP by stanfordnlp.

the class SceneGraphImageCleaner method splitAttributeConjunctions.

/**
 * Splits attributes of the form X and Y or X, Y and Z if
 * all elements were observed somewhere in some other image.
 */
public void splitAttributeConjunctions(SceneGraphImage img) {
    if (ALL_ATTRIBUTES.isEmpty()) {
        System.err.println("WARNING: List of attributes is empty! Won't split any conjunctions.");
        return;
    }
    List<SceneGraphImageAttribute> newAttrs = Generics.newLinkedList();
    for (SceneGraphImageAttribute attr : img.attributes) {
        if (SceneGraphImageUtils.containsLemma(attr.attributeGloss, "and") || SceneGraphImageUtils.containsLemma(attr.attributeGloss, "&")) {
            List<List<CoreLabel>> parts = Generics.newLinkedList();
            boolean shouldSplit = true;
            List<CoreLabel> current = Generics.newLinkedList();
            for (int i = 0, sz = attr.attributeGloss.size(); i <= sz; i++) {
                CoreLabel word = i < sz ? attr.attributeGloss.get(i) : null;
                if (word == null || word.lemma().equals("and") || word.lemma().equals(",") || word.lemma().equals("&")) {
                    if (current.isEmpty()) {
                        continue;
                    }
                    if (!ALL_ATTRIBUTES.contains(lemmaGloss(current))) {
                        shouldSplit = false;
                        break;
                    }
                    parts.add(current);
                    current = Generics.newLinkedList();
                } else {
                    current.add(word);
                }
            }
            if (shouldSplit && parts.size() > 0) {
                attr.attributeGloss = parts.get(0);
                attr.attribute = attr.attributeGloss();
                attr.object = attr.attributeGloss();
                attr.text[2] = attr.attributeGloss();
                for (int i = 1, sz = parts.size(); i < sz; i++) {
                    SceneGraphImageAttribute attr2 = attr.clone();
                    attr2.attributeGloss = parts.get(i);
                    attr2.attribute = attr2.attributeGloss();
                    attr2.object = attr2.attributeGloss();
                    attr2.text[2] = attr2.attributeGloss();
                    newAttrs.add(attr2);
                }
            }
        }
    }
    for (SceneGraphImageAttribute attr : newAttrs) {
        img.addAttribute(attr);
    }
}

Example 5 with SceneGraphImageAttribute

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageAttribute in project CoreNLP by stanfordnlp.

the class SceneGraphImageFilter method filterRegions.

private static void filterRegions(List<SceneGraphImage> images, int threshold) {
    int regionCount = 0;
    int filterCount = 0;
    int imgCount = 0;
    int removedEntireImgCount = 0;
    int removedPartialImgCount = 0;
    for (SceneGraphImage img : images) {
        imgCount++;
        List<SceneGraphImageRegion> toDelete = Generics.newLinkedList();
        for (SceneGraphImageRegion region : img.regions) {
            regionCount++;
            boolean delete = false;
            for (SceneGraphImageAttribute attr : region.attributes) {
                if (attributeCounter.getCount(attr.attributeLemmaGloss()) < threshold || entityCounter.getCount(attr.subjectLemmaGloss()) < threshold) {
                    delete = true;
                    break;
                }
            }
            if (delete) {
                toDelete.add(region);
                continue;
            }
            for (SceneGraphImageRelationship reln : region.relationships) {
                if (entityCounter.getCount(reln.objectLemmaGloss()) < threshold || entityCounter.getCount(reln.subjectLemmaGloss()) < threshold || relationCounter.getCount(reln.predicateLemmaGloss()) < threshold) {
                    delete = true;
                    break;
                }
            }
            if (delete) {
                toDelete.add(region);
                continue;
            }
        }
        for (SceneGraphImageRegion region : toDelete) {
            img.removeRegion(region);
            filterCount++;
        }
        if (!toDelete.isEmpty()) {
            removedPartialImgCount++;
        }
        if (img.regions.isEmpty()) {
            removedEntireImgCount++;
        }
    }
    System.err.printf("%d\t%f\t%f\t%f %n", filterCount, filterCount * 100.0 / regionCount, removedPartialImgCount * 100.0 / imgCount, removedEntireImgCount * 100.0 / imgCount);
}