Examples with SceneGraphImage edu.stanford.nlp.scenegraph.image.SceneGraphImage used on opensource projects

Example 11 with SceneGraphImage

use of edu.stanford.nlp.scenegraph.image.SceneGraphImage in project CoreNLP by stanfordnlp.

the class RuleBasedParser method main.

public static void main(String[] args) throws IOException {
    AbstractSceneGraphParser parser = new RuleBasedParser();
    SceneGraphEvaluation eval = new SceneGraphEvaluation();
    if (args.length < 1) {
        System.err.println("Processing from stdin. Enter one sentence per line.");
        System.err.print("> ");
        Scanner scanner = new Scanner(System.in);
        String line;
        while ((line = scanner.nextLine()) != null) {
            SceneGraph scene = parser.parse(line);
            System.err.println(scene.toReadableString());
            System.err.println("------------------------");
            System.err.print("> ");
        }
        scanner.close();
    } else {
        String filename = args[0];
        BufferedReader reader = IOUtils.readerFromString(filename);
        PrintWriter predWriter = IOUtils.getPrintWriter(args[1]);
        PrintWriter goldWriter = IOUtils.getPrintWriter(args[2]);
        if (args.length > 1 && args[1].equals("n")) {
            List<SceneGraphImage> images = Generics.newLinkedList();
            for (String line = reader.readLine(); line != null; line = reader.readLine()) {
                SceneGraphImage img = SceneGraphImage.readFromJSON(line);
                if (img == null) {
                    continue;
                }
                images.add(img);
            }
            countDoubleNumMods(images);
        } else {
            double count = 0.0;
            double f1Sum = 0.0;
            for (String line = reader.readLine(); line != null; line = reader.readLine()) {
                SceneGraphImage img = SceneGraphImage.readFromJSON(line);
                if (img == null) {
                    continue;
                }
                for (SceneGraphImageRegion region : img.regions) {
                    count += 1.0;
                    SceneGraph scene = parser.parse(region.phrase);
                    System.err.println(region.phrase);
                    System.out.println(scene.toJSON(img.id, img.url, region.phrase));
                    System.err.println(scene.toReadableString());
                    System.err.println(region.toReadableString());
                    Triple<Double, Double, Double> scores = eval.evaluate(scene, region);
                    System.err.printf("Prec: %f, Recall: %f, F1: %f%n", scores.first, scores.second, scores.third);
                    eval.toSmatchString(scene, region, predWriter, goldWriter);
                    f1Sum += scores.third;
                    System.err.println("------------------------");
                }
            }
            System.err.println("#########################################################");
            System.err.printf("Macro-averaged F1: %f%n", f1Sum / count);
            System.err.println("#########################################################");
        }
    }
}

Example 12 with SceneGraphImage

use of edu.stanford.nlp.scenegraph.image.SceneGraphImage in project CoreNLP by stanfordnlp.

the class BoWSceneGraphParser method main.

public static void main(String[] args) throws IOException {
    Properties props = StringUtils.argsToProperties(args, numArgs);
    boolean train = PropertiesUtils.getBool(props, "train", false);
    boolean enforceSubtree = PropertiesUtils.getBool(props, "enforceSubtree", true);
    boolean includeAllObjects = PropertiesUtils.getBool(props, "includeAllObjects", false);
    boolean verbose = PropertiesUtils.getBool(props, "verbose", false);
    String modelPath = PropertiesUtils.getString(props, "model", DEFAULT_MODEL_PATH);
    String entityModelPath = PropertiesUtils.getString(props, "entityModel", DEFAULT_ENTITY_MODEL_PATH);
    String inputPath = PropertiesUtils.getString(props, "input", null);
    String embeddingsPath = PropertiesUtils.getString(props, "embeddings", null);
    if (modelPath == null || entityModelPath == null || embeddingsPath == null) {
        System.err.printf("Usage java %s -model <model.gz>" + " -entityModel <entityModel.gz> -embeddings <wordVectors.gz> [-input <input.json> -train -verbose -enforceSubtree -evalFilePrefix <run0>]%n", BoWSceneGraphParser.class.getCanonicalName());
        return;
    }
    boolean interactive = (inputPath == null);
    Embedding embeddings = new Embedding(embeddingsPath);
    if (!train) {
        BoWSceneGraphParser parser = new BoWSceneGraphParser(modelPath, entityModelPath, embeddings);
        parser.enforceSubtree = enforceSubtree;
        parser.includeAllObjects = includeAllObjects;
        if (!interactive) {
            BufferedReader reader = IOUtils.readerFromString(inputPath);
            SceneGraphEvaluation eval = new SceneGraphEvaluation();
            String evalFilePrefix = PropertiesUtils.getString(props, "evalFilePrefix", null);
            PrintWriter predWriter = null;
            PrintWriter goldWriter = null;
            if (evalFilePrefix != null) {
                String predEvalFilePath = evalFilePrefix + ".smatch";
                String goldEvalFilePath = evalFilePrefix + "_gold.smatch";
                predWriter = IOUtils.getPrintWriter(predEvalFilePath);
                goldWriter = IOUtils.getPrintWriter(goldEvalFilePath);
            }
            double count = 0.0;
            double f1Sum = 0.0;
            for (String line = reader.readLine(); line != null; line = reader.readLine()) {
                SceneGraphImage img = SceneGraphImage.readFromJSON(line);
                if (img == null) {
                    continue;
                }
                for (SceneGraphImageRegion region : img.regions) {
                    count += 1.0;
                    SemanticGraph sg = region.getEnhancedSemanticGraph();
                    SceneGraph scene = parser.parse(sg);
                    System.out.println(scene.toJSON(img.id, img.url, region.phrase));
                    Triple<Double, Double, Double> scores = eval.evaluate(scene, region);
                    if (evalFilePrefix != null) {
                        eval.toSmatchString(scene, region, predWriter, goldWriter);
                    }
                    if (verbose) {
                        System.err.println(region.phrase);
                        System.err.println(scene.toReadableString());
                        System.err.println(region.toReadableString());
                        System.err.printf("Prec: %f, Recall: %f, F1: %f%n", scores.first, scores.second, scores.third);
                        System.err.println("------------------------");
                    }
                    f1Sum += scores.third;
                }
            }
            System.err.println("#########################################################");
            System.err.printf("Macro-averaged F1: %f%n", f1Sum / count);
            System.err.println("#########################################################");
        } else {
            System.err.println("Processing from stdin. Enter one sentence per line.");
            System.err.print("> ");
            Scanner scanner = new Scanner(System.in);
            String line;
            while ((line = scanner.nextLine()) != null) {
                SceneGraph scene = parser.parse(line);
                System.err.println(scene.toReadableString());
                System.err.println("------------------------");
                System.err.print("> ");
            }
            scanner.close();
        }
    } else {
        BoWSceneGraphParser parser = new BoWSceneGraphParser(null, entityModelPath, embeddings);
        parser.enforceSubtree = enforceSubtree;
        parser.train(inputPath, modelPath);
    }
}

Example 13 with SceneGraphImage

use of edu.stanford.nlp.scenegraph.image.SceneGraphImage in project CoreNLP by stanfordnlp.

the class BoWSceneGraphParser method getTrainingExamples.

/**
 * Generate training examples.
 *
 * @param trainingFile Path to JSON file with training images and scene graphs.
 * @param sampleNeg Whether to sample the same number of negative examples as positive examples.
 * @return Dataset to train a classifier.
 * @throws IOException
 */
public Dataset<String, String> getTrainingExamples(String trainingFile, boolean sampleNeg) throws IOException {
    Dataset<String, String> dataset = new Dataset<String, String>();
    Dataset<String, String> negDataset = new Dataset<String, String>();
    /* Load images. */
    List<SceneGraphImage> images = loadImages(trainingFile);
    for (SceneGraphImage image : images) {
        for (SceneGraphImageRegion region : image.regions) {
            SemanticGraph sg = region.getEnhancedSemanticGraph();
            SemanticGraphEnhancer.processQuanftificationModifiers(sg);
            SemanticGraphEnhancer.collapseCompounds(sg);
            SemanticGraphEnhancer.collapseParticles(sg);
            SemanticGraphEnhancer.resolvePronouns(sg);
            Set<Integer> entityPairs = Generics.newHashSet();
            List<Triple<IndexedWord, IndexedWord, String>> relationTriples = this.sentenceMatcher.getRelationTriples(region);
            for (Triple<IndexedWord, IndexedWord, String> triple : relationTriples) {
                IndexedWord iw1 = sg.getNodeByIndexSafe(triple.first.index());
                IndexedWord iw2 = sg.getNodeByIndexSafe(triple.second.index());
                if (iw1 != null && iw2 != null && (!enforceSubtree || SceneGraphUtils.inSameSubTree(sg, iw1, iw2))) {
                    entityClassifer.predictEntity(iw1, this.embeddings);
                    entityClassifer.predictEntity(iw2, this.embeddings);
                    BoWExample example = new BoWExample(iw1, iw2, sg);
                    dataset.add(example.extractFeatures(featureSets), triple.third);
                }
                entityPairs.add((triple.first.index() << 4) + triple.second.index());
            }
            /* Add negative examples. */
            List<IndexedWord> entities = EntityExtractor.extractEntities(sg);
            List<IndexedWord> attributes = EntityExtractor.extractAttributes(sg);
            for (IndexedWord e : entities) {
                entityClassifer.predictEntity(e, this.embeddings);
            }
            for (IndexedWord a : attributes) {
                entityClassifer.predictEntity(a, this.embeddings);
            }
            for (IndexedWord e1 : entities) {
                for (IndexedWord e2 : entities) {
                    if (e1.index() == e2.index()) {
                        continue;
                    }
                    int entityPair = (e1.index() << 4) + e2.index();
                    if (!entityPairs.contains(entityPair) && (!enforceSubtree || SceneGraphUtils.inSameSubTree(sg, e1, e2))) {
                        BoWExample example = new BoWExample(e1, e2, sg);
                        negDataset.add(example.extractFeatures(featureSets), NONE_RELATION);
                    }
                }
            }
            for (IndexedWord e : entities) {
                for (IndexedWord a : attributes) {
                    int entityPair = (e.index() << 4) + a.index();
                    if (!entityPairs.contains(entityPair) && (!enforceSubtree || SceneGraphUtils.inSameSubTree(sg, e, a))) {
                        BoWExample example = new BoWExample(e, a, sg);
                        negDataset.add(example.extractFeatures(featureSets), NONE_RELATION);
                    }
                }
            }
        }
    }
    /* Sample from negative examples to make the training set
     * more balanced. */
    if (sampleNeg && dataset.size() < negDataset.size()) {
        negDataset = negDataset.getRandomSubDataset(dataset.size() * 1.0 / negDataset.size(), 42);
    }
    dataset.addAll(negDataset);
    return dataset;
}

Example 14 with SceneGraphImage

use of edu.stanford.nlp.scenegraph.image.SceneGraphImage in project CoreNLP by stanfordnlp.

the class GroundTruthConverter method main.

public static void main(String[] args) throws IOException {
    BufferedReader reader = IOUtils.readerFromString(args[0]);
    for (String line = reader.readLine(); line != null; line = reader.readLine()) {
        SceneGraphImage img = SceneGraphImage.readFromJSON(line);
        if (img == null) {
            continue;
        }
        for (SceneGraphImageRegion region : img.regions) {
            SceneGraphImage predictedImg = new SceneGraphImage();
            predictedImg.id = img.id;
            predictedImg.url = img.url;
            predictedImg.height = img.height;
            predictedImg.width = img.width;
            Set<Integer> objectIds = Generics.newHashSet();
            for (SceneGraphImageAttribute attr : region.attributes) {
                objectIds.add(img.objects.indexOf(attr.subject));
            }
            for (SceneGraphImageRelationship reln : region.relationships) {
                objectIds.add(img.objects.indexOf(reln.subject));
                objectIds.add(img.objects.indexOf(reln.object));
            }
            predictedImg.objects = Generics.newArrayList();
            for (Integer objectId : objectIds) {
                predictedImg.objects.add(img.objects.get(objectId));
            }
            SceneGraphImageRegion newRegion = new SceneGraphImageRegion();
            newRegion.phrase = region.phrase;
            newRegion.x = region.x;
            newRegion.y = region.y;
            newRegion.h = region.h;
            newRegion.w = region.w;
            newRegion.attributes = Generics.newHashSet();
            newRegion.relationships = Generics.newHashSet();
            predictedImg.regions = Generics.newArrayList();
            predictedImg.regions.add(newRegion);
            predictedImg.attributes = Generics.newLinkedList();
            for (SceneGraphImageAttribute attr : region.attributes) {
                SceneGraphImageAttribute attrCopy = attr.clone();
                attrCopy.region = newRegion;
                attrCopy.image = predictedImg;
                predictedImg.addAttribute(attrCopy);
            }
            predictedImg.relationships = Generics.newLinkedList();
            for (SceneGraphImageRelationship reln : region.relationships) {
                SceneGraphImageRelationship relnCopy = reln.clone();
                relnCopy.image = predictedImg;
                relnCopy.region = newRegion;
                predictedImg.addRelationship(relnCopy);
            }
            System.out.println(predictedImg.toJSON());
        }
    }
}

Example 15 with SceneGraphImage

use of edu.stanford.nlp.scenegraph.image.SceneGraphImage in project CoreNLP by stanfordnlp.

the class RuleBasedParser method countDoubleNumMods.

public static void countDoubleNumMods(List<SceneGraphImage> images) {
    RuleBasedParser parser = new RuleBasedParser();
    int doubleMatches = 0;
    for (SceneGraphImage img : images) {
        for (SceneGraphImageRegion region : img.regions) {
            SceneGraph scene = parser.parse(region.phrase);
            SemgrexMatcher matcher = NUMMOD_PATTERN.matcher(scene.sg);
            int matches = 0;
            while (matcher.findNextMatchingNode()) {
                matches++;
            }
            if (matches > 1) {
                System.err.println(region.phrase);
                doubleMatches++;
            }
        }
    }
    System.err.println(doubleMatches);
}