Examples with SceneGraphImageRegion edu.stanford.nlp.scenegraph.image.SceneGraphImageRegion used on opensource projects

Example 6 with SceneGraphImageRegion

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageRegion in project CoreNLP by stanfordnlp.

the class SceneGraphImageFilter method filterRegions.

private static void filterRegions(List<SceneGraphImage> images, int threshold) {
    int regionCount = 0;
    int filterCount = 0;
    int imgCount = 0;
    int removedEntireImgCount = 0;
    int removedPartialImgCount = 0;
    for (SceneGraphImage img : images) {
        imgCount++;
        List<SceneGraphImageRegion> toDelete = Generics.newLinkedList();
        for (SceneGraphImageRegion region : img.regions) {
            regionCount++;
            boolean delete = false;
            for (SceneGraphImageAttribute attr : region.attributes) {
                if (attributeCounter.getCount(attr.attributeLemmaGloss()) < threshold || entityCounter.getCount(attr.subjectLemmaGloss()) < threshold) {
                    delete = true;
                    break;
                }
            }
            if (delete) {
                toDelete.add(region);
                continue;
            }
            for (SceneGraphImageRelationship reln : region.relationships) {
                if (entityCounter.getCount(reln.objectLemmaGloss()) < threshold || entityCounter.getCount(reln.subjectLemmaGloss()) < threshold || relationCounter.getCount(reln.predicateLemmaGloss()) < threshold) {
                    delete = true;
                    break;
                }
            }
            if (delete) {
                toDelete.add(region);
                continue;
            }
        }
        for (SceneGraphImageRegion region : toDelete) {
            img.removeRegion(region);
            filterCount++;
        }
        if (!toDelete.isEmpty()) {
            removedPartialImgCount++;
        }
        if (img.regions.isEmpty()) {
            removedEntireImgCount++;
        }
    }
    System.err.printf("%d\t%f\t%f\t%f %n", filterCount, filterCount * 100.0 / regionCount, removedPartialImgCount * 100.0 / imgCount, removedEntireImgCount * 100.0 / imgCount);
}

Example 7 with SceneGraphImageRegion

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageRegion in project CoreNLP by stanfordnlp.

the class ObjectSceneGraphParser method main.

public static void main(String[] args) throws IOException {
    ObjectSceneGraphParser parser = new ObjectSceneGraphParser(args[1]);
    BufferedReader reader = IOUtils.readerFromString(args[0]);
    SceneGraphEvaluation eval = new SceneGraphEvaluation();
    String evalFilePrefix = args[2];
    String embeddingsPath = args[3];
    parser.embeddings = new Embedding(embeddingsPath);
    PrintWriter predWriter = null;
    PrintWriter goldWriter = null;
    if (evalFilePrefix != null) {
        String predEvalFilePath = evalFilePrefix + ".smatch";
        String goldEvalFilePath = evalFilePrefix + "_gold.smatch";
        predWriter = IOUtils.getPrintWriter(predEvalFilePath);
        goldWriter = IOUtils.getPrintWriter(goldEvalFilePath);
    }
    for (String line = reader.readLine(); line != null; line = reader.readLine()) {
        SceneGraphImage img = SceneGraphImage.readFromJSON(line);
        if (img == null) {
            continue;
        }
        for (SceneGraphImageRegion region : img.regions) {
            SemanticGraph sg = region.getEnhancedSemanticGraph();
            SceneGraph scene = parser.parse(sg);
            System.out.println(scene.toJSON(img.id, img.url, region.phrase));
            if (evalFilePrefix != null) {
                eval.toSmatchString(scene, region, predWriter, goldWriter);
            }
        }
    }
}

Example 8 with SceneGraphImageRegion

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageRegion in project CoreNLP by stanfordnlp.

the class RuleBasedParser method main.

public static void main(String[] args) throws IOException {
    AbstractSceneGraphParser parser = new RuleBasedParser();
    SceneGraphEvaluation eval = new SceneGraphEvaluation();
    if (args.length < 1) {
        System.err.println("Processing from stdin. Enter one sentence per line.");
        System.err.print("> ");
        Scanner scanner = new Scanner(System.in);
        String line;
        while ((line = scanner.nextLine()) != null) {
            SceneGraph scene = parser.parse(line);
            System.err.println(scene.toReadableString());
            System.err.println("------------------------");
            System.err.print("> ");
        }
        scanner.close();
    } else {
        String filename = args[0];
        BufferedReader reader = IOUtils.readerFromString(filename);
        PrintWriter predWriter = IOUtils.getPrintWriter(args[1]);
        PrintWriter goldWriter = IOUtils.getPrintWriter(args[2]);
        if (args.length > 1 && args[1].equals("n")) {
            List<SceneGraphImage> images = Generics.newLinkedList();
            for (String line = reader.readLine(); line != null; line = reader.readLine()) {
                SceneGraphImage img = SceneGraphImage.readFromJSON(line);
                if (img == null) {
                    continue;
                }
                images.add(img);
            }
            countDoubleNumMods(images);
        } else {
            double count = 0.0;
            double f1Sum = 0.0;
            for (String line = reader.readLine(); line != null; line = reader.readLine()) {
                SceneGraphImage img = SceneGraphImage.readFromJSON(line);
                if (img == null) {
                    continue;
                }
                for (SceneGraphImageRegion region : img.regions) {
                    count += 1.0;
                    SceneGraph scene = parser.parse(region.phrase);
                    System.err.println(region.phrase);
                    System.out.println(scene.toJSON(img.id, img.url, region.phrase));
                    System.err.println(scene.toReadableString());
                    System.err.println(region.toReadableString());
                    Triple<Double, Double, Double> scores = eval.evaluate(scene, region);
                    System.err.printf("Prec: %f, Recall: %f, F1: %f%n", scores.first, scores.second, scores.third);
                    eval.toSmatchString(scene, region, predWriter, goldWriter);
                    f1Sum += scores.third;
                    System.err.println("------------------------");
                }
            }
            System.err.println("#########################################################");
            System.err.printf("Macro-averaged F1: %f%n", f1Sum / count);
            System.err.println("#########################################################");
        }
    }
}

Example 9 with SceneGraphImageRegion

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageRegion in project CoreNLP by stanfordnlp.

the class BoWSceneGraphParser method main.

public static void main(String[] args) throws IOException {
    Properties props = StringUtils.argsToProperties(args, numArgs);
    boolean train = PropertiesUtils.getBool(props, "train", false);
    boolean enforceSubtree = PropertiesUtils.getBool(props, "enforceSubtree", true);
    boolean includeAllObjects = PropertiesUtils.getBool(props, "includeAllObjects", false);
    boolean verbose = PropertiesUtils.getBool(props, "verbose", false);
    String modelPath = PropertiesUtils.getString(props, "model", DEFAULT_MODEL_PATH);
    String entityModelPath = PropertiesUtils.getString(props, "entityModel", DEFAULT_ENTITY_MODEL_PATH);
    String inputPath = PropertiesUtils.getString(props, "input", null);
    String embeddingsPath = PropertiesUtils.getString(props, "embeddings", null);
    if (modelPath == null || entityModelPath == null || embeddingsPath == null) {
        System.err.printf("Usage java %s -model <model.gz>" + " -entityModel <entityModel.gz> -embeddings <wordVectors.gz> [-input <input.json> -train -verbose -enforceSubtree -evalFilePrefix <run0>]%n", BoWSceneGraphParser.class.getCanonicalName());
        return;
    }
    boolean interactive = (inputPath == null);
    Embedding embeddings = new Embedding(embeddingsPath);
    if (!train) {
        BoWSceneGraphParser parser = new BoWSceneGraphParser(modelPath, entityModelPath, embeddings);
        parser.enforceSubtree = enforceSubtree;
        parser.includeAllObjects = includeAllObjects;
        if (!interactive) {
            BufferedReader reader = IOUtils.readerFromString(inputPath);
            SceneGraphEvaluation eval = new SceneGraphEvaluation();
            String evalFilePrefix = PropertiesUtils.getString(props, "evalFilePrefix", null);
            PrintWriter predWriter = null;
            PrintWriter goldWriter = null;
            if (evalFilePrefix != null) {
                String predEvalFilePath = evalFilePrefix + ".smatch";
                String goldEvalFilePath = evalFilePrefix + "_gold.smatch";
                predWriter = IOUtils.getPrintWriter(predEvalFilePath);
                goldWriter = IOUtils.getPrintWriter(goldEvalFilePath);
            }
            double count = 0.0;
            double f1Sum = 0.0;
            for (String line = reader.readLine(); line != null; line = reader.readLine()) {
                SceneGraphImage img = SceneGraphImage.readFromJSON(line);
                if (img == null) {
                    continue;
                }
                for (SceneGraphImageRegion region : img.regions) {
                    count += 1.0;
                    SemanticGraph sg = region.getEnhancedSemanticGraph();
                    SceneGraph scene = parser.parse(sg);
                    System.out.println(scene.toJSON(img.id, img.url, region.phrase));
                    Triple<Double, Double, Double> scores = eval.evaluate(scene, region);
                    if (evalFilePrefix != null) {
                        eval.toSmatchString(scene, region, predWriter, goldWriter);
                    }
                    if (verbose) {
                        System.err.println(region.phrase);
                        System.err.println(scene.toReadableString());
                        System.err.println(region.toReadableString());
                        System.err.printf("Prec: %f, Recall: %f, F1: %f%n", scores.first, scores.second, scores.third);
                        System.err.println("------------------------");
                    }
                    f1Sum += scores.third;
                }
            }
            System.err.println("#########################################################");
            System.err.printf("Macro-averaged F1: %f%n", f1Sum / count);
            System.err.println("#########################################################");
        } else {
            System.err.println("Processing from stdin. Enter one sentence per line.");
            System.err.print("> ");
            Scanner scanner = new Scanner(System.in);
            String line;
            while ((line = scanner.nextLine()) != null) {
                SceneGraph scene = parser.parse(line);
                System.err.println(scene.toReadableString());
                System.err.println("------------------------");
                System.err.print("> ");
            }
            scanner.close();
        }
    } else {
        BoWSceneGraphParser parser = new BoWSceneGraphParser(null, entityModelPath, embeddings);
        parser.enforceSubtree = enforceSubtree;
        parser.train(inputPath, modelPath);
    }
}

Example 10 with SceneGraphImageRegion

use of edu.stanford.nlp.scenegraph.image.SceneGraphImageRegion in project CoreNLP by stanfordnlp.

the class BoWSceneGraphParser method getTrainingExamples.

/**
 * Generate training examples.
 *
 * @param trainingFile Path to JSON file with training images and scene graphs.
 * @param sampleNeg Whether to sample the same number of negative examples as positive examples.
 * @return Dataset to train a classifier.
 * @throws IOException
 */
public Dataset<String, String> getTrainingExamples(String trainingFile, boolean sampleNeg) throws IOException {
    Dataset<String, String> dataset = new Dataset<String, String>();
    Dataset<String, String> negDataset = new Dataset<String, String>();
    /* Load images. */
    List<SceneGraphImage> images = loadImages(trainingFile);
    for (SceneGraphImage image : images) {
        for (SceneGraphImageRegion region : image.regions) {
            SemanticGraph sg = region.getEnhancedSemanticGraph();
            SemanticGraphEnhancer.processQuanftificationModifiers(sg);
            SemanticGraphEnhancer.collapseCompounds(sg);
            SemanticGraphEnhancer.collapseParticles(sg);
            SemanticGraphEnhancer.resolvePronouns(sg);
            Set<Integer> entityPairs = Generics.newHashSet();
            List<Triple<IndexedWord, IndexedWord, String>> relationTriples = this.sentenceMatcher.getRelationTriples(region);
            for (Triple<IndexedWord, IndexedWord, String> triple : relationTriples) {
                IndexedWord iw1 = sg.getNodeByIndexSafe(triple.first.index());
                IndexedWord iw2 = sg.getNodeByIndexSafe(triple.second.index());
                if (iw1 != null && iw2 != null && (!enforceSubtree || SceneGraphUtils.inSameSubTree(sg, iw1, iw2))) {
                    entityClassifer.predictEntity(iw1, this.embeddings);
                    entityClassifer.predictEntity(iw2, this.embeddings);
                    BoWExample example = new BoWExample(iw1, iw2, sg);
                    dataset.add(example.extractFeatures(featureSets), triple.third);
                }
                entityPairs.add((triple.first.index() << 4) + triple.second.index());
            }
            /* Add negative examples. */
            List<IndexedWord> entities = EntityExtractor.extractEntities(sg);
            List<IndexedWord> attributes = EntityExtractor.extractAttributes(sg);
            for (IndexedWord e : entities) {
                entityClassifer.predictEntity(e, this.embeddings);
            }
            for (IndexedWord a : attributes) {
                entityClassifer.predictEntity(a, this.embeddings);
            }
            for (IndexedWord e1 : entities) {
                for (IndexedWord e2 : entities) {
                    if (e1.index() == e2.index()) {
                        continue;
                    }
                    int entityPair = (e1.index() << 4) + e2.index();
                    if (!entityPairs.contains(entityPair) && (!enforceSubtree || SceneGraphUtils.inSameSubTree(sg, e1, e2))) {
                        BoWExample example = new BoWExample(e1, e2, sg);
                        negDataset.add(example.extractFeatures(featureSets), NONE_RELATION);
                    }
                }
            }
            for (IndexedWord e : entities) {
                for (IndexedWord a : attributes) {
                    int entityPair = (e.index() << 4) + a.index();
                    if (!entityPairs.contains(entityPair) && (!enforceSubtree || SceneGraphUtils.inSameSubTree(sg, e, a))) {
                        BoWExample example = new BoWExample(e, a, sg);
                        negDataset.add(example.extractFeatures(featureSets), NONE_RELATION);
                    }
                }
            }
        }
    }
    /* Sample from negative examples to make the training set
     * more balanced. */
    if (sampleNeg && dataset.size() < negDataset.size()) {
        negDataset = negDataset.getRandomSubDataset(dataset.size() * 1.0 / negDataset.size(), 42);
    }
    dataset.addAll(negDataset);
    return dataset;
}