Examples with TestNLPGraph com.graphaware.nlp.util.TestNLPGraph used on opensource projects

Example 16 with TestNLPGraph

use of com.graphaware.nlp.util.TestNLPGraph in project neo4j-nlp-stanfordnlp by graphaware.

the class NasaLessonsLearnedTest method testNASACustomNER.

@Test
public void testNASACustomNER() {
    String modelsPath = getClass().getClassLoader().getResource("").getPath();
    executeInTransaction("CALL ga.nlp.config.model.workdir({p0})", buildSeqParameters(modelsPath), emptyConsumer());
    String text = "Apollo 1, initially designated AS-204, was the first manned mission of the United States Apollo program, which had as its ultimate goal a manned lunar landing.";
    String q = "CALL ga.nlp.processor.train({textProcessor: \"com.graphaware.nlp.processor.stanford.StanfordTextProcessor\", modelIdentifier: \"test-ner\", alg: \"ner\", inputFile: 'nasa-train.tsv', trainingParameters: {iter: 10}})";
    executeInTransaction(q, emptyConsumer());
    String t = "CALL ga.nlp.processor.test({textProcessor: \"com.graphaware.nlp.processor.stanford.StanfordTextProcessor\", modelIdentifier: \"test-ner\", alg: \"ner\", inputFile: 'nasa-test.tsv', trainingParameters: {iter: 10}})";
    executeInTransaction(t, emptyConsumer());
    // Create pipeline
    String addPipelineQuery = "CALL ga.nlp.processor.addPipeline({textProcessor: 'com.graphaware.nlp.processor.stanford.StanfordTextProcessor', name: 'customNER', processingSteps: {tokenize: true, ner: true, sentiment: false, dependency: true, customNER: \"test-ner\"}})";
    executeInTransaction(addPipelineQuery, emptyConsumer());
    // Import some text
    executeInTransaction("CREATE (n:Document) SET n.text = {text}", Collections.singletonMap("text", text), emptyConsumer());
    // Annotate
    executeInTransaction("MATCH (n:Document) CALL ga.nlp.annotate({text: n.text, id:id(n), pipeline: 'customNER', checkLanguage:false}) YIELD result MERGE (n)-[:HAS_ANNOTATED_TEXT]->(result)", emptyConsumer());
    executeInTransaction("MATCH (n:Tag) RETURN [x IN labels(n) | x] AS labels, n.value AS val", (result -> {
        while (result.hasNext()) {
            Map<String, Object> record = result.next();
            List<String> labels = (List<String>) record.get("labels");
            System.out.println(labels);
            System.out.println(record.get("val").toString());
        }
    }));
    TestNLPGraph testNLPGraph = new TestNLPGraph(getDatabase());
    testNLPGraph.assertTagWithValueHasNE("Apollo 1", "MISSION");
// // Check if some labels are there
// executeInTransaction("MATCH (n:NER_Mission) RETURN count(n) AS c", (result -> {
// assertTrue((long) result.next().get("c") > 0);
// }));
}

Example 17 with TestNLPGraph

use of com.graphaware.nlp.util.TestNLPGraph in project neo4j-nlp-stanfordnlp by graphaware.

the class SocialNetworkNERTest method testCustomNER.

@Test
public void testCustomNER() {
    String modelsPath = getClass().getClassLoader().getResource("").getPath();
    executeInTransaction("CALL ga.nlp.config.model.workdir({p0})", buildSeqParameters(modelsPath), emptyConsumer());
    String text = "But then came the fake news, News Feed addiction, violence on Facebook  Live, cyberbullying, abusive ad targeting, election interference and, most recently, the Cambridge Analytica app data privacy scandals. All the while, Facebook either willfully believed the worst case scenarios could never come true, was naive to their existence or calculated the benefits and growth outweighed the risks. And when finally confronted, Facebook often dragged its feet before admitting the extent of the issues.\n" + "\n" + "Inside the social network’s offices, the bonds began to fray. An ethics problem metastisized into a morale problem. Slogans took on sinister second meanings. The Kool-Aid tasted different.";
    String q = "CALL ga.nlp.processor.train({textProcessor: \"com.graphaware.nlp.processor.stanford.StanfordTextProcessor\", modelIdentifier: \"test-ner\", alg: \"ner\", inputFile: 'social-network-train.tsv', trainingParameters: {iter: 10}})";
    executeInTransaction(q, emptyConsumer());
    String t = "CALL ga.nlp.processor.test({textProcessor: \"com.graphaware.nlp.processor.stanford.StanfordTextProcessor\", modelIdentifier: \"test-ner\", alg: \"ner\", inputFile: 'nasa-test.tsv', trainingParameters: {iter: 10}})";
    // executeInTransaction(t, emptyConsumer());
    // Create pipeline
    String addPipelineQuery = "CALL ga.nlp.processor.addPipeline({textProcessor: 'com.graphaware.nlp.processor.stanford.StanfordTextProcessor', name: 'customNER', processingSteps: {tokenize: true, ner: true, sentiment: false, dependency: true, customNER: \"test-ner\"}})";
    executeInTransaction(addPipelineQuery, emptyConsumer());
    // Import some text
    executeInTransaction("CREATE (n:Document) SET n.text = {text}", Collections.singletonMap("text", text), emptyConsumer());
    String text2 = "Even Twitter’s massive growth excitement for years was characterized by the challenge of finding mass sustained appeal — “Why say what I had for lunch?” Now, Snapchat’s massive growth with millions of passionate users is characterized by some persistent user misunderstanding of Snapchat use cases — some ask, “Why show something I did if it gets deleted?” And there are unexplored adaptations to broaden the appeal of Snapchat Stories and strengthen chat.";
    executeInTransaction("CREATE (n:Document) SET n.text = {text}", Collections.singletonMap("text", text2), emptyConsumer());
    // Annotate
    executeInTransaction("MATCH (n:Document) CALL ga.nlp.annotate({text: n.text, id:id(n), pipeline: 'customNER', checkLanguage:false}) YIELD result MERGE (n)-[:HAS_ANNOTATED_TEXT]->(result)", emptyConsumer());
    executeInTransaction("MATCH (n:Tag) RETURN [x IN labels(n) | x] AS labels, n.value AS val", (result -> {
        while (result.hasNext()) {
            Map<String, Object> record = result.next();
            List<String> labels = (List<String>) record.get("labels");
            System.out.println(labels);
            System.out.println(record.get("val").toString());
        }
    }));
    TestNLPGraph testNLPGraph = new TestNLPGraph(getDatabase());
    testNLPGraph.assertTagWithValueHasNE("Facebook", "SOCIALNETWORK");
    testNLPGraph.assertTagWithValueHasNE("Snapchat", "SOCIALNETWORK");
    testNLPGraph.assertTagWithValueHasNE("Twitter", "SOCIALNETWORK");
// // Check if some labels are there
// executeInTransaction("MATCH (n:NER_Mission) RETURN count(n) AS c", (result -> {
// assertTrue((long) result.next().get("c") > 0);
// }));
}

Example 18 with TestNLPGraph

use of com.graphaware.nlp.util.TestNLPGraph in project neo4j-nlp-stanfordnlp by graphaware.

the class TextProcessorIntegrationTest method testAnnotationWithPipelineWithoutNERProcessingStep.

@Test
public void testAnnotationWithPipelineWithoutNERProcessingStep() {
    clearDb();
    executeInTransaction("CALL ga.nlp.processor.addPipeline({name:\"customie\", stopWords:\"hello,build\", textProcessor:'" + StanfordTextProcessor.class.getName() + "', processingSteps:{tokenize:true, ner: false}})", (result -> {
        assertTrue(result.hasNext());
    }));
    String text = "My name is John Doe and I work in Switzerland";
    executeInTransaction("CALL ga.nlp.annotate({text:'" + text + "', pipeline:'customie', id:'test', checkLanguage:false})", (result -> {
        assertTrue(result.hasNext());
    }));
    TestNLPGraph tester = new TestNLPGraph(getDatabase());
    tester.assertNodesCount("NER_Person", 0);
    tester.assertNodesCount("NER_Location", 0);
    tester.assertNodesCount("NER_O", 0);
}

Example 19 with TestNLPGraph

use of com.graphaware.nlp.util.TestNLPGraph in project neo4j-nlp by graphaware.

the class AnnotatedTextPersistenceTest method testTagOccurrenceGetAValue.

@Test
public void testTagOccurrenceGetAValue() {
    clearDb();
    AnnotatedText annotatedText = createAnnotatedTextWithSameTagInSameTextWithDifferentPos();
    TestNLPGraph test = new TestNLPGraph(getDatabase());
    try (Transaction tx = getDatabase().beginTx()) {
        getNLPManager().getPersister(AnnotatedText.class).persist(annotatedText, "test", "1");
        tx.success();
    }
    test.assertTagOccurrenceWithValueExist("cool");
    executeInTransaction("MATCH (n:TagOccurrence) WHERE n.value = 'cool' RETURN n", (result -> {
        assertTrue(result.hasNext());
        Node n = (Node) result.next().get("n");
        String[] ners = (String[]) n.getProperty("ne");
        assertTrue(Arrays.asList(ners).contains("NER_Cool0"));
    }));
}

Example 20 with TestNLPGraph

use of com.graphaware.nlp.util.TestNLPGraph in project neo4j-nlp by graphaware.

the class AnnotatedTextPersistenceTest method testTagsHavingTwoDifferentPOSInDifferentSentencesShouldReflectBothPOS.

@Test
public void testTagsHavingTwoDifferentPOSInDifferentSentencesShouldReflectBothPOS() {
    String text = "The discipline of preparing and peer reviewing formal engineering reports leads to a high degree of accuracy and technical rigor.";
    String text2 = "During this effort to establish accurate crack information, it was discovered that several cracks were kinked rather than extending in a self-similar crack growth direction as was implied by the sketches and analyses reports in the briefing charts.";
    TestNLPGraph test = new TestNLPGraph(getDatabase());
    AnnotatedText at1 = createAnnotatedTextFor(text, "reports", "VGB");
    try (Transaction tx = getDatabase().beginTx()) {
        getNLPManager().getPersister(AnnotatedText.class).persist(at1, "test-a", "1");
        tx.success();
    }
    test.assertTagWithValueHasPos("reports", "VGB");
    AnnotatedText at2 = createAnnotatedTextFor(text2, "reports", "NNS");
    try (Transaction tx = getDatabase().beginTx()) {
        getNLPManager().getPersister(AnnotatedText.class).persist(at2, "test-b", String.valueOf(System.currentTimeMillis()));
        tx.success();
    }
    test.assertTagWithValueHasPos("reports", "VGB");
    test.assertTagWithValueHasPos("reports", "NNS");
}