Examples with PipelineSpecification com.graphaware.nlp.dsl.request.PipelineSpecification used on opensource projects

Example 1 with PipelineSpecification

use of com.graphaware.nlp.dsl.request.PipelineSpecification in project neo4j-nlp by graphaware.

the class AnnotationPersistenceIntegrationTest method testAnnotationWithCustomPipeline.

@Test
public void testAnnotationWithCustomPipeline() {
    Map<String, Object> spec = new HashMap<>();
    spec.put("name", "my-pipeline");
    spec.put("textProcessor", StubTextProcessor.class.getName());
    spec.put("processingSteps", Collections.singletonMap("tokenize", true));
    spec.put("excludedNER", Collections.singletonList("test"));
    PipelineSpecification pipelineSpecification = PipelineSpecification.fromMap(spec);
    getNLPManager().addPipeline(pipelineSpecification);
    try (Transaction tx = getDatabase().beginTx()) {
        manager.annotateTextAndPersist("hello my name is John. I am working for IBM. I live in Italy", "123-fff", true, pipelineSpecification);
        tx.success();
    }
    TestNLPGraph tester = new TestNLPGraph(getDatabase());
    tester.assertNodesCount("test", 0);
}

Example 2 with PipelineSpecification

use of com.graphaware.nlp.dsl.request.PipelineSpecification in project neo4j-nlp by graphaware.

the class DynamicConfigurationTest method testConfigurationCanStoreAndRetrievePipelines.

@Test
public void testConfigurationCanStoreAndRetrievePipelines() {
    DynamicConfiguration configuration = new DynamicConfiguration(getDatabase());
    PipelineSpecification specification = new PipelineSpecification("custom", StubTextProcessor.class.getName());
    specification.setStopWords("hello,hihi");
    specification.setThreadNumber(4);
    configuration.storeCustomPipeline(specification);
    try (Transaction tx = getDatabase().beginTx()) {
        assertTrue(keyValueStore.hasKey("GA__NLP__PIPELINE_custom"));
        tx.success();
    }
}

Example 3 with PipelineSpecification

use of com.graphaware.nlp.dsl.request.PipelineSpecification in project neo4j-nlp by graphaware.

the class DynamicConfigurationTest method testConfigurationValuesShouldBeLoadedFromPreviousState.

@Test
public void testConfigurationValuesShouldBeLoadedFromPreviousState() throws Exception {
    resetSingleton();
    ObjectMapper mapper = new ObjectMapper();
    mapper.enableDefaultTyping(ObjectMapper.DefaultTyping.NON_FINAL, JsonTypeInfo.As.PROPERTY);
    keyValueStore = new GraphKeyValueStore(getDatabase());
    PipelineSpecification specification = new PipelineSpecification("custom", StubTextProcessor.class.getName());
    specification.setStopWords("hello,hihi");
    specification.setThreadNumber(4);
    try (Transaction tx = getDatabase().beginTx()) {
        String writeValueAsString = mapper.writeValueAsString(specification);
        keyValueStore.set("GA__NLP__PIPELINE_custom", writeValueAsString);
        keyValueStore.set("GA__NLP__SETTING_fallbackLanguage", "en");
        tx.success();
    }
    GraphAwareRuntime runtime = GraphAwareRuntimeFactory.createRuntime(getDatabase());
    runtime.registerModule(new NLPModule("NLP", NLPConfiguration.defaultConfiguration(), getDatabase()));
    runtime.start();
    runtime.waitUntilStarted();
    assertTrue(getStartedRuntime(getDatabase()).getModule(NLPModule.class).getNlpManager().getTextProcessorsManager().getTextProcessor(StubTextProcessor.class.getName()).getPipelines().contains("custom"));
    assertTrue(getStartedRuntime(getDatabase()).getModule(NLPModule.class).getNlpManager().getConfiguration().hasSettingValue(SettingsConstants.FALLBACK_LANGUAGE));
}

Example 4 with PipelineSpecification

use of com.graphaware.nlp.dsl.request.PipelineSpecification in project neo4j-nlp by graphaware.

the class DynamicConfigurationTest method testConfigurationCanLoadCustomPipelineAsObject.

@Test
public void testConfigurationCanLoadCustomPipelineAsObject() {
    DynamicConfiguration configuration = new DynamicConfiguration(getDatabase());
    PipelineSpecification specification = new PipelineSpecification("custom", StubTextProcessor.class.getName());
    specification.setStopWords("hello,hihi");
    specification.setThreadNumber(4);
    configuration.storeCustomPipeline(specification);
    PipelineSpecification pipelineSpecification = configuration.loadCustomPipelines().get(0);
    assertEquals(specification.getName(), pipelineSpecification.getName());
    assertEquals(specification.getStopWords(), pipelineSpecification.getStopWords());
    assertEquals(specification.getTextProcessor(), pipelineSpecification.getTextProcessor());
    assertEquals(specification.getThreadNumber(), pipelineSpecification.getThreadNumber());
}

Example 5 with PipelineSpecification

use of com.graphaware.nlp.dsl.request.PipelineSpecification in project neo4j-nlp by graphaware.

the class WorkflowTextProcessor method handle.

@Override
public void handle(WorkflowInputEntry entry) {
    if (entry instanceof WorkflowInputEndOfQueueEntry) {
        super.checkAndHandle(new WorkflowProcessorEndOfQueueEntry());
        return;
    }
    if (isValid()) {
        long start = -System.currentTimeMillis();
        String lang = NLPManager.getInstance().checkTextLanguage(entry.getText(), getConfiguration().checkLanguage());
        System.out.println("Time for getting lang: " + (System.currentTimeMillis() + start));
        String pipeline = NLPManager.getInstance().getPipeline(getConfiguration().getPipeline());
        PipelineSpecification pipelineSpecification = NLPManager.getInstance().getConfiguration().loadPipeline(pipeline);
        if (null == pipelineSpecification) {
            throw new RuntimeException("No pipeline " + pipeline);
        }
        AnnotatedText annotateText = textProcessor.annotateText(entry.getText(), lang, pipelineSpecification);
        super.checkAndHandle(new WorkflowProcessorOutputEntry(annotateText, entry.getId()));
    } else {
        LOG.warn("The Processor " + this.getName() + " is in an invalid state");
        return;
    }
}