Search in sources :

Example 1 with UimaTaskBase

use of org.dkpro.lab.uima.task.impl.UimaTaskBase in project dkpro-lab by dkpro.

the class PosExampleCrf method run.

@Test
public void run() throws Exception {
    // Route logging through log4j
    System.setProperty("org.apache.uima.logger.class", "org.apache.uima.util.impl.Log4jLogger_impl");
    clean();
    Task preprocessingTask = new UimaTaskBase() {

        @Discriminator
        String corpusPath;

        {
            setType("Preprocessing");
        }

        @Override
        public CollectionReaderDescription getCollectionReaderDescription(TaskContext aContext) throws ResourceInitializationException, IOException {
            return createReader(NegraExportReader.class, NegraExportReader.PARAM_SOURCE_LOCATION, corpusPath, NegraExportReader.PARAM_LANGUAGE, "de");
        }

        @Override
        public AnalysisEngineDescription getAnalysisEngineDescription(TaskContext aContext) throws ResourceInitializationException, IOException {
            File xmiDir = aContext.getFolder("XMI", AccessMode.READWRITE);
            return createEngine(createEngine(SnowballStemmer.class), createEngine(XmiWriter.class, XmiWriter.PARAM_TARGET_LOCATION, xmiDir.getAbsolutePath(), XmiWriter.PARAM_COMPRESSION, CompressionMethod.GZIP));
        }
    };
    Task featureExtractionTask = new UimaTaskBase() {

        {
            setType("FeatureExtraction");
        }

        @Override
        public CollectionReaderDescription getCollectionReaderDescription(TaskContext aContext) throws ResourceInitializationException, IOException {
            File xmiDir = aContext.getFolder("XMI", AccessMode.READONLY);
            return createReader(XmiReader.class, XmiReader.PARAM_SOURCE_LOCATION, xmiDir.getAbsolutePath(), XmiReader.PARAM_PATTERNS, new String[] { "[+]**/*.xmi.gz" });
        }

        @Override
        public AnalysisEngineDescription getAnalysisEngineDescription(TaskContext aContext) throws ResourceInitializationException, IOException {
            File modelDir = aContext.getFolder("MODEL", AccessMode.READWRITE);
            return createEngine(createEngineDescription(ExamplePosAnnotator.class, ExamplePosAnnotator.PARAM_DATA_WRITER_FACTORY_CLASS_NAME, DefaultMalletCRFDataWriterFactory.class.getName(), DefaultMalletCRFDataWriterFactory.PARAM_OUTPUT_DIRECTORY, modelDir.getAbsolutePath()));
        }
    };
    Task trainingTask = new ExecutableTaskBase() {

        {
            setType("TrainingTask");
        }

        @Override
        public void execute(TaskContext aContext) throws Exception {
            File dir = aContext.getFolder("MODEL", AccessMode.READWRITE);
            JarClassifierBuilder<?> classifierBuilder = JarClassifierBuilder.fromTrainingDirectory(dir);
            classifierBuilder.trainClassifier(dir, new String[0]);
            classifierBuilder.packageClassifier(dir);
        }
    };
    Task analysisTask = new UimaTaskBase() {

        {
            setType("AnalysisTask");
        }

        @Override
        public CollectionReaderDescription getCollectionReaderDescription(TaskContext aContext) throws ResourceInitializationException, IOException {
            return createReaderDescription(TextReader.class, TextReader.PARAM_SOURCE_LOCATION, "src/test/resources/text", TextReader.PARAM_PATTERNS, new String[] { "[+]**/*.txt" }, TextReader.PARAM_LANGUAGE, "de");
        }

        @Override
        public AnalysisEngineDescription getAnalysisEngineDescription(TaskContext aContext) throws ResourceInitializationException, IOException {
            File model = new File(aContext.getFolder("MODEL", AccessMode.READONLY), "model.jar");
            File tsv = new File(aContext.getFolder("TSV", AccessMode.READWRITE), "output.tsv");
            return createEngine(createEngineDescription(BreakIteratorSegmenter.class), createEngineDescription(SnowballStemmer.class), createEngineDescription(ExamplePosAnnotator.class, GenericJarClassifierFactory.PARAM_CLASSIFIER_JAR_PATH, model.getAbsolutePath()), createEngineDescription(ImsCwbWriter.class, ImsCwbWriter.PARAM_TARGET_LOCATION, tsv));
        }
    };
    ParameterSpace pSpace = new ParameterSpace(Dimension.create("corpusPath", CORPUS_PATH));
    featureExtractionTask.addImport(preprocessingTask, "XMI");
    trainingTask.addImport(featureExtractionTask, "MODEL");
    analysisTask.addImport(trainingTask, "MODEL");
    DefaultBatchTask batch = new DefaultBatchTask();
    batch.setParameterSpace(pSpace);
    batch.setExecutionPolicy(ExecutionPolicy.USE_EXISTING);
    batch.addTask(preprocessingTask);
    batch.addTask(featureExtractionTask);
    batch.addTask(trainingTask);
    batch.addTask(analysisTask);
    Lab.getInstance().run(batch);
}
Also used : Task(org.dkpro.lab.task.Task) DefaultBatchTask(org.dkpro.lab.task.impl.DefaultBatchTask) UimaTaskBase(org.dkpro.lab.uima.task.impl.UimaTaskBase) TaskContext(org.dkpro.lab.engine.TaskContext) SnowballStemmer(de.tudarmstadt.ukp.dkpro.core.snowball.SnowballStemmer) XmiWriter(de.tudarmstadt.ukp.dkpro.core.io.xmi.XmiWriter) ExecutableTaskBase(org.dkpro.lab.task.impl.ExecutableTaskBase) ExamplePosAnnotator(org.dkpro.lab.ml.example.ExamplePosAnnotator) ParameterSpace(org.dkpro.lab.task.ParameterSpace) BreakIteratorSegmenter(de.tudarmstadt.ukp.dkpro.core.tokit.BreakIteratorSegmenter) ImsCwbWriter(de.tudarmstadt.ukp.dkpro.core.io.imscwb.ImsCwbWriter) File(java.io.File) DefaultBatchTask(org.dkpro.lab.task.impl.DefaultBatchTask) Test(org.junit.Test)

Example 2 with UimaTaskBase

use of org.dkpro.lab.uima.task.impl.UimaTaskBase in project dkpro-lab by dkpro.

the class PosExampleMaxEnt method run.

@Test
public void run() throws Exception {
    // Route logging through log4j
    System.setProperty("org.apache.uima.logger.class", "org.apache.uima.util.impl.Log4jLogger_impl");
    clean();
    Task preprocessingTask = new UimaTaskBase() {

        @Discriminator
        String corpusPath;

        {
            setType("Preprocessing");
        }

        @Override
        public CollectionReaderDescription getCollectionReaderDescription(TaskContext aContext) throws ResourceInitializationException, IOException {
            return createReader(NegraExportReader.class, NegraExportReader.PARAM_SOURCE_LOCATION, corpusPath, NegraExportReader.PARAM_LANGUAGE, "de");
        }

        @Override
        public AnalysisEngineDescription getAnalysisEngineDescription(TaskContext aContext) throws ResourceInitializationException, IOException {
            File xmiDir = aContext.getFolder("XMI", AccessMode.READWRITE);
            return createEngine(createEngine(SnowballStemmer.class), createEngine(XmiWriter.class, XmiWriter.PARAM_TARGET_LOCATION, xmiDir.getAbsolutePath(), XmiWriter.PARAM_COMPRESSION, CompressionMethod.GZIP));
        }
    };
    Task featureExtractionTask = new UimaTaskBase() {

        {
            setType("FeatureExtraction");
        }

        @Override
        public CollectionReaderDescription getCollectionReaderDescription(TaskContext aContext) throws ResourceInitializationException, IOException {
            File xmiDir = aContext.getFolder("XMI", AccessMode.READONLY);
            return createReader(XmiReader.class, XmiReader.PARAM_SOURCE_LOCATION, xmiDir.getAbsolutePath(), XmiReader.PARAM_PATTERNS, new String[] { "[+]**/*.xmi.gz" });
        }

        @Override
        public AnalysisEngineDescription getAnalysisEngineDescription(TaskContext aContext) throws ResourceInitializationException, IOException {
            File modelDir = aContext.getFolder("MODEL", AccessMode.READWRITE);
            return createEngine(createEngineDescription(ExamplePosAnnotator.class, ExamplePosAnnotator.PARAM_DATA_WRITER_FACTORY_CLASS_NAME, ViterbiDataWriterFactory.class.getName(), ViterbiDataWriterFactory.PARAM_OUTPUT_DIRECTORY, modelDir.getAbsolutePath(), ViterbiDataWriterFactory.PARAM_DELEGATED_DATA_WRITER_FACTORY_CLASS, DefaultMaxentDataWriterFactory.class.getName()));
        }
    };
    Task trainingTask = new ExecutableTaskBase() {

        @Discriminator
        private int iterations;

        @Discriminator
        private int cutoff;

        {
            setType("TrainingTask");
        }

        @Override
        public void execute(TaskContext aContext) throws Exception {
            File dir = aContext.getFolder("MODEL", AccessMode.READWRITE);
            JarClassifierBuilder<?> classifierBuilder = JarClassifierBuilder.fromTrainingDirectory(dir);
            classifierBuilder.trainClassifier(dir, new String[] { String.valueOf(iterations), String.valueOf(cutoff) });
            classifierBuilder.packageClassifier(dir);
        }
    };
    Task analysisTask = new UimaTaskBase() {

        {
            setType("AnalysisTask");
        }

        @Override
        public CollectionReaderDescription getCollectionReaderDescription(TaskContext aContext) throws ResourceInitializationException, IOException {
            return createReaderDescription(TextReader.class, TextReader.PARAM_SOURCE_LOCATION, "src/test/resources/text/**/*.txt", TextReader.PARAM_LANGUAGE, "de");
        }

        @Override
        public AnalysisEngineDescription getAnalysisEngineDescription(TaskContext aContext) throws ResourceInitializationException, IOException {
            File model = new File(aContext.getFolder("MODEL", AccessMode.READONLY), "model.jar");
            File tsv = new File(aContext.getFolder("TSV", AccessMode.READWRITE), "output.tsv");
            return createEngine(createEngineDescription(BreakIteratorSegmenter.class), createEngineDescription(SnowballStemmer.class), createEngineDescription(ExamplePosAnnotator.class, GenericJarClassifierFactory.PARAM_CLASSIFIER_JAR_PATH, model.getAbsolutePath()), createEngineDescription(ImsCwbWriter.class, ImsCwbWriter.PARAM_TARGET_LOCATION, tsv));
        }
    };
    ParameterSpace pSpace = new ParameterSpace(Dimension.create("corpusPath", CORPUS_PATH), Dimension.create("iterations", 20, 50, 100), Dimension.create("cutoff", 5));
    featureExtractionTask.addImport(preprocessingTask, "XMI");
    trainingTask.addImport(featureExtractionTask, "MODEL");
    analysisTask.addImport(trainingTask, "MODEL");
    DefaultBatchTask batch = new DefaultBatchTask();
    batch.setParameterSpace(pSpace);
    batch.setExecutionPolicy(ExecutionPolicy.USE_EXISTING);
    batch.addTask(preprocessingTask);
    batch.addTask(featureExtractionTask);
    batch.addTask(trainingTask);
    batch.addTask(analysisTask);
    Lab.getInstance().run(batch);
}
Also used : Task(org.dkpro.lab.task.Task) DefaultBatchTask(org.dkpro.lab.task.impl.DefaultBatchTask) UimaTaskBase(org.dkpro.lab.uima.task.impl.UimaTaskBase) TaskContext(org.dkpro.lab.engine.TaskContext) SnowballStemmer(de.tudarmstadt.ukp.dkpro.core.snowball.SnowballStemmer) XmiWriter(de.tudarmstadt.ukp.dkpro.core.io.xmi.XmiWriter) ExecutableTaskBase(org.dkpro.lab.task.impl.ExecutableTaskBase) ExamplePosAnnotator(org.dkpro.lab.ml.example.ExamplePosAnnotator) ParameterSpace(org.dkpro.lab.task.ParameterSpace) BreakIteratorSegmenter(de.tudarmstadt.ukp.dkpro.core.tokit.BreakIteratorSegmenter) ImsCwbWriter(de.tudarmstadt.ukp.dkpro.core.io.imscwb.ImsCwbWriter) File(java.io.File) DefaultBatchTask(org.dkpro.lab.task.impl.DefaultBatchTask) Test(org.junit.Test)

Aggregations

ImsCwbWriter (de.tudarmstadt.ukp.dkpro.core.io.imscwb.ImsCwbWriter)2 XmiWriter (de.tudarmstadt.ukp.dkpro.core.io.xmi.XmiWriter)2 SnowballStemmer (de.tudarmstadt.ukp.dkpro.core.snowball.SnowballStemmer)2 BreakIteratorSegmenter (de.tudarmstadt.ukp.dkpro.core.tokit.BreakIteratorSegmenter)2 File (java.io.File)2 TaskContext (org.dkpro.lab.engine.TaskContext)2 ExamplePosAnnotator (org.dkpro.lab.ml.example.ExamplePosAnnotator)2 ParameterSpace (org.dkpro.lab.task.ParameterSpace)2 Task (org.dkpro.lab.task.Task)2 DefaultBatchTask (org.dkpro.lab.task.impl.DefaultBatchTask)2 ExecutableTaskBase (org.dkpro.lab.task.impl.ExecutableTaskBase)2 UimaTaskBase (org.dkpro.lab.uima.task.impl.UimaTaskBase)2 Test (org.junit.Test)2