Examples with DataIndexer opennlp.model.DataIndexer used on opensource projects

Example 1 with DataIndexer

use of opennlp.model.DataIndexer in project talismane by joliciel-informatique.

the class MaxentModelTrainer method trainModel.

@Override
public ClassificationModel trainModel(ClassificationEventStream corpusEventStream, Map<String, List<String>> descriptors) throws IOException {
    MaxentModel maxentModel = null;
    EventStream eventStream = new OpenNLPEventStream(corpusEventStream);
    DataIndexer dataIndexer = new TwoPassRealValueDataIndexer(eventStream, cutoff);
    GISTrainer trainer = new GISTrainer(true);
    if (this.getSmoothing() > 0) {
        trainer.setSmoothing(true);
        trainer.setSmoothingObservation(this.getSmoothing());
    } else if (this.getSigma() > 0) {
        trainer.setGaussianSigma(this.getSigma());
    }
    maxentModel = trainer.trainModel(iterations, dataIndexer, cutoff);
    MaximumEntropyModel model = new MaximumEntropyModel(maxentModel, config, descriptors);
    model.addModelAttribute("cutoff", this.getCutoff());
    model.addModelAttribute("iterations", this.getIterations());
    model.addModelAttribute("sigma", this.getSigma());
    model.addModelAttribute("smoothing", this.getSmoothing());
    model.getModelAttributes().putAll(corpusEventStream.getAttributes());
    return model;
}