use of com.joliciel.talismane.machineLearning.features.RuntimeEnvironment in project jochre by urieli.
the class Jochre method doCommandTestFeature.
/**
* Test a feature on a particular shape.
*/
public void doCommandTestFeature(int shapeId) {
// just a utility for testing a feature on a particular shape
ShapeFeature<?> feature = new VerticalElongationFeature();
GraphicsDao graphicsDao = GraphicsDao.getInstance(jochreSession);
if (shapeId > 0) {
Shape shape = graphicsDao.loadShape(shapeId);
shape.writeImageToLog();
RuntimeEnvironment env = new RuntimeEnvironment();
feature.check(shape, env);
} else {
// String result = "false";
// TrainingServiceLocator trainingServiceLocator =
// locator.getTrainingServiceLocator();
// TrainingService trainingService =
// trainingServiceLocator.getTrainingService();
// List<Integer> shapeIds =
// trainingService.findShapesForFeature("ג", feature, result);
List<Integer> shapeIds = graphicsDao.findShapeIds("—");
Map<Object, Integer> outcomeMap = new HashMap<>();
for (int oneShapeId : shapeIds) {
Shape shape = graphicsDao.loadShape(oneShapeId);
shape.writeImageToLog();
RuntimeEnvironment env = new RuntimeEnvironment();
FeatureResult<?> weightedOutcome = feature.check(shape, env);
Object outcome = weightedOutcome.getOutcome();
Integer count = outcomeMap.get(outcome);
if (count == null)
outcomeMap.put(outcome, 1);
else
outcomeMap.put(outcome, count.intValue() + 1);
}
LOG.debug("Outcomes:");
for (Object outcome : outcomeMap.keySet()) {
LOG.debug("Outcome " + outcome.toString() + ", count " + outcomeMap.get(outcome));
}
}
}
use of com.joliciel.talismane.machineLearning.features.RuntimeEnvironment in project jochre by urieli.
the class JochreMergeEventStream method next.
@Override
public ClassificationEvent next() {
ClassificationEvent event = null;
if (this.hasNext()) {
LOG.debug("next event, " + mergeCandidate.getFirstShape() + ", " + mergeCandidate.getSecondShape());
List<FeatureResult<?>> featureResults = new ArrayList<>();
// analyse features
for (MergeFeature<?> feature : mergeFeatures) {
RuntimeEnvironment env = new RuntimeEnvironment();
FeatureResult<?> featureResult = feature.check(mergeCandidate, env);
if (featureResult != null) {
featureResults.add(featureResult);
if (LOG.isTraceEnabled()) {
LOG.trace(featureResult.toString());
}
}
}
MergeOutcome outcome = MergeOutcome.DO_NOT_MERGE;
boolean shouldMerge = false;
if (mergeCandidate.getFirstShape().getLetter().startsWith("|")) {
if (mergeCandidate.getSecondShape().getLetter().length() == 0 || mergeCandidate.getSecondShape().getLetter().endsWith("|"))
shouldMerge = true;
} else if (mergeCandidate.getSecondShape().getLetter().endsWith("|")) {
if (mergeCandidate.getFirstShape().getLetter().length() == 0)
shouldMerge = true;
}
if (shouldMerge)
outcome = MergeOutcome.DO_MERGE;
if (outcome.equals(MergeOutcome.DO_MERGE))
yesCount++;
else
noCount++;
LOG.debug("Outcome: " + outcome);
event = new ClassificationEvent(featureResults, outcome.name());
// set mergeCandidate to null so that hasNext can retrieve the next
// one.
this.mergeCandidate = null;
}
return event;
}
use of com.joliciel.talismane.machineLearning.features.RuntimeEnvironment in project jochre by urieli.
the class ShapeMerger method checkMerge.
/**
* Given two sequential shape, returns the probability of a merge.
*/
public double checkMerge(Shape shape1, Shape shape2) {
ShapePair mergeCandidate = new ShapePair(shape1, shape2);
if (LOG.isTraceEnabled())
LOG.trace("mergeCandidate: " + mergeCandidate);
List<FeatureResult<?>> featureResults = new ArrayList<FeatureResult<?>>();
// analyse features
for (MergeFeature<?> feature : mergeFeatures) {
RuntimeEnvironment env = new RuntimeEnvironment();
FeatureResult<?> featureResult = feature.check(mergeCandidate, env);
if (featureResult != null) {
featureResults.add(featureResult);
if (LOG.isTraceEnabled()) {
LOG.trace(featureResult.toString());
}
}
}
List<Decision> decisions = decisionMaker.decide(featureResults);
double yesProb = 0.0;
for (Decision decision : decisions) {
if (decision.getOutcome().equals(MergeOutcome.DO_MERGE)) {
yesProb = decision.getProbability();
break;
}
}
if (LOG.isTraceEnabled()) {
LOG.trace("yesProb: " + yesProb);
}
return yesProb;
}
use of com.joliciel.talismane.machineLearning.features.RuntimeEnvironment in project talismane by joliciel-informatique.
the class PosTagEventStream method next.
@Override
public ClassificationEvent next() throws TalismaneException, IOException {
ClassificationEvent event = null;
if (this.hasNext()) {
PosTaggedToken taggedToken = currentSentence.get(currentIndex++);
String classification = taggedToken.getTag().getCode();
if (LOG.isDebugEnabled())
LOG.debug("next event, token: " + taggedToken.getToken().getAnalyisText() + " : " + classification);
PosTaggerContext context = new PosTaggerContextImpl(taggedToken.getToken(), currentHistory);
List<FeatureResult<?>> posTagFeatureResults = new ArrayList<FeatureResult<?>>();
for (PosTaggerFeature<?> posTaggerFeature : posTaggerFeatures) {
RuntimeEnvironment env = new RuntimeEnvironment();
FeatureResult<?> featureResult = posTaggerFeature.check(context, env);
if (featureResult != null)
posTagFeatureResults.add(featureResult);
}
if (LOG.isTraceEnabled()) {
LOG.trace("Token: " + taggedToken.getToken().getAnalyisText());
SortedSet<String> featureResultSet = posTagFeatureResults.stream().map(f -> f.toString()).collect(Collectors.toCollection(() -> new TreeSet<String>()));
for (String featureResultString : featureResultSet) {
LOG.trace(featureResultString);
}
}
event = new ClassificationEvent(posTagFeatureResults, classification);
currentHistory.addPosTaggedToken(taggedToken);
if (currentIndex == currentSentence.size()) {
currentSentence = null;
}
}
return event;
}
use of com.joliciel.talismane.machineLearning.features.RuntimeEnvironment in project talismane by joliciel-informatique.
the class LanguageDetectorEventStream method next.
@Override
public ClassificationEvent next() throws TalismaneException {
LanguageTaggedText languageTaggedText = this.corpusReader.nextText();
List<FeatureResult<?>> featureResults = new ArrayList<FeatureResult<?>>();
for (LanguageDetectorFeature<?> feature : features) {
RuntimeEnvironment env = new RuntimeEnvironment();
FeatureResult<?> featureResult = feature.check(languageTaggedText.getText(), env);
if (featureResult != null)
featureResults.add(featureResult);
}
String classification = languageTaggedText.getLanguage().toLanguageTag();
if (LOG.isTraceEnabled()) {
for (FeatureResult<?> result : featureResults) {
LOG.trace(result.toString());
}
LOG.trace("classification: " + classification);
}
ClassificationEvent event = new ClassificationEvent(featureResults, classification);
return event;
}
Aggregations