use of org.apache.tika.language.LanguageIdentifier in project lucene-solr by apache.
the class TikaLanguageIdentifierUpdateProcessor method detectLanguage.
@Override
protected List<DetectedLanguage> detectLanguage(SolrInputDocument doc) {
List<DetectedLanguage> languages = new ArrayList<>();
String content = concatFields(doc);
if (content.length() != 0) {
LanguageIdentifier identifier = new LanguageIdentifier(content);
// FIXME: Hack - we get the distance from toString and calculate our own certainty score
Double distance = Double.parseDouble(tikaSimilarityPattern.matcher(identifier.toString()).replaceFirst("$1"));
// This formula gives: 0.02 => 0.8, 0.1 => 0.5 which is a better sweetspot than isReasonablyCertain()
Double certainty = 1 - (5 * distance);
if (certainty < 0)
certainty = 0d;
DetectedLanguage language = new DetectedLanguage(identifier.getLanguage(), certainty);
languages.add(language);
log.debug("Language detected as " + language + " with a certainty of " + language.getCertainty() + " (Tika distance=" + identifier.toString() + ")");
} else {
log.debug("No input text to detect language from, returning empty list");
}
return languages;
}
Aggregations