Examples with Token com.joliciel.talismane.tokeniser.Token used on opensource projects

Example 11 with Token

use of com.joliciel.talismane.tokeniser.Token in project talismane by joliciel-informatique.

the class NLetterPrefixFeature method checkInternal.

@Override
public FeatureResult<String> checkInternal(TokenWrapper tokenWrapper, RuntimeEnvironment env) throws TalismaneException {
    TokenWrapper innerWrapper = this.getToken(tokenWrapper, env);
    if (innerWrapper == null)
        return null;
    Token token = innerWrapper.getToken();
    FeatureResult<String> result = null;
    FeatureResult<Integer> nResult = nFeature.check(innerWrapper, env);
    if (nResult != null) {
        int n = nResult.getOutcome();
        String firstWord = token.getAnalyisText().trim();
        if (firstWord.indexOf(' ') >= 0) {
            firstWord = firstWord.substring(0, firstWord.indexOf(' '));
        }
        if (firstWord.length() > n) {
            String prefix = firstWord.substring(0, n);
            result = this.generateResult(prefix);
        }
    }
    return result;
}

Example 12 with Token

use of com.joliciel.talismane.tokeniser.Token in project talismane by joliciel-informatique.

the class NLetterSuffixFeature method checkInternal.

@Override
public FeatureResult<String> checkInternal(TokenWrapper tokenWrapper, RuntimeEnvironment env) throws TalismaneException {
    TokenWrapper innerWrapper = this.getToken(tokenWrapper, env);
    if (innerWrapper == null)
        return null;
    Token token = innerWrapper.getToken();
    FeatureResult<String> result = null;
    String lastWord = token.getAnalyisText().trim();
    if (lastWord.indexOf(' ') >= 0) {
        int lastSpace = lastWord.lastIndexOf(' ');
        lastWord = lastWord.substring(lastSpace + 1);
    }
    FeatureResult<Integer> nResult = nFeature.check(innerWrapper, env);
    if (nResult != null) {
        int n = nResult.getOutcome();
        if (lastWord.length() > n) {
            String suffix = lastWord.substring(lastWord.length() - n);
            result = this.generateResult(suffix);
        }
    }
    return result;
}

Example 13 with Token

use of com.joliciel.talismane.tokeniser.Token in project talismane by joliciel-informatique.

the class OrRangeFeature method checkInternal.

@Override
public FeatureResult<Boolean> checkInternal(TokenWrapper tokenWrapper, RuntimeEnvironment env) throws TalismaneException {
    TokenWrapper innerWrapper = this.getToken(tokenWrapper, env);
    if (innerWrapper == null)
        return null;
    Token token = innerWrapper.getToken();
    FeatureResult<Boolean> featureResult = null;
    FeatureResult<Integer> startResult = startFeature.check(innerWrapper, env);
    FeatureResult<Integer> endResult = endFeature.check(innerWrapper, env);
    if (startResult != null && endResult != null) {
        int start = startResult.getOutcome();
        int end = endResult.getOutcome();
        if (start < 0)
            start = 0;
        if (end > token.getTokenSequence().size() - 1)
            end = token.getTokenSequence().size() - 1;
        if (start <= end) {
            Boolean result = Boolean.FALSE;
            for (int i = start; i <= end; i++) {
                Token oneToken = token.getTokenSequence().get(i);
                FeatureResult<Boolean> criterionResult = this.criterion.check(oneToken, env);
                if (criterionResult == null) {
                    result = null;
                    break;
                }
                result = result || criterionResult.getOutcome();
            }
            if (result != null) {
                featureResult = this.generateResult(result);
            }
        }
    }
    return featureResult;
}

Example 14 with Token

use of com.joliciel.talismane.tokeniser.Token in project talismane by joliciel-informatique.

the class TokenAtAddressFunction method checkInternal.

@Override
public FeatureResult<TokenWrapper> checkInternal(TokenWrapper tokenWrapper, RuntimeEnvironment env) throws TalismaneException {
    TokenWrapper innerWrapper = this.getToken(tokenWrapper, env);
    if (innerWrapper == null)
        return null;
    Token token = innerWrapper.getToken();
    FeatureResult<TokenWrapper> result = null;
    FeatureResult<Integer> indexResult = indexFeature.check(innerWrapper, env);
    if (indexResult != null) {
        int i = indexResult.getOutcome();
        if (i >= 0 && i < token.getTokenSequence().size()) {
            Token indexedToken = token.getTokenSequence().get(i);
            result = this.generateResult(indexedToken);
        }
    }
    return result;
}

Example 15 with Token

use of com.joliciel.talismane.tokeniser.Token in project talismane by joliciel-informatique.

the class TokenIndexFeature method checkInternal.

@Override
public FeatureResult<Integer> checkInternal(TokenWrapper tokenWrapper, RuntimeEnvironment env) throws TalismaneException {
    TokenWrapper innerWrapper = this.getToken(tokenWrapper, env);
    if (innerWrapper == null)
        return null;
    Token token = innerWrapper.getToken();
    FeatureResult<Integer> result = null;
    int index = token.getIndex();
    result = this.generateResult(index);
    return result;
}