use of org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.ITokenFactory in project asterixdb by apache.
the class GramTokensDescriptor method createEvaluatorFactory.
@Override
public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
return new IScalarEvaluatorFactory() {
private static final long serialVersionUID = 1L;
@Override
public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
ITokenFactory tokenFactory = new UTF8NGramTokenFactory();
NGramUTF8StringBinaryTokenizer tokenizer = new NGramUTF8StringBinaryTokenizer(3, true, true, true, tokenFactory);
return new GramTokensEvaluator(args, ctx, tokenizer, BuiltinType.ASTRING);
}
};
}
use of org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.ITokenFactory in project asterixdb by apache.
the class HashedGramTokensDescriptor method createEvaluatorFactory.
@Override
public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
return new IScalarEvaluatorFactory() {
private static final long serialVersionUID = 1L;
@Override
public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
ITokenFactory tokenFactory = new HashedUTF8NGramTokenFactory();
NGramUTF8StringBinaryTokenizer tokenizer = new NGramUTF8StringBinaryTokenizer(3, true, true, true, tokenFactory);
return new GramTokensEvaluator(args, ctx, tokenizer, BuiltinType.AINT32);
}
};
}
use of org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.ITokenFactory in project asterixdb by apache.
the class HashedWordTokensDescriptor method createEvaluatorFactory.
@Override
public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
return new IScalarEvaluatorFactory() {
private static final long serialVersionUID = 1L;
@Override
public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
ITokenFactory tokenFactory = new HashedUTF8WordTokenFactory();
IBinaryTokenizer tokenizer = new DelimitedUTF8StringBinaryTokenizer(true, true, tokenFactory);
return new WordTokensEvaluator(args, ctx, tokenizer, BuiltinType.AINT32);
}
};
}
use of org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.ITokenFactory in project asterixdb by apache.
the class WordTokensDescriptor method createEvaluatorFactory.
@Override
public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
return new IScalarEvaluatorFactory() {
private static final long serialVersionUID = 1L;
@Override
public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
ITokenFactory tokenFactory = new UTF8WordTokenFactory();
IBinaryTokenizer tokenizer = new DelimitedUTF8StringBinaryTokenizer(true, true, tokenFactory);
return new WordTokensEvaluator(args, ctx, tokenizer, BuiltinType.ASTRING);
}
};
}
use of org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.ITokenFactory in project asterixdb by apache.
the class LSMInvertedIndexTestUtils method createHashedNGramInvIndexTestContext.
public static LSMInvertedIndexTestContext createHashedNGramInvIndexTestContext(LSMInvertedIndexTestHarness harness, InvertedIndexType invIndexType) throws IOException, HyracksDataException {
ISerializerDeserializer[] fieldSerdes = getHashedIndexFieldSerdes(invIndexType);
ITokenFactory tokenFactory = new HashedUTF8NGramTokenFactory();
IBinaryTokenizerFactory tokenizerFactory = new NGramUTF8StringBinaryTokenizerFactory(TEST_GRAM_LENGTH, true, true, false, tokenFactory);
LSMInvertedIndexTestContext testCtx = LSMInvertedIndexTestContext.create(harness, fieldSerdes, fieldSerdes.length - 1, tokenizerFactory, invIndexType, null, null, null, null, null, null);
return testCtx;
}
Aggregations