Examples with MockTokenizer org.apache.lucene.analysis.MockTokenizer used on opensource projects

Example 36 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestDaitchMokotoffSoundexFilterFactory method testSettingInject.

public void testSettingInject() throws Exception {
    Map<String, String> parameters = new HashMap<>();
    parameters.put("inject", "false");
    DaitchMokotoffSoundexFilterFactory factory = new DaitchMokotoffSoundexFilterFactory(parameters);
    Tokenizer inputStream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
    inputStream.setReader(new StringReader("international"));
    TokenStream filteredStream = factory.create(inputStream);
    assertEquals(DaitchMokotoffSoundexFilter.class, filteredStream.getClass());
    assertTokenStreamContents(filteredStream, new String[] { "063963" });
}

Example 37 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestPhoneticFilter method assertAlgorithm.

static void assertAlgorithm(Encoder encoder, boolean inject, String input, String[] expected) throws Exception {
    Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
    tokenizer.setReader(new StringReader(input));
    PhoneticFilter filter = new PhoneticFilter(tokenizer, encoder, inject);
    assertTokenStreamContents(filter, expected);
}

Example 38 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestPhoneticFilter method testRandomStrings.

/** blast some random strings through the analyzer */
public void testRandomStrings() throws IOException {
    Encoder[] encoders = new Encoder[] { new Metaphone(), new DoubleMetaphone(), new Soundex(), new RefinedSoundex(), new Caverphone2() };
    for (final Encoder e : encoders) {
        Analyzer a = new Analyzer() {

            @Override
            protected TokenStreamComponents createComponents(String fieldName) {
                Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
                return new TokenStreamComponents(tokenizer, new PhoneticFilter(tokenizer, e, false));
            }
        };
        checkRandomData(random(), a, 1000 * RANDOM_MULTIPLIER);
        a.close();
        Analyzer b = new Analyzer() {

            @Override
            protected TokenStreamComponents createComponents(String fieldName) {
                Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
                return new TokenStreamComponents(tokenizer, new PhoneticFilter(tokenizer, e, false));
            }
        };
        checkRandomData(random(), b, 1000 * RANDOM_MULTIPLIER);
        b.close();
    }
}

Example 39 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestBeiderMorseFilter method testCustomAttribute.

public void testCustomAttribute() throws IOException {
    TokenStream stream = new MockTokenizer(MockTokenizer.KEYWORD, false);
    ((Tokenizer) stream).setReader(new StringReader("D'Angelo"));
    stream = new PatternKeywordMarkerFilter(stream, Pattern.compile(".*"));
    stream = new BeiderMorseFilter(stream, new PhoneticEngine(NameType.GENERIC, RuleType.EXACT, true));
    KeywordAttribute keyAtt = stream.addAttribute(KeywordAttribute.class);
    stream.reset();
    int i = 0;
    while (stream.incrementToken()) {
        assertTrue(keyAtt.isKeyword());
        i++;
    }
    assertEquals(12, i);
    stream.end();
    stream.close();
}

Example 40 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestIndexSorting method testRandom2.

public void testRandom2() throws Exception {
    int numDocs = atLeast(100);
    FieldType POSITIONS_TYPE = new FieldType(TextField.TYPE_NOT_STORED);
    POSITIONS_TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
    POSITIONS_TYPE.freeze();
    FieldType TERM_VECTORS_TYPE = new FieldType(TextField.TYPE_NOT_STORED);
    TERM_VECTORS_TYPE.setStoreTermVectors(true);
    TERM_VECTORS_TYPE.freeze();
    Analyzer a = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new MockTokenizer();
            return new TokenStreamComponents(tokenizer, tokenizer);
        }
    };
    List<Document> docs = new ArrayList<>();
    for (int i = 0; i < numDocs; i++) {
        int id = i * 10;
        Document doc = new Document();
        doc.add(new StringField("id", Integer.toString(id), Store.YES));
        doc.add(new StringField("docs", "#all#", Store.NO));
        PositionsTokenStream positions = new PositionsTokenStream();
        positions.setId(id);
        doc.add(new Field("positions", positions, POSITIONS_TYPE));
        doc.add(new NumericDocValuesField("numeric", id));
        String value = IntStream.range(0, id).mapToObj(k -> Integer.toString(id)).collect(Collectors.joining(" "));
        TextField norms = new TextField("norms", value, Store.NO);
        doc.add(norms);
        doc.add(new BinaryDocValuesField("binary", new BytesRef(Integer.toString(id))));
        doc.add(new SortedDocValuesField("sorted", new BytesRef(Integer.toString(id))));
        doc.add(new SortedSetDocValuesField("multi_valued_string", new BytesRef(Integer.toString(id))));
        doc.add(new SortedSetDocValuesField("multi_valued_string", new BytesRef(Integer.toString(id + 1))));
        doc.add(new SortedNumericDocValuesField("multi_valued_numeric", id));
        doc.add(new SortedNumericDocValuesField("multi_valued_numeric", id + 1));
        doc.add(new Field("term_vectors", Integer.toString(id), TERM_VECTORS_TYPE));
        byte[] bytes = new byte[4];
        NumericUtils.intToSortableBytes(id, bytes, 0);
        doc.add(new BinaryPoint("points", bytes));
        docs.add(doc);
    }
    // Must use the same seed for both RandomIndexWriters so they behave identically
    long seed = random().nextLong();
    // We add document alread in ID order for the first writer:
    Directory dir1 = newFSDirectory(createTempDir());
    Random random1 = new Random(seed);
    IndexWriterConfig iwc1 = newIndexWriterConfig(random1, a);
    // for testing norms field
    iwc1.setSimilarity(new NormsSimilarity(iwc1.getSimilarity()));
    // preserve docIDs
    iwc1.setMergePolicy(newLogMergePolicy());
    if (VERBOSE) {
        System.out.println("TEST: now index pre-sorted");
    }
    RandomIndexWriter w1 = new RandomIndexWriter(random1, dir1, iwc1);
    for (Document doc : docs) {
        ((PositionsTokenStream) ((Field) doc.getField("positions")).tokenStreamValue()).setId(Integer.parseInt(doc.get("id")));
        w1.addDocument(doc);
    }
    // We shuffle documents, but set index sort, for the second writer:
    Directory dir2 = newFSDirectory(createTempDir());
    Random random2 = new Random(seed);
    IndexWriterConfig iwc2 = newIndexWriterConfig(random2, a);
    // for testing norms field
    iwc2.setSimilarity(new NormsSimilarity(iwc2.getSimilarity()));
    Sort sort = new Sort(new SortField("numeric", SortField.Type.INT));
    iwc2.setIndexSort(sort);
    Collections.shuffle(docs, random());
    if (VERBOSE) {
        System.out.println("TEST: now index with index-time sorting");
    }
    RandomIndexWriter w2 = new RandomIndexWriter(random2, dir2, iwc2);
    int count = 0;
    int commitAtCount = TestUtil.nextInt(random(), 1, numDocs - 1);
    for (Document doc : docs) {
        ((PositionsTokenStream) ((Field) doc.getField("positions")).tokenStreamValue()).setId(Integer.parseInt(doc.get("id")));
        if (count++ == commitAtCount) {
            // Ensure forceMerge really does merge
            w2.commit();
        }
        w2.addDocument(doc);
    }
    if (VERBOSE) {
        System.out.println("TEST: now force merge");
    }
    w2.forceMerge(1);
    DirectoryReader r1 = w1.getReader();
    DirectoryReader r2 = w2.getReader();
    if (VERBOSE) {
        System.out.println("TEST: now compare r1=" + r1 + " r2=" + r2);
    }
    assertEquals(sort, getOnlyLeafReader(r2).getMetaData().getSort());
    assertReaderEquals("left: sorted by hand; right: sorted by Lucene", r1, r2);
    IOUtils.close(w1, w2, r1, r2, dir1, dir2);
}