Examples with StandardAnalyzer org.apache.lucene.analysis.standard.StandardAnalyzer used on opensource projects

Example 6 with StandardAnalyzer

use of org.apache.lucene.analysis.standard.StandardAnalyzer in project elasticsearch by elastic.

the class ParentFieldMapperTests method testNoParentNullFieldCreatedIfNoParentSpecified.

public void testNoParentNullFieldCreatedIfNoParentSpecified() throws Exception {
    Index index = new Index("_index", "testUUID");
    IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, Settings.EMPTY);
    NamedAnalyzer namedAnalyzer = new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer());
    IndexAnalyzers indexAnalyzers = new IndexAnalyzers(indexSettings, namedAnalyzer, namedAnalyzer, namedAnalyzer, Collections.emptyMap(), Collections.emptyMap());
    SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap());
    MapperService mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry(), similarityService, new IndicesModule(emptyList()).getMapperRegistry(), () -> null);
    XContentBuilder mappingSource = jsonBuilder().startObject().startObject("some_type").startObject("properties").endObject().endObject().endObject();
    mapperService.merge("some_type", new CompressedXContent(mappingSource.string()), MergeReason.MAPPING_UPDATE, false);
    Set<String> allFields = new HashSet<>(mapperService.simpleMatchToIndexNames("*"));
    assertTrue(allFields.contains("_parent"));
    assertFalse(allFields.contains("_parent#null"));
}

Example 7 with StandardAnalyzer

use of org.apache.lucene.analysis.standard.StandardAnalyzer in project elasticsearch by elastic.

the class SimpleQueryParserTests method testAnalyzeWildcard.

public void testAnalyzeWildcard() {
    SimpleQueryParser.Settings settings = new SimpleQueryParser.Settings();
    settings.analyzeWildcard(true);
    Map<String, Float> weights = new HashMap<>();
    weights.put("field1", 1.0f);
    SimpleQueryParser parser = new MockSimpleQueryParser(new StandardAnalyzer(), weights, -1, settings);
    for (Operator op : Operator.values()) {
        BooleanClause.Occur defaultOp = op.toBooleanClauseOccur();
        parser.setDefaultOperator(defaultOp);
        Query query = parser.parse("first foo-bar-foobar* last");
        Query expectedQuery = new BooleanQuery.Builder().add(new BooleanClause(new TermQuery(new Term("field1", "first")), defaultOp)).add(new BooleanQuery.Builder().add(new BooleanClause(new TermQuery(new Term("field1", "foo")), defaultOp)).add(new BooleanClause(new TermQuery(new Term("field1", "bar")), defaultOp)).add(new BooleanClause(new PrefixQuery(new Term("field1", "foobar")), defaultOp)).build(), defaultOp).add(new BooleanClause(new TermQuery(new Term("field1", "last")), defaultOp)).build();
        assertThat(query, equalTo(expectedQuery));
    }
}

Example 8 with StandardAnalyzer

use of org.apache.lucene.analysis.standard.StandardAnalyzer in project elasticsearch by elastic.

the class SimpleQueryParserTests method testQuoteFieldSuffix.

public void testQuoteFieldSuffix() {
    SimpleQueryParser.Settings sqpSettings = new SimpleQueryParser.Settings();
    sqpSettings.quoteFieldSuffix(".quote");
    Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetaData.SETTING_INDEX_UUID, "some_uuid").put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
    IndexMetaData indexState = IndexMetaData.builder("index").settings(indexSettings).build();
    IndexSettings settings = new IndexSettings(indexState, Settings.EMPTY);
    QueryShardContext mockShardContext = new QueryShardContext(0, settings, null, null, null, null, null, xContentRegistry(), null, null, System::currentTimeMillis) {

        @Override
        public MappedFieldType fieldMapper(String name) {
            return new MockFieldMapper.FakeFieldType();
        }
    };
    SimpleQueryParser parser = new SimpleQueryParser(new StandardAnalyzer(), Collections.singletonMap("foo", 1f), -1, sqpSettings, mockShardContext);
    assertEquals(new TermQuery(new Term("foo", "bar")), parser.parse("bar"));
    assertEquals(new TermQuery(new Term("foo.quote", "bar")), parser.parse("\"bar\""));
    // Now check what happens if foo.quote does not exist
    mockShardContext = new QueryShardContext(0, settings, null, null, null, null, null, xContentRegistry(), null, null, System::currentTimeMillis) {

        @Override
        public MappedFieldType fieldMapper(String name) {
            if (name.equals("foo.quote")) {
                return null;
            }
            return new MockFieldMapper.FakeFieldType();
        }
    };
    parser = new SimpleQueryParser(new StandardAnalyzer(), Collections.singletonMap("foo", 1f), -1, sqpSettings, mockShardContext);
    assertEquals(new TermQuery(new Term("foo", "bar")), parser.parse("bar"));
    assertEquals(new TermQuery(new Term("foo", "bar")), parser.parse("\"bar\""));
}

Example 9 with StandardAnalyzer

use of org.apache.lucene.analysis.standard.StandardAnalyzer in project elasticsearch by elastic.

the class AbstractTermVectorsTestCase method indexDocsWithLucene.

protected DirectoryReader indexDocsWithLucene(TestDoc[] testDocs) throws IOException {
    Map<String, Analyzer> mapping = new HashMap<>();
    for (TestFieldSetting field : testDocs[0].fieldSettings) {
        if (field.storedPayloads) {
            mapping.put(field.name, new Analyzer() {

                @Override
                protected TokenStreamComponents createComponents(String fieldName) {
                    Tokenizer tokenizer = new StandardTokenizer();
                    TokenFilter filter = new LowerCaseFilter(tokenizer);
                    filter = new TypeAsPayloadTokenFilter(filter);
                    return new TokenStreamComponents(tokenizer, filter);
                }
            });
        }
    }
    PerFieldAnalyzerWrapper wrapper = new PerFieldAnalyzerWrapper(new StandardAnalyzer(CharArraySet.EMPTY_SET), mapping);
    Directory dir = new RAMDirectory();
    IndexWriterConfig conf = new IndexWriterConfig(wrapper);
    conf.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
    IndexWriter writer = new IndexWriter(dir, conf);
    for (TestDoc doc : testDocs) {
        Document d = new Document();
        d.add(new Field("id", doc.id, StringField.TYPE_STORED));
        for (int i = 0; i < doc.fieldContent.length; i++) {
            FieldType type = new FieldType(TextField.TYPE_STORED);
            TestFieldSetting fieldSetting = doc.fieldSettings[i];
            type.setStoreTermVectorOffsets(fieldSetting.storedOffset);
            type.setStoreTermVectorPayloads(fieldSetting.storedPayloads);
            type.setStoreTermVectorPositions(fieldSetting.storedPositions || fieldSetting.storedPayloads || fieldSetting.storedOffset);
            type.setStoreTermVectors(true);
            type.freeze();
            d.add(new Field(fieldSetting.name, doc.fieldContent[i], type));
        }
        writer.updateDocument(new Term("id", doc.id), d);
        writer.commit();
    }
    writer.close();
    return DirectoryReader.open(dir);
}

Example 10 with StandardAnalyzer

use of org.apache.lucene.analysis.standard.StandardAnalyzer in project elasticsearch by elastic.

the class TermVectorsUnitTests method writeEmptyTermVector.

private void writeEmptyTermVector(TermVectorsResponse outResponse) throws IOException {
    Directory dir = newDirectory();
    IndexWriterConfig conf = new IndexWriterConfig(new StandardAnalyzer());
    conf.setOpenMode(OpenMode.CREATE);
    IndexWriter writer = new IndexWriter(dir, conf);
    FieldType type = new FieldType(TextField.TYPE_STORED);
    type.setStoreTermVectorOffsets(true);
    type.setStoreTermVectorPayloads(false);
    type.setStoreTermVectorPositions(true);
    type.setStoreTermVectors(true);
    type.freeze();
    Document d = new Document();
    d.add(new Field("id", "abc", StringField.TYPE_STORED));
    writer.updateDocument(new Term("id", "abc"), d);
    writer.commit();
    writer.close();
    DirectoryReader dr = DirectoryReader.open(dir);
    IndexSearcher s = new IndexSearcher(dr);
    TopDocs search = s.search(new TermQuery(new Term("id", "abc")), 1);
    ScoreDoc[] scoreDocs = search.scoreDocs;
    int doc = scoreDocs[0].doc;
    Fields fields = dr.getTermVectors(doc);
    EnumSet<Flag> flags = EnumSet.of(Flag.Positions, Flag.Offsets);
    outResponse.setFields(fields, null, flags, fields);
    outResponse.setExists(true);
    dr.close();
    dir.close();
}