Examples with ContentPath org.opensearch.index.mapper.ContentPath used on opensource projects

Example 1 with ContentPath

use of org.opensearch.index.mapper.ContentPath in project OpenSearch by opensearch-project.

the class QueryBuilderStoreTests method testStoringQueryBuilders.

public void testStoringQueryBuilders() throws IOException {
    try (Directory directory = newDirectory()) {
        TermQueryBuilder[] queryBuilders = new TermQueryBuilder[randomIntBetween(1, 16)];
        IndexWriterConfig config = new IndexWriterConfig(new WhitespaceAnalyzer());
        config.setMergePolicy(NoMergePolicy.INSTANCE);
        Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build();
        BinaryFieldMapper fieldMapper = PercolatorFieldMapper.Builder.createQueryBuilderFieldBuilder(new Mapper.BuilderContext(settings, new ContentPath(0)));
        Version version = Version.CURRENT;
        try (IndexWriter indexWriter = new IndexWriter(directory, config)) {
            for (int i = 0; i < queryBuilders.length; i++) {
                queryBuilders[i] = new TermQueryBuilder(randomAlphaOfLength(4), randomAlphaOfLength(8));
                ParseContext parseContext = mock(ParseContext.class);
                ParseContext.Document document = new ParseContext.Document();
                when(parseContext.doc()).thenReturn(document);
                PercolatorFieldMapper.createQueryBuilderField(version, fieldMapper, queryBuilders[i], parseContext);
                indexWriter.addDocument(document);
            }
        }
        QueryShardContext queryShardContext = mock(QueryShardContext.class);
        when(queryShardContext.indexVersionCreated()).thenReturn(version);
        when(queryShardContext.getWriteableRegistry()).thenReturn(writableRegistry());
        when(queryShardContext.getXContentRegistry()).thenReturn(xContentRegistry());
        when(queryShardContext.getForField(fieldMapper.fieldType())).thenReturn(new BytesBinaryIndexFieldData(fieldMapper.name(), CoreValuesSourceType.BYTES));
        when(queryShardContext.fieldMapper(Mockito.anyString())).thenAnswer(invocation -> {
            final String fieldName = (String) invocation.getArguments()[0];
            return new KeywordFieldMapper.KeywordFieldType(fieldName);
        });
        PercolateQuery.QueryStore queryStore = PercolateQueryBuilder.createStore(fieldMapper.fieldType(), queryShardContext);
        try (IndexReader indexReader = DirectoryReader.open(directory)) {
            LeafReaderContext leafContext = indexReader.leaves().get(0);
            CheckedFunction<Integer, Query, IOException> queries = queryStore.getQueries(leafContext);
            assertEquals(queryBuilders.length, leafContext.reader().numDocs());
            for (int i = 0; i < queryBuilders.length; i++) {
                TermQuery query = (TermQuery) queries.apply(i);
                assertEquals(queryBuilders[i].fieldName(), query.getTerm().field());
                assertEquals(queryBuilders[i].value(), query.getTerm().text());
            }
        }
    }
}

Example 2 with ContentPath

use of org.opensearch.index.mapper.ContentPath in project OpenSearch by opensearch-project.

the class InternalEngineTests method dynamicUpdate.

private Mapping dynamicUpdate() {
    BuilderContext context = new BuilderContext(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build(), new ContentPath());
    final RootObjectMapper root = new RootObjectMapper.Builder("some_type").build(context);
    return new Mapping(Version.CURRENT, root, new MetadataFieldMapper[0], emptyMap());
}

Example 3 with ContentPath

use of org.opensearch.index.mapper.ContentPath in project OpenSearch by opensearch-project.

the class AbstractFieldDataTestCase method getForField.

public <IFD extends IndexFieldData<?>> IFD getForField(String type, String fieldName, boolean docValues) {
    final MappedFieldType fieldType;
    final BuilderContext context = new BuilderContext(indexService.getIndexSettings().getSettings(), new ContentPath(1));
    if (type.equals("string")) {
        if (docValues) {
            fieldType = new KeywordFieldMapper.Builder(fieldName).build(context).fieldType();
        } else {
            fieldType = new TextFieldMapper.Builder(fieldName, createDefaultIndexAnalyzers()).fielddata(true).build(context).fieldType();
        }
    } else if (type.equals("float")) {
        fieldType = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.FLOAT, false, true).docValues(docValues).build(context).fieldType();
    } else if (type.equals("double")) {
        fieldType = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.DOUBLE, false, true).docValues(docValues).build(context).fieldType();
    } else if (type.equals("long")) {
        fieldType = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.LONG, false, true).docValues(docValues).build(context).fieldType();
    } else if (type.equals("int")) {
        fieldType = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.INTEGER, false, true).docValues(docValues).build(context).fieldType();
    } else if (type.equals("short")) {
        fieldType = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.SHORT, false, true).docValues(docValues).build(context).fieldType();
    } else if (type.equals("byte")) {
        fieldType = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.BYTE, false, true).docValues(docValues).build(context).fieldType();
    } else if (type.equals("geo_point")) {
        fieldType = new GeoPointFieldMapper.Builder(fieldName).docValues(docValues).build(context).fieldType();
    } else if (type.equals("binary")) {
        fieldType = new BinaryFieldMapper.Builder(fieldName, docValues).build(context).fieldType();
    } else {
        throw new UnsupportedOperationException(type);
    }
    return shardContext.getForField(fieldType);
}

Example 4 with ContentPath

use of org.opensearch.index.mapper.ContentPath in project OpenSearch by opensearch-project.

the class FilterFieldDataTests method testFilterByFrequency.

public void testFilterByFrequency() throws Exception {
    Random random = random();
    for (int i = 0; i < 1000; i++) {
        Document d = new Document();
        d.add(new StringField("id", "" + i, Field.Store.NO));
        if (i % 100 == 0) {
            d.add(new StringField("high_freq", "100", Field.Store.NO));
            d.add(new StringField("low_freq", "100", Field.Store.NO));
            d.add(new StringField("med_freq", "100", Field.Store.NO));
        }
        if (i % 10 == 0) {
            d.add(new StringField("high_freq", "10", Field.Store.NO));
            d.add(new StringField("med_freq", "10", Field.Store.NO));
        }
        if (i % 5 == 0) {
            d.add(new StringField("high_freq", "5", Field.Store.NO));
        }
        writer.addDocument(d);
    }
    writer.forceMerge(1, true);
    List<LeafReaderContext> contexts = refreshReader();
    final BuilderContext builderCtx = new BuilderContext(indexService.getIndexSettings().getSettings(), new ContentPath(1));
    {
        indexService.clearCaches(false, true);
        MappedFieldType ft = new TextFieldMapper.Builder("high_freq", createDefaultIndexAnalyzers()).fielddata(true).fielddataFrequencyFilter(0, random.nextBoolean() ? 100 : 0.5d, 0).build(builderCtx).fieldType();
        IndexOrdinalsFieldData fieldData = shardContext.getForField(ft);
        for (LeafReaderContext context : contexts) {
            LeafOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
            SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues();
            assertThat(2L, equalTo(bytesValues.getValueCount()));
            assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("10"));
            assertThat(bytesValues.lookupOrd(1).utf8ToString(), equalTo("100"));
        }
    }
    {
        indexService.clearCaches(false, true);
        MappedFieldType ft = new TextFieldMapper.Builder("high_freq", createDefaultIndexAnalyzers()).fielddata(true).fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d / 200.0d, 201, 100).build(builderCtx).fieldType();
        IndexOrdinalsFieldData fieldData = shardContext.getForField(ft);
        for (LeafReaderContext context : contexts) {
            LeafOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
            SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues();
            assertThat(1L, equalTo(bytesValues.getValueCount()));
            assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("5"));
        }
    }
    {
        // test # docs with value
        indexService.clearCaches(false, true);
        MappedFieldType ft = new TextFieldMapper.Builder("med_freq", createDefaultIndexAnalyzers()).fielddata(true).fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d / 200.0d, Integer.MAX_VALUE, 101).build(builderCtx).fieldType();
        IndexOrdinalsFieldData fieldData = shardContext.getForField(ft);
        for (LeafReaderContext context : contexts) {
            LeafOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
            SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues();
            assertThat(2L, equalTo(bytesValues.getValueCount()));
            assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("10"));
            assertThat(bytesValues.lookupOrd(1).utf8ToString(), equalTo("100"));
        }
    }
    {
        indexService.clearCaches(false, true);
        MappedFieldType ft = new TextFieldMapper.Builder("med_freq", createDefaultIndexAnalyzers()).fielddata(true).fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d / 200.0d, Integer.MAX_VALUE, 101).build(builderCtx).fieldType();
        IndexOrdinalsFieldData fieldData = shardContext.getForField(ft);
        for (LeafReaderContext context : contexts) {
            LeafOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
            SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues();
            assertThat(2L, equalTo(bytesValues.getValueCount()));
            assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("10"));
            assertThat(bytesValues.lookupOrd(1).utf8ToString(), equalTo("100"));
        }
    }
}

Example 5 with ContentPath

use of org.opensearch.index.mapper.ContentPath in project OpenSearch by opensearch-project.

the class IndexFieldDataServiceTests method testClearField.

public void testClearField() throws Exception {
    final IndexService indexService = createIndex("test");
    final IndicesService indicesService = getInstanceFromNode(IndicesService.class);
    // copy the ifdService since we can set the listener only once.
    final IndexFieldDataService ifdService = new IndexFieldDataService(indexService.getIndexSettings(), indicesService.getIndicesFieldDataCache(), indicesService.getCircuitBreakerService(), indexService.mapperService());
    final BuilderContext ctx = new BuilderContext(indexService.getIndexSettings().getSettings(), new ContentPath(1));
    final MappedFieldType mapper1 = new TextFieldMapper.Builder("field_1", createDefaultIndexAnalyzers()).fielddata(true).build(ctx).fieldType();
    final MappedFieldType mapper2 = new TextFieldMapper.Builder("field_2", createDefaultIndexAnalyzers()).fielddata(true).build(ctx).fieldType();
    final IndexWriter writer = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new KeywordAnalyzer()));
    Document doc = new Document();
    doc.add(new StringField("field_1", "thisisastring", Store.NO));
    doc.add(new StringField("field_2", "thisisanotherstring", Store.NO));
    writer.addDocument(doc);
    final IndexReader reader = DirectoryReader.open(writer);
    final AtomicInteger onCacheCalled = new AtomicInteger();
    final AtomicInteger onRemovalCalled = new AtomicInteger();
    ifdService.setListener(new IndexFieldDataCache.Listener() {

        @Override
        public void onCache(ShardId shardId, String fieldName, Accountable ramUsage) {
            onCacheCalled.incrementAndGet();
        }

        @Override
        public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, long sizeInBytes) {
            onRemovalCalled.incrementAndGet();
        }
    });
    IndexFieldData<?> ifd1 = ifdService.getForField(mapper1, "test", () -> {
        throw new UnsupportedOperationException();
    });
    IndexFieldData<?> ifd2 = ifdService.getForField(mapper2, "test", () -> {
        throw new UnsupportedOperationException();
    });
    LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0);
    LeafFieldData loadField1 = ifd1.load(leafReaderContext);
    LeafFieldData loadField2 = ifd2.load(leafReaderContext);
    assertEquals(2, onCacheCalled.get());
    assertEquals(0, onRemovalCalled.get());
    ifdService.clearField("field_1");
    assertEquals(2, onCacheCalled.get());
    assertEquals(1, onRemovalCalled.get());
    ifdService.clearField("field_1");
    assertEquals(2, onCacheCalled.get());
    assertEquals(1, onRemovalCalled.get());
    ifdService.clearField("field_2");
    assertEquals(2, onCacheCalled.get());
    assertEquals(2, onRemovalCalled.get());
    reader.close();
    loadField1.close();
    loadField2.close();
    writer.close();
    ifdService.clear();
}