Examples with BuilderContext org.opensearch.index.mapper.Mapper.BuilderContext used on opensource projects

Example 1 with BuilderContext

use of org.opensearch.index.mapper.Mapper.BuilderContext in project OpenSearch by opensearch-project.

the class InternalEngineTests method dynamicUpdate.

private Mapping dynamicUpdate() {
    BuilderContext context = new BuilderContext(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build(), new ContentPath());
    final RootObjectMapper root = new RootObjectMapper.Builder("some_type").build(context);
    return new Mapping(Version.CURRENT, root, new MetadataFieldMapper[0], emptyMap());
}

Example 2 with BuilderContext

use of org.opensearch.index.mapper.Mapper.BuilderContext in project OpenSearch by opensearch-project.

the class AbstractFieldDataTestCase method getForField.

public <IFD extends IndexFieldData<?>> IFD getForField(String type, String fieldName, boolean docValues) {
    final MappedFieldType fieldType;
    final BuilderContext context = new BuilderContext(indexService.getIndexSettings().getSettings(), new ContentPath(1));
    if (type.equals("string")) {
        if (docValues) {
            fieldType = new KeywordFieldMapper.Builder(fieldName).build(context).fieldType();
        } else {
            fieldType = new TextFieldMapper.Builder(fieldName, createDefaultIndexAnalyzers()).fielddata(true).build(context).fieldType();
        }
    } else if (type.equals("float")) {
        fieldType = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.FLOAT, false, true).docValues(docValues).build(context).fieldType();
    } else if (type.equals("double")) {
        fieldType = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.DOUBLE, false, true).docValues(docValues).build(context).fieldType();
    } else if (type.equals("long")) {
        fieldType = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.LONG, false, true).docValues(docValues).build(context).fieldType();
    } else if (type.equals("int")) {
        fieldType = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.INTEGER, false, true).docValues(docValues).build(context).fieldType();
    } else if (type.equals("short")) {
        fieldType = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.SHORT, false, true).docValues(docValues).build(context).fieldType();
    } else if (type.equals("byte")) {
        fieldType = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.BYTE, false, true).docValues(docValues).build(context).fieldType();
    } else if (type.equals("geo_point")) {
        fieldType = new GeoPointFieldMapper.Builder(fieldName).docValues(docValues).build(context).fieldType();
    } else if (type.equals("binary")) {
        fieldType = new BinaryFieldMapper.Builder(fieldName, docValues).build(context).fieldType();
    } else {
        throw new UnsupportedOperationException(type);
    }
    return shardContext.getForField(fieldType);
}

Example 3 with BuilderContext

use of org.opensearch.index.mapper.Mapper.BuilderContext in project OpenSearch by opensearch-project.

the class FilterFieldDataTests method testFilterByFrequency.

public void testFilterByFrequency() throws Exception {
    Random random = random();
    for (int i = 0; i < 1000; i++) {
        Document d = new Document();
        d.add(new StringField("id", "" + i, Field.Store.NO));
        if (i % 100 == 0) {
            d.add(new StringField("high_freq", "100", Field.Store.NO));
            d.add(new StringField("low_freq", "100", Field.Store.NO));
            d.add(new StringField("med_freq", "100", Field.Store.NO));
        }
        if (i % 10 == 0) {
            d.add(new StringField("high_freq", "10", Field.Store.NO));
            d.add(new StringField("med_freq", "10", Field.Store.NO));
        }
        if (i % 5 == 0) {
            d.add(new StringField("high_freq", "5", Field.Store.NO));
        }
        writer.addDocument(d);
    }
    writer.forceMerge(1, true);
    List<LeafReaderContext> contexts = refreshReader();
    final BuilderContext builderCtx = new BuilderContext(indexService.getIndexSettings().getSettings(), new ContentPath(1));
    {
        indexService.clearCaches(false, true);
        MappedFieldType ft = new TextFieldMapper.Builder("high_freq", createDefaultIndexAnalyzers()).fielddata(true).fielddataFrequencyFilter(0, random.nextBoolean() ? 100 : 0.5d, 0).build(builderCtx).fieldType();
        IndexOrdinalsFieldData fieldData = shardContext.getForField(ft);
        for (LeafReaderContext context : contexts) {
            LeafOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
            SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues();
            assertThat(2L, equalTo(bytesValues.getValueCount()));
            assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("10"));
            assertThat(bytesValues.lookupOrd(1).utf8ToString(), equalTo("100"));
        }
    }
    {
        indexService.clearCaches(false, true);
        MappedFieldType ft = new TextFieldMapper.Builder("high_freq", createDefaultIndexAnalyzers()).fielddata(true).fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d / 200.0d, 201, 100).build(builderCtx).fieldType();
        IndexOrdinalsFieldData fieldData = shardContext.getForField(ft);
        for (LeafReaderContext context : contexts) {
            LeafOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
            SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues();
            assertThat(1L, equalTo(bytesValues.getValueCount()));
            assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("5"));
        }
    }
    {
        // test # docs with value
        indexService.clearCaches(false, true);
        MappedFieldType ft = new TextFieldMapper.Builder("med_freq", createDefaultIndexAnalyzers()).fielddata(true).fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d / 200.0d, Integer.MAX_VALUE, 101).build(builderCtx).fieldType();
        IndexOrdinalsFieldData fieldData = shardContext.getForField(ft);
        for (LeafReaderContext context : contexts) {
            LeafOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
            SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues();
            assertThat(2L, equalTo(bytesValues.getValueCount()));
            assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("10"));
            assertThat(bytesValues.lookupOrd(1).utf8ToString(), equalTo("100"));
        }
    }
    {
        indexService.clearCaches(false, true);
        MappedFieldType ft = new TextFieldMapper.Builder("med_freq", createDefaultIndexAnalyzers()).fielddata(true).fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d / 200.0d, Integer.MAX_VALUE, 101).build(builderCtx).fieldType();
        IndexOrdinalsFieldData fieldData = shardContext.getForField(ft);
        for (LeafReaderContext context : contexts) {
            LeafOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
            SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues();
            assertThat(2L, equalTo(bytesValues.getValueCount()));
            assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("10"));
            assertThat(bytesValues.lookupOrd(1).utf8ToString(), equalTo("100"));
        }
    }
}

Example 4 with BuilderContext

use of org.opensearch.index.mapper.Mapper.BuilderContext in project OpenSearch by opensearch-project.

the class IndexFieldDataServiceTests method testClearField.

public void testClearField() throws Exception {
    final IndexService indexService = createIndex("test");
    final IndicesService indicesService = getInstanceFromNode(IndicesService.class);
    // copy the ifdService since we can set the listener only once.
    final IndexFieldDataService ifdService = new IndexFieldDataService(indexService.getIndexSettings(), indicesService.getIndicesFieldDataCache(), indicesService.getCircuitBreakerService(), indexService.mapperService());
    final BuilderContext ctx = new BuilderContext(indexService.getIndexSettings().getSettings(), new ContentPath(1));
    final MappedFieldType mapper1 = new TextFieldMapper.Builder("field_1", createDefaultIndexAnalyzers()).fielddata(true).build(ctx).fieldType();
    final MappedFieldType mapper2 = new TextFieldMapper.Builder("field_2", createDefaultIndexAnalyzers()).fielddata(true).build(ctx).fieldType();
    final IndexWriter writer = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new KeywordAnalyzer()));
    Document doc = new Document();
    doc.add(new StringField("field_1", "thisisastring", Store.NO));
    doc.add(new StringField("field_2", "thisisanotherstring", Store.NO));
    writer.addDocument(doc);
    final IndexReader reader = DirectoryReader.open(writer);
    final AtomicInteger onCacheCalled = new AtomicInteger();
    final AtomicInteger onRemovalCalled = new AtomicInteger();
    ifdService.setListener(new IndexFieldDataCache.Listener() {

        @Override
        public void onCache(ShardId shardId, String fieldName, Accountable ramUsage) {
            onCacheCalled.incrementAndGet();
        }

        @Override
        public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, long sizeInBytes) {
            onRemovalCalled.incrementAndGet();
        }
    });
    IndexFieldData<?> ifd1 = ifdService.getForField(mapper1, "test", () -> {
        throw new UnsupportedOperationException();
    });
    IndexFieldData<?> ifd2 = ifdService.getForField(mapper2, "test", () -> {
        throw new UnsupportedOperationException();
    });
    LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0);
    LeafFieldData loadField1 = ifd1.load(leafReaderContext);
    LeafFieldData loadField2 = ifd2.load(leafReaderContext);
    assertEquals(2, onCacheCalled.get());
    assertEquals(0, onRemovalCalled.get());
    ifdService.clearField("field_1");
    assertEquals(2, onCacheCalled.get());
    assertEquals(1, onRemovalCalled.get());
    ifdService.clearField("field_1");
    assertEquals(2, onCacheCalled.get());
    assertEquals(1, onRemovalCalled.get());
    ifdService.clearField("field_2");
    assertEquals(2, onCacheCalled.get());
    assertEquals(2, onRemovalCalled.get());
    reader.close();
    loadField1.close();
    loadField2.close();
    writer.close();
    ifdService.clear();
}

Example 5 with BuilderContext

use of org.opensearch.index.mapper.Mapper.BuilderContext in project OpenSearch by opensearch-project.

the class IndexFieldDataServiceTests method testGetForFieldDefaults.

public void testGetForFieldDefaults() {
    final IndexService indexService = createIndex("test");
    final IndicesService indicesService = getInstanceFromNode(IndicesService.class);
    final IndexFieldDataService ifdService = new IndexFieldDataService(indexService.getIndexSettings(), indicesService.getIndicesFieldDataCache(), indicesService.getCircuitBreakerService(), indexService.mapperService());
    final BuilderContext ctx = new BuilderContext(indexService.getIndexSettings().getSettings(), new ContentPath(1));
    final MappedFieldType stringMapper = new KeywordFieldMapper.Builder("string").build(ctx).fieldType();
    ifdService.clear();
    IndexFieldData<?> fd = ifdService.getForField(stringMapper, "test", () -> {
        throw new UnsupportedOperationException();
    });
    assertTrue(fd instanceof SortedSetOrdinalsIndexFieldData);
    for (MappedFieldType mapper : Arrays.asList(new NumberFieldMapper.Builder("int", NumberFieldMapper.NumberType.BYTE, false, true).build(ctx).fieldType(), new NumberFieldMapper.Builder("int", NumberFieldMapper.NumberType.SHORT, false, true).build(ctx).fieldType(), new NumberFieldMapper.Builder("int", NumberFieldMapper.NumberType.INTEGER, false, true).build(ctx).fieldType(), new NumberFieldMapper.Builder("long", NumberFieldMapper.NumberType.LONG, false, true).build(ctx).fieldType())) {
        ifdService.clear();
        fd = ifdService.getForField(mapper, "test", () -> {
            throw new UnsupportedOperationException();
        });
        assertTrue(fd instanceof SortedNumericIndexFieldData);
    }
    final MappedFieldType floatMapper = new NumberFieldMapper.Builder("float", NumberFieldMapper.NumberType.FLOAT, false, true).build(ctx).fieldType();
    ifdService.clear();
    fd = ifdService.getForField(floatMapper, "test", () -> {
        throw new UnsupportedOperationException();
    });
    assertTrue(fd instanceof SortedNumericIndexFieldData);
    final MappedFieldType doubleMapper = new NumberFieldMapper.Builder("double", NumberFieldMapper.NumberType.DOUBLE, false, true).build(ctx).fieldType();
    ifdService.clear();
    fd = ifdService.getForField(doubleMapper, "test", () -> {
        throw new UnsupportedOperationException();
    });
    assertTrue(fd instanceof SortedNumericIndexFieldData);
}