Examples with MapperService org.elasticsearch.index.mapper.MapperService used on opensource projects

Example 36 with MapperService

use of org.elasticsearch.index.mapper.MapperService in project crate by crate.

the class LuceneQueryBuilderTest method createMapperService.

private MapperService createMapperService(Index index, Settings indexSettings, AnalysisService analysisService) {
    DynamicArrayFieldMapperBuilderFactoryProvider arrayMapperProvider = new DynamicArrayFieldMapperBuilderFactoryProvider();
    arrayMapperProvider.dynamicArrayFieldMapperBuilderFactory = new ArrayMapper.BuilderFactory();
    IndicesModule indicesModule = new IndicesModule();
    indicesModule.registerMapper(ArrayMapper.CONTENT_TYPE, new ArrayMapper.TypeParser());
    return new MapperService(index, indexSettings, analysisService, new SimilarityLookupService(index, indexSettings), null, indicesModule.getMapperRegistry(), arrayMapperProvider);
}

Example 37 with MapperService

use of org.elasticsearch.index.mapper.MapperService in project elasticsearch-skywalker by jprante.

the class TransportSkywalkerAction method shardOperation.

@Override
protected ShardSkywalkerResponse shardOperation(ShardSkywalkerRequest request) throws ElasticsearchException {
    synchronized (mutex) {
        IndexService indexService = indicesService.indexServiceSafe(request.index());
        InternalIndexShard indexShard = (InternalIndexShard) indexService.shardSafe(request.shardId());
        MapperService mapperService = indexService.mapperService();
        Engine.Searcher searcher = indexShard.acquireSearcher("skywalker_action");
        try {
            IndexReader reader = searcher.reader();
            Skywalker skywalker = new Skywalker(reader);
            Map<String, Object> response = new HashMap();
            Directory directory = indexShard.store().directory();
            List indexFiles = new ArrayList();
            for (String f : skywalker.getIndexFiles(directory)) {
                Map indexFile = new HashMap();
                indexFile.put("name", f);
                indexFile.put("function", skywalker.getFileFunction(f));
                indexFiles.add(indexFile);
            }
            response.put("indexFiles", indexFiles);
            skywalker.getStoreMetadata(response, indexShard.store().getMetadata());
            response.put("indexVersion", skywalker.getVersion());
            response.put("directoryImpl", skywalker.getDirImpl());
            response.put("numDocs", reader.numDocs());
            response.put("maxDoc", reader.maxDoc());
            response.put("hasDeletions", reader.hasDeletions());
            response.put("numDeletedDocs", reader.numDeletedDocs());
            Set<FieldTermCount> ftc = skywalker.getFieldTermCounts();
            response.put("numTerms", skywalker.getNumTerms());
            Map indexFormatInfo = new HashMap();
            FormatDetails details = skywalker.getFormatDetails();
            indexFormatInfo.put("version", details.getVersion());
            indexFormatInfo.put("genericName", details.getGenericName());
            indexFormatInfo.put("capabilities", details.getCapabilities());
            response.put("indexFormat", indexFormatInfo);
            List commits = new ArrayList();
            Iterator<Segment> it = indexShard.engine().segments().iterator();
            while (it.hasNext()) {
                Segment segment = it.next();
                Map m = new HashMap();
                m.put("segment", segment.getName());
                m.put("count", segment.getNumDocs());
                m.put("deleted", segment.getDeletedDocs());
                m.put("generation", segment.getGeneration());
                m.put("sizeInBytes", segment.getSizeInBytes());
                m.put("version", segment.getVersion());
                m.put("committed", segment.committed);
                m.put("compound", segment.compound);
                m.put("size", segment.getSize().toString());
                commits.add(m);
            }
            response.put("commits", commits);
            List fieldInfos = new ArrayList();
            for (FieldInfo fi : MultiFields.getMergedFieldInfos(reader)) {
                fieldInfos.add(skywalker.getFieldInfo(mapperService, fi));
            }
            response.put("fieldInfos", fieldInfos);
            List termList = new ArrayList();
            for (TermStats ts : skywalker.getTopTerms(50)) {
                Map m = new HashMap();
                m.put("field", ts.field());
                m.put("text", ts.text());
                m.put("docFreq", ts.docFreq());
                termList.add(m);
            }
            response.put("topterms", termList);
            return new ShardSkywalkerResponse(request.index(), request.shardId()).setResponse(response);
        } catch (Exception ex) {
            throw new ElasticsearchException(ex.getMessage(), ex);
        }
    }
}

Example 38 with MapperService

use of org.elasticsearch.index.mapper.MapperService in project crate by crate.

the class TransportCreatePartitionsAction method executeCreateIndices.

/**
 * This code is more or less the same as the stuff in {@link MetadataCreateIndexService}
 * but optimized for bulk operation without separate mapping/alias/index settings.
 */
private ClusterState executeCreateIndices(ClusterState currentState, CreatePartitionsRequest request) throws Exception {
    List<String> indicesToCreate = new ArrayList<>(request.indices().size());
    List<String> removalReasons = new ArrayList<>(request.indices().size());
    List<Index> createdIndices = new ArrayList<>(request.indices().size());
    try {
        validateAndFilterExistingIndices(currentState, indicesToCreate, request);
        if (indicesToCreate.isEmpty()) {
            return currentState;
        }
        Map<String, Map<String, Object>> mappings = new HashMap<>();
        Map<String, AliasMetadata> templatesAliases = new HashMap<>();
        List<String> templateNames = new ArrayList<>();
        List<IndexTemplateMetadata> templates = findTemplates(request, currentState);
        applyTemplates(mappings, templatesAliases, templateNames, templates);
        Metadata.Builder newMetadataBuilder = Metadata.builder(currentState.metadata());
        for (String index : indicesToCreate) {
            Settings indexSettings = createIndexSettings(currentState, templates);
            shardLimitValidator.validateShardLimit(indexSettings, currentState);
            int routingNumShards = IndexMetadata.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(indexSettings);
            String testIndex = indicesToCreate.get(0);
            IndexMetadata.Builder tmpImdBuilder = IndexMetadata.builder(testIndex).setRoutingNumShards(routingNumShards);
            // Set up everything, now locally create the index to see that things are ok, and apply
            final IndexMetadata tmpImd = tmpImdBuilder.settings(indexSettings).build();
            ActiveShardCount waitForActiveShards = tmpImd.getWaitForActiveShards();
            if (!waitForActiveShards.validate(tmpImd.getNumberOfReplicas())) {
                throw new IllegalArgumentException("invalid wait_for_active_shards[" + waitForActiveShards + "]: cannot be greater than number of shard copies [" + (tmpImd.getNumberOfReplicas() + 1) + "]");
            }
            // create the index here (on the master) to validate it can be created, as well as adding the mapping
            IndexService indexService = indicesService.createIndex(tmpImd, Collections.emptyList(), false);
            createdIndices.add(indexService.index());
            // now add the mappings
            MapperService mapperService = indexService.mapperService();
            if (!mappings.isEmpty()) {
                assert mappings.size() == 1 : "Must have at most 1 mapping type";
                var entry = mappings.entrySet().iterator().next();
                try {
                    mapperService.merge(entry.getKey(), entry.getValue(), MapperService.MergeReason.MAPPING_UPDATE);
                } catch (MapperParsingException mpe) {
                    removalReasons.add("failed on parsing mappings on index creation");
                    throw mpe;
                }
            }
            // now, update the mappings with the actual source
            HashMap<String, MappingMetadata> mappingsMetadata = new HashMap<>();
            DocumentMapper mapper = mapperService.documentMapper();
            if (mapper != null) {
                MappingMetadata mappingMd = new MappingMetadata(mapper);
                mappingsMetadata.put(mapper.type(), mappingMd);
            }
            final IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(index).setRoutingNumShards(routingNumShards).settings(indexSettings);
            for (MappingMetadata mappingMd : mappingsMetadata.values()) {
                indexMetadataBuilder.putMapping(mappingMd);
            }
            for (AliasMetadata aliasMetadata : templatesAliases.values()) {
                indexMetadataBuilder.putAlias(aliasMetadata);
            }
            indexMetadataBuilder.state(IndexMetadata.State.OPEN);
            final IndexMetadata indexMetadata;
            try {
                indexMetadata = indexMetadataBuilder.build();
            } catch (Exception e) {
                removalReasons.add("failed to build index metadata");
                throw e;
            }
            logger.info("[{}] creating index, cause [bulk], templates {}, shards [{}]/[{}], mappings {}", index, templateNames, indexMetadata.getNumberOfShards(), indexMetadata.getNumberOfReplicas(), mappings.keySet());
            indexService.getIndexEventListener().beforeIndexAddedToCluster(indexMetadata.getIndex(), indexMetadata.getSettings());
            newMetadataBuilder.put(indexMetadata, false);
            removalReasons.add("cleaning up after validating index on master");
        }
        Metadata newMetadata = newMetadataBuilder.build();
        ClusterState updatedState = ClusterState.builder(currentState).metadata(newMetadata).build();
        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(updatedState.routingTable());
        for (String index : indicesToCreate) {
            routingTableBuilder.addAsNew(updatedState.metadata().index(index));
        }
        return allocationService.reroute(ClusterState.builder(updatedState).routingTable(routingTableBuilder.build()).build(), "bulk-index-creation");
    } finally {
        for (int i = 0; i < createdIndices.size(); i++) {
            // Index was already partially created - need to clean up
            String removalReason = removalReasons.size() > i ? removalReasons.get(i) : "failed to create index";
            indicesService.removeIndex(createdIndices.get(i), IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED, removalReason);
        }
    }
}

Example 39 with MapperService

use of org.elasticsearch.index.mapper.MapperService in project crate by crate.

the class MetadataIndexUpgradeService method checkMappingsCompatibility.

/**
 * Checks the mappings for compatibility with the current version
 */
private void checkMappingsCompatibility(IndexMetadata indexMetadata) {
    try {
        // We cannot instantiate real analysis server or similarity service at this point because the node
        // might not have been started yet. However, we don't really need real analyzers or similarities at
        // this stage - so we can fake it using constant maps accepting every key.
        // This is ok because all used similarities and analyzers for this index were known before the upgrade.
        // Missing analyzers and similarities plugin will still trigger the appropriate error during the
        // actual upgrade.
        IndexSettings indexSettings = new IndexSettings(indexMetadata, this.settings);
        final NamedAnalyzer fakeDefault = new NamedAnalyzer("fake_default", AnalyzerScope.INDEX, new Analyzer() {

            @Override
            protected TokenStreamComponents createComponents(String fieldName) {
                throw new UnsupportedOperationException("shouldn't be here");
            }
        });
        final Map<String, NamedAnalyzer> analyzerMap = new AbstractMap<String, NamedAnalyzer>() {

            @Override
            public NamedAnalyzer get(Object key) {
                assert key instanceof String : "key must be a string but was: " + key.getClass();
                return new NamedAnalyzer((String) key, AnalyzerScope.INDEX, fakeDefault.analyzer());
            }

            // this entrySet impl isn't fully correct but necessary as IndexAnalyzers will iterate
            // over all analyzers to close them
            @Override
            public Set<Entry<String, NamedAnalyzer>> entrySet() {
                return Collections.emptySet();
            }
        };
        try (IndexAnalyzers fakeIndexAnalzyers = new IndexAnalyzers(indexSettings, fakeDefault, fakeDefault, fakeDefault, analyzerMap, analyzerMap, analyzerMap)) {
            MapperService mapperService = new MapperService(indexSettings, fakeIndexAnalzyers, xContentRegistry, mapperRegistry, () -> null);
            mapperService.merge(indexMetadata, MapperService.MergeReason.MAPPING_RECOVERY);
        }
    } catch (Exception ex) {
        // Wrap the inner exception so we have the index name in the exception message
        throw new IllegalStateException("unable to upgrade the mappings for the index [" + indexMetadata.getIndex() + "]", ex);
    }
}

Example 40 with MapperService

use of org.elasticsearch.index.mapper.MapperService in project crate by crate.

the class ArrayMapperTest method mapper.

/**
 * create index with type and mapping and validate DocumentMapper serialization
 */
private DocumentMapper mapper(String indexName, String mapping) throws IOException {
    IndicesModule indicesModule = new IndicesModule(List.of());
    MapperService mapperService = MapperTestUtils.newMapperService(NamedXContentRegistry.EMPTY, createTempDir(), Settings.EMPTY, indicesModule, indexName);
    DocumentMapperParser parser = mapperService.documentMapperParser();
    DocumentMapper defaultMapper = parser.parse(TYPE, new CompressedXContent(mapping));
    XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
    builder.startObject();
    defaultMapper.toXContent(builder, ToXContent.EMPTY_PARAMS);
    builder.endObject();
    String rebuildMapping = Strings.toString(builder);
    return parser.parse(TYPE, new CompressedXContent(rebuildMapping));
}