Examples with DocumentMapper org.elasticsearch.index.mapper.DocumentMapper used on opensource projects

Example 91 with DocumentMapper

use of org.elasticsearch.index.mapper.DocumentMapper in project crate by crate.

the class TransportCreatePartitionsAction method executeCreateIndices.

/**
 * This code is more or less the same as the stuff in {@link MetadataCreateIndexService}
 * but optimized for bulk operation without separate mapping/alias/index settings.
 */
private ClusterState executeCreateIndices(ClusterState currentState, CreatePartitionsRequest request) throws Exception {
    List<String> indicesToCreate = new ArrayList<>(request.indices().size());
    List<String> removalReasons = new ArrayList<>(request.indices().size());
    List<Index> createdIndices = new ArrayList<>(request.indices().size());
    try {
        validateAndFilterExistingIndices(currentState, indicesToCreate, request);
        if (indicesToCreate.isEmpty()) {
            return currentState;
        }
        Map<String, Map<String, Object>> mappings = new HashMap<>();
        Map<String, AliasMetadata> templatesAliases = new HashMap<>();
        List<String> templateNames = new ArrayList<>();
        List<IndexTemplateMetadata> templates = findTemplates(request, currentState);
        applyTemplates(mappings, templatesAliases, templateNames, templates);
        Metadata.Builder newMetadataBuilder = Metadata.builder(currentState.metadata());
        for (String index : indicesToCreate) {
            Settings indexSettings = createIndexSettings(currentState, templates);
            shardLimitValidator.validateShardLimit(indexSettings, currentState);
            int routingNumShards = IndexMetadata.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(indexSettings);
            String testIndex = indicesToCreate.get(0);
            IndexMetadata.Builder tmpImdBuilder = IndexMetadata.builder(testIndex).setRoutingNumShards(routingNumShards);
            // Set up everything, now locally create the index to see that things are ok, and apply
            final IndexMetadata tmpImd = tmpImdBuilder.settings(indexSettings).build();
            ActiveShardCount waitForActiveShards = tmpImd.getWaitForActiveShards();
            if (!waitForActiveShards.validate(tmpImd.getNumberOfReplicas())) {
                throw new IllegalArgumentException("invalid wait_for_active_shards[" + waitForActiveShards + "]: cannot be greater than number of shard copies [" + (tmpImd.getNumberOfReplicas() + 1) + "]");
            }
            // create the index here (on the master) to validate it can be created, as well as adding the mapping
            IndexService indexService = indicesService.createIndex(tmpImd, Collections.emptyList(), false);
            createdIndices.add(indexService.index());
            // now add the mappings
            MapperService mapperService = indexService.mapperService();
            if (!mappings.isEmpty()) {
                assert mappings.size() == 1 : "Must have at most 1 mapping type";
                var entry = mappings.entrySet().iterator().next();
                try {
                    mapperService.merge(entry.getKey(), entry.getValue(), MapperService.MergeReason.MAPPING_UPDATE);
                } catch (MapperParsingException mpe) {
                    removalReasons.add("failed on parsing mappings on index creation");
                    throw mpe;
                }
            }
            // now, update the mappings with the actual source
            HashMap<String, MappingMetadata> mappingsMetadata = new HashMap<>();
            DocumentMapper mapper = mapperService.documentMapper();
            if (mapper != null) {
                MappingMetadata mappingMd = new MappingMetadata(mapper);
                mappingsMetadata.put(mapper.type(), mappingMd);
            }
            final IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(index).setRoutingNumShards(routingNumShards).settings(indexSettings);
            for (MappingMetadata mappingMd : mappingsMetadata.values()) {
                indexMetadataBuilder.putMapping(mappingMd);
            }
            for (AliasMetadata aliasMetadata : templatesAliases.values()) {
                indexMetadataBuilder.putAlias(aliasMetadata);
            }
            indexMetadataBuilder.state(IndexMetadata.State.OPEN);
            final IndexMetadata indexMetadata;
            try {
                indexMetadata = indexMetadataBuilder.build();
            } catch (Exception e) {
                removalReasons.add("failed to build index metadata");
                throw e;
            }
            logger.info("[{}] creating index, cause [bulk], templates {}, shards [{}]/[{}], mappings {}", index, templateNames, indexMetadata.getNumberOfShards(), indexMetadata.getNumberOfReplicas(), mappings.keySet());
            indexService.getIndexEventListener().beforeIndexAddedToCluster(indexMetadata.getIndex(), indexMetadata.getSettings());
            newMetadataBuilder.put(indexMetadata, false);
            removalReasons.add("cleaning up after validating index on master");
        }
        Metadata newMetadata = newMetadataBuilder.build();
        ClusterState updatedState = ClusterState.builder(currentState).metadata(newMetadata).build();
        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(updatedState.routingTable());
        for (String index : indicesToCreate) {
            routingTableBuilder.addAsNew(updatedState.metadata().index(index));
        }
        return allocationService.reroute(ClusterState.builder(updatedState).routingTable(routingTableBuilder.build()).build(), "bulk-index-creation");
    } finally {
        for (int i = 0; i < createdIndices.size(); i++) {
            // Index was already partially created - need to clean up
            String removalReason = removalReasons.size() > i ? removalReasons.get(i) : "failed to create index";
            indicesService.removeIndex(createdIndices.get(i), IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED, removalReason);
        }
    }
}

Example 92 with DocumentMapper

use of org.elasticsearch.index.mapper.DocumentMapper in project crate by crate.

the class MetadataMappingService method refreshIndexMapping.

private boolean refreshIndexMapping(IndexService indexService, IndexMetadata.Builder builder) {
    boolean dirty = false;
    String index = indexService.index().getName();
    try {
        List<String> updatedTypes = new ArrayList<>();
        DocumentMapper mapper = indexService.mapperService().documentMapper();
        if (mapper != null) {
            final String type = mapper.type();
            if (!mapper.mappingSource().equals(builder.mapping(type).source())) {
                updatedTypes.add(type);
            }
        }
        // if a single type is not up-to-date, re-send everything
        if (updatedTypes.isEmpty() == false) {
            LOGGER.warn("[{}] re-syncing mappings with cluster state because of types [{}]", index, updatedTypes);
            dirty = true;
            if (mapper != null) {
                builder.putMapping(new MappingMetadata(mapper));
            }
        }
    } catch (Exception e) {
        LOGGER.warn(() -> new ParameterizedMessage("[{}] failed to refresh-mapping in cluster state", index), e);
    }
    return dirty;
}

Example 93 with DocumentMapper

use of org.elasticsearch.index.mapper.DocumentMapper in project crate by crate.

the class IndexShard method tombstoneDocSupplier.

private EngineConfig.TombstoneDocSupplier tombstoneDocSupplier() {
    final RootObjectMapper.Builder noopRootMapper = new RootObjectMapper.Builder("default");
    final DocumentMapper noopDocumentMapper = mapperService == null ? null : new DocumentMapper.Builder(noopRootMapper, mapperService).build(mapperService);
    return new EngineConfig.TombstoneDocSupplier() {

        @Override
        public ParsedDocument newDeleteTombstoneDoc(String id) {
            return mapperService.documentMapper().createDeleteTombstoneDoc(shardId.getIndexName(), id);
        }

        @Override
        public ParsedDocument newNoopTombstoneDoc(String reason) {
            return noopDocumentMapper.createNoopTombstoneDoc(shardId.getIndexName(), reason);
        }
    };
}

Example 94 with DocumentMapper

use of org.elasticsearch.index.mapper.DocumentMapper in project crate by crate.

the class ArrayMapperTest method testParseNull.

@Test
public void testParseNull() throws Exception {
    // @formatter: off
    String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject(TYPE).startObject("properties").startObject("array_field").field("type", ArrayMapper.CONTENT_TYPE).startObject(ArrayMapper.INNER_TYPE).field("type", "double").endObject().endObject().endObject().endObject().endObject());
    // @formatter: on
    DocumentMapper mapper = mapper(INDEX, mapping);
    BytesReference bytesReference = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().nullField("array_field").endObject());
    SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
    ParsedDocument parsedDoc = mapper.parse(sourceToParse);
    assertThat(parsedDoc.docs().size(), is(1));
    assertThat(parsedDoc.docs().get(0).getField("array_field"), is(nullValue()));
}

Example 95 with DocumentMapper

use of org.elasticsearch.index.mapper.DocumentMapper in project crate by crate.

the class ArrayMapperTest method mapper.

/**
 * create index with type and mapping and validate DocumentMapper serialization
 */
private DocumentMapper mapper(String indexName, String mapping) throws IOException {
    IndicesModule indicesModule = new IndicesModule(List.of());
    MapperService mapperService = MapperTestUtils.newMapperService(NamedXContentRegistry.EMPTY, createTempDir(), Settings.EMPTY, indicesModule, indexName);
    DocumentMapperParser parser = mapperService.documentMapperParser();
    DocumentMapper defaultMapper = parser.parse(TYPE, new CompressedXContent(mapping));
    XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
    builder.startObject();
    defaultMapper.toXContent(builder, ToXContent.EMPTY_PARAMS);
    builder.endObject();
    String rebuildMapping = Strings.toString(builder);
    return parser.parse(TYPE, new CompressedXContent(rebuildMapping));
}