Examples with DocumentMapper org.elasticsearch.index.mapper.DocumentMapper used on opensource projects

Example 36 with DocumentMapper

use of org.elasticsearch.index.mapper.DocumentMapper in project crate by crate.

the class TransportBulkCreateIndicesAction method executeCreateIndices.

/**
     * This code is more or less the same as the stuff in {@link MetaDataCreateIndexService}
     * but optimized for bulk operation without separate mapping/alias/index settings.
     */
ClusterState executeCreateIndices(ClusterState currentState, BulkCreateIndicesRequest request) throws Exception {
    List<String> indicesToCreate = new ArrayList<>(request.indices().size());
    String removalReason = null;
    String testIndex = null;
    try {
        validateAndFilterExistingIndices(currentState, indicesToCreate, request);
        if (indicesToCreate.isEmpty()) {
            return currentState;
        }
        Map<String, IndexMetaData.Custom> customs = Maps.newHashMap();
        Map<String, Map<String, Object>> mappings = Maps.newHashMap();
        Map<String, AliasMetaData> templatesAliases = Maps.newHashMap();
        List<String> templateNames = Lists.newArrayList();
        List<IndexTemplateMetaData> templates = findTemplates(request, currentState, indexTemplateFilter);
        applyTemplates(customs, mappings, templatesAliases, templateNames, templates);
        File mappingsDir = new File(environment.configFile().toFile(), "mappings");
        if (mappingsDir.isDirectory()) {
            addMappingFromMappingsFile(mappings, mappingsDir, request);
        }
        Settings indexSettings = createIndexSettings(currentState, templates);
        testIndex = indicesToCreate.get(0);
        indicesService.createIndex(testIndex, indexSettings, clusterService.localNode().getId());
        // now add the mappings
        IndexService indexService = indicesService.indexServiceSafe(testIndex);
        MapperService mapperService = indexService.mapperService();
        // first, add the default mapping
        if (mappings.containsKey(MapperService.DEFAULT_MAPPING)) {
            try {
                mapperService.merge(MapperService.DEFAULT_MAPPING, new CompressedXContent(XContentFactory.jsonBuilder().map(mappings.get(MapperService.DEFAULT_MAPPING)).string()), MapperService.MergeReason.MAPPING_UPDATE, false);
            } catch (Exception e) {
                removalReason = "failed on parsing default mapping on index creation";
                throw new MapperParsingException("mapping [" + MapperService.DEFAULT_MAPPING + "]", e);
            }
        }
        for (Map.Entry<String, Map<String, Object>> entry : mappings.entrySet()) {
            if (entry.getKey().equals(MapperService.DEFAULT_MAPPING)) {
                continue;
            }
            try {
                // apply the default here, its the first time we parse it
                mapperService.merge(entry.getKey(), new CompressedXContent(XContentFactory.jsonBuilder().map(entry.getValue()).string()), MapperService.MergeReason.MAPPING_UPDATE, false);
            } catch (Exception e) {
                removalReason = "failed on parsing mappings on index creation";
                throw new MapperParsingException("mapping [" + entry.getKey() + "]", e);
            }
        }
        IndexQueryParserService indexQueryParserService = indexService.queryParserService();
        for (AliasMetaData aliasMetaData : templatesAliases.values()) {
            if (aliasMetaData.filter() != null) {
                aliasValidator.validateAliasFilter(aliasMetaData.alias(), aliasMetaData.filter().uncompressed(), indexQueryParserService);
            }
        }
        // now, update the mappings with the actual source
        Map<String, MappingMetaData> mappingsMetaData = Maps.newHashMap();
        for (DocumentMapper mapper : mapperService.docMappers(true)) {
            MappingMetaData mappingMd = new MappingMetaData(mapper);
            mappingsMetaData.put(mapper.type(), mappingMd);
        }
        MetaData.Builder newMetaDataBuilder = MetaData.builder(currentState.metaData());
        for (String index : indicesToCreate) {
            Settings newIndexSettings = createIndexSettings(currentState, templates);
            final IndexMetaData.Builder indexMetaDataBuilder = IndexMetaData.builder(index).settings(newIndexSettings);
            for (MappingMetaData mappingMd : mappingsMetaData.values()) {
                indexMetaDataBuilder.putMapping(mappingMd);
            }
            for (AliasMetaData aliasMetaData : templatesAliases.values()) {
                indexMetaDataBuilder.putAlias(aliasMetaData);
            }
            for (Map.Entry<String, IndexMetaData.Custom> customEntry : customs.entrySet()) {
                indexMetaDataBuilder.putCustom(customEntry.getKey(), customEntry.getValue());
            }
            indexMetaDataBuilder.state(IndexMetaData.State.OPEN);
            final IndexMetaData indexMetaData;
            try {
                indexMetaData = indexMetaDataBuilder.build();
            } catch (Exception e) {
                removalReason = "failed to build index metadata";
                throw e;
            }
            logger.info("[{}] creating index, cause [bulk], templates {}, shards [{}]/[{}], mappings {}", index, templateNames, indexMetaData.getNumberOfShards(), indexMetaData.getNumberOfReplicas(), mappings.keySet());
            indexService.indicesLifecycle().beforeIndexAddedToCluster(new Index(index), indexMetaData.getSettings());
            newMetaDataBuilder.put(indexMetaData, false);
        }
        MetaData newMetaData = newMetaDataBuilder.build();
        ClusterState updatedState = ClusterState.builder(currentState).metaData(newMetaData).build();
        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(updatedState.routingTable());
        for (String index : indicesToCreate) {
            routingTableBuilder.addAsNew(updatedState.metaData().index(index));
        }
        RoutingAllocation.Result routingResult = allocationService.reroute(ClusterState.builder(updatedState).routingTable(routingTableBuilder).build(), "bulk-index-creation");
        updatedState = ClusterState.builder(updatedState).routingResult(routingResult).build();
        removalReason = "cleaning up after validating index on master";
        return updatedState;
    } finally {
        if (testIndex != null) {
            // index was partially created - need to clean up
            indicesService.deleteIndex(testIndex, removalReason != null ? removalReason : "failed to create index");
        }
    }
}

Example 37 with DocumentMapper

use of org.elasticsearch.index.mapper.DocumentMapper in project elasticsearch-opennlp-plugin by spinscale.

the class OpenNlpMappingTest method testSimpleMappings.

@Test
public void testSimpleMappings() throws Exception {
    String mapping = copyToStringFromClasspath("/test-mapping.json");
    DocumentMapper docMapper = mapperParser.parse(mapping);
    String sampleText = copyToStringFromClasspath("/sample-text.txt");
    BytesReference json = jsonBuilder().startObject().field("_id", 1).field("someField", sampleText).endObject().bytes();
    Document doc = docMapper.parse(json).rootDoc();
    assertThat(doc.get(docMapper.mappers().smartName("someField").mapper().names().indexName()), is(sampleText));
    assertThat(doc.getFields("someField.name").length, is(2));
    assertThat(doc.getFields("someField.name")[0].stringValue(), is("Jack Nicholson"));
    assertThat(doc.getFields("someField.name")[1].stringValue(), is("Kobe Bryant"));
    assertThat(doc.get(docMapper.mappers().smartName("someField.date").mapper().names().indexName()), is("tomorrow"));
    assertThat(doc.get(docMapper.mappers().smartName("someField.location").mapper().names().indexName()), is("Munich"));
    // re-parse it
    String builtMapping = docMapper.mappingSource().string();
    docMapper = mapperParser.parse(builtMapping);
    json = jsonBuilder().startObject().field("_id", 1).field("someField", sampleText).endObject().bytes();
    doc = docMapper.parse(json).rootDoc();
    assertThat(doc.get(docMapper.mappers().smartName("someField").mapper().names().indexName()), is(sampleText));
    assertThat(doc.getFields("someField.name").length, is(2));
    assertThat(doc.getFields("someField.name")[0].stringValue(), is("Jack Nicholson"));
    assertThat(doc.getFields("someField.name")[1].stringValue(), is("Kobe Bryant"));
    assertThat(doc.get(docMapper.mappers().smartName("someField.date").mapper().names().indexName()), is("tomorrow"));
    assertThat(doc.get(docMapper.mappers().smartName("someField.location").mapper().names().indexName()), is("Munich"));
}

Example 38 with DocumentMapper

use of org.elasticsearch.index.mapper.DocumentMapper in project crate by crate.

the class ArrayMapperTest method testInvalidArrayNonConvertableType.

@Test
public void testInvalidArrayNonConvertableType() throws Exception {
    String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject(TYPE).startObject("properties").startObject("array_field").field("type", ArrayMapper.CONTENT_TYPE).startObject(ArrayMapper.INNER_TYPE).field("type", "double").endObject().endObject().endObject().endObject().endObject());
    DocumentMapper mapper = mapper(INDEX, mapping);
    expectedException.expect(MapperParsingException.class);
    expectedException.expectMessage("failed to parse field [array_field] of type [double]");
    BytesReference bytesReference = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().array("array_field", true, false, true).endObject());
    SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
    mapper.parse(sourceToParse);
}

Example 39 with DocumentMapper

use of org.elasticsearch.index.mapper.DocumentMapper in project crate by crate.

the class ArrayMapperTest method testCopyToFieldsOfInnerMapping.

@Test
public void testCopyToFieldsOfInnerMapping() throws Exception {
    // @formatter:off
    String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject(TYPE).startObject("properties").startObject("string_array").field("type", ArrayMapper.CONTENT_TYPE).startObject(ArrayMapper.INNER_TYPE).field("type", "text").field("index", "true").field("copy_to", "string_array_ft").endObject().endObject().endObject().endObject().endObject());
    // @formatter:on
    DocumentMapper mapper = mapper(INDEX, mapping);
    FieldMapper arrayMapper = (FieldMapper) mapper.mappers().getMapper("string_array");
    assertThat(arrayMapper, is(instanceOf(ArrayMapper.class)));
    assertThat(arrayMapper.copyTo().copyToFields(), contains("string_array_ft"));
    // @formatter:on
    BytesReference bytesReference = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().startArray("string_array").value("foo").value("bar").endArray().endObject());
    SourceToParse sourceToParse = new SourceToParse(INDEX, "1", bytesReference, XContentType.JSON);
    ParsedDocument doc = mapper.parse(sourceToParse);
    // @formatter:off
    List<String> copyValues = new ArrayList<>();
    Document document = doc.docs().get(0);
    IndexableField[] fields = document.getFields("string_array_ft");
    for (var field : fields) {
        if (field.fieldType().docValuesType() == DocValuesType.SORTED_SET) {
            copyValues.add(field.binaryValue().utf8ToString());
        }
    }
    assertThat(copyValues, containsInAnyOrder("foo", "bar"));
}

Example 40 with DocumentMapper

use of org.elasticsearch.index.mapper.DocumentMapper in project crate by crate.

the class ArrayMapperTest method testParseNullOnObjectArray.

@Test
public void testParseNullOnObjectArray() throws Exception {
    // @formatter: on
    String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject(TYPE).startObject("properties").startObject("array_field").field("type", ArrayMapper.CONTENT_TYPE).startObject(ArrayMapper.INNER_TYPE).field("type", "object").startObject("properties").endObject().endObject().endObject().endObject().endObject().endObject());
    // @formatter: off
    DocumentMapper mapper = mapper(INDEX, mapping);
    BytesReference bytesReference = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().nullField("array_field").endObject());
    SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
    ParsedDocument parsedDoc = mapper.parse(sourceToParse);
    assertThat(parsedDoc.docs().size(), is(1));
    assertThat(parsedDoc.docs().get(0).getField("array_field"), is(nullValue()));
}