Examples with CompressedXContent org.opensearch.common.compress.CompressedXContent used on opensource projects

Example 11 with CompressedXContent

use of org.opensearch.common.compress.CompressedXContent in project OpenSearch by opensearch-project.

the class PercolatorFieldMapperTests method addQueryFieldMappings.

private void addQueryFieldMappings() throws Exception {
    fieldName = randomAlphaOfLength(4);
    String percolatorMapper = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("properties").startObject(fieldName).field("type", "percolator").endObject().endObject().endObject());
    mapperService.merge(MapperService.SINGLE_MAPPING_NAME, new CompressedXContent(percolatorMapper), MapperService.MergeReason.MAPPING_UPDATE);
    fieldType = (PercolatorFieldMapper.PercolatorFieldType) mapperService.fieldType(fieldName);
}

Example 12 with CompressedXContent

use of org.opensearch.common.compress.CompressedXContent in project OpenSearch by opensearch-project.

the class PercolatorFieldMapperTests method testNestedPercolatorField.

// percolator field can be nested under an object field, but only one query can be specified per document
public void testNestedPercolatorField() throws Exception {
    String typeName = MapperService.SINGLE_MAPPING_NAME;
    String percolatorMapper = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject(typeName).startObject("properties").startObject("object_field").field("type", "object").startObject("properties").startObject("query_field").field("type", "percolator").endObject().endObject().endObject().endObject().endObject().endObject());
    mapperService.merge(typeName, new CompressedXContent(percolatorMapper), MapperService.MergeReason.MAPPING_UPDATE);
    QueryBuilder queryBuilder = matchQuery("field", "value");
    ParsedDocument doc = mapperService.documentMapper().parse(new SourceToParse("test", "1", BytesReference.bytes(jsonBuilder().startObject().startObject("object_field").field("query_field", queryBuilder).endObject().endObject()), XContentType.JSON));
    // also includes all other meta fields
    assertThat(doc.rootDoc().getFields().size(), equalTo(12));
    IndexableField queryBuilderField = doc.rootDoc().getField("object_field.query_field.query_builder_field");
    assertTrue(queryBuilderField.fieldType().omitNorms());
    IndexableField extractionResultField = doc.rootDoc().getField("object_field.query_field.extraction_result");
    assertTrue(extractionResultField.fieldType().omitNorms());
    BytesRef queryBuilderAsBytes = queryBuilderField.binaryValue();
    assertQueryBuilder(queryBuilderAsBytes, queryBuilder);
    doc = mapperService.documentMapper().parse(new SourceToParse("test", "1", BytesReference.bytes(jsonBuilder().startObject().startArray("object_field").startObject().field("query_field", queryBuilder).endObject().endArray().endObject()), XContentType.JSON));
    // also includes all other meta fields
    assertThat(doc.rootDoc().getFields().size(), equalTo(12));
    queryBuilderAsBytes = doc.rootDoc().getField("object_field.query_field.query_builder_field").binaryValue();
    assertQueryBuilder(queryBuilderAsBytes, queryBuilder);
    MapperParsingException e = expectThrows(MapperParsingException.class, () -> {
        mapperService.documentMapper().parse(new SourceToParse("test", "1", BytesReference.bytes(jsonBuilder().startObject().startArray("object_field").startObject().field("query_field", queryBuilder).endObject().startObject().field("query_field", queryBuilder).endObject().endArray().endObject()), XContentType.JSON));
    });
    assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
    assertThat(e.getCause().getMessage(), equalTo("a document can only contain one percolator query"));
}

Example 13 with CompressedXContent

use of org.opensearch.common.compress.CompressedXContent in project OpenSearch by opensearch-project.

the class MetadataIndexTemplateService method addComponentTemplate.

// Package visible for testing
ClusterState addComponentTemplate(final ClusterState currentState, final boolean create, final String name, final ComponentTemplate template) throws Exception {
    final ComponentTemplate existing = currentState.metadata().componentTemplates().get(name);
    if (create && existing != null) {
        throw new IllegalArgumentException("component template [" + name + "] already exists");
    }
    CompressedXContent mappings = template.template().mappings();
    String stringMappings = mappings == null ? null : mappings.string();
    // We may need to normalize index settings, so do that also
    Settings finalSettings = template.template().settings();
    if (finalSettings != null) {
        finalSettings = Settings.builder().put(finalSettings).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX).build();
    }
    // Collect all the composable (index) templates that use this component template, we'll use
    // this for validating that they're still going to be valid after this component template
    // has been updated
    final Map<String, ComposableIndexTemplate> templatesUsingComponent = currentState.metadata().templatesV2().entrySet().stream().filter(e -> e.getValue().composedOf().contains(name)).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
    // if we're updating a component template, let's check if it's part of any V2 template that will yield the CT update invalid
    if (create == false && finalSettings != null) {
        // if the CT is specifying the `index.hidden` setting it cannot be part of any global template
        if (IndexMetadata.INDEX_HIDDEN_SETTING.exists(finalSettings)) {
            List<String> globalTemplatesThatUseThisComponent = new ArrayList<>();
            for (Map.Entry<String, ComposableIndexTemplate> entry : templatesUsingComponent.entrySet()) {
                ComposableIndexTemplate templateV2 = entry.getValue();
                if (templateV2.indexPatterns().stream().anyMatch(Regex::isMatchAllPattern)) {
                    // global templates don't support configuring the `index.hidden` setting so we don't need to resolve the settings as
                    // no other component template can remove this setting from the resolved settings, so just invalidate this update
                    globalTemplatesThatUseThisComponent.add(entry.getKey());
                }
            }
            if (globalTemplatesThatUseThisComponent.isEmpty() == false) {
                throw new IllegalArgumentException("cannot update component template [" + name + "] because the following global templates would resolve to specifying the [" + IndexMetadata.SETTING_INDEX_HIDDEN + "] setting: [" + String.join(",", globalTemplatesThatUseThisComponent) + "]");
            }
        }
    }
    // Mappings in component templates don't include _doc, so update the mappings to include this single type
    if (stringMappings != null) {
        Map<String, Object> parsedMappings = MapperService.parseMapping(xContentRegistry, stringMappings);
        if (parsedMappings.size() > 0) {
            stringMappings = Strings.toString(XContentFactory.jsonBuilder().startObject().field(MapperService.SINGLE_MAPPING_NAME, parsedMappings).endObject());
        }
    }
    final Template finalTemplate = new Template(finalSettings, stringMappings == null ? null : new CompressedXContent(stringMappings), template.template().aliases());
    final ComponentTemplate finalComponentTemplate = new ComponentTemplate(finalTemplate, template.version(), template.metadata());
    if (finalComponentTemplate.equals(existing)) {
        return currentState;
    }
    validateTemplate(finalSettings, stringMappings, indicesService, xContentRegistry);
    validate(name, finalComponentTemplate);
    // Validate all composable index templates that use this component template
    if (templatesUsingComponent.size() > 0) {
        ClusterState tempStateWithComponentTemplateAdded = ClusterState.builder(currentState).metadata(Metadata.builder(currentState.metadata()).put(name, finalComponentTemplate)).build();
        Exception validationFailure = null;
        for (Map.Entry<String, ComposableIndexTemplate> entry : templatesUsingComponent.entrySet()) {
            final String composableTemplateName = entry.getKey();
            final ComposableIndexTemplate composableTemplate = entry.getValue();
            try {
                validateCompositeTemplate(tempStateWithComponentTemplateAdded, composableTemplateName, composableTemplate, indicesService, xContentRegistry);
            } catch (Exception e) {
                if (validationFailure == null) {
                    validationFailure = new IllegalArgumentException("updating component template [" + name + "] results in invalid composable template [" + composableTemplateName + "] after templates are merged", e);
                } else {
                    validationFailure.addSuppressed(e);
                }
            }
        }
        if (validationFailure != null) {
            throw validationFailure;
        }
    }
    logger.info("{} component template [{}]", existing == null ? "adding" : "updating", name);
    return ClusterState.builder(currentState).metadata(Metadata.builder(currentState.metadata()).put(name, finalComponentTemplate)).build();
}

Example 14 with CompressedXContent

use of org.opensearch.common.compress.CompressedXContent in project OpenSearch by opensearch-project.

the class MetadataIndexTemplateService method addIndexTemplateV2.

public ClusterState addIndexTemplateV2(final ClusterState currentState, final boolean create, final String name, final ComposableIndexTemplate template) throws Exception {
    final ComposableIndexTemplate existing = currentState.metadata().templatesV2().get(name);
    if (create && existing != null) {
        throw new IllegalArgumentException("index template [" + name + "] already exists");
    }
    Map<String, List<String>> overlaps = findConflictingV2Templates(currentState, name, template.indexPatterns(), true, template.priorityOrZero());
    overlaps.remove(name);
    if (overlaps.size() > 0) {
        String error = String.format(Locale.ROOT, "index template [%s] has index patterns %s matching patterns from " + "existing templates [%s] with patterns (%s) that have the same priority [%d], multiple index templates may not " + "match during index creation, please use a different priority", name, template.indexPatterns(), Strings.collectionToCommaDelimitedString(overlaps.keySet()), overlaps.entrySet().stream().map(e -> e.getKey() + " => " + e.getValue()).collect(Collectors.joining(",")), template.priorityOrZero());
        throw new IllegalArgumentException(error);
    }
    overlaps = findConflictingV1Templates(currentState, name, template.indexPatterns());
    if (overlaps.size() > 0) {
        String warning = String.format(Locale.ROOT, "index template [%s] has index patterns %s matching patterns from " + "existing older templates [%s] with patterns (%s); this template [%s] will take precedence during new index creation", name, template.indexPatterns(), Strings.collectionToCommaDelimitedString(overlaps.keySet()), overlaps.entrySet().stream().map(e -> e.getKey() + " => " + e.getValue()).collect(Collectors.joining(",")), name);
        logger.warn(warning);
        HeaderWarning.addWarning(warning);
    }
    ComposableIndexTemplate finalIndexTemplate = template;
    Template innerTemplate = template.template();
    if (innerTemplate != null) {
        // We may need to normalize index settings, so do that also
        Settings finalSettings = innerTemplate.settings();
        if (finalSettings != null) {
            finalSettings = Settings.builder().put(finalSettings).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX).build();
        }
        // If an inner template was specified, its mappings may need to be
        // adjusted (to add _doc) and it should be validated
        CompressedXContent mappings = innerTemplate.mappings();
        String stringMappings = mappings == null ? null : mappings.string();
        // Mappings in index templates don't include _doc, so update the mappings to include this single type
        if (stringMappings != null) {
            Map<String, Object> parsedMappings = MapperService.parseMapping(xContentRegistry, stringMappings);
            if (parsedMappings.size() > 0) {
                stringMappings = Strings.toString(XContentFactory.jsonBuilder().startObject().field(MapperService.SINGLE_MAPPING_NAME, parsedMappings).endObject());
            }
        }
        final Template finalTemplate = new Template(finalSettings, stringMappings == null ? null : new CompressedXContent(stringMappings), innerTemplate.aliases());
        finalIndexTemplate = new ComposableIndexTemplate(template.indexPatterns(), finalTemplate, template.composedOf(), template.priority(), template.version(), template.metadata(), template.getDataStreamTemplate());
    }
    if (finalIndexTemplate.equals(existing)) {
        return currentState;
    }
    validate(name, finalIndexTemplate);
    validateDataStreamsStillReferenced(currentState, name, finalIndexTemplate);
    // mappings, and aliases are valid after it's been composed with the component templates
    try {
        validateCompositeTemplate(currentState, name, finalIndexTemplate, indicesService, xContentRegistry);
    } catch (Exception e) {
        throw new IllegalArgumentException("composable template [" + name + "] template after composition " + (finalIndexTemplate.composedOf().size() > 0 ? "with component templates " + finalIndexTemplate.composedOf() + " " : "") + "is invalid", e);
    }
    logger.info("{} index template [{}] for index patterns {}", existing == null ? "adding" : "updating", name, template.indexPatterns());
    return ClusterState.builder(currentState).metadata(Metadata.builder(currentState.metadata()).put(name, finalIndexTemplate)).build();
}

Example 15 with CompressedXContent

use of org.opensearch.common.compress.CompressedXContent in project OpenSearch by opensearch-project.

the class MetadataIndexTemplateService method collectMappings.

/**
 * Collect the given v2 template into an ordered list of mappings.
 */
public static List<CompressedXContent> collectMappings(final ClusterState state, final String templateName, final String indexName) throws Exception {
    final ComposableIndexTemplate template = state.metadata().templatesV2().get(templateName);
    assert template != null : "attempted to resolve mappings for a template [" + templateName + "] that did not exist in the cluster state";
    if (template == null) {
        return Collections.emptyList();
    }
    final Map<String, ComponentTemplate> componentTemplates = state.metadata().componentTemplates();
    List<CompressedXContent> mappings = template.composedOf().stream().map(componentTemplates::get).filter(Objects::nonNull).map(ComponentTemplate::template).map(Template::mappings).filter(Objects::nonNull).collect(Collectors.toCollection(LinkedList::new));
    // Add the actual index template's mappings, since it takes the highest precedence
    Optional.ofNullable(template.template()).map(Template::mappings).ifPresent(mappings::add);
    if (template.getDataStreamTemplate() != null && indexName.startsWith(DataStream.BACKING_INDEX_PREFIX)) {
        // add a default mapping for the timestamp field, at the lowest precedence, to make bootstrapping data streams more
        // straightforward as all backing indices are required to have a timestamp field
        String timestampFieldName = template.getDataStreamTemplate().getTimestampField().getName();
        mappings.add(0, new CompressedXContent(getTimestampFieldMapping(timestampFieldName)));
    }
    // Only include timestamp mapping snippet if creating backing index.
    if (indexName.startsWith(DataStream.BACKING_INDEX_PREFIX)) {
        // Only if template has data stream definition this should be added and
        // adding this template last, since timestamp field should have highest precedence:
        Optional.ofNullable(template.getDataStreamTemplate()).map(ComposableIndexTemplate.DataStreamTemplate::getDataStreamMappingSnippet).map(mapping -> {
            try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
                builder.value(mapping);
                return new CompressedXContent(BytesReference.bytes(builder));
            } catch (IOException e) {
                throw new UncheckedIOException(e);
            }
        }).ifPresent(mappings::add);
    }
    return Collections.unmodifiableList(mappings);
}