Examples with IndexTemplateMetadata org.opensearch.cluster.metadata.IndexTemplateMetadata used on opensource projects

Example 11 with IndexTemplateMetadata

use of org.opensearch.cluster.metadata.IndexTemplateMetadata in project OpenSearch by opensearch-project.

the class ClusterSerializationTests method testObjectReuseWhenApplyingClusterStateDiff.

public void testObjectReuseWhenApplyingClusterStateDiff() throws Exception {
    IndexMetadata indexMetadata = IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(10).numberOfReplicas(1).build();
    IndexTemplateMetadata indexTemplateMetadata = IndexTemplateMetadata.builder("test-template").patterns(Arrays.asList(generateRandomStringArray(10, 100, false, false))).build();
    Metadata metadata = Metadata.builder().put(indexMetadata, true).put(indexTemplateMetadata).build();
    RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
    ClusterState clusterState1 = ClusterState.builder(new ClusterName("clusterName1")).metadata(metadata).routingTable(routingTable).build();
    BytesStreamOutput outStream = new BytesStreamOutput();
    outStream.setVersion(Version.CURRENT);
    clusterState1.writeTo(outStream);
    StreamInput inStream = new NamedWriteableAwareStreamInput(outStream.bytes().streamInput(), new NamedWriteableRegistry(ClusterModule.getNamedWriteables()));
    ClusterState serializedClusterState1 = ClusterState.readFrom(inStream, newNode("node4"));
    // Create a new, albeit equal, IndexMetadata object
    ClusterState clusterState2 = ClusterState.builder(clusterState1).incrementVersion().metadata(Metadata.builder().put(IndexMetadata.builder(indexMetadata).numberOfReplicas(1).build(), true)).build();
    assertNotSame("Should have created a new, equivalent, IndexMetadata object in clusterState2", clusterState1.metadata().index("test"), clusterState2.metadata().index("test"));
    ClusterState serializedClusterState2 = updateUsingSerialisedDiff(serializedClusterState1, clusterState2.diff(clusterState1));
    assertSame("Unchanged metadata should not create new IndexMetadata objects", serializedClusterState1.metadata().index("test"), serializedClusterState2.metadata().index("test"));
    assertSame("Unchanged routing table should not create new IndexRoutingTable objects", serializedClusterState1.routingTable().index("test"), serializedClusterState2.routingTable().index("test"));
    // Create a new and different IndexMetadata object
    ClusterState clusterState3 = ClusterState.builder(clusterState1).incrementVersion().metadata(Metadata.builder().put(IndexMetadata.builder(indexMetadata).numberOfReplicas(2).build(), true)).build();
    ClusterState serializedClusterState3 = updateUsingSerialisedDiff(serializedClusterState2, clusterState3.diff(clusterState2));
    assertNotEquals("Should have a new IndexMetadata object", serializedClusterState2.metadata().index("test"), serializedClusterState3.metadata().index("test"));
    assertSame("Unchanged routing table should not create new IndexRoutingTable objects", serializedClusterState2.routingTable().index("test"), serializedClusterState3.routingTable().index("test"));
    assertSame("nodes", serializedClusterState2.nodes(), serializedClusterState3.nodes());
    assertSame("blocks", serializedClusterState2.blocks(), serializedClusterState3.blocks());
    assertSame("template", serializedClusterState2.metadata().templates().get("test-template"), serializedClusterState3.metadata().templates().get("test-template"));
}

Example 12 with IndexTemplateMetadata

use of org.opensearch.cluster.metadata.IndexTemplateMetadata in project OpenSearch by opensearch-project.

the class IngestService method resolvePipelines.

public static boolean resolvePipelines(final DocWriteRequest<?> originalRequest, final IndexRequest indexRequest, final Metadata metadata) {
    if (indexRequest.isPipelineResolved() == false) {
        final String requestPipeline = indexRequest.getPipeline();
        indexRequest.setPipeline(NOOP_PIPELINE_NAME);
        indexRequest.setFinalPipeline(NOOP_PIPELINE_NAME);
        String defaultPipeline = null;
        String finalPipeline = null;
        IndexMetadata indexMetadata = null;
        // start to look for default or final pipelines via settings found in the index meta data
        if (originalRequest != null) {
            indexMetadata = metadata.indices().get(originalRequest.index());
        }
        // check the alias for the index request (this is how normal index requests are modeled)
        if (indexMetadata == null && indexRequest.index() != null) {
            IndexAbstraction indexAbstraction = metadata.getIndicesLookup().get(indexRequest.index());
            if (indexAbstraction != null) {
                indexMetadata = indexAbstraction.getWriteIndex();
            }
        }
        // check the alias for the action request (this is how upserts are modeled)
        if (indexMetadata == null && originalRequest != null && originalRequest.index() != null) {
            IndexAbstraction indexAbstraction = metadata.getIndicesLookup().get(originalRequest.index());
            if (indexAbstraction != null) {
                indexMetadata = indexAbstraction.getWriteIndex();
            }
        }
        if (indexMetadata != null) {
            final Settings indexSettings = indexMetadata.getSettings();
            if (IndexSettings.DEFAULT_PIPELINE.exists(indexSettings)) {
                // find the default pipeline if one is defined from an existing index setting
                defaultPipeline = IndexSettings.DEFAULT_PIPELINE.get(indexSettings);
                indexRequest.setPipeline(defaultPipeline);
            }
            if (IndexSettings.FINAL_PIPELINE.exists(indexSettings)) {
                // find the final pipeline if one is defined from an existing index setting
                finalPipeline = IndexSettings.FINAL_PIPELINE.get(indexSettings);
                indexRequest.setFinalPipeline(finalPipeline);
            }
        } else if (indexRequest.index() != null) {
            // the index does not exist yet (and this is a valid request), so match index
            // templates to look for pipelines in either a matching V2 template (which takes
            // precedence), or if a V2 template does not match, any V1 templates
            String v2Template = MetadataIndexTemplateService.findV2Template(metadata, indexRequest.index(), false);
            if (v2Template != null) {
                Settings settings = MetadataIndexTemplateService.resolveSettings(metadata, v2Template);
                if (IndexSettings.DEFAULT_PIPELINE.exists(settings)) {
                    defaultPipeline = IndexSettings.DEFAULT_PIPELINE.get(settings);
                // we can not break in case a lower-order template has a final pipeline that we need to collect
                }
                if (IndexSettings.FINAL_PIPELINE.exists(settings)) {
                    finalPipeline = IndexSettings.FINAL_PIPELINE.get(settings);
                // we can not break in case a lower-order template has a default pipeline that we need to collect
                }
                indexRequest.setPipeline(defaultPipeline != null ? defaultPipeline : NOOP_PIPELINE_NAME);
                indexRequest.setFinalPipeline(finalPipeline != null ? finalPipeline : NOOP_PIPELINE_NAME);
            } else {
                List<IndexTemplateMetadata> templates = MetadataIndexTemplateService.findV1Templates(metadata, indexRequest.index(), null);
                // order of templates are highest order first
                for (final IndexTemplateMetadata template : templates) {
                    final Settings settings = template.settings();
                    if (defaultPipeline == null && IndexSettings.DEFAULT_PIPELINE.exists(settings)) {
                        defaultPipeline = IndexSettings.DEFAULT_PIPELINE.get(settings);
                    // we can not break in case a lower-order template has a final pipeline that we need to collect
                    }
                    if (finalPipeline == null && IndexSettings.FINAL_PIPELINE.exists(settings)) {
                        finalPipeline = IndexSettings.FINAL_PIPELINE.get(settings);
                    // we can not break in case a lower-order template has a default pipeline that we need to collect
                    }
                    if (defaultPipeline != null && finalPipeline != null) {
                        // we can break if we have already collected a default and final pipeline
                        break;
                    }
                }
                indexRequest.setPipeline(defaultPipeline != null ? defaultPipeline : NOOP_PIPELINE_NAME);
                indexRequest.setFinalPipeline(finalPipeline != null ? finalPipeline : NOOP_PIPELINE_NAME);
            }
        }
        if (requestPipeline != null) {
            indexRequest.setPipeline(requestPipeline);
        }
        /*
             * We have to track whether or not the pipeline for this request has already been resolved. It can happen that the
             * pipeline for this request has already been derived yet we execute this loop again. That occurs if the bulk request
             * has been forwarded by a non-ingest coordinating node to an ingest node. In this case, the coordinating node will have
             * already resolved the pipeline for this request. It is important that we are able to distinguish this situation as we
             * can not double-resolve the pipeline because we will not be able to distinguish the case of the pipeline having been
             * set from a request pipeline parameter versus having been set by the resolution. We need to be able to distinguish
             * these cases as we need to reject the request if the pipeline was set by a required pipeline and there is a request
             * pipeline parameter too.
             */
        indexRequest.isPipelineResolved(true);
    }
    // return whether this index request has a pipeline
    return NOOP_PIPELINE_NAME.equals(indexRequest.getPipeline()) == false || NOOP_PIPELINE_NAME.equals(indexRequest.getFinalPipeline()) == false;
}

Example 13 with IndexTemplateMetadata

use of org.opensearch.cluster.metadata.IndexTemplateMetadata in project OpenSearch by opensearch-project.

the class OpenSearchAssertions method assertIndexTemplateExists.

/**
 * Assert that an index template exists
 */
public static void assertIndexTemplateExists(GetIndexTemplatesResponse templatesResponse, String name) {
    List<String> templateNames = new ArrayList<>();
    for (IndexTemplateMetadata indexTemplateMetadata : templatesResponse.getIndexTemplates()) {
        templateNames.add(indexTemplateMetadata.name());
    }
    assertThat(templateNames, hasItem(name));
}

Example 14 with IndexTemplateMetadata

use of org.opensearch.cluster.metadata.IndexTemplateMetadata in project OpenSearch by opensearch-project.

the class GatewayMetaStateTests method randomMetadataWithIndexTemplates.

private static Metadata randomMetadataWithIndexTemplates(String... templates) {
    Metadata.Builder builder = Metadata.builder();
    for (String template : templates) {
        IndexTemplateMetadata templateMetadata = IndexTemplateMetadata.builder(template).settings(settings(Version.CURRENT).put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), randomIntBetween(0, 3)).put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), randomIntBetween(1, 5))).patterns(randomIndexPatterns()).build();
        builder.put(templateMetadata);
    }
    for (int i = 0; i < randomIntBetween(1, 5); i++) {
        builder.put(IndexMetadata.builder(randomAlphaOfLength(10)).settings(settings(Version.CURRENT)).numberOfReplicas(randomIntBetween(0, 3)).numberOfShards(randomIntBetween(1, 5)));
    }
    return builder.build();
}