Examples with IndexNameExpressionResolver org.opensearch.cluster.metadata.IndexNameExpressionResolver used on opensource projects

Example 21 with IndexNameExpressionResolver

use of org.opensearch.cluster.metadata.IndexNameExpressionResolver in project OpenSearch by opensearch-project.

the class MetadataRolloverServiceTests method testValidation.

public void testValidation() throws Exception {
    final String rolloverTarget;
    final String sourceIndexName;
    final String defaultRolloverIndexName;
    final boolean useDataStream = randomBoolean();
    final Metadata.Builder builder = Metadata.builder();
    if (useDataStream) {
        DataStream dataStream = DataStreamTests.randomInstance();
        rolloverTarget = dataStream.getName();
        sourceIndexName = dataStream.getIndices().get(dataStream.getIndices().size() - 1).getName();
        defaultRolloverIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1);
        ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList(dataStream.getName() + "*"), null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate());
        builder.put("template", template);
        for (Index index : dataStream.getIndices()) {
            builder.put(DataStreamTestHelper.getIndexMetadataBuilderForIndex(index));
        }
        builder.put(dataStream);
    } else {
        String indexPrefix = "logs-index-00000";
        rolloverTarget = "logs-alias";
        sourceIndexName = indexPrefix + "1";
        defaultRolloverIndexName = indexPrefix + "2";
        final IndexMetadata.Builder indexMetadata = IndexMetadata.builder(sourceIndexName).putAlias(AliasMetadata.builder(rolloverTarget).writeIndex(true).build()).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1);
        builder.put(indexMetadata);
    }
    final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metadata(builder).build();
    MetadataCreateIndexService createIndexService = mock(MetadataCreateIndexService.class);
    MetadataIndexAliasesService metadataIndexAliasesService = mock(MetadataIndexAliasesService.class);
    IndexNameExpressionResolver mockIndexNameExpressionResolver = mock(IndexNameExpressionResolver.class);
    when(mockIndexNameExpressionResolver.resolveDateMathExpression(any())).then(returnsFirstArg());
    MetadataRolloverService rolloverService = new MetadataRolloverService(null, createIndexService, metadataIndexAliasesService, mockIndexNameExpressionResolver);
    String newIndexName = useDataStream == false && randomBoolean() ? "logs-index-9" : null;
    MetadataRolloverService.RolloverResult rolloverResult = rolloverService.rolloverClusterState(clusterState, rolloverTarget, newIndexName, new CreateIndexRequest("_na_"), null, randomBoolean(), true);
    newIndexName = newIndexName == null ? defaultRolloverIndexName : newIndexName;
    assertEquals(sourceIndexName, rolloverResult.sourceIndexName);
    assertEquals(newIndexName, rolloverResult.rolloverIndexName);
    assertSame(rolloverResult.clusterState, clusterState);
    verify(createIndexService).validateIndexName(any(), same(clusterState));
    verifyNoMoreInteractions(createIndexService);
    verifyNoMoreInteractions(metadataIndexAliasesService);
    reset(createIndexService);
    doThrow(new InvalidIndexNameException("test", "invalid")).when(createIndexService).validateIndexName(any(), any());
    expectThrows(InvalidIndexNameException.class, () -> rolloverService.rolloverClusterState(clusterState, rolloverTarget, null, new CreateIndexRequest("_na_"), null, randomBoolean(), randomBoolean()));
    verify(createIndexService).validateIndexName(any(), same(clusterState));
    verifyNoMoreInteractions(createIndexService);
    verifyNoMoreInteractions(metadataIndexAliasesService);
}

Example 22 with IndexNameExpressionResolver

use of org.opensearch.cluster.metadata.IndexNameExpressionResolver in project OpenSearch by opensearch-project.

the class MetadataRolloverServiceTests method testRolloverClusterStateForDataStreamNoTemplate.

public void testRolloverClusterStateForDataStreamNoTemplate() throws Exception {
    final DataStream dataStream = DataStreamTests.randomInstance();
    Metadata.Builder builder = Metadata.builder();
    for (Index index : dataStream.getIndices()) {
        builder.put(DataStreamTestHelper.getIndexMetadataBuilderForIndex(index));
    }
    builder.put(dataStream);
    final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metadata(builder).build();
    ThreadPool testThreadPool = mock(ThreadPool.class);
    ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool);
    Environment env = mock(Environment.class);
    AllocationService allocationService = mock(AllocationService.class);
    IndicesService indicesService = mockIndicesServices();
    IndexNameExpressionResolver mockIndexNameExpressionResolver = mock(IndexNameExpressionResolver.class);
    MetadataCreateIndexService createIndexService = new MetadataCreateIndexService(Settings.EMPTY, clusterService, indicesService, allocationService, null, null, env, null, testThreadPool, null, new SystemIndices(emptyMap()), false);
    MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService(clusterService, indicesService, new AliasValidator(), null, xContentRegistry());
    MetadataRolloverService rolloverService = new MetadataRolloverService(testThreadPool, createIndexService, indexAliasesService, mockIndexNameExpressionResolver);
    MaxDocsCondition condition = new MaxDocsCondition(randomNonNegativeLong());
    List<Condition<?>> metConditions = Collections.singletonList(condition);
    CreateIndexRequest createIndexRequest = new CreateIndexRequest("_na_");
    Exception e = expectThrows(IllegalArgumentException.class, () -> rolloverService.rolloverClusterState(clusterState, dataStream.getName(), null, createIndexRequest, metConditions, false, randomBoolean()));
    assertThat(e.getMessage(), equalTo("no matching index template found for data stream [" + dataStream.getName() + "]"));
}

Example 23 with IndexNameExpressionResolver

use of org.opensearch.cluster.metadata.IndexNameExpressionResolver in project OpenSearch by opensearch-project.

the class MetadataRolloverServiceTests method testRolloverClusterStateForDataStream.

public void testRolloverClusterStateForDataStream() throws Exception {
    final DataStream dataStream = DataStreamTests.randomInstance();
    ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList(dataStream.getName() + "*"), null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate());
    Metadata.Builder builder = Metadata.builder();
    builder.put("template", template);
    for (Index index : dataStream.getIndices()) {
        builder.put(DataStreamTestHelper.getIndexMetadataBuilderForIndex(index));
    }
    builder.put(dataStream);
    final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metadata(builder).build();
    ThreadPool testThreadPool = new TestThreadPool(getTestName());
    try {
        Mapper.BuilderContext builderContext = new Mapper.BuilderContext(Settings.EMPTY, new ContentPath(0));
        DateFieldMapper dateFieldMapper = new DateFieldMapper.Builder("@timestamp", DateFieldMapper.Resolution.MILLISECONDS, null, false, Version.CURRENT).build(builderContext);
        MetadataFieldMapper mockedTimestampField = mock(MetadataFieldMapper.class);
        when(mockedTimestampField.name()).thenReturn("_data_stream_timestamp");
        MappedFieldType mockedTimestampFieldType = mock(MappedFieldType.class);
        when(mockedTimestampFieldType.name()).thenReturn("_data_stream_timestamp");
        when(mockedTimestampField.fieldType()).thenReturn(mockedTimestampFieldType);
        when(mockedTimestampField.copyTo()).thenReturn(FieldMapper.CopyTo.empty());
        when(mockedTimestampField.multiFields()).thenReturn(FieldMapper.MultiFields.empty());
        MappingLookup mappingLookup = new MappingLookup(Arrays.asList(mockedTimestampField, dateFieldMapper), Collections.emptyList(), Collections.emptyList(), 0, new StandardAnalyzer());
        ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool);
        Environment env = mock(Environment.class);
        when(env.sharedDataDir()).thenReturn(null);
        AllocationService allocationService = mock(AllocationService.class);
        when(allocationService.reroute(any(ClusterState.class), any(String.class))).then(i -> i.getArguments()[0]);
        DocumentMapper documentMapper = mock(DocumentMapper.class);
        when(documentMapper.mappers()).thenReturn(mappingLookup);
        when(documentMapper.type()).thenReturn("_doc");
        CompressedXContent mapping = new CompressedXContent("{\"_doc\":" + generateMapping(dataStream.getTimeStampField().getName(), "date") + "}");
        when(documentMapper.mappingSource()).thenReturn(mapping);
        RoutingFieldMapper routingFieldMapper = mock(RoutingFieldMapper.class);
        when(routingFieldMapper.required()).thenReturn(false);
        when(documentMapper.routingFieldMapper()).thenReturn(routingFieldMapper);
        IndicesService indicesService = mockIndicesServices(documentMapper);
        IndexNameExpressionResolver mockIndexNameExpressionResolver = mock(IndexNameExpressionResolver.class);
        when(mockIndexNameExpressionResolver.resolveDateMathExpression(any())).then(returnsFirstArg());
        final SystemIndices systemIndices = new SystemIndices(emptyMap());
        ShardLimitValidator shardLimitValidator = new ShardLimitValidator(Settings.EMPTY, clusterService, systemIndices);
        MetadataCreateIndexService createIndexService = new MetadataCreateIndexService(Settings.EMPTY, clusterService, indicesService, allocationService, null, shardLimitValidator, env, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, testThreadPool, null, systemIndices, false);
        MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService(clusterService, indicesService, new AliasValidator(), null, xContentRegistry());
        MetadataRolloverService rolloverService = new MetadataRolloverService(testThreadPool, createIndexService, indexAliasesService, mockIndexNameExpressionResolver);
        MaxDocsCondition condition = new MaxDocsCondition(randomNonNegativeLong());
        List<Condition<?>> metConditions = Collections.singletonList(condition);
        CreateIndexRequest createIndexRequest = new CreateIndexRequest("_na_");
        long before = testThreadPool.absoluteTimeInMillis();
        MetadataRolloverService.RolloverResult rolloverResult = rolloverService.rolloverClusterState(clusterState, dataStream.getName(), null, createIndexRequest, metConditions, randomBoolean(), false);
        long after = testThreadPool.absoluteTimeInMillis();
        String sourceIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration());
        String newIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1);
        assertEquals(sourceIndexName, rolloverResult.sourceIndexName);
        assertEquals(newIndexName, rolloverResult.rolloverIndexName);
        Metadata rolloverMetadata = rolloverResult.clusterState.metadata();
        assertEquals(dataStream.getIndices().size() + 1, rolloverMetadata.indices().size());
        IndexMetadata rolloverIndexMetadata = rolloverMetadata.index(newIndexName);
        IndexAbstraction ds = rolloverMetadata.getIndicesLookup().get(dataStream.getName());
        assertThat(ds.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM));
        assertThat(ds.getIndices(), hasSize(dataStream.getIndices().size() + 1));
        assertThat(ds.getIndices(), hasItem(rolloverMetadata.index(sourceIndexName)));
        assertThat(ds.getIndices(), hasItem(rolloverIndexMetadata));
        assertThat(ds.getWriteIndex(), equalTo(rolloverIndexMetadata));
        RolloverInfo info = rolloverMetadata.index(sourceIndexName).getRolloverInfos().get(dataStream.getName());
        assertThat(info.getTime(), lessThanOrEqualTo(after));
        assertThat(info.getTime(), greaterThanOrEqualTo(before));
        assertThat(info.getMetConditions(), hasSize(1));
        assertThat(info.getMetConditions().get(0).value(), equalTo(condition.value()));
    } finally {
        testThreadPool.shutdown();
    }
}

Example 24 with IndexNameExpressionResolver

use of org.opensearch.cluster.metadata.IndexNameExpressionResolver in project OpenSearch by opensearch-project.

the class MetadataRolloverServiceTests method testGenerateRolloverIndexName.

public void testGenerateRolloverIndexName() {
    String invalidIndexName = randomAlphaOfLength(10) + "A";
    IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY));
    expectThrows(IllegalArgumentException.class, () -> MetadataRolloverService.generateRolloverIndexName(invalidIndexName, indexNameExpressionResolver));
    int num = randomIntBetween(0, 100);
    final String indexPrefix = randomAlphaOfLength(10);
    String indexEndingInNumbers = indexPrefix + "-" + num;
    assertThat(MetadataRolloverService.generateRolloverIndexName(indexEndingInNumbers, indexNameExpressionResolver), equalTo(indexPrefix + "-" + String.format(Locale.ROOT, "%06d", num + 1)));
    assertThat(MetadataRolloverService.generateRolloverIndexName("index-name-1", indexNameExpressionResolver), equalTo("index-name-000002"));
    assertThat(MetadataRolloverService.generateRolloverIndexName("index-name-2", indexNameExpressionResolver), equalTo("index-name-000003"));
    assertEquals("<index-name-{now/d}-000002>", MetadataRolloverService.generateRolloverIndexName("<index-name-{now/d}-1>", indexNameExpressionResolver));
}

Example 25 with IndexNameExpressionResolver

use of org.opensearch.cluster.metadata.IndexNameExpressionResolver in project OpenSearch by opensearch-project.

the class MetadataRolloverServiceTests method testRolloverClusterState.

/**
 * Test the main rolloverClusterState method. This does not validate every detail to depth, rather focuses on observing that each
 * parameter is used for the purpose intended.
 */
public void testRolloverClusterState() throws Exception {
    final String aliasName = "logs-alias";
    final String indexPrefix = "logs-index-00000";
    String sourceIndexName = indexPrefix + "1";
    final IndexMetadata.Builder indexMetadata = IndexMetadata.builder(sourceIndexName).putAlias(AliasMetadata.builder(aliasName).writeIndex(true).build()).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1);
    final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metadata(Metadata.builder().put(indexMetadata)).build();
    ThreadPool testThreadPool = new TestThreadPool(getTestName());
    try {
        ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool);
        Environment env = mock(Environment.class);
        when(env.sharedDataDir()).thenReturn(null);
        AllocationService allocationService = mock(AllocationService.class);
        when(allocationService.reroute(any(ClusterState.class), any(String.class))).then(i -> i.getArguments()[0]);
        IndicesService indicesService = mockIndicesServices();
        IndexNameExpressionResolver mockIndexNameExpressionResolver = mock(IndexNameExpressionResolver.class);
        when(mockIndexNameExpressionResolver.resolveDateMathExpression(any())).then(returnsFirstArg());
        final SystemIndices systemIndices = new SystemIndices(emptyMap());
        ShardLimitValidator shardLimitValidator = new ShardLimitValidator(Settings.EMPTY, clusterService, systemIndices);
        MetadataCreateIndexService createIndexService = new MetadataCreateIndexService(Settings.EMPTY, clusterService, indicesService, allocationService, null, shardLimitValidator, env, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, testThreadPool, null, systemIndices, false);
        MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService(clusterService, indicesService, new AliasValidator(), null, xContentRegistry());
        MetadataRolloverService rolloverService = new MetadataRolloverService(testThreadPool, createIndexService, indexAliasesService, mockIndexNameExpressionResolver);
        MaxDocsCondition condition = new MaxDocsCondition(randomNonNegativeLong());
        List<Condition<?>> metConditions = Collections.singletonList(condition);
        String newIndexName = randomBoolean() ? "logs-index-9" : null;
        int numberOfShards = randomIntBetween(1, 5);
        CreateIndexRequest createIndexRequest = new CreateIndexRequest("_na_");
        createIndexRequest.settings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards));
        long before = testThreadPool.absoluteTimeInMillis();
        MetadataRolloverService.RolloverResult rolloverResult = rolloverService.rolloverClusterState(clusterState, aliasName, newIndexName, createIndexRequest, metConditions, randomBoolean(), false);
        long after = testThreadPool.absoluteTimeInMillis();
        newIndexName = newIndexName == null ? indexPrefix + "2" : newIndexName;
        assertEquals(sourceIndexName, rolloverResult.sourceIndexName);
        assertEquals(newIndexName, rolloverResult.rolloverIndexName);
        Metadata rolloverMetadata = rolloverResult.clusterState.metadata();
        assertEquals(2, rolloverMetadata.indices().size());
        IndexMetadata rolloverIndexMetadata = rolloverMetadata.index(newIndexName);
        assertThat(rolloverIndexMetadata.getNumberOfShards(), equalTo(numberOfShards));
        IndexAbstraction alias = rolloverMetadata.getIndicesLookup().get(aliasName);
        assertThat(alias.getType(), equalTo(IndexAbstraction.Type.ALIAS));
        assertThat(alias.getIndices(), hasSize(2));
        assertThat(alias.getIndices(), hasItem(rolloverMetadata.index(sourceIndexName)));
        assertThat(alias.getIndices(), hasItem(rolloverIndexMetadata));
        assertThat(alias.getWriteIndex(), equalTo(rolloverIndexMetadata));
        RolloverInfo info = rolloverMetadata.index(sourceIndexName).getRolloverInfos().get(aliasName);
        assertThat(info.getTime(), lessThanOrEqualTo(after));
        assertThat(info.getTime(), greaterThanOrEqualTo(before));
        assertThat(info.getMetConditions(), hasSize(1));
        assertThat(info.getMetConditions().get(0).value(), equalTo(condition.value()));
    } finally {
        testThreadPool.shutdown();
    }
}