use of org.opensearch.cluster.metadata.MetadataIndexAliasesService in project OpenSearch by opensearch-project.
the class MetadataRolloverServiceTests method testRolloverClusterStateForDataStreamNoTemplate.
public void testRolloverClusterStateForDataStreamNoTemplate() throws Exception {
final DataStream dataStream = DataStreamTests.randomInstance();
Metadata.Builder builder = Metadata.builder();
for (Index index : dataStream.getIndices()) {
builder.put(DataStreamTestHelper.getIndexMetadataBuilderForIndex(index));
}
builder.put(dataStream);
final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metadata(builder).build();
ThreadPool testThreadPool = mock(ThreadPool.class);
ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool);
Environment env = mock(Environment.class);
AllocationService allocationService = mock(AllocationService.class);
IndicesService indicesService = mockIndicesServices();
IndexNameExpressionResolver mockIndexNameExpressionResolver = mock(IndexNameExpressionResolver.class);
MetadataCreateIndexService createIndexService = new MetadataCreateIndexService(Settings.EMPTY, clusterService, indicesService, allocationService, null, null, env, null, testThreadPool, null, new SystemIndices(emptyMap()), false);
MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService(clusterService, indicesService, new AliasValidator(), null, xContentRegistry());
MetadataRolloverService rolloverService = new MetadataRolloverService(testThreadPool, createIndexService, indexAliasesService, mockIndexNameExpressionResolver);
MaxDocsCondition condition = new MaxDocsCondition(randomNonNegativeLong());
List<Condition<?>> metConditions = Collections.singletonList(condition);
CreateIndexRequest createIndexRequest = new CreateIndexRequest("_na_");
Exception e = expectThrows(IllegalArgumentException.class, () -> rolloverService.rolloverClusterState(clusterState, dataStream.getName(), null, createIndexRequest, metConditions, false, randomBoolean()));
assertThat(e.getMessage(), equalTo("no matching index template found for data stream [" + dataStream.getName() + "]"));
}
use of org.opensearch.cluster.metadata.MetadataIndexAliasesService in project OpenSearch by opensearch-project.
the class MetadataRolloverServiceTests method testRolloverClusterState.
/**
* Test the main rolloverClusterState method. This does not validate every detail to depth, rather focuses on observing that each
* parameter is used for the purpose intended.
*/
public void testRolloverClusterState() throws Exception {
final String aliasName = "logs-alias";
final String indexPrefix = "logs-index-00000";
String sourceIndexName = indexPrefix + "1";
final IndexMetadata.Builder indexMetadata = IndexMetadata.builder(sourceIndexName).putAlias(AliasMetadata.builder(aliasName).writeIndex(true).build()).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1);
final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metadata(Metadata.builder().put(indexMetadata)).build();
ThreadPool testThreadPool = new TestThreadPool(getTestName());
try {
ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool);
Environment env = mock(Environment.class);
when(env.sharedDataFile()).thenReturn(null);
AllocationService allocationService = mock(AllocationService.class);
when(allocationService.reroute(any(ClusterState.class), any(String.class))).then(i -> i.getArguments()[0]);
IndicesService indicesService = mockIndicesServices();
IndexNameExpressionResolver mockIndexNameExpressionResolver = mock(IndexNameExpressionResolver.class);
when(mockIndexNameExpressionResolver.resolveDateMathExpression(any())).then(returnsFirstArg());
ShardLimitValidator shardLimitValidator = new ShardLimitValidator(Settings.EMPTY, clusterService);
MetadataCreateIndexService createIndexService = new MetadataCreateIndexService(Settings.EMPTY, clusterService, indicesService, allocationService, null, shardLimitValidator, env, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, testThreadPool, null, new SystemIndices(emptyMap()), false);
MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService(clusterService, indicesService, new AliasValidator(), null, xContentRegistry());
MetadataRolloverService rolloverService = new MetadataRolloverService(testThreadPool, createIndexService, indexAliasesService, mockIndexNameExpressionResolver);
MaxDocsCondition condition = new MaxDocsCondition(randomNonNegativeLong());
List<Condition<?>> metConditions = Collections.singletonList(condition);
String newIndexName = randomBoolean() ? "logs-index-9" : null;
int numberOfShards = randomIntBetween(1, 5);
CreateIndexRequest createIndexRequest = new CreateIndexRequest("_na_");
createIndexRequest.settings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards));
long before = testThreadPool.absoluteTimeInMillis();
MetadataRolloverService.RolloverResult rolloverResult = rolloverService.rolloverClusterState(clusterState, aliasName, newIndexName, createIndexRequest, metConditions, randomBoolean(), false);
long after = testThreadPool.absoluteTimeInMillis();
newIndexName = newIndexName == null ? indexPrefix + "2" : newIndexName;
assertEquals(sourceIndexName, rolloverResult.sourceIndexName);
assertEquals(newIndexName, rolloverResult.rolloverIndexName);
Metadata rolloverMetadata = rolloverResult.clusterState.metadata();
assertEquals(2, rolloverMetadata.indices().size());
IndexMetadata rolloverIndexMetadata = rolloverMetadata.index(newIndexName);
assertThat(rolloverIndexMetadata.getNumberOfShards(), equalTo(numberOfShards));
IndexAbstraction alias = rolloverMetadata.getIndicesLookup().get(aliasName);
assertThat(alias.getType(), equalTo(IndexAbstraction.Type.ALIAS));
assertThat(alias.getIndices(), hasSize(2));
assertThat(alias.getIndices(), hasItem(rolloverMetadata.index(sourceIndexName)));
assertThat(alias.getIndices(), hasItem(rolloverIndexMetadata));
assertThat(alias.getWriteIndex(), equalTo(rolloverIndexMetadata));
RolloverInfo info = rolloverMetadata.index(sourceIndexName).getRolloverInfos().get(aliasName);
assertThat(info.getTime(), lessThanOrEqualTo(after));
assertThat(info.getTime(), greaterThanOrEqualTo(before));
assertThat(info.getMetConditions(), hasSize(1));
assertThat(info.getMetConditions().get(0).value(), equalTo(condition.value()));
} finally {
testThreadPool.shutdown();
}
}
use of org.opensearch.cluster.metadata.MetadataIndexAliasesService in project OpenSearch by opensearch-project.
the class MetadataRolloverServiceTests method testValidation.
public void testValidation() throws Exception {
final String rolloverTarget;
final String sourceIndexName;
final String defaultRolloverIndexName;
final boolean useDataStream = randomBoolean();
final Metadata.Builder builder = Metadata.builder();
if (useDataStream) {
DataStream dataStream = DataStreamTests.randomInstance();
rolloverTarget = dataStream.getName();
sourceIndexName = dataStream.getIndices().get(dataStream.getIndices().size() - 1).getName();
defaultRolloverIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1);
ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList(dataStream.getName() + "*"), null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate());
builder.put("template", template);
for (Index index : dataStream.getIndices()) {
builder.put(DataStreamTestHelper.getIndexMetadataBuilderForIndex(index));
}
builder.put(dataStream);
} else {
String indexPrefix = "logs-index-00000";
rolloverTarget = "logs-alias";
sourceIndexName = indexPrefix + "1";
defaultRolloverIndexName = indexPrefix + "2";
final IndexMetadata.Builder indexMetadata = IndexMetadata.builder(sourceIndexName).putAlias(AliasMetadata.builder(rolloverTarget).writeIndex(true).build()).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1);
builder.put(indexMetadata);
}
final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metadata(builder).build();
MetadataCreateIndexService createIndexService = mock(MetadataCreateIndexService.class);
MetadataIndexAliasesService metadataIndexAliasesService = mock(MetadataIndexAliasesService.class);
IndexNameExpressionResolver mockIndexNameExpressionResolver = mock(IndexNameExpressionResolver.class);
when(mockIndexNameExpressionResolver.resolveDateMathExpression(any())).then(returnsFirstArg());
MetadataRolloverService rolloverService = new MetadataRolloverService(null, createIndexService, metadataIndexAliasesService, mockIndexNameExpressionResolver);
String newIndexName = useDataStream == false && randomBoolean() ? "logs-index-9" : null;
MetadataRolloverService.RolloverResult rolloverResult = rolloverService.rolloverClusterState(clusterState, rolloverTarget, newIndexName, new CreateIndexRequest("_na_"), null, randomBoolean(), true);
newIndexName = newIndexName == null ? defaultRolloverIndexName : newIndexName;
assertEquals(sourceIndexName, rolloverResult.sourceIndexName);
assertEquals(newIndexName, rolloverResult.rolloverIndexName);
assertSame(rolloverResult.clusterState, clusterState);
verify(createIndexService).validateIndexName(any(), same(clusterState));
verifyNoMoreInteractions(createIndexService);
verifyNoMoreInteractions(metadataIndexAliasesService);
reset(createIndexService);
doThrow(new InvalidIndexNameException("test", "invalid")).when(createIndexService).validateIndexName(any(), any());
expectThrows(InvalidIndexNameException.class, () -> rolloverService.rolloverClusterState(clusterState, rolloverTarget, null, new CreateIndexRequest("_na_"), null, randomBoolean(), randomBoolean()));
verify(createIndexService).validateIndexName(any(), same(clusterState));
verifyNoMoreInteractions(createIndexService);
verifyNoMoreInteractions(metadataIndexAliasesService);
}
use of org.opensearch.cluster.metadata.MetadataIndexAliasesService in project OpenSearch by opensearch-project.
the class MetadataRolloverServiceTests method testRolloverClusterStateForDataStream.
public void testRolloverClusterStateForDataStream() throws Exception {
final DataStream dataStream = DataStreamTests.randomInstance();
ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList(dataStream.getName() + "*"), null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate());
Metadata.Builder builder = Metadata.builder();
builder.put("template", template);
for (Index index : dataStream.getIndices()) {
builder.put(DataStreamTestHelper.getIndexMetadataBuilderForIndex(index));
}
builder.put(dataStream);
final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metadata(builder).build();
ThreadPool testThreadPool = new TestThreadPool(getTestName());
try {
Mapper.BuilderContext builderContext = new Mapper.BuilderContext(Settings.EMPTY, new ContentPath(0));
DateFieldMapper dateFieldMapper = new DateFieldMapper.Builder("@timestamp", DateFieldMapper.Resolution.MILLISECONDS, null, false, Version.CURRENT).build(builderContext);
MetadataFieldMapper mockedTimestampField = mock(MetadataFieldMapper.class);
when(mockedTimestampField.name()).thenReturn("_data_stream_timestamp");
MappedFieldType mockedTimestampFieldType = mock(MappedFieldType.class);
when(mockedTimestampFieldType.name()).thenReturn("_data_stream_timestamp");
when(mockedTimestampField.fieldType()).thenReturn(mockedTimestampFieldType);
when(mockedTimestampField.copyTo()).thenReturn(FieldMapper.CopyTo.empty());
when(mockedTimestampField.multiFields()).thenReturn(FieldMapper.MultiFields.empty());
MappingLookup mappingLookup = new MappingLookup(Arrays.asList(mockedTimestampField, dateFieldMapper), Collections.emptyList(), Collections.emptyList(), 0, new StandardAnalyzer());
ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool);
Environment env = mock(Environment.class);
when(env.sharedDataFile()).thenReturn(null);
AllocationService allocationService = mock(AllocationService.class);
when(allocationService.reroute(any(ClusterState.class), any(String.class))).then(i -> i.getArguments()[0]);
DocumentMapper documentMapper = mock(DocumentMapper.class);
when(documentMapper.mappers()).thenReturn(mappingLookup);
when(documentMapper.type()).thenReturn("_doc");
CompressedXContent mapping = new CompressedXContent("{\"_doc\":" + generateMapping(dataStream.getTimeStampField().getName(), "date") + "}");
when(documentMapper.mappingSource()).thenReturn(mapping);
RoutingFieldMapper routingFieldMapper = mock(RoutingFieldMapper.class);
when(routingFieldMapper.required()).thenReturn(false);
when(documentMapper.routingFieldMapper()).thenReturn(routingFieldMapper);
IndicesService indicesService = mockIndicesServices(documentMapper);
IndexNameExpressionResolver mockIndexNameExpressionResolver = mock(IndexNameExpressionResolver.class);
when(mockIndexNameExpressionResolver.resolveDateMathExpression(any())).then(returnsFirstArg());
ShardLimitValidator shardLimitValidator = new ShardLimitValidator(Settings.EMPTY, clusterService);
MetadataCreateIndexService createIndexService = new MetadataCreateIndexService(Settings.EMPTY, clusterService, indicesService, allocationService, null, shardLimitValidator, env, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, testThreadPool, null, new SystemIndices(emptyMap()), false);
MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService(clusterService, indicesService, new AliasValidator(), null, xContentRegistry());
MetadataRolloverService rolloverService = new MetadataRolloverService(testThreadPool, createIndexService, indexAliasesService, mockIndexNameExpressionResolver);
MaxDocsCondition condition = new MaxDocsCondition(randomNonNegativeLong());
List<Condition<?>> metConditions = Collections.singletonList(condition);
CreateIndexRequest createIndexRequest = new CreateIndexRequest("_na_");
long before = testThreadPool.absoluteTimeInMillis();
MetadataRolloverService.RolloverResult rolloverResult = rolloverService.rolloverClusterState(clusterState, dataStream.getName(), null, createIndexRequest, metConditions, randomBoolean(), false);
long after = testThreadPool.absoluteTimeInMillis();
String sourceIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration());
String newIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1);
assertEquals(sourceIndexName, rolloverResult.sourceIndexName);
assertEquals(newIndexName, rolloverResult.rolloverIndexName);
Metadata rolloverMetadata = rolloverResult.clusterState.metadata();
assertEquals(dataStream.getIndices().size() + 1, rolloverMetadata.indices().size());
IndexMetadata rolloverIndexMetadata = rolloverMetadata.index(newIndexName);
IndexAbstraction ds = rolloverMetadata.getIndicesLookup().get(dataStream.getName());
assertThat(ds.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM));
assertThat(ds.getIndices(), hasSize(dataStream.getIndices().size() + 1));
assertThat(ds.getIndices(), hasItem(rolloverMetadata.index(sourceIndexName)));
assertThat(ds.getIndices(), hasItem(rolloverIndexMetadata));
assertThat(ds.getWriteIndex(), equalTo(rolloverIndexMetadata));
RolloverInfo info = rolloverMetadata.index(sourceIndexName).getRolloverInfos().get(dataStream.getName());
assertThat(info.getTime(), lessThanOrEqualTo(after));
assertThat(info.getTime(), greaterThanOrEqualTo(before));
assertThat(info.getMetConditions(), hasSize(1));
assertThat(info.getMetConditions().get(0).value(), equalTo(condition.value()));
} finally {
testThreadPool.shutdown();
}
}
use of org.opensearch.cluster.metadata.MetadataIndexAliasesService in project OpenSearch by opensearch-project.
the class TransportRolloverActionTests method testConditionEvaluationWhenAliasToWriteAndReadIndicesConsidersOnlyPrimariesFromWriteIndex.
public void testConditionEvaluationWhenAliasToWriteAndReadIndicesConsidersOnlyPrimariesFromWriteIndex() throws Exception {
final TransportService mockTransportService = mock(TransportService.class);
final ClusterService mockClusterService = mock(ClusterService.class);
final DiscoveryNode mockNode = mock(DiscoveryNode.class);
when(mockNode.getId()).thenReturn("mocknode");
when(mockClusterService.localNode()).thenReturn(mockNode);
final ThreadPool mockThreadPool = mock(ThreadPool.class);
final MetadataCreateIndexService mockCreateIndexService = mock(MetadataCreateIndexService.class);
final IndexNameExpressionResolver mockIndexNameExpressionResolver = mock(IndexNameExpressionResolver.class);
when(mockIndexNameExpressionResolver.resolveDateMathExpression(any())).thenReturn("logs-index-000003");
final ActionFilters mockActionFilters = mock(ActionFilters.class);
final MetadataIndexAliasesService mdIndexAliasesService = mock(MetadataIndexAliasesService.class);
final Client mockClient = mock(Client.class);
final Map<String, IndexStats> indexStats = new HashMap<>();
int total = randomIntBetween(500, 1000);
indexStats.put("logs-index-000001", createIndexStats(200L, total));
indexStats.put("logs-index-000002", createIndexStats(300L, total));
final IndicesStatsResponse statsResponse = createAliasToMultipleIndicesStatsResponse(indexStats);
doAnswer(invocation -> {
Object[] args = invocation.getArguments();
assert args.length == 3;
ActionListener<IndicesStatsResponse> listener = (ActionListener<IndicesStatsResponse>) args[2];
listener.onResponse(statsResponse);
return null;
}).when(mockClient).execute(eq(IndicesStatsAction.INSTANCE), any(ActionRequest.class), any(ActionListener.class));
assert statsResponse.getPrimaries().getDocs().getCount() == 500L;
assert statsResponse.getTotal().getDocs().getCount() == (total + total);
final IndexMetadata.Builder indexMetadata = IndexMetadata.builder("logs-index-000001").putAlias(AliasMetadata.builder("logs-alias").writeIndex(false).build()).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1);
final IndexMetadata.Builder indexMetadata2 = IndexMetadata.builder("logs-index-000002").putAlias(AliasMetadata.builder("logs-alias").writeIndex(true).build()).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1);
final ClusterState stateBefore = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder().put(indexMetadata).put(indexMetadata2)).build();
when(mockCreateIndexService.applyCreateIndexRequest(any(), any(), anyBoolean())).thenReturn(stateBefore);
when(mdIndexAliasesService.applyAliasActions(any(), any())).thenReturn(stateBefore);
MetadataRolloverService rolloverService = new MetadataRolloverService(mockThreadPool, mockCreateIndexService, mdIndexAliasesService, mockIndexNameExpressionResolver);
final TransportRolloverAction transportRolloverAction = new TransportRolloverAction(mockTransportService, mockClusterService, mockThreadPool, mockActionFilters, mockIndexNameExpressionResolver, rolloverService, mockClient);
// For given alias, verify that condition evaluation fails when the condition doc count is greater than the primaries doc count
// (primaries from only write index is considered)
PlainActionFuture<RolloverResponse> future = new PlainActionFuture<>();
RolloverRequest rolloverRequest = new RolloverRequest("logs-alias", "logs-index-000003");
rolloverRequest.addMaxIndexDocsCondition(500L);
rolloverRequest.dryRun(true);
transportRolloverAction.masterOperation(mock(Task.class), rolloverRequest, stateBefore, future);
RolloverResponse response = future.actionGet();
assertThat(response.getOldIndex(), equalTo("logs-index-000002"));
assertThat(response.getNewIndex(), equalTo("logs-index-000003"));
assertThat(response.isDryRun(), equalTo(true));
assertThat(response.isRolledOver(), equalTo(false));
assertThat(response.getConditionStatus().size(), equalTo(1));
assertThat(response.getConditionStatus().get("[max_docs: 500]"), is(false));
// For given alias, verify that the condition evaluation is successful when condition doc count is less than the primaries doc count
// (primaries from only write index is considered)
future = new PlainActionFuture<>();
rolloverRequest = new RolloverRequest("logs-alias", "logs-index-000003");
rolloverRequest.addMaxIndexDocsCondition(300L);
rolloverRequest.dryRun(true);
transportRolloverAction.masterOperation(mock(Task.class), rolloverRequest, stateBefore, future);
response = future.actionGet();
assertThat(response.getOldIndex(), equalTo("logs-index-000002"));
assertThat(response.getNewIndex(), equalTo("logs-index-000003"));
assertThat(response.isDryRun(), equalTo(true));
assertThat(response.isRolledOver(), equalTo(false));
assertThat(response.getConditionStatus().size(), equalTo(1));
assertThat(response.getConditionStatus().get("[max_docs: 300]"), is(true));
}
Aggregations