use of org.opensearch.indices.SystemIndices in project OpenSearch by opensearch-project.
the class TransportBulkAction method doInternalExecute.
protected void doInternalExecute(Task task, BulkRequest bulkRequest, String executorName, ActionListener<BulkResponse> listener) {
final long startTime = relativeTime();
final AtomicArray<BulkItemResponse> responses = new AtomicArray<>(bulkRequest.requests.size());
boolean hasIndexRequestsWithPipelines = false;
final Metadata metadata = clusterService.state().getMetadata();
final Version minNodeVersion = clusterService.state().getNodes().getMinNodeVersion();
for (DocWriteRequest<?> actionRequest : bulkRequest.requests) {
IndexRequest indexRequest = getIndexWriteRequest(actionRequest);
if (indexRequest != null) {
// Each index request needs to be evaluated, because this method also modifies the IndexRequest
boolean indexRequestHasPipeline = IngestService.resolvePipelines(actionRequest, indexRequest, metadata);
hasIndexRequestsWithPipelines |= indexRequestHasPipeline;
}
if (actionRequest instanceof IndexRequest) {
IndexRequest ir = (IndexRequest) actionRequest;
ir.checkAutoIdWithOpTypeCreateSupportedByVersion(minNodeVersion);
if (ir.getAutoGeneratedTimestamp() != IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP) {
throw new IllegalArgumentException("autoGeneratedTimestamp should not be set externally");
}
}
}
if (hasIndexRequestsWithPipelines) {
// this path is never taken.
try {
if (Assertions.ENABLED) {
final boolean arePipelinesResolved = bulkRequest.requests().stream().map(TransportBulkAction::getIndexWriteRequest).filter(Objects::nonNull).allMatch(IndexRequest::isPipelineResolved);
assert arePipelinesResolved : bulkRequest;
}
if (clusterService.localNode().isIngestNode()) {
processBulkIndexIngestRequest(task, bulkRequest, executorName, listener);
} else {
ingestForwarder.forwardIngestRequest(BulkAction.INSTANCE, bulkRequest, listener);
}
} catch (Exception e) {
listener.onFailure(e);
}
return;
}
final boolean includesSystem = includesSystem(bulkRequest, clusterService.state().metadata().getIndicesLookup(), systemIndices);
if (includesSystem || needToCheck()) {
// Attempt to create all the indices that we're going to need during the bulk before we start.
// Step 1: collect all the indices in the request
final Map<String, Boolean> indices = bulkRequest.requests.stream().filter(request -> request.opType() != DocWriteRequest.OpType.DELETE || request.versionType() == VersionType.EXTERNAL || request.versionType() == VersionType.EXTERNAL_GTE).collect(Collectors.toMap(DocWriteRequest::index, DocWriteRequest::isRequireAlias, (v1, v2) -> v1 || v2));
/* Step 2: filter that to indices that don't exist and we can create. At the same time build a map of indices we can't create
* that we'll use when we try to run the requests. */
final Map<String, IndexNotFoundException> indicesThatCannotBeCreated = new HashMap<>();
Set<String> autoCreateIndices = new HashSet<>();
ClusterState state = clusterService.state();
for (Map.Entry<String, Boolean> indexAndFlag : indices.entrySet()) {
boolean shouldAutoCreate;
final String index = indexAndFlag.getKey();
try {
shouldAutoCreate = shouldAutoCreate(index, state);
} catch (IndexNotFoundException e) {
shouldAutoCreate = false;
indicesThatCannotBeCreated.put(index, e);
}
// We should only auto create if we are not requiring it to be an alias
if (shouldAutoCreate && (indexAndFlag.getValue() == false)) {
autoCreateIndices.add(index);
}
}
// Step 3: create all the indices that are missing, if there are any missing. start the bulk after all the creates come back.
if (autoCreateIndices.isEmpty()) {
executeBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated);
} else {
final AtomicInteger counter = new AtomicInteger(autoCreateIndices.size());
for (String index : autoCreateIndices) {
createIndex(index, bulkRequest.timeout(), minNodeVersion, new ActionListener<CreateIndexResponse>() {
@Override
public void onResponse(CreateIndexResponse result) {
if (counter.decrementAndGet() == 0) {
threadPool.executor(executorName).execute(new ActionRunnable<BulkResponse>(listener) {
@Override
protected void doRun() {
executeBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated);
}
});
}
}
@Override
public void onFailure(Exception e) {
if (!(ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException)) {
// fail all requests involving this index, if create didn't work
for (int i = 0; i < bulkRequest.requests.size(); i++) {
DocWriteRequest<?> request = bulkRequest.requests.get(i);
if (request != null && setResponseFailureIfIndexMatches(responses, i, request, index, e)) {
bulkRequest.requests.set(i, null);
}
}
}
if (counter.decrementAndGet() == 0) {
final ActionListener<BulkResponse> wrappedListener = ActionListener.wrap(listener::onResponse, inner -> {
inner.addSuppressed(e);
listener.onFailure(inner);
});
threadPool.executor(executorName).execute(new ActionRunnable<BulkResponse>(wrappedListener) {
@Override
protected void doRun() {
executeBulk(task, bulkRequest, startTime, wrappedListener, responses, indicesThatCannotBeCreated);
}
@Override
public void onRejection(Exception rejectedException) {
rejectedException.addSuppressed(e);
super.onRejection(rejectedException);
}
});
}
}
});
}
}
} else {
executeBulk(task, bulkRequest, startTime, listener, responses, emptyMap());
}
}
use of org.opensearch.indices.SystemIndices in project OpenSearch by opensearch-project.
the class TransportBulkActionTests method testOnlySystem.
public void testOnlySystem() {
SortedMap<String, IndexAbstraction> indicesLookup = new TreeMap<>();
Settings settings = Settings.builder().put("index.version.created", Version.CURRENT).build();
indicesLookup.put(".foo", new Index(IndexMetadata.builder(".foo").settings(settings).system(true).numberOfShards(1).numberOfReplicas(0).build()));
indicesLookup.put(".bar", new Index(IndexMetadata.builder(".bar").settings(settings).system(true).numberOfShards(1).numberOfReplicas(0).build()));
SystemIndices systemIndices = new SystemIndices(singletonMap("plugin", singletonList(new SystemIndexDescriptor(".test", ""))));
List<String> onlySystem = Arrays.asList(".foo", ".bar");
assertTrue(bulkAction.isOnlySystem(buildBulkRequest(onlySystem), indicesLookup, systemIndices));
onlySystem = Arrays.asList(".foo", ".bar", ".test");
assertTrue(bulkAction.isOnlySystem(buildBulkRequest(onlySystem), indicesLookup, systemIndices));
List<String> nonSystem = Arrays.asList("foo", "bar");
assertFalse(bulkAction.isOnlySystem(buildBulkRequest(nonSystem), indicesLookup, systemIndices));
List<String> mixed = Arrays.asList(".foo", ".test", "other");
assertFalse(bulkAction.isOnlySystem(buildBulkRequest(mixed), indicesLookup, systemIndices));
}
use of org.opensearch.indices.SystemIndices in project OpenSearch by opensearch-project.
the class TransportBulkActionIndicesThatCannotBeCreatedTests method indicesThatCannotBeCreatedTestCase.
private void indicesThatCannotBeCreatedTestCase(Set<String> expected, BulkRequest bulkRequest, Function<String, Boolean> shouldAutoCreate) {
ClusterService clusterService = mock(ClusterService.class);
ClusterState state = mock(ClusterState.class);
when(state.getMetadata()).thenReturn(Metadata.EMPTY_METADATA);
when(state.metadata()).thenReturn(Metadata.EMPTY_METADATA);
when(clusterService.state()).thenReturn(state);
DiscoveryNodes discoveryNodes = mock(DiscoveryNodes.class);
when(state.getNodes()).thenReturn(discoveryNodes);
when(discoveryNodes.getMinNodeVersion()).thenReturn(VersionUtils.randomCompatibleVersion(random(), Version.CURRENT));
DiscoveryNode localNode = mock(DiscoveryNode.class);
when(clusterService.localNode()).thenReturn(localNode);
when(localNode.isIngestNode()).thenReturn(randomBoolean());
final ThreadPool threadPool = mock(ThreadPool.class);
final ExecutorService direct = OpenSearchExecutors.newDirectExecutorService();
when(threadPool.executor(anyString())).thenReturn(direct);
TransportBulkAction action = new TransportBulkAction(threadPool, mock(TransportService.class), clusterService, null, null, null, mock(ActionFilters.class), null, null, new IndexingPressureService(Settings.EMPTY, new ClusterService(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null)), new SystemIndices(emptyMap())) {
@Override
void executeBulk(Task task, BulkRequest bulkRequest, long startTimeNanos, ActionListener<BulkResponse> listener, AtomicArray<BulkItemResponse> responses, Map<String, IndexNotFoundException> indicesThatCannotBeCreated) {
assertEquals(expected, indicesThatCannotBeCreated.keySet());
}
@Override
boolean needToCheck() {
// Use "null" to mean "no indices can be created so don't bother checking"
return null != shouldAutoCreate;
}
@Override
boolean shouldAutoCreate(String index, ClusterState state) {
return shouldAutoCreate.apply(index);
}
@Override
void createIndex(String index, TimeValue timeout, Version minNodeVersion, ActionListener<CreateIndexResponse> listener) {
// If we try to create an index just immediately assume it worked
listener.onResponse(new CreateIndexResponse(true, true, index) {
});
}
};
action.doExecute(null, bulkRequest, null);
}
use of org.opensearch.indices.SystemIndices in project OpenSearch by opensearch-project.
the class MetadataCreateIndexServiceTests method testIndexLifecycleNameSetting.
public void testIndexLifecycleNameSetting() {
// see: https://github.com/opensearch-project/OpenSearch/issues/1019
final Settings ilnSetting = Settings.builder().put("index.lifecycle.name", "dummy").build();
withTemporaryClusterService(((clusterService, threadPool) -> {
MetadataCreateIndexService checkerService = new MetadataCreateIndexService(Settings.EMPTY, clusterService, null, null, null, createTestShardLimitService(randomIntBetween(1, 1000), clusterService), new Environment(Settings.builder().put("path.home", "dummy").build(), null), new IndexScopedSettings(ilnSetting, Collections.emptySet()), threadPool, null, new SystemIndices(Collections.emptyMap()), true);
final List<String> validationErrors = checkerService.getIndexSettingsValidationErrors(ilnSetting, true);
assertThat(validationErrors.size(), is(1));
assertThat(validationErrors.get(0), is("expected [index.lifecycle.name] to be private but it was not"));
}));
}
use of org.opensearch.indices.SystemIndices in project OpenSearch by opensearch-project.
the class TransportGetAliasesActionTests method testDeprecationWarningEmittedWhenRequestingNonExistingAliasInSystemPattern.
public void testDeprecationWarningEmittedWhenRequestingNonExistingAliasInSystemPattern() {
ClusterState state = systemIndexTestClusterState();
SystemIndices systemIndices = new SystemIndices(Collections.singletonMap(this.getTestName(), Collections.singletonList(new SystemIndexDescriptor(".y", "an index that doesn't exist"))));
GetAliasesRequest request = new GetAliasesRequest(".y");
ImmutableOpenMap<String, List<AliasMetadata>> aliases = ImmutableOpenMap.<String, List<AliasMetadata>>builder().build();
final String[] concreteIndices = {};
assertEquals(state.metadata().findAliases(request, concreteIndices), aliases);
ImmutableOpenMap<String, List<AliasMetadata>> result = TransportGetAliasesAction.postProcess(request, concreteIndices, aliases, state, false, systemIndices);
assertThat(result.size(), equalTo(0));
assertWarnings("this request accesses aliases with names reserved for system indices: [.y], but in a future major version, direct" + "access to system indices and their aliases will not be allowed");
}
Aggregations