Search in sources :

Example 1 with PutIndexTemplateRequest

use of org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest in project OpenSearch by opensearch-project.

the class RestPutIndexTemplateAction method prepareRequest.

@Override
public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
    PutIndexTemplateRequest putRequest = new PutIndexTemplateRequest(request.param("name"));
    if (request.hasParam("template")) {
        deprecationLogger.deprecate("put_index_template_deprecated_parameter", "Deprecated parameter [template] used, replaced by [index_patterns]");
        putRequest.patterns(Collections.singletonList(request.param("template")));
    } else {
        putRequest.patterns(Arrays.asList(request.paramAsStringArray("index_patterns", Strings.EMPTY_ARRAY)));
    }
    putRequest.order(request.paramAsInt("order", putRequest.order()));
    putRequest.masterNodeTimeout(request.paramAsTime("master_timeout", putRequest.masterNodeTimeout()));
    putRequest.create(request.paramAsBoolean("create", false));
    putRequest.cause(request.param("cause", ""));
    Map<String, Object> sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2();
    sourceAsMap = RestCreateIndexAction.prepareMappings(sourceAsMap);
    putRequest.source(sourceAsMap);
    return channel -> client.admin().indices().putTemplate(putRequest, new RestToXContentListener<>(channel));
}
Also used : POST(org.opensearch.rest.RestRequest.Method.POST) Arrays(java.util.Arrays) NodeClient(org.opensearch.client.node.NodeClient) Collections.unmodifiableList(java.util.Collections.unmodifiableList) RestRequest(org.opensearch.rest.RestRequest) PutIndexTemplateRequest(org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest) IOException(java.io.IOException) Strings(org.opensearch.common.Strings) DeprecationLogger(org.opensearch.common.logging.DeprecationLogger) XContentHelper(org.opensearch.common.xcontent.XContentHelper) List(java.util.List) RestToXContentListener(org.opensearch.rest.action.RestToXContentListener) Arrays.asList(java.util.Arrays.asList) Map(java.util.Map) BaseRestHandler(org.opensearch.rest.BaseRestHandler) PUT(org.opensearch.rest.RestRequest.Method.PUT) Collections(java.util.Collections) PutIndexTemplateRequest(org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest)

Example 2 with PutIndexTemplateRequest

use of org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest in project OpenSearch by opensearch-project.

the class TemplateUpgradeServiceTests method testUpdateTemplates.

@SuppressWarnings("unchecked")
public void testUpdateTemplates() {
    int additionsCount = randomIntBetween(0, 5);
    int deletionsCount = randomIntBetween(0, 3);
    List<ActionListener<AcknowledgedResponse>> putTemplateListeners = new ArrayList<>();
    List<ActionListener<AcknowledgedResponse>> deleteTemplateListeners = new ArrayList<>();
    Client mockClient = mock(Client.class);
    AdminClient mockAdminClient = mock(AdminClient.class);
    IndicesAdminClient mockIndicesAdminClient = mock(IndicesAdminClient.class);
    when(mockClient.admin()).thenReturn(mockAdminClient);
    when(mockAdminClient.indices()).thenReturn(mockIndicesAdminClient);
    doAnswer(invocation -> {
        Object[] args = invocation.getArguments();
        assert args.length == 2;
        PutIndexTemplateRequest request = (PutIndexTemplateRequest) args[0];
        assertThat(request.name(), equalTo("add_template_" + request.order()));
        putTemplateListeners.add((ActionListener) args[1]);
        return null;
    }).when(mockIndicesAdminClient).putTemplate(any(PutIndexTemplateRequest.class), any(ActionListener.class));
    doAnswer(invocation -> {
        Object[] args = invocation.getArguments();
        assert args.length == 2;
        DeleteIndexTemplateRequest request = (DeleteIndexTemplateRequest) args[0];
        assertThat(request.name(), startsWith("remove_template_"));
        deleteTemplateListeners.add((ActionListener) args[1]);
        return null;
    }).when(mockIndicesAdminClient).deleteTemplate(any(DeleteIndexTemplateRequest.class), any(ActionListener.class));
    Set<String> deletions = new HashSet<>(deletionsCount);
    for (int i = 0; i < deletionsCount; i++) {
        deletions.add("remove_template_" + i);
    }
    Map<String, BytesReference> additions = new HashMap<>(additionsCount);
    for (int i = 0; i < additionsCount; i++) {
        additions.put("add_template_" + i, new BytesArray("{\"index_patterns\" : \"*\", \"order\" : " + i + "}"));
    }
    final TemplateUpgradeService service = new TemplateUpgradeService(mockClient, clusterService, threadPool, Collections.emptyList());
    IllegalStateException ise = expectThrows(IllegalStateException.class, () -> service.upgradeTemplates(additions, deletions));
    assertThat(ise.getMessage(), containsString("template upgrade service should always happen in a system context"));
    // +2 to skip tryFinishUpgrade
    service.upgradesInProgress.set(additionsCount + deletionsCount + 2);
    final ThreadContext threadContext = threadPool.getThreadContext();
    try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
        threadContext.markAsSystemContext();
        service.upgradeTemplates(additions, deletions);
    }
    assertThat(putTemplateListeners, hasSize(additionsCount));
    assertThat(deleteTemplateListeners, hasSize(deletionsCount));
    for (int i = 0; i < additionsCount; i++) {
        if (randomBoolean()) {
            putTemplateListeners.get(i).onFailure(new RuntimeException("test - ignore"));
        } else {
            putTemplateListeners.get(i).onResponse(new AcknowledgedResponse(randomBoolean()) {
            });
        }
    }
    for (int i = 0; i < deletionsCount; i++) {
        if (randomBoolean()) {
            int prevUpdatesInProgress = service.upgradesInProgress.get();
            deleteTemplateListeners.get(i).onFailure(new RuntimeException("test - ignore"));
            assertThat(prevUpdatesInProgress - service.upgradesInProgress.get(), equalTo(1));
        } else {
            int prevUpdatesInProgress = service.upgradesInProgress.get();
            deleteTemplateListeners.get(i).onResponse(new AcknowledgedResponse(randomBoolean()) {
            });
            assertThat(prevUpdatesInProgress - service.upgradesInProgress.get(), equalTo(1));
        }
    }
    // tryFinishUpgrade was skipped
    assertThat(service.upgradesInProgress.get(), equalTo(2));
}
Also used : BytesReference(org.opensearch.common.bytes.BytesReference) BytesArray(org.opensearch.common.bytes.BytesArray) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) ThreadContext(org.opensearch.common.util.concurrent.ThreadContext) AcknowledgedResponse(org.opensearch.action.support.master.AcknowledgedResponse) PutIndexTemplateRequest(org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest) IndicesAdminClient(org.opensearch.client.IndicesAdminClient) Matchers.containsString(org.hamcrest.Matchers.containsString) DeleteIndexTemplateRequest(org.opensearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest) ActionListener(org.opensearch.action.ActionListener) Client(org.opensearch.client.Client) AdminClient(org.opensearch.client.AdminClient) IndicesAdminClient(org.opensearch.client.IndicesAdminClient) AdminClient(org.opensearch.client.AdminClient) IndicesAdminClient(org.opensearch.client.IndicesAdminClient) HashSet(java.util.HashSet)

Example 3 with PutIndexTemplateRequest

use of org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest in project OpenSearch by opensearch-project.

the class TemplateUpgradeServiceTests method testClusterStateUpdate.

@SuppressWarnings("unchecked")
public void testClusterStateUpdate() throws InterruptedException {
    final AtomicReference<ActionListener<AcknowledgedResponse>> addedListener = new AtomicReference<>();
    final AtomicReference<ActionListener<AcknowledgedResponse>> changedListener = new AtomicReference<>();
    final AtomicReference<ActionListener<AcknowledgedResponse>> removedListener = new AtomicReference<>();
    final Semaphore updateInvocation = new Semaphore(0);
    final Semaphore calculateInvocation = new Semaphore(0);
    final Semaphore changedInvocation = new Semaphore(0);
    final Semaphore finishInvocation = new Semaphore(0);
    Metadata metadata = randomMetadata(IndexTemplateMetadata.builder("user_template").patterns(randomIndexPatterns()).build(), IndexTemplateMetadata.builder("removed_test_template").patterns(randomIndexPatterns()).build(), IndexTemplateMetadata.builder("changed_test_template").patterns(randomIndexPatterns()).build());
    Client mockClient = mock(Client.class);
    AdminClient mockAdminClient = mock(AdminClient.class);
    IndicesAdminClient mockIndicesAdminClient = mock(IndicesAdminClient.class);
    when(mockClient.admin()).thenReturn(mockAdminClient);
    when(mockAdminClient.indices()).thenReturn(mockIndicesAdminClient);
    doAnswer(invocation -> {
        Object[] args = invocation.getArguments();
        assert args.length == 2;
        PutIndexTemplateRequest request = (PutIndexTemplateRequest) args[0];
        if (request.name().equals("added_test_template")) {
            assertThat(addedListener.getAndSet((ActionListener) args[1]), nullValue());
        } else if (request.name().equals("changed_test_template")) {
            assertThat(changedListener.getAndSet((ActionListener) args[1]), nullValue());
        } else {
            fail("unexpected put template call for " + request.name());
        }
        return null;
    }).when(mockIndicesAdminClient).putTemplate(any(PutIndexTemplateRequest.class), any(ActionListener.class));
    doAnswer(invocation -> {
        Object[] args = invocation.getArguments();
        assert args.length == 2;
        DeleteIndexTemplateRequest request = (DeleteIndexTemplateRequest) args[0];
        assertThat(request.name(), startsWith("removed_test_template"));
        assertThat(removedListener.getAndSet((ActionListener) args[1]), nullValue());
        return null;
    }).when(mockIndicesAdminClient).deleteTemplate(any(DeleteIndexTemplateRequest.class), any(ActionListener.class));
    new TemplateUpgradeService(mockClient, clusterService, threadPool, Arrays.asList(templates -> {
        assertNull(templates.put("added_test_template", IndexTemplateMetadata.builder("added_test_template").patterns(Collections.singletonList("*")).build()));
        return templates;
    }, templates -> {
        assertNotNull(templates.remove("removed_test_template"));
        return templates;
    }, templates -> {
        assertNotNull(templates.put("changed_test_template", IndexTemplateMetadata.builder("changed_test_template").patterns(Collections.singletonList("*")).order(10).build()));
        return templates;
    })) {

        @Override
        void tryFinishUpgrade(AtomicBoolean anyUpgradeFailed) {
            super.tryFinishUpgrade(anyUpgradeFailed);
            finishInvocation.release();
        }

        @Override
        void upgradeTemplates(Map<String, BytesReference> changes, Set<String> deletions) {
            super.upgradeTemplates(changes, deletions);
            updateInvocation.release();
        }

        @Override
        Optional<Tuple<Map<String, BytesReference>, Set<String>>> calculateTemplateChanges(ImmutableOpenMap<String, IndexTemplateMetadata> templates) {
            final Optional<Tuple<Map<String, BytesReference>, Set<String>>> ans = super.calculateTemplateChanges(templates);
            calculateInvocation.release();
            return ans;
        }

        @Override
        public void clusterChanged(ClusterChangedEvent event) {
            super.clusterChanged(event);
            changedInvocation.release();
        }
    };
    ClusterState prevState = ClusterState.EMPTY_STATE;
    ClusterState state = ClusterState.builder(prevState).nodes(DiscoveryNodes.builder().add(new DiscoveryNode("node1", "node1", buildNewFakeTransportAddress(), emptyMap(), MASTER_DATA_ROLES, Version.CURRENT)).localNodeId("node1").masterNodeId("node1").build()).metadata(metadata).build();
    setState(clusterService, state);
    changedInvocation.acquire();
    assertThat(changedInvocation.availablePermits(), equalTo(0));
    calculateInvocation.acquire();
    assertThat(calculateInvocation.availablePermits(), equalTo(0));
    updateInvocation.acquire();
    assertThat(updateInvocation.availablePermits(), equalTo(0));
    assertThat(finishInvocation.availablePermits(), equalTo(0));
    assertThat(addedListener.get(), notNullValue());
    assertThat(changedListener.get(), notNullValue());
    assertThat(removedListener.get(), notNullValue());
    prevState = state;
    state = ClusterState.builder(prevState).metadata(Metadata.builder(state.metadata()).removeTemplate("user_template")).build();
    setState(clusterService, state);
    // Make sure that update wasn't invoked since we are still running
    changedInvocation.acquire();
    assertThat(changedInvocation.availablePermits(), equalTo(0));
    assertThat(calculateInvocation.availablePermits(), equalTo(0));
    assertThat(updateInvocation.availablePermits(), equalTo(0));
    assertThat(finishInvocation.availablePermits(), equalTo(0));
    addedListener.getAndSet(null).onResponse(new AcknowledgedResponse(true) {
    });
    changedListener.getAndSet(null).onResponse(new AcknowledgedResponse(true) {
    });
    removedListener.getAndSet(null).onResponse(new AcknowledgedResponse(true) {
    });
    // 3 upgrades should be completed, in addition to the final calculate
    finishInvocation.acquire(3);
    assertThat(finishInvocation.availablePermits(), equalTo(0));
    calculateInvocation.acquire();
    assertThat(calculateInvocation.availablePermits(), equalTo(0));
    setState(clusterService, state);
    // Make sure that update was called this time since we are no longer running
    changedInvocation.acquire();
    assertThat(changedInvocation.availablePermits(), equalTo(0));
    calculateInvocation.acquire();
    assertThat(calculateInvocation.availablePermits(), equalTo(0));
    updateInvocation.acquire();
    assertThat(updateInvocation.availablePermits(), equalTo(0));
    assertThat(finishInvocation.availablePermits(), equalTo(0));
    addedListener.getAndSet(null).onFailure(new RuntimeException("test - ignore"));
    changedListener.getAndSet(null).onFailure(new RuntimeException("test - ignore"));
    removedListener.getAndSet(null).onFailure(new RuntimeException("test - ignore"));
    finishInvocation.acquire(3);
    assertThat(finishInvocation.availablePermits(), equalTo(0));
    calculateInvocation.acquire();
    assertThat(calculateInvocation.availablePermits(), equalTo(0));
    setState(clusterService, state);
    // Make sure that update wasn't called this time since the index template metadata didn't change
    changedInvocation.acquire();
    assertThat(changedInvocation.availablePermits(), equalTo(0));
    assertThat(calculateInvocation.availablePermits(), equalTo(0));
    assertThat(updateInvocation.availablePermits(), equalTo(0));
    assertThat(finishInvocation.availablePermits(), equalTo(0));
}
Also used : ImmutableOpenMap(org.opensearch.common.collect.ImmutableOpenMap) Arrays(java.util.Arrays) PutIndexTemplateRequest(org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest) TestThreadPool(org.opensearch.threadpool.TestThreadPool) Version(org.opensearch.Version) ClusterServiceUtils.setState(org.opensearch.test.ClusterServiceUtils.setState) CoreMatchers.startsWith(org.hamcrest.CoreMatchers.startsWith) ThreadContext(org.opensearch.common.util.concurrent.ThreadContext) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) After(org.junit.After) Map(java.util.Map) Mockito.doAnswer(org.mockito.Mockito.doAnswer) ActionListener(org.opensearch.action.ActionListener) DeleteIndexTemplateRequest(org.opensearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest) Client(org.opensearch.client.Client) AdminClient(org.opensearch.client.AdminClient) Matchers.notNullValue(org.hamcrest.Matchers.notNullValue) OpenSearchTestCase(org.opensearch.test.OpenSearchTestCase) Set(java.util.Set) DiscoveryNodeRole(org.opensearch.cluster.node.DiscoveryNodeRole) Collectors(java.util.stream.Collectors) Tuple(org.opensearch.common.collect.Tuple) List(java.util.List) BytesArray(org.opensearch.common.bytes.BytesArray) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Optional(java.util.Optional) Mockito.any(org.mockito.Mockito.any) Matchers.containsString(org.hamcrest.Matchers.containsString) Mockito.mock(org.mockito.Mockito.mock) IntStream(java.util.stream.IntStream) DiscoveryNodes(org.opensearch.cluster.node.DiscoveryNodes) BytesReference(org.opensearch.common.bytes.BytesReference) ThreadPool(org.opensearch.threadpool.ThreadPool) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) AtomicReference(java.util.concurrent.atomic.AtomicReference) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) ClusterState(org.opensearch.cluster.ClusterState) Matchers.hasSize(org.hamcrest.Matchers.hasSize) CoreMatchers.nullValue(org.hamcrest.CoreMatchers.nullValue) Before(org.junit.Before) Collections.emptyMap(java.util.Collections.emptyMap) Matchers.empty(org.hamcrest.Matchers.empty) ClusterServiceUtils.createClusterService(org.opensearch.test.ClusterServiceUtils.createClusterService) Semaphore(java.util.concurrent.Semaphore) Mockito.when(org.mockito.Mockito.when) AcknowledgedResponse(org.opensearch.action.support.master.AcknowledgedResponse) IndicesAdminClient(org.opensearch.client.IndicesAdminClient) ClusterService(org.opensearch.cluster.service.ClusterService) Collections(java.util.Collections) ClusterChangedEvent(org.opensearch.cluster.ClusterChangedEvent) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) Set(java.util.Set) HashSet(java.util.HashSet) PutIndexTemplateRequest(org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest) IndicesAdminClient(org.opensearch.client.IndicesAdminClient) ClusterChangedEvent(org.opensearch.cluster.ClusterChangedEvent) Semaphore(java.util.concurrent.Semaphore) Matchers.containsString(org.hamcrest.Matchers.containsString) DeleteIndexTemplateRequest(org.opensearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest) ImmutableOpenMap(org.opensearch.common.collect.ImmutableOpenMap) Client(org.opensearch.client.Client) AdminClient(org.opensearch.client.AdminClient) IndicesAdminClient(org.opensearch.client.IndicesAdminClient) BytesReference(org.opensearch.common.bytes.BytesReference) ClusterState(org.opensearch.cluster.ClusterState) AcknowledgedResponse(org.opensearch.action.support.master.AcknowledgedResponse) AtomicReference(java.util.concurrent.atomic.AtomicReference) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ActionListener(org.opensearch.action.ActionListener) ImmutableOpenMap(org.opensearch.common.collect.ImmutableOpenMap) Map(java.util.Map) HashMap(java.util.HashMap) Collections.emptyMap(java.util.Collections.emptyMap) Tuple(org.opensearch.common.collect.Tuple) AdminClient(org.opensearch.client.AdminClient) IndicesAdminClient(org.opensearch.client.IndicesAdminClient)

Example 4 with PutIndexTemplateRequest

use of org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest in project OpenSearch by opensearch-project.

the class TemplateUpgradeService method upgradeTemplates.

void upgradeTemplates(Map<String, BytesReference> changes, Set<String> deletions) {
    final AtomicBoolean anyUpgradeFailed = new AtomicBoolean(false);
    if (threadPool.getThreadContext().isSystemContext() == false) {
        throw new IllegalStateException("template updates from the template upgrade service should always happen in a system context");
    }
    for (Map.Entry<String, BytesReference> change : changes.entrySet()) {
        PutIndexTemplateRequest request = new PutIndexTemplateRequest(change.getKey()).source(change.getValue(), XContentType.JSON);
        request.masterNodeTimeout(TimeValue.timeValueMinutes(1));
        client.admin().indices().putTemplate(request, new ActionListener<AcknowledgedResponse>() {

            @Override
            public void onResponse(AcknowledgedResponse response) {
                if (response.isAcknowledged() == false) {
                    anyUpgradeFailed.set(true);
                    logger.warn("Error updating template [{}], request was not acknowledged", change.getKey());
                }
                tryFinishUpgrade(anyUpgradeFailed);
            }

            @Override
            public void onFailure(Exception e) {
                anyUpgradeFailed.set(true);
                logger.warn(new ParameterizedMessage("Error updating template [{}]", change.getKey()), e);
                tryFinishUpgrade(anyUpgradeFailed);
            }
        });
    }
    for (String template : deletions) {
        DeleteIndexTemplateRequest request = new DeleteIndexTemplateRequest(template);
        request.masterNodeTimeout(TimeValue.timeValueMinutes(1));
        client.admin().indices().deleteTemplate(request, new ActionListener<AcknowledgedResponse>() {

            @Override
            public void onResponse(AcknowledgedResponse response) {
                if (response.isAcknowledged() == false) {
                    anyUpgradeFailed.set(true);
                    logger.warn("Error deleting template [{}], request was not acknowledged", template);
                }
                tryFinishUpgrade(anyUpgradeFailed);
            }

            @Override
            public void onFailure(Exception e) {
                anyUpgradeFailed.set(true);
                if (e instanceof IndexTemplateMissingException == false) {
                    // we might attempt to delete the same template from different nodes - so that's ok if template doesn't exist
                    // otherwise we need to warn
                    logger.warn(new ParameterizedMessage("Error deleting template [{}]", template), e);
                }
                tryFinishUpgrade(anyUpgradeFailed);
            }
        });
    }
}
Also used : BytesReference(org.opensearch.common.bytes.BytesReference) IndexTemplateMissingException(org.opensearch.indices.IndexTemplateMissingException) AcknowledgedResponse(org.opensearch.action.support.master.AcknowledgedResponse) PutIndexTemplateRequest(org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest) DeleteIndexTemplateRequest(org.opensearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest) IndexTemplateMissingException(org.opensearch.indices.IndexTemplateMissingException) IOException(java.io.IOException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) ImmutableOpenMap(org.opensearch.common.collect.ImmutableOpenMap) HashMap(java.util.HashMap) Map(java.util.Map) Collections.singletonMap(java.util.Collections.singletonMap)

Aggregations

PutIndexTemplateRequest (org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest)4 HashMap (java.util.HashMap)3 Map (java.util.Map)3 DeleteIndexTemplateRequest (org.opensearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest)3 AcknowledgedResponse (org.opensearch.action.support.master.AcknowledgedResponse)3 BytesReference (org.opensearch.common.bytes.BytesReference)3 IOException (java.io.IOException)2 ArrayList (java.util.ArrayList)2 Arrays (java.util.Arrays)2 Collections (java.util.Collections)2 HashSet (java.util.HashSet)2 List (java.util.List)2 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)2 Matchers.containsString (org.hamcrest.Matchers.containsString)2 ActionListener (org.opensearch.action.ActionListener)2 AdminClient (org.opensearch.client.AdminClient)2 Client (org.opensearch.client.Client)2 IndicesAdminClient (org.opensearch.client.IndicesAdminClient)2 BytesArray (org.opensearch.common.bytes.BytesArray)2 ThreadContext (org.opensearch.common.util.concurrent.ThreadContext)2