Search in sources :

Example 1 with ThreadContext

use of org.opensearch.common.util.concurrent.ThreadContext in project OpenSearch by opensearch-project.

the class ClusterApplierService method submitStateUpdateTask.

private void submitStateUpdateTask(final String source, final ClusterStateTaskConfig config, final Function<ClusterState, ClusterState> executor, final ClusterApplyListener listener) {
    if (!lifecycle.started()) {
        return;
    }
    final ThreadContext threadContext = threadPool.getThreadContext();
    final Supplier<ThreadContext.StoredContext> supplier = threadContext.newRestorableContext(true);
    try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
        threadContext.markAsSystemContext();
        final UpdateTask updateTask = new UpdateTask(config.priority(), source, new SafeClusterApplyListener(listener, supplier, logger), executor);
        if (config.timeout() != null) {
            threadPoolExecutor.execute(updateTask, config.timeout(), () -> threadPool.generic().execute(() -> listener.onFailure(source, new ProcessClusterEventTimeoutException(config.timeout(), source))));
        } else {
            threadPoolExecutor.execute(updateTask);
        }
    } catch (OpenSearchRejectedExecutionException e) {
        // to be done here...
        if (!lifecycle.stoppedOrClosed()) {
            throw e;
        }
    }
}
Also used : ThreadContext(org.opensearch.common.util.concurrent.ThreadContext) ProcessClusterEventTimeoutException(org.opensearch.cluster.metadata.ProcessClusterEventTimeoutException) OpenSearchRejectedExecutionException(org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException)

Example 2 with ThreadContext

use of org.opensearch.common.util.concurrent.ThreadContext in project OpenSearch by opensearch-project.

the class MasterService method submitStateUpdateTasks.

/**
 * Submits a batch of cluster state update tasks; submitted updates are guaranteed to be processed together,
 * potentially with more tasks of the same executor.
 *
 * @param source   the source of the cluster state update task
 * @param tasks    a map of update tasks and their corresponding listeners
 * @param config   the cluster state update task configuration
 * @param executor the cluster state update task executor; tasks
 *                 that share the same executor will be executed
 *                 batches on this executor
 * @param <T>      the type of the cluster state update task state
 */
public <T> void submitStateUpdateTasks(final String source, final Map<T, ClusterStateTaskListener> tasks, final ClusterStateTaskConfig config, final ClusterStateTaskExecutor<T> executor) {
    if (!lifecycle.started()) {
        return;
    }
    final ThreadContext threadContext = threadPool.getThreadContext();
    final Supplier<ThreadContext.StoredContext> supplier = threadContext.newRestorableContext(true);
    try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
        threadContext.markAsSystemContext();
        List<Batcher.UpdateTask> safeTasks = tasks.entrySet().stream().map(e -> taskBatcher.new UpdateTask(config.priority(), source, e.getKey(), safe(e.getValue(), supplier), executor)).collect(Collectors.toList());
        taskBatcher.submitTasks(safeTasks, config.timeout());
    } catch (OpenSearchRejectedExecutionException e) {
        // to be done here...
        if (!lifecycle.stoppedOrClosed()) {
            throw e;
        }
    }
}
Also used : ClusterStateTaskListener(org.opensearch.cluster.ClusterStateTaskListener) OpenSearchRejectedExecutionException(org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException) AckedClusterStateTaskListener(org.opensearch.cluster.AckedClusterStateTaskListener) DiscoveryNodes(org.opensearch.cluster.node.DiscoveryNodes) Arrays(java.util.Arrays) Metadata(org.opensearch.cluster.metadata.Metadata) CountDown(org.opensearch.common.util.concurrent.CountDown) PrioritizedOpenSearchThreadPoolExecutor(org.opensearch.common.util.concurrent.PrioritizedOpenSearchThreadPoolExecutor) ThreadPool(org.opensearch.threadpool.ThreadPool) Priority(org.opensearch.common.Priority) Node(org.opensearch.node.Node) FutureUtils(org.opensearch.common.util.concurrent.FutureUtils) ThreadContext(org.opensearch.common.util.concurrent.ThreadContext) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) OpenSearchExecutors(org.opensearch.common.util.concurrent.OpenSearchExecutors) Supplier(java.util.function.Supplier) ClusterState(org.opensearch.cluster.ClusterState) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) ClusterTasksResult(org.opensearch.cluster.ClusterStateTaskExecutor.ClusterTasksResult) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) Locale(java.util.Locale) OpenSearchExecutors.daemonThreadFactory(org.opensearch.common.util.concurrent.OpenSearchExecutors.daemonThreadFactory) Assertions(org.opensearch.Assertions) Map(java.util.Map) ClusterStatePublisher(org.opensearch.cluster.coordination.ClusterStatePublisher) ClusterStateTaskConfig(org.opensearch.cluster.ClusterStateTaskConfig) ClusterSettings(org.opensearch.common.settings.ClusterSettings) ProcessClusterEventTimeoutException(org.opensearch.cluster.metadata.ProcessClusterEventTimeoutException) Setting(org.opensearch.common.settings.Setting) TimeValue(org.opensearch.common.unit.TimeValue) ClusterStateTaskExecutor(org.opensearch.cluster.ClusterStateTaskExecutor) Settings(org.opensearch.common.settings.Settings) Discovery(org.opensearch.discovery.Discovery) Collectors(java.util.stream.Collectors) Nullable(org.opensearch.common.Nullable) FailedToCommitClusterStateException(org.opensearch.cluster.coordination.FailedToCommitClusterStateException) Objects(java.util.Objects) TimeUnit(java.util.concurrent.TimeUnit) AbstractLifecycleComponent(org.opensearch.common.component.AbstractLifecycleComponent) List(java.util.List) Logger(org.apache.logging.log4j.Logger) Builder(org.opensearch.cluster.ClusterState.Builder) RoutingTable(org.opensearch.cluster.routing.RoutingTable) LogManager(org.apache.logging.log4j.LogManager) Collections(java.util.Collections) Text(org.opensearch.common.text.Text) Scheduler(org.opensearch.threadpool.Scheduler) ClusterChangedEvent(org.opensearch.cluster.ClusterChangedEvent) ThreadContext(org.opensearch.common.util.concurrent.ThreadContext) OpenSearchRejectedExecutionException(org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException)

Example 3 with ThreadContext

use of org.opensearch.common.util.concurrent.ThreadContext in project OpenSearch by opensearch-project.

the class Netty4HttpServerTransportTests method testBadRequest.

public void testBadRequest() throws InterruptedException {
    final AtomicReference<Throwable> causeReference = new AtomicReference<>();
    final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() {

        @Override
        public void dispatchRequest(final RestRequest request, final RestChannel channel, final ThreadContext threadContext) {
            logger.error("--> Unexpected successful request [{}]", FakeRestRequest.requestToString(request));
            throw new AssertionError();
        }

        @Override
        public void dispatchBadRequest(final RestChannel channel, final ThreadContext threadContext, final Throwable cause) {
            causeReference.set(cause);
            try {
                final OpenSearchException e = new OpenSearchException("you sent a bad request and you should feel bad");
                channel.sendResponse(new BytesRestResponse(channel, BAD_REQUEST, e));
            } catch (final IOException e) {
                throw new AssertionError(e);
            }
        }
    };
    final Settings settings;
    final int maxInitialLineLength;
    final Setting<ByteSizeValue> httpMaxInitialLineLengthSetting = HttpTransportSettings.SETTING_HTTP_MAX_INITIAL_LINE_LENGTH;
    if (randomBoolean()) {
        maxInitialLineLength = httpMaxInitialLineLengthSetting.getDefault(Settings.EMPTY).bytesAsInt();
        settings = createSettings();
    } else {
        maxInitialLineLength = randomIntBetween(1, 8192);
        settings = createBuilderWithPort().put(httpMaxInitialLineLengthSetting.getKey(), maxInitialLineLength + "b").build();
    }
    try (Netty4HttpServerTransport transport = new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool, xContentRegistry(), dispatcher, clusterSettings, new SharedGroupFactory(settings))) {
        transport.start();
        final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
        try (Netty4HttpClient client = new Netty4HttpClient()) {
            final String url = "/" + new String(new byte[maxInitialLineLength], Charset.forName("UTF-8"));
            final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, url);
            final FullHttpResponse response = client.send(remoteAddress.address(), request);
            try {
                assertThat(response.status(), equalTo(HttpResponseStatus.BAD_REQUEST));
                assertThat(new String(response.content().array(), Charset.forName("UTF-8")), containsString("you sent a bad request and you should feel bad"));
            } finally {
                response.release();
            }
        }
    }
    assertNotNull(causeReference.get());
    assertThat(causeReference.get(), instanceOf(TooLongFrameException.class));
}
Also used : DefaultFullHttpRequest(io.netty.handler.codec.http.DefaultFullHttpRequest) FullHttpRequest(io.netty.handler.codec.http.FullHttpRequest) TooLongFrameException(io.netty.handler.codec.TooLongFrameException) TransportAddress(org.opensearch.common.transport.TransportAddress) ByteSizeValue(org.opensearch.common.unit.ByteSizeValue) Matchers.containsString(org.hamcrest.Matchers.containsString) HttpServerTransport(org.opensearch.http.HttpServerTransport) NullDispatcher(org.opensearch.http.NullDispatcher) BytesRestResponse(org.opensearch.rest.BytesRestResponse) FullHttpResponse(io.netty.handler.codec.http.FullHttpResponse) Settings(org.opensearch.common.settings.Settings) HttpTransportSettings(org.opensearch.http.HttpTransportSettings) ClusterSettings(org.opensearch.common.settings.ClusterSettings) DefaultFullHttpRequest(io.netty.handler.codec.http.DefaultFullHttpRequest) ThreadContext(org.opensearch.common.util.concurrent.ThreadContext) AtomicReference(java.util.concurrent.atomic.AtomicReference) RestChannel(org.opensearch.rest.RestChannel) SharedGroupFactory(org.opensearch.transport.SharedGroupFactory) IOException(java.io.IOException) FakeRestRequest(org.opensearch.test.rest.FakeRestRequest) RestRequest(org.opensearch.rest.RestRequest) OpenSearchException(org.opensearch.OpenSearchException)

Example 4 with ThreadContext

use of org.opensearch.common.util.concurrent.ThreadContext in project OpenSearch by opensearch-project.

the class Netty4HttpServerTransportTests method testLargeCompressedResponse.

public void testLargeCompressedResponse() throws InterruptedException {
    final String responseString = randomAlphaOfLength(4 * 1024 * 1024);
    final String url = "/thing";
    final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() {

        @Override
        public void dispatchRequest(final RestRequest request, final RestChannel channel, final ThreadContext threadContext) {
            if (url.equals(request.uri())) {
                channel.sendResponse(new BytesRestResponse(OK, responseString));
            } else {
                logger.error("--> Unexpected successful uri [{}]", request.uri());
                throw new AssertionError();
            }
        }

        @Override
        public void dispatchBadRequest(final RestChannel channel, final ThreadContext threadContext, final Throwable cause) {
            logger.error(new ParameterizedMessage("--> Unexpected bad request [{}]", FakeRestRequest.requestToString(channel.request())), cause);
            throw new AssertionError();
        }
    };
    try (Netty4HttpServerTransport transport = new Netty4HttpServerTransport(Settings.EMPTY, networkService, bigArrays, threadPool, xContentRegistry(), dispatcher, clusterSettings, new SharedGroupFactory(Settings.EMPTY))) {
        transport.start();
        final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
        try (Netty4HttpClient client = new Netty4HttpClient()) {
            DefaultFullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, url);
            request.headers().add(HttpHeaderNames.ACCEPT_ENCODING, randomFrom("deflate", "gzip"));
            long numOfHugeAllocations = getHugeAllocationCount();
            final FullHttpResponse response = client.send(remoteAddress.address(), request);
            try {
                assertThat(getHugeAllocationCount(), equalTo(numOfHugeAllocations));
                assertThat(response.status(), equalTo(HttpResponseStatus.OK));
                byte[] bytes = new byte[response.content().readableBytes()];
                response.content().readBytes(bytes);
                assertThat(new String(bytes, StandardCharsets.UTF_8), equalTo(responseString));
            } finally {
                response.release();
            }
        }
    }
}
Also used : DefaultFullHttpRequest(io.netty.handler.codec.http.DefaultFullHttpRequest) TransportAddress(org.opensearch.common.transport.TransportAddress) ThreadContext(org.opensearch.common.util.concurrent.ThreadContext) RestChannel(org.opensearch.rest.RestChannel) SharedGroupFactory(org.opensearch.transport.SharedGroupFactory) Matchers.containsString(org.hamcrest.Matchers.containsString) HttpServerTransport(org.opensearch.http.HttpServerTransport) NullDispatcher(org.opensearch.http.NullDispatcher) FakeRestRequest(org.opensearch.test.rest.FakeRestRequest) RestRequest(org.opensearch.rest.RestRequest) BytesRestResponse(org.opensearch.rest.BytesRestResponse) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) FullHttpResponse(io.netty.handler.codec.http.FullHttpResponse)

Example 5 with ThreadContext

use of org.opensearch.common.util.concurrent.ThreadContext in project OpenSearch by opensearch-project.

the class Netty4HttpServerTransportTests method testReadTimeout.

public void testReadTimeout() throws Exception {
    final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() {

        @Override
        public void dispatchRequest(final RestRequest request, final RestChannel channel, final ThreadContext threadContext) {
            logger.error("--> Unexpected successful request [{}]", FakeRestRequest.requestToString(request));
            throw new AssertionError("Should not have received a dispatched request");
        }

        @Override
        public void dispatchBadRequest(final RestChannel channel, final ThreadContext threadContext, final Throwable cause) {
            logger.error(new ParameterizedMessage("--> Unexpected bad request [{}]", FakeRestRequest.requestToString(channel.request())), cause);
            throw new AssertionError("Should not have received a dispatched request");
        }
    };
    Settings settings = createBuilderWithPort().put(HttpTransportSettings.SETTING_HTTP_READ_TIMEOUT.getKey(), new TimeValue(randomIntBetween(100, 300))).build();
    NioEventLoopGroup group = new NioEventLoopGroup();
    try (Netty4HttpServerTransport transport = new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool, xContentRegistry(), dispatcher, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), new SharedGroupFactory(settings))) {
        transport.start();
        final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
        CountDownLatch channelClosedLatch = new CountDownLatch(1);
        Bootstrap clientBootstrap = new Bootstrap().option(ChannelOption.ALLOCATOR, NettyAllocator.getAllocator()).channel(NioSocketChannel.class).handler(new ChannelInitializer<SocketChannel>() {

            @Override
            protected void initChannel(SocketChannel ch) {
                ch.pipeline().addLast(new ChannelHandlerAdapter() {
                });
            }
        }).group(group);
        ChannelFuture connect = clientBootstrap.connect(remoteAddress.address());
        connect.channel().closeFuture().addListener(future -> channelClosedLatch.countDown());
        assertTrue("Channel should be closed due to read timeout", channelClosedLatch.await(1, TimeUnit.MINUTES));
    } finally {
        group.shutdownGracefully().await();
    }
}
Also used : ChannelFuture(io.netty.channel.ChannelFuture) SocketChannel(io.netty.channel.socket.SocketChannel) NioSocketChannel(io.netty.channel.socket.nio.NioSocketChannel) ClusterSettings(org.opensearch.common.settings.ClusterSettings) TransportAddress(org.opensearch.common.transport.TransportAddress) ThreadContext(org.opensearch.common.util.concurrent.ThreadContext) RestChannel(org.opensearch.rest.RestChannel) SharedGroupFactory(org.opensearch.transport.SharedGroupFactory) HttpServerTransport(org.opensearch.http.HttpServerTransport) NullDispatcher(org.opensearch.http.NullDispatcher) CountDownLatch(java.util.concurrent.CountDownLatch) ChannelHandlerAdapter(io.netty.channel.ChannelHandlerAdapter) FakeRestRequest(org.opensearch.test.rest.FakeRestRequest) RestRequest(org.opensearch.rest.RestRequest) Bootstrap(io.netty.bootstrap.Bootstrap) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) ChannelInitializer(io.netty.channel.ChannelInitializer) Settings(org.opensearch.common.settings.Settings) HttpTransportSettings(org.opensearch.http.HttpTransportSettings) ClusterSettings(org.opensearch.common.settings.ClusterSettings) TimeValue(org.opensearch.common.unit.TimeValue) NioEventLoopGroup(io.netty.channel.nio.NioEventLoopGroup)

Aggregations

ThreadContext (org.opensearch.common.util.concurrent.ThreadContext)87 Matchers.containsString (org.hamcrest.Matchers.containsString)35 Settings (org.opensearch.common.settings.Settings)30 IOException (java.io.IOException)28 List (java.util.List)22 IndexNameExpressionResolver (org.opensearch.cluster.metadata.IndexNameExpressionResolver)22 ClusterSettings (org.opensearch.common.settings.ClusterSettings)18 ClusterState (org.opensearch.cluster.ClusterState)16 FakeRestRequest (org.opensearch.test.rest.FakeRestRequest)16 RestChannel (org.opensearch.rest.RestChannel)15 RestRequest (org.opensearch.rest.RestRequest)15 ParameterizedMessage (org.apache.logging.log4j.message.ParameterizedMessage)14 ActionListener (org.opensearch.action.ActionListener)14 ThreadPool (org.opensearch.threadpool.ThreadPool)13 Tuple (org.opensearch.common.collect.Tuple)12 TransportAddress (org.opensearch.common.transport.TransportAddress)12 TimeValue (org.opensearch.common.unit.TimeValue)11 HttpServerTransport (org.opensearch.http.HttpServerTransport)11 OpenSearchTestCase (org.opensearch.test.OpenSearchTestCase)11 Version (org.opensearch.Version)10