Examples with ThreadPool org.opensearch.threadpool.ThreadPool used on opensource projects

Example 1 with ThreadPool

use of org.opensearch.threadpool.ThreadPool in project OpenSearch by opensearch-project.

the class ActionModule method initRestHandlers.

public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
    List<AbstractCatAction> catActions = new ArrayList<>();
    Consumer<RestHandler> registerHandler = handler -> {
        if (handler instanceof AbstractCatAction) {
            catActions.add((AbstractCatAction) handler);
        }
        restController.registerHandler(handler);
    };
    registerHandler.accept(new RestAddVotingConfigExclusionAction());
    registerHandler.accept(new RestClearVotingConfigExclusionsAction());
    registerHandler.accept(new RestMainAction());
    registerHandler.accept(new RestNodesInfoAction(settingsFilter));
    registerHandler.accept(new RestRemoteClusterInfoAction());
    registerHandler.accept(new RestNodesStatsAction());
    registerHandler.accept(new RestNodesUsageAction());
    registerHandler.accept(new RestNodesHotThreadsAction());
    registerHandler.accept(new RestClusterAllocationExplainAction());
    registerHandler.accept(new RestClusterStatsAction());
    registerHandler.accept(new RestClusterStateAction(settingsFilter));
    registerHandler.accept(new RestClusterHealthAction());
    registerHandler.accept(new RestClusterUpdateSettingsAction());
    registerHandler.accept(new RestClusterGetSettingsAction(settings, clusterSettings, settingsFilter));
    registerHandler.accept(new RestClusterRerouteAction(settingsFilter));
    registerHandler.accept(new RestClusterSearchShardsAction());
    registerHandler.accept(new RestPendingClusterTasksAction());
    registerHandler.accept(new RestPutRepositoryAction());
    registerHandler.accept(new RestGetRepositoriesAction(settingsFilter));
    registerHandler.accept(new RestDeleteRepositoryAction());
    registerHandler.accept(new RestVerifyRepositoryAction());
    registerHandler.accept(new RestCleanupRepositoryAction());
    registerHandler.accept(new RestGetSnapshotsAction());
    registerHandler.accept(new RestCreateSnapshotAction());
    registerHandler.accept(new RestCloneSnapshotAction());
    registerHandler.accept(new RestRestoreSnapshotAction());
    registerHandler.accept(new RestDeleteSnapshotAction());
    registerHandler.accept(new RestSnapshotsStatusAction());
    registerHandler.accept(new RestGetIndicesAction());
    registerHandler.accept(new RestIndicesStatsAction());
    registerHandler.accept(new RestIndicesSegmentsAction());
    registerHandler.accept(new RestIndicesShardStoresAction());
    registerHandler.accept(new RestGetAliasesAction());
    registerHandler.accept(new RestIndexDeleteAliasesAction());
    registerHandler.accept(new RestIndexPutAliasAction());
    registerHandler.accept(new RestIndicesAliasesAction());
    registerHandler.accept(new RestCreateIndexAction());
    registerHandler.accept(new RestResizeHandler.RestShrinkIndexAction());
    registerHandler.accept(new RestResizeHandler.RestSplitIndexAction());
    registerHandler.accept(new RestResizeHandler.RestCloneIndexAction());
    registerHandler.accept(new RestRolloverIndexAction());
    registerHandler.accept(new RestDeleteIndexAction());
    registerHandler.accept(new RestCloseIndexAction());
    registerHandler.accept(new RestOpenIndexAction());
    registerHandler.accept(new RestAddIndexBlockAction());
    registerHandler.accept(new RestUpdateSettingsAction());
    registerHandler.accept(new RestGetSettingsAction());
    registerHandler.accept(new RestAnalyzeAction());
    registerHandler.accept(new RestGetIndexTemplateAction());
    registerHandler.accept(new RestPutIndexTemplateAction());
    registerHandler.accept(new RestDeleteIndexTemplateAction());
    registerHandler.accept(new RestPutComponentTemplateAction());
    registerHandler.accept(new RestGetComponentTemplateAction());
    registerHandler.accept(new RestDeleteComponentTemplateAction());
    registerHandler.accept(new RestPutComposableIndexTemplateAction());
    registerHandler.accept(new RestGetComposableIndexTemplateAction());
    registerHandler.accept(new RestDeleteComposableIndexTemplateAction());
    registerHandler.accept(new RestSimulateIndexTemplateAction());
    registerHandler.accept(new RestSimulateTemplateAction());
    registerHandler.accept(new RestPutMappingAction());
    registerHandler.accept(new RestGetMappingAction(threadPool));
    registerHandler.accept(new RestGetFieldMappingAction());
    registerHandler.accept(new RestRefreshAction());
    registerHandler.accept(new RestFlushAction());
    registerHandler.accept(new RestSyncedFlushAction());
    registerHandler.accept(new RestForceMergeAction());
    registerHandler.accept(new RestUpgradeAction());
    registerHandler.accept(new RestUpgradeStatusAction());
    registerHandler.accept(new RestClearIndicesCacheAction());
    registerHandler.accept(new RestIndexAction());
    registerHandler.accept(new CreateHandler());
    registerHandler.accept(new AutoIdHandler(nodesInCluster));
    registerHandler.accept(new RestGetAction());
    registerHandler.accept(new RestGetSourceAction());
    registerHandler.accept(new RestMultiGetAction(settings));
    registerHandler.accept(new RestDeleteAction());
    registerHandler.accept(new RestCountAction());
    registerHandler.accept(new RestTermVectorsAction());
    registerHandler.accept(new RestMultiTermVectorsAction());
    registerHandler.accept(new RestBulkAction(settings));
    registerHandler.accept(new RestUpdateAction());
    registerHandler.accept(new RestSearchAction());
    registerHandler.accept(new RestSearchScrollAction());
    registerHandler.accept(new RestClearScrollAction());
    registerHandler.accept(new RestMultiSearchAction(settings));
    registerHandler.accept(new RestValidateQueryAction());
    registerHandler.accept(new RestExplainAction());
    registerHandler.accept(new RestRecoveryAction());
    registerHandler.accept(new RestReloadSecureSettingsAction());
    // Scripts API
    registerHandler.accept(new RestGetStoredScriptAction());
    registerHandler.accept(new RestPutStoredScriptAction());
    registerHandler.accept(new RestDeleteStoredScriptAction());
    registerHandler.accept(new RestGetScriptContextAction());
    registerHandler.accept(new RestGetScriptLanguageAction());
    registerHandler.accept(new RestFieldCapabilitiesAction());
    // Tasks API
    registerHandler.accept(new RestListTasksAction(nodesInCluster));
    registerHandler.accept(new RestGetTaskAction());
    registerHandler.accept(new RestCancelTasksAction(nodesInCluster));
    // Ingest API
    registerHandler.accept(new RestPutPipelineAction());
    registerHandler.accept(new RestGetPipelineAction());
    registerHandler.accept(new RestDeletePipelineAction());
    registerHandler.accept(new RestSimulatePipelineAction());
    // Dangling indices API
    registerHandler.accept(new RestListDanglingIndicesAction());
    registerHandler.accept(new RestImportDanglingIndexAction());
    registerHandler.accept(new RestDeleteDanglingIndexAction());
    // Data Stream API
    registerHandler.accept(new RestCreateDataStreamAction());
    registerHandler.accept(new RestDeleteDataStreamAction());
    registerHandler.accept(new RestGetDataStreamsAction());
    registerHandler.accept(new RestResolveIndexAction());
    registerHandler.accept(new RestDataStreamsStatsAction());
    // CAT API
    registerHandler.accept(new RestAllocationAction());
    registerHandler.accept(new RestShardsAction());
    registerHandler.accept(new RestMasterAction());
    registerHandler.accept(new RestNodesAction());
    registerHandler.accept(new RestTasksAction(nodesInCluster));
    registerHandler.accept(new RestIndicesAction());
    registerHandler.accept(new RestSegmentsAction());
    // Fully qualified to prevent interference with rest.action.count.RestCountAction
    registerHandler.accept(new org.opensearch.rest.action.cat.RestCountAction());
    // Fully qualified to prevent interference with rest.action.indices.RestRecoveryAction
    registerHandler.accept(new RestCatRecoveryAction());
    registerHandler.accept(new RestHealthAction());
    registerHandler.accept(new org.opensearch.rest.action.cat.RestPendingClusterTasksAction());
    registerHandler.accept(new RestAliasAction());
    registerHandler.accept(new RestThreadPoolAction());
    registerHandler.accept(new RestPluginsAction());
    registerHandler.accept(new RestFielddataAction());
    registerHandler.accept(new RestNodeAttrsAction());
    registerHandler.accept(new RestRepositoriesAction());
    registerHandler.accept(new RestSnapshotAction());
    registerHandler.accept(new RestTemplatesAction());
    for (ActionPlugin plugin : actionPlugins) {
        for (RestHandler handler : plugin.getRestHandlers(settings, restController, clusterSettings, indexScopedSettings, settingsFilter, indexNameExpressionResolver, nodesInCluster)) {
            registerHandler.accept(handler);
        }
    }
    registerHandler.accept(new RestCatAction(catActions));
}

Example 2 with ThreadPool

use of org.opensearch.threadpool.ThreadPool in project OpenSearch by opensearch-project.

the class RetryTests method blockExecutor.

/**
 * Blocks the named executor by getting its only thread running a task blocked on a CyclicBarrier and fills the queue with a noop task.
 * So requests to use this queue should get {@link OpenSearchRejectedExecutionException}s.
 */
private CyclicBarrier blockExecutor(String name, String node) throws Exception {
    ThreadPool threadPool = internalCluster().getInstance(ThreadPool.class, node);
    CyclicBarrier barrier = new CyclicBarrier(2);
    logger.info("Blocking the [{}] executor", name);
    threadPool.executor(name).execute(() -> {
        try {
            threadPool.executor(name).execute(() -> {
            });
            barrier.await();
            logger.info("Blocked the [{}] executor", name);
            barrier.await();
            logger.info("Unblocking the [{}] executor", name);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    });
    barrier.await();
    blockedExecutors.add(barrier);
    return barrier;
}

Example 3 with ThreadPool

use of org.opensearch.threadpool.ThreadPool in project OpenSearch by opensearch-project.

the class IndexingPressureIT method testWriteBytesAreIncremented.

public void testWriteBytesAreIncremented() throws Exception {
    assertAcked(prepareCreate(INDEX_NAME, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)));
    ensureGreen(INDEX_NAME);
    Tuple<String, String> primaryReplicaNodeNames = getPrimaryReplicaNodeNames();
    String primaryName = primaryReplicaNodeNames.v1();
    String replicaName = primaryReplicaNodeNames.v2();
    String coordinatingOnlyNode = getCoordinatingOnlyNode();
    final CountDownLatch replicationSendPointReached = new CountDownLatch(1);
    final CountDownLatch latchBlockingReplicationSend = new CountDownLatch(1);
    TransportService primaryService = internalCluster().getInstance(TransportService.class, primaryName);
    final MockTransportService primaryTransportService = (MockTransportService) primaryService;
    TransportService replicaService = internalCluster().getInstance(TransportService.class, replicaName);
    final MockTransportService replicaTransportService = (MockTransportService) replicaService;
    primaryTransportService.addSendBehavior((connection, requestId, action, request, options) -> {
        if (action.equals(TransportShardBulkAction.ACTION_NAME + "[r]")) {
            try {
                replicationSendPointReached.countDown();
                latchBlockingReplicationSend.await();
            } catch (InterruptedException e) {
                throw new IllegalStateException(e);
            }
        }
        connection.sendRequest(requestId, action, request, options);
    });
    final ThreadPool replicaThreadPool = replicaTransportService.getThreadPool();
    final Releasable replicaRelease = blockReplicas(replicaThreadPool);
    final BulkRequest bulkRequest = new BulkRequest();
    int totalRequestSize = 0;
    for (int i = 0; i < 80; ++i) {
        IndexRequest request = new IndexRequest(INDEX_NAME).id(UUIDs.base64UUID()).source(Collections.singletonMap("key", randomAlphaOfLength(50)));
        totalRequestSize += request.ramBytesUsed();
        assertTrue(request.ramBytesUsed() > request.source().length());
        bulkRequest.add(request);
    }
    final long bulkRequestSize = bulkRequest.ramBytesUsed();
    final long bulkShardRequestSize = totalRequestSize;
    try {
        final ActionFuture<BulkResponse> successFuture = client(coordinatingOnlyNode).bulk(bulkRequest);
        replicationSendPointReached.await();
        IndexingPressure primaryWriteLimits = internalCluster().getInstance(IndexingPressureService.class, primaryName).getShardIndexingPressure();
        IndexingPressure replicaWriteLimits = internalCluster().getInstance(IndexingPressureService.class, replicaName).getShardIndexingPressure();
        ;
        IndexingPressure coordinatingWriteLimits = internalCluster().getInstance(IndexingPressureService.class, coordinatingOnlyNode).getShardIndexingPressure();
        ;
        assertThat(primaryWriteLimits.getCurrentCombinedCoordinatingAndPrimaryBytes(), greaterThan(bulkShardRequestSize));
        assertThat(primaryWriteLimits.getCurrentPrimaryBytes(), greaterThan(bulkShardRequestSize));
        assertEquals(0, primaryWriteLimits.getCurrentCoordinatingBytes());
        assertEquals(0, primaryWriteLimits.getCurrentReplicaBytes());
        assertEquals(0, replicaWriteLimits.getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertEquals(0, replicaWriteLimits.getCurrentCoordinatingBytes());
        assertEquals(0, replicaWriteLimits.getCurrentPrimaryBytes());
        assertEquals(0, replicaWriteLimits.getCurrentReplicaBytes());
        assertEquals(bulkRequestSize, coordinatingWriteLimits.getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertEquals(bulkRequestSize, coordinatingWriteLimits.getCurrentCoordinatingBytes());
        assertEquals(0, coordinatingWriteLimits.getCurrentPrimaryBytes());
        assertEquals(0, coordinatingWriteLimits.getCurrentReplicaBytes());
        latchBlockingReplicationSend.countDown();
        IndexRequest request = new IndexRequest(INDEX_NAME).id(UUIDs.base64UUID()).source(Collections.singletonMap("key", randomAlphaOfLength(50)));
        final BulkRequest secondBulkRequest = new BulkRequest();
        secondBulkRequest.add(request);
        // Use the primary or the replica data node as the coordinating node this time
        boolean usePrimaryAsCoordinatingNode = randomBoolean();
        final ActionFuture<BulkResponse> secondFuture;
        if (usePrimaryAsCoordinatingNode) {
            secondFuture = client(primaryName).bulk(secondBulkRequest);
        } else {
            secondFuture = client(replicaName).bulk(secondBulkRequest);
        }
        final long secondBulkRequestSize = secondBulkRequest.ramBytesUsed();
        final long secondBulkShardRequestSize = request.ramBytesUsed();
        if (usePrimaryAsCoordinatingNode) {
            assertBusy(() -> {
                assertThat(primaryWriteLimits.getCurrentCombinedCoordinatingAndPrimaryBytes(), greaterThan(bulkShardRequestSize + secondBulkRequestSize));
                assertEquals(secondBulkRequestSize, primaryWriteLimits.getCurrentCoordinatingBytes());
                assertThat(primaryWriteLimits.getCurrentPrimaryBytes(), greaterThan(bulkShardRequestSize + secondBulkRequestSize));
                assertEquals(0, replicaWriteLimits.getCurrentCombinedCoordinatingAndPrimaryBytes());
                assertEquals(0, replicaWriteLimits.getCurrentCoordinatingBytes());
                assertEquals(0, replicaWriteLimits.getCurrentPrimaryBytes());
            });
        } else {
            assertThat(primaryWriteLimits.getCurrentCombinedCoordinatingAndPrimaryBytes(), greaterThan(bulkShardRequestSize));
            assertEquals(secondBulkRequestSize, replicaWriteLimits.getCurrentCombinedCoordinatingAndPrimaryBytes());
            assertEquals(secondBulkRequestSize, replicaWriteLimits.getCurrentCoordinatingBytes());
            assertEquals(0, replicaWriteLimits.getCurrentPrimaryBytes());
        }
        assertEquals(bulkRequestSize, coordinatingWriteLimits.getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertBusy(() -> assertThat(replicaWriteLimits.getCurrentReplicaBytes(), greaterThan(bulkShardRequestSize + secondBulkShardRequestSize)));
        replicaRelease.close();
        successFuture.actionGet();
        secondFuture.actionGet();
        assertEquals(0, primaryWriteLimits.getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertEquals(0, primaryWriteLimits.getCurrentCoordinatingBytes());
        assertEquals(0, primaryWriteLimits.getCurrentPrimaryBytes());
        assertEquals(0, primaryWriteLimits.getCurrentReplicaBytes());
        assertEquals(0, replicaWriteLimits.getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertEquals(0, replicaWriteLimits.getCurrentCoordinatingBytes());
        assertEquals(0, replicaWriteLimits.getCurrentPrimaryBytes());
        assertEquals(0, replicaWriteLimits.getCurrentReplicaBytes());
        assertEquals(0, coordinatingWriteLimits.getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertEquals(0, coordinatingWriteLimits.getCurrentCoordinatingBytes());
        assertEquals(0, coordinatingWriteLimits.getCurrentPrimaryBytes());
        assertEquals(0, coordinatingWriteLimits.getCurrentReplicaBytes());
    } finally {
        if (replicationSendPointReached.getCount() > 0) {
            replicationSendPointReached.countDown();
        }
        replicaRelease.close();
        if (latchBlockingReplicationSend.getCount() > 0) {
            latchBlockingReplicationSend.countDown();
        }
        replicaRelease.close();
        primaryTransportService.clearAllRules();
    }
}

Example 4 with ThreadPool

use of org.opensearch.threadpool.ThreadPool in project OpenSearch by opensearch-project.

the class ShardIndexingPressureIT method testShardIndexingPressureTrackingDuringBulkWrites.

public void testShardIndexingPressureTrackingDuringBulkWrites() throws Exception {
    assertAcked(prepareCreate(INDEX_NAME, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)));
    ensureGreen(INDEX_NAME);
    Tuple<String, String> primaryReplicaNodeNames = getPrimaryReplicaNodeNames(INDEX_NAME);
    String primaryName = primaryReplicaNodeNames.v1();
    String replicaName = primaryReplicaNodeNames.v2();
    String coordinatingOnlyNode = getCoordinatingOnlyNode();
    final CountDownLatch replicationSendPointReached = new CountDownLatch(1);
    final CountDownLatch latchBlockingReplicationSend = new CountDownLatch(1);
    TransportService primaryService = internalCluster().getInstance(TransportService.class, primaryName);
    final MockTransportService primaryTransportService = (MockTransportService) primaryService;
    TransportService replicaService = internalCluster().getInstance(TransportService.class, replicaName);
    final MockTransportService replicaTransportService = (MockTransportService) replicaService;
    primaryTransportService.addSendBehavior((connection, requestId, action, request, options) -> {
        if (action.equals(TransportShardBulkAction.ACTION_NAME + "[r]")) {
            try {
                replicationSendPointReached.countDown();
                latchBlockingReplicationSend.await();
            } catch (InterruptedException e) {
                throw new IllegalStateException(e);
            }
        }
        connection.sendRequest(requestId, action, request, options);
    });
    final ThreadPool replicaThreadPool = replicaTransportService.getThreadPool();
    final Releasable replicaRelease = blockReplicas(replicaThreadPool);
    final BulkRequest bulkRequest = new BulkRequest();
    int totalRequestSize = 0;
    for (int i = 0; i < 80; ++i) {
        IndexRequest request = new IndexRequest(INDEX_NAME).id(UUIDs.base64UUID()).source(Collections.singletonMap("key", randomAlphaOfLength(50)));
        totalRequestSize += request.ramBytesUsed();
        assertTrue(request.ramBytesUsed() > request.source().length());
        bulkRequest.add(request);
    }
    final long bulkShardRequestSize = totalRequestSize + (RamUsageEstimator.shallowSizeOfInstance(BulkItemRequest.class) * 80) + RamUsageEstimator.shallowSizeOfInstance(BulkShardRequest.class);
    try {
        final ActionFuture<BulkResponse> successFuture = client(coordinatingOnlyNode).bulk(bulkRequest);
        replicationSendPointReached.await();
        IndexService indexService = internalCluster().getInstance(IndicesService.class, primaryName).iterator().next();
        Index index = indexService.getIndexSettings().getIndex();
        ShardId shardId = new ShardId(index, 0);
        ShardIndexingPressureTracker primaryShardTracker = internalCluster().getInstance(IndexingPressureService.class, primaryName).getShardIndexingPressure().getShardIndexingPressureTracker(shardId);
        ShardIndexingPressureTracker replicaShardTracker = internalCluster().getInstance(IndexingPressureService.class, replicaName).getShardIndexingPressure().getShardIndexingPressureTracker(shardId);
        ShardIndexingPressureTracker coordinatingShardTracker = internalCluster().getInstance(IndexingPressureService.class, coordinatingOnlyNode).getShardIndexingPressure().getShardIndexingPressureTracker(shardId);
        assertThat(primaryShardTracker.getCommonOperationTracker().getCurrentCombinedCoordinatingAndPrimaryBytes(), equalTo(bulkShardRequestSize));
        assertThat(primaryShardTracker.getPrimaryOperationTracker().getStatsTracker().getCurrentBytes(), equalTo(bulkShardRequestSize));
        assertEquals(0, primaryShardTracker.getCoordinatingOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, primaryShardTracker.getReplicaOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, replicaShardTracker.getCommonOperationTracker().getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertEquals(0, replicaShardTracker.getCoordinatingOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, replicaShardTracker.getPrimaryOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, replicaShardTracker.getReplicaOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(bulkShardRequestSize, coordinatingShardTracker.getCommonOperationTracker().getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertEquals(bulkShardRequestSize, coordinatingShardTracker.getCoordinatingOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, coordinatingShardTracker.getPrimaryOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, coordinatingShardTracker.getReplicaOperationTracker().getStatsTracker().getCurrentBytes());
        latchBlockingReplicationSend.countDown();
        IndexRequest request = new IndexRequest(INDEX_NAME).id(UUIDs.base64UUID()).source(Collections.singletonMap("key", randomAlphaOfLength(50)));
        final BulkRequest secondBulkRequest = new BulkRequest();
        secondBulkRequest.add(request);
        // Use the primary or the replica data node as the coordinating node this time
        boolean usePrimaryAsCoordinatingNode = randomBoolean();
        final ActionFuture<BulkResponse> secondFuture;
        if (usePrimaryAsCoordinatingNode) {
            secondFuture = client(primaryName).bulk(secondBulkRequest);
        } else {
            secondFuture = client(replicaName).bulk(secondBulkRequest);
        }
        final long secondBulkShardRequestSize = request.ramBytesUsed() + RamUsageEstimator.shallowSizeOfInstance(BulkItemRequest.class) + RamUsageEstimator.shallowSizeOfInstance(BulkShardRequest.class);
        if (usePrimaryAsCoordinatingNode) {
            assertBusy(() -> {
                assertThat(primaryShardTracker.getCommonOperationTracker().getCurrentCombinedCoordinatingAndPrimaryBytes(), equalTo(bulkShardRequestSize + secondBulkShardRequestSize));
                assertEquals(secondBulkShardRequestSize, primaryShardTracker.getCoordinatingOperationTracker().getStatsTracker().getCurrentBytes());
                assertThat(primaryShardTracker.getPrimaryOperationTracker().getStatsTracker().getCurrentBytes(), equalTo(bulkShardRequestSize + secondBulkShardRequestSize));
                assertEquals(0, replicaShardTracker.getCommonOperationTracker().getCurrentCombinedCoordinatingAndPrimaryBytes());
                assertEquals(0, replicaShardTracker.getCoordinatingOperationTracker().getStatsTracker().getCurrentBytes());
                assertEquals(0, replicaShardTracker.getPrimaryOperationTracker().getStatsTracker().getCurrentBytes());
            });
        } else {
            assertThat(primaryShardTracker.getCommonOperationTracker().getCurrentCombinedCoordinatingAndPrimaryBytes(), equalTo(bulkShardRequestSize));
            assertEquals(secondBulkShardRequestSize, replicaShardTracker.getCommonOperationTracker().getCurrentCombinedCoordinatingAndPrimaryBytes());
            assertEquals(secondBulkShardRequestSize, replicaShardTracker.getCoordinatingOperationTracker().getStatsTracker().getCurrentBytes());
            assertEquals(0, replicaShardTracker.getPrimaryOperationTracker().getStatsTracker().getCurrentBytes());
        }
        assertEquals(bulkShardRequestSize, coordinatingShardTracker.getCommonOperationTracker().getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertBusy(() -> assertThat(replicaShardTracker.getReplicaOperationTracker().getStatsTracker().getCurrentBytes(), equalTo(bulkShardRequestSize + secondBulkShardRequestSize)));
        replicaRelease.close();
        successFuture.actionGet();
        secondFuture.actionGet();
        assertEquals(0, primaryShardTracker.getCommonOperationTracker().getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertEquals(0, primaryShardTracker.getCoordinatingOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, primaryShardTracker.getPrimaryOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, primaryShardTracker.getReplicaOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, replicaShardTracker.getCommonOperationTracker().getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertEquals(0, replicaShardTracker.getCoordinatingOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, replicaShardTracker.getReplicaOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, replicaShardTracker.getReplicaOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, coordinatingShardTracker.getCommonOperationTracker().getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertEquals(0, coordinatingShardTracker.getCoordinatingOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, coordinatingShardTracker.getPrimaryOperationTracker().getStatsTracker().getCurrentBytes());
        assertEquals(0, coordinatingShardTracker.getReplicaOperationTracker().getStatsTracker().getCurrentBytes());
    } finally {
        if (replicationSendPointReached.getCount() > 0) {
            replicationSendPointReached.countDown();
        }
        replicaRelease.close();
        if (latchBlockingReplicationSend.getCount() > 0) {
            latchBlockingReplicationSend.countDown();
        }
        replicaRelease.close();
        primaryTransportService.clearAllRules();
    }
}

Example 5 with ThreadPool

use of org.opensearch.threadpool.ThreadPool in project OpenSearch by opensearch-project.

the class ShardIndexingPressureSettingsIT method testShardIndexingPressureRequestSizeWindowSettingUpdate.

public void testShardIndexingPressureRequestSizeWindowSettingUpdate() throws Exception {
    final BulkRequest bulkRequest = new BulkRequest();
    int totalRequestSize = 0;
    for (int i = 0; i < 80; ++i) {
        IndexRequest request = new IndexRequest(INDEX_NAME).id(UUIDs.base64UUID()).source(Collections.singletonMap("key", randomAlphaOfLength(50)));
        totalRequestSize += request.ramBytesUsed();
        assertTrue(request.ramBytesUsed() > request.source().length());
        bulkRequest.add(request);
    }
    final long bulkShardRequestSize = totalRequestSize + (RamUsageEstimator.shallowSizeOfInstance(BulkItemRequest.class) * 80) + RamUsageEstimator.shallowSizeOfInstance(BulkShardRequest.class);
    Settings settings = Settings.builder().put(ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENABLED.getKey(), true).put(ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENFORCED.getKey(), true).put(IndexingPressure.MAX_INDEXING_BYTES.getKey(), (long) (bulkShardRequestSize * 1.2) + "B").put(ShardIndexingPressureSettings.REQUEST_SIZE_WINDOW.getKey(), 1).put(ShardIndexingPressureMemoryManager.THROUGHPUT_DEGRADATION_LIMITS.getKey(), 1).build();
    restartCluster(settings);
    assertAcked(prepareCreate(INDEX_NAME, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)));
    ensureGreen(INDEX_NAME);
    Tuple<String, String> primaryReplicaNodeNames = getPrimaryReplicaNodeNames(INDEX_NAME);
    String primaryName = primaryReplicaNodeNames.v1();
    String replicaName = primaryReplicaNodeNames.v2();
    String coordinatingOnlyNode = getCoordinatingOnlyNode();
    IndexService indexService = internalCluster().getInstance(IndicesService.class, primaryName).iterator().next();
    Index index = indexService.getIndexSettings().getIndex();
    ShardId shardId = new ShardId(index, 0);
    boolean randomBoolean = randomBoolean();
    // Send first request which gets successful
    ActionFuture<BulkResponse> successFuture;
    if (randomBoolean) {
        successFuture = client(coordinatingOnlyNode).bulk(bulkRequest);
    } else {
        successFuture = client(primaryName).bulk(bulkRequest);
    }
    successFuture.actionGet();
    final CountDownLatch replicationSendPointReached = new CountDownLatch(1);
    final CountDownLatch latchBlockingReplicationSend = new CountDownLatch(1);
    TransportService primaryService = internalCluster().getInstance(TransportService.class, primaryName);
    final MockTransportService primaryTransportService = (MockTransportService) primaryService;
    TransportService replicaService = internalCluster().getInstance(TransportService.class, replicaName);
    final MockTransportService replicaTransportService = (MockTransportService) replicaService;
    primaryTransportService.addSendBehavior((connection, requestId, action, request, options) -> {
        if (action.equals(TransportShardBulkAction.ACTION_NAME + "[r]")) {
            try {
                replicationSendPointReached.countDown();
                latchBlockingReplicationSend.await();
            } catch (InterruptedException e) {
                throw new IllegalStateException(e);
            }
        }
        connection.sendRequest(requestId, action, request, options);
    });
    final ThreadPool replicaThreadPool = replicaTransportService.getThreadPool();
    final Releasable replicaRelease = blockReplicas(replicaThreadPool);
    // Send one more requests which remains outstanding for delayed time
    if (randomBoolean) {
        successFuture = client(coordinatingOnlyNode).bulk(bulkRequest);
    } else {
        successFuture = client(primaryName).bulk(bulkRequest);
    }
    // Delay to breach the success time stamp threshold
    Thread.sleep(3000);
    latchBlockingReplicationSend.countDown();
    replicaRelease.close();
    successFuture.actionGet();
    // This request breaches the threshold and hence will be rejected
    if (randomBoolean) {
        expectThrows(OpenSearchRejectedExecutionException.class, () -> client(coordinatingOnlyNode).bulk(bulkRequest).actionGet());
    } else {
        expectThrows(OpenSearchRejectedExecutionException.class, () -> client(primaryName).bulk(bulkRequest).actionGet());
    }
    // Update the outstanding threshold setting to see no rejections
    ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
    updateSettingsRequest.persistentSettings(Settings.builder().put(ShardIndexingPressureSettings.REQUEST_SIZE_WINDOW.getKey(), 10));
    assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
    // Verify no rejection with similar request pattern
    if (randomBoolean) {
        successFuture = client(coordinatingOnlyNode).bulk(bulkRequest);
        successFuture.actionGet();
        successFuture = client(coordinatingOnlyNode).bulk(bulkRequest);
        Thread.sleep(10);
        successFuture.actionGet();
        client(coordinatingOnlyNode).bulk(bulkRequest).actionGet();
        ShardIndexingPressureTracker coordinatingShardTracker = internalCluster().getInstance(IndexingPressureService.class, coordinatingOnlyNode).getShardIndexingPressure().getShardIndexingPressureTracker(shardId);
        assertEquals(1, coordinatingShardTracker.getCoordinatingOperationTracker().getRejectionTracker().getTotalRejections());
        assertEquals(1, coordinatingShardTracker.getCoordinatingOperationTracker().getRejectionTracker().getThroughputDegradationLimitsBreachedRejections());
    } else {
        successFuture = client(primaryName).bulk(bulkRequest);
        successFuture.actionGet();
        successFuture = client(primaryName).bulk(bulkRequest);
        Thread.sleep(10);
        successFuture.actionGet();
        client(primaryName).bulk(bulkRequest).actionGet();
        ShardIndexingPressureTracker primaryShardTracker = internalCluster().getInstance(IndexingPressureService.class, primaryName).getShardIndexingPressure().getShardIndexingPressureTracker(shardId);
        assertEquals(1, primaryShardTracker.getCoordinatingOperationTracker().getRejectionTracker().getTotalRejections());
        assertEquals(1, primaryShardTracker.getCoordinatingOperationTracker().getRejectionTracker().getThroughputDegradationLimitsBreachedRejections());
    }
}