Examples with IndexingPressureService org.opensearch.index.IndexingPressureService used on opensource projects

Example 1 with IndexingPressureService

use of org.opensearch.index.IndexingPressureService in project OpenSearch by opensearch-project.

the class TransportBulkActionIndicesThatCannotBeCreatedTests method indicesThatCannotBeCreatedTestCase.

private void indicesThatCannotBeCreatedTestCase(Set<String> expected, BulkRequest bulkRequest, Function<String, Boolean> shouldAutoCreate) {
    ClusterService clusterService = mock(ClusterService.class);
    ClusterState state = mock(ClusterState.class);
    when(state.getMetadata()).thenReturn(Metadata.EMPTY_METADATA);
    when(state.metadata()).thenReturn(Metadata.EMPTY_METADATA);
    when(clusterService.state()).thenReturn(state);
    DiscoveryNodes discoveryNodes = mock(DiscoveryNodes.class);
    when(state.getNodes()).thenReturn(discoveryNodes);
    when(discoveryNodes.getMinNodeVersion()).thenReturn(VersionUtils.randomCompatibleVersion(random(), Version.CURRENT));
    DiscoveryNode localNode = mock(DiscoveryNode.class);
    when(clusterService.localNode()).thenReturn(localNode);
    when(localNode.isIngestNode()).thenReturn(randomBoolean());
    final ThreadPool threadPool = mock(ThreadPool.class);
    final ExecutorService direct = OpenSearchExecutors.newDirectExecutorService();
    when(threadPool.executor(anyString())).thenReturn(direct);
    TransportBulkAction action = new TransportBulkAction(threadPool, mock(TransportService.class), clusterService, null, null, null, mock(ActionFilters.class), null, null, new IndexingPressureService(Settings.EMPTY, new ClusterService(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null)), new SystemIndices(emptyMap())) {

        @Override
        void executeBulk(Task task, BulkRequest bulkRequest, long startTimeNanos, ActionListener<BulkResponse> listener, AtomicArray<BulkItemResponse> responses, Map<String, IndexNotFoundException> indicesThatCannotBeCreated) {
            assertEquals(expected, indicesThatCannotBeCreated.keySet());
        }

        @Override
        boolean needToCheck() {
            // Use "null" to mean "no indices can be created so don't bother checking"
            return null != shouldAutoCreate;
        }

        @Override
        boolean shouldAutoCreate(String index, ClusterState state) {
            return shouldAutoCreate.apply(index);
        }

        @Override
        void createIndex(String index, TimeValue timeout, Version minNodeVersion, ActionListener<CreateIndexResponse> listener) {
            // If we try to create an index just immediately assume it worked
            listener.onResponse(new CreateIndexResponse(true, true, index) {
            });
        }
    };
    action.doExecute(null, bulkRequest, null);
}

Example 2 with IndexingPressureService

use of org.opensearch.index.IndexingPressureService in project OpenSearch by opensearch-project.

the class TransportWriteActionForIndexingPressureTests method testIndexingPressureOperationStartedForReplicaNode.

public void testIndexingPressureOperationStartedForReplicaNode() {
    final ShardId shardId = new ShardId("test", "_na_", 0);
    final ClusterState state = state(shardId.getIndexName(), true, ShardRoutingState.STARTED, ShardRoutingState.STARTED);
    setState(clusterService, state);
    final ShardRouting replicaRouting = state.getRoutingTable().shardRoutingTable(shardId).replicaShards().get(0);
    final ReplicationTask task = maybeTask();
    final Settings settings = Settings.builder().put(ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENABLED.getKey(), false).build();
    this.indexingPressureService = new IndexingPressureService(settings, clusterService);
    TestAction action = new TestAction(settings, "internal:testAction", transportService, clusterService, shardStateAction, threadPool);
    action.handleReplicaRequest(new TransportReplicationAction.ConcreteReplicaRequest<>(new TestRequest(), replicaRouting.allocationId().getId(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()), createTransportChannel(new PlainActionFuture<>()), task);
    IndexingPressurePerShardStats shardStats = this.indexingPressureService.shardStats(CommonStatsFlags.ALL).getIndexingPressureShardStats(shardId);
    assertPhase(task, "finished");
    assertTrue(Objects.isNull(shardStats));
}

Example 3 with IndexingPressureService

use of org.opensearch.index.IndexingPressureService in project OpenSearch by opensearch-project.

the class TransportResyncReplicationActionTests method testResyncDoesNotBlockOnPrimaryAction.

public void testResyncDoesNotBlockOnPrimaryAction() throws Exception {
    try (ClusterService clusterService = createClusterService(threadPool)) {
        final String indexName = randomAlphaOfLength(5);
        setState(clusterService, state(indexName, true, ShardRoutingState.STARTED));
        setState(clusterService, ClusterState.builder(clusterService.state()).blocks(ClusterBlocks.builder().addGlobalBlock(NoMasterBlockService.NO_MASTER_BLOCK_ALL).addIndexBlock(indexName, IndexMetadata.INDEX_WRITE_BLOCK)));
        try (MockNioTransport transport = new MockNioTransport(Settings.EMPTY, Version.CURRENT, threadPool, new NetworkService(emptyList()), PageCacheRecycler.NON_RECYCLING_INSTANCE, new NamedWriteableRegistry(emptyList()), new NoneCircuitBreakerService())) {
            final MockTransportService transportService = new MockTransportService(Settings.EMPTY, transport, threadPool, NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet());
            transportService.start();
            transportService.acceptIncomingRequests();
            final ShardStateAction shardStateAction = new ShardStateAction(clusterService, transportService, null, null, threadPool);
            final IndexMetadata indexMetadata = clusterService.state().metadata().index(indexName);
            final Index index = indexMetadata.getIndex();
            final ShardId shardId = new ShardId(index, 0);
            final IndexShardRoutingTable shardRoutingTable = clusterService.state().routingTable().shardRoutingTable(shardId);
            final ShardRouting primaryShardRouting = clusterService.state().routingTable().shardRoutingTable(shardId).primaryShard();
            final String allocationId = primaryShardRouting.allocationId().getId();
            final long primaryTerm = indexMetadata.primaryTerm(shardId.id());
            final AtomicInteger acquiredPermits = new AtomicInteger();
            final IndexShard indexShard = mock(IndexShard.class);
            when(indexShard.indexSettings()).thenReturn(new IndexSettings(indexMetadata, Settings.EMPTY));
            when(indexShard.shardId()).thenReturn(shardId);
            when(indexShard.routingEntry()).thenReturn(primaryShardRouting);
            when(indexShard.getPendingPrimaryTerm()).thenReturn(primaryTerm);
            when(indexShard.getOperationPrimaryTerm()).thenReturn(primaryTerm);
            when(indexShard.getActiveOperationsCount()).then(i -> acquiredPermits.get());
            doAnswer(invocation -> {
                ActionListener<Releasable> callback = (ActionListener<Releasable>) invocation.getArguments()[0];
                acquiredPermits.incrementAndGet();
                callback.onResponse(acquiredPermits::decrementAndGet);
                return null;
            }).when(indexShard).acquirePrimaryOperationPermit(any(ActionListener.class), anyString(), any(), eq(true));
            when(indexShard.getReplicationGroup()).thenReturn(new ReplicationGroup(shardRoutingTable, clusterService.state().metadata().index(index).inSyncAllocationIds(shardId.id()), shardRoutingTable.getAllAllocationIds(), 0));
            final IndexService indexService = mock(IndexService.class);
            when(indexService.getShard(eq(shardId.id()))).thenReturn(indexShard);
            final IndicesService indexServices = mock(IndicesService.class);
            when(indexServices.indexServiceSafe(eq(index))).thenReturn(indexService);
            final TransportResyncReplicationAction action = new TransportResyncReplicationAction(Settings.EMPTY, transportService, clusterService, indexServices, threadPool, shardStateAction, new ActionFilters(new HashSet<>()), new IndexingPressureService(Settings.EMPTY, clusterService), new SystemIndices(emptyMap()));
            assertThat(action.globalBlockLevel(), nullValue());
            assertThat(action.indexBlockLevel(), nullValue());
            final Task task = mock(Task.class);
            when(task.getId()).thenReturn(randomNonNegativeLong());
            final byte[] bytes = "{}".getBytes(Charset.forName("UTF-8"));
            final ResyncReplicationRequest request = new ResyncReplicationRequest(shardId, 42L, 100, new Translog.Operation[] { new Translog.Index("type", "id", 0, primaryTerm, 0L, bytes, null, -1) });
            final PlainActionFuture<ResyncReplicationResponse> listener = new PlainActionFuture<>();
            action.sync(request, task, allocationId, primaryTerm, listener);
            assertThat(listener.get().getShardInfo().getFailed(), equalTo(0));
            assertThat(listener.isDone(), is(true));
        }
    }
}

Example 4 with IndexingPressureService

use of org.opensearch.index.IndexingPressureService in project OpenSearch by opensearch-project.

the class TransportWriteActionForIndexingPressureTests method testIndexingPressureOperationStartedForPrimaryShard.

public void testIndexingPressureOperationStartedForPrimaryShard() {
    final ShardId shardId = new ShardId("test", "_na_", 0);
    final ClusterState state = state(shardId.getIndexName(), true, ShardRoutingState.STARTED, ShardRoutingState.STARTED);
    setState(clusterService, state);
    final ShardRouting replicaRouting = state.getRoutingTable().shardRoutingTable(shardId).replicaShards().get(0);
    final ReplicationTask task = maybeTask();
    final Settings settings = Settings.builder().put(ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENABLED.getKey(), true).build();
    this.indexingPressureService = new IndexingPressureService(settings, clusterService);
    TestAction action = new TestAction(settings, "internal:testActionWithExceptions", transportService, clusterService, shardStateAction, threadPool);
    action.handlePrimaryRequest(new TransportReplicationAction.ConcreteReplicaRequest<>(new TestRequest(), replicaRouting.allocationId().getId(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()), createTransportChannel(new PlainActionFuture<>()), task);
    CommonStatsFlags statsFlag = new CommonStatsFlags();
    statsFlag.includeAllShardIndexingPressureTrackers(true);
    IndexingPressurePerShardStats shardStats = this.indexingPressureService.shardStats(statsFlag).getIndexingPressureShardStats(shardId);
    assertPhase(task, "finished");
    assertTrue(!Objects.isNull(shardStats));
    assertEquals(100, shardStats.getTotalPrimaryBytes());
}

Example 5 with IndexingPressureService

use of org.opensearch.index.IndexingPressureService in project OpenSearch by opensearch-project.

the class TransportWriteActionForIndexingPressureTests method testIndexingPressureOperationStartedForLocalPrimaryNode.

public void testIndexingPressureOperationStartedForLocalPrimaryNode() {
    final ShardId shardId = new ShardId("test", "_na_", 0);
    final ClusterState state = state(shardId.getIndexName(), true, ShardRoutingState.STARTED, ShardRoutingState.STARTED);
    setState(clusterService, state);
    final ShardRouting replicaRouting = state.getRoutingTable().shardRoutingTable(shardId).replicaShards().get(0);
    final ReplicationTask task = maybeTask();
    final Settings settings = Settings.builder().put(ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENABLED.getKey(), false).build();
    this.indexingPressureService = new IndexingPressureService(settings, clusterService);
    TestAction action = new TestAction(settings, "internal:testAction", transportService, clusterService, shardStateAction, threadPool);
    action.handlePrimaryRequest(new TransportReplicationAction.ConcreteShardRequest<>(new TestRequest(), replicaRouting.allocationId().getId(), randomNonNegativeLong(), true, true), createTransportChannel(new PlainActionFuture<>()), task);
    IndexingPressurePerShardStats shardStats = this.indexingPressureService.shardStats(CommonStatsFlags.ALL).getIndexingPressureShardStats(shardId);
    assertPhase(task, "finished");
    assertTrue(Objects.isNull(shardStats));
}