use of org.opensearch.index.stats.IndexingPressurePerShardStats in project OpenSearch by opensearch-project.
the class TransportWriteActionForIndexingPressureTests method testIndexingPressureOperationStartedForReplicaShard.
public void testIndexingPressureOperationStartedForReplicaShard() {
final ShardId shardId = new ShardId("test", "_na_", 0);
final ClusterState state = state(shardId.getIndexName(), true, ShardRoutingState.STARTED, ShardRoutingState.STARTED);
setState(clusterService, state);
final ShardRouting replicaRouting = state.getRoutingTable().shardRoutingTable(shardId).replicaShards().get(0);
final ReplicationTask task = maybeTask();
final Settings settings = Settings.builder().put(ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENABLED.getKey(), true).build();
this.indexingPressureService = new IndexingPressureService(settings, clusterService);
TestAction action = new TestAction(settings, "internal:testAction", transportService, clusterService, shardStateAction, threadPool);
action.handleReplicaRequest(new TransportReplicationAction.ConcreteReplicaRequest<>(new TestRequest(), replicaRouting.allocationId().getId(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()), createTransportChannel(new PlainActionFuture<>()), task);
CommonStatsFlags statsFlag = new CommonStatsFlags();
statsFlag.includeAllShardIndexingPressureTrackers(true);
IndexingPressurePerShardStats shardStats = this.indexingPressureService.shardStats(statsFlag).getIndexingPressureShardStats(shardId);
assertPhase(task, "finished");
assertTrue(!Objects.isNull(shardStats));
assertEquals(100, shardStats.getTotalReplicaBytes());
}
use of org.opensearch.index.stats.IndexingPressurePerShardStats in project OpenSearch by opensearch-project.
the class ShardIndexingPressure method coldStats.
ShardIndexingPressureStats coldStats() {
Map<ShardId, IndexingPressurePerShardStats> statsPerShard = new HashMap<>();
boolean isEnforcedMode = shardIndexingPressureSettings.isShardIndexingPressureEnforced();
for (Map.Entry<ShardId, ShardIndexingPressureTracker> shardEntry : memoryManager.getShardIndexingPressureColdStore().entrySet()) {
IndexingPressurePerShardStats shardStats = new IndexingPressurePerShardStats(shardEntry.getValue(), isEnforcedMode);
statsPerShard.put(shardEntry.getKey(), shardStats);
}
return new ShardIndexingPressureStats(statsPerShard, memoryManager.getTotalNodeLimitsBreachedRejections(), memoryManager.getTotalLastSuccessfulRequestLimitsBreachedRejections(), memoryManager.getTotalThroughputDegradationLimitsBreachedRejections(), shardIndexingPressureSettings.isShardIndexingPressureEnabled(), isEnforcedMode);
}
use of org.opensearch.index.stats.IndexingPressurePerShardStats in project OpenSearch by opensearch-project.
the class ShardIndexingPressure method shardStats.
ShardIndexingPressureStats shardStats() {
Map<ShardId, IndexingPressurePerShardStats> statsPerShard = new HashMap<>();
boolean isEnforcedMode = shardIndexingPressureSettings.isShardIndexingPressureEnforced();
for (Map.Entry<ShardId, ShardIndexingPressureTracker> shardEntry : memoryManager.getShardIndexingPressureHotStore().entrySet()) {
IndexingPressurePerShardStats shardStats = new IndexingPressurePerShardStats(shardEntry.getValue(), isEnforcedMode);
statsPerShard.put(shardEntry.getKey(), shardStats);
}
return new ShardIndexingPressureStats(statsPerShard, memoryManager.getTotalNodeLimitsBreachedRejections(), memoryManager.getTotalLastSuccessfulRequestLimitsBreachedRejections(), memoryManager.getTotalThroughputDegradationLimitsBreachedRejections(), shardIndexingPressureSettings.isShardIndexingPressureEnabled(), isEnforcedMode);
}
use of org.opensearch.index.stats.IndexingPressurePerShardStats in project OpenSearch by opensearch-project.
the class ShardIndexingPressureTests method testReplicaShardRejectionViaThroughputDegradationParam.
public void testReplicaShardRejectionViaThroughputDegradationParam() throws InterruptedException {
Settings settings = Settings.builder().put(IndexingPressure.MAX_INDEXING_BYTES.getKey(), "10KB").put(ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENABLED.getKey(), true).put(ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENFORCED.getKey(), true).put(ShardIndexingPressureMemoryManager.THROUGHPUT_DEGRADATION_LIMITS.getKey(), 1).put(ShardIndexingPressureSettings.REQUEST_SIZE_WINDOW.getKey(), 1).build();
ShardIndexingPressure shardIndexingPressure = new ShardIndexingPressure(settings, clusterService);
Index index = new Index("IndexName", "UUID");
ShardId shardId = new ShardId(index, 0);
try (Releasable replica = shardIndexingPressure.markReplicaOperationStarted(shardId, 1 * 1024, false);
Releasable replica1 = shardIndexingPressure.markReplicaOperationStarted(shardId, 3 * 1024, false)) {
assertEquals(4 * 1024, shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId).getCurrentReplicaBytes());
assertEquals((long) (4 * 1024 / 0.85), shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId).getCurrentReplicaLimits());
// Adding delay in the current in flight request to mimic throughput degradation
Thread.sleep(100);
}
expectThrows(OpenSearchRejectedExecutionException.class, () -> shardIndexingPressure.markReplicaOperationStarted(shardId, 12 * 1024, false));
IndexingPressurePerShardStats shardStats = shardIndexingPressure.coldStats().getIndexingPressureShardStats(shardId);
assertEquals(1, shardStats.getReplicaRejections());
assertEquals(1, shardStats.getReplicaThroughputDegradationLimitsBreachedRejections());
assertEquals(0, shardStats.getCurrentReplicaBytes());
assertEquals(4 * 1024, shardStats.getTotalReplicaBytes());
assertEquals(15, shardStats.getCurrentReplicaLimits());
IndexingPressureStats nodeStats = shardIndexingPressure.stats();
assertEquals(1, nodeStats.getReplicaRejections());
assertEquals(0, nodeStats.getCurrentReplicaBytes());
}
use of org.opensearch.index.stats.IndexingPressurePerShardStats in project OpenSearch by opensearch-project.
the class ShardIndexingPressureTests method testReplicaShardRejectionViaSuccessfulRequestsParam.
public void testReplicaShardRejectionViaSuccessfulRequestsParam() throws InterruptedException {
Settings settings = Settings.builder().put(IndexingPressure.MAX_INDEXING_BYTES.getKey(), "10KB").put(ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENABLED.getKey(), true).put(ShardIndexingPressureMemoryManager.MAX_OUTSTANDING_REQUESTS.getKey(), 1).put(ShardIndexingPressureMemoryManager.SUCCESSFUL_REQUEST_ELAPSED_TIMEOUT.getKey(), "20ms").put(ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENFORCED.getKey(), true).build();
ShardIndexingPressure shardIndexingPressure = new ShardIndexingPressure(settings, clusterService);
Index index = new Index("IndexName", "UUID");
ShardId shardId = new ShardId(index, 0);
try (Releasable replica = shardIndexingPressure.markReplicaOperationStarted(shardId, 1 * 1024, false)) {
assertEquals(1 * 1024, shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId).getCurrentReplicaBytes());
assertEquals((long) (1 * 1024 / 0.85), shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId).getCurrentReplicaLimits());
}
IndexingPressurePerShardStats shardStats = shardIndexingPressure.coldStats().getIndexingPressureShardStats(shardId);
assertEquals(0, shardStats.getCurrentReplicaBytes());
assertEquals(1 * 1024, shardStats.getTotalReplicaBytes());
assertEquals(15, shardStats.getCurrentReplicaLimits());
Thread.sleep(25);
// Total Bytes are 14*1024 and node limit is 15*1024
try (Releasable replica = shardIndexingPressure.markReplicaOperationStarted(shardId, 10 * 1024, false);
Releasable replica1 = shardIndexingPressure.markReplicaOperationStarted(shardId, 2 * 1024, false)) {
expectThrows(OpenSearchRejectedExecutionException.class, () -> shardIndexingPressure.markReplicaOperationStarted(shardId, 2 * 1024, false));
}
shardStats = shardIndexingPressure.coldStats().getIndexingPressureShardStats(shardId);
assertEquals(1, shardStats.getReplicaRejections());
assertEquals(0, shardStats.getCurrentReplicaBytes());
assertEquals(1, shardStats.getReplicaLastSuccessfulRequestLimitsBreachedRejections());
IndexingPressureStats nodeStats = shardIndexingPressure.stats();
assertEquals(1, nodeStats.getReplicaRejections());
assertEquals(0, nodeStats.getCurrentReplicaBytes());
}
Aggregations