Examples with IndexingPressureStats org.opensearch.index.stats.IndexingPressureStats used on opensource projects

Example 11 with IndexingPressureStats

use of org.opensearch.index.stats.IndexingPressureStats in project OpenSearch by opensearch-project.

the class IndexingPressureServiceTests method testLocalPrimaryOperationForIndexingPressure.

public void testLocalPrimaryOperationForIndexingPressure() {
    IndexingPressureService service = new IndexingPressureService(settings, clusterService);
    Index index = new Index("IndexName", "UUID");
    ShardId shardId = new ShardId(index, 0);
    Settings.Builder updated = Settings.builder();
    clusterSettings.updateDynamicSettings(Settings.builder().put(ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENABLED.getKey(), false).build(), Settings.builder().put(settings), updated, getTestClass().getName());
    clusterSettings.applySettings(updated.build());
    Releasable releasable = service.markPrimaryOperationLocalToCoordinatingNodeStarted(shardId, 1024);
    IndexingPressurePerShardStats shardStats = service.shardStats(CommonStatsFlags.ALL).getIndexingPressureShardStats(shardId);
    assertNull(shardStats);
    IndexingPressureStats nodeStats = service.nodeStats();
    assertEquals(1024, nodeStats.getCurrentPrimaryBytes());
    releasable.close();
}

Example 12 with IndexingPressureStats

use of org.opensearch.index.stats.IndexingPressureStats in project OpenSearch by opensearch-project.

the class ShardIndexingPressureTests method testReplicaShardRejectionViaThroughputDegradationParam.

public void testReplicaShardRejectionViaThroughputDegradationParam() throws InterruptedException {
    Settings settings = Settings.builder().put(IndexingPressure.MAX_INDEXING_BYTES.getKey(), "10KB").put(ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENABLED.getKey(), true).put(ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENFORCED.getKey(), true).put(ShardIndexingPressureMemoryManager.THROUGHPUT_DEGRADATION_LIMITS.getKey(), 1).put(ShardIndexingPressureSettings.REQUEST_SIZE_WINDOW.getKey(), 1).build();
    ShardIndexingPressure shardIndexingPressure = new ShardIndexingPressure(settings, clusterService);
    Index index = new Index("IndexName", "UUID");
    ShardId shardId = new ShardId(index, 0);
    try (Releasable replica = shardIndexingPressure.markReplicaOperationStarted(shardId, 1 * 1024, false);
        Releasable replica1 = shardIndexingPressure.markReplicaOperationStarted(shardId, 3 * 1024, false)) {
        assertEquals(4 * 1024, shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId).getCurrentReplicaBytes());
        assertEquals((long) (4 * 1024 / 0.85), shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId).getCurrentReplicaLimits());
        // Adding delay in the current in flight request to mimic throughput degradation
        Thread.sleep(100);
    }
    expectThrows(OpenSearchRejectedExecutionException.class, () -> shardIndexingPressure.markReplicaOperationStarted(shardId, 12 * 1024, false));
    IndexingPressurePerShardStats shardStats = shardIndexingPressure.coldStats().getIndexingPressureShardStats(shardId);
    assertEquals(1, shardStats.getReplicaRejections());
    assertEquals(1, shardStats.getReplicaThroughputDegradationLimitsBreachedRejections());
    assertEquals(0, shardStats.getCurrentReplicaBytes());
    assertEquals(4 * 1024, shardStats.getTotalReplicaBytes());
    assertEquals(15, shardStats.getCurrentReplicaLimits());
    IndexingPressureStats nodeStats = shardIndexingPressure.stats();
    assertEquals(1, nodeStats.getReplicaRejections());
    assertEquals(0, nodeStats.getCurrentReplicaBytes());
}

Example 13 with IndexingPressureStats

use of org.opensearch.index.stats.IndexingPressureStats in project OpenSearch by opensearch-project.

the class ShardIndexingPressureTests method testReplicaShardRejectionViaSuccessfulRequestsParam.

public void testReplicaShardRejectionViaSuccessfulRequestsParam() throws InterruptedException {
    Settings settings = Settings.builder().put(IndexingPressure.MAX_INDEXING_BYTES.getKey(), "10KB").put(ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENABLED.getKey(), true).put(ShardIndexingPressureMemoryManager.MAX_OUTSTANDING_REQUESTS.getKey(), 1).put(ShardIndexingPressureMemoryManager.SUCCESSFUL_REQUEST_ELAPSED_TIMEOUT.getKey(), "20ms").put(ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENFORCED.getKey(), true).build();
    ShardIndexingPressure shardIndexingPressure = new ShardIndexingPressure(settings, clusterService);
    Index index = new Index("IndexName", "UUID");
    ShardId shardId = new ShardId(index, 0);
    try (Releasable replica = shardIndexingPressure.markReplicaOperationStarted(shardId, 1 * 1024, false)) {
        assertEquals(1 * 1024, shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId).getCurrentReplicaBytes());
        assertEquals((long) (1 * 1024 / 0.85), shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId).getCurrentReplicaLimits());
    }
    IndexingPressurePerShardStats shardStats = shardIndexingPressure.coldStats().getIndexingPressureShardStats(shardId);
    assertEquals(0, shardStats.getCurrentReplicaBytes());
    assertEquals(1 * 1024, shardStats.getTotalReplicaBytes());
    assertEquals(15, shardStats.getCurrentReplicaLimits());
    Thread.sleep(25);
    // Total Bytes are 14*1024 and node limit is 15*1024
    try (Releasable replica = shardIndexingPressure.markReplicaOperationStarted(shardId, 10 * 1024, false);
        Releasable replica1 = shardIndexingPressure.markReplicaOperationStarted(shardId, 2 * 1024, false)) {
        expectThrows(OpenSearchRejectedExecutionException.class, () -> shardIndexingPressure.markReplicaOperationStarted(shardId, 2 * 1024, false));
    }
    shardStats = shardIndexingPressure.coldStats().getIndexingPressureShardStats(shardId);
    assertEquals(1, shardStats.getReplicaRejections());
    assertEquals(0, shardStats.getCurrentReplicaBytes());
    assertEquals(1, shardStats.getReplicaLastSuccessfulRequestLimitsBreachedRejections());
    IndexingPressureStats nodeStats = shardIndexingPressure.stats();
    assertEquals(1, nodeStats.getReplicaRejections());
    assertEquals(0, nodeStats.getCurrentReplicaBytes());
}

Example 14 with IndexingPressureStats

use of org.opensearch.index.stats.IndexingPressureStats in project OpenSearch by opensearch-project.

the class ShardIndexingPressureTests method testAvoidDoubleAccounting.

public void testAvoidDoubleAccounting() {
    ShardIndexingPressure shardIndexingPressure = new ShardIndexingPressure(settings, clusterService);
    Index index = new Index("IndexName", "UUID");
    ShardId shardId = new ShardId(index, 0);
    try (Releasable coordinating = shardIndexingPressure.markCoordinatingOperationStarted(shardId, 10, false);
        Releasable primary = shardIndexingPressure.markPrimaryOperationLocalToCoordinatingNodeStarted(shardId, 15)) {
        IndexingPressureStats nodeStats = shardIndexingPressure.stats();
        assertEquals(10, nodeStats.getCurrentCoordinatingBytes());
        assertEquals(15, nodeStats.getCurrentPrimaryBytes());
        assertEquals(10, nodeStats.getCurrentCombinedCoordinatingAndPrimaryBytes());
        IndexingPressurePerShardStats shardStats = shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId);
        assertEquals(10, shardStats.getCurrentCoordinatingBytes());
        assertEquals(15, shardStats.getCurrentPrimaryBytes());
        assertEquals(10, shardStats.getCurrentCombinedCoordinatingAndPrimaryBytes());
    }
    IndexingPressureStats nodeStats = shardIndexingPressure.stats();
    assertEquals(0, nodeStats.getCurrentCoordinatingBytes());
    assertEquals(0, nodeStats.getCurrentPrimaryBytes());
    assertEquals(0, nodeStats.getCurrentCombinedCoordinatingAndPrimaryBytes());
    assertEquals(10, nodeStats.getTotalCoordinatingBytes());
    assertEquals(15, nodeStats.getTotalPrimaryBytes());
    assertEquals(10, nodeStats.getTotalCombinedCoordinatingAndPrimaryBytes());
    IndexingPressurePerShardStats shardStoreStats = shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId);
    assertNull(shardStoreStats);
    IndexingPressurePerShardStats shardStats = shardIndexingPressure.coldStats().getIndexingPressureShardStats(shardId);
    assertEquals(0, shardStats.getCurrentCoordinatingBytes());
    assertEquals(0, shardStats.getCurrentPrimaryBytes());
    assertEquals(0, shardStats.getCurrentCombinedCoordinatingAndPrimaryBytes());
    assertEquals(10, shardStats.getTotalCoordinatingBytes());
    assertEquals(15, shardStats.getTotalPrimaryBytes());
    assertEquals(10, shardStats.getTotalCombinedCoordinatingAndPrimaryBytes());
}

Example 15 with IndexingPressureStats

use of org.opensearch.index.stats.IndexingPressureStats in project OpenSearch by opensearch-project.

the class ShardIndexingPressureTests method testCoordinatingPrimaryShardRejectionViaThroughputDegradationParam.

public void testCoordinatingPrimaryShardRejectionViaThroughputDegradationParam() throws InterruptedException {
    Settings settings = Settings.builder().put(IndexingPressure.MAX_INDEXING_BYTES.getKey(), "10KB").put(ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENABLED.getKey(), true).put(ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENFORCED.getKey(), true).put(ShardIndexingPressureMemoryManager.THROUGHPUT_DEGRADATION_LIMITS.getKey(), 1).put(ShardIndexingPressureSettings.REQUEST_SIZE_WINDOW.getKey(), 1).build();
    ShardIndexingPressure shardIndexingPressure = new ShardIndexingPressure(settings, clusterService);
    Index index = new Index("IndexName", "UUID");
    ShardId shardId = new ShardId(index, 0);
    boolean randomBoolean = randomBoolean();
    try (Releasable coordinating = shardIndexingPressure.markCoordinatingOperationStarted(shardId, 1 * 1024, false);
        Releasable coordinating1 = shardIndexingPressure.markCoordinatingOperationStarted(shardId, 3 * 1024, false);
        Releasable primary = shardIndexingPressure.markPrimaryOperationStarted(shardId, 1 * 1024, false);
        Releasable primary1 = shardIndexingPressure.markPrimaryOperationStarted(shardId, 3 * 1024, false)) {
        assertEquals(4 * 1024, shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId).getCurrentCoordinatingBytes());
        assertEquals(4 * 1024, shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId).getCurrentPrimaryBytes());
        assertEquals(8 * 1024, shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId).getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertEquals((long) (8 * 1024 / 0.85), shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId).getCurrentPrimaryAndCoordinatingLimits());
        // Adding delay in the current in flight request to mimic throughput degradation
        Thread.sleep(100);
    }
    if (randomBoolean) {
        expectThrows(OpenSearchRejectedExecutionException.class, () -> shardIndexingPressure.markCoordinatingOperationStarted(shardId, 8 * 1024, false));
    } else {
        expectThrows(OpenSearchRejectedExecutionException.class, () -> shardIndexingPressure.markPrimaryOperationStarted(shardId, 8 * 1024, false));
    }
    IndexingPressurePerShardStats shardStats = shardIndexingPressure.coldStats().getIndexingPressureShardStats(shardId);
    if (randomBoolean) {
        assertEquals(1, shardStats.getCoordinatingRejections());
        assertEquals(1, shardStats.getCoordinatingThroughputDegradationLimitsBreachedRejections());
        assertEquals(0, shardStats.getCurrentCoordinatingBytes());
        assertEquals(4 * 1024, shardStats.getTotalCoordinatingBytes());
    } else {
        assertEquals(1, shardStats.getPrimaryRejections());
        assertEquals(1, shardStats.getPrimaryThroughputDegradationLimitsBreachedRejections());
        assertEquals(0, shardStats.getCurrentPrimaryBytes());
        assertEquals(4 * 1024, shardStats.getTotalPrimaryBytes());
    }
    assertEquals(10, shardStats.getCurrentPrimaryAndCoordinatingLimits());
    assertEquals(0, shardStats.getCurrentCombinedCoordinatingAndPrimaryBytes());
    assertEquals(8 * 1024, shardStats.getTotalCombinedCoordinatingAndPrimaryBytes());
    IndexingPressureStats nodeStats = shardIndexingPressure.stats();
    if (randomBoolean) {
        assertEquals(1, nodeStats.getCoordinatingRejections());
        assertEquals(0, nodeStats.getCurrentCoordinatingBytes());
    } else {
        assertEquals(1, nodeStats.getPrimaryRejections());
        assertEquals(0, nodeStats.getCurrentPrimaryBytes());
    }
}