Examples with IndexingPressureStats org.opensearch.index.stats.IndexingPressureStats used on opensource projects

Example 1 with IndexingPressureStats

use of org.opensearch.index.stats.IndexingPressureStats in project OpenSearch by opensearch-project.

the class ShardIndexingPressureConcurrentExecutionTests method testReplicaThreadedUpdateToShardLimitsAndRejections.

public void testReplicaThreadedUpdateToShardLimitsAndRejections() throws Exception {
    final int NUM_THREADS = 100;
    final Thread[] threads = new Thread[NUM_THREADS];
    final Releasable[] releasables = new Releasable[NUM_THREADS];
    AtomicInteger rejectionCount = new AtomicInteger();
    ShardIndexingPressure shardIndexingPressure = new ShardIndexingPressure(settings, clusterService);
    Index index = new Index("IndexName", "UUID");
    ShardId shardId1 = new ShardId(index, 0);
    for (int i = 0; i < NUM_THREADS; i++) {
        int counter = i;
        threads[i] = new Thread(() -> {
            try {
                releasables[counter] = shardIndexingPressure.markReplicaOperationStarted(shardId1, 300, false);
            } catch (OpenSearchRejectedExecutionException e) {
                rejectionCount.addAndGet(1);
            }
        });
        threads[i].start();
    }
    for (Thread t : threads) {
        t.join();
    }
    IndexingPressureStats nodeStats = shardIndexingPressure.stats();
    assertEquals(rejectionCount.get(), nodeStats.getReplicaRejections());
    assertTrue(nodeStats.getCurrentReplicaBytes() < 50 * 300);
    ShardIndexingPressureStats shardStats = shardIndexingPressure.shardStats();
    assertTrue(shardStats.getIndexingPressureShardStats(shardId1).getCurrentReplicaBytes() < 50 * 300);
    for (Releasable releasable : releasables) {
        if (releasable != null) {
            releasable.close();
        }
    }
    nodeStats = shardIndexingPressure.stats();
    assertEquals(rejectionCount.get(), nodeStats.getReplicaRejections());
    assertEquals(0, nodeStats.getCurrentReplicaBytes());
    IndexingPressurePerShardStats shardStoreStats = shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId1);
    assertNull(shardStoreStats);
    shardStats = shardIndexingPressure.coldStats();
    assertEquals(rejectionCount.get(), shardStats.getIndexingPressureShardStats(shardId1).getReplicaNodeLimitsBreachedRejections());
    assertEquals(0, shardStats.getIndexingPressureShardStats(shardId1).getCurrentReplicaBytes());
    assertEquals(15, shardStats.getIndexingPressureShardStats(shardId1).getCurrentReplicaLimits());
}

Example 2 with IndexingPressureStats

use of org.opensearch.index.stats.IndexingPressureStats in project OpenSearch by opensearch-project.

the class ShardIndexingPressureConcurrentExecutionTests method testCoordinatingPrimaryThreadedUpdateToShardLimitsAndRejections.

public void testCoordinatingPrimaryThreadedUpdateToShardLimitsAndRejections() throws Exception {
    final int NUM_THREADS = 100;
    final Thread[] threads = new Thread[NUM_THREADS];
    final Releasable[] releasables = new Releasable[NUM_THREADS];
    AtomicInteger rejectionCount = new AtomicInteger();
    ShardIndexingPressure shardIndexingPressure = new ShardIndexingPressure(settings, clusterService);
    Index index = new Index("IndexName", "UUID");
    ShardId shardId1 = new ShardId(index, 0);
    boolean randomBoolean = randomBoolean();
    for (int i = 0; i < NUM_THREADS; i++) {
        int counter = i;
        threads[i] = new Thread(() -> {
            try {
                if (randomBoolean) {
                    releasables[counter] = shardIndexingPressure.markCoordinatingOperationStarted(shardId1, 200, false);
                } else {
                    releasables[counter] = shardIndexingPressure.markPrimaryOperationStarted(shardId1, 200, false);
                }
            } catch (OpenSearchRejectedExecutionException e) {
                rejectionCount.addAndGet(1);
            }
        });
        threads[i].start();
    }
    for (Thread t : threads) {
        t.join();
    }
    IndexingPressureStats nodeStats = shardIndexingPressure.stats();
    ShardIndexingPressureStats shardStats = shardIndexingPressure.shardStats();
    if (randomBoolean) {
        assertEquals(rejectionCount.get(), nodeStats.getCoordinatingRejections());
        assertTrue(shardStats.getIndexingPressureShardStats(shardId1).getCurrentCoordinatingBytes() < 50 * 200);
    } else {
        assertTrue(shardStats.getIndexingPressureShardStats(shardId1).getCurrentPrimaryBytes() < 50 * 200);
        assertEquals(rejectionCount.get(), nodeStats.getPrimaryRejections());
    }
    assertTrue(nodeStats.getCurrentCombinedCoordinatingAndPrimaryBytes() < 50 * 200);
    assertTrue(shardStats.getIndexingPressureShardStats(shardId1).getCurrentCombinedCoordinatingAndPrimaryBytes() < 50 * 200);
    for (Releasable releasable : releasables) {
        if (releasable != null) {
            releasable.close();
        }
    }
    nodeStats = shardIndexingPressure.stats();
    IndexingPressurePerShardStats shardStoreStats = shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId1);
    assertNull(shardStoreStats);
    shardStats = shardIndexingPressure.coldStats();
    if (randomBoolean) {
        assertEquals(rejectionCount.get(), nodeStats.getCoordinatingRejections());
        assertEquals(rejectionCount.get(), shardStats.getIndexingPressureShardStats(shardId1).getCoordinatingNodeLimitsBreachedRejections());
        assertEquals(0, shardStats.getIndexingPressureShardStats(shardId1).getCurrentCoordinatingBytes());
    } else {
        assertEquals(rejectionCount.get(), nodeStats.getPrimaryRejections());
        assertEquals(rejectionCount.get(), shardStats.getIndexingPressureShardStats(shardId1).getPrimaryNodeLimitsBreachedRejections());
        assertEquals(0, shardStats.getIndexingPressureShardStats(shardId1).getCurrentPrimaryBytes());
    }
    assertEquals(0, nodeStats.getCurrentCombinedCoordinatingAndPrimaryBytes());
    assertEquals(0, shardStats.getIndexingPressureShardStats(shardId1).getCurrentCombinedCoordinatingAndPrimaryBytes());
    assertEquals(10, shardStats.getIndexingPressureShardStats(shardId1).getCurrentPrimaryAndCoordinatingLimits());
}

Example 3 with IndexingPressureStats

use of org.opensearch.index.stats.IndexingPressureStats in project OpenSearch by opensearch-project.

the class ShardIndexingPressureTests method testReplicaShardRejectionSkippedInShadowModeViaThroughputDegradationParam.

public void testReplicaShardRejectionSkippedInShadowModeViaThroughputDegradationParam() throws InterruptedException {
    Settings settings = Settings.builder().put(IndexingPressure.MAX_INDEXING_BYTES.getKey(), "10KB").put(ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENABLED.getKey(), true).put(ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENFORCED.getKey(), false).put(ShardIndexingPressureMemoryManager.THROUGHPUT_DEGRADATION_LIMITS.getKey(), 1).put(ShardIndexingPressureSettings.REQUEST_SIZE_WINDOW.getKey(), 1).build();
    ShardIndexingPressure shardIndexingPressure = new ShardIndexingPressure(settings, clusterService);
    Index index = new Index("IndexName", "UUID");
    ShardId shardId = new ShardId(index, 0);
    try (Releasable replica = shardIndexingPressure.markReplicaOperationStarted(shardId, 1 * 1024, false);
        Releasable replica1 = shardIndexingPressure.markReplicaOperationStarted(shardId, 3 * 1024, false)) {
        assertEquals(4 * 1024, shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId).getCurrentReplicaBytes());
        assertEquals((long) (4 * 1024 / 0.85), shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId).getCurrentReplicaLimits());
        // Adding delay in the current in flight request to mimic throughput degradation
        Thread.sleep(100);
    }
    Releasable replica = shardIndexingPressure.markReplicaOperationStarted(shardId, 12 * 1024, false);
    replica.close();
    IndexingPressurePerShardStats shardStats = shardIndexingPressure.coldStats().getIndexingPressureShardStats(shardId);
    assertEquals(0, shardStats.getReplicaRejections());
    assertEquals(1, shardStats.getReplicaThroughputDegradationLimitsBreachedRejections());
    assertEquals(0, shardStats.getCurrentReplicaBytes());
    assertEquals(16 * 1024, shardStats.getTotalReplicaBytes());
    assertEquals(15, shardStats.getCurrentReplicaLimits());
    IndexingPressureStats nodeStats = shardIndexingPressure.stats();
    assertEquals(0, nodeStats.getReplicaRejections());
    assertEquals(0, nodeStats.getCurrentReplicaBytes());
}

Example 4 with IndexingPressureStats

use of org.opensearch.index.stats.IndexingPressureStats in project OpenSearch by opensearch-project.

the class ShardIndexingPressureTests method testCoordinatingPrimaryRejections.

public void testCoordinatingPrimaryRejections() {
    ShardIndexingPressure shardIndexingPressure = new ShardIndexingPressure(settings, clusterService);
    Index index = new Index("IndexName", "UUID");
    ShardId shardId = new ShardId(index, 0);
    try (Releasable coordinating = shardIndexingPressure.markCoordinatingOperationStarted(shardId, 1024 * 3, false);
        Releasable primary = shardIndexingPressure.markPrimaryOperationStarted(shardId, 1024 * 3, false);
        Releasable replica = shardIndexingPressure.markReplicaOperationStarted(shardId, 1024 * 3, false)) {
        if (randomBoolean()) {
            expectThrows(OpenSearchRejectedExecutionException.class, () -> shardIndexingPressure.markCoordinatingOperationStarted(shardId, 1024 * 2, false));
            IndexingPressureStats nodeStats = shardIndexingPressure.stats();
            assertEquals(1, nodeStats.getCoordinatingRejections());
            assertEquals(1024 * 6, nodeStats.getCurrentCombinedCoordinatingAndPrimaryBytes());
            IndexingPressurePerShardStats shardStats = shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId);
            assertEquals(1, shardStats.getCoordinatingRejections());
            assertEquals(1024 * 6, shardStats.getCurrentCombinedCoordinatingAndPrimaryBytes());
            assertEquals(1, shardStats.getCoordinatingNodeLimitsBreachedRejections());
        } else {
            expectThrows(OpenSearchRejectedExecutionException.class, () -> shardIndexingPressure.markPrimaryOperationStarted(shardId, 1024 * 2, false));
            IndexingPressureStats nodeStats = shardIndexingPressure.stats();
            assertEquals(1, nodeStats.getPrimaryRejections());
            assertEquals(1024 * 6, nodeStats.getCurrentCombinedCoordinatingAndPrimaryBytes());
            IndexingPressurePerShardStats shardStats = shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId);
            assertEquals(1, shardStats.getPrimaryRejections());
            assertEquals(1024 * 6, nodeStats.getCurrentCombinedCoordinatingAndPrimaryBytes());
            assertEquals(1, shardStats.getPrimaryNodeLimitsBreachedRejections());
        }
        long preForceRejections = shardIndexingPressure.stats().getPrimaryRejections();
        long preForcedShardRejections = shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId).getPrimaryRejections();
        // Primary can be forced
        Releasable forced = shardIndexingPressure.markPrimaryOperationStarted(shardId, 1024 * 2, true);
        assertEquals(preForceRejections, shardIndexingPressure.stats().getPrimaryRejections());
        assertEquals(1024 * 8, shardIndexingPressure.stats().getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertEquals(preForcedShardRejections, shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId).getPrimaryRejections());
        assertEquals(1024 * 8, shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId).getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertEquals(preForcedShardRejections, shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId).getPrimaryNodeLimitsBreachedRejections());
        forced.close();
        // Local to coordinating node primary actions not rejected
        IndexingPressureStats preLocalNodeStats = shardIndexingPressure.stats();
        IndexingPressurePerShardStats preLocalShardStats = shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId);
        Releasable local = shardIndexingPressure.markPrimaryOperationLocalToCoordinatingNodeStarted(shardId, 1024 * 2);
        assertEquals(preLocalNodeStats.getPrimaryRejections(), shardIndexingPressure.stats().getPrimaryRejections());
        assertEquals(1024 * 6, shardIndexingPressure.stats().getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertEquals(preLocalNodeStats.getCurrentPrimaryBytes() + 1024 * 2, shardIndexingPressure.stats().getCurrentPrimaryBytes());
        assertEquals(preLocalShardStats.getPrimaryRejections(), shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId).getPrimaryRejections());
        assertEquals(1024 * 6, shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId).getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertEquals(preLocalShardStats.getCurrentPrimaryBytes() + 1024 * 2, shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId).getCurrentPrimaryBytes());
        assertEquals(preLocalShardStats.getPrimaryNodeLimitsBreachedRejections(), shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId).getPrimaryNodeLimitsBreachedRejections());
        local.close();
    }
    assertEquals(1024 * 8, shardIndexingPressure.stats().getTotalCombinedCoordinatingAndPrimaryBytes());
    assertNull(shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId));
    assertEquals(1024 * 8, shardIndexingPressure.coldStats().getIndexingPressureShardStats(shardId).getTotalCombinedCoordinatingAndPrimaryBytes());
}

Example 5 with IndexingPressureStats

use of org.opensearch.index.stats.IndexingPressureStats in project OpenSearch by opensearch-project.

the class ShardIndexingPressureTests method testCoordinatingPrimaryShardRejectionSkippedInShadowModeViaSuccessfulRequestsParam.

public void testCoordinatingPrimaryShardRejectionSkippedInShadowModeViaSuccessfulRequestsParam() throws InterruptedException {
    Settings settings = Settings.builder().put(IndexingPressure.MAX_INDEXING_BYTES.getKey(), "10KB").put(ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENABLED.getKey(), true).put(ShardIndexingPressureMemoryManager.MAX_OUTSTANDING_REQUESTS.getKey(), 1).put(ShardIndexingPressureMemoryManager.SUCCESSFUL_REQUEST_ELAPSED_TIMEOUT.getKey(), "20ms").put(ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENFORCED.getKey(), false).build();
    ShardIndexingPressure shardIndexingPressure = new ShardIndexingPressure(settings, clusterService);
    Index index = new Index("IndexName", "UUID");
    ShardId shardId = new ShardId(index, 0);
    boolean randomBoolean = randomBoolean();
    try (Releasable coordinating = shardIndexingPressure.markCoordinatingOperationStarted(shardId, 1 * 1024, false);
        Releasable primary = shardIndexingPressure.markPrimaryOperationStarted(shardId, 1 * 1024, false)) {
        assertEquals(1 * 1024, shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId).getCurrentCoordinatingBytes());
        assertEquals(1 * 1024, shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId).getCurrentPrimaryBytes());
        assertEquals(2 * 1024, shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId).getCurrentCombinedCoordinatingAndPrimaryBytes());
        assertEquals((long) (2 * 1024 / 0.85), shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId).getCurrentPrimaryAndCoordinatingLimits());
    }
    IndexingPressurePerShardStats shardStats = shardIndexingPressure.coldStats().getIndexingPressureShardStats(shardId);
    assertEquals(0, shardStats.getCurrentCoordinatingBytes());
    assertEquals(0, shardStats.getCurrentPrimaryBytes());
    assertEquals(0, shardStats.getCurrentCombinedCoordinatingAndPrimaryBytes());
    assertEquals(1 * 1024, shardStats.getTotalCoordinatingBytes());
    assertEquals(1 * 1024, shardStats.getTotalPrimaryBytes());
    assertEquals(2 * 1024, shardStats.getTotalCombinedCoordinatingAndPrimaryBytes());
    assertEquals(10, shardStats.getCurrentPrimaryAndCoordinatingLimits());
    Thread.sleep(25);
    // Total Bytes are 9*1024 and node limit is 10*1024
    if (randomBoolean) {
        try (Releasable coordinating = shardIndexingPressure.markCoordinatingOperationStarted(shardId, 7 * 1024, false);
            Releasable coordinating1 = shardIndexingPressure.markCoordinatingOperationStarted(shardId, 1 * 1024, false)) {
            Releasable coordinating2 = shardIndexingPressure.markCoordinatingOperationStarted(shardId, 1 * 1024, false);
            coordinating2.close();
        }
    } else {
        try (Releasable primary = shardIndexingPressure.markPrimaryOperationStarted(shardId, 7 * 1024, false);
            Releasable primary1 = shardIndexingPressure.markPrimaryOperationStarted(shardId, 1 * 1024, false)) {
            Releasable primary2 = shardIndexingPressure.markPrimaryOperationStarted(shardId, 1 * 1024, false);
            primary2.close();
        }
    }
    shardStats = shardIndexingPressure.coldStats().getIndexingPressureShardStats(shardId);
    if (randomBoolean) {
        assertEquals(0, shardStats.getCoordinatingRejections());
        assertEquals(0, shardStats.getCurrentCoordinatingBytes());
        assertEquals(1, shardStats.getCoordinatingLastSuccessfulRequestLimitsBreachedRejections());
    } else {
        assertEquals(0, shardStats.getPrimaryRejections());
        assertEquals(0, shardStats.getCurrentPrimaryBytes());
        assertEquals(1, shardStats.getPrimaryLastSuccessfulRequestLimitsBreachedRejections());
    }
    IndexingPressureStats nodeStats = shardIndexingPressure.stats();
    if (randomBoolean) {
        assertEquals(0, nodeStats.getCoordinatingRejections());
        assertEquals(0, nodeStats.getCurrentCoordinatingBytes());
    } else {
        assertEquals(0, nodeStats.getPrimaryRejections());
        assertEquals(0, nodeStats.getCurrentPrimaryBytes());
    }
}