Examples with ShardIndexingPressureStats org.opensearch.index.stats.ShardIndexingPressureStats used on opensource projects

Example 1 with ShardIndexingPressureStats

use of org.opensearch.index.stats.ShardIndexingPressureStats in project OpenSearch by opensearch-project.

the class ShardIndexingPressureConcurrentExecutionTests method testReplicaThreadedUpdateToShardLimitsAndRejections.

public void testReplicaThreadedUpdateToShardLimitsAndRejections() throws Exception {
    final int NUM_THREADS = 100;
    final Thread[] threads = new Thread[NUM_THREADS];
    final Releasable[] releasables = new Releasable[NUM_THREADS];
    AtomicInteger rejectionCount = new AtomicInteger();
    ShardIndexingPressure shardIndexingPressure = new ShardIndexingPressure(settings, clusterService);
    Index index = new Index("IndexName", "UUID");
    ShardId shardId1 = new ShardId(index, 0);
    for (int i = 0; i < NUM_THREADS; i++) {
        int counter = i;
        threads[i] = new Thread(() -> {
            try {
                releasables[counter] = shardIndexingPressure.markReplicaOperationStarted(shardId1, 300, false);
            } catch (OpenSearchRejectedExecutionException e) {
                rejectionCount.addAndGet(1);
            }
        });
        threads[i].start();
    }
    for (Thread t : threads) {
        t.join();
    }
    IndexingPressureStats nodeStats = shardIndexingPressure.stats();
    assertEquals(rejectionCount.get(), nodeStats.getReplicaRejections());
    assertTrue(nodeStats.getCurrentReplicaBytes() < 50 * 300);
    ShardIndexingPressureStats shardStats = shardIndexingPressure.shardStats();
    assertTrue(shardStats.getIndexingPressureShardStats(shardId1).getCurrentReplicaBytes() < 50 * 300);
    for (Releasable releasable : releasables) {
        if (releasable != null) {
            releasable.close();
        }
    }
    nodeStats = shardIndexingPressure.stats();
    assertEquals(rejectionCount.get(), nodeStats.getReplicaRejections());
    assertEquals(0, nodeStats.getCurrentReplicaBytes());
    IndexingPressurePerShardStats shardStoreStats = shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId1);
    assertNull(shardStoreStats);
    shardStats = shardIndexingPressure.coldStats();
    assertEquals(rejectionCount.get(), shardStats.getIndexingPressureShardStats(shardId1).getReplicaNodeLimitsBreachedRejections());
    assertEquals(0, shardStats.getIndexingPressureShardStats(shardId1).getCurrentReplicaBytes());
    assertEquals(15, shardStats.getIndexingPressureShardStats(shardId1).getCurrentReplicaLimits());
}

Example 2 with ShardIndexingPressureStats

use of org.opensearch.index.stats.ShardIndexingPressureStats in project OpenSearch by opensearch-project.

the class ShardIndexingPressureConcurrentExecutionTests method testCoordinatingPrimaryThreadedUpdateToShardLimitsAndRejections.

public void testCoordinatingPrimaryThreadedUpdateToShardLimitsAndRejections() throws Exception {
    final int NUM_THREADS = 100;
    final Thread[] threads = new Thread[NUM_THREADS];
    final Releasable[] releasables = new Releasable[NUM_THREADS];
    AtomicInteger rejectionCount = new AtomicInteger();
    ShardIndexingPressure shardIndexingPressure = new ShardIndexingPressure(settings, clusterService);
    Index index = new Index("IndexName", "UUID");
    ShardId shardId1 = new ShardId(index, 0);
    boolean randomBoolean = randomBoolean();
    for (int i = 0; i < NUM_THREADS; i++) {
        int counter = i;
        threads[i] = new Thread(() -> {
            try {
                if (randomBoolean) {
                    releasables[counter] = shardIndexingPressure.markCoordinatingOperationStarted(shardId1, 200, false);
                } else {
                    releasables[counter] = shardIndexingPressure.markPrimaryOperationStarted(shardId1, 200, false);
                }
            } catch (OpenSearchRejectedExecutionException e) {
                rejectionCount.addAndGet(1);
            }
        });
        threads[i].start();
    }
    for (Thread t : threads) {
        t.join();
    }
    IndexingPressureStats nodeStats = shardIndexingPressure.stats();
    ShardIndexingPressureStats shardStats = shardIndexingPressure.shardStats();
    if (randomBoolean) {
        assertEquals(rejectionCount.get(), nodeStats.getCoordinatingRejections());
        assertTrue(shardStats.getIndexingPressureShardStats(shardId1).getCurrentCoordinatingBytes() < 50 * 200);
    } else {
        assertTrue(shardStats.getIndexingPressureShardStats(shardId1).getCurrentPrimaryBytes() < 50 * 200);
        assertEquals(rejectionCount.get(), nodeStats.getPrimaryRejections());
    }
    assertTrue(nodeStats.getCurrentCombinedCoordinatingAndPrimaryBytes() < 50 * 200);
    assertTrue(shardStats.getIndexingPressureShardStats(shardId1).getCurrentCombinedCoordinatingAndPrimaryBytes() < 50 * 200);
    for (Releasable releasable : releasables) {
        if (releasable != null) {
            releasable.close();
        }
    }
    nodeStats = shardIndexingPressure.stats();
    IndexingPressurePerShardStats shardStoreStats = shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId1);
    assertNull(shardStoreStats);
    shardStats = shardIndexingPressure.coldStats();
    if (randomBoolean) {
        assertEquals(rejectionCount.get(), nodeStats.getCoordinatingRejections());
        assertEquals(rejectionCount.get(), shardStats.getIndexingPressureShardStats(shardId1).getCoordinatingNodeLimitsBreachedRejections());
        assertEquals(0, shardStats.getIndexingPressureShardStats(shardId1).getCurrentCoordinatingBytes());
    } else {
        assertEquals(rejectionCount.get(), nodeStats.getPrimaryRejections());
        assertEquals(rejectionCount.get(), shardStats.getIndexingPressureShardStats(shardId1).getPrimaryNodeLimitsBreachedRejections());
        assertEquals(0, shardStats.getIndexingPressureShardStats(shardId1).getCurrentPrimaryBytes());
    }
    assertEquals(0, nodeStats.getCurrentCombinedCoordinatingAndPrimaryBytes());
    assertEquals(0, shardStats.getIndexingPressureShardStats(shardId1).getCurrentCombinedCoordinatingAndPrimaryBytes());
    assertEquals(10, shardStats.getIndexingPressureShardStats(shardId1).getCurrentPrimaryAndCoordinatingLimits());
}

Example 3 with ShardIndexingPressureStats

use of org.opensearch.index.stats.ShardIndexingPressureStats in project OpenSearch by opensearch-project.

the class ShardIndexingPressure method coldStats.

ShardIndexingPressureStats coldStats() {
    Map<ShardId, IndexingPressurePerShardStats> statsPerShard = new HashMap<>();
    boolean isEnforcedMode = shardIndexingPressureSettings.isShardIndexingPressureEnforced();
    for (Map.Entry<ShardId, ShardIndexingPressureTracker> shardEntry : memoryManager.getShardIndexingPressureColdStore().entrySet()) {
        IndexingPressurePerShardStats shardStats = new IndexingPressurePerShardStats(shardEntry.getValue(), isEnforcedMode);
        statsPerShard.put(shardEntry.getKey(), shardStats);
    }
    return new ShardIndexingPressureStats(statsPerShard, memoryManager.getTotalNodeLimitsBreachedRejections(), memoryManager.getTotalLastSuccessfulRequestLimitsBreachedRejections(), memoryManager.getTotalThroughputDegradationLimitsBreachedRejections(), shardIndexingPressureSettings.isShardIndexingPressureEnabled(), isEnforcedMode);
}

Example 4 with ShardIndexingPressureStats

use of org.opensearch.index.stats.ShardIndexingPressureStats in project OpenSearch by opensearch-project.

the class ShardIndexingPressure method shardStats.

ShardIndexingPressureStats shardStats() {
    Map<ShardId, IndexingPressurePerShardStats> statsPerShard = new HashMap<>();
    boolean isEnforcedMode = shardIndexingPressureSettings.isShardIndexingPressureEnforced();
    for (Map.Entry<ShardId, ShardIndexingPressureTracker> shardEntry : memoryManager.getShardIndexingPressureHotStore().entrySet()) {
        IndexingPressurePerShardStats shardStats = new IndexingPressurePerShardStats(shardEntry.getValue(), isEnforcedMode);
        statsPerShard.put(shardEntry.getKey(), shardStats);
    }
    return new ShardIndexingPressureStats(statsPerShard, memoryManager.getTotalNodeLimitsBreachedRejections(), memoryManager.getTotalLastSuccessfulRequestLimitsBreachedRejections(), memoryManager.getTotalThroughputDegradationLimitsBreachedRejections(), shardIndexingPressureSettings.isShardIndexingPressureEnabled(), isEnforcedMode);
}