Examples with IndexingPressurePerShardStats org.opensearch.index.stats.IndexingPressurePerShardStats used on opensource projects

Example 1 with IndexingPressurePerShardStats

use of org.opensearch.index.stats.IndexingPressurePerShardStats in project OpenSearch by opensearch-project.

the class ShardIndexingPressureConcurrentExecutionTests method testCoordinatingPrimaryThreadedUpdateToShardLimits.

public void testCoordinatingPrimaryThreadedUpdateToShardLimits() throws Exception {
    final int NUM_THREADS = scaledRandomIntBetween(100, 500);
    ShardIndexingPressure shardIndexingPressure = new ShardIndexingPressure(settings, clusterService);
    Index index = new Index("IndexName", "UUID");
    ShardId shardId1 = new ShardId(index, 0);
    boolean randomBoolean = randomBoolean();
    Releasable[] releasable;
    if (randomBoolean) {
        releasable = fireConcurrentRequests(NUM_THREADS, shardIndexingPressure, shardId1, 15, OperationType.COORDINATING);
    } else {
        releasable = fireConcurrentRequests(NUM_THREADS, shardIndexingPressure, shardId1, 15, OperationType.PRIMARY);
    }
    if (randomBoolean) {
        assertEquals(NUM_THREADS * 15, shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId1).getCurrentCoordinatingBytes());
    } else {
        assertEquals(NUM_THREADS * 15, shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId1).getCurrentPrimaryBytes());
    }
    assertEquals(NUM_THREADS * 15, shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId1).getCurrentCombinedCoordinatingAndPrimaryBytes());
    assertTrue((double) (NUM_THREADS * 15) / shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId1).getCurrentPrimaryAndCoordinatingLimits() < 0.95);
    assertTrue((double) (NUM_THREADS * 15) / shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId1).getCurrentPrimaryAndCoordinatingLimits() > 0.75);
    for (int i = 0; i < NUM_THREADS; i++) {
        releasable[i].close();
    }
    IndexingPressurePerShardStats shardStoreStats = shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId1);
    assertNull(shardStoreStats);
    if (randomBoolean) {
        assertEquals(0, shardIndexingPressure.coldStats().getIndexingPressureShardStats(shardId1).getCurrentCoordinatingBytes());
    } else {
        assertEquals(0, shardIndexingPressure.coldStats().getIndexingPressureShardStats(shardId1).getCurrentPrimaryBytes());
    }
    assertEquals(0, shardIndexingPressure.coldStats().getIndexingPressureShardStats(shardId1).getCurrentCombinedCoordinatingAndPrimaryBytes());
    assertEquals(10, shardIndexingPressure.coldStats().getIndexingPressureShardStats(shardId1).getCurrentPrimaryAndCoordinatingLimits());
}

Example 2 with IndexingPressurePerShardStats

use of org.opensearch.index.stats.IndexingPressurePerShardStats in project OpenSearch by opensearch-project.

the class ShardIndexingPressureConcurrentExecutionTests method testReplicaThreadedSimultaneousUpdateToShardLimits.

public void testReplicaThreadedSimultaneousUpdateToShardLimits() throws Exception {
    final int NUM_THREADS = scaledRandomIntBetween(100, 500);
    ShardIndexingPressure shardIndexingPressure = new ShardIndexingPressure(settings, clusterService);
    Index index = new Index("IndexName", "UUID");
    ShardId shardId1 = new ShardId(index, 0);
    fireAndCompleteConcurrentRequests(NUM_THREADS, shardIndexingPressure, shardId1, 100, OperationType.REPLICA);
    IndexingPressurePerShardStats shardStoreStats = shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId1);
    assertNull(shardStoreStats);
    assertEquals(0, shardIndexingPressure.coldStats().getIndexingPressureShardStats(shardId1).getCurrentReplicaBytes());
    assertEquals(15, shardIndexingPressure.coldStats().getIndexingPressureShardStats(shardId1).getCurrentReplicaLimits());
}

Example 3 with IndexingPressurePerShardStats

use of org.opensearch.index.stats.IndexingPressurePerShardStats in project OpenSearch by opensearch-project.

the class ShardIndexingPressureConcurrentExecutionTests method testReplicaThreadedUpdateToShardLimitsAndRejections.

public void testReplicaThreadedUpdateToShardLimitsAndRejections() throws Exception {
    final int NUM_THREADS = 100;
    final Thread[] threads = new Thread[NUM_THREADS];
    final Releasable[] releasables = new Releasable[NUM_THREADS];
    AtomicInteger rejectionCount = new AtomicInteger();
    ShardIndexingPressure shardIndexingPressure = new ShardIndexingPressure(settings, clusterService);
    Index index = new Index("IndexName", "UUID");
    ShardId shardId1 = new ShardId(index, 0);
    for (int i = 0; i < NUM_THREADS; i++) {
        int counter = i;
        threads[i] = new Thread(() -> {
            try {
                releasables[counter] = shardIndexingPressure.markReplicaOperationStarted(shardId1, 300, false);
            } catch (OpenSearchRejectedExecutionException e) {
                rejectionCount.addAndGet(1);
            }
        });
        threads[i].start();
    }
    for (Thread t : threads) {
        t.join();
    }
    IndexingPressureStats nodeStats = shardIndexingPressure.stats();
    assertEquals(rejectionCount.get(), nodeStats.getReplicaRejections());
    assertTrue(nodeStats.getCurrentReplicaBytes() < 50 * 300);
    ShardIndexingPressureStats shardStats = shardIndexingPressure.shardStats();
    assertTrue(shardStats.getIndexingPressureShardStats(shardId1).getCurrentReplicaBytes() < 50 * 300);
    for (Releasable releasable : releasables) {
        if (releasable != null) {
            releasable.close();
        }
    }
    nodeStats = shardIndexingPressure.stats();
    assertEquals(rejectionCount.get(), nodeStats.getReplicaRejections());
    assertEquals(0, nodeStats.getCurrentReplicaBytes());
    IndexingPressurePerShardStats shardStoreStats = shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId1);
    assertNull(shardStoreStats);
    shardStats = shardIndexingPressure.coldStats();
    assertEquals(rejectionCount.get(), shardStats.getIndexingPressureShardStats(shardId1).getReplicaNodeLimitsBreachedRejections());
    assertEquals(0, shardStats.getIndexingPressureShardStats(shardId1).getCurrentReplicaBytes());
    assertEquals(15, shardStats.getIndexingPressureShardStats(shardId1).getCurrentReplicaLimits());
}

Example 4 with IndexingPressurePerShardStats

use of org.opensearch.index.stats.IndexingPressurePerShardStats in project OpenSearch by opensearch-project.

the class ShardIndexingPressureConcurrentExecutionTests method testCoordinatingPrimaryThreadedSimultaneousUpdateToShardLimits.

public void testCoordinatingPrimaryThreadedSimultaneousUpdateToShardLimits() throws Exception {
    final int NUM_THREADS = scaledRandomIntBetween(100, 500);
    ShardIndexingPressure shardIndexingPressure = new ShardIndexingPressure(settings, clusterService);
    Index index = new Index("IndexName", "UUID");
    ShardId shardId1 = new ShardId(index, 0);
    boolean randomBoolean = randomBoolean();
    if (randomBoolean) {
        fireAndCompleteConcurrentRequests(NUM_THREADS, shardIndexingPressure, shardId1, 100, OperationType.COORDINATING);
    } else {
        fireAndCompleteConcurrentRequests(NUM_THREADS, shardIndexingPressure, shardId1, 100, OperationType.PRIMARY);
    }
    IndexingPressurePerShardStats shardStoreStats = shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId1);
    assertNull(shardStoreStats);
    if (randomBoolean) {
        assertEquals(0, shardIndexingPressure.coldStats().getIndexingPressureShardStats(shardId1).getCurrentCoordinatingBytes());
    } else {
        assertEquals(0, shardIndexingPressure.coldStats().getIndexingPressureShardStats(shardId1).getCurrentPrimaryBytes());
    }
    assertEquals(0, shardIndexingPressure.coldStats().getIndexingPressureShardStats(shardId1).getCurrentCombinedCoordinatingAndPrimaryBytes());
    assertEquals(10, shardIndexingPressure.coldStats().getIndexingPressureShardStats(shardId1).getCurrentPrimaryAndCoordinatingLimits());
}

Example 5 with IndexingPressurePerShardStats

use of org.opensearch.index.stats.IndexingPressurePerShardStats in project OpenSearch by opensearch-project.

the class ShardIndexingPressureConcurrentExecutionTests method testCoordinatingPrimaryThreadedUpdateToShardLimitsAndRejections.

public void testCoordinatingPrimaryThreadedUpdateToShardLimitsAndRejections() throws Exception {
    final int NUM_THREADS = 100;
    final Thread[] threads = new Thread[NUM_THREADS];
    final Releasable[] releasables = new Releasable[NUM_THREADS];
    AtomicInteger rejectionCount = new AtomicInteger();
    ShardIndexingPressure shardIndexingPressure = new ShardIndexingPressure(settings, clusterService);
    Index index = new Index("IndexName", "UUID");
    ShardId shardId1 = new ShardId(index, 0);
    boolean randomBoolean = randomBoolean();
    for (int i = 0; i < NUM_THREADS; i++) {
        int counter = i;
        threads[i] = new Thread(() -> {
            try {
                if (randomBoolean) {
                    releasables[counter] = shardIndexingPressure.markCoordinatingOperationStarted(shardId1, 200, false);
                } else {
                    releasables[counter] = shardIndexingPressure.markPrimaryOperationStarted(shardId1, 200, false);
                }
            } catch (OpenSearchRejectedExecutionException e) {
                rejectionCount.addAndGet(1);
            }
        });
        threads[i].start();
    }
    for (Thread t : threads) {
        t.join();
    }
    IndexingPressureStats nodeStats = shardIndexingPressure.stats();
    ShardIndexingPressureStats shardStats = shardIndexingPressure.shardStats();
    if (randomBoolean) {
        assertEquals(rejectionCount.get(), nodeStats.getCoordinatingRejections());
        assertTrue(shardStats.getIndexingPressureShardStats(shardId1).getCurrentCoordinatingBytes() < 50 * 200);
    } else {
        assertTrue(shardStats.getIndexingPressureShardStats(shardId1).getCurrentPrimaryBytes() < 50 * 200);
        assertEquals(rejectionCount.get(), nodeStats.getPrimaryRejections());
    }
    assertTrue(nodeStats.getCurrentCombinedCoordinatingAndPrimaryBytes() < 50 * 200);
    assertTrue(shardStats.getIndexingPressureShardStats(shardId1).getCurrentCombinedCoordinatingAndPrimaryBytes() < 50 * 200);
    for (Releasable releasable : releasables) {
        if (releasable != null) {
            releasable.close();
        }
    }
    nodeStats = shardIndexingPressure.stats();
    IndexingPressurePerShardStats shardStoreStats = shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId1);
    assertNull(shardStoreStats);
    shardStats = shardIndexingPressure.coldStats();
    if (randomBoolean) {
        assertEquals(rejectionCount.get(), nodeStats.getCoordinatingRejections());
        assertEquals(rejectionCount.get(), shardStats.getIndexingPressureShardStats(shardId1).getCoordinatingNodeLimitsBreachedRejections());
        assertEquals(0, shardStats.getIndexingPressureShardStats(shardId1).getCurrentCoordinatingBytes());
    } else {
        assertEquals(rejectionCount.get(), nodeStats.getPrimaryRejections());
        assertEquals(rejectionCount.get(), shardStats.getIndexingPressureShardStats(shardId1).getPrimaryNodeLimitsBreachedRejections());
        assertEquals(0, shardStats.getIndexingPressureShardStats(shardId1).getCurrentPrimaryBytes());
    }
    assertEquals(0, nodeStats.getCurrentCombinedCoordinatingAndPrimaryBytes());
    assertEquals(0, shardStats.getIndexingPressureShardStats(shardId1).getCurrentCombinedCoordinatingAndPrimaryBytes());
    assertEquals(10, shardStats.getIndexingPressureShardStats(shardId1).getCurrentPrimaryAndCoordinatingLimits());
}