Examples with ClusterSettings org.opensearch.common.settings.ClusterSettings used on opensource projects

Example 66 with ClusterSettings

use of org.opensearch.common.settings.ClusterSettings in project OpenSearch by opensearch-project.

the class HierarchyCircuitBreakerServiceTests method testAllocationBucketsBreaker.

public void testAllocationBucketsBreaker() {
    Settings clusterSettings = Settings.builder().put(HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "100b").put(HierarchyCircuitBreakerService.USE_REAL_MEMORY_USAGE_SETTING.getKey(), "false").build();
    try (HierarchyCircuitBreakerService service = new HierarchyCircuitBreakerService(clusterSettings, Collections.emptyList(), new ClusterSettings(clusterSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))) {
        long parentLimitBytes = service.getParentLimit();
        assertEquals(new ByteSizeValue(100, ByteSizeUnit.BYTES).getBytes(), parentLimitBytes);
        CircuitBreaker breaker = service.getBreaker(CircuitBreaker.REQUEST);
        MultiBucketConsumerService.MultiBucketConsumer multiBucketConsumer = new MultiBucketConsumerService.MultiBucketConsumer(10000, breaker);
        // make sure used bytes is greater than the total circuit breaker limit
        breaker.addWithoutBreaking(200);
        // make sure that we check on the the following call
        for (int i = 0; i < 1023; i++) {
            multiBucketConsumer.accept(0);
        }
        CircuitBreakingException exception = expectThrows(CircuitBreakingException.class, () -> multiBucketConsumer.accept(1024));
        assertThat(exception.getMessage(), containsString("[parent] Data too large, data for [allocated_buckets] would be"));
        assertThat(exception.getMessage(), containsString("which is larger than the limit of [100/100b]"));
    }
}

Example 67 with ClusterSettings

use of org.opensearch.common.settings.ClusterSettings in project OpenSearch by opensearch-project.

the class HierarchyCircuitBreakerServiceTests method testParentDoesOverLimitCheck.

public void testParentDoesOverLimitCheck() {
    long g1RegionSize = JvmInfo.jvmInfo().getG1RegionSize();
    Settings clusterSettings = Settings.builder().put(HierarchyCircuitBreakerService.USE_REAL_MEMORY_USAGE_SETTING.getKey(), Boolean.TRUE).put(HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "50%").build();
    boolean saveTheDay = randomBoolean();
    AtomicBoolean overLimitTriggered = new AtomicBoolean();
    final HierarchyCircuitBreakerService service = new HierarchyCircuitBreakerService(clusterSettings, Collections.emptyList(), new ClusterSettings(clusterSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), trackRealMemoryUsage -> memoryUsed -> {
        assertTrue(overLimitTriggered.compareAndSet(false, true));
        if (saveTheDay) {
            return new HierarchyCircuitBreakerService.MemoryUsage(memoryUsed.baseUsage / 2, memoryUsed.totalUsage - (memoryUsed.baseUsage / 2), memoryUsed.transientChildUsage, memoryUsed.permanentChildUsage);
        } else {
            return memoryUsed;
        }
    });
    int allocationSize = g1RegionSize > 0 ? (int) (g1RegionSize / 2) : 1024 * 1024;
    int allocationCount = (int) (JvmInfo.jvmInfo().getConfiguredMaxHeapSize() / allocationSize) + 1;
    List<byte[]> data = new ArrayList<>();
    try {
        for (int i = 0; i < allocationCount && overLimitTriggered.get() == false; ++i) {
            data.add(new byte[allocationSize]);
            service.checkParentLimit(0, "test");
        }
        assertTrue(saveTheDay);
    } catch (CircuitBreakingException e) {
        assertFalse(saveTheDay);
    }
    logger.trace("black hole [{}]", data.hashCode());
}

Example 68 with ClusterSettings

use of org.opensearch.common.settings.ClusterSettings in project OpenSearch by opensearch-project.

the class HierarchyCircuitBreakerServiceTests method testParentTriggersG1GCBeforeBreaking.

/**
 * "Integration test" checking that we ask the G1 over limit check before parent breaking.
 * Given that it depends on GC, the main assertion that we do not get a circuit breaking exception in the threads towards
 * the end of the test is not enabled. The following tests checks this in more unit test style.
 */
public void testParentTriggersG1GCBeforeBreaking() throws InterruptedException, TimeoutException, BrokenBarrierException {
    assumeTrue("Only G1GC can utilize the over limit check", JvmInfo.jvmInfo().useG1GC().equals("true"));
    long g1RegionSize = JvmInfo.jvmInfo().getG1RegionSize();
    assumeTrue("Must have region size", g1RegionSize > 0);
    Settings clusterSettings = Settings.builder().put(HierarchyCircuitBreakerService.USE_REAL_MEMORY_USAGE_SETTING.getKey(), Boolean.TRUE).put(HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "50%").build();
    AtomicInteger leaderTriggerCount = new AtomicInteger();
    AtomicReference<Consumer<Boolean>> onOverLimit = new AtomicReference<>(leader -> {
    });
    AtomicLong time = new AtomicLong(randomLongBetween(Long.MIN_VALUE / 2, Long.MAX_VALUE / 2));
    long interval = randomLongBetween(1, 1000);
    final HierarchyCircuitBreakerService service = new HierarchyCircuitBreakerService(clusterSettings, Collections.emptyList(), new ClusterSettings(clusterSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), trackRealMemoryUsage -> new HierarchyCircuitBreakerService.G1OverLimitStrategy(JvmInfo.jvmInfo(), HierarchyCircuitBreakerService::realMemoryUsage, HierarchyCircuitBreakerService.createYoungGcCountSupplier(), time::get, interval, TimeValue.timeValueSeconds(30)) {

        @Override
        void overLimitTriggered(boolean leader) {
            if (leader) {
                leaderTriggerCount.incrementAndGet();
            }
            onOverLimit.get().accept(leader);
        }
    });
    long maxHeap = JvmInfo.jvmInfo().getConfiguredMaxHeapSize();
    int regionCount = Math.toIntExact((maxHeap / 2 + g1RegionSize - 1) / g1RegionSize);
    // First setup a host of large byte[]'s, must be Humongous objects since those are cleaned during a young phase (no concurrent cycle
    // necessary, which is hard to control in the test).
    List<byte[]> data = new ArrayList<>();
    for (int i = 0; i < regionCount; ++i) {
        data.add(new byte[(int) (JvmInfo.jvmInfo().getG1RegionSize() / 2)]);
    }
    try {
        service.checkParentLimit(0, "test");
        fail("must exceed memory limit");
    } catch (CircuitBreakingException e) {
    // OK
    }
    time.addAndGet(randomLongBetween(interval, interval + 10));
    onOverLimit.set(leader -> {
        if (leader) {
            data.clear();
        }
    });
    logger.trace("black hole [{}]", data.hashCode());
    int threadCount = randomIntBetween(1, 10);
    CyclicBarrier barrier = new CyclicBarrier(threadCount + 1);
    List<Thread> threads = new ArrayList<>(threadCount);
    for (int i = 0; i < threadCount; ++i) {
        threads.add(new Thread(() -> {
            try {
                barrier.await(10, TimeUnit.SECONDS);
                service.checkParentLimit(0, "test-thread");
            } catch (InterruptedException | BrokenBarrierException | TimeoutException e) {
                throw new AssertionError(e);
            } catch (CircuitBreakingException e) {
                // very rare
                logger.info("Thread got semi-unexpected circuit breaking exception", e);
            }
        }));
    }
    threads.forEach(Thread::start);
    barrier.await(20, TimeUnit.SECONDS);
    for (Thread thread : threads) {
        thread.join(10000);
    }
    threads.forEach(thread -> assertFalse(thread.isAlive()));
    assertThat(leaderTriggerCount.get(), equalTo(2));
}

Example 69 with ClusterSettings

use of org.opensearch.common.settings.ClusterSettings in project OpenSearch by opensearch-project.

the class ShardLimitValidatorTests method createTestShardLimitService.

/**
 * Creates a {@link ShardLimitValidator} for testing with the given setting and a mocked cluster service.
 *
 * @param maxShardsPerNode the value to use for the max shards per node setting
 * @return a test instance
 */
public static ShardLimitValidator createTestShardLimitService(int maxShardsPerNode) {
    // Use a mocked clusterService - for unit tests we won't be updating the setting anyway.
    ClusterService clusterService = mock(ClusterService.class);
    Settings limitOnlySettings = Settings.builder().put(SETTING_CLUSTER_MAX_SHARDS_PER_NODE.getKey(), maxShardsPerNode).build();
    when(clusterService.getClusterSettings()).thenReturn(new ClusterSettings(limitOnlySettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
    return new ShardLimitValidator(limitOnlySettings, clusterService, new SystemIndices(emptyMap()));
}

Example 70 with ClusterSettings

use of org.opensearch.common.settings.ClusterSettings in project OpenSearch by opensearch-project.

the class ClusterServiceUtils method createMasterService.

public static MasterService createMasterService(ThreadPool threadPool, ClusterState initialClusterState) {
    MasterService clusterManagerService = new MasterService(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test_cluster_manager_node").build(), new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), threadPool);
    AtomicReference<ClusterState> clusterStateRef = new AtomicReference<>(initialClusterState);
    clusterManagerService.setClusterStatePublisher((event, publishListener, ackListener) -> {
        clusterStateRef.set(event.state());
        publishListener.onResponse(null);
    });
    clusterManagerService.setClusterStateSupplier(clusterStateRef::get);
    clusterManagerService.start();
    return clusterManagerService;
}