Examples with ImmutableOpenMap org.opensearch.common.collect.ImmutableOpenMap used on opensource projects

Example 41 with ImmutableOpenMap

use of org.opensearch.common.collect.ImmutableOpenMap in project OpenSearch by opensearch-project.

the class DiskThresholdMonitorTests method testDoesNotSubmitRerouteTaskTooFrequently.

public void testDoesNotSubmitRerouteTaskTooFrequently() {
    final ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
    AtomicLong currentTime = new AtomicLong();
    AtomicReference<ActionListener<ClusterState>> listenerReference = new AtomicReference<>();
    DiskThresholdMonitor monitor = new DiskThresholdMonitor(Settings.EMPTY, () -> clusterState, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null, currentTime::get, (reason, priority, listener) -> {
        assertNotNull(listener);
        assertThat(priority, equalTo(Priority.HIGH));
        assertTrue(listenerReference.compareAndSet(null, listener));
    }) {

        @Override
        protected void updateIndicesReadOnly(Set<String> indicesToMarkReadOnly, ActionListener<Void> listener, boolean readOnly) {
            throw new AssertionError("unexpected");
        }
    };
    final ImmutableOpenMap.Builder<String, DiskUsage> allDisksOkBuilder;
    allDisksOkBuilder = ImmutableOpenMap.builder();
    allDisksOkBuilder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, 50));
    allDisksOkBuilder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, 50));
    final ImmutableOpenMap<String, DiskUsage> allDisksOk = allDisksOkBuilder.build();
    final ImmutableOpenMap.Builder<String, DiskUsage> oneDiskAboveWatermarkBuilder = ImmutableOpenMap.builder();
    oneDiskAboveWatermarkBuilder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(5, 9)));
    oneDiskAboveWatermarkBuilder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, 50));
    final ImmutableOpenMap<String, DiskUsage> oneDiskAboveWatermark = oneDiskAboveWatermarkBuilder.build();
    // should not reroute when all disks are ok
    currentTime.addAndGet(randomLongBetween(0, 120000));
    monitor.onNewInfo(clusterInfo(allDisksOk));
    assertNull(listenerReference.get());
    // should reroute when one disk goes over the watermark
    currentTime.addAndGet(randomLongBetween(0, 120000));
    monitor.onNewInfo(clusterInfo(oneDiskAboveWatermark));
    assertNotNull(listenerReference.get());
    listenerReference.getAndSet(null).onResponse(clusterState);
    if (randomBoolean()) {
        // should not re-route again within the reroute interval
        currentTime.addAndGet(randomLongBetween(0, DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING.get(Settings.EMPTY).millis()));
        monitor.onNewInfo(clusterInfo(allDisksOk));
        assertNull(listenerReference.get());
    }
    // should reroute again when one disk is still over the watermark
    currentTime.addAndGet(randomLongBetween(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING.get(Settings.EMPTY).millis() + 1, 120000));
    monitor.onNewInfo(clusterInfo(oneDiskAboveWatermark));
    assertNotNull(listenerReference.get());
    final ActionListener<ClusterState> rerouteListener1 = listenerReference.getAndSet(null);
    // should not re-route again before reroute has completed
    currentTime.addAndGet(randomLongBetween(0, 120000));
    monitor.onNewInfo(clusterInfo(allDisksOk));
    assertNull(listenerReference.get());
    // complete reroute
    rerouteListener1.onResponse(clusterState);
    if (randomBoolean()) {
        // should not re-route again within the reroute interval
        currentTime.addAndGet(randomLongBetween(0, DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING.get(Settings.EMPTY).millis()));
        monitor.onNewInfo(clusterInfo(allDisksOk));
        assertNull(listenerReference.get());
    }
    // should reroute again after the reroute interval
    currentTime.addAndGet(randomLongBetween(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING.get(Settings.EMPTY).millis() + 1, 120000));
    monitor.onNewInfo(clusterInfo(allDisksOk));
    assertNotNull(listenerReference.get());
    listenerReference.getAndSet(null).onResponse(null);
    // should not reroute again when it is not required
    currentTime.addAndGet(randomLongBetween(0, 120000));
    monitor.onNewInfo(clusterInfo(allDisksOk));
    assertNull(listenerReference.get());
    // should reroute again when one disk has reserved space that pushes it over the high watermark
    final ImmutableOpenMap.Builder<ClusterInfo.NodeAndPath, ClusterInfo.ReservedSpace> builder = ImmutableOpenMap.builder(1);
    builder.put(new ClusterInfo.NodeAndPath("node1", "/foo/bar"), new ClusterInfo.ReservedSpace.Builder().add(new ShardId("baz", "quux", 0), between(41, 100)).build());
    final ImmutableOpenMap<ClusterInfo.NodeAndPath, ClusterInfo.ReservedSpace> reservedSpaces = builder.build();
    currentTime.addAndGet(randomLongBetween(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING.get(Settings.EMPTY).millis() + 1, 120000));
    monitor.onNewInfo(clusterInfo(allDisksOk, reservedSpaces));
    assertNotNull(listenerReference.get());
    listenerReference.getAndSet(null).onResponse(null);
}

Example 42 with ImmutableOpenMap

use of org.opensearch.common.collect.ImmutableOpenMap in project OpenSearch by opensearch-project.

the class IndexShardConstraintDeciderOverlapTests method testHighWatermarkBreachWithLowShardCount.

/**
 * High watermark breach blocks new shard allocations to affected nodes. If shard count on such
 * nodes is low, this will cause IndexShardPerNodeConstraint to breach.
 *
 * This test verifies that this doesn't lead to unassigned shards, and there are no hot spots in eligible
 * nodes.
 */
public void testHighWatermarkBreachWithLowShardCount() {
    setupInitialCluster(3, 15, 10, 1);
    addNodesWithIndexing(1, "high_watermark_node_", 6, 5, 1);
    // Disk threshold settings enabled
    Settings settings = Settings.builder().put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true).put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), 0.7).put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), 0.8).put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), 0.95).put("cluster.routing.allocation.node_concurrent_recoveries", 1).put("cluster.routing.allocation.cluster_concurrent_recoveries", 1).build();
    // Build Shard size and disk usages
    ImmutableOpenMap.Builder<String, DiskUsage> usagesBuilder = ImmutableOpenMap.builder();
    // 20% used
    usagesBuilder.put("node_0", new DiskUsage("node_0", "node_0", "/dev/null", 100, 80));
    // 45% used
    usagesBuilder.put("node_1", new DiskUsage("node_1", "node_1", "/dev/null", 100, 55));
    // 65% used
    usagesBuilder.put("node_2", new DiskUsage("node_2", "node_2", "/dev/null", 100, 35));
    // 90%
    usagesBuilder.put("high_watermark_node_0", new DiskUsage("high_watermark_node_0", "high_watermark_node_0", "/dev/null", 100, 10));
    // used
    ImmutableOpenMap<String, DiskUsage> usages = usagesBuilder.build();
    ImmutableOpenMap.Builder<String, Long> shardSizesBuilder = ImmutableOpenMap.builder();
    // Each
    clusterState.getRoutingTable().allShards().forEach(shard -> shardSizesBuilder.put(shardIdentifierFromRouting(shard), 1L));
    // shard
    // is 1
    // byte
    ImmutableOpenMap<String, Long> shardSizes = shardSizesBuilder.build();
    final ImmutableOpenMap<ClusterInfo.NodeAndPath, ClusterInfo.ReservedSpace> reservedSpace = new ImmutableOpenMap.Builder<ClusterInfo.NodeAndPath, ClusterInfo.ReservedSpace>().fPut(getNodeAndDevNullPath("node_0"), getReservedSpace()).fPut(getNodeAndDevNullPath("node_1"), getReservedSpace()).fPut(getNodeAndDevNullPath("node_2"), getReservedSpace()).fPut(getNodeAndDevNullPath("high_watermark_node_0"), getReservedSpace()).build();
    final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes, reservedSpace);
    ClusterInfoService cis = () -> clusterInfo;
    allocation = createAllocationService(settings, cis);
    allocateAndCheckIndexShardHotSpots(false, 3, "node_0", "node_1", "node_2");
    assertForIndexShardHotSpots(true, 4);
    assertTrue(clusterState.getRoutingTable().shardsWithState(UNASSIGNED).isEmpty());
    assertTrue(clusterState.getRoutingNodes().node("high_watermark_node_0").isEmpty());
    /* Shard sizes that would breach high watermark on node_2 if allocated.
         */
    addIndices("big_index_", 1, 10, 0);
    ImmutableOpenMap.Builder<String, Long> bigIndexShardSizeBuilder = ImmutableOpenMap.builder(shardSizes);
    clusterState.getRoutingNodes().unassigned().forEach(shard -> bigIndexShardSizeBuilder.put(shardIdentifierFromRouting(shard), 20L));
    shardSizes = bigIndexShardSizeBuilder.build();
    final ClusterInfo bigIndexClusterInfo = new DevNullClusterInfo(usages, usages, shardSizes, reservedSpace);
    cis = () -> bigIndexClusterInfo;
    allocation = createAllocationService(settings, cis);
    allocateAndCheckIndexShardHotSpots(false, 2, "node_0", "node_1");
    assertForIndexShardHotSpots(true, 4);
    assertTrue(clusterState.getRoutingTable().shardsWithState(UNASSIGNED).isEmpty());
    for (ShardRouting shard : clusterState.getRoutingTable().index("big_index_0").shardsWithState(STARTED)) {
        assertNotEquals("node_2", shard.currentNodeId());
    }
}

Example 43 with ImmutableOpenMap

use of org.opensearch.common.collect.ImmutableOpenMap in project OpenSearch by opensearch-project.

the class SnapshotsInProgressSerializationTests method randomSnapshot.

private Entry randomSnapshot() {
    Snapshot snapshot = new Snapshot(randomAlphaOfLength(10), new SnapshotId(randomAlphaOfLength(10), randomAlphaOfLength(10)));
    boolean includeGlobalState = randomBoolean();
    boolean partial = randomBoolean();
    int numberOfIndices = randomIntBetween(0, 10);
    List<IndexId> indices = new ArrayList<>();
    for (int i = 0; i < numberOfIndices; i++) {
        indices.add(new IndexId(randomAlphaOfLength(10), randomAlphaOfLength(10)));
    }
    long startTime = randomLong();
    long repositoryStateId = randomLong();
    ImmutableOpenMap.Builder<ShardId, SnapshotsInProgress.ShardSnapshotStatus> builder = ImmutableOpenMap.builder();
    final List<Index> esIndices = indices.stream().map(i -> new Index(i.getName(), randomAlphaOfLength(10))).collect(Collectors.toList());
    List<String> dataStreams = Arrays.asList(generateRandomStringArray(10, 10, false));
    for (Index idx : esIndices) {
        int shardsCount = randomIntBetween(1, 10);
        for (int j = 0; j < shardsCount; j++) {
            ShardId shardId = new ShardId(idx, j);
            String nodeId = randomAlphaOfLength(10);
            ShardState shardState = randomFrom(ShardState.values());
            builder.put(shardId, shardState == ShardState.QUEUED ? SnapshotsInProgress.ShardSnapshotStatus.UNASSIGNED_QUEUED : new SnapshotsInProgress.ShardSnapshotStatus(nodeId, shardState, shardState.failed() ? randomAlphaOfLength(10) : null, "1"));
        }
    }
    ImmutableOpenMap<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shards = builder.build();
    return new Entry(snapshot, includeGlobalState, partial, randomState(shards), indices, dataStreams, startTime, repositoryStateId, shards, null, SnapshotInfoTests.randomUserMetadata(), VersionUtils.randomVersion(random()));
}