Examples with ClusterSettings org.elasticsearch.common.settings.ClusterSettings used on opensource projects

Example 91 with ClusterSettings

use of org.elasticsearch.common.settings.ClusterSettings in project crate by crate.

the class PublicationTransportHandlerTests method testDiffSerializationFailure.

public void testDiffSerializationFailure() {
    DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test").build(), random());
    final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
    final DiscoveryNode localNode = new DiscoveryNode("localNode", buildNewFakeTransportAddress(), Version.CURRENT);
    final TransportService transportService = new CapturingTransport().createTransportService(Settings.EMPTY, deterministicTaskQueue.getThreadPool(), x -> localNode, clusterSettings);
    final PublicationTransportHandler handler = new PublicationTransportHandler(transportService, writableRegistry(), pu -> null, (pu, l) -> {
    });
    transportService.start();
    transportService.acceptIncomingRequests();
    final DiscoveryNode otherNode = new DiscoveryNode("otherNode", buildNewFakeTransportAddress(), Version.CURRENT);
    final ClusterState clusterState = CoordinationStateTests.clusterState(2L, 1L, DiscoveryNodes.builder().add(localNode).add(otherNode).localNodeId(localNode.getId()).build(), VotingConfiguration.EMPTY_CONFIG, VotingConfiguration.EMPTY_CONFIG, 0L);
    final ClusterState unserializableClusterState = new ClusterState(clusterState.version(), clusterState.stateUUID(), clusterState) {

        @Override
        public Diff<ClusterState> diff(ClusterState previousState) {
            return new Diff<ClusterState>() {

                @Override
                public ClusterState apply(ClusterState part) {
                    fail("this diff shouldn't be applied");
                    return part;
                }

                @Override
                public void writeTo(StreamOutput out) throws IOException {
                    throw new IOException("Simulated failure of diff serialization");
                }
            };
        }
    };
    ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> handler.newPublicationContext(new ClusterChangedEvent("test", unserializableClusterState, clusterState)));
    assertNotNull(e.getCause());
    assertThat(e.getCause(), instanceOf(IOException.class));
    assertThat(e.getCause().getMessage(), containsString("Simulated failure of diff serialization"));
}

Example 92 with ClusterSettings

use of org.elasticsearch.common.settings.ClusterSettings in project crate by crate.

the class MasterServiceTests method testLongClusterStateUpdateLogging.

// To ensure that we log cluster state events on WARN level
@TestLogging("org.elasticsearch.cluster.service:WARN")
public void testLongClusterStateUpdateLogging() throws Exception {
    MockLogAppender mockAppender = new MockLogAppender();
    mockAppender.start();
    mockAppender.addExpectation(new MockLogAppender.UnseenEventExpectation("test1 shouldn't log because it was fast enough", MasterService.class.getCanonicalName(), Level.WARN, "*took*test1*"));
    mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test2", MasterService.class.getCanonicalName(), Level.WARN, "*took [*], which is over [10s], to compute cluster state update for [test2]"));
    mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test3", MasterService.class.getCanonicalName(), Level.WARN, "*took [*], which is over [10s], to compute cluster state update for [test3]"));
    mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test4", MasterService.class.getCanonicalName(), Level.WARN, "*took [*], which is over [10s], to compute cluster state update for [test4]"));
    mockAppender.addExpectation(new MockLogAppender.UnseenEventExpectation("test5 should not log despite publishing slowly", MasterService.class.getCanonicalName(), Level.WARN, "*took*test5*"));
    mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test6 should log due to slow and failing publication", MasterService.class.getCanonicalName(), Level.WARN, "took [*] and then failed to publish updated cluster state (version: *, uuid: *) for [test6]:*"));
    Logger clusterLogger = LogManager.getLogger(MasterService.class);
    Loggers.addAppender(clusterLogger, mockAppender);
    try (MasterService masterService = new MasterService(Settings.builder().put(ClusterName.CLUSTER_NAME_SETTING.getKey(), MasterServiceTests.class.getSimpleName()).put(Node.NODE_NAME_SETTING.getKey(), "test_node").build(), new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), threadPool)) {
        final DiscoveryNode localNode = new DiscoveryNode("node1", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
        final ClusterState initialClusterState = ClusterState.builder(new ClusterName(MasterServiceTests.class.getSimpleName())).nodes(DiscoveryNodes.builder().add(localNode).localNodeId(localNode.getId()).masterNodeId(localNode.getId())).blocks(ClusterBlocks.EMPTY_CLUSTER_BLOCK).build();
        final AtomicReference<ClusterState> clusterStateRef = new AtomicReference<>(initialClusterState);
        masterService.setClusterStatePublisher((event, publishListener, ackListener) -> {
            if (event.source().contains("test5")) {
                relativeTimeInMillis += MasterService.MASTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.get(Settings.EMPTY).millis() + randomLongBetween(1, 1000000);
            }
            if (event.source().contains("test6")) {
                relativeTimeInMillis += MasterService.MASTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.get(Settings.EMPTY).millis() + randomLongBetween(1, 1000000);
                throw new ElasticsearchException("simulated error during slow publication which should trigger logging");
            }
            clusterStateRef.set(event.state());
            publishListener.onResponse(null);
        });
        masterService.setClusterStateSupplier(clusterStateRef::get);
        masterService.start();
        final CountDownLatch latch = new CountDownLatch(6);
        final CountDownLatch processedFirstTask = new CountDownLatch(1);
        masterService.submitStateUpdateTask("test1", new ClusterStateUpdateTask() {

            @Override
            public ClusterState execute(ClusterState currentState) {
                relativeTimeInMillis += randomLongBetween(0L, MasterService.MASTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.get(Settings.EMPTY).millis());
                return currentState;
            }

            @Override
            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                latch.countDown();
                processedFirstTask.countDown();
            }

            @Override
            public void onFailure(String source, Exception e) {
                fail();
            }
        });
        processedFirstTask.await();
        masterService.submitStateUpdateTask("test2", new ClusterStateUpdateTask() {

            @Override
            public ClusterState execute(ClusterState currentState) {
                relativeTimeInMillis += MasterService.MASTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.get(Settings.EMPTY).millis() + randomLongBetween(1, 1000000);
                throw new IllegalArgumentException("Testing handling of exceptions in the cluster state task");
            }

            @Override
            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                fail();
            }

            @Override
            public void onFailure(String source, Exception e) {
                latch.countDown();
            }
        });
        masterService.submitStateUpdateTask("test3", new ClusterStateUpdateTask() {

            @Override
            public ClusterState execute(ClusterState currentState) {
                relativeTimeInMillis += MasterService.MASTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.get(Settings.EMPTY).millis() + randomLongBetween(1, 1000000);
                return ClusterState.builder(currentState).incrementVersion().build();
            }

            @Override
            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                latch.countDown();
            }

            @Override
            public void onFailure(String source, Exception e) {
                fail();
            }
        });
        masterService.submitStateUpdateTask("test4", new ClusterStateUpdateTask() {

            @Override
            public ClusterState execute(ClusterState currentState) {
                relativeTimeInMillis += MasterService.MASTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.get(Settings.EMPTY).millis() + randomLongBetween(1, 1000000);
                return currentState;
            }

            @Override
            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                latch.countDown();
            }

            @Override
            public void onFailure(String source, Exception e) {
                fail();
            }
        });
        masterService.submitStateUpdateTask("test5", new ClusterStateUpdateTask() {

            @Override
            public ClusterState execute(ClusterState currentState) {
                return ClusterState.builder(currentState).incrementVersion().build();
            }

            @Override
            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                latch.countDown();
            }

            @Override
            public void onFailure(String source, Exception e) {
                fail();
            }
        });
        masterService.submitStateUpdateTask("test6", new ClusterStateUpdateTask() {

            @Override
            public ClusterState execute(ClusterState currentState) {
                return ClusterState.builder(currentState).incrementVersion().build();
            }

            @Override
            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                fail();
            }

            @Override
            public void onFailure(String source, Exception e) {
                // maybe we should notify here?
                fail();
            }
        });
        // Additional update task to make sure all previous logging made it to the loggerName
        // We don't check logging for this on since there is no guarantee that it will occur before our check
        masterService.submitStateUpdateTask("test7", new ClusterStateUpdateTask() {

            @Override
            public ClusterState execute(ClusterState currentState) {
                return currentState;
            }

            @Override
            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                latch.countDown();
            }

            @Override
            public void onFailure(String source, Exception e) {
                fail();
            }
        });
        latch.await();
    } finally {
        Loggers.removeAppender(clusterLogger, mockAppender);
        mockAppender.stop();
    }
    mockAppender.assertAllExpectationsMatched();
}

Example 93 with ClusterSettings

use of org.elasticsearch.common.settings.ClusterSettings in project crate by crate.

the class MasterServiceTests method testAcking.

public void testAcking() throws InterruptedException {
    final DiscoveryNode node1 = new DiscoveryNode("node1", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
    final DiscoveryNode node2 = new DiscoveryNode("node2", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
    final DiscoveryNode node3 = new DiscoveryNode("node3", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
    try (MasterService masterService = new MasterService(Settings.builder().put(ClusterName.CLUSTER_NAME_SETTING.getKey(), MasterServiceTests.class.getSimpleName()).put(Node.NODE_NAME_SETTING.getKey(), "test_node").build(), new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), threadPool)) {
        final ClusterState initialClusterState = ClusterState.builder(new ClusterName(MasterServiceTests.class.getSimpleName())).nodes(DiscoveryNodes.builder().add(node1).add(node2).add(node3).localNodeId(node1.getId()).masterNodeId(node1.getId())).blocks(ClusterBlocks.EMPTY_CLUSTER_BLOCK).build();
        final AtomicReference<ClusterStatePublisher> publisherRef = new AtomicReference<>();
        masterService.setClusterStatePublisher((e, pl, al) -> publisherRef.get().publish(e, pl, al));
        masterService.setClusterStateSupplier(() -> initialClusterState);
        masterService.start();
        // check that we don't time out before even committing the cluster state
        {
            final CountDownLatch latch = new CountDownLatch(1);
            publisherRef.set((clusterChangedEvent, publishListener, ackListener) -> publishListener.onFailure(new FailedToCommitClusterStateException("mock exception")));
            masterService.submitStateUpdateTask("test2", new AckedClusterStateUpdateTask<Void>(null, null) {

                @Override
                public ClusterState execute(ClusterState currentState) {
                    return ClusterState.builder(currentState).build();
                }

                @Override
                public TimeValue ackTimeout() {
                    return TimeValue.ZERO;
                }

                @Override
                public TimeValue timeout() {
                    return null;
                }

                @Override
                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                    fail();
                }

                @Override
                protected Void newResponse(boolean acknowledged) {
                    fail();
                    return null;
                }

                @Override
                public void onFailure(String source, Exception e) {
                    latch.countDown();
                }

                @Override
                public void onAckTimeout() {
                    fail();
                }
            });
            latch.await();
        }
        // check that we timeout if commit took too long
        {
            final CountDownLatch latch = new CountDownLatch(2);
            final TimeValue ackTimeout = TimeValue.timeValueMillis(randomInt(100));
            publisherRef.set((clusterChangedEvent, publishListener, ackListener) -> {
                publishListener.onResponse(null);
                ackListener.onCommit(TimeValue.timeValueMillis(ackTimeout.millis() + randomInt(100)));
                ackListener.onNodeAck(node1, null);
                ackListener.onNodeAck(node2, null);
                ackListener.onNodeAck(node3, null);
            });
            masterService.submitStateUpdateTask("test2", new AckedClusterStateUpdateTask<Void>(null, null) {

                @Override
                public ClusterState execute(ClusterState currentState) {
                    return ClusterState.builder(currentState).build();
                }

                @Override
                public TimeValue ackTimeout() {
                    return ackTimeout;
                }

                @Override
                public TimeValue timeout() {
                    return null;
                }

                @Override
                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                    latch.countDown();
                }

                @Override
                protected Void newResponse(boolean acknowledged) {
                    fail();
                    return null;
                }

                @Override
                public void onFailure(String source, Exception e) {
                    fail();
                }

                @Override
                public void onAckTimeout() {
                    latch.countDown();
                }
            });
            latch.await();
        }
    }
}

Example 94 with ClusterSettings

use of org.elasticsearch.common.settings.ClusterSettings in project crate by crate.

the class DiskThresholdDeciderTests method testCanRemainWithShardRelocatingAway.

public void testCanRemainWithShardRelocatingAway() {
    Settings diskSettings = Settings.builder().put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true).put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_INCLUDE_RELOCATIONS_SETTING.getKey(), true).put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "60%").put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "70%").build();
    // We have an index with 2 primary shards each taking 40 bytes. Each node has 100 bytes available
    ImmutableOpenMap.Builder<String, DiskUsage> usagesBuilder = ImmutableOpenMap.builder();
    // 80% used
    usagesBuilder.put("node1", new DiskUsage("node1", "n1", "/dev/null", 100, 20));
    // 0% used
    usagesBuilder.put("node2", new DiskUsage("node2", "n2", "/dev/null", 100, 100));
    ImmutableOpenMap<String, DiskUsage> usages = usagesBuilder.build();
    ImmutableOpenMap.Builder<String, Long> shardSizesBuilder = ImmutableOpenMap.builder();
    shardSizesBuilder.put("[test][0][p]", 40L);
    shardSizesBuilder.put("[test][1][p]", 40L);
    shardSizesBuilder.put("[foo][0][p]", 10L);
    ImmutableOpenMap<String, Long> shardSizes = shardSizesBuilder.build();
    final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes);
    DiskThresholdDecider diskThresholdDecider = makeDecider(diskSettings);
    Metadata metadata = Metadata.builder().put(IndexMetadata.builder("test").settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "false"))).put(IndexMetadata.builder("foo").settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "false"))).build();
    RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).addAsNew(metadata.index("foo")).build();
    DiscoveryNode discoveryNode1 = new DiscoveryNode("node1", buildNewFakeTransportAddress(), emptyMap(), MASTER_DATA_ROLES, Version.CURRENT);
    DiscoveryNode discoveryNode2 = new DiscoveryNode("node2", buildNewFakeTransportAddress(), emptyMap(), MASTER_DATA_ROLES, Version.CURRENT);
    DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().add(discoveryNode1).add(discoveryNode2).build();
    ClusterState baseClusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metadata(metadata).routingTable(initialRoutingTable).nodes(discoveryNodes).build();
    // Two shards consuming each 80% of disk space while 70% is allowed, so shard 0 isn't allowed here
    ShardRouting firstRouting = TestShardRouting.newShardRouting("test", 0, "node1", null, true, ShardRoutingState.STARTED);
    ShardRouting secondRouting = TestShardRouting.newShardRouting("test", 1, "node1", null, true, ShardRoutingState.STARTED);
    RoutingNode firstRoutingNode = new RoutingNode("node1", discoveryNode1, firstRouting, secondRouting);
    RoutingTable.Builder builder = RoutingTable.builder().add(IndexRoutingTable.builder(firstRouting.index()).addIndexShard(new IndexShardRoutingTable.Builder(firstRouting.shardId()).addShard(firstRouting).build()).addIndexShard(new IndexShardRoutingTable.Builder(secondRouting.shardId()).addShard(secondRouting).build()));
    ClusterState clusterState = ClusterState.builder(baseClusterState).routingTable(builder.build()).build();
    RoutingAllocation routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), clusterState, clusterInfo, System.nanoTime());
    routingAllocation.debugDecision(true);
    Decision decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation);
    assertThat(decision.type(), equalTo(Decision.Type.NO));
    assertThat(decision.getExplanation(), containsString("the shard cannot remain on this node because it is above the high watermark cluster setting " + "[cluster.routing.allocation.disk.watermark.high=70%] and there is less than the required [30.0%] free disk on node, " + "actual free: [20.0%]"));
    // Two shards consuming each 80% of disk space while 70% is allowed, but one is relocating, so shard 0 can stay
    firstRouting = TestShardRouting.newShardRouting("test", 0, "node1", null, true, ShardRoutingState.STARTED);
    secondRouting = TestShardRouting.newShardRouting("test", 1, "node1", "node2", true, ShardRoutingState.RELOCATING);
    ShardRouting fooRouting = TestShardRouting.newShardRouting("foo", 0, null, true, ShardRoutingState.UNASSIGNED);
    firstRoutingNode = new RoutingNode("node1", discoveryNode1, firstRouting, secondRouting);
    builder = RoutingTable.builder().add(IndexRoutingTable.builder(firstRouting.index()).addIndexShard(new IndexShardRoutingTable.Builder(firstRouting.shardId()).addShard(firstRouting).build()).addIndexShard(new IndexShardRoutingTable.Builder(secondRouting.shardId()).addShard(secondRouting).build()));
    clusterState = ClusterState.builder(baseClusterState).routingTable(builder.build()).build();
    routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), clusterState, clusterInfo, System.nanoTime());
    routingAllocation.debugDecision(true);
    decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation);
    assertThat(decision.type(), equalTo(Decision.Type.YES));
    assertEquals("there is enough disk on this node for the shard to remain, free: [60b]", decision.getExplanation());
    decision = diskThresholdDecider.canAllocate(fooRouting, firstRoutingNode, routingAllocation);
    assertThat(decision.type(), equalTo(Decision.Type.NO));
    if (fooRouting.recoverySource().getType() == RecoverySource.Type.EMPTY_STORE) {
        assertThat(decision.getExplanation(), containsString("the node is above the high watermark cluster setting [cluster.routing.allocation.disk.watermark.high=70%], using " + "more disk space than the maximum allowed [70.0%], actual free: [20.0%]"));
    } else {
        assertThat(decision.getExplanation(), containsString("the node is above the low watermark cluster setting [cluster.routing.allocation.disk.watermark.low=60%], using more " + "disk space than the maximum allowed [60.0%], actual free: [20.0%]"));
    }
    // Creating AllocationService instance and the services it depends on...
    ClusterInfoService cis = () -> {
        logger.info("--> calling fake getClusterInfo");
        return clusterInfo;
    };
    AllocationDeciders deciders = new AllocationDeciders(new HashSet<>(Arrays.asList(new SameShardAllocationDecider(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), diskThresholdDecider)));
    AllocationService strategy = new AllocationService(deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
    // Ensure that the reroute call doesn't alter the routing table, since the first primary is relocating away
    // and therefor we will have sufficient disk space on node1.
    ClusterState result = strategy.reroute(clusterState, "reroute");
    assertThat(result, equalTo(clusterState));
    assertThat(result.routingTable().index("test").getShards().get(0).primaryShard().state(), equalTo(STARTED));
    assertThat(result.routingTable().index("test").getShards().get(0).primaryShard().currentNodeId(), equalTo("node1"));
    assertThat(result.routingTable().index("test").getShards().get(0).primaryShard().relocatingNodeId(), nullValue());
    assertThat(result.routingTable().index("test").getShards().get(1).primaryShard().state(), equalTo(RELOCATING));
    assertThat(result.routingTable().index("test").getShards().get(1).primaryShard().currentNodeId(), equalTo("node1"));
    assertThat(result.routingTable().index("test").getShards().get(1).primaryShard().relocatingNodeId(), equalTo("node2"));
}

Example 95 with ClusterSettings

use of org.elasticsearch.common.settings.ClusterSettings in project crate by crate.

the class EnableAllocationShortCircuitTests method createAllocationService.

private static AllocationService createAllocationService(Settings.Builder settings, ClusterPlugin plugin) {
    final ClusterSettings emptyClusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
    List<AllocationDecider> deciders = new ArrayList<>(ClusterModule.createAllocationDeciders(settings.build(), emptyClusterSettings, Collections.singletonList(plugin)));
    return new MockAllocationService(new AllocationDeciders(deciders), new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
}