Examples with InternalTestCluster org.elasticsearch.test.InternalTestCluster used on opensource projects

Example 11 with InternalTestCluster

use of org.elasticsearch.test.InternalTestCluster in project elasticsearch by elastic.

the class InternalTestClusterTests method testBeforeTest.

public void testBeforeTest() throws Exception {
    final boolean autoManageMinMasterNodes = randomBoolean();
    long clusterSeed = randomLong();
    final boolean masterNodes;
    final int minNumDataNodes;
    final int maxNumDataNodes;
    if (autoManageMinMasterNodes) {
        masterNodes = randomBoolean();
        minNumDataNodes = randomIntBetween(0, 3);
        maxNumDataNodes = randomIntBetween(minNumDataNodes, 4);
    } else {
        // if we manage min master nodes, we need to lock down the number of nodes
        minNumDataNodes = randomIntBetween(0, 4);
        maxNumDataNodes = minNumDataNodes;
        masterNodes = false;
    }
    final int numClientNodes = randomIntBetween(0, 2);
    final String clusterName1 = "shared1";
    final String clusterName2 = "shared2";
    NodeConfigurationSource nodeConfigurationSource = new NodeConfigurationSource() {

        @Override
        public Settings nodeSettings(int nodeOrdinal) {
            final Settings.Builder settings = Settings.builder().put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), 2 * ((masterNodes ? InternalTestCluster.DEFAULT_HIGH_NUM_MASTER_NODES : 0) + maxNumDataNodes + numClientNodes)).put(NetworkModule.HTTP_ENABLED.getKey(), false).put(NetworkModule.TRANSPORT_TYPE_KEY, MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME);
            if (autoManageMinMasterNodes == false) {
                assert minNumDataNodes == maxNumDataNodes;
                assert masterNodes == false;
                settings.put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minNumDataNodes / 2 + 1);
            }
            return settings.build();
        }

        @Override
        public Settings transportClientSettings() {
            return Settings.builder().put(NetworkModule.TRANSPORT_TYPE_KEY, MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME).build();
        }
    };
    boolean enableHttpPipelining = randomBoolean();
    String nodePrefix = "foobar";
    Path baseDir = createTempDir();
    final List<Class<? extends Plugin>> mockPlugins = Arrays.asList(MockTcpTransportPlugin.class, TestZenDiscovery.TestPlugin.class);
    InternalTestCluster cluster0 = new InternalTestCluster(clusterSeed, baseDir, masterNodes, autoManageMinMasterNodes, minNumDataNodes, maxNumDataNodes, clusterName1, nodeConfigurationSource, numClientNodes, enableHttpPipelining, nodePrefix, mockPlugins, Function.identity());
    InternalTestCluster cluster1 = new InternalTestCluster(clusterSeed, baseDir, masterNodes, autoManageMinMasterNodes, minNumDataNodes, maxNumDataNodes, clusterName2, nodeConfigurationSource, numClientNodes, enableHttpPipelining, nodePrefix, mockPlugins, Function.identity());
    assertClusters(cluster0, cluster1, false);
    long seed = randomLong();
    try {
        {
            Random random = new Random(seed);
            cluster0.beforeTest(random, random.nextDouble());
        }
        {
            Random random = new Random(seed);
            cluster1.beforeTest(random, random.nextDouble());
        }
        assertArrayEquals(cluster0.getNodeNames(), cluster1.getNodeNames());
        Iterator<Client> iterator1 = cluster1.getClients().iterator();
        for (Client client : cluster0.getClients()) {
            assertTrue(iterator1.hasNext());
            Client other = iterator1.next();
            assertSettings(client.settings(), other.settings(), false);
        }
        assertArrayEquals(cluster0.getNodeNames(), cluster1.getNodeNames());
        assertMMNinNodeSetting(cluster0, cluster0.numMasterNodes());
        assertMMNinNodeSetting(cluster1, cluster0.numMasterNodes());
        cluster0.afterTest();
        cluster1.afterTest();
    } finally {
        IOUtils.close(cluster0, cluster1);
    }
}

Example 12 with InternalTestCluster

use of org.elasticsearch.test.InternalTestCluster in project elasticsearch by elastic.

the class InternalTestClusterTests method testDifferentRolesMaintainPathOnRestart.

public void testDifferentRolesMaintainPathOnRestart() throws Exception {
    final Path baseDir = createTempDir();
    final int numNodes = 5;
    InternalTestCluster cluster = new InternalTestCluster(randomLong(), baseDir, true, true, 0, 0, "test", new NodeConfigurationSource() {

        @Override
        public Settings nodeSettings(int nodeOrdinal) {
            return Settings.builder().put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), numNodes).put(NetworkModule.HTTP_ENABLED.getKey(), false).put(NetworkModule.TRANSPORT_TYPE_KEY, MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME).put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), 0).build();
        }

        @Override
        public Settings transportClientSettings() {
            return Settings.builder().put(NetworkModule.TRANSPORT_TYPE_KEY, MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME).build();
        }
    }, 0, randomBoolean(), "", Arrays.asList(MockTcpTransportPlugin.class, TestZenDiscovery.TestPlugin.class), Function.identity());
    cluster.beforeTest(random(), 0.0);
    try {
        Map<DiscoveryNode.Role, Set<String>> pathsPerRole = new HashMap<>();
        for (int i = 0; i < numNodes; i++) {
            final DiscoveryNode.Role role = i == numNodes - 1 && pathsPerRole.containsKey(MASTER) == false ? // last noe and still no master ofr the cluster
            MASTER : randomFrom(MASTER, DiscoveryNode.Role.DATA, DiscoveryNode.Role.INGEST);
            final String node;
            switch(role) {
                case MASTER:
                    node = cluster.startMasterOnlyNode(Settings.EMPTY);
                    break;
                case DATA:
                    node = cluster.startDataOnlyNode(Settings.EMPTY);
                    break;
                case INGEST:
                    node = cluster.startCoordinatingOnlyNode(Settings.EMPTY);
                    break;
                default:
                    throw new IllegalStateException("get your story straight");
            }
            Set<String> rolePaths = pathsPerRole.computeIfAbsent(role, k -> new HashSet<>());
            for (Path path : getNodePaths(cluster, node)) {
                assertTrue(rolePaths.add(path.toString()));
            }
        }
        cluster.fullRestart();
        Map<DiscoveryNode.Role, Set<String>> result = new HashMap<>();
        for (String name : cluster.getNodeNames()) {
            DiscoveryNode node = cluster.getInstance(ClusterService.class, name).localNode();
            List<String> paths = Arrays.stream(getNodePaths(cluster, name)).map(Path::toString).collect(Collectors.toList());
            if (node.isMasterNode()) {
                result.computeIfAbsent(MASTER, k -> new HashSet<>()).addAll(paths);
            } else if (node.isDataNode()) {
                result.computeIfAbsent(DATA, k -> new HashSet<>()).addAll(paths);
            } else {
                result.computeIfAbsent(INGEST, k -> new HashSet<>()).addAll(paths);
            }
        }
        assertThat(result.size(), equalTo(pathsPerRole.size()));
        for (DiscoveryNode.Role role : result.keySet()) {
            assertThat("path are not the same for " + role, result.get(role), equalTo(pathsPerRole.get(role)));
        }
    } finally {
        cluster.close();
    }
}

Example 13 with InternalTestCluster

use of org.elasticsearch.test.InternalTestCluster in project elasticsearch by elastic.

the class InternalTestClusterTests method testDataFolderAssignmentAndCleaning.

public void testDataFolderAssignmentAndCleaning() throws IOException, InterruptedException {
    long clusterSeed = randomLong();
    boolean masterNodes = randomBoolean();
    // we need one stable node
    final int minNumDataNodes = 2;
    final int maxNumDataNodes = 2;
    final int numClientNodes = randomIntBetween(0, 2);
    final String clusterName1 = "shared1";
    NodeConfigurationSource nodeConfigurationSource = new NodeConfigurationSource() {

        @Override
        public Settings nodeSettings(int nodeOrdinal) {
            return Settings.builder().put(NetworkModule.HTTP_ENABLED.getKey(), false).put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), 2 + (masterNodes ? InternalTestCluster.DEFAULT_HIGH_NUM_MASTER_NODES : 0) + maxNumDataNodes + numClientNodes).put(NetworkModule.TRANSPORT_TYPE_KEY, MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME).build();
        }

        @Override
        public Settings transportClientSettings() {
            return Settings.builder().put(NetworkModule.TRANSPORT_TYPE_KEY, MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME).build();
        }
    };
    boolean enableHttpPipelining = randomBoolean();
    String nodePrefix = "test";
    Path baseDir = createTempDir();
    InternalTestCluster cluster = new InternalTestCluster(clusterSeed, baseDir, masterNodes, true, minNumDataNodes, maxNumDataNodes, clusterName1, nodeConfigurationSource, numClientNodes, enableHttpPipelining, nodePrefix, Arrays.asList(MockTcpTransportPlugin.class, TestZenDiscovery.TestPlugin.class), Function.identity());
    try {
        cluster.beforeTest(random(), 0.0);
        final int originalMasterCount = cluster.numMasterNodes();
        assertMMNinNodeSetting(cluster, originalMasterCount);
        final Map<String, Path[]> shardNodePaths = new HashMap<>();
        for (String name : cluster.getNodeNames()) {
            shardNodePaths.put(name, getNodePaths(cluster, name));
        }
        String poorNode = randomFrom(cluster.getNodeNames());
        Path dataPath = getNodePaths(cluster, poorNode)[0];
        final Path testMarker = dataPath.resolve("testMarker");
        Files.createDirectories(testMarker);
        int expectedMasterCount = originalMasterCount;
        if (cluster.getInstance(ClusterService.class, poorNode).localNode().isMasterNode()) {
            expectedMasterCount--;
        }
        cluster.stopRandomNode(InternalTestCluster.nameFilter(poorNode));
        if (expectedMasterCount != originalMasterCount) {
            // check for updated
            assertMMNinClusterSetting(cluster, expectedMasterCount);
        }
        // stopping a node half way shouldn't clean data
        assertFileExists(testMarker);
        final String stableNode = randomFrom(cluster.getNodeNames());
        final Path stableDataPath = getNodePaths(cluster, stableNode)[0];
        final Path stableTestMarker = stableDataPath.resolve("stableTestMarker");
        assertThat(stableDataPath, not(dataPath));
        Files.createDirectories(stableTestMarker);
        final String newNode1 = cluster.startNode();
        expectedMasterCount++;
        assertThat(getNodePaths(cluster, newNode1)[0], equalTo(dataPath));
        // starting a node should re-use data folders and not clean it
        assertFileExists(testMarker);
        if (expectedMasterCount > 1) {
            // this is the first master, it's in cluster state settings won't be updated
            assertMMNinClusterSetting(cluster, expectedMasterCount);
        }
        assertMMNinNodeSetting(newNode1, cluster, expectedMasterCount);
        final String newNode2 = cluster.startNode();
        expectedMasterCount++;
        assertMMNinClusterSetting(cluster, expectedMasterCount);
        final Path newDataPath = getNodePaths(cluster, newNode2)[0];
        final Path newTestMarker = newDataPath.resolve("newTestMarker");
        assertThat(newDataPath, not(dataPath));
        Files.createDirectories(newTestMarker);
        cluster.beforeTest(random(), 0.0);
        // the cluster should be reset for a new test, cleaning up the extra path we made
        assertFileNotExists(newTestMarker);
        // a new unknown node used this path, it should be cleaned
        assertFileNotExists(testMarker);
        // but leaving the structure of existing, reused nodes
        assertFileExists(stableTestMarker);
        for (String name : cluster.getNodeNames()) {
            assertThat("data paths for " + name + " changed", getNodePaths(cluster, name), equalTo(shardNodePaths.get(name)));
        }
        cluster.beforeTest(random(), 0.0);
        // but leaving the structure of existing, reused nodes
        assertFileExists(stableTestMarker);
        for (String name : cluster.getNodeNames()) {
            assertThat("data paths for " + name + " changed", getNodePaths(cluster, name), equalTo(shardNodePaths.get(name)));
        }
        assertMMNinNodeSetting(cluster, originalMasterCount);
    } finally {
        cluster.close();
    }
}

Example 14 with InternalTestCluster

use of org.elasticsearch.test.InternalTestCluster in project crate by crate.

the class ClusterInfoServiceIT method testClusterInfoServiceCollectsInformation.

public void testClusterInfoServiceCollectsInformation() throws Exception {
    internalCluster().startNodes(2);
    assertAcked(prepareCreate("test").setSettings(Settings.builder().put(Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), 0).put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1).put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE).build()));
    ensureGreen("test");
    InternalTestCluster internalTestCluster = internalCluster();
    // Get the cluster info service on the master node
    final InternalClusterInfoService infoService = (InternalClusterInfoService) internalTestCluster.getInstance(ClusterInfoService.class, internalTestCluster.getMasterName());
    infoService.setUpdateFrequency(TimeValue.timeValueMillis(200));
    infoService.onMaster();
    ClusterInfo info = infoService.refresh();
    assertNotNull("info should not be null", info);
    ImmutableOpenMap<String, DiskUsage> leastUsages = info.getNodeLeastAvailableDiskUsages();
    ImmutableOpenMap<String, DiskUsage> mostUsages = info.getNodeMostAvailableDiskUsages();
    ImmutableOpenMap<String, Long> shardSizes = info.shardSizes;
    assertNotNull(leastUsages);
    assertNotNull(shardSizes);
    assertThat("some usages are populated", leastUsages.values().size(), Matchers.equalTo(2));
    assertThat("some shard sizes are populated", shardSizes.values().size(), greaterThan(0));
    for (ObjectCursor<DiskUsage> usage : leastUsages.values()) {
        logger.info("--> usage: {}", usage.value);
        assertThat("usage has be retrieved", usage.value.getFreeBytes(), greaterThan(0L));
    }
    for (ObjectCursor<DiskUsage> usage : mostUsages.values()) {
        logger.info("--> usage: {}", usage.value);
        assertThat("usage has be retrieved", usage.value.getFreeBytes(), greaterThan(0L));
    }
    for (ObjectCursor<Long> size : shardSizes.values()) {
        logger.info("--> shard size: {}", size.value);
        assertThat("shard size is greater than 0", size.value, greaterThanOrEqualTo(0L));
    }
    ClusterService clusterService = internalTestCluster.getInstance(ClusterService.class, internalTestCluster.getMasterName());
    ClusterState state = clusterService.state();
    for (ShardRouting shard : state.routingTable().allShards()) {
        String dataPath = info.getDataPath(shard);
        assertNotNull(dataPath);
        String nodeId = shard.currentNodeId();
        DiscoveryNode discoveryNode = state.getNodes().get(nodeId);
        IndicesService indicesService = internalTestCluster.getInstance(IndicesService.class, discoveryNode.getName());
        IndexService indexService = indicesService.indexService(shard.index());
        IndexShard indexShard = indexService.getShardOrNull(shard.id());
        assertEquals(indexShard.shardPath().getRootDataPath().toString(), dataPath);
    }
}

Example 15 with InternalTestCluster

use of org.elasticsearch.test.InternalTestCluster in project crate by crate.

the class MetadataWriteDataNodesIT method indexDirectoryExists.

private boolean indexDirectoryExists(String nodeName, Index index) {
    NodeEnvironment nodeEnv = ((InternalTestCluster) cluster()).getInstance(NodeEnvironment.class, nodeName);
    Path[] paths = nodeEnv.indexPaths(index);
    for (Path path : paths) {
        if (Files.exists(path)) {
            return true;
        }
    }
    return false;
}