Examples with ShardId org.elasticsearch.index.shard.ShardId used on opensource projects

Example 96 with ShardId

use of org.elasticsearch.index.shard.ShardId in project elasticsearch by elastic.

the class NodeJoinControllerTests method testElectionBasedOnConflictingNodes.

/**
     * Tests tha node can become a master, even though the last cluster state it knows contains
     * nodes that conflict with the joins it got and needs to become a master
     */
public void testElectionBasedOnConflictingNodes() throws InterruptedException, ExecutionException {
    final DiscoveryNode masterNode = clusterService.localNode();
    final DiscoveryNode otherNode = new DiscoveryNode("other_node", buildNewFakeTransportAddress(), emptyMap(), EnumSet.allOf(DiscoveryNode.Role.class), Version.CURRENT);
    // simulate master going down with stale nodes in it's cluster state (for example when min master nodes is set to 2)
    // also add some shards to that node
    DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(clusterService.state().nodes());
    discoBuilder.masterNodeId(null);
    discoBuilder.add(otherNode);
    ClusterState.Builder stateBuilder = ClusterState.builder(clusterService.state()).nodes(discoBuilder);
    if (randomBoolean()) {
        IndexMetaData indexMetaData = IndexMetaData.builder("test").settings(Settings.builder().put(SETTING_VERSION_CREATED, Version.CURRENT).put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 1).put(SETTING_CREATION_DATE, System.currentTimeMillis())).build();
        IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(indexMetaData.getIndex());
        RoutingTable.Builder routing = new RoutingTable.Builder();
        routing.addAsNew(indexMetaData);
        final ShardId shardId = new ShardId("test", "_na_", 0);
        IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId);
        final DiscoveryNode primaryNode = randomBoolean() ? masterNode : otherNode;
        final DiscoveryNode replicaNode = primaryNode.equals(masterNode) ? otherNode : masterNode;
        final boolean primaryStarted = randomBoolean();
        indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting("test", 0, primaryNode.getId(), null, true, primaryStarted ? ShardRoutingState.STARTED : ShardRoutingState.INITIALIZING, primaryStarted ? null : new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, "getting there")));
        if (primaryStarted) {
            boolean replicaStared = randomBoolean();
            indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting("test", 0, replicaNode.getId(), null, false, replicaStared ? ShardRoutingState.STARTED : ShardRoutingState.INITIALIZING, replicaStared ? null : new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "getting there")));
        } else {
            indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting("test", 0, null, null, false, ShardRoutingState.UNASSIGNED, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "life sucks")));
        }
        indexRoutingTableBuilder.addIndexShard(indexShardRoutingBuilder.build());
        IndexRoutingTable indexRoutingTable = indexRoutingTableBuilder.build();
        IndexMetaData updatedIndexMetaData = updateActiveAllocations(indexRoutingTable, indexMetaData);
        stateBuilder.metaData(MetaData.builder().put(updatedIndexMetaData, false).generateClusterUuidIfNeeded()).routingTable(RoutingTable.builder().add(indexRoutingTable).build());
    }
    setState(clusterService, stateBuilder.build());
    // conflict on node id or address
    final DiscoveryNode conflictingNode = randomBoolean() ? new DiscoveryNode(otherNode.getId(), randomBoolean() ? otherNode.getAddress() : buildNewFakeTransportAddress(), otherNode.getAttributes(), otherNode.getRoles(), Version.CURRENT) : new DiscoveryNode("conflicting_address_node", otherNode.getAddress(), otherNode.getAttributes(), otherNode.getRoles(), Version.CURRENT);
    nodeJoinController.startElectionContext();
    final SimpleFuture joinFuture = joinNodeAsync(conflictingNode);
    final CountDownLatch elected = new CountDownLatch(1);
    nodeJoinController.waitToBeElectedAsMaster(1, TimeValue.timeValueHours(5), new NodeJoinController.ElectionCallback() {

        @Override
        public void onElectedAsMaster(ClusterState state) {
            elected.countDown();
        }

        @Override
        public void onFailure(Throwable t) {
            logger.error("failed to be elected as master", t);
            throw new AssertionError("failed to be elected as master", t);
        }
    });
    elected.await();
    // throw any exception
    joinFuture.get();
    final ClusterState finalState = clusterService.state();
    final DiscoveryNodes finalNodes = finalState.nodes();
    assertTrue(finalNodes.isLocalNodeElectedMaster());
    assertThat(finalNodes.getLocalNode(), equalTo(masterNode));
    assertThat(finalNodes.getSize(), equalTo(2));
    assertThat(finalNodes.get(conflictingNode.getId()), equalTo(conflictingNode));
    List<ShardRouting> activeShardsOnRestartedNode = StreamSupport.stream(finalState.getRoutingNodes().node(conflictingNode.getId()).spliterator(), false).filter(ShardRouting::active).collect(Collectors.toList());
    assertThat(activeShardsOnRestartedNode, empty());
}

Example 97 with ShardId

use of org.elasticsearch.index.shard.ShardId in project elasticsearch by elastic.

the class ZenDiscoveryUnitTests method testValidateOnUnsupportedIndexVersionCreated.

public void testValidateOnUnsupportedIndexVersionCreated() throws Exception {
    final int iters = randomIntBetween(3, 10);
    for (int i = 0; i < iters; i++) {
        ClusterState.Builder stateBuilder = ClusterState.builder(ClusterName.DEFAULT);
        final DiscoveryNode otherNode = new DiscoveryNode("other_node", buildNewFakeTransportAddress(), emptyMap(), EnumSet.allOf(DiscoveryNode.Role.class), Version.CURRENT);
        MembershipAction.ValidateJoinRequestRequestHandler request = new MembershipAction.ValidateJoinRequestRequestHandler();
        final boolean incompatible = randomBoolean();
        IndexMetaData indexMetaData = IndexMetaData.builder("test").settings(Settings.builder().put(SETTING_VERSION_CREATED, incompatible ? VersionUtils.getPreviousVersion(Version.CURRENT.minimumIndexCompatibilityVersion()) : VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumIndexCompatibilityVersion(), Version.CURRENT)).put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0).put(SETTING_CREATION_DATE, System.currentTimeMillis())).state(IndexMetaData.State.OPEN).build();
        IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(indexMetaData.getIndex());
        RoutingTable.Builder routing = new RoutingTable.Builder();
        routing.addAsNew(indexMetaData);
        final ShardId shardId = new ShardId("test", "_na_", 0);
        IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId);
        final DiscoveryNode primaryNode = otherNode;
        indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting("test", 0, primaryNode.getId(), null, true, ShardRoutingState.INITIALIZING, new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, "getting there")));
        indexRoutingTableBuilder.addIndexShard(indexShardRoutingBuilder.build());
        IndexRoutingTable indexRoutingTable = indexRoutingTableBuilder.build();
        IndexMetaData updatedIndexMetaData = updateActiveAllocations(indexRoutingTable, indexMetaData);
        stateBuilder.metaData(MetaData.builder().put(updatedIndexMetaData, false).generateClusterUuidIfNeeded()).routingTable(RoutingTable.builder().add(indexRoutingTable).build());
        if (incompatible) {
            IllegalStateException ex = expectThrows(IllegalStateException.class, () -> request.messageReceived(new MembershipAction.ValidateJoinRequest(stateBuilder.build()), null));
            assertEquals("index [test] version not supported: " + VersionUtils.getPreviousVersion(Version.CURRENT.minimumCompatibilityVersion()) + " minimum compatible index version is: " + Version.CURRENT.minimumCompatibilityVersion(), ex.getMessage());
        } else {
            AtomicBoolean sendResponse = new AtomicBoolean(false);
            request.messageReceived(new MembershipAction.ValidateJoinRequest(stateBuilder.build()), new TransportChannel() {

                @Override
                public String action() {
                    return null;
                }

                @Override
                public String getProfileName() {
                    return null;
                }

                @Override
                public long getRequestId() {
                    return 0;
                }

                @Override
                public String getChannelType() {
                    return null;
                }

                @Override
                public void sendResponse(TransportResponse response) throws IOException {
                    sendResponse.set(true);
                }

                @Override
                public void sendResponse(TransportResponse response, TransportResponseOptions options) throws IOException {
                }

                @Override
                public void sendResponse(Exception exception) throws IOException {
                }
            });
            assertTrue(sendResponse.get());
        }
    }
}

Example 98 with ShardId

use of org.elasticsearch.index.shard.ShardId in project elasticsearch by elastic.

the class NodeEnvironmentTests method testShardLock.

public void testShardLock() throws Exception {
    final NodeEnvironment env = newNodeEnvironment();
    Index index = new Index("foo", "fooUUID");
    ShardLock fooLock = env.shardLock(new ShardId(index, 0));
    assertEquals(new ShardId(index, 0), fooLock.getShardId());
    try {
        env.shardLock(new ShardId(index, 0));
        fail("shard is locked");
    } catch (ShardLockObtainFailedException ex) {
    // expected
    }
    for (Path path : env.indexPaths(index)) {
        Files.createDirectories(path.resolve("0"));
        Files.createDirectories(path.resolve("1"));
    }
    try {
        env.lockAllForIndex(index, idxSettings, randomIntBetween(0, 10));
        fail("shard 0 is locked");
    } catch (ShardLockObtainFailedException ex) {
    // expected
    }
    fooLock.close();
    // can lock again?
    env.shardLock(new ShardId(index, 0)).close();
    List<ShardLock> locks = env.lockAllForIndex(index, idxSettings, randomIntBetween(0, 10));
    try {
        env.shardLock(new ShardId(index, 0));
        fail("shard is locked");
    } catch (ShardLockObtainFailedException ex) {
    // expected
    }
    IOUtils.close(locks);
    assertTrue("LockedShards: " + env.lockedShards(), env.lockedShards().isEmpty());
    env.close();
}

Example 99 with ShardId

use of org.elasticsearch.index.shard.ShardId in project elasticsearch by elastic.

the class NodeEnvironmentTests method testDeleteSafe.

public void testDeleteSafe() throws Exception {
    final NodeEnvironment env = newNodeEnvironment();
    final Index index = new Index("foo", "fooUUID");
    ShardLock fooLock = env.shardLock(new ShardId(index, 0));
    assertEquals(new ShardId(index, 0), fooLock.getShardId());
    for (Path path : env.indexPaths(index)) {
        Files.createDirectories(path.resolve("0"));
        Files.createDirectories(path.resolve("1"));
    }
    try {
        env.deleteShardDirectorySafe(new ShardId(index, 0), idxSettings);
        fail("shard is locked");
    } catch (ShardLockObtainFailedException ex) {
    // expected
    }
    for (Path path : env.indexPaths(index)) {
        assertTrue(Files.exists(path.resolve("0")));
        assertTrue(Files.exists(path.resolve("1")));
    }
    env.deleteShardDirectorySafe(new ShardId(index, 1), idxSettings);
    for (Path path : env.indexPaths(index)) {
        assertTrue(Files.exists(path.resolve("0")));
        assertFalse(Files.exists(path.resolve("1")));
    }
    try {
        env.deleteIndexDirectorySafe(index, randomIntBetween(0, 10), idxSettings);
        fail("shard is locked");
    } catch (ShardLockObtainFailedException ex) {
    // expected
    }
    fooLock.close();
    for (Path path : env.indexPaths(index)) {
        assertTrue(Files.exists(path));
    }
    final AtomicReference<Throwable> threadException = new AtomicReference<>();
    final CountDownLatch latch = new CountDownLatch(1);
    final CountDownLatch blockLatch = new CountDownLatch(1);
    final CountDownLatch start = new CountDownLatch(1);
    if (randomBoolean()) {
        Thread t = new Thread(new AbstractRunnable() {

            @Override
            public void onFailure(Exception e) {
                logger.error("unexpected error", e);
                threadException.set(e);
                latch.countDown();
                blockLatch.countDown();
            }

            @Override
            protected void doRun() throws Exception {
                start.await();
                try (ShardLock autoCloses = env.shardLock(new ShardId(index, 0))) {
                    blockLatch.countDown();
                    Thread.sleep(randomIntBetween(1, 10));
                }
                latch.countDown();
            }
        });
        t.start();
    } else {
        latch.countDown();
        blockLatch.countDown();
    }
    start.countDown();
    blockLatch.await();
    env.deleteIndexDirectorySafe(index, 5000, idxSettings);
    assertNull(threadException.get());
    for (Path path : env.indexPaths(index)) {
        assertFalse(Files.exists(path));
    }
    latch.await();
    assertTrue("LockedShards: " + env.lockedShards(), env.lockedShards().isEmpty());
    env.close();
}

Example 100 with ShardId

use of org.elasticsearch.index.shard.ShardId in project elasticsearch by elastic.

the class NodeEnvironmentTests method testCustomDataPaths.

public void testCustomDataPaths() throws Exception {
    String[] dataPaths = tmpPaths();
    NodeEnvironment env = newNodeEnvironment(dataPaths, "/tmp", Settings.EMPTY);
    final Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_INDEX_UUID, "myindexUUID").build();
    IndexSettings s1 = IndexSettingsModule.newIndexSettings("myindex", indexSettings);
    IndexSettings s2 = IndexSettingsModule.newIndexSettings("myindex", Settings.builder().put(indexSettings).put(IndexMetaData.SETTING_DATA_PATH, "/tmp/foo").build());
    Index index = new Index("myindex", "myindexUUID");
    ShardId sid = new ShardId(index, 0);
    assertFalse("no settings should mean no custom data path", s1.hasCustomDataPath());
    assertTrue("settings with path_data should have a custom data path", s2.hasCustomDataPath());
    assertThat(env.availableShardPaths(sid), equalTo(env.availableShardPaths(sid)));
    assertThat(env.resolveCustomLocation(s2, sid), equalTo(PathUtils.get("/tmp/foo/0/" + index.getUUID() + "/0")));
    assertThat("shard paths with a custom data_path should contain only regular paths", env.availableShardPaths(sid), equalTo(stringsToPaths(dataPaths, "nodes/0/indices/" + index.getUUID() + "/0")));
    assertThat("index paths uses the regular template", env.indexPaths(index), equalTo(stringsToPaths(dataPaths, "nodes/0/indices/" + index.getUUID())));
    IndexSettings s3 = new IndexSettings(s2.getIndexMetaData(), Settings.builder().put(NodeEnvironment.ADD_NODE_LOCK_ID_TO_CUSTOM_PATH.getKey(), false).build());
    assertThat(env.availableShardPaths(sid), equalTo(env.availableShardPaths(sid)));
    assertThat(env.resolveCustomLocation(s3, sid), equalTo(PathUtils.get("/tmp/foo/" + index.getUUID() + "/0")));
    assertThat("shard paths with a custom data_path should contain only regular paths", env.availableShardPaths(sid), equalTo(stringsToPaths(dataPaths, "nodes/0/indices/" + index.getUUID() + "/0")));
    assertThat("index paths uses the regular template", env.indexPaths(index), equalTo(stringsToPaths(dataPaths, "nodes/0/indices/" + index.getUUID())));
    env.close();
}