Examples with ShardId org.elasticsearch.index.shard.ShardId used on opensource projects

Example 91 with ShardId

use of org.elasticsearch.index.shard.ShardId in project elasticsearch by elastic.

the class FailedShardsRoutingTests method testFailAllReplicasInitializingOnPrimaryFail.

public void testFailAllReplicasInitializingOnPrimaryFail() {
    AllocationService allocation = createAllocationService(Settings.builder().build());
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(2)).build();
    RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
    ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
    ShardId shardId = new ShardId(metaData.index("test").getIndex(), 0);
    // add 4 nodes
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")).add(newNode("node3")).add(newNode("node4"))).build();
    clusterState = ClusterState.builder(clusterState).routingTable(allocation.reroute(clusterState, "reroute").routingTable()).build();
    assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
    assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(2));
    // start primary shards
    clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
    assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(1));
    assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(2));
    // start one replica so it can take over.
    clusterState = allocation.applyStartedShards(clusterState, Collections.singletonList(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).get(0)));
    assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(2));
    assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
    ShardRouting startedReplica = clusterState.getRoutingNodes().activeReplica(shardId);
    // fail the primary shard, check replicas get removed as well...
    ShardRouting primaryShardToFail = clusterState.routingTable().index("test").shard(0).primaryShard();
    ClusterState newState = allocation.applyFailedShard(clusterState, primaryShardToFail);
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    // the primary gets allocated on another node, replicas are initializing
    assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(1));
    assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(2));
    ShardRouting newPrimaryShard = clusterState.routingTable().index("test").shard(0).primaryShard();
    assertThat(newPrimaryShard, not(equalTo(primaryShardToFail)));
    assertThat(newPrimaryShard.allocationId(), equalTo(startedReplica.allocationId()));
}

Example 92 with ShardId

use of org.elasticsearch.index.shard.ShardId in project elasticsearch by elastic.

the class NodeVersionAllocationDeciderTests method testRebalanceDoesNotAllocatePrimaryAndReplicasOnDifferentVersionNodes.

public void testRebalanceDoesNotAllocatePrimaryAndReplicasOnDifferentVersionNodes() {
    ShardId shard1 = new ShardId("test1", "_na_", 0);
    ShardId shard2 = new ShardId("test2", "_na_", 0);
    final DiscoveryNode newNode = new DiscoveryNode("newNode", buildNewFakeTransportAddress(), emptyMap(), MASTER_DATA_ROLES, Version.CURRENT);
    final DiscoveryNode oldNode1 = new DiscoveryNode("oldNode1", buildNewFakeTransportAddress(), emptyMap(), MASTER_DATA_ROLES, VersionUtils.getPreviousVersion());
    final DiscoveryNode oldNode2 = new DiscoveryNode("oldNode2", buildNewFakeTransportAddress(), emptyMap(), MASTER_DATA_ROLES, VersionUtils.getPreviousVersion());
    AllocationId allocationId1P = AllocationId.newInitializing();
    AllocationId allocationId1R = AllocationId.newInitializing();
    AllocationId allocationId2P = AllocationId.newInitializing();
    AllocationId allocationId2R = AllocationId.newInitializing();
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder(shard1.getIndexName()).settings(settings(Version.CURRENT).put(Settings.EMPTY)).numberOfShards(1).numberOfReplicas(1).putInSyncAllocationIds(0, Sets.newHashSet(allocationId1P.getId(), allocationId1R.getId()))).put(IndexMetaData.builder(shard2.getIndexName()).settings(settings(Version.CURRENT).put(Settings.EMPTY)).numberOfShards(1).numberOfReplicas(1).putInSyncAllocationIds(0, Sets.newHashSet(allocationId2P.getId(), allocationId2R.getId()))).build();
    RoutingTable routingTable = RoutingTable.builder().add(IndexRoutingTable.builder(shard1.getIndex()).addIndexShard(new IndexShardRoutingTable.Builder(shard1).addShard(TestShardRouting.newShardRouting(shard1.getIndexName(), shard1.getId(), newNode.getId(), null, true, ShardRoutingState.STARTED, allocationId1P)).addShard(TestShardRouting.newShardRouting(shard1.getIndexName(), shard1.getId(), oldNode1.getId(), null, false, ShardRoutingState.STARTED, allocationId1R)).build())).add(IndexRoutingTable.builder(shard2.getIndex()).addIndexShard(new IndexShardRoutingTable.Builder(shard2).addShard(TestShardRouting.newShardRouting(shard2.getIndexName(), shard2.getId(), newNode.getId(), null, true, ShardRoutingState.STARTED, allocationId2P)).addShard(TestShardRouting.newShardRouting(shard2.getIndexName(), shard2.getId(), oldNode1.getId(), null, false, ShardRoutingState.STARTED, allocationId2R)).build())).build();
    ClusterState state = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).nodes(DiscoveryNodes.builder().add(newNode).add(oldNode1).add(oldNode2)).build();
    AllocationDeciders allocationDeciders = new AllocationDeciders(Settings.EMPTY, Collections.singleton(new NodeVersionAllocationDecider(Settings.EMPTY)));
    AllocationService strategy = new MockAllocationService(Settings.EMPTY, allocationDeciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
    state = strategy.reroute(state, new AllocationCommands(), true, false).getClusterState();
    // the two indices must stay as is, the replicas cannot move to oldNode2 because versions don't match
    assertThat(state.routingTable().index(shard2.getIndex()).shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(0));
    assertThat(state.routingTable().index(shard1.getIndex()).shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(0));
}

Example 93 with ShardId

use of org.elasticsearch.index.shard.ShardId in project elasticsearch by elastic.

the class DiskThresholdDeciderUnitTests method testSizeShrinkIndex.

public void testSizeShrinkIndex() {
    ImmutableOpenMap.Builder<String, Long> shardSizes = ImmutableOpenMap.builder();
    shardSizes.put("[test][0][p]", 10L);
    shardSizes.put("[test][1][p]", 100L);
    shardSizes.put("[test][2][p]", 500L);
    shardSizes.put("[test][3][p]", 500L);
    ClusterInfo info = new DevNullClusterInfo(ImmutableOpenMap.of(), ImmutableOpenMap.of(), shardSizes.build());
    MetaData.Builder metaBuilder = MetaData.builder();
    metaBuilder.put(IndexMetaData.builder("test").settings(settings(Version.CURRENT).put("index.uuid", "1234")).numberOfShards(4).numberOfReplicas(0));
    metaBuilder.put(IndexMetaData.builder("target").settings(settings(Version.CURRENT).put("index.uuid", "5678").put("index.shrink.source.name", "test").put("index.shrink.source.uuid", "1234")).numberOfShards(1).numberOfReplicas(0));
    metaBuilder.put(IndexMetaData.builder("target2").settings(settings(Version.CURRENT).put("index.uuid", "9101112").put("index.shrink.source.name", "test").put("index.shrink.source.uuid", "1234")).numberOfShards(2).numberOfReplicas(0));
    MetaData metaData = metaBuilder.build();
    RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
    routingTableBuilder.addAsNew(metaData.index("test"));
    routingTableBuilder.addAsNew(metaData.index("target"));
    routingTableBuilder.addAsNew(metaData.index("target2"));
    ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTableBuilder.build()).build();
    AllocationService allocationService = createAllocationService();
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
    clusterState = allocationService.reroute(clusterState, "foo");
    clusterState = allocationService.applyStartedShards(clusterState, clusterState.getRoutingTable().index("test").shardsWithState(ShardRoutingState.UNASSIGNED));
    RoutingAllocation allocation = new RoutingAllocation(null, clusterState.getRoutingNodes(), clusterState, info, 0, false);
    final Index index = new Index("test", "1234");
    ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), true, LocalShardsRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    test_0 = ShardRoutingHelper.initialize(test_0, "node1");
    test_0 = ShardRoutingHelper.moveToStarted(test_0);
    ShardRouting test_1 = ShardRouting.newUnassigned(new ShardId(index, 1), true, LocalShardsRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    test_1 = ShardRoutingHelper.initialize(test_1, "node2");
    test_1 = ShardRoutingHelper.moveToStarted(test_1);
    ShardRouting test_2 = ShardRouting.newUnassigned(new ShardId(index, 2), true, LocalShardsRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    test_2 = ShardRoutingHelper.initialize(test_2, "node1");
    ShardRouting test_3 = ShardRouting.newUnassigned(new ShardId(index, 3), true, LocalShardsRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    test_3 = ShardRoutingHelper.initialize(test_3, "node1");
    assertEquals(500L, DiskThresholdDecider.getExpectedShardSize(test_3, allocation, 0));
    assertEquals(500L, DiskThresholdDecider.getExpectedShardSize(test_2, allocation, 0));
    assertEquals(100L, DiskThresholdDecider.getExpectedShardSize(test_1, allocation, 0));
    assertEquals(10L, DiskThresholdDecider.getExpectedShardSize(test_0, allocation, 0));
    ShardRouting target = ShardRouting.newUnassigned(new ShardId(new Index("target", "5678"), 0), true, LocalShardsRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    assertEquals(1110L, DiskThresholdDecider.getExpectedShardSize(target, allocation, 0));
    ShardRouting target2 = ShardRouting.newUnassigned(new ShardId(new Index("target2", "9101112"), 0), true, LocalShardsRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    assertEquals(110L, DiskThresholdDecider.getExpectedShardSize(target2, allocation, 0));
    target2 = ShardRouting.newUnassigned(new ShardId(new Index("target2", "9101112"), 1), true, LocalShardsRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    assertEquals(1000L, DiskThresholdDecider.getExpectedShardSize(target2, allocation, 0));
}

Example 94 with ShardId

use of org.elasticsearch.index.shard.ShardId in project elasticsearch by elastic.

the class DiskThresholdDeciderUnitTests method testCanRemainUsesLeastAvailableSpace.

public void testCanRemainUsesLeastAvailableSpace() {
    ClusterSettings nss = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
    DiskThresholdDecider decider = new DiskThresholdDecider(Settings.EMPTY, nss);
    ImmutableOpenMap.Builder<ShardRouting, String> shardRoutingMap = ImmutableOpenMap.builder();
    DiscoveryNode node_0 = new DiscoveryNode("node_0", buildNewFakeTransportAddress(), Collections.emptyMap(), new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT);
    DiscoveryNode node_1 = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Collections.emptyMap(), new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT);
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)).build();
    final IndexMetaData indexMetaData = metaData.index("test");
    ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 0), true, StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    test_0 = ShardRoutingHelper.initialize(test_0, node_0.getId());
    test_0 = ShardRoutingHelper.moveToStarted(test_0);
    shardRoutingMap.put(test_0, "/node0/least");
    ShardRouting test_1 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 1), true, StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    test_1 = ShardRoutingHelper.initialize(test_1, node_1.getId());
    test_1 = ShardRoutingHelper.moveToStarted(test_1);
    shardRoutingMap.put(test_1, "/node1/least");
    ShardRouting test_2 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 2), true, StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    test_2 = ShardRoutingHelper.initialize(test_2, node_1.getId());
    test_2 = ShardRoutingHelper.moveToStarted(test_2);
    shardRoutingMap.put(test_2, "/node1/most");
    ShardRouting test_3 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 3), true, StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    test_3 = ShardRoutingHelper.initialize(test_3, node_1.getId());
    test_3 = ShardRoutingHelper.moveToStarted(test_3);
    // Intentionally not in the shardRoutingMap. We want to test what happens when we don't know where it is.
    RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetaData).build();
    ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
    logger.info("--> adding two nodes");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(node_0).add(node_1)).build();
    // actual test -- after all that bloat :)
    ImmutableOpenMap.Builder<String, DiskUsage> leastAvailableUsages = ImmutableOpenMap.builder();
    // 90% used
    leastAvailableUsages.put("node_0", new DiskUsage("node_0", "node_0", "/node0/least", 100, 10));
    // 91% used
    leastAvailableUsages.put("node_1", new DiskUsage("node_1", "node_1", "/node1/least", 100, 9));
    ImmutableOpenMap.Builder<String, DiskUsage> mostAvailableUsage = ImmutableOpenMap.builder();
    // 10% used
    mostAvailableUsage.put("node_0", new DiskUsage("node_0", "node_0", "/node0/most", 100, 90));
    // 10% used
    mostAvailableUsage.put("node_1", new DiskUsage("node_1", "node_1", "/node1/most", 100, 90));
    ImmutableOpenMap.Builder<String, Long> shardSizes = ImmutableOpenMap.builder();
    // 10 bytes
    shardSizes.put("[test][0][p]", 10L);
    shardSizes.put("[test][1][p]", 10L);
    shardSizes.put("[test][2][p]", 10L);
    final ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages.build(), mostAvailableUsage.build(), shardSizes.build(), shardRoutingMap.build());
    RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, Collections.singleton(decider)), clusterState.getRoutingNodes(), clusterState, clusterInfo, System.nanoTime(), false);
    allocation.debugDecision(true);
    Decision decision = decider.canRemain(test_0, new RoutingNode("node_0", node_0), allocation);
    assertEquals(Decision.Type.YES, decision.type());
    assertThat(((Decision.Single) decision).getExplanation(), containsString("there is enough disk on this node for the shard to remain, free: [10b]"));
    decision = decider.canRemain(test_1, new RoutingNode("node_1", node_1), allocation);
    assertEquals(Decision.Type.NO, decision.type());
    assertThat(((Decision.Single) decision).getExplanation(), containsString("the shard cannot remain on this node because it is " + "above the high watermark cluster setting [cluster.routing.allocation.disk.watermark.high=90%] and there is less than " + "the required [10.0%] free disk on node, actual free: [9.0%]"));
    try {
        decider.canRemain(test_0, new RoutingNode("node_1", node_1), allocation);
        fail("not allocated on this node");
    } catch (IllegalArgumentException ex) {
    // not allocated on that node
    }
    try {
        decider.canRemain(test_1, new RoutingNode("node_0", node_0), allocation);
        fail("not allocated on this node");
    } catch (IllegalArgumentException ex) {
    // not allocated on that node
    }
    decision = decider.canRemain(test_2, new RoutingNode("node_1", node_1), allocation);
    assertEquals("can stay since allocated on a different path with enough space", Decision.Type.YES, decision.type());
    assertThat(((Decision.Single) decision).getExplanation(), containsString("this shard is not allocated on the most utilized disk and can remain"));
    decision = decider.canRemain(test_2, new RoutingNode("node_1", node_1), allocation);
    assertEquals("can stay since we don't have information about this shard", Decision.Type.YES, decision.type());
    assertThat(((Decision.Single) decision).getExplanation(), containsString("this shard is not allocated on the most utilized disk and can remain"));
}

Example 95 with ShardId

use of org.elasticsearch.index.shard.ShardId in project elasticsearch by elastic.

the class SharedClusterSnapshotRestoreIT method testDeleteOrphanSnapshot.

public void testDeleteOrphanSnapshot() throws Exception {
    Client client = client();
    logger.info("-->  creating repository");
    final String repositoryName = "test-repo";
    assertAcked(client.admin().cluster().preparePutRepository(repositoryName).setType("mock").setSettings(Settings.builder().put("location", randomRepoPath()).put("compress", randomBoolean()).put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
    logger.info("--> create the index");
    final String idxName = "test-idx";
    createIndex(idxName);
    ensureGreen();
    ClusterService clusterService = internalCluster().getInstance(ClusterService.class, internalCluster().getMasterName());
    final CountDownLatch countDownLatch = new CountDownLatch(1);
    logger.info("--> snapshot");
    final String snapshotName = "test-snap";
    CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repositoryName, snapshotName).setWaitForCompletion(true).setIndices(idxName).get();
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
    logger.info("--> emulate an orphan snapshot");
    RepositoriesService repositoriesService = internalCluster().getInstance(RepositoriesService.class, internalCluster().getMasterName());
    final RepositoryData repositoryData = repositoriesService.repository(repositoryName).getRepositoryData();
    final IndexId indexId = repositoryData.resolveIndexId(idxName);
    clusterService.submitStateUpdateTask("orphan snapshot test", new ClusterStateUpdateTask() {

        @Override
        public ClusterState execute(ClusterState currentState) {
            // Simulate orphan snapshot
            ImmutableOpenMap.Builder<ShardId, ShardSnapshotStatus> shards = ImmutableOpenMap.builder();
            shards.put(new ShardId(idxName, "_na_", 0), new ShardSnapshotStatus("unknown-node", State.ABORTED));
            shards.put(new ShardId(idxName, "_na_", 1), new ShardSnapshotStatus("unknown-node", State.ABORTED));
            shards.put(new ShardId(idxName, "_na_", 2), new ShardSnapshotStatus("unknown-node", State.ABORTED));
            List<Entry> entries = new ArrayList<>();
            entries.add(new Entry(new Snapshot(repositoryName, createSnapshotResponse.getSnapshotInfo().snapshotId()), true, false, State.ABORTED, Collections.singletonList(indexId), System.currentTimeMillis(), repositoryData.getGenId(), shards.build()));
            return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, new SnapshotsInProgress(Collections.unmodifiableList(entries))).build();
        }

        @Override
        public void onFailure(String source, Exception e) {
            fail();
        }

        @Override
        public void clusterStateProcessed(String source, ClusterState oldState, final ClusterState newState) {
            countDownLatch.countDown();
        }
    });
    countDownLatch.await();
    logger.info("--> try deleting the orphan snapshot");
    assertAcked(client.admin().cluster().prepareDeleteSnapshot(repositoryName, snapshotName).get("10s"));
}