Examples with IndexMetaData org.elasticsearch.cluster.metadata.IndexMetaData used on opensource projects

Example 16 with IndexMetaData

use of org.elasticsearch.cluster.metadata.IndexMetaData in project elasticsearch by elastic.

the class OperationRoutingTests method testBWC.

/**
     * Ensures that all changes to the hash-function / shard selection are BWC
     */
public void testBWC() {
    Map<String, Integer> termToShard = new TreeMap<>();
    termToShard.put("sEERfFzPSI", 1);
    termToShard.put("cNRiIrjzYd", 7);
    termToShard.put("BgfLBXUyWT", 5);
    termToShard.put("cnepjZhQnb", 3);
    termToShard.put("OKCmuYkeCK", 6);
    termToShard.put("OutXGRQUja", 5);
    termToShard.put("yCdyocKWou", 1);
    termToShard.put("KXuNWWNgVj", 2);
    termToShard.put("DGJOYrpESx", 4);
    termToShard.put("upLDybdTGs", 5);
    termToShard.put("yhZhzCPQby", 1);
    termToShard.put("EyCVeiCouA", 1);
    termToShard.put("tFyVdQauWR", 6);
    termToShard.put("nyeRYDnDQr", 6);
    termToShard.put("hswhrppvDH", 0);
    termToShard.put("BSiWvDOsNE", 5);
    termToShard.put("YHicpFBSaY", 1);
    termToShard.put("EquPtdKaBZ", 4);
    termToShard.put("rSjLZHCDfT", 5);
    termToShard.put("qoZALVcite", 7);
    termToShard.put("yDCCPVBiCm", 7);
    termToShard.put("ngizYtQgGK", 5);
    termToShard.put("FYQRIBcNqz", 0);
    termToShard.put("EBzEDAPODe", 2);
    termToShard.put("YePigbXgKb", 1);
    termToShard.put("PeGJjomyik", 3);
    termToShard.put("cyQIvDmyYD", 7);
    termToShard.put("yIEfZrYfRk", 5);
    termToShard.put("kblouyFUbu", 7);
    termToShard.put("xvIGbRiGJF", 3);
    termToShard.put("KWimwsREPf", 4);
    termToShard.put("wsNavvIcdk", 7);
    termToShard.put("xkWaPcCmpT", 0);
    termToShard.put("FKKTOnJMDy", 7);
    termToShard.put("RuLzobYixn", 2);
    termToShard.put("mFohLeFRvF", 4);
    termToShard.put("aAMXnamRJg", 7);
    termToShard.put("zKBMYJDmBI", 0);
    termToShard.put("ElSVuJQQuw", 7);
    termToShard.put("pezPtTQAAm", 7);
    termToShard.put("zBjjNEjAex", 2);
    termToShard.put("PGgHcLNPYX", 7);
    termToShard.put("hOkpeQqTDF", 3);
    termToShard.put("chZXraUPBH", 7);
    termToShard.put("FAIcSmmNXq", 5);
    termToShard.put("EZmDicyayC", 0);
    termToShard.put("GRIueBeIyL", 7);
    termToShard.put("qCChjGZYLp", 3);
    termToShard.put("IsSZQwwnUT", 3);
    termToShard.put("MGlxLFyyCK", 3);
    termToShard.put("YmscwrKSpB", 0);
    termToShard.put("czSljcjMop", 5);
    termToShard.put("XhfGWwNlng", 1);
    termToShard.put("cWpKJjlzgj", 7);
    termToShard.put("eDzIfMKbvk", 1);
    termToShard.put("WFFWYBfnTb", 0);
    termToShard.put("oDdHJxGxja", 7);
    termToShard.put("PDOQQqgIKE", 1);
    termToShard.put("bGEIEBLATe", 6);
    termToShard.put("xpRkJPWVpu", 2);
    termToShard.put("kTwZnPEeIi", 2);
    termToShard.put("DifcuqSsKk", 1);
    termToShard.put("CEmLmljpXe", 5);
    termToShard.put("cuNKtLtyJQ", 7);
    termToShard.put("yNjiAnxAmt", 5);
    termToShard.put("bVDJDCeaFm", 2);
    termToShard.put("vdnUhGLFtl", 0);
    termToShard.put("LnqSYezXbr", 5);
    termToShard.put("EzHgydDCSR", 3);
    termToShard.put("ZSKjhJlcpn", 1);
    termToShard.put("WRjUoZwtUz", 3);
    termToShard.put("RiBbcCdIgk", 4);
    termToShard.put("yizTqyjuDn", 4);
    termToShard.put("QnFjcpcZUT", 4);
    termToShard.put("agYhXYUUpl", 7);
    termToShard.put("UOjiTugjNC", 7);
    termToShard.put("nICGuWTdfV", 0);
    termToShard.put("NrnSmcnUVF", 2);
    termToShard.put("ZSzFcbpDqP", 3);
    termToShard.put("YOhahLSzzE", 5);
    termToShard.put("iWswCilUaT", 1);
    termToShard.put("zXAamKsRwj", 2);
    termToShard.put("aqGsrUPHFq", 5);
    termToShard.put("eDItImYWTS", 1);
    termToShard.put("JAYDZMRcpW", 4);
    termToShard.put("lmvAaEPflK", 7);
    termToShard.put("IKuOwPjKCx", 5);
    termToShard.put("schsINzlYB", 1);
    termToShard.put("OqbFNxrKrF", 2);
    termToShard.put("QrklDfvEJU", 6);
    termToShard.put("VLxKRKdLbx", 4);
    termToShard.put("imoydNTZhV", 1);
    termToShard.put("uFZyTyOMRO", 4);
    termToShard.put("nVAZVMPNNx", 3);
    termToShard.put("rPIdESYaAO", 5);
    termToShard.put("nbZWPWJsIM", 0);
    termToShard.put("wRZXPSoEgd", 3);
    termToShard.put("nGzpgwsSBc", 4);
    termToShard.put("AITyyoyLLs", 4);
    IndexMetaData metaData = IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(8).numberOfReplicas(1).build();
    for (Map.Entry<String, Integer> entry : termToShard.entrySet()) {
        String key = entry.getKey();
        int shard = randomBoolean() ? OperationRouting.generateShardId(metaData, key, null) : OperationRouting.generateShardId(metaData, "foobar", key);
        assertEquals(shard, entry.getValue().intValue());
    }
}

Example 17 with IndexMetaData

use of org.elasticsearch.cluster.metadata.IndexMetaData in project elasticsearch by elastic.

the class RoutingBackwardCompatibilityTests method testBackwardCompatibility.

public void testBackwardCompatibility() throws Exception {
    try (BufferedReader reader = new BufferedReader(new InputStreamReader(RoutingBackwardCompatibilityTests.class.getResourceAsStream("/org/elasticsearch/cluster/routing/shard_routes.txt"), "UTF-8"))) {
        for (String line = reader.readLine(); line != null; line = reader.readLine()) {
            if (line.startsWith("#")) {
                // comment
                continue;
            }
            String[] parts = line.split("\t");
            assertEquals(Arrays.toString(parts), 7, parts.length);
            final String index = parts[0];
            final int numberOfShards = Integer.parseInt(parts[1]);
            final String type = parts[2];
            final String id = parts[3];
            final String routing = "null".equals(parts[4]) ? null : parts[4];
            // not needed anymore - old hashing is gone
            final int pre20ExpectedShardId = Integer.parseInt(parts[5]);
            final int currentExpectedShard = Integer.parseInt(parts[6]);
            OperationRouting operationRouting = new OperationRouting(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
            for (Version version : VersionUtils.allReleasedVersions()) {
                if (version.onOrAfter(Version.V_2_0_0) == false) {
                    // unsupported version, no need to test
                    continue;
                }
                final Settings settings = settings(version).build();
                IndexMetaData indexMetaData = IndexMetaData.builder(index).settings(settings).numberOfShards(numberOfShards).numberOfReplicas(randomInt(3)).build();
                MetaData.Builder metaData = MetaData.builder().put(indexMetaData, false);
                RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetaData).build();
                ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
                final int shardId = operationRouting.indexShards(clusterState, index, id, routing).shardId().getId();
                assertEquals(currentExpectedShard, shardId);
            }
        }
    }
}

Example 18 with IndexMetaData

use of org.elasticsearch.cluster.metadata.IndexMetaData in project elasticsearch by elastic.

the class OperationRoutingTests method testPartitionedIndexShrunk.

public void testPartitionedIndexShrunk() {
    Map<String, Map<String, Integer>> routingIdToShard = new HashMap<>();
    Map<String, Integer> routingA = new HashMap<>();
    routingA.put("a_0", 1);
    routingA.put("a_1", 2);
    routingA.put("a_2", 2);
    routingA.put("a_3", 2);
    routingA.put("a_4", 1);
    routingA.put("a_5", 2);
    routingIdToShard.put("a", routingA);
    Map<String, Integer> routingB = new HashMap<>();
    routingB.put("b_0", 0);
    routingB.put("b_1", 0);
    routingB.put("b_2", 0);
    routingB.put("b_3", 0);
    routingB.put("b_4", 3);
    routingB.put("b_5", 3);
    routingIdToShard.put("b", routingB);
    Map<String, Integer> routingC = new HashMap<>();
    routingC.put("c_0", 1);
    routingC.put("c_1", 1);
    routingC.put("c_2", 0);
    routingC.put("c_3", 0);
    routingC.put("c_4", 0);
    routingC.put("c_5", 1);
    routingIdToShard.put("c", routingC);
    Map<String, Integer> routingD = new HashMap<>();
    routingD.put("d_0", 2);
    routingD.put("d_1", 2);
    routingD.put("d_2", 3);
    routingD.put("d_3", 3);
    routingD.put("d_4", 3);
    routingD.put("d_5", 3);
    routingIdToShard.put("d", routingD);
    IndexMetaData metaData = IndexMetaData.builder("test").settings(settings(Version.CURRENT)).setRoutingNumShards(8).numberOfShards(4).routingPartitionSize(3).numberOfReplicas(1).build();
    for (Map.Entry<String, Map<String, Integer>> routingIdEntry : routingIdToShard.entrySet()) {
        String routing = routingIdEntry.getKey();
        for (Map.Entry<String, Integer> idEntry : routingIdEntry.getValue().entrySet()) {
            String id = idEntry.getKey();
            int shard = idEntry.getValue();
            assertEquals(shard, OperationRouting.generateShardId(metaData, id, routing));
        }
    }
}

Example 19 with IndexMetaData

use of org.elasticsearch.cluster.metadata.IndexMetaData in project elasticsearch by elastic.

the class AbstractIndicesClusterStateServiceTestCase method assertClusterStateMatchesNodeState.

/**
     * Checks if cluster state matches internal state of IndicesClusterStateService instance
     *
     * @param state cluster state used for matching
     */
public void assertClusterStateMatchesNodeState(ClusterState state, IndicesClusterStateService indicesClusterStateService) {
    MockIndicesService indicesService = (MockIndicesService) indicesClusterStateService.indicesService;
    ConcurrentMap<ShardId, ShardRouting> failedShardsCache = indicesClusterStateService.failedShardsCache;
    RoutingNode localRoutingNode = state.getRoutingNodes().node(state.getNodes().getLocalNodeId());
    if (localRoutingNode != null) {
        if (enableRandomFailures == false) {
            assertThat("failed shard cache should be empty", failedShardsCache.values(), empty());
        }
        // check that all shards in local routing nodes have been allocated
        for (ShardRouting shardRouting : localRoutingNode) {
            Index index = shardRouting.index();
            IndexMetaData indexMetaData = state.metaData().getIndexSafe(index);
            MockIndexShard shard = indicesService.getShardOrNull(shardRouting.shardId());
            ShardRouting failedShard = failedShardsCache.get(shardRouting.shardId());
            if (enableRandomFailures) {
                if (shard == null && failedShard == null) {
                    fail("Shard with id " + shardRouting + " expected but missing in indicesService and failedShardsCache");
                }
                if (failedShard != null && failedShard.isSameAllocation(shardRouting) == false) {
                    fail("Shard cache has not been properly cleaned for " + failedShard);
                }
            } else {
                if (shard == null) {
                    fail("Shard with id " + shardRouting + " expected but missing in indicesService");
                }
            }
            if (shard != null) {
                AllocatedIndex<? extends Shard> indexService = indicesService.indexService(index);
                assertTrue("Index " + index + " expected but missing in indicesService", indexService != null);
                // index metadata has been updated
                assertThat(indexService.getIndexSettings().getIndexMetaData(), equalTo(indexMetaData));
                // shard has been created
                if (enableRandomFailures == false || failedShard == null) {
                    assertTrue("Shard with id " + shardRouting + " expected but missing in indexService", shard != null);
                    // shard has latest shard routing
                    assertThat(shard.routingEntry(), equalTo(shardRouting));
                }
                if (shard.routingEntry().primary() && shard.routingEntry().active()) {
                    IndexShardRoutingTable shardRoutingTable = state.routingTable().shardRoutingTable(shard.shardId());
                    Set<String> activeIds = shardRoutingTable.activeShards().stream().map(r -> r.allocationId().getId()).collect(Collectors.toSet());
                    Set<String> initializingIds = shardRoutingTable.getAllInitializingShards().stream().map(r -> r.allocationId().getId()).collect(Collectors.toSet());
                    assertThat(shard.routingEntry() + " isn't updated with active aIDs", shard.activeAllocationIds, equalTo(activeIds));
                    assertThat(shard.routingEntry() + " isn't updated with init aIDs", shard.initializingAllocationIds, equalTo(initializingIds));
                }
            }
        }
    }
    // all other shards / indices have been cleaned up
    for (AllocatedIndex<? extends Shard> indexService : indicesService) {
        assertTrue(state.metaData().getIndexSafe(indexService.index()) != null);
        boolean shardsFound = false;
        for (Shard shard : indexService) {
            shardsFound = true;
            ShardRouting persistedShardRouting = shard.routingEntry();
            ShardRouting shardRouting = localRoutingNode.getByShardId(persistedShardRouting.shardId());
            if (shardRouting == null) {
                fail("Shard with id " + persistedShardRouting + " locally exists but missing in routing table");
            }
            if (shardRouting.equals(persistedShardRouting) == false) {
                fail("Local shard " + persistedShardRouting + " has stale routing" + shardRouting);
            }
        }
        if (shardsFound == false) {
            if (enableRandomFailures) {
                // check if we have shards of that index in failedShardsCache
                // if yes, we might not have cleaned the index as failedShardsCache can be populated by another thread
                assertFalse(failedShardsCache.keySet().stream().noneMatch(shardId -> shardId.getIndex().equals(indexService.index())));
            } else {
                fail("index service for index " + indexService.index() + " has no shards");
            }
        }
    }
}

Example 20 with IndexMetaData

use of org.elasticsearch.cluster.metadata.IndexMetaData in project elasticsearch by elastic.

the class OpenCloseIndexIT method checkIndexState.

private void checkIndexState(IndexMetaData.State expectedState, String... indices) {
    ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().execute().actionGet();
    for (String index : indices) {
        IndexMetaData indexMetaData = clusterStateResponse.getState().metaData().indices().get(index);
        assertThat(indexMetaData, notNullValue());
        assertThat(indexMetaData.getState(), equalTo(expectedState));
    }
}