Examples with UnassignedInfo org.elasticsearch.cluster.routing.UnassignedInfo used on opensource projects

Example 51 with UnassignedInfo

use of org.elasticsearch.cluster.routing.UnassignedInfo in project elasticsearch by elastic.

the class ReplicaShardAllocator method processExistingRecoveries.

/**
     * Process existing recoveries of replicas and see if we need to cancel them if we find a better
     * match. Today, a better match is one that has full sync id match compared to not having one in
     * the previous recovery.
     */
public void processExistingRecoveries(RoutingAllocation allocation) {
    MetaData metaData = allocation.metaData();
    RoutingNodes routingNodes = allocation.routingNodes();
    List<Runnable> shardCancellationActions = new ArrayList<>();
    for (RoutingNode routingNode : routingNodes) {
        for (ShardRouting shard : routingNode) {
            if (shard.primary()) {
                continue;
            }
            if (shard.initializing() == false) {
                continue;
            }
            if (shard.relocatingNodeId() != null) {
                continue;
            }
            // if we are allocating a replica because of index creation, no need to go and find a copy, there isn't one...
            if (shard.unassignedInfo() != null && shard.unassignedInfo().getReason() == UnassignedInfo.Reason.INDEX_CREATED) {
                continue;
            }
            AsyncShardFetch.FetchResult<NodeStoreFilesMetaData> shardStores = fetchData(shard, allocation);
            if (shardStores.hasData() == false) {
                logger.trace("{}: fetching new stores for initializing shard", shard);
                // still fetching
                continue;
            }
            ShardRouting primaryShard = allocation.routingNodes().activePrimary(shard.shardId());
            assert primaryShard != null : "the replica shard can be allocated on at least one node, so there must be an active primary";
            TransportNodesListShardStoreMetaData.StoreFilesMetaData primaryStore = findStore(primaryShard, allocation, shardStores);
            if (primaryStore == null) {
                // if we can't find the primary data, it is probably because the primary shard is corrupted (and listing failed)
                // just let the recovery find it out, no need to do anything about it for the initializing shard
                logger.trace("{}: no primary shard store found or allocated, letting actual allocation figure it out", shard);
                continue;
            }
            MatchingNodes matchingNodes = findMatchingNodes(shard, allocation, primaryStore, shardStores, false);
            if (matchingNodes.getNodeWithHighestMatch() != null) {
                DiscoveryNode currentNode = allocation.nodes().get(shard.currentNodeId());
                DiscoveryNode nodeWithHighestMatch = matchingNodes.getNodeWithHighestMatch();
                // current node will not be in matchingNodes as it is filtered away by SameShardAllocationDecider
                final String currentSyncId;
                if (shardStores.getData().containsKey(currentNode)) {
                    currentSyncId = shardStores.getData().get(currentNode).storeFilesMetaData().syncId();
                } else {
                    currentSyncId = null;
                }
                if (currentNode.equals(nodeWithHighestMatch) == false && Objects.equals(currentSyncId, primaryStore.syncId()) == false && matchingNodes.isNodeMatchBySyncID(nodeWithHighestMatch)) {
                    // we found a better match that has a full sync id match, the existing allocation is not fully synced
                    // so we found a better one, cancel this one
                    logger.debug("cancelling allocation of replica on [{}], sync id match found on node [{}]", currentNode, nodeWithHighestMatch);
                    UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.REALLOCATED_REPLICA, "existing allocation of replica to [" + currentNode + "] cancelled, sync id match found on node [" + nodeWithHighestMatch + "]", null, 0, allocation.getCurrentNanoTime(), System.currentTimeMillis(), false, UnassignedInfo.AllocationStatus.NO_ATTEMPT);
                    // don't cancel shard in the loop as it will cause a ConcurrentModificationException
                    shardCancellationActions.add(() -> routingNodes.failShard(logger, shard, unassignedInfo, metaData.getIndexSafe(shard.index()), allocation.changes()));
                }
            }
        }
    }
    for (Runnable action : shardCancellationActions) {
        action.run();
    }
}

Example 52 with UnassignedInfo

use of org.elasticsearch.cluster.routing.UnassignedInfo in project elasticsearch by elastic.

the class ClusterAllocationExplainIT method testAllocationFilteringOnIndexCreation.

public void testAllocationFilteringOnIndexCreation() throws Exception {
    logger.info("--> starting 2 nodes");
    internalCluster().startNodes(2);
    logger.info("--> creating an index with 1 primary, 0 replicas, with allocation filtering so the primary can't be assigned");
    createIndexAndIndexData(1, 0, Settings.builder().put("index.routing.allocation.include._name", "non_existent_node").build(), ActiveShardCount.NONE);
    boolean includeYesDecisions = randomBoolean();
    boolean includeDiskInfo = randomBoolean();
    ClusterAllocationExplanation explanation = runExplain(true, includeYesDecisions, includeDiskInfo);
    ShardId shardId = explanation.getShard();
    boolean isPrimary = explanation.isPrimary();
    ShardRoutingState shardRoutingState = explanation.getShardState();
    DiscoveryNode currentNode = explanation.getCurrentNode();
    UnassignedInfo unassignedInfo = explanation.getUnassignedInfo();
    ClusterInfo clusterInfo = explanation.getClusterInfo();
    AllocateUnassignedDecision allocateDecision = explanation.getShardAllocationDecision().getAllocateDecision();
    MoveDecision moveDecision = explanation.getShardAllocationDecision().getMoveDecision();
    // verify shard info
    assertEquals("idx", shardId.getIndexName());
    assertEquals(0, shardId.getId());
    assertTrue(isPrimary);
    // verify current node info
    assertNotEquals(ShardRoutingState.STARTED, shardRoutingState);
    assertNull(currentNode);
    // verify unassigned info
    assertNotNull(unassignedInfo);
    assertEquals(Reason.INDEX_CREATED, unassignedInfo.getReason());
    assertEquals(AllocationStatus.DECIDERS_NO, unassignedInfo.getLastAllocationStatus());
    // verify cluster info
    verifyClusterInfo(clusterInfo, includeDiskInfo, 2);
    // verify decision objects
    assertTrue(allocateDecision.isDecisionTaken());
    assertFalse(moveDecision.isDecisionTaken());
    assertEquals(AllocationDecision.NO, allocateDecision.getAllocationDecision());
    assertEquals("cannot allocate because allocation is not permitted to any of the nodes", allocateDecision.getExplanation());
    assertNull(allocateDecision.getAllocationId());
    assertNull(allocateDecision.getTargetNode());
    assertEquals(0L, allocateDecision.getConfiguredDelayInMillis());
    assertEquals(0L, allocateDecision.getRemainingDelayInMillis());
    assertEquals(2, allocateDecision.getNodeDecisions().size());
    for (NodeAllocationResult result : allocateDecision.getNodeDecisions()) {
        assertNotNull(result.getNode());
        assertEquals(AllocationDecision.NO, result.getNodeDecision());
        if (includeYesDecisions) {
            assertThat(result.getCanAllocateDecision().getDecisions().size(), greaterThan(1));
        } else {
            assertEquals(1, result.getCanAllocateDecision().getDecisions().size());
        }
        for (Decision d : result.getCanAllocateDecision().getDecisions()) {
            if (d.label().equals("filter")) {
                assertEquals(Decision.Type.NO, d.type());
                assertEquals("node does not match index setting [index.routing.allocation.include] filters " + "[_name:\"non_existent_node\"]", d.getExplanation());
            }
        }
    }
    // verify JSON output
    try (XContentParser parser = getParser(explanation)) {
        verifyShardInfo(parser, true, includeDiskInfo, ShardRoutingState.UNASSIGNED);
        parser.nextToken();
        assertEquals("can_allocate", parser.currentName());
        parser.nextToken();
        String allocationDecision = parser.text();
        assertTrue(allocationDecision.equals(AllocationDecision.NO.toString()) || allocationDecision.equals(AllocationDecision.AWAITING_INFO.toString()));
        parser.nextToken();
        assertEquals("allocate_explanation", parser.currentName());
        parser.nextToken();
        if (allocationDecision.equals("awaiting_info")) {
            assertEquals("cannot allocate because information about existing shard data is still being retrieved " + "from some of the nodes", parser.text());
        } else {
            assertEquals("cannot allocate because allocation is not permitted to any of the nodes", parser.text());
        }
        Map<String, AllocationDecision> nodeDecisions = new HashMap<>();
        for (String nodeName : internalCluster().getNodeNames()) {
            nodeDecisions.put(nodeName, AllocationDecision.NO);
        }
        verifyNodeDecisions(parser, nodeDecisions, includeYesDecisions, false);
        assertEquals(Token.END_OBJECT, parser.nextToken());
    }
}

Example 53 with UnassignedInfo

use of org.elasticsearch.cluster.routing.UnassignedInfo in project elasticsearch by elastic.

the class ClusterAllocationExplainIT method testUnassignedPrimaryWithExistingIndex.

public void testUnassignedPrimaryWithExistingIndex() throws Exception {
    logger.info("--> starting 2 nodes");
    internalCluster().startNodes(2);
    logger.info("--> creating an index with 1 primary, 0 replicas");
    createIndexAndIndexData(1, 0);
    logger.info("--> stopping the node with the primary");
    internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName()));
    ensureStableCluster(1);
    boolean includeYesDecisions = randomBoolean();
    boolean includeDiskInfo = randomBoolean();
    ClusterAllocationExplanation explanation = runExplain(true, includeYesDecisions, includeDiskInfo);
    ShardId shardId = explanation.getShard();
    boolean isPrimary = explanation.isPrimary();
    ShardRoutingState shardState = explanation.getShardState();
    DiscoveryNode currentNode = explanation.getCurrentNode();
    UnassignedInfo unassignedInfo = explanation.getUnassignedInfo();
    ClusterInfo clusterInfo = explanation.getClusterInfo();
    AllocateUnassignedDecision allocateDecision = explanation.getShardAllocationDecision().getAllocateDecision();
    MoveDecision moveDecision = explanation.getShardAllocationDecision().getMoveDecision();
    // verify shard info
    assertEquals("idx", shardId.getIndexName());
    assertEquals(0, shardId.getId());
    assertTrue(isPrimary);
    // verify current node info
    assertNotEquals(ShardRoutingState.STARTED, shardState);
    assertNull(currentNode);
    // verify unassigned info
    assertNotNull(unassignedInfo);
    assertEquals(Reason.NODE_LEFT, unassignedInfo.getReason());
    assertTrue(unassignedInfo.getLastAllocationStatus() == AllocationStatus.FETCHING_SHARD_DATA || unassignedInfo.getLastAllocationStatus() == AllocationStatus.NO_VALID_SHARD_COPY);
    // verify cluster info
    verifyClusterInfo(clusterInfo, includeDiskInfo, 1);
    // verify decision objects
    assertTrue(allocateDecision.isDecisionTaken());
    assertFalse(moveDecision.isDecisionTaken());
    assertTrue(allocateDecision.getAllocationDecision() == AllocationDecision.NO_VALID_SHARD_COPY || allocateDecision.getAllocationDecision() == AllocationDecision.AWAITING_INFO);
    if (allocateDecision.getAllocationDecision() == AllocationDecision.NO_VALID_SHARD_COPY) {
        assertEquals("cannot allocate because a previous copy of the primary shard existed but can no longer be " + "found on the nodes in the cluster", allocateDecision.getExplanation());
    } else {
        assertEquals("cannot allocate because information about existing shard data is still being retrieved from some of the nodes", allocateDecision.getExplanation());
    }
    assertNull(allocateDecision.getAllocationId());
    assertNull(allocateDecision.getTargetNode());
    assertEquals(0L, allocateDecision.getConfiguredDelayInMillis());
    assertEquals(0L, allocateDecision.getRemainingDelayInMillis());
    if (allocateDecision.getAllocationDecision() == AllocationDecision.NO_VALID_SHARD_COPY) {
        assertEquals(1, allocateDecision.getNodeDecisions().size());
        // verify JSON output
        try (XContentParser parser = getParser(explanation)) {
            verifyShardInfo(parser, true, includeDiskInfo, ShardRoutingState.UNASSIGNED);
            parser.nextToken();
            assertEquals("can_allocate", parser.currentName());
            parser.nextToken();
            assertEquals(AllocationDecision.NO_VALID_SHARD_COPY.toString(), parser.text());
            parser.nextToken();
            assertEquals("allocate_explanation", parser.currentName());
            parser.nextToken();
            assertEquals("cannot allocate because a previous copy of the primary shard existed but can no longer be found " + "on the nodes in the cluster", parser.text());
            verifyStaleShardCopyNodeDecisions(parser, 1, Collections.emptySet());
        }
    }
}

Example 54 with UnassignedInfo

use of org.elasticsearch.cluster.routing.UnassignedInfo in project elasticsearch by elastic.

the class ClusterAllocationExplainIT method testWorseBalance.

public void testWorseBalance() throws Exception {
    logger.info("--> starting a single node");
    internalCluster().startNode();
    ensureStableCluster(1);
    logger.info("--> creating an index with 5 shards, all allocated to the single node");
    createIndexAndIndexData(5, 0);
    logger.info("--> setting balancing threshold really high, so it won't be met");
    client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put("cluster.routing.allocation.balance.threshold", 1000.0f)).get();
    logger.info("--> starting another node, with the rebalance threshold so high, it should not get any shards");
    internalCluster().startNode();
    ensureStableCluster(2);
    boolean includeYesDecisions = randomBoolean();
    boolean includeDiskInfo = randomBoolean();
    ClusterAllocationExplanation explanation = runExplain(true, includeYesDecisions, includeDiskInfo);
    ShardId shardId = explanation.getShard();
    boolean isPrimary = explanation.isPrimary();
    ShardRoutingState shardRoutingState = explanation.getShardState();
    DiscoveryNode currentNode = explanation.getCurrentNode();
    UnassignedInfo unassignedInfo = explanation.getUnassignedInfo();
    ClusterInfo clusterInfo = explanation.getClusterInfo();
    AllocateUnassignedDecision allocateDecision = explanation.getShardAllocationDecision().getAllocateDecision();
    MoveDecision moveDecision = explanation.getShardAllocationDecision().getMoveDecision();
    // verify shard info
    assertEquals("idx", shardId.getIndexName());
    assertEquals(0, shardId.getId());
    assertTrue(isPrimary);
    // verify current node info
    assertEquals(ShardRoutingState.STARTED, shardRoutingState);
    assertNotNull(currentNode);
    // verify unassigned info
    assertNull(unassignedInfo);
    // verify cluster info
    verifyClusterInfo(clusterInfo, includeDiskInfo, 2);
    // verify decision object
    assertFalse(allocateDecision.isDecisionTaken());
    assertTrue(moveDecision.isDecisionTaken());
    assertEquals(AllocationDecision.NO, moveDecision.getAllocationDecision());
    assertEquals("cannot rebalance as no target node exists that can both allocate this shard and improve the cluster balance", moveDecision.getExplanation());
    assertTrue(moveDecision.canRemain());
    assertFalse(moveDecision.forceMove());
    assertTrue(moveDecision.canRebalanceCluster());
    assertNotNull(moveDecision.getCanRemainDecision());
    assertNull(moveDecision.getTargetNode());
    assertEquals(1, moveDecision.getCurrentNodeRanking());
    // verifying cluster rebalance decision object
    assertNotNull(moveDecision.getClusterRebalanceDecision());
    assertEquals(Decision.Type.YES, moveDecision.getClusterRebalanceDecision().type());
    for (Decision d : moveDecision.getClusterRebalanceDecision().getDecisions()) {
        assertEquals(Decision.Type.YES, d.type());
        assertNotNull(d.getExplanation());
    }
    // verify node decisions
    assertEquals(1, moveDecision.getNodeDecisions().size());
    NodeAllocationResult result = moveDecision.getNodeDecisions().get(0);
    assertNotNull(result.getNode());
    assertEquals(1, result.getWeightRanking());
    assertEquals(AllocationDecision.WORSE_BALANCE, result.getNodeDecision());
    if (includeYesDecisions) {
        assertThat(result.getCanAllocateDecision().getDecisions().size(), greaterThan(0));
    } else {
        assertEquals(0, result.getCanAllocateDecision().getDecisions().size());
    }
    for (Decision d : result.getCanAllocateDecision().getDecisions()) {
        assertEquals(Decision.Type.YES, d.type());
        assertNotNull(d.getExplanation());
    }
    // verify JSON output
    try (XContentParser parser = getParser(explanation)) {
        verifyShardInfo(parser, true, includeDiskInfo, ShardRoutingState.STARTED);
        parser.nextToken();
        assertEquals("can_remain_on_current_node", parser.currentName());
        parser.nextToken();
        assertEquals(AllocationDecision.YES.toString(), parser.text());
        parser.nextToken();
        assertEquals("can_rebalance_cluster", parser.currentName());
        parser.nextToken();
        assertEquals(AllocationDecision.YES.toString(), parser.text());
        parser.nextToken();
        assertEquals("can_rebalance_to_other_node", parser.currentName());
        parser.nextToken();
        assertEquals(AllocationDecision.NO.toString(), parser.text());
        parser.nextToken();
        assertEquals("rebalance_explanation", parser.currentName());
        parser.nextToken();
        assertEquals("cannot rebalance as no target node exists that can both allocate this shard and improve the cluster balance", parser.text());
        verifyNodeDecisions(parser, allNodeDecisions(AllocationDecision.WORSE_BALANCE, true), includeYesDecisions, false);
        assertEquals(Token.END_OBJECT, parser.nextToken());
    }
}

Example 55 with UnassignedInfo

use of org.elasticsearch.cluster.routing.UnassignedInfo in project elasticsearch by elastic.

the class ClusterAllocationExplainIT method testCannotAllocateStaleReplicaExplanation.

public void testCannotAllocateStaleReplicaExplanation() throws Exception {
    logger.info("--> starting 3 nodes");
    final String masterNode = internalCluster().startNode();
    // start replica node first, so it's path will be used first when we start a node after
    // stopping all of them at end of test.
    final String replicaNode = internalCluster().startNode();
    final String primaryNode = internalCluster().startNode();
    logger.info("--> creating an index with 1 primary and 1 replica");
    createIndexAndIndexData(1, 1, Settings.builder().put("index.routing.allocation.include._name", primaryNode).put("index.routing.allocation.exclude._name", masterNode).build(), ActiveShardCount.ONE);
    client().admin().indices().prepareUpdateSettings("idx").setSettings(Settings.builder().put("index.routing.allocation.include._name", (String) null)).get();
    ensureGreen();
    assertThat(replicaNode().getName(), equalTo(replicaNode));
    assertThat(primaryNodeName(), equalTo(primaryNode));
    logger.info("--> stop node with the replica shard");
    internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNode));
    logger.info("--> index more data, now the replica is stale");
    indexData();
    logger.info("--> stop the node with the primary");
    internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNode));
    logger.info("--> restart the node with the stale replica");
    String restartedNode = internalCluster().startDataOnlyNode();
    // wait for the master to finish processing join.
    ensureClusterSizeConsistency();
    // wait until the system has fetched shard data and we know there is no valid shard copy
    assertBusy(() -> {
        ClusterAllocationExplanation explanation = client().admin().cluster().prepareAllocationExplain().setIndex("idx").setShard(0).setPrimary(true).get().getExplanation();
        assertTrue(explanation.getShardAllocationDecision().getAllocateDecision().isDecisionTaken());
        assertEquals(AllocationDecision.NO_VALID_SHARD_COPY, explanation.getShardAllocationDecision().getAllocateDecision().getAllocationDecision());
    });
    boolean includeYesDecisions = randomBoolean();
    boolean includeDiskInfo = randomBoolean();
    ClusterAllocationExplanation explanation = runExplain(true, includeYesDecisions, includeDiskInfo);
    ShardId shardId = explanation.getShard();
    boolean isPrimary = explanation.isPrimary();
    ShardRoutingState shardRoutingState = explanation.getShardState();
    DiscoveryNode currentNode = explanation.getCurrentNode();
    UnassignedInfo unassignedInfo = explanation.getUnassignedInfo();
    AllocateUnassignedDecision allocateDecision = explanation.getShardAllocationDecision().getAllocateDecision();
    MoveDecision moveDecision = explanation.getShardAllocationDecision().getMoveDecision();
    // verify shard info
    assertEquals("idx", shardId.getIndexName());
    assertEquals(0, shardId.getId());
    assertTrue(isPrimary);
    // verify current node info
    assertEquals(ShardRoutingState.UNASSIGNED, shardRoutingState);
    assertNull(currentNode);
    // verify unassigned info
    assertNotNull(unassignedInfo);
    // verify decision object
    assertTrue(allocateDecision.isDecisionTaken());
    assertFalse(moveDecision.isDecisionTaken());
    assertEquals(AllocationDecision.NO_VALID_SHARD_COPY, allocateDecision.getAllocationDecision());
    assertEquals(2, allocateDecision.getNodeDecisions().size());
    for (NodeAllocationResult nodeAllocationResult : allocateDecision.getNodeDecisions()) {
        if (nodeAllocationResult.getNode().getName().equals(restartedNode)) {
            assertNotNull(nodeAllocationResult.getShardStoreInfo());
            assertNotNull(nodeAllocationResult.getShardStoreInfo().getAllocationId());
            assertFalse(nodeAllocationResult.getShardStoreInfo().isInSync());
            assertNull(nodeAllocationResult.getShardStoreInfo().getStoreException());
        } else {
            assertNotNull(nodeAllocationResult.getShardStoreInfo());
            assertNull(nodeAllocationResult.getShardStoreInfo().getAllocationId());
            assertFalse(nodeAllocationResult.getShardStoreInfo().isInSync());
            assertNull(nodeAllocationResult.getShardStoreInfo().getStoreException());
        }
    }
    // verify JSON output
    try (XContentParser parser = getParser(explanation)) {
        verifyShardInfo(parser, true, includeDiskInfo, ShardRoutingState.UNASSIGNED);
        parser.nextToken();
        assertEquals("can_allocate", parser.currentName());
        parser.nextToken();
        assertEquals(AllocationDecision.NO_VALID_SHARD_COPY.toString(), parser.text());
        parser.nextToken();
        assertEquals("allocate_explanation", parser.currentName());
        parser.nextToken();
        assertEquals("cannot allocate because all found copies of the shard are either stale or corrupt", parser.text());
        verifyStaleShardCopyNodeDecisions(parser, 2, Collections.singleton(restartedNode));
    }
}