Examples with NodeGatewayStartedShards org.elasticsearch.gateway.TransportNodesListGatewayStartedShards.NodeGatewayStartedShards used on opensource projects

Example 6 with NodeGatewayStartedShards

use of org.elasticsearch.gateway.TransportNodesListGatewayStartedShards.NodeGatewayStartedShards in project elasticsearch by elastic.

the class PrimaryShardAllocator method makeAllocationDecision.

@Override
public AllocateUnassignedDecision makeAllocationDecision(final ShardRouting unassignedShard, final RoutingAllocation allocation, final Logger logger) {
    if (isResponsibleFor(unassignedShard) == false) {
        // this allocator is not responsible for allocating this shard
        return AllocateUnassignedDecision.NOT_TAKEN;
    }
    final boolean explain = allocation.debugDecision();
    final FetchResult<NodeGatewayStartedShards> shardState = fetchData(unassignedShard, allocation);
    if (shardState.hasData() == false) {
        allocation.setHasPendingAsyncFetch();
        List<NodeAllocationResult> nodeDecisions = null;
        if (explain) {
            nodeDecisions = buildDecisionsForAllNodes(unassignedShard, allocation);
        }
        return AllocateUnassignedDecision.no(AllocationStatus.FETCHING_SHARD_DATA, nodeDecisions);
    }
    // don't create a new IndexSetting object for every shard as this could cause a lot of garbage
    // on cluster restart if we allocate a boat load of shards
    final IndexMetaData indexMetaData = allocation.metaData().getIndexSafe(unassignedShard.index());
    final Set<String> inSyncAllocationIds = indexMetaData.inSyncAllocationIds(unassignedShard.id());
    final boolean snapshotRestore = unassignedShard.recoverySource().getType() == RecoverySource.Type.SNAPSHOT;
    final boolean recoverOnAnyNode = recoverOnAnyNode(indexMetaData);
    assert inSyncAllocationIds.isEmpty() == false;
    // use in-sync allocation ids to select nodes
    final NodeShardsResult nodeShardsResult = buildNodeShardsResult(unassignedShard, snapshotRestore || recoverOnAnyNode, allocation.getIgnoreNodes(unassignedShard.shardId()), inSyncAllocationIds, shardState, logger);
    final boolean enoughAllocationsFound = nodeShardsResult.orderedAllocationCandidates.size() > 0;
    logger.debug("[{}][{}]: found {} allocation candidates of {} based on allocation ids: [{}]", unassignedShard.index(), unassignedShard.id(), nodeShardsResult.orderedAllocationCandidates.size(), unassignedShard, inSyncAllocationIds);
    if (enoughAllocationsFound == false) {
        if (snapshotRestore) {
            // let BalancedShardsAllocator take care of allocating this shard
            logger.debug("[{}][{}]: missing local data, will restore from [{}]", unassignedShard.index(), unassignedShard.id(), unassignedShard.recoverySource());
            return AllocateUnassignedDecision.NOT_TAKEN;
        } else if (recoverOnAnyNode) {
            // let BalancedShardsAllocator take care of allocating this shard
            logger.debug("[{}][{}]: missing local data, recover from any node", unassignedShard.index(), unassignedShard.id());
            return AllocateUnassignedDecision.NOT_TAKEN;
        } else {
            // We have a shard that was previously allocated, but we could not find a valid shard copy to allocate the primary.
            // We could just be waiting for the node that holds the primary to start back up, in which case the allocation for
            // this shard will be picked up when the node joins and we do another allocation reroute
            logger.debug("[{}][{}]: not allocating, number_of_allocated_shards_found [{}]", unassignedShard.index(), unassignedShard.id(), nodeShardsResult.allocationsFound);
            return AllocateUnassignedDecision.no(AllocationStatus.NO_VALID_SHARD_COPY, explain ? buildNodeDecisions(null, shardState, inSyncAllocationIds) : null);
        }
    }
    NodesToAllocate nodesToAllocate = buildNodesToAllocate(allocation, nodeShardsResult.orderedAllocationCandidates, unassignedShard, false);
    DiscoveryNode node = null;
    String allocationId = null;
    boolean throttled = false;
    if (nodesToAllocate.yesNodeShards.isEmpty() == false) {
        DecidedNode decidedNode = nodesToAllocate.yesNodeShards.get(0);
        logger.debug("[{}][{}]: allocating [{}] to [{}] on primary allocation", unassignedShard.index(), unassignedShard.id(), unassignedShard, decidedNode.nodeShardState.getNode());
        node = decidedNode.nodeShardState.getNode();
        allocationId = decidedNode.nodeShardState.allocationId();
    } else if (nodesToAllocate.throttleNodeShards.isEmpty() && !nodesToAllocate.noNodeShards.isEmpty()) {
        // The deciders returned a NO decision for all nodes with shard copies, so we check if primary shard
        // can be force-allocated to one of the nodes.
        nodesToAllocate = buildNodesToAllocate(allocation, nodeShardsResult.orderedAllocationCandidates, unassignedShard, true);
        if (nodesToAllocate.yesNodeShards.isEmpty() == false) {
            final DecidedNode decidedNode = nodesToAllocate.yesNodeShards.get(0);
            final NodeGatewayStartedShards nodeShardState = decidedNode.nodeShardState;
            logger.debug("[{}][{}]: allocating [{}] to [{}] on forced primary allocation", unassignedShard.index(), unassignedShard.id(), unassignedShard, nodeShardState.getNode());
            node = nodeShardState.getNode();
            allocationId = nodeShardState.allocationId();
        } else if (nodesToAllocate.throttleNodeShards.isEmpty() == false) {
            logger.debug("[{}][{}]: throttling allocation [{}] to [{}] on forced primary allocation", unassignedShard.index(), unassignedShard.id(), unassignedShard, nodesToAllocate.throttleNodeShards);
            throttled = true;
        } else {
            logger.debug("[{}][{}]: forced primary allocation denied [{}]", unassignedShard.index(), unassignedShard.id(), unassignedShard);
        }
    } else {
        // we are throttling this, since we are allowed to allocate to this node but there are enough allocations
        // taking place on the node currently, ignore it for now
        logger.debug("[{}][{}]: throttling allocation [{}] to [{}] on primary allocation", unassignedShard.index(), unassignedShard.id(), unassignedShard, nodesToAllocate.throttleNodeShards);
        throttled = true;
    }
    List<NodeAllocationResult> nodeResults = null;
    if (explain) {
        nodeResults = buildNodeDecisions(nodesToAllocate, shardState, inSyncAllocationIds);
    }
    if (allocation.hasPendingAsyncFetch()) {
        return AllocateUnassignedDecision.no(AllocationStatus.FETCHING_SHARD_DATA, nodeResults);
    } else if (node != null) {
        return AllocateUnassignedDecision.yes(node, allocationId, nodeResults, false);
    } else if (throttled) {
        return AllocateUnassignedDecision.throttle(nodeResults);
    } else {
        return AllocateUnassignedDecision.no(AllocationStatus.DECIDERS_NO, nodeResults, true);
    }
}

Example 7 with NodeGatewayStartedShards

use of org.elasticsearch.gateway.TransportNodesListGatewayStartedShards.NodeGatewayStartedShards in project elasticsearch by elastic.

the class PrimaryShardAllocator method buildNodeDecisions.

/**
     * Builds a map of nodes to the corresponding allocation decisions for those nodes.
     */
private static List<NodeAllocationResult> buildNodeDecisions(NodesToAllocate nodesToAllocate, FetchResult<NodeGatewayStartedShards> fetchedShardData, Set<String> inSyncAllocationIds) {
    List<NodeAllocationResult> nodeResults = new ArrayList<>();
    Collection<NodeGatewayStartedShards> ineligibleShards;
    if (nodesToAllocate != null) {
        final Set<DiscoveryNode> discoNodes = new HashSet<>();
        nodeResults.addAll(Stream.of(nodesToAllocate.yesNodeShards, nodesToAllocate.throttleNodeShards, nodesToAllocate.noNodeShards).flatMap(Collection::stream).map(dnode -> {
            discoNodes.add(dnode.nodeShardState.getNode());
            return new NodeAllocationResult(dnode.nodeShardState.getNode(), shardStoreInfo(dnode.nodeShardState, inSyncAllocationIds), dnode.decision);
        }).collect(Collectors.toList()));
        ineligibleShards = fetchedShardData.getData().values().stream().filter(shardData -> discoNodes.contains(shardData.getNode()) == false).collect(Collectors.toList());
    } else {
        // there were no shard copies that were eligible for being assigned the allocation,
        // so all fetched shard data are ineligible shards
        ineligibleShards = fetchedShardData.getData().values();
    }
    nodeResults.addAll(ineligibleShards.stream().map(shardData -> new NodeAllocationResult(shardData.getNode(), shardStoreInfo(shardData, inSyncAllocationIds), null)).collect(Collectors.toList()));
    return nodeResults;
}

Example 8 with NodeGatewayStartedShards

use of org.elasticsearch.gateway.TransportNodesListGatewayStartedShards.NodeGatewayStartedShards in project crate by crate.

the class PrimaryShardAllocator method makeAllocationDecision.

@Override
public AllocateUnassignedDecision makeAllocationDecision(final ShardRouting unassignedShard, final RoutingAllocation allocation, final Logger logger) {
    if (isResponsibleFor(unassignedShard) == false) {
        // this allocator is not responsible for allocating this shard
        return AllocateUnassignedDecision.NOT_TAKEN;
    }
    final boolean explain = allocation.debugDecision();
    final FetchResult<NodeGatewayStartedShards> shardState = fetchData(unassignedShard, allocation);
    if (shardState.hasData() == false) {
        allocation.setHasPendingAsyncFetch();
        List<NodeAllocationResult> nodeDecisions = null;
        if (explain) {
            nodeDecisions = buildDecisionsForAllNodes(unassignedShard, allocation);
        }
        return AllocateUnassignedDecision.no(AllocationStatus.FETCHING_SHARD_DATA, nodeDecisions);
    }
    // don't create a new IndexSetting object for every shard as this could cause a lot of garbage
    // on cluster restart if we allocate a boat load of shards
    final IndexMetadata indexMetadata = allocation.metadata().getIndexSafe(unassignedShard.index());
    final Set<String> inSyncAllocationIds = indexMetadata.inSyncAllocationIds(unassignedShard.id());
    final boolean snapshotRestore = unassignedShard.recoverySource().getType() == RecoverySource.Type.SNAPSHOT;
    assert inSyncAllocationIds.isEmpty() == false;
    // use in-sync allocation ids to select nodes
    final NodeShardsResult nodeShardsResult = buildNodeShardsResult(unassignedShard, snapshotRestore, allocation.getIgnoreNodes(unassignedShard.shardId()), inSyncAllocationIds, shardState, logger);
    final boolean enoughAllocationsFound = nodeShardsResult.orderedAllocationCandidates.size() > 0;
    logger.debug("[{}][{}]: found {} allocation candidates of {} based on allocation ids: [{}]", unassignedShard.index(), unassignedShard.id(), nodeShardsResult.orderedAllocationCandidates.size(), unassignedShard, inSyncAllocationIds);
    if (enoughAllocationsFound == false) {
        if (snapshotRestore) {
            // let BalancedShardsAllocator take care of allocating this shard
            logger.debug("[{}][{}]: missing local data, will restore from [{}]", unassignedShard.index(), unassignedShard.id(), unassignedShard.recoverySource());
            return AllocateUnassignedDecision.NOT_TAKEN;
        } else {
            // We have a shard that was previously allocated, but we could not find a valid shard copy to allocate the primary.
            // We could just be waiting for the node that holds the primary to start back up, in which case the allocation for
            // this shard will be picked up when the node joins and we do another allocation reroute
            logger.debug("[{}][{}]: not allocating, number_of_allocated_shards_found [{}]", unassignedShard.index(), unassignedShard.id(), nodeShardsResult.allocationsFound);
            return AllocateUnassignedDecision.no(AllocationStatus.NO_VALID_SHARD_COPY, explain ? buildNodeDecisions(null, shardState, inSyncAllocationIds) : null);
        }
    }
    NodesToAllocate nodesToAllocate = buildNodesToAllocate(allocation, nodeShardsResult.orderedAllocationCandidates, unassignedShard, false);
    DiscoveryNode node = null;
    String allocationId = null;
    boolean throttled = false;
    if (nodesToAllocate.yesNodeShards.isEmpty() == false) {
        DecidedNode decidedNode = nodesToAllocate.yesNodeShards.get(0);
        logger.debug("[{}][{}]: allocating [{}] to [{}] on primary allocation", unassignedShard.index(), unassignedShard.id(), unassignedShard, decidedNode.nodeShardState.getNode());
        node = decidedNode.nodeShardState.getNode();
        allocationId = decidedNode.nodeShardState.allocationId();
    } else if (nodesToAllocate.throttleNodeShards.isEmpty() && !nodesToAllocate.noNodeShards.isEmpty()) {
        // The deciders returned a NO decision for all nodes with shard copies, so we check if primary shard
        // can be force-allocated to one of the nodes.
        nodesToAllocate = buildNodesToAllocate(allocation, nodeShardsResult.orderedAllocationCandidates, unassignedShard, true);
        if (nodesToAllocate.yesNodeShards.isEmpty() == false) {
            final DecidedNode decidedNode = nodesToAllocate.yesNodeShards.get(0);
            final NodeGatewayStartedShards nodeShardState = decidedNode.nodeShardState;
            logger.debug("[{}][{}]: allocating [{}] to [{}] on forced primary allocation", unassignedShard.index(), unassignedShard.id(), unassignedShard, nodeShardState.getNode());
            node = nodeShardState.getNode();
            allocationId = nodeShardState.allocationId();
        } else if (nodesToAllocate.throttleNodeShards.isEmpty() == false) {
            logger.debug("[{}][{}]: throttling allocation [{}] to [{}] on forced primary allocation", unassignedShard.index(), unassignedShard.id(), unassignedShard, nodesToAllocate.throttleNodeShards);
            throttled = true;
        } else {
            logger.debug("[{}][{}]: forced primary allocation denied [{}]", unassignedShard.index(), unassignedShard.id(), unassignedShard);
        }
    } else {
        // we are throttling this, since we are allowed to allocate to this node but there are enough allocations
        // taking place on the node currently, ignore it for now
        logger.debug("[{}][{}]: throttling allocation [{}] to [{}] on primary allocation", unassignedShard.index(), unassignedShard.id(), unassignedShard, nodesToAllocate.throttleNodeShards);
        throttled = true;
    }
    List<NodeAllocationResult> nodeResults = null;
    if (explain) {
        nodeResults = buildNodeDecisions(nodesToAllocate, shardState, inSyncAllocationIds);
    }
    if (allocation.hasPendingAsyncFetch()) {
        return AllocateUnassignedDecision.no(AllocationStatus.FETCHING_SHARD_DATA, nodeResults);
    } else if (node != null) {
        return AllocateUnassignedDecision.yes(node, allocationId, nodeResults, false);
    } else if (throttled) {
        return AllocateUnassignedDecision.throttle(nodeResults);
    } else {
        return AllocateUnassignedDecision.no(AllocationStatus.DECIDERS_NO, nodeResults, true);
    }
}