Examples with ShardLockObtainFailedException org.elasticsearch.env.ShardLockObtainFailedException used on opensource projects

Example 11 with ShardLockObtainFailedException

use of org.elasticsearch.env.ShardLockObtainFailedException in project elasticsearch by elastic.

the class PrimaryShardAllocator method buildNodeShardsResult.

/**
     * Builds a list of nodes. If matchAnyShard is set to false, only nodes that have an allocation id matching
     * inSyncAllocationIds are added to the list. Otherwise, any node that has a shard is added to the list, but
     * entries with matching allocation id are always at the front of the list.
     */
protected static NodeShardsResult buildNodeShardsResult(ShardRouting shard, boolean matchAnyShard, Set<String> ignoreNodes, Set<String> inSyncAllocationIds, FetchResult<NodeGatewayStartedShards> shardState, Logger logger) {
    List<NodeGatewayStartedShards> nodeShardStates = new ArrayList<>();
    int numberOfAllocationsFound = 0;
    for (NodeGatewayStartedShards nodeShardState : shardState.getData().values()) {
        DiscoveryNode node = nodeShardState.getNode();
        String allocationId = nodeShardState.allocationId();
        if (ignoreNodes.contains(node.getId())) {
            continue;
        }
        if (nodeShardState.storeException() == null) {
            if (allocationId == null) {
                logger.trace("[{}] on node [{}] has no shard state information", shard, nodeShardState.getNode());
            } else {
                logger.trace("[{}] on node [{}] has allocation id [{}]", shard, nodeShardState.getNode(), allocationId);
            }
        } else {
            final String finalAllocationId = allocationId;
            if (nodeShardState.storeException() instanceof ShardLockObtainFailedException) {
                logger.trace((Supplier<?>) () -> new ParameterizedMessage("[{}] on node [{}] has allocation id [{}] but the store can not be opened as it's locked, treating as valid shard", shard, nodeShardState.getNode(), finalAllocationId), nodeShardState.storeException());
            } else {
                logger.trace((Supplier<?>) () -> new ParameterizedMessage("[{}] on node [{}] has allocation id [{}] but the store can not be opened, treating as no allocation id", shard, nodeShardState.getNode(), finalAllocationId), nodeShardState.storeException());
                allocationId = null;
            }
        }
        if (allocationId != null) {
            assert nodeShardState.storeException() == null || nodeShardState.storeException() instanceof ShardLockObtainFailedException : "only allow store that can be opened or that throws a ShardLockObtainFailedException while being opened but got a store throwing " + nodeShardState.storeException();
            numberOfAllocationsFound++;
            if (matchAnyShard || inSyncAllocationIds.contains(nodeShardState.allocationId())) {
                nodeShardStates.add(nodeShardState);
            }
        }
    }
    // allocation preference
    final Comparator<NodeGatewayStartedShards> comparator;
    if (matchAnyShard) {
        // prefer shards with matching allocation ids
        Comparator<NodeGatewayStartedShards> matchingAllocationsFirst = Comparator.comparing((NodeGatewayStartedShards state) -> inSyncAllocationIds.contains(state.allocationId())).reversed();
        comparator = matchingAllocationsFirst.thenComparing(NO_STORE_EXCEPTION_FIRST_COMPARATOR).thenComparing(PRIMARY_FIRST_COMPARATOR);
    } else {
        comparator = NO_STORE_EXCEPTION_FIRST_COMPARATOR.thenComparing(PRIMARY_FIRST_COMPARATOR);
    }
    nodeShardStates.sort(comparator);
    if (logger.isTraceEnabled()) {
        logger.trace("{} candidates for allocation: {}", shard, nodeShardStates.stream().map(s -> s.getNode().getName()).collect(Collectors.joining(", ")));
    }
    return new NodeShardsResult(nodeShardStates, numberOfAllocationsFound);
}

Example 12 with ShardLockObtainFailedException

use of org.elasticsearch.env.ShardLockObtainFailedException in project elasticsearch by elastic.

the class InternalTestCluster method assertAfterTest.

@Override
public void assertAfterTest() throws IOException {
    super.assertAfterTest();
    assertRequestsFinished();
    for (NodeAndClient nodeAndClient : nodes.values()) {
        NodeEnvironment env = nodeAndClient.node().getNodeEnvironment();
        Set<ShardId> shardIds = env.lockedShards();
        for (ShardId id : shardIds) {
            try {
                env.shardLock(id, TimeUnit.SECONDS.toMillis(5)).close();
            } catch (ShardLockObtainFailedException ex) {
                fail("Shard " + id + " is still locked after 5 sec waiting");
            }
        }
    }
}

Example 13 with ShardLockObtainFailedException

use of org.elasticsearch.env.ShardLockObtainFailedException in project elasticsearch by elastic.

the class ExceptionSerializationTests method testBWCShardLockObtainFailedException.

public void testBWCShardLockObtainFailedException() throws IOException {
    ShardId shardId = new ShardId("foo", "_na_", 1);
    ShardLockObtainFailedException orig = new ShardLockObtainFailedException(shardId, "boom");
    Exception ex = serialize((Exception) orig, randomFrom(Version.V_5_0_0, Version.V_5_0_1));
    assertThat(ex, instanceOf(NotSerializableExceptionWrapper.class));
    assertEquals("shard_lock_obtain_failed_exception: [foo][1]: boom", ex.getMessage());
}

Example 14 with ShardLockObtainFailedException

use of org.elasticsearch.env.ShardLockObtainFailedException in project crate by crate.

the class IndicesClusterStateService method deleteIndices.

/**
 * Deletes indices (with shard data).
 *
 * @param event cluster change event
 */
private void deleteIndices(final ClusterChangedEvent event) {
    final ClusterState previousState = event.previousState();
    final ClusterState state = event.state();
    final String localNodeId = state.nodes().getLocalNodeId();
    assert localNodeId != null;
    for (Index index : event.indicesDeleted()) {
        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("[{}] cleaning index, no longer part of the metadata", index);
        }
        AllocatedIndex<? extends Shard> indexService = indicesService.indexService(index);
        final IndexSettings indexSettings;
        if (indexService != null) {
            indexSettings = indexService.getIndexSettings();
            indicesService.removeIndex(index, DELETED, "index no longer part of the metadata");
        } else if (previousState.metadata().hasIndex(index)) {
            // The deleted index was part of the previous cluster state, but not loaded on the local node
            final IndexMetadata metadata = previousState.metadata().index(index);
            indexSettings = new IndexSettings(metadata, settings);
            indicesService.deleteUnassignedIndex("deleted index was not assigned to local node", metadata, state);
        } else {
            // asserting that the previous cluster state is not initialized/recovered.
            assert previousState.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK);
            final IndexMetadata metadata = indicesService.verifyIndexIsDeleted(index, event.state());
            if (metadata != null) {
                indexSettings = new IndexSettings(metadata, settings);
            } else {
                indexSettings = null;
            }
        }
        if (indexSettings != null) {
            threadPool.generic().execute(new AbstractRunnable() {

                @Override
                public void onFailure(Exception e) {
                    LOGGER.warn(() -> new ParameterizedMessage("[{}] failed to complete pending deletion for index", index), e);
                }

                @Override
                protected void doRun() throws Exception {
                    try {
                        // we are waiting until we can lock the index / all shards on the node and then we ack the delete of the store
                        // to the master. If we can't acquire the locks here immediately there might be a shard of this index still
                        // holding on to the lock due to a "currently canceled recovery" or so. The shard will delete itself BEFORE the
                        // lock is released so it's guaranteed to be deleted by the time we get the lock
                        indicesService.processPendingDeletes(index, indexSettings, new TimeValue(30, TimeUnit.MINUTES));
                    } catch (ShardLockObtainFailedException exc) {
                        LOGGER.warn("[{}] failed to lock all shards for index - timed out after 30 seconds", index);
                    } catch (InterruptedException e) {
                        LOGGER.warn("[{}] failed to lock all shards for index - interrupted", index);
                    }
                }
            });
        }
    }
}