Examples with IllegalIndexShardStateException org.elasticsearch.index.shard.IllegalIndexShardStateException used on opensource projects

Example 1 with IllegalIndexShardStateException

use of org.elasticsearch.index.shard.IllegalIndexShardStateException in project crate by crate.

the class BlobRecoverySourceHandler method finalizeRecovery.

/**
     * finalizes the recovery process
     */
public void finalizeRecovery() {
    if (shard.state() == IndexShardState.CLOSED) {
        throw new IndexShardClosedException(request.shardId());
    }
    cancellableThreads.checkForCancel();
    StopWatch stopWatch = new StopWatch().start();
    logger.trace("[{}][{}] finalizing recovery to {}", indexName, shardId, request.targetNode());
    cancellableThreads.execute(new Interruptable() {

        @Override
        public void run() throws InterruptedException {
            // Send the FINALIZE request to the target node. The finalize request
            // clears unreferenced translog files, refreshes the engine now that
            // new segments are available, and enables garbage collection of
            // tombstone files. The shard is also moved to the POST_RECOVERY phase
            // during this time
            transportService.submitRequest(request.targetNode(), RecoveryTarget.Actions.FINALIZE, new RecoveryFinalizeRecoveryRequest(request.recoveryId(), request.shardId()), TransportRequestOptions.builder().withTimeout(recoverySettings.internalActionLongTimeout()).build(), EmptyTransportResponseHandler.INSTANCE_SAME).txGet();
        }
    });
    if (request.markAsRelocated()) {
        // TODO what happens if the recovery process fails afterwards, we need to mark this back to started
        try {
            shard.relocated("to " + request.targetNode());
        } catch (IllegalIndexShardStateException e) {
        // we can ignore this exception since, on the other node, when it moved to phase3
        // it will also send shard started, which might cause the index shard we work against
        // to move be closed by the time we get to the the relocated method
        }
    }
    stopWatch.stop();
    logger.trace("[{}][{}] finalizing recovery to {}: took [{}]", indexName, shardId, request.targetNode(), stopWatch.totalTime());
}

Example 2 with IllegalIndexShardStateException

use of org.elasticsearch.index.shard.IllegalIndexShardStateException in project elasticsearch by elastic.

the class PeerRecoveryTargetService method doRecovery.

private void doRecovery(final long recoveryId) {
    final StartRecoveryRequest request;
    final CancellableThreads cancellableThreads;
    final RecoveryState.Timer timer;
    try (RecoveryRef recoveryRef = onGoingRecoveries.getRecovery(recoveryId)) {
        if (recoveryRef == null) {
            logger.trace("not running recovery with id [{}] - can not find it (probably finished)", recoveryId);
            return;
        }
        final RecoveryTarget recoveryTarget = recoveryRef.target();
        cancellableThreads = recoveryTarget.cancellableThreads();
        timer = recoveryTarget.state().getTimer();
        try {
            assert recoveryTarget.sourceNode() != null : "can not do a recovery without a source node";
            request = getStartRecoveryRequest(recoveryTarget);
            logger.trace("{} preparing shard for peer recovery", recoveryTarget.shardId());
            recoveryTarget.indexShard().prepareForIndexRecovery();
        } catch (final Exception e) {
            // this will be logged as warning later on...
            logger.trace("unexpected error while preparing shard for peer recovery, failing recovery", e);
            onGoingRecoveries.failRecovery(recoveryId, new RecoveryFailedException(recoveryTarget.state(), "failed to prepare shard for recovery", e), true);
            return;
        }
    }
    try {
        logger.trace("{} starting recovery from {}", request.shardId(), request.sourceNode());
        final AtomicReference<RecoveryResponse> responseHolder = new AtomicReference<>();
        cancellableThreads.execute(() -> responseHolder.set(transportService.submitRequest(request.sourceNode(), PeerRecoverySourceService.Actions.START_RECOVERY, request, new FutureTransportResponseHandler<RecoveryResponse>() {

            @Override
            public RecoveryResponse newInstance() {
                return new RecoveryResponse();
            }
        }).txGet()));
        final RecoveryResponse recoveryResponse = responseHolder.get();
        final TimeValue recoveryTime = new TimeValue(timer.time());
        // do this through ongoing recoveries to remove it from the collection
        onGoingRecoveries.markRecoveryAsDone(recoveryId);
        if (logger.isTraceEnabled()) {
            StringBuilder sb = new StringBuilder();
            sb.append('[').append(request.shardId().getIndex().getName()).append(']').append('[').append(request.shardId().id()).append("] ");
            sb.append("recovery completed from ").append(request.sourceNode()).append(", took[").append(recoveryTime).append("]\n");
            sb.append("   phase1: recovered_files [").append(recoveryResponse.phase1FileNames.size()).append("]").append(" with " + "total_size of [").append(new ByteSizeValue(recoveryResponse.phase1TotalSize)).append("]").append(", took [").append(timeValueMillis(recoveryResponse.phase1Time)).append("], throttling_wait [").append(timeValueMillis(recoveryResponse.phase1ThrottlingWaitTime)).append(']').append("\n");
            sb.append("         : reusing_files   [").append(recoveryResponse.phase1ExistingFileNames.size()).append("] with " + "total_size of [").append(new ByteSizeValue(recoveryResponse.phase1ExistingTotalSize)).append("]\n");
            sb.append("   phase2: start took [").append(timeValueMillis(recoveryResponse.startTime)).append("]\n");
            sb.append("         : recovered [").append(recoveryResponse.phase2Operations).append("]").append(" transaction log " + "operations").append(", took [").append(timeValueMillis(recoveryResponse.phase2Time)).append("]").append("\n");
            logger.trace("{}", sb);
        } else {
            logger.debug("{} recovery done from [{}], took [{}]", request.shardId(), request.sourceNode(), recoveryTime);
        }
    } catch (CancellableThreads.ExecutionCancelledException e) {
        logger.trace("recovery cancelled", e);
    } catch (Exception e) {
        if (logger.isTraceEnabled()) {
            logger.trace((Supplier<?>) () -> new ParameterizedMessage("[{}][{}] Got exception on recovery", request.shardId().getIndex().getName(), request.shardId().id()), e);
        }
        Throwable cause = ExceptionsHelper.unwrapCause(e);
        if (cause instanceof CancellableThreads.ExecutionCancelledException) {
            // this can also come from the source wrapped in a RemoteTransportException
            onGoingRecoveries.failRecovery(recoveryId, new RecoveryFailedException(request, "source has canceled the recovery", cause), false);
            return;
        }
        if (cause instanceof RecoveryEngineException) {
            // unwrap an exception that was thrown as part of the recovery
            cause = cause.getCause();
        }
        // do it twice, in case we have double transport exception
        cause = ExceptionsHelper.unwrapCause(cause);
        if (cause instanceof RecoveryEngineException) {
            // unwrap an exception that was thrown as part of the recovery
            cause = cause.getCause();
        }
        if (cause instanceof IllegalIndexShardStateException || cause instanceof IndexNotFoundException || cause instanceof ShardNotFoundException) {
            // if the target is not ready yet, retry
            retryRecovery(recoveryId, "remote shard not ready", recoverySettings.retryDelayStateSync(), recoverySettings.activityTimeout());
            return;
        }
        if (cause instanceof DelayRecoveryException) {
            retryRecovery(recoveryId, cause, recoverySettings.retryDelayStateSync(), recoverySettings.activityTimeout());
            return;
        }
        if (cause instanceof ConnectTransportException) {
            logger.debug("delaying recovery of {} for [{}] due to networking error [{}]", request.shardId(), recoverySettings.retryDelayNetwork(), cause.getMessage());
            retryRecovery(recoveryId, cause.getMessage(), recoverySettings.retryDelayNetwork(), recoverySettings.activityTimeout());
            return;
        }
        if (cause instanceof AlreadyClosedException) {
            onGoingRecoveries.failRecovery(recoveryId, new RecoveryFailedException(request, "source shard is closed", cause), false);
            return;
        }
        onGoingRecoveries.failRecovery(recoveryId, new RecoveryFailedException(request, e), true);
    }
}

Example 3 with IllegalIndexShardStateException

use of org.elasticsearch.index.shard.IllegalIndexShardStateException in project crate by crate.

the class ShardCollectSource method getDocCollectors.

private Collection<CrateCollector.Builder> getDocCollectors(JobCollectContext jobCollectContext, RoutedCollectPhase collectPhase, boolean requiresScroll, Map<String, List<Integer>> indexShards) {
    List<CrateCollector.Builder> crateCollectors = new ArrayList<>();
    for (Map.Entry<String, List<Integer>> entry : indexShards.entrySet()) {
        String indexName = entry.getKey();
        try {
            indicesService.indexServiceSafe(indexName);
        } catch (IndexNotFoundException e) {
            if (PartitionName.isPartition(indexName)) {
                continue;
            }
            throw e;
        }
        for (Integer shardNum : entry.getValue()) {
            ShardId shardId = new ShardId(indexName, shardNum);
            try {
                ShardCollectorProvider shardCollectorProvider = getCollectorProviderSafe(shardId);
                CrateCollector.Builder collector = shardCollectorProvider.getCollectorBuilder(collectPhase, requiresScroll, jobCollectContext);
                crateCollectors.add(collector);
            } catch (ShardNotFoundException | IllegalIndexShardStateException e) {
                // and the reader required in the fetchPhase would be missing.
                if (Symbols.containsColumn(collectPhase.toCollect(), DocSysColumns.FETCHID)) {
                    throw e;
                }
                crateCollectors.add(remoteCollectorFactory.createCollector(shardId.getIndex(), shardId.id(), collectPhase, jobCollectContext.queryPhaseRamAccountingContext()));
            } catch (InterruptedException e) {
                throw Throwables.propagate(e);
            } catch (Throwable t) {
                throw new UnhandledServerException(t);
            }
        }
    }
    return crateCollectors;
}

Example 4 with IllegalIndexShardStateException

use of org.elasticsearch.index.shard.IllegalIndexShardStateException in project crate by crate.

the class ShardCollectSource method createMultiShardScoreDocCollector.

private CrateCollector createMultiShardScoreDocCollector(RoutedCollectPhase collectPhase, BatchConsumer consumer, JobCollectContext jobCollectContext, String localNodeId) {
    Map<String, Map<String, List<Integer>>> locations = collectPhase.routing().locations();
    SharedShardContexts sharedShardContexts = jobCollectContext.sharedShardContexts();
    Map<String, List<Integer>> indexShards = locations.get(localNodeId);
    List<OrderedDocCollector> orderedDocCollectors = new ArrayList<>();
    for (Map.Entry<String, List<Integer>> entry : indexShards.entrySet()) {
        String indexName = entry.getKey();
        for (Integer shardNum : entry.getValue()) {
            ShardId shardId = new ShardId(indexName, shardNum);
            SharedShardContext context = sharedShardContexts.getOrCreateContext(shardId);
            try {
                ShardCollectorProvider shardCollectorProvider = getCollectorProviderSafe(shardId);
                orderedDocCollectors.add(shardCollectorProvider.getOrderedCollector(collectPhase, context, jobCollectContext, consumer.requiresScroll()));
            } catch (ShardNotFoundException | IllegalIndexShardStateException e) {
                throw e;
            } catch (IndexNotFoundException e) {
                if (PartitionName.isPartition(indexName)) {
                    break;
                }
                throw e;
            } catch (Throwable t) {
                throw new UnhandledServerException(t);
            }
        }
    }
    OrderBy orderBy = collectPhase.orderBy();
    assert orderBy != null : "orderBy must not be null";
    return BatchIteratorCollectorBridge.newInstance(OrderedLuceneBatchIteratorFactory.newInstance(orderedDocCollectors, collectPhase.toCollect().size(), OrderingByPosition.rowOrdering(OrderByPositionVisitor.orderByPositions(orderBy.orderBySymbols(), collectPhase.toCollect()), orderBy.reverseFlags(), orderBy.nullsFirst()), executor, consumer.requiresScroll()), consumer);
}

Example 5 with IllegalIndexShardStateException

use of org.elasticsearch.index.shard.IllegalIndexShardStateException in project crate by crate.

the class ShardCollectSource method getShardsCollector.

private CrateCollector getShardsCollector(RoutedCollectPhase collectPhase, RoutedCollectPhase normalizedPhase, String localNodeId, BatchConsumer consumer) {
    Map<String, Map<String, List<Integer>>> locations = collectPhase.routing().locations();
    List<UnassignedShard> unassignedShards = new ArrayList<>();
    List<Object[]> rows = new ArrayList<>();
    Map<String, List<Integer>> indexShardsMap = locations.get(localNodeId);
    for (Map.Entry<String, List<Integer>> indexShards : indexShardsMap.entrySet()) {
        String indexName = indexShards.getKey();
        List<Integer> shards = indexShards.getValue();
        IndexService indexService = indicesService.indexService(indexName);
        if (indexService == null) {
            for (Integer shard : shards) {
                unassignedShards.add(toUnassignedShard(new ShardId(indexName, UnassignedShard.markAssigned(shard))));
            }
            continue;
        }
        for (Integer shard : shards) {
            if (UnassignedShard.isUnassigned(shard)) {
                unassignedShards.add(toUnassignedShard(new ShardId(indexName, UnassignedShard.markAssigned(shard))));
                continue;
            }
            ShardId shardId = new ShardId(indexName, shard);
            try {
                ShardCollectorProvider shardCollectorProvider = getCollectorProviderSafe(shardId);
                Object[] row = shardCollectorProvider.getRowForShard(normalizedPhase);
                if (row != null) {
                    rows.add(row);
                }
            } catch (ShardNotFoundException | IllegalIndexShardStateException e) {
                unassignedShards.add(toUnassignedShard(shardId));
            } catch (Throwable t) {
                t.printStackTrace();
                throw new UnhandledServerException(t);
            }
        }
    }
    if (!unassignedShards.isEmpty()) {
        // because otherwise if _node was also selected it would contain something which is wrong
        for (Row row : systemCollectSource.toRowsIterableTransformation(collectPhase, false).apply(unassignedShards)) {
            rows.add(row.materialize());
        }
    }
    if (collectPhase.orderBy() != null) {
        rows.sort(OrderingByPosition.arrayOrdering(collectPhase).reverse());
    }
    return BatchIteratorCollectorBridge.newInstance(RowsBatchIterator.newInstance(Iterables.transform(rows, Buckets.arrayToRowFunction()), collectPhase.outputTypes().size()), consumer);
}