Examples with AlreadyClosedException org.apache.lucene.store.AlreadyClosedException used on opensource projects

Example 6 with AlreadyClosedException

use of org.apache.lucene.store.AlreadyClosedException in project elasticsearch by elastic.

the class PeerRecoveryTargetService method doRecovery.

private void doRecovery(final long recoveryId) {
    final StartRecoveryRequest request;
    final CancellableThreads cancellableThreads;
    final RecoveryState.Timer timer;
    try (RecoveryRef recoveryRef = onGoingRecoveries.getRecovery(recoveryId)) {
        if (recoveryRef == null) {
            logger.trace("not running recovery with id [{}] - can not find it (probably finished)", recoveryId);
            return;
        }
        final RecoveryTarget recoveryTarget = recoveryRef.target();
        cancellableThreads = recoveryTarget.cancellableThreads();
        timer = recoveryTarget.state().getTimer();
        try {
            assert recoveryTarget.sourceNode() != null : "can not do a recovery without a source node";
            request = getStartRecoveryRequest(recoveryTarget);
            logger.trace("{} preparing shard for peer recovery", recoveryTarget.shardId());
            recoveryTarget.indexShard().prepareForIndexRecovery();
        } catch (final Exception e) {
            // this will be logged as warning later on...
            logger.trace("unexpected error while preparing shard for peer recovery, failing recovery", e);
            onGoingRecoveries.failRecovery(recoveryId, new RecoveryFailedException(recoveryTarget.state(), "failed to prepare shard for recovery", e), true);
            return;
        }
    }
    try {
        logger.trace("{} starting recovery from {}", request.shardId(), request.sourceNode());
        final AtomicReference<RecoveryResponse> responseHolder = new AtomicReference<>();
        cancellableThreads.execute(() -> responseHolder.set(transportService.submitRequest(request.sourceNode(), PeerRecoverySourceService.Actions.START_RECOVERY, request, new FutureTransportResponseHandler<RecoveryResponse>() {

            @Override
            public RecoveryResponse newInstance() {
                return new RecoveryResponse();
            }
        }).txGet()));
        final RecoveryResponse recoveryResponse = responseHolder.get();
        final TimeValue recoveryTime = new TimeValue(timer.time());
        // do this through ongoing recoveries to remove it from the collection
        onGoingRecoveries.markRecoveryAsDone(recoveryId);
        if (logger.isTraceEnabled()) {
            StringBuilder sb = new StringBuilder();
            sb.append('[').append(request.shardId().getIndex().getName()).append(']').append('[').append(request.shardId().id()).append("] ");
            sb.append("recovery completed from ").append(request.sourceNode()).append(", took[").append(recoveryTime).append("]\n");
            sb.append("   phase1: recovered_files [").append(recoveryResponse.phase1FileNames.size()).append("]").append(" with " + "total_size of [").append(new ByteSizeValue(recoveryResponse.phase1TotalSize)).append("]").append(", took [").append(timeValueMillis(recoveryResponse.phase1Time)).append("], throttling_wait [").append(timeValueMillis(recoveryResponse.phase1ThrottlingWaitTime)).append(']').append("\n");
            sb.append("         : reusing_files   [").append(recoveryResponse.phase1ExistingFileNames.size()).append("] with " + "total_size of [").append(new ByteSizeValue(recoveryResponse.phase1ExistingTotalSize)).append("]\n");
            sb.append("   phase2: start took [").append(timeValueMillis(recoveryResponse.startTime)).append("]\n");
            sb.append("         : recovered [").append(recoveryResponse.phase2Operations).append("]").append(" transaction log " + "operations").append(", took [").append(timeValueMillis(recoveryResponse.phase2Time)).append("]").append("\n");
            logger.trace("{}", sb);
        } else {
            logger.debug("{} recovery done from [{}], took [{}]", request.shardId(), request.sourceNode(), recoveryTime);
        }
    } catch (CancellableThreads.ExecutionCancelledException e) {
        logger.trace("recovery cancelled", e);
    } catch (Exception e) {
        if (logger.isTraceEnabled()) {
            logger.trace((Supplier<?>) () -> new ParameterizedMessage("[{}][{}] Got exception on recovery", request.shardId().getIndex().getName(), request.shardId().id()), e);
        }
        Throwable cause = ExceptionsHelper.unwrapCause(e);
        if (cause instanceof CancellableThreads.ExecutionCancelledException) {
            // this can also come from the source wrapped in a RemoteTransportException
            onGoingRecoveries.failRecovery(recoveryId, new RecoveryFailedException(request, "source has canceled the recovery", cause), false);
            return;
        }
        if (cause instanceof RecoveryEngineException) {
            // unwrap an exception that was thrown as part of the recovery
            cause = cause.getCause();
        }
        // do it twice, in case we have double transport exception
        cause = ExceptionsHelper.unwrapCause(cause);
        if (cause instanceof RecoveryEngineException) {
            // unwrap an exception that was thrown as part of the recovery
            cause = cause.getCause();
        }
        if (cause instanceof IllegalIndexShardStateException || cause instanceof IndexNotFoundException || cause instanceof ShardNotFoundException) {
            // if the target is not ready yet, retry
            retryRecovery(recoveryId, "remote shard not ready", recoverySettings.retryDelayStateSync(), recoverySettings.activityTimeout());
            return;
        }
        if (cause instanceof DelayRecoveryException) {
            retryRecovery(recoveryId, cause, recoverySettings.retryDelayStateSync(), recoverySettings.activityTimeout());
            return;
        }
        if (cause instanceof ConnectTransportException) {
            logger.debug("delaying recovery of {} for [{}] due to networking error [{}]", request.shardId(), recoverySettings.retryDelayNetwork(), cause.getMessage());
            retryRecovery(recoveryId, cause.getMessage(), recoverySettings.retryDelayNetwork(), recoverySettings.activityTimeout());
            return;
        }
        if (cause instanceof AlreadyClosedException) {
            onGoingRecoveries.failRecovery(recoveryId, new RecoveryFailedException(request, "source shard is closed", cause), false);
            return;
        }
        onGoingRecoveries.failRecovery(recoveryId, new RecoveryFailedException(request, e), true);
    }
}

Example 7 with AlreadyClosedException

use of org.apache.lucene.store.AlreadyClosedException in project elasticsearch by elastic.

the class RemoteClusterConnectionTests method testCloseWhileConcurrentlyConnecting.

public void testCloseWhileConcurrentlyConnecting() throws IOException, InterruptedException, BrokenBarrierException {
    List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
    try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT);
        MockTransportService seedTransport1 = startTransport("seed_node_1", knownNodes, Version.CURRENT);
        MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) {
        DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
        DiscoveryNode seedNode1 = seedTransport1.getLocalDiscoNode();
        knownNodes.add(seedTransport.getLocalDiscoNode());
        knownNodes.add(discoverableTransport.getLocalDiscoNode());
        knownNodes.add(seedTransport1.getLocalDiscoNode());
        Collections.shuffle(knownNodes, random());
        List<DiscoveryNode> seedNodes = Arrays.asList(seedNode1, seedNode);
        Collections.shuffle(seedNodes, random());
        try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
            service.start();
            service.acceptIncomingRequests();
            try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", seedNodes, service, Integer.MAX_VALUE, n -> true)) {
                int numThreads = randomIntBetween(4, 10);
                Thread[] threads = new Thread[numThreads];
                CyclicBarrier barrier = new CyclicBarrier(numThreads + 1);
                for (int i = 0; i < threads.length; i++) {
                    final int numConnectionAttempts = randomIntBetween(10, 100);
                    threads[i] = new Thread() {

                        @Override
                        public void run() {
                            try {
                                barrier.await();
                                CountDownLatch latch = new CountDownLatch(numConnectionAttempts);
                                for (int i = 0; i < numConnectionAttempts; i++) {
                                    AtomicReference<RuntimeException> executed = new AtomicReference<>();
                                    ActionListener<Void> listener = ActionListener.wrap(x -> {
                                        if (executed.compareAndSet(null, new RuntimeException())) {
                                            latch.countDown();
                                        } else {
                                            throw new AssertionError("shit's been called twice", executed.get());
                                        }
                                    }, x -> {
                                        if (executed.compareAndSet(null, new RuntimeException())) {
                                            latch.countDown();
                                        } else {
                                            throw new AssertionError("shit's been called twice", executed.get());
                                        }
                                        if (x instanceof RejectedExecutionException || x instanceof AlreadyClosedException || x instanceof CancellableThreads.ExecutionCancelledException) {
                                        } else {
                                            throw new AssertionError(x);
                                        }
                                    });
                                    connection.updateSeedNodes(seedNodes, listener);
                                }
                                latch.await();
                            } catch (Exception ex) {
                                throw new AssertionError(ex);
                            }
                        }
                    };
                    threads[i].start();
                }
                barrier.await();
                connection.close();
            }
        }
    }
}

Example 8 with AlreadyClosedException

use of org.apache.lucene.store.AlreadyClosedException in project elasticsearch by elastic.

the class UnicastZenPing method ping.

/**
     * a variant of {@link #ping(Consumer, TimeValue)}, but allows separating the scheduling duration
     * from the duration used for request level time outs. This is useful for testing
     */
protected void ping(final Consumer<PingCollection> resultsConsumer, final TimeValue scheduleDuration, final TimeValue requestDuration) {
    final List<DiscoveryNode> seedNodes;
    try {
        seedNodes = resolveHostsLists(unicastZenPingExecutorService, logger, configuredHosts, limitPortCounts, transportService, UNICAST_NODE_PREFIX, resolveTimeout);
    } catch (InterruptedException e) {
        throw new RuntimeException(e);
    }
    seedNodes.addAll(hostsProvider.buildDynamicNodes());
    final DiscoveryNodes nodes = contextProvider.nodes();
    // add all possible master nodes that were active in the last known cluster configuration
    for (ObjectCursor<DiscoveryNode> masterNode : nodes.getMasterNodes().values()) {
        seedNodes.add(masterNode.value);
    }
    final ConnectionProfile connectionProfile = ConnectionProfile.buildSingleChannelProfile(TransportRequestOptions.Type.REG, requestDuration, requestDuration);
    final PingingRound pingingRound = new PingingRound(pingingRoundIdGenerator.incrementAndGet(), seedNodes, resultsConsumer, nodes.getLocalNode(), connectionProfile);
    activePingingRounds.put(pingingRound.id(), pingingRound);
    final AbstractRunnable pingSender = new AbstractRunnable() {

        @Override
        public void onFailure(Exception e) {
            if (e instanceof AlreadyClosedException == false) {
                logger.warn("unexpected error while pinging", e);
            }
        }

        @Override
        protected void doRun() throws Exception {
            sendPings(requestDuration, pingingRound);
        }
    };
    threadPool.generic().execute(pingSender);
    threadPool.schedule(TimeValue.timeValueMillis(scheduleDuration.millis() / 3), ThreadPool.Names.GENERIC, pingSender);
    threadPool.schedule(TimeValue.timeValueMillis(scheduleDuration.millis() / 3 * 2), ThreadPool.Names.GENERIC, pingSender);
    threadPool.schedule(scheduleDuration, ThreadPool.Names.GENERIC, new AbstractRunnable() {

        @Override
        protected void doRun() throws Exception {
            finishPingingRound(pingingRound);
        }

        @Override
        public void onFailure(Exception e) {
            logger.warn("unexpected error while finishing pinging round", e);
        }
    });
}

Example 9 with AlreadyClosedException

use of org.apache.lucene.store.AlreadyClosedException in project elasticsearch by elastic.

the class ShadowEngineTests method testFailEngineOnCorruption.

public void testFailEngineOnCorruption() throws IOException {
    ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null);
    primaryEngine.index(indexForDoc(doc));
    primaryEngine.flush();
    MockDirectoryWrapper leaf = DirectoryUtils.getLeaf(replicaEngine.config().getStore().directory(), MockDirectoryWrapper.class);
    leaf.setRandomIOExceptionRate(1.0);
    leaf.setRandomIOExceptionRateOnOpen(1.0);
    try {
        replicaEngine.refresh("foo");
        fail("exception expected");
    } catch (Exception ex) {
    }
    try {
        Engine.Searcher searchResult = replicaEngine.acquireSearcher("test");
        MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
        MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1));
        searchResult.close();
        fail("exception expected");
    } catch (AlreadyClosedException ex) {
    // all is well
    }
}

Example 10 with AlreadyClosedException

use of org.apache.lucene.store.AlreadyClosedException in project elasticsearch by elastic.

the class InternalEngineTests method testForceMergeAndClose.

public void testForceMergeAndClose() throws IOException, InterruptedException {
    int numIters = randomIntBetween(2, 10);
    for (int j = 0; j < numIters; j++) {
        try (Store store = createStore()) {
            final InternalEngine engine = createEngine(store, createTempDir());
            final CountDownLatch startGun = new CountDownLatch(1);
            final CountDownLatch indexed = new CountDownLatch(1);
            Thread thread = new Thread() {

                @Override
                public void run() {
                    try {
                        try {
                            startGun.await();
                        } catch (InterruptedException e) {
                            throw new RuntimeException(e);
                        }
                        int i = 0;
                        while (true) {
                            int numDocs = randomIntBetween(1, 20);
                            for (int j = 0; j < numDocs; j++) {
                                i++;
                                ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, testDocument(), B_1, null);
                                Engine.Index index = indexForDoc(doc);
                                engine.index(index);
                            }
                            engine.refresh("test");
                            indexed.countDown();
                            try {
                                engine.forceMerge(randomBoolean(), 1, false, randomBoolean(), randomBoolean());
                            } catch (IOException e) {
                                return;
                            }
                        }
                    } catch (AlreadyClosedException ex) {
                    // fine
                    } catch (IOException e) {
                        throw new AssertionError(e);
                    }
                }
            };
            thread.start();
            startGun.countDown();
            int someIters = randomIntBetween(1, 10);
            for (int i = 0; i < someIters; i++) {
                engine.forceMerge(randomBoolean(), 1, false, randomBoolean(), randomBoolean());
            }
            indexed.await();
            IOUtils.close(engine);
            thread.join();
        }
    }
}