Examples with ShardLock org.opensearch.env.ShardLock used on opensource projects

Example 1 with ShardLock

use of org.opensearch.env.ShardLock in project OpenSearch by opensearch-project.

the class IndexShardIT method testLockTryingToDelete.

public void testLockTryingToDelete() throws Exception {
    createIndex("test");
    ensureGreen();
    NodeEnvironment env = getInstanceFromNode(NodeEnvironment.class);
    ClusterService cs = getInstanceFromNode(ClusterService.class);
    final Index index = cs.state().metadata().index("test").getIndex();
    Path[] shardPaths = env.availableShardPaths(new ShardId(index, 0));
    logger.info("--> paths: [{}]", (Object) shardPaths);
    // Should not be able to acquire the lock because it's already open
    try {
        NodeEnvironment.acquireFSLockForPaths(IndexSettingsModule.newIndexSettings("test", Settings.EMPTY), shardPaths);
        fail("should not have been able to acquire the lock");
    } catch (LockObtainFailedException e) {
        assertTrue("msg: " + e.getMessage(), e.getMessage().contains("unable to acquire write.lock"));
    }
    // Test without the regular shard lock to assume we can acquire it
    // (worst case, meaning that the shard lock could be acquired and
    // we're green to delete the shard's directory)
    ShardLock sLock = new DummyShardLock(new ShardId(index, 0));
    try {
        env.deleteShardDirectoryUnderLock(sLock, IndexSettingsModule.newIndexSettings("test", Settings.EMPTY));
        fail("should not have been able to delete the directory");
    } catch (LockObtainFailedException e) {
        assertTrue("msg: " + e.getMessage(), e.getMessage().contains("unable to acquire write.lock"));
    }
}

Example 2 with ShardLock

use of org.opensearch.env.ShardLock in project OpenSearch by opensearch-project.

the class StoreTests method testOnCloseCallback.

public void testOnCloseCallback() throws IOException {
    final ShardId shardId = new ShardId(new Index(randomRealisticUnicodeOfCodepointLengthBetween(1, 10), "_na_"), randomIntBetween(0, 100));
    final AtomicInteger count = new AtomicInteger(0);
    final ShardLock lock = new DummyShardLock(shardId);
    Store store = new Store(shardId, INDEX_SETTINGS, StoreTests.newDirectory(random()), lock, theLock -> {
        assertEquals(shardId, theLock.getShardId());
        assertEquals(lock, theLock);
        count.incrementAndGet();
    });
    assertEquals(count.get(), 0);
    final int iters = randomIntBetween(1, 10);
    for (int i = 0; i < iters; i++) {
        store.close();
    }
    assertEquals(count.get(), 1);
}

Example 3 with ShardLock

use of org.opensearch.env.ShardLock in project OpenSearch by opensearch-project.

the class IndicesService method processPendingDeletes.

/**
 * Processes all pending deletes for the given index. This method will acquire all locks for the given index and will
 * process all pending deletes for this index. Pending deletes might occur if the OS doesn't allow deletion of files because
 * they are used by a different process ie. on Windows where files might still be open by a virus scanner. On a shared
 * filesystem a replica might not have been closed when the primary is deleted causing problems on delete calls so we
 * schedule there deletes later.
 * @param index the index to process the pending deletes for
 * @param timeout the timeout used for processing pending deletes
 */
@Override
public void processPendingDeletes(Index index, IndexSettings indexSettings, TimeValue timeout) throws IOException, InterruptedException, ShardLockObtainFailedException {
    logger.debug("{} processing pending deletes", index);
    final long startTimeNS = System.nanoTime();
    final List<ShardLock> shardLocks = nodeEnv.lockAllForIndex(index, indexSettings, "process pending deletes", timeout.millis());
    int numRemoved = 0;
    try {
        Map<ShardId, ShardLock> locks = new HashMap<>();
        for (ShardLock lock : shardLocks) {
            locks.put(lock.getShardId(), lock);
        }
        final List<PendingDelete> remove;
        synchronized (pendingDeletes) {
            remove = pendingDeletes.remove(index);
        }
        if (remove != null && remove.isEmpty() == false) {
            numRemoved = remove.size();
            // make sure we delete indices first
            CollectionUtil.timSort(remove);
            // ensure we retry after 10 sec
            final long maxSleepTimeMs = 10 * 1000;
            long sleepTime = 10;
            do {
                if (remove.isEmpty()) {
                    break;
                }
                Iterator<PendingDelete> iterator = remove.iterator();
                while (iterator.hasNext()) {
                    PendingDelete delete = iterator.next();
                    if (delete.deleteIndex) {
                        assert delete.shardId == -1;
                        logger.debug("{} deleting index store reason [{}]", index, "pending delete");
                        try {
                            nodeEnv.deleteIndexDirectoryUnderLock(index, indexSettings);
                            iterator.remove();
                        } catch (IOException ex) {
                            logger.debug(() -> new ParameterizedMessage("{} retry pending delete", index), ex);
                        }
                    } else {
                        assert delete.shardId != -1;
                        ShardLock shardLock = locks.get(new ShardId(delete.index, delete.shardId));
                        if (shardLock != null) {
                            try {
                                deleteShardStore("pending delete", shardLock, delete.settings);
                                iterator.remove();
                            } catch (IOException ex) {
                                logger.debug(() -> new ParameterizedMessage("{} retry pending delete", shardLock.getShardId()), ex);
                            }
                        } else {
                            logger.warn("{} no shard lock for pending delete", delete.shardId);
                            iterator.remove();
                        }
                    }
                }
                if (remove.isEmpty() == false) {
                    logger.warn("{} still pending deletes present for shards {} - retrying", index, remove.toString());
                    Thread.sleep(sleepTime);
                    // increase the sleep time gradually
                    sleepTime = Math.min(maxSleepTimeMs, sleepTime * 2);
                    logger.debug("{} schedule pending delete retry after {} ms", index, sleepTime);
                }
            } while ((System.nanoTime() - startTimeNS) < timeout.nanos());
        }
    } finally {
        IOUtils.close(shardLocks);
        if (numRemoved > 0) {
            int remainingUncompletedDeletes = numUncompletedDeletes.addAndGet(-numRemoved);
            assert remainingUncompletedDeletes >= 0;
        }
    }
}

Example 4 with ShardLock

use of org.opensearch.env.ShardLock in project OpenSearch by opensearch-project.

the class Store method tryOpenIndex.

/**
 * Tries to open an index for the given location. This includes reading the
 * segment infos and possible corruption markers. If the index can not
 * be opened, an exception is thrown
 */
public static void tryOpenIndex(Path indexLocation, ShardId shardId, NodeEnvironment.ShardLocker shardLocker, Logger logger) throws IOException, ShardLockObtainFailedException {
    try (ShardLock lock = shardLocker.lock(shardId, "open index", TimeUnit.SECONDS.toMillis(5));
        Directory dir = new NIOFSDirectory(indexLocation)) {
        failIfCorrupted(dir);
        SegmentInfos segInfo = Lucene.readSegmentInfos(dir);
        logger.trace("{} loaded segment info [{}]", shardId, segInfo);
    }
}

Example 5 with ShardLock

use of org.opensearch.env.ShardLock in project OpenSearch by opensearch-project.

the class IndexService method createShard.

public synchronized IndexShard createShard(final ShardRouting routing, final Consumer<ShardId> globalCheckpointSyncer, final RetentionLeaseSyncer retentionLeaseSyncer) throws IOException {
    Objects.requireNonNull(retentionLeaseSyncer);
    /*
         * TODO: we execute this in parallel but it's a synced method. Yet, we might
         * be able to serialize the execution via the cluster state in the future. for now we just
         * keep it synced.
         */
    if (closed.get()) {
        throw new IllegalStateException("Can't create shard " + routing.shardId() + ", closed");
    }
    final Settings indexSettings = this.indexSettings.getSettings();
    final ShardId shardId = routing.shardId();
    boolean success = false;
    Store store = null;
    IndexShard indexShard = null;
    ShardLock lock = null;
    try {
        lock = nodeEnv.shardLock(shardId, "starting shard", TimeUnit.SECONDS.toMillis(5));
        eventListener.beforeIndexShardCreated(shardId, indexSettings);
        ShardPath path;
        try {
            path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings.customDataPath());
        } catch (IllegalStateException ex) {
            logger.warn("{} failed to load shard path, trying to remove leftover", shardId);
            try {
                ShardPath.deleteLeftoverShardDirectory(logger, nodeEnv, lock, this.indexSettings);
                path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings.customDataPath());
            } catch (Exception inner) {
                ex.addSuppressed(inner);
                throw ex;
            }
        }
        if (path == null) {
            // TODO: we should, instead, hold a "bytes reserved" of how large we anticipate this shard will be, e.g. for a shard
            // that's being relocated/replicated we know how large it will become once it's done copying:
            // Count up how many shards are currently on each data path:
            Map<Path, Integer> dataPathToShardCount = new HashMap<>();
            for (IndexShard shard : this) {
                Path dataPath = shard.shardPath().getRootStatePath();
                Integer curCount = dataPathToShardCount.get(dataPath);
                if (curCount == null) {
                    curCount = 0;
                }
                dataPathToShardCount.put(dataPath, curCount + 1);
            }
            path = ShardPath.selectNewPathForShard(nodeEnv, shardId, this.indexSettings, routing.getExpectedShardSize() == ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE ? getAvgShardSizeInBytes() : routing.getExpectedShardSize(), dataPathToShardCount);
            logger.debug("{} creating using a new path [{}]", shardId, path);
        } else {
            logger.debug("{} creating using an existing path [{}]", shardId, path);
        }
        if (shards.containsKey(shardId.id())) {
            throw new IllegalStateException(shardId + " already exists");
        }
        logger.debug("creating shard_id {}", shardId);
        // if we are on a shared FS we only own the shard (ie. we can safely delete it) if we are the primary.
        final Engine.Warmer engineWarmer = (reader) -> {
            IndexShard shard = getShardOrNull(shardId.getId());
            if (shard != null) {
                warmer.warm(reader, shard, IndexService.this.indexSettings);
            }
        };
        Directory directory = directoryFactory.newDirectory(this.indexSettings, path);
        store = new Store(shardId, this.indexSettings, directory, lock, new StoreCloseListener(shardId, () -> eventListener.onStoreClosed(shardId)));
        eventListener.onStoreCreated(shardId);
        indexShard = new IndexShard(routing, this.indexSettings, path, store, indexSortSupplier, indexCache, mapperService, similarityService, engineFactory, engineConfigFactory, eventListener, readerWrapper, threadPool, bigArrays, engineWarmer, searchOperationListeners, indexingOperationListeners, () -> globalCheckpointSyncer.accept(shardId), retentionLeaseSyncer, circuitBreakerService);
        eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created");
        eventListener.afterIndexShardCreated(indexShard);
        shards = newMapBuilder(shards).put(shardId.id(), indexShard).immutableMap();
        success = true;
        return indexShard;
    } catch (ShardLockObtainFailedException e) {
        throw new IOException("failed to obtain in-memory shard lock", e);
    } finally {
        if (success == false) {
            if (lock != null) {
                IOUtils.closeWhileHandlingException(lock);
            }
            closeShard("initialization failed", shardId, indexShard, store, eventListener);
        }
    }
}