Examples with ShardPath org.elasticsearch.index.shard.ShardPath used on opensource projects

Example 6 with ShardPath

use of org.elasticsearch.index.shard.ShardPath in project elasticsearch by elastic.

the class NodeEnvironment method assertCanWrite.

/**
     * This is a best effort to ensure that we actually have write permissions to write in all our data directories.
     * This prevents disasters if nodes are started under the wrong username etc.
     */
private void assertCanWrite() throws IOException {
    for (Path path : nodeDataPaths()) {
        // check node-paths are writable
        tryWriteTempFile(path);
    }
    for (String indexFolderName : this.availableIndexFolders()) {
        for (Path indexPath : this.resolveIndexFolder(indexFolderName)) {
            // check index paths are writable
            Path indexStatePath = indexPath.resolve(MetaDataStateFormat.STATE_DIR_NAME);
            tryWriteTempFile(indexStatePath);
            tryWriteTempFile(indexPath);
            try (DirectoryStream<Path> stream = Files.newDirectoryStream(indexPath)) {
                for (Path shardPath : stream) {
                    String fileName = shardPath.getFileName().toString();
                    if (Files.isDirectory(shardPath) && fileName.chars().allMatch(Character::isDigit)) {
                        Path indexDir = shardPath.resolve(ShardPath.INDEX_FOLDER_NAME);
                        Path statePath = shardPath.resolve(MetaDataStateFormat.STATE_DIR_NAME);
                        Path translogDir = shardPath.resolve(ShardPath.TRANSLOG_FOLDER_NAME);
                        tryWriteTempFile(indexDir);
                        tryWriteTempFile(translogDir);
                        tryWriteTempFile(statePath);
                        tryWriteTempFile(shardPath);
                    }
                }
            }
        }
    }
}

Example 7 with ShardPath

use of org.elasticsearch.index.shard.ShardPath in project elasticsearch by elastic.

the class DiskUsageTests method testFillShardLevelInfo.

public void testFillShardLevelInfo() {
    final Index index = new Index("test", "0xdeadbeef");
    ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), false, PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    test_0 = ShardRoutingHelper.initialize(test_0, "node1");
    test_0 = ShardRoutingHelper.moveToStarted(test_0);
    Path test0Path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve("0");
    CommonStats commonStats0 = new CommonStats();
    commonStats0.store = new StoreStats(100);
    ShardRouting test_1 = ShardRouting.newUnassigned(new ShardId(index, 1), false, PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    test_1 = ShardRoutingHelper.initialize(test_1, "node2");
    test_1 = ShardRoutingHelper.moveToStarted(test_1);
    Path test1Path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve("1");
    CommonStats commonStats1 = new CommonStats();
    commonStats1.store = new StoreStats(1000);
    ShardStats[] stats = new ShardStats[] { new ShardStats(test_0, new ShardPath(false, test0Path, test0Path, test_0.shardId()), commonStats0, null, null), new ShardStats(test_1, new ShardPath(false, test1Path, test1Path, test_1.shardId()), commonStats1, null, null) };
    ImmutableOpenMap.Builder<String, Long> shardSizes = ImmutableOpenMap.builder();
    ImmutableOpenMap.Builder<ShardRouting, String> routingToPath = ImmutableOpenMap.builder();
    ClusterState state = ClusterState.builder(new ClusterName("blarg")).version(0).build();
    InternalClusterInfoService.buildShardLevelInfo(logger, stats, shardSizes, routingToPath, state);
    assertEquals(2, shardSizes.size());
    assertTrue(shardSizes.containsKey(ClusterInfo.shardIdentifierFromRouting(test_0)));
    assertTrue(shardSizes.containsKey(ClusterInfo.shardIdentifierFromRouting(test_1)));
    assertEquals(100L, shardSizes.get(ClusterInfo.shardIdentifierFromRouting(test_0)).longValue());
    assertEquals(1000L, shardSizes.get(ClusterInfo.shardIdentifierFromRouting(test_1)).longValue());
    assertEquals(2, routingToPath.size());
    assertTrue(routingToPath.containsKey(test_0));
    assertTrue(routingToPath.containsKey(test_1));
    assertEquals(test0Path.getParent().getParent().getParent().toAbsolutePath().toString(), routingToPath.get(test_0));
    assertEquals(test1Path.getParent().getParent().getParent().toAbsolutePath().toString(), routingToPath.get(test_1));
}

Example 8 with ShardPath

use of org.elasticsearch.index.shard.ShardPath in project crate by crate.

the class BlobShard method getBlobDataDir.

private Path getBlobDataDir(Settings indexSettings, ShardPath shardPath, @Nullable Path globalBlobPath) {
    String tableBlobPath = indexSettings.get(BlobIndicesService.SETTING_INDEX_BLOBS_PATH);
    Path blobPath;
    if (tableBlobPath == null) {
        if (globalBlobPath == null) {
            return shardPath.getDataPath().resolve(BLOBS_SUB_PATH);
        }
        assert BlobIndicesService.ensureExistsAndWritable(globalBlobPath) : "global blob path must exist and be writable";
        blobPath = globalBlobPath;
    } else {
        blobPath = PathUtils.get(tableBlobPath);
    }
    // rootDataPath is /<path.data>/<clusterName>/nodes/<nodeLock>/
    Path rootDataPath = shardPath.getRootDataPath();
    Path clusterDataDir = rootDataPath.getParent().getParent();
    Path pathToShard = clusterDataDir.relativize(shardPath.getShardStatePath());
    // this generates <blobs.path>/nodes/<nodeLock>/<path-to-shard>/blobs
    return blobPath.resolve(pathToShard).resolve(BLOBS_SUB_PATH);
}

Example 9 with ShardPath

use of org.elasticsearch.index.shard.ShardPath in project elasticsearch by elastic.

the class IndexService method createShard.

public synchronized IndexShard createShard(ShardRouting routing) throws IOException {
    final boolean primary = routing.primary();
    /*
         * TODO: we execute this in parallel but it's a synced method. Yet, we might
         * be able to serialize the execution via the cluster state in the future. for now we just
         * keep it synced.
         */
    if (closed.get()) {
        throw new IllegalStateException("Can't create shard " + routing.shardId() + ", closed");
    }
    final Settings indexSettings = this.indexSettings.getSettings();
    final ShardId shardId = routing.shardId();
    boolean success = false;
    Store store = null;
    IndexShard indexShard = null;
    ShardLock lock = null;
    try {
        lock = nodeEnv.shardLock(shardId, TimeUnit.SECONDS.toMillis(5));
        eventListener.beforeIndexShardCreated(shardId, indexSettings);
        ShardPath path;
        try {
            path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings);
        } catch (IllegalStateException ex) {
            logger.warn("{} failed to load shard path, trying to remove leftover", shardId);
            try {
                ShardPath.deleteLeftoverShardDirectory(logger, nodeEnv, lock, this.indexSettings);
                path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings);
            } catch (Exception inner) {
                ex.addSuppressed(inner);
                throw ex;
            }
        }
        if (path == null) {
            // TODO: we should, instead, hold a "bytes reserved" of how large we anticipate this shard will be, e.g. for a shard
            // that's being relocated/replicated we know how large it will become once it's done copying:
            // Count up how many shards are currently on each data path:
            Map<Path, Integer> dataPathToShardCount = new HashMap<>();
            for (IndexShard shard : this) {
                Path dataPath = shard.shardPath().getRootStatePath();
                Integer curCount = dataPathToShardCount.get(dataPath);
                if (curCount == null) {
                    curCount = 0;
                }
                dataPathToShardCount.put(dataPath, curCount + 1);
            }
            path = ShardPath.selectNewPathForShard(nodeEnv, shardId, this.indexSettings, routing.getExpectedShardSize() == ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE ? getAvgShardSizeInBytes() : routing.getExpectedShardSize(), dataPathToShardCount);
            logger.debug("{} creating using a new path [{}]", shardId, path);
        } else {
            logger.debug("{} creating using an existing path [{}]", shardId, path);
        }
        if (shards.containsKey(shardId.id())) {
            throw new IllegalStateException(shardId + " already exists");
        }
        logger.debug("creating shard_id {}", shardId);
        // if we are on a shared FS we only own the shard (ie. we can safely delete it) if we are the primary.
        final boolean canDeleteShardContent = this.indexSettings.isOnSharedFilesystem() == false || (primary && this.indexSettings.isOnSharedFilesystem());
        final Engine.Warmer engineWarmer = (searcher) -> {
            IndexShard shard = getShardOrNull(shardId.getId());
            if (shard != null) {
                warmer.warm(searcher, shard, IndexService.this.indexSettings);
            }
        };
        store = new Store(shardId, this.indexSettings, indexStore.newDirectoryService(path), lock, new StoreCloseListener(shardId, canDeleteShardContent, () -> eventListener.onStoreClosed(shardId)));
        if (useShadowEngine(primary, this.indexSettings)) {
            indexShard = new ShadowIndexShard(routing, this.indexSettings, path, store, indexCache, mapperService, similarityService, indexFieldData, engineFactory, eventListener, searcherWrapper, threadPool, bigArrays, engineWarmer, searchOperationListeners);
        // no indexing listeners - shadow  engines don't index
        } else {
            indexShard = new IndexShard(routing, this.indexSettings, path, store, indexCache, mapperService, similarityService, indexFieldData, engineFactory, eventListener, searcherWrapper, threadPool, bigArrays, engineWarmer, () -> globalCheckpointSyncer.accept(shardId), searchOperationListeners, indexingOperationListeners);
        }
        eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created");
        eventListener.afterIndexShardCreated(indexShard);
        shards = newMapBuilder(shards).put(shardId.id(), indexShard).immutableMap();
        success = true;
        return indexShard;
    } catch (ShardLockObtainFailedException e) {
        throw new IOException("failed to obtain in-memory shard lock", e);
    } finally {
        if (success == false) {
            if (lock != null) {
                IOUtils.closeWhileHandlingException(lock);
            }
            closeShard("initialization failed", shardId, indexShard, store, eventListener);
        }
    }
}

Example 10 with ShardPath

use of org.elasticsearch.index.shard.ShardPath in project elasticsearch by elastic.

the class FsDirectoryServiceTests method testHasSleepWrapperOnSharedFS.

public void testHasSleepWrapperOnSharedFS() throws IOException {
    Settings build = randomBoolean() ? Settings.builder().put(IndexMetaData.SETTING_SHARED_FILESYSTEM, true).build() : Settings.builder().put(IndexMetaData.SETTING_SHADOW_REPLICAS, true).build();
    ;
    IndexSettings settings = IndexSettingsModule.newIndexSettings("foo", build);
    IndexStore store = new IndexStore(settings);
    Path tempDir = createTempDir().resolve(settings.getUUID()).resolve("0");
    Files.createDirectories(tempDir);
    ShardPath path = new ShardPath(false, tempDir, tempDir, new ShardId(settings.getIndex(), 0));
    FsDirectoryService fsDirectoryService = new FsDirectoryService(settings, store, path);
    Directory directory = fsDirectoryService.newDirectory();
    assertTrue(directory.getClass().toString(), directory instanceof SleepingLockWrapper);
}