Examples with ShardPath org.elasticsearch.index.shard.ShardPath used on opensource projects

Example 11 with ShardPath

use of org.elasticsearch.index.shard.ShardPath in project elasticsearch by elastic.

the class FsDirectoryServiceTests method doTestPreload.

private void doTestPreload(String... preload) throws IOException {
    Settings build = Settings.builder().put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), "mmapfs").putArray(IndexModule.INDEX_STORE_PRE_LOAD_SETTING.getKey(), preload).build();
    IndexSettings settings = IndexSettingsModule.newIndexSettings("foo", build);
    IndexStore store = new IndexStore(settings);
    Path tempDir = createTempDir().resolve(settings.getUUID()).resolve("0");
    Files.createDirectories(tempDir);
    ShardPath path = new ShardPath(false, tempDir, tempDir, new ShardId(settings.getIndex(), 0));
    FsDirectoryService fsDirectoryService = new FsDirectoryService(settings, store, path);
    Directory directory = fsDirectoryService.newDirectory();
    assertFalse(directory instanceof SleepingLockWrapper);
    if (preload.length == 0) {
        assertTrue(directory.toString(), directory instanceof MMapDirectory);
        assertFalse(((MMapDirectory) directory).getPreload());
    } else if (Arrays.asList(preload).contains("*")) {
        assertTrue(directory.toString(), directory instanceof MMapDirectory);
        assertTrue(((MMapDirectory) directory).getPreload());
    } else {
        assertTrue(directory.toString(), directory instanceof FileSwitchDirectory);
        FileSwitchDirectory fsd = (FileSwitchDirectory) directory;
        assertTrue(fsd.getPrimaryDir() instanceof MMapDirectory);
        assertTrue(((MMapDirectory) fsd.getPrimaryDir()).getPreload());
        assertTrue(fsd.getSecondaryDir() instanceof MMapDirectory);
        assertFalse(((MMapDirectory) fsd.getSecondaryDir()).getPreload());
    }
}

Example 12 with ShardPath

use of org.elasticsearch.index.shard.ShardPath in project elasticsearch by elastic.

the class FsDirectoryServiceTests method testHasNoSleepWrapperOnNormalFS.

public void testHasNoSleepWrapperOnNormalFS() throws IOException {
    Settings build = Settings.builder().put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), "simplefs").build();
    IndexSettings settings = IndexSettingsModule.newIndexSettings("foo", build);
    IndexStore store = new IndexStore(settings);
    Path tempDir = createTempDir().resolve(settings.getUUID()).resolve("0");
    Files.createDirectories(tempDir);
    ShardPath path = new ShardPath(false, tempDir, tempDir, new ShardId(settings.getIndex(), 0));
    FsDirectoryService fsDirectoryService = new FsDirectoryService(settings, store, path);
    Directory directory = fsDirectoryService.newDirectory();
    assertFalse(directory instanceof SleepingLockWrapper);
    assertTrue(directory instanceof SimpleFSDirectory);
}

Example 13 with ShardPath

use of org.elasticsearch.index.shard.ShardPath in project elasticsearch by elastic.

the class IndexFolderUpgraderTests method checkIndex.

private void checkIndex(NodeEnvironment nodeEnv, IndexSettings indexSettings, int numIdxFiles, int numTranslogFiles) throws IOException {
    final Index index = indexSettings.getIndex();
    // ensure index state can be loaded
    IndexMetaData loadLatestState = IndexMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, nodeEnv.indexPaths(index));
    assertNotNull(loadLatestState);
    assertEquals(loadLatestState.getIndex(), index);
    for (int shardId = 0; shardId < indexSettings.getNumberOfShards(); shardId++) {
        // ensure shard path can be loaded
        ShardPath targetShardPath = ShardPath.loadShardPath(logger, nodeEnv, new ShardId(index, shardId), indexSettings);
        assertNotNull(targetShardPath);
        // ensure shard contents are copied over
        final Path translog = targetShardPath.resolveTranslog();
        final Path idx = targetShardPath.resolveIndex();
        // ensure index and translog files are copied over
        assertEquals(numTranslogFiles, FileSystemUtils.files(translog).length);
        assertEquals(numIdxFiles, FileSystemUtils.files(idx).length);
        Path[] files = FileSystemUtils.files(translog);
        final HashSet<Path> translogFiles = new HashSet<>(Arrays.asList(files));
        for (int i = 0; i < numTranslogFiles; i++) {
            final String name = Integer.toString(i);
            translogFiles.contains(translog.resolve(name + ".translog"));
            byte[] content = Files.readAllBytes(translog.resolve(name + ".translog"));
            assertEquals(name, new String(content, StandardCharsets.UTF_8));
        }
        Path[] indexFileList = FileSystemUtils.files(idx);
        final HashSet<Path> idxFiles = new HashSet<>(Arrays.asList(indexFileList));
        for (int i = 0; i < numIdxFiles; i++) {
            final String name = Integer.toString(i);
            idxFiles.contains(idx.resolve(name + ".tst"));
            byte[] content = Files.readAllBytes(idx.resolve(name + ".tst"));
            assertEquals(name, new String(content, StandardCharsets.UTF_8));
        }
    }
}

Example 14 with ShardPath

use of org.elasticsearch.index.shard.ShardPath in project elasticsearch by elastic.

the class IndexFolderUpgraderTests method testUpgradeRealIndex.

/**
     * Run upgrade on a real bwc index
     */
public void testUpgradeRealIndex() throws IOException, URISyntaxException {
    List<Path> indexes = new ArrayList<>();
    try (DirectoryStream<Path> stream = Files.newDirectoryStream(getBwcIndicesPath(), "index-*.zip")) {
        for (Path path : stream) {
            indexes.add(path);
        }
    }
    CollectionUtil.introSort(indexes, (o1, o2) -> o1.getFileName().compareTo(o2.getFileName()));
    final Path path = randomFrom(indexes);
    final String indexName = path.getFileName().toString().replace(".zip", "").toLowerCase(Locale.ROOT);
    try (NodeEnvironment nodeEnvironment = newNodeEnvironment()) {
        Path unzipDir = createTempDir();
        Path unzipDataDir = unzipDir.resolve("data");
        // decompress the index
        try (InputStream stream = Files.newInputStream(path)) {
            TestUtil.unzip(stream, unzipDir);
        }
        // check it is unique
        assertTrue(Files.exists(unzipDataDir));
        Path[] list = FileSystemUtils.files(unzipDataDir);
        if (list.length != 1) {
            throw new IllegalStateException("Backwards index must contain exactly one cluster but was " + list.length);
        }
        // the bwc scripts packs the indices under this path
        Path src = OldIndexUtils.getIndexDir(logger, indexName, path.getFileName().toString(), list[0]);
        assertTrue("[" + path + "] missing index dir: " + src.toString(), Files.exists(src));
        final Path indicesPath = randomFrom(nodeEnvironment.nodePaths()).indicesPath;
        logger.info("--> injecting index [{}] into [{}]", indexName, indicesPath);
        OldIndexUtils.copyIndex(logger, src, src.getFileName().toString(), indicesPath);
        IndexFolderUpgrader.upgradeIndicesIfNeeded(Settings.EMPTY, nodeEnvironment);
        // ensure old index folder is deleted
        Set<String> indexFolders = nodeEnvironment.availableIndexFolders();
        assertEquals(indexFolders.size(), 1);
        // ensure index metadata is moved
        IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, nodeEnvironment.resolveIndexFolder(indexFolders.iterator().next()));
        assertNotNull(indexMetaData);
        Index index = indexMetaData.getIndex();
        assertEquals(index.getName(), indexName);
        Set<ShardId> shardIds = nodeEnvironment.findAllShardIds(index);
        // ensure all shards are moved
        assertEquals(shardIds.size(), indexMetaData.getNumberOfShards());
        for (ShardId shardId : shardIds) {
            final ShardPath shardPath = ShardPath.loadShardPath(logger, nodeEnvironment, shardId, new IndexSettings(indexMetaData, Settings.EMPTY));
            final Path translog = shardPath.resolveTranslog();
            final Path idx = shardPath.resolveIndex();
            final Path state = shardPath.getShardStatePath().resolve(MetaDataStateFormat.STATE_DIR_NAME);
            assertTrue(shardPath.exists());
            assertTrue(Files.exists(translog));
            assertTrue(Files.exists(idx));
            assertTrue(Files.exists(state));
        }
    }
}

Example 15 with ShardPath

use of org.elasticsearch.index.shard.ShardPath in project elasticsearch by elastic.

the class DiskUsageTests method testFillShardsWithShadowIndices.

public void testFillShardsWithShadowIndices() {
    final Index index = new Index("non-shadow", "0xcafe0000");
    ShardRouting s0 = ShardRouting.newUnassigned(new ShardId(index, 0), false, PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    s0 = ShardRoutingHelper.initialize(s0, "node1");
    s0 = ShardRoutingHelper.moveToStarted(s0);
    Path i0Path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve("0");
    CommonStats commonStats0 = new CommonStats();
    commonStats0.store = new StoreStats(100);
    final Index index2 = new Index("shadow", "0xcafe0001");
    ShardRouting s1 = ShardRouting.newUnassigned(new ShardId(index2, 0), false, PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    s1 = ShardRoutingHelper.initialize(s1, "node2");
    s1 = ShardRoutingHelper.moveToStarted(s1);
    Path i1Path = createTempDir().resolve("indices").resolve(index2.getUUID()).resolve("0");
    CommonStats commonStats1 = new CommonStats();
    commonStats1.store = new StoreStats(1000);
    ShardStats[] stats = new ShardStats[] { new ShardStats(s0, new ShardPath(false, i0Path, i0Path, s0.shardId()), commonStats0, null, null), new ShardStats(s1, new ShardPath(false, i1Path, i1Path, s1.shardId()), commonStats1, null, null) };
    ImmutableOpenMap.Builder<String, Long> shardSizes = ImmutableOpenMap.builder();
    ImmutableOpenMap.Builder<ShardRouting, String> routingToPath = ImmutableOpenMap.builder();
    ClusterState state = ClusterState.builder(new ClusterName("blarg")).version(0).metaData(MetaData.builder().put(IndexMetaData.builder("non-shadow").settings(Settings.builder().put(IndexMetaData.SETTING_INDEX_UUID, "0xcafe0000").put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)).numberOfShards(1).numberOfReplicas(0)).put(IndexMetaData.builder("shadow").settings(Settings.builder().put(IndexMetaData.SETTING_INDEX_UUID, "0xcafe0001").put(IndexMetaData.SETTING_SHADOW_REPLICAS, true).put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))).build();
    logger.info("--> calling buildShardLevelInfo with state: {}", state);
    InternalClusterInfoService.buildShardLevelInfo(logger, stats, shardSizes, routingToPath, state);
    assertEquals(2, shardSizes.size());
    assertTrue(shardSizes.containsKey(ClusterInfo.shardIdentifierFromRouting(s0)));
    assertTrue(shardSizes.containsKey(ClusterInfo.shardIdentifierFromRouting(s1)));
    assertEquals(100L, shardSizes.get(ClusterInfo.shardIdentifierFromRouting(s0)).longValue());
    assertEquals(0L, shardSizes.get(ClusterInfo.shardIdentifierFromRouting(s1)).longValue());
}