Examples with ShardPath org.opensearch.index.shard.ShardPath used on opensource projects

Example 1 with ShardPath

use of org.opensearch.index.shard.ShardPath in project OpenSearch by opensearch-project.

the class NodeEnvironment method assertCanWrite.

/**
 * This is a best effort to ensure that we actually have write permissions to write in all our data directories.
 * This prevents disasters if nodes are started under the wrong username etc.
 */
private void assertCanWrite() throws IOException {
    for (Path path : nodeDataPaths()) {
        // check node-paths are writable
        tryWriteTempFile(path);
    }
    for (String indexFolderName : this.availableIndexFolders()) {
        for (Path indexPath : this.resolveIndexFolder(indexFolderName)) {
            // check index paths are writable
            Path indexStatePath = indexPath.resolve(MetadataStateFormat.STATE_DIR_NAME);
            tryWriteTempFile(indexStatePath);
            tryWriteTempFile(indexPath);
            try (DirectoryStream<Path> stream = Files.newDirectoryStream(indexPath)) {
                for (Path shardPath : stream) {
                    String fileName = shardPath.getFileName().toString();
                    if (Files.isDirectory(shardPath) && fileName.chars().allMatch(Character::isDigit)) {
                        Path indexDir = shardPath.resolve(ShardPath.INDEX_FOLDER_NAME);
                        Path statePath = shardPath.resolve(MetadataStateFormat.STATE_DIR_NAME);
                        Path translogDir = shardPath.resolve(ShardPath.TRANSLOG_FOLDER_NAME);
                        tryWriteTempFile(indexDir);
                        tryWriteTempFile(translogDir);
                        tryWriteTempFile(statePath);
                        tryWriteTempFile(shardPath);
                    }
                }
            }
        }
    }
}

Example 2 with ShardPath

use of org.opensearch.index.shard.ShardPath in project OpenSearch by opensearch-project.

the class FsDirectoryFactoryTests method newDirectory.

private Directory newDirectory(Settings settings) throws IOException {
    IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("foo", settings);
    Path tempDir = createTempDir().resolve(idxSettings.getUUID()).resolve("0");
    Files.createDirectories(tempDir);
    ShardPath path = new ShardPath(false, tempDir, tempDir, new ShardId(idxSettings.getIndex(), 0));
    return new FsDirectoryFactory().newDirectory(idxSettings, path);
}

Example 3 with ShardPath

use of org.opensearch.index.shard.ShardPath in project OpenSearch by opensearch-project.

the class IndicesServiceTests method testPendingTasks.

public void testPendingTasks() throws Exception {
    final IndexService indexService = createIndex("test");
    final Index index = indexService.index();
    final IndexSettings indexSettings = indexService.getIndexSettings();
    final IndexShard indexShard = indexService.getShardOrNull(0);
    assertNotNull(indexShard);
    assertTrue(indexShard.routingEntry().started());
    final ShardPath shardPath = indexShard.shardPath();
    assertEquals(ShardPath.loadShardPath(logger, getNodeEnvironment(), indexShard.shardId(), indexSettings.customDataPath()), shardPath);
    final IndicesService indicesService = getIndicesService();
    expectThrows(ShardLockObtainFailedException.class, () -> indicesService.processPendingDeletes(index, indexSettings, TimeValue.timeValueMillis(0)));
    assertTrue(shardPath.exists());
    int numPending = 1;
    if (randomBoolean()) {
        indicesService.addPendingDelete(indexShard.shardId(), indexSettings);
    } else {
        if (randomBoolean()) {
            numPending++;
            indicesService.addPendingDelete(indexShard.shardId(), indexSettings);
        }
        indicesService.addPendingDelete(index, indexSettings);
    }
    assertAcked(client().admin().indices().prepareClose("test"));
    assertTrue(shardPath.exists());
    ensureGreen("test");
    assertEquals(indicesService.numPendingDeletes(index), numPending);
    assertTrue(indicesService.hasUncompletedPendingDeletes());
    expectThrows(ShardLockObtainFailedException.class, () -> indicesService.processPendingDeletes(index, indexSettings, TimeValue.timeValueMillis(0)));
    assertEquals(indicesService.numPendingDeletes(index), numPending);
    assertTrue(indicesService.hasUncompletedPendingDeletes());
    final boolean hasBogus = randomBoolean();
    if (hasBogus) {
        indicesService.addPendingDelete(new ShardId(index, 0), indexSettings);
        indicesService.addPendingDelete(new ShardId(index, 1), indexSettings);
        indicesService.addPendingDelete(new ShardId("bogus", "_na_", 1), indexSettings);
        assertEquals(indicesService.numPendingDeletes(index), numPending + 2);
        assertTrue(indicesService.hasUncompletedPendingDeletes());
    }
    assertAcked(client().admin().indices().prepareDelete("test"));
    assertBusy(() -> {
        try {
            indicesService.processPendingDeletes(index, indexSettings, TimeValue.timeValueMillis(0));
            assertEquals(indicesService.numPendingDeletes(index), 0);
        } catch (final Exception e) {
            fail(e.getMessage());
        }
    });
    assertBusy(() -> {
        // "bogus" index has not been removed
        assertThat(indicesService.hasUncompletedPendingDeletes(), equalTo(hasBogus));
        assertFalse(shardPath.exists());
    });
}

Example 4 with ShardPath

use of org.opensearch.index.shard.ShardPath in project OpenSearch by opensearch-project.

the class IndicesStatsResponseTests method testGetIndices.

public void testGetIndices() {
    List<ShardStats> shards = new ArrayList<>();
    int noOfIndexes = randomIntBetween(2, 5);
    List<String> expectedIndexes = new ArrayList<>();
    Map<String, AtomicLong> expectedIndexToPrimaryShardsCount = new HashMap<>();
    Map<String, AtomicLong> expectedIndexToTotalShardsCount = new HashMap<>();
    for (int indCnt = 0; indCnt < noOfIndexes; indCnt++) {
        Index index = createIndex(randomAlphaOfLength(9));
        expectedIndexes.add(index.getName());
        int numShards = randomIntBetween(1, 5);
        for (int shardId = 0; shardId < numShards; shardId++) {
            ShardId shId = new ShardId(index, shardId);
            Path path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve(String.valueOf(shardId));
            ShardPath shardPath = new ShardPath(false, path, path, shId);
            ShardRouting routing = createShardRouting(index, shId, (shardId == 0));
            shards.add(new ShardStats(routing, shardPath, null, null, null, null));
            AtomicLong primaryShardsCounter = expectedIndexToPrimaryShardsCount.computeIfAbsent(index.getName(), k -> new AtomicLong(0L));
            if (routing.primary()) {
                primaryShardsCounter.incrementAndGet();
            }
            AtomicLong shardsCounter = expectedIndexToTotalShardsCount.computeIfAbsent(index.getName(), k -> new AtomicLong(0L));
            shardsCounter.incrementAndGet();
        }
    }
    final IndicesStatsResponse indicesStatsResponse = new IndicesStatsResponse(shards.toArray(new ShardStats[shards.size()]), 0, 0, 0, null);
    Map<String, IndexStats> indexStats = indicesStatsResponse.getIndices();
    assertThat(indexStats.size(), is(noOfIndexes));
    assertThat(indexStats.keySet(), containsInAnyOrder(expectedIndexes.toArray(new String[0])));
    for (String index : indexStats.keySet()) {
        IndexStats stat = indexStats.get(index);
        ShardStats[] shardStats = stat.getShards();
        long primaryCount = 0L;
        long totalCount = shardStats.length;
        for (ShardStats shardStat : shardStats) {
            if (shardStat.getShardRouting().primary()) {
                primaryCount++;
            }
        }
        assertThat(primaryCount, is(expectedIndexToPrimaryShardsCount.get(index).get()));
        assertThat(totalCount, is(expectedIndexToTotalShardsCount.get(index).get()));
    }
}

Example 5 with ShardPath

use of org.opensearch.index.shard.ShardPath in project OpenSearch by opensearch-project.

the class TruncateTranslogAction method isTranslogClean.

private boolean isTranslogClean(ShardPath shardPath, ClusterState clusterState, String translogUUID) throws IOException {
    // perform clean check of translog instead of corrupted marker file
    try {
        final Path translogPath = shardPath.resolveTranslog();
        final long translogGlobalCheckpoint = Translog.readGlobalCheckpoint(translogPath, translogUUID);
        final IndexMetadata indexMetadata = clusterState.metadata().getIndexSafe(shardPath.getShardId().getIndex());
        final IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY);
        final TranslogConfig translogConfig = new TranslogConfig(shardPath.getShardId(), translogPath, indexSettings, BigArrays.NON_RECYCLING_INSTANCE);
        long primaryTerm = indexSettings.getIndexMetadata().primaryTerm(shardPath.getShardId().id());
        // We open translog to check for corruption, do not clean anything.
        final TranslogDeletionPolicy retainAllTranslogPolicy = new DefaultTranslogDeletionPolicy(Long.MAX_VALUE, Long.MAX_VALUE, Integer.MAX_VALUE) {

            @Override
            public long minTranslogGenRequired(List<TranslogReader> readers, TranslogWriter writer) {
                long minGen = writer.generation;
                for (TranslogReader reader : readers) {
                    minGen = Math.min(reader.generation, minGen);
                }
                return minGen;
            }
        };
        try (Translog translog = new Translog(translogConfig, translogUUID, retainAllTranslogPolicy, () -> translogGlobalCheckpoint, () -> primaryTerm, seqNo -> {
        });
            Translog.Snapshot snapshot = translog.newSnapshot(0, Long.MAX_VALUE)) {
            // noinspection StatementWithEmptyBody we are just checking that we can iterate through the whole snapshot
            while (snapshot.next() != null) {
            }
        }
        return true;
    } catch (TranslogCorruptedException e) {
        return false;
    }
}