Examples with DummyShardLock org.opensearch.test.DummyShardLock used on opensource projects

Example 16 with DummyShardLock

use of org.opensearch.test.DummyShardLock in project OpenSearch by opensearch-project.

the class StoreTests method testCanOpenIndex.

public void testCanOpenIndex() throws IOException {
    final ShardId shardId = new ShardId("index", "_na_", 1);
    IndexWriterConfig iwc = newIndexWriterConfig();
    Path tempDir = createTempDir();
    final BaseDirectoryWrapper dir = newFSDirectory(tempDir);
    assertFalse(StoreUtils.canOpenIndex(logger, tempDir, shardId, (id, l, d) -> new DummyShardLock(id)));
    IndexWriter writer = new IndexWriter(dir, iwc);
    Document doc = new Document();
    doc.add(new StringField("id", "1", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    writer.addDocument(doc);
    writer.commit();
    writer.close();
    assertTrue(StoreUtils.canOpenIndex(logger, tempDir, shardId, (id, l, d) -> new DummyShardLock(id)));
    Store store = new Store(shardId, INDEX_SETTINGS, dir, new DummyShardLock(shardId));
    store.markStoreCorrupted(new CorruptIndexException("foo", "bar"));
    assertFalse(StoreUtils.canOpenIndex(logger, tempDir, shardId, (id, l, d) -> new DummyShardLock(id)));
    store.close();
}

Example 17 with DummyShardLock

use of org.opensearch.test.DummyShardLock in project OpenSearch by opensearch-project.

the class StoreTests method testHistoryUUIDCanBeForced.

public void testHistoryUUIDCanBeForced() throws IOException {
    final ShardId shardId = new ShardId("index", "_na_", 1);
    try (Store store = new Store(shardId, INDEX_SETTINGS, StoreTests.newDirectory(random()), new DummyShardLock(shardId))) {
        store.createEmpty(Version.LATEST);
        SegmentInfos segmentInfos = Lucene.readSegmentInfos(store.directory());
        assertThat(segmentInfos.getUserData(), hasKey(Engine.HISTORY_UUID_KEY));
        final String oldHistoryUUID = segmentInfos.getUserData().get(Engine.HISTORY_UUID_KEY);
        store.bootstrapNewHistory();
        segmentInfos = Lucene.readSegmentInfos(store.directory());
        assertThat(segmentInfos.getUserData(), hasKey(Engine.HISTORY_UUID_KEY));
        assertThat(segmentInfos.getUserData().get(Engine.HISTORY_UUID_KEY), not(equalTo(oldHistoryUUID)));
    }
}

Example 18 with DummyShardLock

use of org.opensearch.test.DummyShardLock in project OpenSearch by opensearch-project.

the class StoreTests method testCorruptionMarkerVersionCheck.

public void testCorruptionMarkerVersionCheck() throws IOException {
    final ShardId shardId = new ShardId("index", "_na_", 1);
    // I use ram dir to prevent that virusscanner being a PITA
    final Directory dir = new ByteBuffersDirectory();
    try (Store store = new Store(shardId, INDEX_SETTINGS, dir, new DummyShardLock(shardId))) {
        final String corruptionMarkerName = Store.CORRUPTED_MARKER_NAME_PREFIX + UUIDs.randomBase64UUID();
        try (IndexOutput output = dir.createOutput(corruptionMarkerName, IOContext.DEFAULT)) {
            CodecUtil.writeHeader(output, Store.CODEC, Store.CORRUPTED_MARKER_CODEC_VERSION + randomFrom(1, 2, -1, -2, -3));
        // we only need the header to trigger the exception
        }
        final IOException ioException = expectThrows(IOException.class, store::failIfCorrupted);
        assertThat(ioException, anyOf(instanceOf(IndexFormatTooOldException.class), instanceOf(IndexFormatTooNewException.class)));
        assertThat(ioException.getMessage(), containsString(corruptionMarkerName));
    }
}

Example 19 with DummyShardLock

use of org.opensearch.test.DummyShardLock in project OpenSearch by opensearch-project.

the class StoreTests method testEnsureIndexHasHistoryUUID.

public void testEnsureIndexHasHistoryUUID() throws IOException {
    final ShardId shardId = new ShardId("index", "_na_", 1);
    try (Store store = new Store(shardId, INDEX_SETTINGS, StoreTests.newDirectory(random()), new DummyShardLock(shardId))) {
        store.createEmpty(Version.LATEST);
        // remove the history uuid
        IndexWriterConfig iwc = new IndexWriterConfig(null).setCommitOnClose(false).setMergePolicy(NoMergePolicy.INSTANCE).setOpenMode(IndexWriterConfig.OpenMode.APPEND);
        try (IndexWriter writer = new IndexWriter(store.directory(), iwc)) {
            Map<String, String> newCommitData = new HashMap<>();
            for (Map.Entry<String, String> entry : writer.getLiveCommitData()) {
                if (entry.getKey().equals(Engine.HISTORY_UUID_KEY) == false) {
                    newCommitData.put(entry.getKey(), entry.getValue());
                }
            }
            writer.setLiveCommitData(newCommitData.entrySet());
            writer.commit();
        }
        store.ensureIndexHasHistoryUUID();
        SegmentInfos segmentInfos = Lucene.readSegmentInfos(store.directory());
        assertThat(segmentInfos.getUserData(), hasKey(Engine.HISTORY_UUID_KEY));
    }
}

Example 20 with DummyShardLock

use of org.opensearch.test.DummyShardLock in project OpenSearch by opensearch-project.

the class FsRepositoryTests method testSnapshotAndRestore.

public void testSnapshotAndRestore() throws IOException, InterruptedException {
    ThreadPool threadPool = new TestThreadPool(getClass().getSimpleName());
    try (Directory directory = newDirectory()) {
        Path repo = createTempDir();
        Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath()).put(Environment.PATH_REPO_SETTING.getKey(), repo.toAbsolutePath()).putList(Environment.PATH_DATA_SETTING.getKey(), tmpPaths()).put("location", repo).put("compress", randomBoolean()).put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES).build();
        int numDocs = indexDocs(directory);
        RepositoryMetadata metadata = new RepositoryMetadata("test", "fs", settings);
        FsRepository repository = new FsRepository(metadata, new Environment(settings, null), NamedXContentRegistry.EMPTY, BlobStoreTestUtil.mockClusterService(), new RecoverySettings(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)));
        repository.start();
        final Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_INDEX_UUID, "myindexUUID").build();
        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("myindex", indexSettings);
        ShardId shardId = new ShardId(idxSettings.getIndex(), 1);
        Store store = new Store(shardId, idxSettings, directory, new DummyShardLock(shardId));
        SnapshotId snapshotId = new SnapshotId("test", "test");
        IndexId indexId = new IndexId(idxSettings.getIndex().getName(), idxSettings.getUUID());
        IndexCommit indexCommit = Lucene.getIndexCommit(Lucene.readSegmentInfos(store.directory()), store.directory());
        final PlainActionFuture<String> future1 = PlainActionFuture.newFuture();
        runGeneric(threadPool, () -> {
            IndexShardSnapshotStatus snapshotStatus = IndexShardSnapshotStatus.newInitializing(null);
            repository.snapshotShard(store, null, snapshotId, indexId, indexCommit, null, snapshotStatus, Version.CURRENT, Collections.emptyMap(), future1);
            future1.actionGet();
            IndexShardSnapshotStatus.Copy copy = snapshotStatus.asCopy();
            assertEquals(copy.getTotalFileCount(), copy.getIncrementalFileCount());
        });
        final String shardGeneration = future1.actionGet();
        Lucene.cleanLuceneIndex(directory);
        expectThrows(org.apache.lucene.index.IndexNotFoundException.class, () -> Lucene.readSegmentInfos(directory));
        DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
        ShardRouting routing = ShardRouting.newUnassigned(shardId, true, new RecoverySource.SnapshotRecoverySource("test", new Snapshot("foo", snapshotId), Version.CURRENT, indexId), new UnassignedInfo(UnassignedInfo.Reason.EXISTING_INDEX_RESTORED, ""));
        routing = ShardRoutingHelper.initialize(routing, localNode.getId(), 0);
        RecoveryState state = new RecoveryState(routing, localNode, null);
        final PlainActionFuture<Void> futureA = PlainActionFuture.newFuture();
        runGeneric(threadPool, () -> repository.restoreShard(store, snapshotId, indexId, shardId, state, futureA));
        futureA.actionGet();
        assertTrue(state.getIndex().recoveredBytes() > 0);
        assertEquals(0, state.getIndex().reusedFileCount());
        assertEquals(indexCommit.getFileNames().size(), state.getIndex().recoveredFileCount());
        assertEquals(numDocs, Lucene.readSegmentInfos(directory).totalMaxDoc());
        deleteRandomDoc(store.directory());
        SnapshotId incSnapshotId = new SnapshotId("test1", "test1");
        IndexCommit incIndexCommit = Lucene.getIndexCommit(Lucene.readSegmentInfos(store.directory()), store.directory());
        Collection<String> commitFileNames = incIndexCommit.getFileNames();
        final PlainActionFuture<String> future2 = PlainActionFuture.newFuture();
        runGeneric(threadPool, () -> {
            IndexShardSnapshotStatus snapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration);
            repository.snapshotShard(store, null, incSnapshotId, indexId, incIndexCommit, null, snapshotStatus, Version.CURRENT, Collections.emptyMap(), future2);
            future2.actionGet();
            IndexShardSnapshotStatus.Copy copy = snapshotStatus.asCopy();
            assertEquals(2, copy.getIncrementalFileCount());
            assertEquals(commitFileNames.size(), copy.getTotalFileCount());
        });
        // roll back to the first snap and then incrementally restore
        RecoveryState firstState = new RecoveryState(routing, localNode, null);
        final PlainActionFuture<Void> futureB = PlainActionFuture.newFuture();
        runGeneric(threadPool, () -> repository.restoreShard(store, snapshotId, indexId, shardId, firstState, futureB));
        futureB.actionGet();
        assertEquals("should reuse everything except of .liv and .si", commitFileNames.size() - 2, firstState.getIndex().reusedFileCount());
        RecoveryState secondState = new RecoveryState(routing, localNode, null);
        final PlainActionFuture<Void> futureC = PlainActionFuture.newFuture();
        runGeneric(threadPool, () -> repository.restoreShard(store, incSnapshotId, indexId, shardId, secondState, futureC));
        futureC.actionGet();
        assertEquals(secondState.getIndex().reusedFileCount(), commitFileNames.size() - 2);
        assertEquals(secondState.getIndex().recoveredFileCount(), 2);
        List<RecoveryState.FileDetail> recoveredFiles = secondState.getIndex().fileDetails().stream().filter(f -> f.reused() == false).collect(Collectors.toList());
        Collections.sort(recoveredFiles, Comparator.comparing(RecoveryState.FileDetail::name));
        assertTrue(recoveredFiles.get(0).name(), recoveredFiles.get(0).name().endsWith(".liv"));
        assertTrue(recoveredFiles.get(1).name(), recoveredFiles.get(1).name().endsWith("segments_" + incIndexCommit.getGeneration()));
    } finally {
        terminate(threadPool);
    }
}