Examples with FSDirectory org.apache.lucene.store.FSDirectory used on opensource projects

Example 36 with FSDirectory

use of org.apache.lucene.store.FSDirectory in project jackrabbit-oak by apache.

the class LucenePropertyIndexTest method createIndexCopier.

private IndexCopier createIndexCopier() {
    try {
        return new IndexCopier(executorService, temporaryFolder.getRoot()) {

            @Override
            public Directory wrapForRead(String indexPath, IndexDefinition definition, Directory remote, String dirName) throws IOException {
                Directory ret = super.wrapForRead(indexPath, definition, remote, dirName);
                corDir = getFSDirPath(ret);
                return ret;
            }

            @Override
            public Directory wrapForWrite(IndexDefinition definition, Directory remote, boolean reindexMode, String dirName) throws IOException {
                Directory ret = super.wrapForWrite(definition, remote, reindexMode, dirName);
                cowDir = getFSDirPath(ret);
                return ret;
            }

            private String getFSDirPath(Directory dir) {
                if (dir instanceof CopyOnReadDirectory) {
                    dir = ((CopyOnReadDirectory) dir).getLocal();
                }
                dir = unwrap(dir);
                if (dir instanceof FSDirectory) {
                    return ((FSDirectory) dir).getDirectory().getAbsolutePath();
                }
                return null;
            }

            private Directory unwrap(Directory dir) {
                if (dir instanceof FilterDirectory) {
                    return unwrap(((FilterDirectory) dir).getDelegate());
                }
                return dir;
            }
        };
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

Example 37 with FSDirectory

use of org.apache.lucene.store.FSDirectory in project lucene-solr by apache.

the class TestUtil method hasWindowsFS.

public static boolean hasWindowsFS(Directory dir) {
    dir = FilterDirectory.unwrap(dir);
    if (dir instanceof FSDirectory) {
        Path path = ((FSDirectory) dir).getDirectory();
        FileSystem fs = path.getFileSystem();
        while (fs instanceof FilterFileSystem) {
            FilterFileSystem ffs = (FilterFileSystem) fs;
            if (ffs.getParent() instanceof WindowsFS) {
                return true;
            }
            fs = ffs.getDelegate();
        }
    }
    return false;
}

Example 38 with FSDirectory

use of org.apache.lucene.store.FSDirectory in project lucene-solr by apache.

the class TestIndexWriter method testDeleteUnusedFiles.

public void testDeleteUnusedFiles() throws Exception {
    assumeFalse("test relies on exact filenames", Codec.getDefault() instanceof SimpleTextCodec);
    assumeWorkingMMapOnWindows();
    for (int iter = 0; iter < 2; iter++) {
        // relies on windows semantics
        Path path = createTempDir();
        FileSystem fs = new WindowsFS(path.getFileSystem()).getFileSystem(URI.create("file:///"));
        Path indexPath = new FilterPath(path, fs);
        // NOTE: on Unix, we cannot use MMapDir, because WindowsFS doesn't see/think it keeps file handles open.  Yet, on Windows, we MUST use
        // MMapDir because the windows OS will in fact prevent file deletion for us, and fails otherwise:
        FSDirectory dir;
        if (Constants.WINDOWS) {
            dir = new MMapDirectory(indexPath);
        } else {
            dir = new NIOFSDirectory(indexPath);
        }
        MergePolicy mergePolicy = newLogMergePolicy(true);
        // This test expects all of its segments to be in CFS
        mergePolicy.setNoCFSRatio(1.0);
        mergePolicy.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);
        IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(mergePolicy).setUseCompoundFile(true));
        Document doc = new Document();
        doc.add(newTextField("field", "go", Field.Store.NO));
        w.addDocument(doc);
        DirectoryReader r;
        if (iter == 0) {
            // use NRT
            r = w.getReader();
        } else {
            // don't use NRT
            w.commit();
            r = DirectoryReader.open(dir);
        }
        assertTrue(Files.exists(indexPath.resolve("_0.cfs")));
        assertTrue(Files.exists(indexPath.resolve("_0.cfe")));
        assertTrue(Files.exists(indexPath.resolve("_0.si")));
        if (iter == 1) {
            // we run a full commit so there should be a segments file etc.
            assertTrue(Files.exists(indexPath.resolve("segments_1")));
        } else {
            // this is an NRT reopen - no segments files yet
            assertFalse(Files.exists(indexPath.resolve("segments_1")));
        }
        w.addDocument(doc);
        w.forceMerge(1);
        if (iter == 1) {
            w.commit();
        }
        IndexReader r2 = DirectoryReader.openIfChanged(r);
        assertNotNull(r2);
        assertTrue(r != r2);
        // NOTE: here we rely on "Windows" behavior, ie, even
        // though IW wanted to delete _0.cfs since it was
        // merged away, because we have a reader open
        // against this file, it should still be here:
        assertTrue(Files.exists(indexPath.resolve("_0.cfs")));
        // forceMerge created this
        //assertTrue(files.contains("_2.cfs"));
        w.deleteUnusedFiles();
        // r still holds this file open
        assertTrue(Files.exists(indexPath.resolve("_0.cfs")));
        //assertTrue(files.contains("_2.cfs"));
        r.close();
        if (iter == 0) {
            // on closing NRT reader, it calls writer.deleteUnusedFiles
            assertFalse(Files.exists(indexPath.resolve("_0.cfs")));
        } else {
            // now FSDir can remove it
            dir.deletePendingFiles();
            assertFalse(Files.exists(indexPath.resolve("_0.cfs")));
        }
        w.close();
        r2.close();
        dir.close();
    }
}

Example 39 with FSDirectory

use of org.apache.lucene.store.FSDirectory in project lucene-solr by apache.

the class TestIndexWriter method testWithPendingDeletions.

public void testWithPendingDeletions() throws Exception {
    // irony: currently we don't emulate windows well enough to work on windows!
    assumeFalse("windows is not supported", Constants.WINDOWS);
    Path path = createTempDir();
    // Use WindowsFS to prevent open files from being deleted:
    FileSystem fs = new WindowsFS(path.getFileSystem()).getFileSystem(URI.create("file:///"));
    Path root = new FilterPath(path, fs);
    // MMapDirectory doesn't work because it closes its file handles after mapping!
    try (FSDirectory dir = new SimpleFSDirectory(root)) {
        IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
        IndexWriter w = new IndexWriter(dir, iwc);
        w.commit();
        IndexInput in = dir.openInput("segments_1", IOContext.DEFAULT);
        w.addDocument(new Document());
        w.close();
        assertTrue(dir.checkPendingDeletions());
        // make sure we get NFSF if we try to delete and already-pending-delete file:
        expectThrows(NoSuchFileException.class, () -> {
            dir.deleteFile("segments_1");
        });
        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
            new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())));
        });
        assertTrue(expected.getMessage().contains("still has pending deleted files; cannot initialize IndexWriter"));
        in.close();
    }
}

Example 40 with FSDirectory

use of org.apache.lucene.store.FSDirectory in project lucene-solr by apache.

the class IndexSplitter method split.

public void split(Path destDir, String[] segs) throws IOException {
    Files.createDirectories(destDir);
    FSDirectory destFSDir = FSDirectory.open(destDir);
    SegmentInfos destInfos = new SegmentInfos(infos.getIndexCreatedVersionMajor());
    destInfos.counter = infos.counter;
    for (String n : segs) {
        SegmentCommitInfo infoPerCommit = getInfo(n);
        SegmentInfo info = infoPerCommit.info;
        // Same info just changing the dir:
        SegmentInfo newInfo = new SegmentInfo(destFSDir, info.getVersion(), info.getMinVersion(), info.name, info.maxDoc(), info.getUseCompoundFile(), info.getCodec(), info.getDiagnostics(), info.getId(), new HashMap<>(), null);
        destInfos.add(new SegmentCommitInfo(newInfo, infoPerCommit.getDelCount(), infoPerCommit.getDelGen(), infoPerCommit.getFieldInfosGen(), infoPerCommit.getDocValuesGen()));
        // now copy files over
        Collection<String> files = infoPerCommit.files();
        for (final String srcName : files) {
            Path srcFile = dir.resolve(srcName);
            Path destFile = destDir.resolve(srcName);
            Files.copy(srcFile, destFile);
        }
    }
    destInfos.changed();
    destInfos.commit(destFSDir);
// System.out.println("destDir:"+destDir.getAbsolutePath());
}