Examples with SimpleFSDirectory org.apache.lucene.store.SimpleFSDirectory used on opensource projects

Example 16 with SimpleFSDirectory

use of org.apache.lucene.store.SimpleFSDirectory in project crate by crate.

the class HunspellService method loadDictionary.

/**
 * Loads the hunspell dictionary for the given local.
 *
 * @param locale       The locale of the hunspell dictionary to be loaded.
 * @param nodeSettings The node level settings
 * @param env          The node environment (from which the conf path will be resolved)
 * @return The loaded Hunspell dictionary
 * @throws Exception when loading fails (due to IO errors or malformed dictionary files)
 */
private Dictionary loadDictionary(String locale, Settings nodeSettings, Environment env) throws Exception {
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("Loading hunspell dictionary [{}]...", locale);
    }
    Path dicDir = hunspellDir.resolve(locale);
    if (FileSystemUtils.isAccessibleDirectory(dicDir, LOGGER) == false) {
        throw new ElasticsearchException(String.format(Locale.ROOT, "Could not find hunspell dictionary [%s]", locale));
    }
    // merging node settings with hunspell dictionary specific settings
    Settings dictSettings = HUNSPELL_DICTIONARY_OPTIONS.get(nodeSettings);
    nodeSettings = loadDictionarySettings(dicDir, dictSettings.getByPrefix(locale + "."));
    boolean ignoreCase = nodeSettings.getAsBoolean("ignore_case", defaultIgnoreCase);
    Path[] affixFiles = FileSystemUtils.files(dicDir, "*.aff");
    if (affixFiles.length == 0) {
        throw new ElasticsearchException(String.format(Locale.ROOT, "Missing affix file for hunspell dictionary [%s]", locale));
    }
    if (affixFiles.length != 1) {
        throw new ElasticsearchException(String.format(Locale.ROOT, "Too many affix files exist for hunspell dictionary [%s]", locale));
    }
    InputStream affixStream = null;
    Path[] dicFiles = FileSystemUtils.files(dicDir, "*.dic");
    List<InputStream> dicStreams = new ArrayList<>(dicFiles.length);
    try {
        for (int i = 0; i < dicFiles.length; i++) {
            dicStreams.add(Files.newInputStream(dicFiles[i]));
        }
        affixStream = Files.newInputStream(affixFiles[0]);
        try (Directory tmp = new SimpleFSDirectory(env.tmpFile())) {
            return new Dictionary(tmp, "hunspell", affixStream, dicStreams, ignoreCase);
        }
    } catch (Exception e) {
        LOGGER.error(() -> new ParameterizedMessage("Could not load hunspell dictionary [{}]", locale), e);
        throw e;
    } finally {
        IOUtils.close(affixStream);
        IOUtils.close(dicStreams);
    }
}

Example 17 with SimpleFSDirectory

use of org.apache.lucene.store.SimpleFSDirectory in project crate by crate.

the class FsDirectoryFactory method newFSDirectory.

protected Directory newFSDirectory(Path location, LockFactory lockFactory, IndexSettings indexSettings) throws IOException {
    final String storeType = indexSettings.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.FS.getSettingsKey());
    IndexModule.Type type;
    if (IndexModule.Type.FS.match(storeType)) {
        type = IndexModule.defaultStoreType(IndexModule.NODE_STORE_ALLOW_MMAP.getWithFallback(indexSettings.getNodeSettings()));
    } else {
        type = IndexModule.Type.fromSettingsKey(storeType);
    }
    Set<String> preLoadExtensions = new HashSet<>(indexSettings.getValue(IndexModule.INDEX_STORE_PRE_LOAD_SETTING));
    switch(type) {
        case HYBRIDFS:
            // Use Lucene defaults
            final FSDirectory primaryDirectory = FSDirectory.open(location, lockFactory);
            if (primaryDirectory instanceof MMapDirectory) {
                MMapDirectory mMapDirectory = (MMapDirectory) primaryDirectory;
                return new HybridDirectory(lockFactory, setPreload(mMapDirectory, lockFactory, preLoadExtensions));
            } else {
                return primaryDirectory;
            }
        case MMAPFS:
            return setPreload(new MMapDirectory(location, lockFactory), lockFactory, preLoadExtensions);
        case SIMPLEFS:
            return new SimpleFSDirectory(location, lockFactory);
        case NIOFS:
            return new NIOFSDirectory(location, lockFactory);
        default:
            throw new AssertionError("unexpected built-in store type [" + type + "]");
    }
}

Example 18 with SimpleFSDirectory

use of org.apache.lucene.store.SimpleFSDirectory in project crate by crate.

the class Checkpoint method read.

public static Checkpoint read(Path path) throws IOException {
    try (Directory dir = new SimpleFSDirectory(path.getParent())) {
        try (IndexInput indexInput = dir.openInput(path.getFileName().toString(), IOContext.DEFAULT)) {
            // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here.
            CodecUtil.checksumEntireFile(indexInput);
            final int fileVersion = CodecUtil.checkHeader(indexInput, CHECKPOINT_CODEC, INITIAL_VERSION, CURRENT_VERSION);
            if (fileVersion == INITIAL_VERSION) {
                assert indexInput.length() == V1_FILE_SIZE : indexInput.length();
                return Checkpoint.readCheckpointV5_0_0(indexInput);
            } else if (fileVersion == VERSION_6_0_0) {
                assert indexInput.length() == V2_FILE_SIZE : indexInput.length();
                return Checkpoint.readCheckpointV6_0_0(indexInput);
            } else {
                assert fileVersion == CURRENT_VERSION : fileVersion;
                assert indexInput.length() == V3_FILE_SIZE : indexInput.length();
                return Checkpoint.readCheckpointV6_4_0(indexInput);
            }
        } catch (CorruptIndexException | NoSuchFileException | IndexFormatTooOldException | IndexFormatTooNewException e) {
            throw new TranslogCorruptedException(path.toString(), e);
        }
    }
}

Example 19 with SimpleFSDirectory

use of org.apache.lucene.store.SimpleFSDirectory in project crate by crate.

the class PersistedClusterStateService method nodeMetadata.

/**
 * Returns the node metadata for the given data paths, and checks if the node ids are unique
 * @param dataPaths the data paths to scan
 */
@Nullable
public static NodeMetadata nodeMetadata(Path... dataPaths) throws IOException {
    String nodeId = null;
    Version version = null;
    for (final Path dataPath : dataPaths) {
        final Path indexPath = dataPath.resolve(METADATA_DIRECTORY_NAME);
        if (Files.exists(indexPath)) {
            try (DirectoryReader reader = DirectoryReader.open(new SimpleFSDirectory(dataPath.resolve(METADATA_DIRECTORY_NAME)))) {
                final Map<String, String> userData = reader.getIndexCommit().getUserData();
                assert userData.get(NODE_VERSION_KEY) != null;
                final String thisNodeId = userData.get(NODE_ID_KEY);
                assert thisNodeId != null;
                if (nodeId != null && nodeId.equals(thisNodeId) == false) {
                // Do nothing, because the metadata does not belong to this node
                } else if (nodeId == null) {
                    nodeId = thisNodeId;
                    version = Version.fromId(Integer.parseInt(userData.get(NODE_VERSION_KEY)));
                }
            } catch (IndexNotFoundException e) {
                LOGGER.debug(new ParameterizedMessage("no on-disk state at {}", indexPath), e);
            }
        }
    }
    if (nodeId == null) {
        return null;
    }
    return new NodeMetadata(nodeId, version);
}

Example 20 with SimpleFSDirectory

use of org.apache.lucene.store.SimpleFSDirectory in project crate by crate.

the class FsDirectoryFactoryTests method doTestStoreDirectory.

private void doTestStoreDirectory(Path tempDir, String typeSettingValue, IndexModule.Type type) throws IOException {
    Settings.Builder settingsBuilder = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT);
    if (typeSettingValue != null) {
        settingsBuilder.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), typeSettingValue);
    }
    Settings settings = settingsBuilder.build();
    IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
    FsDirectoryFactory service = new FsDirectoryFactory();
    try (Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE, indexSettings)) {
        switch(type) {
            case HYBRIDFS:
                assertTrue(FsDirectoryFactory.isHybridFs(directory));
                break;
            case NIOFS:
                assertTrue(type + " " + directory.toString(), directory instanceof NIOFSDirectory);
                break;
            case MMAPFS:
                assertTrue(type + " " + directory.toString(), directory instanceof MMapDirectory);
                break;
            case SIMPLEFS:
                assertTrue(type + " " + directory.toString(), directory instanceof SimpleFSDirectory);
                break;
            case FS:
                if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
                    assertTrue(FsDirectoryFactory.isHybridFs(directory));
                } else if (Constants.WINDOWS) {
                    assertTrue(directory.toString(), directory instanceof SimpleFSDirectory);
                } else {
                    assertTrue(directory.toString(), directory instanceof NIOFSDirectory);
                }
                break;
            default:
                fail();
        }
    }
}