Examples with NamespacedStorage com.bakdata.conquery.io.storage.NamespacedStorage used on opensource projects

Example 1 with NamespacedStorage

use of com.bakdata.conquery.io.storage.NamespacedStorage in project conquery by bakdata.

the class XodusStoreFactory method loadNamespacedStores.

private <T extends NamespacedStorage> Queue<T> loadNamespacedStores(String prefix, Function<String, T> creator, Set<String> storesToTest) throws InterruptedException {
    File baseDir = getDirectory().toFile();
    if (baseDir.mkdirs()) {
        log.warn("Had to create Storage Dir at `{}`", baseDir);
    }
    Queue<T> storages = new ConcurrentLinkedQueue<>();
    ExecutorService loaders = Executors.newFixedThreadPool(getNThreads());
    for (File directory : Objects.requireNonNull(baseDir.listFiles((file, name) -> file.isDirectory() && name.startsWith(prefix)))) {
        final String name = directory.getName();
        loaders.submit(() -> {
            try {
                ConqueryMDC.setLocation(directory.toString());
                if (!environmentHasStores(directory, storesToTest)) {
                    log.warn("No valid WorkerStorage found.");
                    return;
                }
                T namespacedStorage = creator.apply(name);
                namespacedStorage.openStores(this);
                log.debug("BEGIN reading Storage");
                namespacedStorage.loadData();
                storages.add(namespacedStorage);
            } catch (Exception e) {
                log.error("Failed reading Storage", e);
            } finally {
                log.debug("DONE reading Storage");
                ConqueryMDC.clearLocation();
            }
        });
    }
    loaders.shutdown();
    while (!loaders.awaitTermination(1, TimeUnit.MINUTES)) {
        log.debug("Waiting for Worker storages to load. {} are already finished.", storages.size());
    }
    log.info("All WorkerStores loaded: {}", storages);
    return storages;
}