Examples with SimilarityCompactor com.cinchapi.concourse.server.storage.db.compaction.similarity.SimilarityCompactor used on opensource projects

Example 1 with SimilarityCompactor

use of com.cinchapi.concourse.server.storage.db.compaction.similarity.SimilarityCompactor in project concourse by cinchapi.

the class Database method start.

@Override
public void start() {
    if (!running) {
        Logger.info("Database configured to store data in {}", directory);
        running = true;
        this.writer = new AwaitableExecutorService(Executors.newCachedThreadPool(ThreadFactories.namingThreadFactory("DatabaseWriter")));
        this.segments.clear();
        ArrayBuilder<Runnable> tasks = ArrayBuilder.builder();
        List<Segment> segments = Collections.synchronizedList(this.segments);
        Stream<Path> files = storage.files();
        files.forEach(file -> tasks.add(() -> {
            try {
                Segment segment = Segment.load(file);
                segments.add(segment);
            } catch (SegmentLoadingException e) {
                Logger.error("Error when trying to load Segment {}", file);
                Logger.error("", e);
            }
        }));
        files.close();
        if (tasks.length() > 0) {
            AwaitableExecutorService loader = new AwaitableExecutorService(Executors.newCachedThreadPool(ThreadFactories.namingThreadFactory("DatabaseLoader")));
            try {
                loader.await((task, error) -> Logger.error("Unexpected error when trying to load Database Segments: {}", error), tasks.build());
            } catch (InterruptedException e) {
                Logger.error("The Database was interrupted while starting...", e);
                Thread.currentThread().interrupt();
                return;
            } finally {
                loader.shutdown();
            }
        }
        // Sort the segments in chronological order
        Collections.sort(this.segments, Segment.TEMPORAL_COMPARATOR);
        // Remove segments that overlap. Segments may overlap if they are
        // duplicates resulting from a botched upgrade or reindex or if they
        // were involved in an optimization pass, but garbage collection
        // didn't run before the server shutdown.
        ListIterator<Segment> lit = segments.listIterator();
        while (lit.hasNext()) {
            if (lit.hasPrevious()) {
                Segment previous = lit.previous();
                lit.next();
                Segment current = lit.next();
                if (current.intersects(previous)) {
                    lit.previous();
                    lit.previous();
                    lit.remove();
                    Logger.warn("Segment {} was not loaded because it contains duplicate data. It has been scheduled for garbage collection.", previous);
                // TODO: mark #previous for garbage collection
                }
            } else {
                lit.next();
            }
        }
        rotate(false);
        memory = new CacheState();
        /*
             * If enabled, setup Compaction to run continuously in the
             * background; trying to perform both "full" and "incremental"
             * compaction. Incremental compaction is opportunistic; attempting
             * frequently, but only occurring if no other conflicting work is
             * happening and only trying to compact one "shift". On the other
             * hand,full compaction runs less frequently, but is very
             * aggressive: blocking until any other conflicting work is done
             * and trying every possible shift.
             */
        // @formatter:off
        compactor = ENABLE_COMPACTION ? new SimilarityCompactor(storage) : NoOpCompactor.instance();
        fullCompaction = ENABLE_COMPACTION ? Executors.newScheduledThreadPool(1, createCompactionThreadFactory(tag, "Full")) : NoOpScheduledExecutorService.instance();
        fullCompaction.scheduleWithFixedDelay(() -> compactor.executeFullCompaction(), FULL_COMPACTION_INITIAL_DELAY_IN_SECONDS, FULL_COMPACTION_RUN_FREQUENCY_IN_SECONDS, TimeUnit.SECONDS);
        incrementalCompaction = ENABLE_COMPACTION ? Executors.newScheduledThreadPool(1, createCompactionThreadFactory(tag, "Incremental")) : NoOpScheduledExecutorService.instance();
        incrementalCompaction.scheduleWithFixedDelay(() -> compactor.tryIncrementalCompaction(), INCREMENTAL_COMPACTION_INITIAL_DELAY_IN_SECONDS, INCREMENTAL_COMPACTION_RUN_FREQUENCY_IN_SECONDS, TimeUnit.SECONDS);
        // @formatter:on
        Logger.info("Database is running with compaction {}.", ENABLE_COMPACTION ? "ON" : "OFF");
    }
}