Examples with Segment com.cinchapi.concourse.server.storage.db.kernel.Segment used on opensource projects

Example 11 with Segment

use of com.cinchapi.concourse.server.storage.db.kernel.Segment in project concourse by cinchapi.

the class Database method getCorpusRecord.

/**
 * Return the CorpusRecord identified by {@code key}.
 *
 * @param key
 * @param query
 * @param toks {@code query} split by whitespace
 * @return the CorpusRecord
 */
private CorpusRecord getCorpusRecord(Text key, Text infix) {
    masterLock.readLock().lock();
    try {
        Composite composite = Composite.create(key, infix);
        Cache<Composite, CorpusRecord> cache = ENABLE_SEARCH_CACHE ? corpusCaches.computeIfAbsent(key, $ -> buildCache()) : DISABLED_CORPUS_CACHE;
        return cache.get(composite, () -> {
            CorpusRecord $ = CorpusRecord.createPartial(key, infix);
            for (Segment segment : segments) {
                segment.corpus().seek(composite, $);
            }
            return $;
        });
    } catch (ExecutionException e) {
        throw CheckedExceptions.wrapAsRuntimeException(e);
    } finally {
        masterLock.readLock().unlock();
    }
}

Example 12 with Segment

use of com.cinchapi.concourse.server.storage.db.kernel.Segment in project concourse by cinchapi.

the class SimilarityCompactor method compact.

@Override
protected List<Segment> compact(Segment... segments) {
    if (segments.length == 2) {
        Segment a = segments[0];
        Segment b = segments[1];
        long requiredDiskSpace = a.length() + b.length();
        if (storage().availableDiskSpace() > requiredDiskSpace && a.similarityWith(b) > minimumSimilarityThreshold) {
            // TODO:
            Segment merged = Segment.create((int) (a.count() + b.count()));
            // create
            // offheap
            Streams.concat(a.writes(), b.writes()).parallel().forEach(write -> merged.acquire(write));
            return ImmutableList.of(merged);
        } else {
            return null;
        }
    } else {
        return null;
    }
}

Example 13 with Segment

use of com.cinchapi.concourse.server.storage.db.kernel.Segment in project concourse by cinchapi.

the class Upgrade0_11_0_1 method rollback.

@Override
protected void rollback() {
    logErrorMessage("Deleting Segment files");
    Environments.iterator(GlobalState.BUFFER_DIRECTORY, GlobalState.DATABASE_DIRECTORY).forEachRemaining(environment -> {
        Path directory = Paths.get(GlobalState.DATABASE_DIRECTORY).resolve(environment).resolve("segments");
        if (directory.toFile().exists()) {
            Iterable<Segment> segments = StorageFormatV3.load(directory);
            for (Segment segment : segments) {
                segment.delete();
            }
        }
    });
}

Example 14 with Segment

use of com.cinchapi.concourse.server.storage.db.kernel.Segment in project concourse by cinchapi.

the class StorageFormatV3 method load.

/**
 * Load the {@link Segment Segments} from {@code directory}
 *
 * @param directory
 * @return an {@link Iterable} containing all the loaded {@link Segment
 *         Segments}
 */
public static Iterable<Segment> load(Path directory) {
    /*
         * NOTE: This implementation is copied from Database#start with a few
         * changes:
         * 1) Segments are loaded seriously instead of asynchronously
         * 2) Overlapping Segments are not removed
         */
    List<Segment> segments = Lists.newArrayList();
    Stream<Path> files = FileSystem.ls(directory);
    files.forEach(file -> {
        try {
            Segment segment = Segment.load(file);
            segments.add(segment);
        } catch (SegmentLoadingException e) {
            Logger.error("Error when trying to load Segment {}", file);
            Logger.error("", e);
        }
    });
    files.close();
    // Sort the segments in chronological order
    Collections.sort(segments, Segment.TEMPORAL_COMPARATOR);
    return segments;
}

Example 15 with Segment

use of com.cinchapi.concourse.server.storage.db.kernel.Segment in project concourse by cinchapi.

the class SimilarityCompactorTest method testSanityCheck.

@Test
public void testSanityCheck() {
    SegmentStorageSystem storage = CompactorTests.getStorageSystem();
    Segment a = Segment.create();
    a.acquire(Write.add("name", Convert.javaToThrift("jeff"), 1));
    a.acquire(Write.add("name", Convert.javaToThrift("ashleah"), 2));
    a.acquire(Write.add("age", Convert.javaToThrift(33), 2));
    a.acquire(Write.add("company", Convert.javaToThrift("Cinchapi"), 1));
    a.transfer(Paths.get(TestData.getTemporaryTestFile()));
    Segment b = Segment.create();
    b.acquire(Write.add("name", Convert.javaToThrift("Jeff"), 1));
    b.acquire(Write.remove("name", Convert.javaToThrift("jeff"), 1));
    b.acquire(Write.remove("name", Convert.javaToThrift("ashleah"), 2));
    b.acquire(Write.add("name", Convert.javaToThrift("Jeffery"), 1));
    b.acquire(Write.add("company", Convert.javaToThrift("Know Full Well"), 2));
    b.acquire(Write.add("age", Convert.javaToThrift(33), 1));
    b.acquire(Write.add("age", Convert.javaToThrift(33.0), 1));
    b.transfer(Paths.get(TestData.getTemporaryTestFile()));
    storage.segments().add(a);
    storage.segments().add(b);
    // seg0
    storage.segments().add(Segment.create());
    SimilarityCompactor compactor = new SimilarityCompactor(storage);
    compactor.minimumSimilarityThreshold(0);
    List<Write> expected = storage.segments().stream().flatMap(segment -> segment.writes()).collect(Collectors.toList());
    compactor.executeFullCompaction();
    List<Write> actual = storage.segments().stream().flatMap(segment -> segment.writes()).collect(Collectors.toList());
    Assert.assertEquals(2, storage.segments().size());
    Assert.assertTrue(expected.size() == actual.size() && expected.containsAll(actual) && // assert that no data
    actual.containsAll(expected));
    // lost...
    actual.forEach(System.out::println);
    for (int i = 0; i < actual.size(); ++i) {
        if (i > 0) {
            Identifier previous = actual.get(i - 1).getRecord();
            Identifier current = actual.get(i).getRecord();
            // Assert that, from table view, all records are grouped
            // together
            Assert.assertTrue(current.longValue() == previous.longValue() || current.longValue() == previous.longValue() + 1);
        }
    }
    Assert.assertEquals(ImmutableList.of(a, b), compactor.garbage());
}