Examples with Identifier com.cinchapi.concourse.server.model.Identifier used on opensource projects

Example 31 with Identifier

use of com.cinchapi.concourse.server.model.Identifier in project concourse by cinchapi.

the class ByteableCollectionsTest method testStreamIteratorThreadSafety.

@Test
public void testStreamIteratorThreadSafety() throws InterruptedException, ExecutionException {
    List<Identifier> values = Lists.newArrayList();
    int count = 10;
    for (int i = 0; i < count; ++i) {
        values.add(Identifier.of(i));
    }
    Path file = Paths.get(TestData.getTemporaryTestFile());
    ByteBuffer bytes = ByteableCollections.toByteBuffer(values);
    FileSystem.writeBytes(bytes, file.toString());
    FileChannel channel = FileSystem.getFileChannel(file);
    ExecutorService executor = Executors.newCachedThreadPool();
    List<Future<?>> futures = Lists.newArrayList();
    for (int i = 0; i < 10; ++i) {
        futures.add(executor.submit(() -> {
            List<Identifier> actual = Lists.newArrayList();
            Iterator<ByteBuffer> it = ByteableCollections.stream(channel, 0, FileSystem.getFileSize(file.toString()), 20);
            while (it.hasNext()) {
                actual.add(Identifier.fromByteBuffer(it.next()));
            }
        }));
    }
    for (Future<?> future : futures) {
        future.get();
    }
}

Example 32 with Identifier

use of com.cinchapi.concourse.server.model.Identifier in project concourse by cinchapi.

the class ManifestTest method testManifestWorksAfterBeingFlushed.

@Test
public void testManifestWorksAfterBeingFlushed() {
    int count = TestData.getScaleCount() * 2;
    Manifest manifest = Manifest.create(count);
    Identifier key = Identifier.of(count);
    manifest.putStart(count, key);
    manifest.putEnd(count * 2, key);
    Range range = manifest.lookup(key);
    Assert.assertEquals(count, range.start());
    Assert.assertEquals(count * 2, range.end());
    manifest.transfer(file);
    range = manifest.lookup(key);
    Assert.assertEquals(count, range.start());
    Assert.assertEquals(count * 2, range.end());
}

Example 33 with Identifier

use of com.cinchapi.concourse.server.model.Identifier in project concourse by cinchapi.

the class ManifestTest method testManifestStreamedEntriesAccuracy.

@Test
public void testManifestStreamedEntriesAccuracy() {
    int threshold = Manifest.MANIFEST_LENGTH_ENTRY_STREAMING_THRESHOLD;
    Manifest.MANIFEST_LENGTH_ENTRY_STREAMING_THRESHOLD = (int) Math.pow(2, 16);
    try {
        Manifest manifest = Manifest.create(100000);
        Text text = Text.wrap(Random.getString());
        int count = 0;
        int start = 0;
        Map<Composite, Range> expected = Maps.newHashMap();
        while (manifest.length() < Manifest.MANIFEST_LENGTH_ENTRY_STREAMING_THRESHOLD) {
            Identifier record = Identifier.of(count);
            int $start = start;
            int end = start + TestData.getScaleCount();
            Range range = new Manifest.Range() {

                @Override
                public long start() {
                    return $start;
                }

                @Override
                public long end() {
                    return end;
                }
            };
            Composite composite = Composite.create(text, record);
            manifest.putStart(start, composite);
            manifest.putEnd(end, composite);
            expected.put(composite, range);
            start = end + 1;
            ++count;
        }
        Path file = Paths.get(TestData.getTemporaryTestFile());
        manifest.transfer(file);
        Manifest $manifest = Manifest.load(file, 0, FileSystem.getFileSize(file.toString()));
        expected.forEach((composite, range) -> {
            Range actual = $manifest.lookup(composite);
            Assert.assertEquals(range.start(), actual.start());
            Assert.assertEquals(range.end(), actual.end());
        });
    } finally {
        Manifest.MANIFEST_LENGTH_ENTRY_STREAMING_THRESHOLD = threshold;
    }
}

Example 34 with Identifier

use of com.cinchapi.concourse.server.model.Identifier in project concourse by cinchapi.

the class SegmentTest method testDataDeduplication.

@Test
public void testDataDeduplication() {
    String key = "name";
    TObject value = Convert.javaToThrift("Fonamey");
    long record = 1;
    // Simulate adding and removing while server is running, but creating
    // new intermediate TObjects
    Write w1 = Write.add(key, Convert.javaToThrift("Fonamey"), record);
    Write w2 = Write.remove(key, Convert.javaToThrift("Fonamey"), record);
    Assert.assertNotSame(w1.getValue(), w2.getValue());
    segment.acquire(w1);
    segment.acquire(w2);
    // Simulate loading data from disk and creating new intermediate because
    // values are not cached when read
    w1 = Write.fromByteBuffer(w1.getBytes());
    w2 = Write.fromByteBuffer(w2.getBytes());
    Assert.assertNotSame(w1.getValue(), w2.getValue());
    segment.acquire(w1);
    segment.acquire(w2);
    int count = TestData.getScaleCount();
    for (int i = 0; i < count; ++i) {
        Write write = Numbers.isEven(i) ? Write.remove(key, value, record) : Write.add(key, value, record);
        write = Write.fromByteBuffer(write.getBytes());
        segment.acquire(write);
    }
    Text name = null;
    Value fonamey = null;
    Identifier one = null;
    Position position = null;
    Iterator<Revision<Identifier, Text, Value>> tit = segment.table().iterator();
    while (tit.hasNext()) {
        Revision<Identifier, Text, Value> revision = tit.next();
        if (one == null) {
            one = revision.getLocator();
        }
        if (name == null) {
            name = revision.getKey();
        }
        if (fonamey == null) {
            fonamey = revision.getValue();
        }
        Assert.assertSame(one, revision.getLocator());
        Assert.assertSame(name, revision.getKey());
        Assert.assertSame(fonamey, revision.getValue());
    }
    Iterator<Revision<Text, Value, Identifier>> iit = segment.index().iterator();
    while (iit.hasNext()) {
        Revision<Text, Value, Identifier> revision = iit.next();
        if (one == null) {
            one = revision.getValue();
        }
        if (name == null) {
            name = revision.getLocator();
        }
        if (fonamey == null) {
            fonamey = revision.getKey();
        }
        Assert.assertSame(one, revision.getValue());
        Assert.assertSame(name, revision.getLocator());
        Assert.assertSame(fonamey, revision.getKey());
    }
    Iterator<Revision<Text, Text, Position>> cit = segment.corpus().iterator();
    while (cit.hasNext()) {
        Revision<Text, Text, Position> revision = cit.next();
        if (position == null) {
            position = revision.getValue();
        }
        if (name == null) {
            name = revision.getLocator();
        }
        Assert.assertSame(position, revision.getValue());
        Assert.assertSame(name, revision.getLocator());
        if (revision.getKey().toString().equals("name")) {
            Assert.assertSame(name, revision.getKey());
        }
    }
}

Example 35 with Identifier

use of com.cinchapi.concourse.server.model.Identifier in project concourse by cinchapi.

the class SegmentTest method testConcurrency.

@Test
public void testConcurrency() throws InterruptedException {
    AtomicBoolean succeeded = new AtomicBoolean(true);
    AwaitableExecutorService executor = new AwaitableExecutorService(Executors.newCachedThreadPool());
    try {
        for (int i = 0; i < 1000; ++i) {
            AtomicBoolean done = new AtomicBoolean(false);
            long record = i;
            String key = Long.toString(record);
            TObject value = Convert.javaToThrift(Long.toString(record));
            Write write = Write.add(key, value, record);
            Identifier pk = Identifier.of(record);
            Text text = Text.wrap(key);
            Thread reader = new Thread(() -> {
                while (!done.get()) {
                    TableRecord tr = TableRecord.create(pk);
                    IndexRecord ir = IndexRecord.create(text);
                    CorpusRecord cr = CorpusRecord.create(text);
                    segment.table().seek(Composite.create(pk), tr);
                    segment.index().seek(Composite.create(text), ir);
                    segment.corpus().seek(Composite.create(text), cr);
                    if (!done.get() && tr.isEmpty() != ir.isEmpty()) {
                        if (!tr.isEmpty() && ir.isEmpty()) {
                            // Later read is empty
                            succeeded.set(false);
                            System.out.println(AnyStrings.format("table empty = {} and index empty = {} for {}", tr.isEmpty(), ir.isEmpty(), record));
                        }
                    }
                    if (!done.get() && ir.isEmpty() != cr.isEmpty()) {
                        if (!ir.isEmpty() && cr.isEmpty()) {
                            // Later read is empty
                            succeeded.set(false);
                            System.out.println(AnyStrings.format("index empty = {} and corpus empty = {} for {}", tr.isEmpty(), cr.isEmpty(), record));
                        }
                    }
                }
                TableRecord tr = TableRecord.create(pk);
                IndexRecord ir = IndexRecord.create(text);
                CorpusRecord cr = CorpusRecord.create(text);
                segment.table().seek(Composite.create(pk), tr);
                segment.index().seek(Composite.create(text), ir);
                segment.corpus().seek(Composite.create(text), cr);
                if (tr.isEmpty()) {
                    succeeded.set(false);
                    System.out.println("After write finished, table still empty for " + record);
                }
                if (ir.isEmpty()) {
                    succeeded.set(false);
                    System.out.println("After write finished, index still empty for " + record);
                }
                if (cr.isEmpty()) {
                    succeeded.set(false);
                    System.out.println("After write finished, corpus still empty for " + record);
                }
            });
            Thread writer = new Thread(() -> {
                try {
                    segment.acquire(write, executor);
                    done.set(true);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            });
            reader.start();
            writer.start();
            writer.join();
            reader.join();
        }
        Assert.assertTrue(succeeded.get());
    } finally {
        executor.shutdown();
    }
}