Examples with OLogSequenceNumber com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OLogSequenceNumber used on opensource projects

Example 81 with OLogSequenceNumber

use of com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OLogSequenceNumber in project orientdb by orientechnologies.

the class ReadWriteDiskCacheTest method testFrequentlyReadItemsAreMovedInAm.

public void testFrequentlyReadItemsAreMovedInAm() throws Exception {
    long fileId = readBuffer.addFile(fileName, writeBuffer);
    OCacheEntry[] entries = new OCacheEntry[10];
    for (int i = 0; i < 10; i++) {
        entries[i] = readBuffer.load(fileId, i, false, writeBuffer, 1, true);
        if (entries[i] == null) {
            entries[i] = readBuffer.allocateNewPage(fileId, writeBuffer, true);
            Assert.assertEquals(entries[i].getPageIndex(), i);
        }
        entries[i].getCachePointer().acquireExclusiveLock();
        entries[i].markDirty();
        final ByteBuffer buffer = entries[i].getCachePointer().getSharedBuffer();
        buffer.position(systemOffset);
        buffer.put(new byte[] { (byte) i, 1, 2, seed, 4, 5, 6, (byte) i });
        setLsn(buffer, new OLogSequenceNumber(1, i));
        entries[i].getCachePointer().releaseExclusiveLock();
        readBuffer.release(entries[i], writeBuffer);
    }
    writeBuffer.flush();
    readBuffer.clear();
    for (int i = 0; i < 10; i++) assertFile(i, new byte[] { (byte) i, 1, 2, seed, 4, 5, 6, (byte) i }, new OLogSequenceNumber(1, i));
    for (int i = 0; i < 8; i++) {
        entries[i] = readBuffer.load(fileId, i, false, writeBuffer, 1, true);
        readBuffer.release(entries[i], writeBuffer);
    }
    for (int i = 2; i < 4; i++) {
        entries[i] = readBuffer.load(fileId, i, false, writeBuffer, 1, true);
        readBuffer.release(entries[i], writeBuffer);
    }
    LRUList am = readBuffer.getAm();
    LRUList a1in = readBuffer.getA1in();
    LRUList a1out = readBuffer.getA1out();
    Assert.assertEquals(am.size(), 2);
    Assert.assertEquals(a1in.size(), 2);
    Assert.assertEquals(a1out.size(), 2);
    final OByteBufferPool bufferPool = OByteBufferPool.instance();
    for (int i = 2; i < 4; i++) {
        OCacheEntry lruEntry = generateEntry(fileId, i, entries[i].getCachePointer().getSharedBuffer(), bufferPool, false, new OLogSequenceNumber(1, i));
        Assert.assertEquals(am.get(fileId, i), lruEntry);
    }
    for (int i = 4; i < 6; i++) {
        OCacheEntry lruEntry = generateRemovedEntry(fileId, i);
        Assert.assertEquals(a1out.get(fileId, i), lruEntry);
    }
    for (int i = 6; i < 8; i++) {
        OCacheEntry lruEntry = generateEntry(fileId, i, entries[i].getCachePointer().getSharedBuffer(), bufferPool, false, new OLogSequenceNumber(1, i));
        Assert.assertEquals(a1in.get(fileId, i), lruEntry);
    }
}

Example 82 with OLogSequenceNumber

use of com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OLogSequenceNumber in project orientdb by orientechnologies.

the class ReadWriteDiskCacheTest method testFrequentlyAddItemsAreMovedInAm.

public void testFrequentlyAddItemsAreMovedInAm() throws Exception {
    long fileId = readBuffer.addFile(fileName, writeBuffer);
    OCacheEntry[] entries = new OCacheEntry[10];
    for (int i = 0; i < 10; i++) {
        entries[i] = readBuffer.load(fileId, i, false, writeBuffer, 1, true);
        if (entries[i] == null) {
            entries[i] = readBuffer.allocateNewPage(fileId, writeBuffer, true);
            Assert.assertEquals(entries[i].getPageIndex(), i);
        }
        entries[i].getCachePointer().acquireExclusiveLock();
        entries[i].markDirty();
        final ByteBuffer buffer = entries[i].getCachePointer().getSharedBuffer();
        buffer.position(systemOffset);
        buffer.put(new byte[] { (byte) i, 1, 2, seed, 4, 5, 6, (byte) i });
        setLsn(buffer, new OLogSequenceNumber(1, i));
        entries[i].getCachePointer().releaseExclusiveLock();
        readBuffer.release(entries[i], writeBuffer);
    }
    LRUList am = readBuffer.getAm();
    LRUList a1in = readBuffer.getA1in();
    LRUList a1out = readBuffer.getA1out();
    Assert.assertEquals(a1in.size(), 4);
    Assert.assertEquals(a1out.size(), 2);
    Assert.assertEquals(am.size(), 0);
    OByteBufferPool bufferPool = OByteBufferPool.instance();
    for (int i = 6; i < 10; i++) {
        OCacheEntry lruEntry = generateEntry(fileId, i, entries[i].getCachePointer().getSharedBuffer(), bufferPool, false, new OLogSequenceNumber(0, 0));
        Assert.assertEquals(a1in.get(fileId, i), lruEntry);
    }
    for (int i = 4; i < 6; i++) {
        OCacheEntry lruEntry = generateRemovedEntry(fileId, i);
        Assert.assertEquals(a1out.get(fileId, i), lruEntry);
    }
    for (int i = 4; i < 6; i++) {
        entries[i] = readBuffer.load(fileId, i, false, writeBuffer, 1, true);
        readBuffer.release(entries[i], writeBuffer);
    }
    Assert.assertEquals(am.size(), 2);
    Assert.assertEquals(a1in.size(), 2);
    Assert.assertEquals(a1out.size(), 2);
    for (int i = 4; i < 6; i++) {
        OCacheEntry lruEntry = generateEntry(fileId, i, entries[i].getCachePointer().getSharedBuffer(), bufferPool, false, new OLogSequenceNumber(1, i));
        Assert.assertEquals(am.get(fileId, i), lruEntry);
    }
    for (int i = 6; i < 8; i++) {
        OCacheEntry lruEntry = generateRemovedEntry(fileId, i);
        Assert.assertEquals(a1out.get(fileId, i), lruEntry);
    }
    for (int i = 8; i < 10; i++) {
        OCacheEntry lruEntry = generateEntry(fileId, i, entries[i].getCachePointer().getSharedBuffer(), bufferPool, false, new OLogSequenceNumber(0, 0));
        Assert.assertEquals(a1in.get(fileId, i), lruEntry);
    }
    writeBuffer.flush();
    for (int i = 0; i < 10; i++) assertFile(i, new byte[] { (byte) i, 1, 2, seed, 4, 5, 6, (byte) i }, new OLogSequenceNumber(1, i));
}

Example 83 with OLogSequenceNumber

use of com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OLogSequenceNumber in project orientdb by orientechnologies.

the class ReadWriteDiskCacheTest method testPrefetchPagesInA1inAmQueue.

public void testPrefetchPagesInA1inAmQueue() throws Exception {
    final long fileId = readBuffer.addFile(fileName, writeBuffer);
    Assert.assertEquals(readBuffer.getMaxSize(), 4);
    // create file with 8 pages, we will push some of them in different queues later
    for (int i = 0; i < 8; i++) {
        OCacheEntry cacheEntry = readBuffer.allocateNewPage(fileId, writeBuffer, true);
        cacheEntry.acquireExclusiveLock();
        try {
            byte[] userData = new byte[userDataSize];
            for (int n = 0; n < userData.length; n++) {
                userData[n] = (byte) (i + 1);
            }
            final ByteBuffer buffer = cacheEntry.getCachePointer().getSharedBuffer();
            buffer.position(systemOffset);
            buffer.put(userData);
            setLsn(buffer, new OLogSequenceNumber(1, i));
            cacheEntry.markDirty();
        } finally {
            cacheEntry.releaseExclusiveLock();
            readBuffer.release(cacheEntry, writeBuffer);
        }
    }
    readBuffer.clear();
    writeBuffer.flush();
    for (int i = 0; i < 8; i++) {
        byte[] userData = new byte[userDataSize];
        for (int n = 0; n < userData.length; n++) {
            userData[n] = (byte) (i + 1);
        }
        assertFile(i, userData, new OLogSequenceNumber(1, i));
    }
    LRUList am = readBuffer.getAm();
    LRUList a1in = readBuffer.getA1in();
    LRUList a1out = readBuffer.getA1out();
    Assert.assertEquals(am.size(), 0);
    Assert.assertEquals(a1out.size(), 0);
    Assert.assertEquals(a1in.size(), 0);
    // put 1 and 2  pages to the a1out queue, page 0 is dropped from buffer
    for (int i = 0; i < 7; i++) {
        OCacheEntry cacheEntry = readBuffer.load(fileId, i, false, writeBuffer, 1, true);
        readBuffer.release(cacheEntry, writeBuffer);
    }
    Assert.assertEquals(am.size(), 0);
    // pages 1 - 2
    Assert.assertEquals(a1out.size(), 2);
    // pages 3 - 6
    Assert.assertEquals(a1in.size(), 4);
    // put 1-th page to the am queue
    OCacheEntry cacheEntry = readBuffer.load(fileId, 1, false, writeBuffer, 1, true);
    readBuffer.release(cacheEntry, writeBuffer);
    // page 1
    Assert.assertEquals(am.size(), 1);
    // page 2 - 3 (removed from a1in because of size limit)
    Assert.assertEquals(a1out.size(), 2);
    // pages 4 - 6
    Assert.assertEquals(a1in.size(), 3);
    // load page 0 and prefetch 1 (in am) 2 and 3 (in a1out)  and 4 (in a1in) , so 5 pages in total but we have room only for 4
    // am max size is 3 and a1in is 1 if cache is filled, so am will contain pages 2 -3 , a1in page 0, because it is in use,
    // the rest of pages will be removed from cache.
    cacheEntry = readBuffer.load(fileId, 0, false, writeBuffer, 4, true);
    readBuffer.release(cacheEntry, writeBuffer);
    // pages 1, 2, 3
    Assert.assertEquals(am.size(), 3);
    // pages 5, 6
    Assert.assertEquals(a1out.size(), 2);
    // page 0
    Assert.assertEquals(a1in.size(), 1);
    Assert.assertNotNull(am.get(fileId, 1));
    Assert.assertNotNull(am.get(fileId, 2));
    Assert.assertNotNull(am.get(fileId, 3));
    Assert.assertNotNull(a1out.get(fileId, 5));
    Assert.assertNotNull(a1out.get(fileId, 6));
    Assert.assertNotNull(a1in.get(fileId, 0));
}

Example 84 with OLogSequenceNumber

use of com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OLogSequenceNumber in project orientdb by orientechnologies.

the class ReadWriteDiskCacheTest method testPrefetchPagesInPinnedPages.

public void testPrefetchPagesInPinnedPages() throws Exception {
    final long fileId = readBuffer.addFile(fileName, writeBuffer);
    Assert.assertEquals(readBuffer.getMaxSize(), 4);
    // create file with 8 pages, we will push some of them in different queues later
    for (int i = 0; i < 8; i++) {
        OCacheEntry cacheEntry = readBuffer.allocateNewPage(fileId, writeBuffer, true);
        cacheEntry.acquireExclusiveLock();
        try {
            byte[] userData = new byte[userDataSize];
            for (int n = 0; n < userData.length; n++) {
                userData[n] = (byte) (i + 1);
            }
            final ByteBuffer buffer = cacheEntry.getCachePointer().getSharedBuffer();
            buffer.position(systemOffset);
            buffer.put(userData);
            setLsn(buffer, new OLogSequenceNumber(1, i));
            cacheEntry.markDirty();
        } finally {
            cacheEntry.releaseExclusiveLock();
            readBuffer.release(cacheEntry, writeBuffer);
        }
    }
    readBuffer.clear();
    writeBuffer.flush();
    for (int i = 0; i < 8; i++) {
        byte[] userData = new byte[userDataSize];
        for (int n = 0; n < userData.length; n++) {
            userData[n] = (byte) (i + 1);
        }
        assertFile(i, userData, new OLogSequenceNumber(1, i));
    }
    LRUList am = readBuffer.getAm();
    LRUList a1in = readBuffer.getA1in();
    LRUList a1out = readBuffer.getA1out();
    Assert.assertEquals(am.size(), 0);
    Assert.assertEquals(a1out.size(), 0);
    Assert.assertEquals(a1in.size(), 0);
    // put 1 and 2  pages to the a1out queue, page 0 is dropped from buffer
    for (int i = 0; i < 7; i++) {
        OCacheEntry cacheEntry = readBuffer.load(fileId, i, false, writeBuffer, 1, true);
        readBuffer.release(cacheEntry, writeBuffer);
    }
    Assert.assertEquals(am.size(), 0);
    // pages 1 - 2
    Assert.assertEquals(a1out.size(), 2);
    // pages 3 - 6
    Assert.assertEquals(a1in.size(), 4);
    // put 1-th page to the am queue
    OCacheEntry cacheEntry = readBuffer.load(fileId, 1, false, writeBuffer, 1, true);
    readBuffer.release(cacheEntry, writeBuffer);
    // page 1
    Assert.assertEquals(am.size(), 1);
    // page 2 - 3 (removed from a1in because of size limit)
    Assert.assertEquals(a1out.size(), 2);
    // pages 4 - 6
    Assert.assertEquals(a1in.size(), 3);
    // move page 4 to pinned pages
    cacheEntry = readBuffer.load(fileId, 4, false, writeBuffer, 1, true);
    readBuffer.pinPage(cacheEntry);
    readBuffer.release(cacheEntry, writeBuffer);
    // page 1
    Assert.assertEquals(am.size(), 1);
    // page 2 - 3
    Assert.assertEquals(a1out.size(), 2);
    // pages 5 - 6
    Assert.assertEquals(a1in.size(), 2);
    // change cache max size so at least single page may be put in a1in queue
    readBuffer.changeMaximumAmountOfMemory(5 * (PAGE_SIZE));
    // load page 0, and prefetch pages 1 (am), 2,3 (a1out) and 4 (pinned page)
    // so 5 pages in total, but we may only load 4 pages (1 page is pinned),so we have
    // page 0 in a1in (as used page), pages 1, 2, 3 in am, pages 5, 6 in a1out
    cacheEntry = readBuffer.load(fileId, 0, false, writeBuffer, 4, true);
    readBuffer.release(cacheEntry, writeBuffer);
    // page 1,2,3
    Assert.assertEquals(am.size(), 3);
    // pages 5, 6
    Assert.assertEquals(a1out.size(), 2);
    // page 0
    Assert.assertEquals(a1in.size(), 1);
    Assert.assertTrue(readBuffer.inPinnedPages(fileId, 4));
    Assert.assertNotNull(am.get(fileId, 1));
    Assert.assertNotNull(am.get(fileId, 2));
    Assert.assertNotNull(am.get(fileId, 3));
    Assert.assertNotNull(a1out.get(fileId, 5));
    Assert.assertNotNull(a1out.get(fileId, 6));
    Assert.assertNotNull(a1in.get(fileId, 0));
    readBuffer.changeMaximumAmountOfMemory(4 * (PAGE_SIZE));
}

Example 85 with OLogSequenceNumber

use of com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OLogSequenceNumber in project orientdb by orientechnologies.

the class ReadWriteDiskCacheTest method assertFile.

private void assertFile(long pageIndex, byte[] value, OLogSequenceNumber lsn) throws IOException {
    String path = storageLocal.getConfiguration().getDirectory() + "/readWriteDiskCacheTest.tst";
    OFileClassic fileClassic = new OFileClassic(path, "r");
    fileClassic.open();
    byte[] content = new byte[userDataSize + systemOffset];
    fileClassic.read(pageIndex * (userDataSize + systemOffset), content, userDataSize + systemOffset);
    Assert.assertEquals(Arrays.copyOfRange(content, systemOffset, userDataSize + systemOffset), value);
    long magicNumber = OLongSerializer.INSTANCE.deserializeNative(content, 0);
    Assert.assertEquals(magicNumber, OWOWCache.MAGIC_NUMBER_WITH_CHECKSUM);
    CRC32 crc32 = new CRC32();
    crc32.update(content, OIntegerSerializer.INT_SIZE + OLongSerializer.LONG_SIZE, content.length - OIntegerSerializer.INT_SIZE - OLongSerializer.LONG_SIZE);
    int crc = OIntegerSerializer.INSTANCE.deserializeNative(content, OLongSerializer.LONG_SIZE);
    Assert.assertEquals(crc, (int) crc32.getValue());
    long segment = OLongSerializer.INSTANCE.deserializeNative(content, ODurablePage.WAL_SEGMENT_OFFSET);
    long position = OLongSerializer.INSTANCE.deserializeNative(content, ODurablePage.WAL_POSITION_OFFSET);
    OLogSequenceNumber readLsn = new OLogSequenceNumber(segment, position);
    Assert.assertEquals(readLsn, lsn);
    fileClassic.close();
}