Examples with ODurablePage com.orientechnologies.orient.core.storage.impl.local.paginated.base.ODurablePage used on opensource projects

Example 1 with ODurablePage

use of com.orientechnologies.orient.core.storage.impl.local.paginated.base.ODurablePage in project orientdb by orientechnologies.

the class OAtomicOperation method commitChanges.

public void commitChanges(OWriteAheadLog writeAheadLog) throws IOException {
    final OSessionStoragePerformanceStatistic sessionStoragePerformanceStatistic = performanceStatisticManager.getSessionPerformanceStatistic();
    if (sessionStoragePerformanceStatistic != null) {
        sessionStoragePerformanceStatistic.startCommitTimer();
        sessionStoragePerformanceStatistic.startComponentOperation("atomic operation", OSessionStoragePerformanceStatistic.ComponentType.GENERAL);
    }
    try {
        if (writeAheadLog != null) {
            for (long deletedFileId : deletedFiles) {
                writeAheadLog.log(new OFileDeletedWALRecord(operationUnitId, deletedFileId));
            }
            for (Map.Entry<Long, FileChanges> fileChangesEntry : fileChanges.entrySet()) {
                final FileChanges fileChanges = fileChangesEntry.getValue();
                final long fileId = fileChangesEntry.getKey();
                if (fileChanges.isNew)
                    writeAheadLog.log(new OFileCreatedWALRecord(operationUnitId, fileChanges.fileName, fileId));
                else if (fileChanges.truncate)
                    writeAheadLog.log(new OFileTruncatedWALRecord(operationUnitId, fileId));
                Iterator<Map.Entry<Long, FilePageChanges>> filePageChangesIterator = fileChanges.pageChangesMap.entrySet().iterator();
                while (filePageChangesIterator.hasNext()) {
                    Map.Entry<Long, FilePageChanges> filePageChangesEntry = filePageChangesIterator.next();
                    //I assume new pages have everytime changes
                    if (filePageChangesEntry.getValue().changes.hasChanges()) {
                        final long pageIndex = filePageChangesEntry.getKey();
                        final FilePageChanges filePageChanges = filePageChangesEntry.getValue();
                        filePageChanges.lsn = writeAheadLog.log(new OUpdatePageRecord(pageIndex, fileId, operationUnitId, filePageChanges.changes));
                    } else
                        filePageChangesIterator.remove();
                }
            }
        }
        for (long deletedFileId : deletedFiles) {
            readCache.deleteFile(deletedFileId, writeCache);
        }
        for (Map.Entry<Long, FileChanges> fileChangesEntry : fileChanges.entrySet()) {
            final FileChanges fileChanges = fileChangesEntry.getValue();
            final long fileId = fileChangesEntry.getKey();
            if (fileChanges.isNew)
                readCache.addFile(fileChanges.fileName, newFileNamesId.get(fileChanges.fileName), writeCache);
            else if (fileChanges.truncate)
                readCache.truncateFile(fileId, writeCache);
            for (Map.Entry<Long, FilePageChanges> filePageChangesEntry : fileChanges.pageChangesMap.entrySet()) {
                final FilePageChanges filePageChanges = filePageChangesEntry.getValue();
                if (!filePageChanges.changes.hasChanges())
                    continue;
                final long pageIndex = filePageChangesEntry.getKey();
                OCacheEntry cacheEntry = filePageChanges.isNew ? null : pageCache.purgePage(fileId, pageIndex, writeCache);
                if (cacheEntry == null) {
                    cacheEntry = readCache.load(fileId, pageIndex, true, writeCache, 1);
                    if (cacheEntry == null) {
                        assert filePageChanges.isNew;
                        do {
                            if (cacheEntry != null)
                                readCache.release(cacheEntry, writeCache);
                            cacheEntry = readCache.allocateNewPage(fileId, writeCache);
                        } while (cacheEntry.getPageIndex() != pageIndex);
                    }
                }
                cacheEntry.acquireExclusiveLock();
                try {
                    ODurablePage durablePage = new ODurablePage(cacheEntry, null);
                    durablePage.restoreChanges(filePageChanges.changes);
                    if (writeAheadLog != null)
                        durablePage.setLsn(filePageChanges.lsn);
                    if (filePageChanges.pinPage)
                        readCache.pinPage(cacheEntry);
                } finally {
                    cacheEntry.releaseExclusiveLock();
                    readCache.release(cacheEntry, writeCache);
                }
            }
        }
    } finally {
        pageCache.reset(writeCache);
        if (sessionStoragePerformanceStatistic != null) {
            sessionStoragePerformanceStatistic.stopCommitTimer();
            sessionStoragePerformanceStatistic.completeComponentOperation();
        }
    }
}

Example 2 with ODurablePage

use of com.orientechnologies.orient.core.storage.impl.local.paginated.base.ODurablePage in project orientdb by orientechnologies.

the class OSBTreeBonsaiWALTest method restoreDataFromWAL.

private void restoreDataFromWAL() throws IOException {
    ODiskWriteAheadLog log = new ODiskWriteAheadLog(4, -1, 10 * 1024L * OWALPage.PAGE_SIZE, null, true, actualStorage, 10);
    OLogSequenceNumber lsn = log.begin();
    List<OWALRecord> atomicUnit = new ArrayList<OWALRecord>();
    boolean atomicChangeIsProcessed = false;
    while (lsn != null) {
        OWALRecord walRecord = log.read(lsn);
        if (walRecord instanceof OOperationUnitBodyRecord)
            atomicUnit.add(walRecord);
        if (!atomicChangeIsProcessed) {
            if (walRecord instanceof OAtomicUnitStartRecord)
                atomicChangeIsProcessed = true;
        } else if (walRecord instanceof OAtomicUnitEndRecord) {
            atomicChangeIsProcessed = false;
            for (OWALRecord restoreRecord : atomicUnit) {
                if (restoreRecord instanceof OAtomicUnitStartRecord || restoreRecord instanceof OAtomicUnitEndRecord || restoreRecord instanceof ONonTxOperationPerformedWALRecord)
                    continue;
                if (restoreRecord instanceof OFileCreatedWALRecord) {
                    final OFileCreatedWALRecord fileCreatedCreatedRecord = (OFileCreatedWALRecord) restoreRecord;
                    final String fileName = fileCreatedCreatedRecord.getFileName().replace("actualSBTree", "expectedSBTree");
                    if (!expectedWriteCache.exists(fileName))
                        expectedReadCache.addFile(fileName, fileCreatedCreatedRecord.getFileId(), expectedWriteCache);
                } else {
                    final OUpdatePageRecord updatePageRecord = (OUpdatePageRecord) restoreRecord;
                    final long fileId = updatePageRecord.getFileId();
                    final long pageIndex = updatePageRecord.getPageIndex();
                    OCacheEntry cacheEntry = expectedReadCache.load(fileId, pageIndex, true, expectedWriteCache, 1);
                    if (cacheEntry == null) {
                        do {
                            if (cacheEntry != null)
                                expectedReadCache.release(cacheEntry, expectedWriteCache);
                            cacheEntry = expectedReadCache.allocateNewPage(fileId, expectedWriteCache);
                        } while (cacheEntry.getPageIndex() != pageIndex);
                    }
                    cacheEntry.acquireExclusiveLock();
                    try {
                        ODurablePage durablePage = new ODurablePage(cacheEntry, null);
                        durablePage.restoreChanges(updatePageRecord.getChanges());
                        durablePage.setLsn(updatePageRecord.getLsn());
                        cacheEntry.markDirty();
                    } finally {
                        cacheEntry.releaseExclusiveLock();
                        expectedReadCache.release(cacheEntry, expectedWriteCache);
                    }
                }
            }
            atomicUnit.clear();
        } else {
            Assert.assertTrue(walRecord instanceof OUpdatePageRecord || walRecord instanceof ONonTxOperationPerformedWALRecord || walRecord instanceof OFileCreatedWALRecord);
        }
        lsn = log.next(lsn);
    }
    Assert.assertTrue(atomicUnit.isEmpty());
    log.close();
}

Example 3 with ODurablePage

use of com.orientechnologies.orient.core.storage.impl.local.paginated.base.ODurablePage in project orientdb by orientechnologies.

the class SBTreeWALTest method restoreDataFromWAL.

private void restoreDataFromWAL() throws IOException {
    ODiskWriteAheadLog log = new ODiskWriteAheadLog(4, -1, 10 * 1024L * OWALPage.PAGE_SIZE, null, true, actualStorage, 10);
    OLogSequenceNumber lsn = log.begin();
    List<OWALRecord> atomicUnit = new ArrayList<OWALRecord>();
    boolean atomicChangeIsProcessed = false;
    while (lsn != null) {
        OWALRecord walRecord = log.read(lsn);
        if (walRecord instanceof OOperationUnitBodyRecord)
            atomicUnit.add(walRecord);
        if (!atomicChangeIsProcessed) {
            if (walRecord instanceof OAtomicUnitStartRecord)
                atomicChangeIsProcessed = true;
        } else if (walRecord instanceof OAtomicUnitEndRecord) {
            atomicChangeIsProcessed = false;
            for (OWALRecord restoreRecord : atomicUnit) {
                if (restoreRecord instanceof OAtomicUnitStartRecord || restoreRecord instanceof OAtomicUnitEndRecord || restoreRecord instanceof ONonTxOperationPerformedWALRecord)
                    continue;
                if (restoreRecord instanceof OFileCreatedWALRecord) {
                    final OFileCreatedWALRecord fileCreatedCreatedRecord = (OFileCreatedWALRecord) restoreRecord;
                    final String fileName = fileCreatedCreatedRecord.getFileName().replace("actualSBTree", "expectedSBTree");
                    if (!expectedWriteCache.exists(fileName))
                        expectedReadCache.addFile(fileName, fileCreatedCreatedRecord.getFileId(), expectedWriteCache);
                } else {
                    final OUpdatePageRecord updatePageRecord = (OUpdatePageRecord) restoreRecord;
                    final long fileId = updatePageRecord.getFileId();
                    final long pageIndex = updatePageRecord.getPageIndex();
                    OCacheEntry cacheEntry = expectedReadCache.load(fileId, pageIndex, true, expectedWriteCache, 1);
                    if (cacheEntry == null) {
                        do {
                            if (cacheEntry != null)
                                expectedReadCache.release(cacheEntry, expectedWriteCache);
                            cacheEntry = expectedReadCache.allocateNewPage(fileId, expectedWriteCache);
                        } while (cacheEntry.getPageIndex() != pageIndex);
                    }
                    cacheEntry.acquireExclusiveLock();
                    try {
                        ODurablePage durablePage = new ODurablePage(cacheEntry, null);
                        durablePage.restoreChanges(updatePageRecord.getChanges());
                        durablePage.setLsn(updatePageRecord.getLsn());
                        cacheEntry.markDirty();
                    } finally {
                        cacheEntry.releaseExclusiveLock();
                        expectedReadCache.release(cacheEntry, expectedWriteCache);
                    }
                }
            }
            atomicUnit.clear();
        } else {
            Assert.assertTrue(walRecord instanceof OUpdatePageRecord || walRecord instanceof ONonTxOperationPerformedWALRecord || walRecord instanceof OFileCreatedWALRecord);
        }
        lsn = log.next(lsn);
    }
    Assert.assertTrue(atomicUnit.isEmpty());
    log.close();
}

Example 4 with ODurablePage

use of com.orientechnologies.orient.core.storage.impl.local.paginated.base.ODurablePage in project orientdb by orientechnologies.

the class OAbstractPaginatedStorage method restoreAtomicUnit.

protected void restoreAtomicUnit(List<OWALRecord> atomicUnit, OModifiableBoolean atLeastOnePageUpdate) throws IOException {
    assert atomicUnit.get(atomicUnit.size() - 1) instanceof OAtomicUnitEndRecord;
    for (OWALRecord walRecord : atomicUnit) {
        if (walRecord instanceof OFileDeletedWALRecord) {
            OFileDeletedWALRecord fileDeletedWALRecord = (OFileDeletedWALRecord) walRecord;
            if (writeCache.exists(fileDeletedWALRecord.getFileId()))
                readCache.deleteFile(fileDeletedWALRecord.getFileId(), writeCache);
        } else if (walRecord instanceof OFileCreatedWALRecord) {
            OFileCreatedWALRecord fileCreatedCreatedWALRecord = (OFileCreatedWALRecord) walRecord;
            if (!writeCache.exists(fileCreatedCreatedWALRecord.getFileName())) {
                readCache.addFile(fileCreatedCreatedWALRecord.getFileName(), fileCreatedCreatedWALRecord.getFileId(), writeCache);
            }
        } else if (walRecord instanceof OUpdatePageRecord) {
            final OUpdatePageRecord updatePageRecord = (OUpdatePageRecord) walRecord;
            long fileId = updatePageRecord.getFileId();
            if (!writeCache.exists(fileId)) {
                String fileName = writeCache.restoreFileById(fileId);
                if (fileName == null) {
                    throw new OStorageException("File with id " + fileId + " was deleted from storage, the rest of operations can not be restored");
                } else {
                    OLogManager.instance().warn(this, "Previously deleted file with name " + fileName + " was deleted but new empty file was added to continue restore process");
                }
            }
            final long pageIndex = updatePageRecord.getPageIndex();
            fileId = writeCache.externalFileId(writeCache.internalFileId(fileId));
            OCacheEntry cacheEntry = readCache.load(fileId, pageIndex, true, writeCache, 1);
            if (cacheEntry == null) {
                do {
                    if (cacheEntry != null)
                        readCache.release(cacheEntry, writeCache);
                    cacheEntry = readCache.allocateNewPage(fileId, writeCache);
                } while (cacheEntry.getPageIndex() != pageIndex);
            }
            final OCachePointer cachePointer = cacheEntry.getCachePointer();
            cachePointer.acquireExclusiveLock();
            try {
                ODurablePage durablePage = new ODurablePage(cacheEntry, null);
                durablePage.restoreChanges(updatePageRecord.getChanges());
                durablePage.setLsn(updatePageRecord.getLsn());
            } finally {
                cachePointer.releaseExclusiveLock();
                readCache.release(cacheEntry, writeCache);
            }
            atLeastOnePageUpdate.setValue(true);
        } else if (walRecord instanceof OAtomicUnitStartRecord) {
            continue;
        } else if (walRecord instanceof OAtomicUnitEndRecord) {
            continue;
        } else {
            OLogManager.instance().error(this, "Invalid WAL record type was passed %s. Given record will be skipped.", walRecord.getClass());
            assert false : "Invalid WAL record type was passed " + walRecord.getClass().getName();
        }
    }
}

Example 5 with ODurablePage

use of com.orientechnologies.orient.core.storage.impl.local.paginated.base.ODurablePage in project orientdb by orientechnologies.

the class OLocalHashTableWALTest method restoreDataFromBatch.

private boolean restoreDataFromBatch(boolean atomicChangeIsProcessed, List<OWALRecord> atomicUnit, List<OWALRecord> records) throws IOException {
    final OReadCache expectedReadCache = ((OAbstractPaginatedStorage) expectedDatabaseDocumentTx.getStorage()).getReadCache();
    final OWriteCache expectedWriteCache = ((OAbstractPaginatedStorage) expectedDatabaseDocumentTx.getStorage()).getWriteCache();
    for (OWALRecord walRecord : records) {
        if (walRecord instanceof OOperationUnitBodyRecord)
            atomicUnit.add(walRecord);
        if (!atomicChangeIsProcessed && walRecord instanceof OAtomicUnitStartRecord) {
            atomicChangeIsProcessed = true;
        } else if (walRecord instanceof OAtomicUnitEndRecord) {
            atomicChangeIsProcessed = false;
            for (OWALRecord restoreRecord : atomicUnit) {
                if (restoreRecord instanceof OAtomicUnitStartRecord || restoreRecord instanceof OAtomicUnitEndRecord || restoreRecord instanceof ONonTxOperationPerformedWALRecord || restoreRecord instanceof OFullCheckpointStartRecord || restoreRecord instanceof OCheckpointEndRecord)
                    continue;
                if (restoreRecord instanceof OUpdatePageRecord) {
                    final OUpdatePageRecord updatePageRecord = (OUpdatePageRecord) restoreRecord;
                    final long fileId = updatePageRecord.getFileId();
                    final long pageIndex = updatePageRecord.getPageIndex();
                    OCacheEntry cacheEntry = expectedReadCache.load(fileId, pageIndex, true, expectedWriteCache, 1);
                    if (cacheEntry == null)
                        do {
                            cacheEntry = expectedReadCache.allocateNewPage(fileId, expectedWriteCache);
                        } while (cacheEntry.getPageIndex() != pageIndex);
                    cacheEntry.acquireExclusiveLock();
                    try {
                        ODurablePage durablePage = new ODurablePage(cacheEntry, null);
                        durablePage.restoreChanges(updatePageRecord.getChanges());
                        durablePage.setLsn(updatePageRecord.getLsn());
                    } finally {
                        cacheEntry.releaseExclusiveLock();
                        expectedReadCache.release(cacheEntry, expectedWriteCache);
                    }
                } else if (restoreRecord instanceof OFileCreatedWALRecord) {
                    final OFileCreatedWALRecord fileCreatedCreatedRecord = (OFileCreatedWALRecord) restoreRecord;
                    String fileName = fileCreatedCreatedRecord.getFileName().replace("actualLocalHashTable", "expectedLocalHashTable");
                    if (!expectedWriteCache.exists(fileName))
                        expectedReadCache.addFile(fileName, fileCreatedCreatedRecord.getFileId(), expectedWriteCache);
                }
            }
            atomicUnit.clear();
        } else {
            Assert.assertTrue(walRecord instanceof OUpdatePageRecord || walRecord instanceof OFileCreatedWALRecord || walRecord instanceof ONonTxOperationPerformedWALRecord || walRecord instanceof OFullCheckpointStartRecord || walRecord instanceof OCheckpointEndRecord || walRecord instanceof OFuzzyCheckpointStartRecord || walRecord instanceof OFuzzyCheckpointEndRecord);
        }
    }
    return atomicChangeIsProcessed;
}