Examples with OModifiableBoolean com.orientechnologies.common.types.OModifiableBoolean used on opensource projects

Example 6 with OModifiableBoolean

use of com.orientechnologies.common.types.OModifiableBoolean in project orientdb by orientechnologies.

the class OAbstractPaginatedStorage method restoreFrom.

protected OLogSequenceNumber restoreFrom(OLogSequenceNumber lsn, OWriteAheadLog writeAheadLog) throws IOException {
    OLogSequenceNumber logSequenceNumber = null;
    OModifiableBoolean atLeastOnePageUpdate = new OModifiableBoolean();
    long recordsProcessed = 0;
    final int reportBatchSize = OGlobalConfiguration.WAL_REPORT_AFTER_OPERATIONS_DURING_RESTORE.getValueAsInteger();
    final Map<OOperationUnitId, List<OWALRecord>> operationUnits = new HashMap<OOperationUnitId, List<OWALRecord>>();
    long lastReportTime = 0;
    try {
        while (lsn != null) {
            logSequenceNumber = lsn;
            OWALRecord walRecord = writeAheadLog.read(lsn);
            if (walRecord instanceof OAtomicUnitEndRecord) {
                OAtomicUnitEndRecord atomicUnitEndRecord = (OAtomicUnitEndRecord) walRecord;
                List<OWALRecord> atomicUnit = operationUnits.remove(atomicUnitEndRecord.getOperationUnitId());
                // in case of data restore from fuzzy checkpoint part of operations may be already flushed to the disk
                if (atomicUnit != null) {
                    atomicUnit.add(walRecord);
                    restoreAtomicUnit(atomicUnit, atLeastOnePageUpdate);
                }
            } else if (walRecord instanceof OAtomicUnitStartRecord) {
                List<OWALRecord> operationList = new ArrayList<OWALRecord>();
                assert !operationUnits.containsKey(((OAtomicUnitStartRecord) walRecord).getOperationUnitId());
                operationUnits.put(((OAtomicUnitStartRecord) walRecord).getOperationUnitId(), operationList);
                operationList.add(walRecord);
            } else if (walRecord instanceof OOperationUnitRecord) {
                OOperationUnitRecord operationUnitRecord = (OOperationUnitRecord) walRecord;
                // in case of data restore from fuzzy checkpoint part of operations may be already flushed to the disk
                List<OWALRecord> operationList = operationUnits.get(operationUnitRecord.getOperationUnitId());
                if (operationList == null) {
                    operationList = new ArrayList<OWALRecord>();
                    operationUnits.put(operationUnitRecord.getOperationUnitId(), operationList);
                }
                operationList.add(operationUnitRecord);
            } else if (walRecord instanceof ONonTxOperationPerformedWALRecord) {
                if (!wereNonTxOperationsPerformedInPreviousOpen) {
                    OLogManager.instance().warn(this, "Non tx operation was used during data modification we will need index rebuild.");
                    wereNonTxOperationsPerformedInPreviousOpen = true;
                }
            } else
                OLogManager.instance().warn(this, "Record %s will be skipped during data restore", walRecord);
            recordsProcessed++;
            final long currentTime = System.currentTimeMillis();
            if (reportBatchSize > 0 && recordsProcessed % reportBatchSize == 0 || currentTime - lastReportTime > WAL_RESTORE_REPORT_INTERVAL) {
                OLogManager.instance().info(this, "%d operations were processed, current LSN is %s last LSN is %s", recordsProcessed, lsn, writeAheadLog.end());
                lastReportTime = currentTime;
            }
            lsn = writeAheadLog.next(lsn);
        }
    } catch (OWALPageBrokenException e) {
        OLogManager.instance().error(this, "Data restore was paused because broken WAL page was found. The rest of changes will be rolled back.");
    } catch (RuntimeException e) {
        OLogManager.instance().error(this, "Data restore was paused because of exception. The rest of changes will be rolled back and WAL files will be backed up." + " Please report issue about this exception to bug tracker and provide WAL files which are backed up in 'wal_backup' directory.");
        backUpWAL(e);
    }
    if (atLeastOnePageUpdate.getValue())
        return logSequenceNumber;
    return null;
}

Example 7 with OModifiableBoolean

use of com.orientechnologies.common.types.OModifiableBoolean in project orientdb by orientechnologies.

the class WOWCacheTest method testDataUpdate.

public void testDataUpdate() throws Exception {
    final NavigableMap<Long, byte[]> pageIndexDataMap = new TreeMap<Long, byte[]>();
    long fileId = wowCache.addFile(fileName);
    Random random = new Random();
    for (int i = 0; i < 600; i++) {
        long pageIndex = random.nextInt(2048);
        byte[] data = new byte[8];
        random.nextBytes(data);
        pageIndexDataMap.put(pageIndex, data);
        final OCachePointer cachePointer = wowCache.load(fileId, pageIndex, 1, true, new OModifiableBoolean())[0];
        cachePointer.acquireExclusiveLock();
        ByteBuffer buffer = cachePointer.getSharedBuffer();
        buffer.position(systemOffset);
        buffer.put(data);
        cachePointer.releaseExclusiveLock();
        wowCache.store(fileId, pageIndex, cachePointer);
        cachePointer.decrementReadersReferrer();
    }
    for (Map.Entry<Long, byte[]> entry : pageIndexDataMap.entrySet()) {
        long pageIndex = entry.getKey();
        byte[] dataOne = entry.getValue();
        OCachePointer cachePointer = wowCache.load(fileId, pageIndex, 1, false, new OModifiableBoolean())[0];
        byte[] dataTwo = new byte[8];
        ByteBuffer buffer = cachePointer.getSharedBuffer();
        buffer.position(systemOffset);
        buffer.get(dataTwo);
        cachePointer.decrementReadersReferrer();
        Assert.assertEquals(dataTwo, dataOne);
    }
    for (int i = 0; i < 300; i++) {
        long desiredIndex = random.nextInt(2048);
        Long pageIndex = pageIndexDataMap.ceilingKey(desiredIndex);
        if (pageIndex == null)
            pageIndex = pageIndexDataMap.floorKey(desiredIndex);
        byte[] data = new byte[8];
        random.nextBytes(data);
        pageIndexDataMap.put(pageIndex, data);
        final OCachePointer cachePointer = wowCache.load(fileId, pageIndex, 1, true, new OModifiableBoolean())[0];
        cachePointer.acquireExclusiveLock();
        ByteBuffer buffer = cachePointer.getSharedBuffer();
        buffer.position(systemOffset);
        buffer.put(data);
        cachePointer.releaseExclusiveLock();
        wowCache.store(fileId, pageIndex, cachePointer);
        cachePointer.decrementReadersReferrer();
    }
    for (Map.Entry<Long, byte[]> entry : pageIndexDataMap.entrySet()) {
        long pageIndex = entry.getKey();
        byte[] dataOne = entry.getValue();
        OCachePointer cachePointer = wowCache.load(fileId, pageIndex, 1, false, new OModifiableBoolean())[0];
        byte[] dataTwo = new byte[8];
        ByteBuffer buffer = cachePointer.getSharedBuffer();
        buffer.position(systemOffset);
        buffer.get(dataTwo);
        cachePointer.decrementReadersReferrer();
        Assert.assertEquals(dataTwo, dataOne);
    }
    wowCache.flush();
    for (Map.Entry<Long, byte[]> entry : pageIndexDataMap.entrySet()) {
        assertFile(entry.getKey(), entry.getValue(), new OLogSequenceNumber(0, 0));
    }
}

Example 8 with OModifiableBoolean

use of com.orientechnologies.common.types.OModifiableBoolean in project orientdb by orientechnologies.

the class O2QCache method doLoad.

private UpdateCacheResult doLoad(long fileId, long pageIndex, boolean checkPinnedPages, boolean addNewPages, OWriteCache writeCache, final int pageCount, final OSessionStoragePerformanceStatistic sessionStoragePerformanceStatistic) throws IOException {
    if (pageCount < 1)
        throw new IllegalArgumentException("Amount of pages to load from cache should be not less than 1 but passed value is " + pageCount);
    boolean removeColdPages = false;
    OCacheEntry cacheEntry = null;
    Lock fileLock;
    Lock[] pageLocks;
    final OModifiableBoolean cacheHit = new OModifiableBoolean(false);
    cacheLock.acquireReadLock();
    try {
        fileLock = fileLockManager.acquireSharedLock(fileId);
        try {
            final PageKey[] pageKeys = new PageKey[pageCount];
            for (int i = 0; i < pageKeys.length; i++) {
                pageKeys[i] = new PageKey(fileId, pageIndex + i);
            }
            pageLocks = pageLockManager.acquireExclusiveLocksInBatch(pageKeys);
            try {
                if (checkPinnedPages)
                    cacheEntry = pinnedPages.get(new PinnedPage(fileId, pageIndex));
                if (cacheEntry == null) {
                    UpdateCacheResult cacheResult = updateCache(fileId, pageIndex, addNewPages, writeCache, pageCount, cacheHit);
                    if (cacheResult == null)
                        return null;
                    cacheEntry = cacheResult.cacheEntry;
                    removeColdPages = cacheResult.removeColdPages;
                } else {
                    cacheHit.setValue(true);
                }
                cacheEntry.incrementUsages();
            } finally {
                for (Lock pageLock : pageLocks) {
                    pageLock.unlock();
                }
            }
        } finally {
            fileLockManager.releaseSharedLock(fileId);
        }
    } finally {
        cacheLock.releaseReadLock();
    }
    if (sessionStoragePerformanceStatistic != null)
        sessionStoragePerformanceStatistic.incrementPageAccessOnCacheLevel(cacheHit.getValue());
    return new UpdateCacheResult(removeColdPages, cacheEntry);
}

Example 9 with OModifiableBoolean

use of com.orientechnologies.common.types.OModifiableBoolean in project orientdb by orientechnologies.

the class O2QCache method restoreQueueWithoutPageLoad.

/**
   * Restores queues state if it is NOT needed to load cache page from disk to cache.
   * <p>
   * Following format is used to store queue state:
   * <p>
   * <ol>
   * <li>File id or -1 if end of queue is reached (int)</li>
   * <li>Page index (long), is absent if end of the queue is reached</li>
   * </ol>
   *
   * @param dataInputStream Stream of file which contains state of the cache.
   * @param queue           Queue, state of which should be restored.
   * @param writeCache      Write cache is used to load data from disk if needed.
   */
private void restoreQueueWithoutPageLoad(OWriteCache writeCache, LRUList queue, DataInputStream dataInputStream) throws IOException {
    // used only for statistics, and there is passed merely as stub
    final OModifiableBoolean cacheHit = new OModifiableBoolean();
    int internalFileId = dataInputStream.readInt();
    while (internalFileId >= 0) {
        final long pageIndex = dataInputStream.readLong();
        try {
            final long fileId = writeCache.externalFileId(internalFileId);
            final OCacheEntry cacheEntry = new OCacheEntry(fileId, pageIndex, null, false);
            Set<Long> pages = filePages.get(fileId);
            if (pages == null) {
                pages = new HashSet<Long>();
                Set<Long> op = filePages.putIfAbsent(fileId, pages);
                if (op != null) {
                    pages = op;
                }
            }
            queue.putToMRU(cacheEntry);
            pages.add(cacheEntry.getPageIndex());
        } finally {
            internalFileId = dataInputStream.readInt();
        }
    }
}

Example 10 with OModifiableBoolean

use of com.orientechnologies.common.types.OModifiableBoolean in project orientdb by orientechnologies.

the class O2QCache method restoreQueueWithPageLoad.

/**
   * Restores queues state if it is needed to load cache page from disk to cache.
   * <p>
   * Following format is used to store queue state:
   * <p>
   * <ol>
   * <li>File id or -1 if end of queue is reached (int)</li>
   * <li>Page index (long), is absent if end of the queue is reached</li>
   * </ol>
   *
   * @param dataInputStream Stream of file which contains state of the cache.
   * @param queue           Queue, state of which should be restored.
   * @param writeCache      Write cache is used to load data from disk if needed.
   */
private void restoreQueueWithPageLoad(OWriteCache writeCache, LRUList queue, DataInputStream dataInputStream) throws IOException {
    // used only for statistics, and there is passed merely as stub
    final OModifiableBoolean cacheHit = new OModifiableBoolean();
    // first step, we will create two tree maps to sort data by position in file and load them with maximum speed and
    // then to put data into the queue to restore position of entries in LRU list.
    final TreeMap<PageKey, OPair<Long, OCacheEntry>> filePositionMap = new TreeMap<PageKey, OPair<Long, OCacheEntry>>();
    final TreeMap<Long, OCacheEntry> queuePositionMap = new TreeMap<Long, OCacheEntry>();
    long position = 0;
    int internalFileId = dataInputStream.readInt();
    while (internalFileId >= 0) {
        final long pageIndex = dataInputStream.readLong();
        try {
            final long fileId = writeCache.externalFileId(internalFileId);
            final OCacheEntry entry = new OCacheEntry(fileId, pageIndex, null, false);
            filePositionMap.put(new PageKey(fileId, pageIndex), new OPair<Long, OCacheEntry>(position, entry));
            queuePositionMap.put(position, entry);
            position++;
        } finally {
            internalFileId = dataInputStream.readInt();
        }
    }
    // second step: load pages sorted by position in file
    for (final Map.Entry<PageKey, OPair<Long, OCacheEntry>> entry : filePositionMap.entrySet()) {
        final PageKey pageKey = entry.getKey();
        final OPair<Long, OCacheEntry> pair = entry.getValue();
        final OCachePointer[] pointers = writeCache.load(pageKey.fileId, pageKey.pageIndex, 1, false, cacheHit);
        if (pointers.length == 0) {
            queuePositionMap.remove(pair.key);
            continue;
        }
        final OCacheEntry cacheEntry = pair.value;
        cacheEntry.setCachePointer(pointers[0]);
    }
    // third step: add pages according to their order in LRU queue
    for (final OCacheEntry cacheEntry : queuePositionMap.values()) {
        final long fileId = cacheEntry.getFileId();
        Set<Long> pages = filePages.get(fileId);
        if (pages == null) {
            pages = new HashSet<Long>();
            Set<Long> op = filePages.putIfAbsent(fileId, pages);
            if (op != null) {
                pages = op;
            }
        }
        queue.putToMRU(cacheEntry);
        pages.add(cacheEntry.getPageIndex());
    }
}