Examples with WALIterator org.apache.ignite.internal.pagemem.wal.WALIterator used on opensource projects

Example 1 with WALIterator

use of org.apache.ignite.internal.pagemem.wal.WALIterator in project ignite by apache.

the class IgniteWalReaderTest method testFillWalAndReadRecords.

/**
 * @throws Exception if failed.
 */
public void testFillWalAndReadRecords() throws Exception {
    setWalAndArchiveToSameValue = false;
    final int cacheObjectsToWrite = 10000;
    final Ignite ignite0 = startGrid("node0");
    ignite0.active(true);
    final Serializable consistentId = (Serializable) ignite0.cluster().localNode().consistentId();
    final String subfolderName = genNewStyleSubfolderName(0, (UUID) consistentId);
    putDummyRecords(ignite0, cacheObjectsToWrite);
    stopGrid("node0");
    final String workDir = U.defaultWorkDirectory();
    final File db = U.resolveWorkDirectory(workDir, DFLT_STORE_DIR, false);
    final File wal = new File(db, "wal");
    final File walArchive = setWalAndArchiveToSameValue ? wal : new File(wal, "archive");
    final MockWalIteratorFactory mockItFactory = new MockWalIteratorFactory(log, PAGE_SIZE, consistentId, subfolderName, WAL_SEGMENTS);
    final WALIterator it = mockItFactory.iterator(wal, walArchive);
    final int cntUsingMockIter = iterateAndCount(it, false);
    log.info("Total records loaded " + cntUsingMockIter);
    assertTrue(cntUsingMockIter > 0);
    assertTrue(cntUsingMockIter > cacheObjectsToWrite);
    final File walArchiveDirWithConsistentId = new File(walArchive, subfolderName);
    final File walWorkDirWithConsistentId = new File(wal, subfolderName);
    final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
    final int cntArchiveDir = iterateAndCount(factory.iteratorArchiveDirectory(walArchiveDirWithConsistentId));
    log.info("Total records loaded using directory : " + cntArchiveDir);
    final int cntArchiveFileByFile = iterateAndCount(factory.iteratorArchiveFiles(walArchiveDirWithConsistentId.listFiles(FileWriteAheadLogManager.WAL_SEGMENT_FILE_FILTER)));
    log.info("Total records loaded using archive directory (file-by-file): " + cntArchiveFileByFile);
    assertTrue(cntArchiveFileByFile > cacheObjectsToWrite);
    assertTrue(cntArchiveDir > cacheObjectsToWrite);
    assertTrue(cntArchiveDir == cntArchiveFileByFile);
    // really count2 may be less because work dir correct loading is not supported yet
    assertTrue("Mock based reader loaded " + cntUsingMockIter + " records " + "but standalone has loaded only " + cntArchiveDir, cntUsingMockIter >= cntArchiveDir);
    final File[] workFiles = walWorkDirWithConsistentId.listFiles(FileWriteAheadLogManager.WAL_SEGMENT_FILE_FILTER);
    final int cntWork = iterateAndCount(factory.iteratorWorkFiles(workFiles));
    log.info("Total records loaded from work: " + cntWork);
    assertTrue("Work iterator loaded [" + cntWork + "] " + "Archive iterator loaded [" + cntArchiveFileByFile + "]; " + "mock iterator [" + cntUsingMockIter + "]", cntWork + cntArchiveFileByFile == cntUsingMockIter);
}

Example 2 with WALIterator

use of org.apache.ignite.internal.pagemem.wal.WALIterator in project ignite by apache.

the class IgniteWalReaderTest method iterateAndCountDataRecord.

/**
 * Iterates over data records, checks each DataRecord and its entries, finds out all transactions in WAL.
 *
 * @param walIter iterator to use.
 * @return count of data records observed for each global TX ID. Contains null for non tx updates.
 * @throws IgniteCheckedException if failure.
 */
private Map<GridCacheVersion, Integer> iterateAndCountDataRecord(final WALIterator walIter, @Nullable final IgniteBiInClosure<Object, Object> cacheObjHnd, @Nullable final IgniteInClosure<DataRecord> dataRecordHnd) throws IgniteCheckedException {
    final Map<GridCacheVersion, Integer> entriesUnderTxFound = new HashMap<>();
    try (WALIterator stIt = walIter) {
        while (stIt.hasNextX()) {
            final IgniteBiTuple<WALPointer, WALRecord> next = stIt.nextX();
            final WALRecord walRecord = next.get2();
            if (walRecord.type() == WALRecord.RecordType.DATA_RECORD && walRecord instanceof DataRecord) {
                final DataRecord dataRecord = (DataRecord) walRecord;
                if (dataRecordHnd != null)
                    dataRecordHnd.apply(dataRecord);
                final List<DataEntry> entries = dataRecord.writeEntries();
                for (DataEntry entry : entries) {
                    final GridCacheVersion globalTxId = entry.nearXidVersion();
                    Object unwrappedKeyObj;
                    Object unwrappedValObj;
                    if (entry instanceof UnwrapDataEntry) {
                        UnwrapDataEntry unwrapDataEntry = (UnwrapDataEntry) entry;
                        unwrappedKeyObj = unwrapDataEntry.unwrappedKey();
                        unwrappedValObj = unwrapDataEntry.unwrappedValue();
                    } else if (entry instanceof LazyDataEntry) {
                        unwrappedKeyObj = null;
                        unwrappedValObj = null;
                    // can't check value
                    } else {
                        final CacheObject val = entry.value();
                        unwrappedValObj = val instanceof BinaryObject ? val : val.value(null, false);
                        final CacheObject key = entry.key();
                        unwrappedKeyObj = key instanceof BinaryObject ? key : key.value(null, false);
                    }
                    if (dumpRecords)
                        log.info("//Entry operation " + entry.op() + "; cache Id" + entry.cacheId() + "; " + "under transaction: " + globalTxId + // ; entry " + entry +
                        "; Key: " + unwrappedKeyObj + "; Value: " + unwrappedValObj);
                    if (cacheObjHnd != null && (unwrappedKeyObj != null || unwrappedValObj != null))
                        cacheObjHnd.apply(unwrappedKeyObj, unwrappedValObj);
                    final Integer entriesUnderTx = entriesUnderTxFound.get(globalTxId);
                    entriesUnderTxFound.put(globalTxId, entriesUnderTx == null ? 1 : entriesUnderTx + 1);
                }
            } else if (walRecord.type() == WALRecord.RecordType.TX_RECORD && walRecord instanceof TxRecord) {
                final TxRecord txRecord = (TxRecord) walRecord;
                final GridCacheVersion globalTxId = txRecord.nearXidVersion();
                if (dumpRecords)
                    log.info("//Tx Record, state: " + txRecord.state() + "; nearTxVersion" + globalTxId);
            }
        }
    }
    return entriesUnderTxFound;
}

Example 3 with WALIterator

use of org.apache.ignite.internal.pagemem.wal.WALIterator in project ignite by apache.

the class IgniteWalReaderTest method scanIterateAndCount.

/**
 * Scan WAL and WAL archive for logical records and its entries.
 *
 * @param factory WAL iterator factory.
 * @param workDir Ignite work directory.
 * @param subfolderName DB subfolder name based on consistent ID.
 * @param minCntEntries minimum expected entries count to find.
 * @param minTxCnt minimum expected transaction count to find.
 * @param objConsumer object handler, called for each object found in logical data records.
 * @param dataRecordHnd data handler record.
 * @throws IgniteCheckedException if failed.
 */
private void scanIterateAndCount(final IgniteWalIteratorFactory factory, final String workDir, final String subfolderName, final int minCntEntries, final int minTxCnt, @Nullable final IgniteBiInClosure<Object, Object> objConsumer, @Nullable final IgniteInClosure<DataRecord> dataRecordHnd) throws IgniteCheckedException {
    final File db = U.resolveWorkDirectory(workDir, DFLT_STORE_DIR, false);
    final File wal = new File(db, "wal");
    final File walArchive = new File(wal, "archive");
    final File walArchiveDirWithConsistentId = new File(walArchive, subfolderName);
    final File[] files = walArchiveDirWithConsistentId.listFiles(FileWriteAheadLogManager.WAL_SEGMENT_FILE_FILTER);
    A.notNull(files, "Can't iterate over files [" + walArchiveDirWithConsistentId + "] Directory is N/A");
    final WALIterator iter = factory.iteratorArchiveFiles(files);
    final Map<GridCacheVersion, Integer> cntArch = iterateAndCountDataRecord(iter, objConsumer, dataRecordHnd);
    int txCntObservedArch = cntArch.size();
    if (cntArch.containsKey(null))
        // exclude non transactional updates
        txCntObservedArch -= 1;
    final int entriesArch = valuesSum(cntArch.values());
    log.info("Total tx found loaded using archive directory (file-by-file): " + txCntObservedArch);
    final File walWorkDirWithNodeSubDir = new File(wal, subfolderName);
    final File[] workFiles = walWorkDirWithNodeSubDir.listFiles(FileWriteAheadLogManager.WAL_SEGMENT_FILE_FILTER);
    final WALIterator tuples = factory.iteratorWorkFiles(workFiles);
    final Map<GridCacheVersion, Integer> cntWork = iterateAndCountDataRecord(tuples, objConsumer, dataRecordHnd);
    int txCntObservedWork = cntWork.size();
    if (cntWork.containsKey(null))
        // exclude non transactional updates
        txCntObservedWork -= 1;
    final int entriesWork = valuesSum(cntWork.values());
    log.info("Archive directory: Tx found " + txCntObservedWork + " entries " + entriesWork);
    assertTrue("entriesArch=" + entriesArch + " + entriesWork=" + entriesWork + " >= minCntEntries=" + minCntEntries, entriesArch + entriesWork >= minCntEntries);
    assertTrue("txCntObservedWork=" + txCntObservedWork + " + txCntObservedArch=" + txCntObservedArch + " >= minTxCnt=" + minTxCnt, txCntObservedWork + txCntObservedArch >= minTxCnt);
}

Example 4 with WALIterator

use of org.apache.ignite.internal.pagemem.wal.WALIterator in project ignite by apache.

the class PageMemoryImpl method tryToRestorePage.

/**
 * Restores page from WAL page snapshot & delta records.
 *
 * @param fullId Full page ID.
 * @param buf Destination byte buffer. Note: synchronization to provide ByteBuffer safety should be done outside
 * this method.
 *
 * @throws IgniteCheckedException If failed to start WAL iteration, if incorrect page type observed in data, etc.
 * @throws AssertionError if it was not possible to restore page, page not found in WAL.
 */
private void tryToRestorePage(FullPageId fullId, ByteBuffer buf) throws IgniteCheckedException {
    Long tmpAddr = null;
    try {
        ByteBuffer curPage = null;
        ByteBuffer lastValidPage = null;
        try (WALIterator it = walMgr.replay(null)) {
            for (IgniteBiTuple<WALPointer, WALRecord> tuple : it) {
                switch(tuple.getValue().type()) {
                    case PAGE_RECORD:
                        PageSnapshot snapshot = (PageSnapshot) tuple.getValue();
                        if (snapshot.fullPageId().equals(fullId)) {
                            if (tmpAddr == null) {
                                assert snapshot.pageData().length <= pageSize() : snapshot.pageData().length;
                                tmpAddr = GridUnsafe.allocateMemory(pageSize());
                            }
                            if (curPage == null)
                                curPage = wrapPointer(tmpAddr, pageSize());
                            PageUtils.putBytes(tmpAddr, 0, snapshot.pageData());
                        }
                        break;
                    case CHECKPOINT_RECORD:
                        CheckpointRecord rec = (CheckpointRecord) tuple.getValue();
                        assert !rec.end();
                        if (curPage != null) {
                            lastValidPage = curPage;
                            curPage = null;
                        }
                        break;
                    case // It means that previous checkpoint was broken.
                    MEMORY_RECOVERY:
                        curPage = null;
                        break;
                    default:
                        if (tuple.getValue() instanceof PageDeltaRecord) {
                            PageDeltaRecord deltaRecord = (PageDeltaRecord) tuple.getValue();
                            if (curPage != null && deltaRecord.pageId() == fullId.pageId() && deltaRecord.groupId() == fullId.groupId()) {
                                assert tmpAddr != null;
                                deltaRecord.applyDelta(this, tmpAddr);
                            }
                        }
                }
            }
        }
        ByteBuffer restored = curPage == null ? lastValidPage : curPage;
        if (restored == null)
            throw new IllegalStateException(String.format("Page is broken. Can't restore it from WAL. (grpId = %d, pageId = %X).", fullId.groupId(), fullId.pageId()));
        buf.put(restored);
    } finally {
        if (tmpAddr != null)
            GridUnsafe.freeMemory(tmpAddr);
    }
}

Example 5 with WALIterator

use of org.apache.ignite.internal.pagemem.wal.WALIterator in project ignite by apache.

the class GridCacheOffheapManager method historicalIterator.

/**
 * {@inheritDoc}
 */
@Override
@Nullable
protected IgniteHistoricalIterator historicalIterator(CachePartitionPartialCountersMap partCntrs) throws IgniteCheckedException {
    if (partCntrs == null || partCntrs.isEmpty())
        return null;
    GridCacheDatabaseSharedManager database = (GridCacheDatabaseSharedManager) grp.shared().database();
    FileWALPointer minPtr = null;
    for (int i = 0; i < partCntrs.size(); i++) {
        int p = partCntrs.partitionAt(i);
        long initCntr = partCntrs.initialUpdateCounterAt(i);
        FileWALPointer startPtr = (FileWALPointer) database.searchPartitionCounter(grp.groupId(), p, initCntr);
        if (startPtr == null)
            throw new IgniteCheckedException("Could not find start pointer for partition [part=" + p + ", partCntrSince=" + initCntr + "]");
        if (minPtr == null || startPtr.compareTo(minPtr) == -1)
            minPtr = startPtr;
    }
    WALIterator it = grp.shared().wal().replay(minPtr);
    return new WALIteratorAdapter(grp, partCntrs, it);
}