Examples with WALPointer org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer used on opensource projects

Example 51 with WALPointer

use of org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer in project ignite by apache.

the class IgniteWithoutArchiverWalIteratorInvalidCrcTest method nodeShouldStartIfLogicalRecordCorruptedAfterCheckpointOrWalStart.

/**
 *  A logical record was corrupted or just doesn't exist because the end of wal is reached, after start checkpoint without end.
 *  -----||------||----X----> OR ----X----->
 *  We recover all before it, and start the node.
 */
@Test
@WithSystemProperty(key = GridCacheDatabaseSharedManager.IGNITE_PDS_SKIP_CHECKPOINT_ON_NODE_STOP, value = "true")
public void nodeShouldStartIfLogicalRecordCorruptedAfterCheckpointOrWalStart() throws Exception {
    startNodeAndPopulate();
    stopGrid(0);
    IgniteWriteAheadLogManager walMgr = ignite.context().cache().context().wal();
    File walDir = U.field(walMgr, "walWorkDir");
    IgniteWalIteratorFactory iterFactory = new IgniteWalIteratorFactory();
    List<FileDescriptor> walFiles = getWalFiles(walDir, iterFactory);
    FileDescriptor lastWalFile = walFiles.get(walFiles.size() - 1);
    List<WALPointer> pointers = WalTestUtils.getPointers(lastWalFile, iterFactory, LOGICAL);
    WalTestUtils.corruptWalSegmentFile(lastWalFile, pointers.get(pointers.size() - 1));
    IgniteEx ex = startGrid(0);
    ex.cluster().active(true);
}

Example 52 with WALPointer

use of org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer in project ignite by apache.

the class IgniteWalReaderTest method iterateAndCount.

/**
 * Iterates on records and closes iterator.
 *
 * @param walIter iterator to count, will be closed.
 * @return count of records.
 * @throws IgniteCheckedException if failed to iterate.
 */
private int iterateAndCount(WALIterator walIter) throws IgniteCheckedException {
    int cnt = 0;
    try (WALIterator it = walIter) {
        while (it.hasNextX()) {
            IgniteBiTuple<WALPointer, WALRecord> tup = it.nextX();
            WALRecord walRecord = tup.get2();
            if (walRecord.type() == DATA_RECORD_V2 || walRecord.type() == MVCC_DATA_RECORD) {
                DataRecord record = (DataRecord) walRecord;
                for (int i = 0; i < record.entryCount(); i++) {
                    DataEntry entry = record.get(i);
                    KeyCacheObject key = entry.key();
                    CacheObject val = entry.value();
                    if (walRecord.type() == DATA_RECORD_V2) {
                        assertEquals(primary, (entry.flags() & DataEntry.PRIMARY_FLAG) != 0);
                        assertEquals(rebalance, (entry.flags() & DataEntry.PRELOAD_FLAG) != 0);
                    }
                    if (DUMP_RECORDS)
                        log.info("Op: " + entry.op() + ", Key: " + key + ", Value: " + val);
                }
            }
            if (DUMP_RECORDS)
                log.info("Record: " + walRecord);
            cnt++;
        }
    }
    return cnt;
}

Example 53 with WALPointer

use of org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer in project ignite by apache.

the class IgniteWalReaderTest method iterateAndCountDataRecord.

/**
 * Iterates over data records, checks each DataRecord and its entries, finds out all transactions in WAL.
 *
 * @param walIter iterator to use.
 * @return count of data records observed for each global TX ID. Contains null for non tx updates.
 * @throws IgniteCheckedException if failure.
 */
private Map<GridCacheVersion, Integer> iterateAndCountDataRecord(WALIterator walIter, @Nullable IgniteBiInClosure<Object, Object> cacheObjHnd, @Nullable IgniteInClosure<DataRecord> dataRecordHnd) throws IgniteCheckedException {
    Map<GridCacheVersion, Integer> entriesUnderTxFound = new HashMap<>();
    try (WALIterator stIt = walIter) {
        while (stIt.hasNextX()) {
            IgniteBiTuple<WALPointer, WALRecord> tup = stIt.nextX();
            WALRecord walRecord = tup.get2();
            WALRecord.RecordType type = walRecord.type();
            // noinspection EnumSwitchStatementWhichMissesCases
            switch(type) {
                case DATA_RECORD_V2:
                // Fallthrough.
                case MVCC_DATA_RECORD:
                    {
                        assert walRecord instanceof DataRecord;
                        DataRecord dataRecord = (DataRecord) walRecord;
                        if (dataRecordHnd != null)
                            dataRecordHnd.apply(dataRecord);
                        for (int i = 0; i < dataRecord.entryCount(); i++) {
                            DataEntry entry = dataRecord.get(i);
                            if (walRecord.type() == DATA_RECORD_V2) {
                                assertEquals(primary, (entry.flags() & DataEntry.PRIMARY_FLAG) != 0);
                                assertEquals(rebalance, (entry.flags() & DataEntry.PRELOAD_FLAG) != 0);
                            }
                            GridCacheVersion globalTxId = entry.nearXidVersion();
                            Object unwrappedKeyObj;
                            Object unwrappedValObj;
                            if (entry instanceof UnwrappedDataEntry) {
                                UnwrappedDataEntry unwrapDataEntry = (UnwrappedDataEntry) entry;
                                unwrappedKeyObj = unwrapDataEntry.unwrappedKey();
                                unwrappedValObj = unwrapDataEntry.unwrappedValue();
                            } else if (entry instanceof MarshalledDataEntry) {
                                unwrappedKeyObj = null;
                                unwrappedValObj = null;
                            // can't check value
                            } else {
                                final CacheObject val = entry.value();
                                unwrappedValObj = val instanceof BinaryObject ? val : val.value(null, false);
                                final CacheObject key = entry.key();
                                unwrappedKeyObj = key instanceof BinaryObject ? key : key.value(null, false);
                            }
                            if (DUMP_RECORDS)
                                log.info("//Entry operation " + entry.op() + "; cache Id" + entry.cacheId() + "; " + "under transaction: " + globalTxId + // ; entry " + entry +
                                "; Key: " + unwrappedKeyObj + "; Value: " + unwrappedValObj);
                            if (cacheObjHnd != null && (unwrappedKeyObj != null || unwrappedValObj != null))
                                cacheObjHnd.apply(unwrappedKeyObj, unwrappedValObj);
                            Integer entriesUnderTx = entriesUnderTxFound.get(globalTxId);
                            entriesUnderTxFound.put(globalTxId, entriesUnderTx == null ? 1 : entriesUnderTx + 1);
                        }
                    }
                    break;
                case TX_RECORD:
                // Fallthrough
                case MVCC_TX_RECORD:
                    {
                        assert walRecord instanceof TxRecord;
                        TxRecord txRecord = (TxRecord) walRecord;
                        GridCacheVersion globalTxId = txRecord.nearXidVersion();
                        if (DUMP_RECORDS)
                            log.info("//Tx Record, state: " + txRecord.state() + "; nearTxVersion" + globalTxId);
                    }
            }
        }
    }
    return entriesUnderTxFound;
}

Example 54 with WALPointer

use of org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer in project ignite by apache.

the class WalOnNodeStartTest method testNoNewMetaPagesSnapshotsOnNodeStart.

/**
 */
@Test
public void testNoNewMetaPagesSnapshotsOnNodeStart() throws Exception {
    IgniteEx ignite = startGrid(0);
    ignite.cluster().state(ClusterState.ACTIVE);
    // Default cache with a lot of partitions required.
    IgniteCache<Object, Object> cache = ignite.getOrCreateCache(DEFAULT_CACHE_NAME);
    for (int k = 0; k < 1024; k++) cache.put(k, k);
    // Graceful caches shutdown with the final checkpoint.
    ignite.cluster().state(ClusterState.INACTIVE);
    WALPointer lastWalPtr = ignite.context().cache().context().database().lastCheckpointMarkWalPointer();
    stopGrid(0);
    ignite = startGrid(0);
    awaitPartitionMapExchange();
    ignite.cluster().state(ClusterState.INACTIVE);
    String walPath = ignite.configuration().getDataStorageConfiguration().getWalPath();
    String walArchivePath = ignite.configuration().getDataStorageConfiguration().getWalArchivePath();
    // Stop grid so there are no ongoing wal records (BLT update and something else maybe).
    stopGrid(0);
    WALIterator replayIter = new IgniteWalIteratorFactory(log).iterator(lastWalPtr.next(), new File(walArchivePath), new File(walPath));
    replayIter.forEach(walPtrAndRecordPair -> {
        WALRecord walRecord = walPtrAndRecordPair.getValue();
        if (walRecord.type() == WALRecord.RecordType.PAGE_RECORD) {
            PageSnapshot pageSnapshot = (PageSnapshot) walRecord;
            ByteBuffer data = pageSnapshot.pageDataBuffer();
            // No metapages should be present in WAL because they all were in correct states already.
            assertThat(PageIO.T_PART_META, not(equalTo(PageIO.getType(data))));
        }
    });
}

Example 55 with WALPointer

use of org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer in project ignite by apache.

the class WalRecoveryTxLogicalRecordsTest method testHistoricalRebalanceIterator.

/**
 * @throws Exception if failed.
 */
@Test
public void testHistoricalRebalanceIterator() throws Exception {
    System.setProperty(IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD, "0");
    extraCcfg = new CacheConfiguration(CACHE_NAME + "2");
    extraCcfg.setAffinity(new RendezvousAffinityFunction(false, PARTS));
    Ignite ignite = startGrid();
    try {
        ignite.cluster().active(true);
        GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) ((IgniteEx) ignite).context().cache().context().database();
        dbMgr.waitForCheckpoint("test");
        // This number depends on wal history size.
        int entries = 25;
        IgniteCache<Integer, Integer> cache = ignite.cache(CACHE_NAME);
        IgniteCache<Integer, Integer> cache2 = ignite.cache(CACHE_NAME + "2");
        for (int i = 0; i < entries; i++) {
            // Put to partition 0.
            cache.put(i * PARTS, i * PARTS);
            // Put to partition 1.
            cache.put(i * PARTS + 1, i * PARTS + 1);
            // Put to another cache.
            cache2.put(i, i);
            dbMgr.waitForCheckpoint("test");
        }
        for (int i = 0; i < entries; i++) {
            assertEquals((Integer) (i * PARTS), cache.get(i * PARTS));
            assertEquals((Integer) (i * PARTS + 1), cache.get(i * PARTS + 1));
            assertEquals((Integer) (i), cache2.get(i));
        }
        CacheGroupContext grp = ((IgniteEx) ignite).context().cache().cacheGroup(CU.cacheId(CACHE_NAME));
        IgniteCacheOffheapManager offh = grp.offheap();
        AffinityTopologyVersion topVer = grp.affinity().lastVersion();
        IgniteDhtDemandedPartitionsMap map;
        for (int i = 0; i < entries; i++) {
            map = new IgniteDhtDemandedPartitionsMap();
            map.addHistorical(0, i, entries, PARTS);
            WALPointer ptr = reserveWalPointerForIterator(grp.shared());
            try (IgniteRebalanceIterator it = offh.rebalanceIterator(map, topVer)) {
                assertNotNull(it);
                assertTrue("Not historical for iteration: " + i, it.historical(0));
                for (int j = i; j < entries; j++) {
                    assertTrue("i=" + i + ", j=" + j, it.hasNextX());
                    CacheDataRow row = it.next();
                    assertEquals(j * PARTS, (int) row.key().value(grp.cacheObjectContext(), false));
                    assertEquals(j * PARTS, (int) row.value().value(grp.cacheObjectContext(), false));
                }
                assertFalse(it.hasNext());
            } finally {
                releaseWalPointerForIterator(grp.shared(), ptr);
            }
            map = new IgniteDhtDemandedPartitionsMap();
            map.addHistorical(1, i, entries, PARTS);
            ptr = reserveWalPointerForIterator(grp.shared());
            try (IgniteRebalanceIterator it = offh.rebalanceIterator(map, topVer)) {
                assertNotNull(it);
                assertTrue("Not historical for iteration: " + i, it.historical(1));
                for (int j = i; j < entries; j++) {
                    assertTrue(it.hasNextX());
                    CacheDataRow row = it.next();
                    assertEquals(j * PARTS + 1, (int) row.key().value(grp.cacheObjectContext(), false));
                    assertEquals(j * PARTS + 1, (int) row.value().value(grp.cacheObjectContext(), false));
                }
                assertFalse(it.hasNext());
            } finally {
                releaseWalPointerForIterator(grp.shared(), ptr);
            }
        }
        stopAllGrids();
        // Check that iterator is valid after restart.
        ignite = startGrid();
        ignite.cluster().active(true);
        grp = ((IgniteEx) ignite).context().cache().cacheGroup(CU.cacheId(CACHE_NAME));
        offh = grp.offheap();
        topVer = grp.affinity().lastVersion();
        for (int i = 0; i < entries; i++) {
            long start = System.currentTimeMillis();
            map = new IgniteDhtDemandedPartitionsMap();
            map.addHistorical(0, i, entries, PARTS);
            WALPointer ptr = reserveWalPointerForIterator(grp.shared());
            try (IgniteRebalanceIterator it = offh.rebalanceIterator(map, topVer)) {
                long end = System.currentTimeMillis();
                info("Time to get iterator: " + (end - start));
                assertTrue("Not historical for iteration: " + i, it.historical(0));
                assertNotNull(it);
                start = System.currentTimeMillis();
                for (int j = i; j < entries; j++) {
                    assertTrue("i=" + i + ", j=" + j, it.hasNextX());
                    CacheDataRow row = it.next();
                    assertEquals(j * PARTS, (int) row.key().value(grp.cacheObjectContext(), false));
                    assertEquals(j * PARTS, (int) row.value().value(grp.cacheObjectContext(), false));
                }
                end = System.currentTimeMillis();
                info("Time to iterate: " + (end - start));
                assertFalse(it.hasNext());
            } finally {
                releaseWalPointerForIterator(grp.shared(), ptr);
            }
            map = new IgniteDhtDemandedPartitionsMap();
            map.addHistorical(1, i, entries, PARTS);
            ptr = reserveWalPointerForIterator(grp.shared());
            try (IgniteRebalanceIterator it = offh.rebalanceIterator(map, topVer)) {
                assertNotNull(it);
                assertTrue("Not historical for iteration: " + i, it.historical(1));
                for (int j = i; j < entries; j++) {
                    assertTrue(it.hasNextX());
                    CacheDataRow row = it.next();
                    assertEquals(j * PARTS + 1, (int) row.key().value(grp.cacheObjectContext(), false));
                    assertEquals(j * PARTS + 1, (int) row.value().value(grp.cacheObjectContext(), false));
                }
                assertFalse(it.hasNext());
            } finally {
                releaseWalPointerForIterator(grp.shared(), ptr);
            }
        }
    } finally {
        stopAllGrids();
        System.clearProperty(IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD);
    }
}