Search in sources :

Example 16 with PageSnapshot

use of org.apache.ignite.internal.pagemem.wal.record.PageSnapshot in project ignite by apache.

the class WalScannerTest method shouldDumpToLogFoundRecord.

/**
 * @throws Exception If failed.
 */
@Test
public void shouldDumpToLogFoundRecord() throws Exception {
    // given: Test logger for interception of logging.
    long expPageId = 984;
    int grpId = 123;
    IgniteLogger log = mock(IgniteLogger.class);
    when(log.isInfoEnabled()).thenReturn(true);
    ArgumentCaptor<String> valCapture = ArgumentCaptor.forClass(String.class);
    doNothing().when(log).info(valCapture.capture());
    WALIterator mockedIter = mockWalIterator(new IgniteBiTuple<>(NULL_PTR, new PageSnapshot(new FullPageId(expPageId, grpId), dummyPage(1024, expPageId), 1024)), new IgniteBiTuple<>(NULL_PTR, new CheckpointRecord(new WALPointer(5738, 0, 0))), new IgniteBiTuple<>(NULL_PTR, new FixCountRecord(grpId, expPageId, 4)));
    IgniteWalIteratorFactory factory = mock(IgniteWalIteratorFactory.class);
    when(factory.iterator(any(IteratorParametersBuilder.class))).thenReturn(mockedIter);
    Set<T2<Integer, Long>> groupAndPageIds = new HashSet<>();
    groupAndPageIds.add(new T2<>(grpId, expPageId));
    // when: Scanning WAL for searching expected page.
    buildWalScanner(withIteratorParameters(), factory).findAllRecordsFor(groupAndPageIds).forEach(printToLog(log));
    // then: Should be find only expected value from log.
    List<String> actualRecords = valCapture.getAllValues();
    assertEquals(actualRecords.size(), 1);
    assertRecord(actualRecords.get(0), "PageSnapshot [", "PAGE_RECORD");
    assertRecord(actualRecords.get(0), "CheckpointRecord [", "CHECKPOINT_RECORD");
    assertRecord(actualRecords.get(0), "FixCountRecord [", "BTREE_FIX_COUNT");
}
Also used : IgniteWalIteratorFactory(org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory) IteratorParametersBuilder(org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder) CheckpointRecord(org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord) FixCountRecord(org.apache.ignite.internal.pagemem.wal.record.delta.FixCountRecord) WALIterator(org.apache.ignite.internal.pagemem.wal.WALIterator) IgniteLogger(org.apache.ignite.IgniteLogger) WALPointer(org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer) T2(org.apache.ignite.internal.util.typedef.T2) PageSnapshot(org.apache.ignite.internal.pagemem.wal.record.PageSnapshot) FullPageId(org.apache.ignite.internal.pagemem.FullPageId) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 17 with PageSnapshot

use of org.apache.ignite.internal.pagemem.wal.record.PageSnapshot in project ignite by apache.

the class WalScannerTest method shouldDumpToFileFoundRecord.

/**
 * @throws Exception If failed.
 */
@Test
public void shouldDumpToFileFoundRecord() throws Exception {
    // given: File for dumping records.
    File targetFile = Paths.get(U.defaultWorkDirectory(), TEST_DUMP_FILE).toFile();
    long expectedPageId = 984;
    int grpId = 123;
    WALIterator mockedIter = mockWalIterator(new IgniteBiTuple<>(NULL_PTR, new PageSnapshot(new FullPageId(expectedPageId, grpId), dummyPage(1024, expectedPageId), 1024)), new IgniteBiTuple<>(NULL_PTR, new CheckpointRecord(new WALPointer(5738, 0, 0))), new IgniteBiTuple<>(NULL_PTR, new FixCountRecord(grpId, expectedPageId, 4)));
    IgniteWalIteratorFactory factory = mock(IgniteWalIteratorFactory.class);
    when(factory.iterator(any(IteratorParametersBuilder.class))).thenReturn(mockedIter);
    Set<T2<Integer, Long>> groupAndPageIds = new HashSet<>();
    groupAndPageIds.add(new T2<>(grpId, expectedPageId));
    List<String> actualRecords;
    try {
        // when: Scanning WAL for searching expected page.
        buildWalScanner(withIteratorParameters(), factory).findAllRecordsFor(groupAndPageIds).forEach(printToFile(targetFile));
        actualRecords = Files.readAllLines(targetFile.toPath());
    } finally {
        targetFile.delete();
    }
    // then: Should be find only expected value from file. PageSnapshot string representation is 11 lines long.
    assertEquals(13, actualRecords.size());
    assertTrue(actualRecords.get(0), actualRecords.get(0).contains("PageSnapshot ["));
    assertTrue(actualRecords.get(11), actualRecords.get(11).contains("CheckpointRecord ["));
    assertTrue(actualRecords.get(12), actualRecords.get(12).contains("FixCountRecord ["));
}
Also used : IgniteWalIteratorFactory(org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory) IteratorParametersBuilder(org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder) CheckpointRecord(org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord) FixCountRecord(org.apache.ignite.internal.pagemem.wal.record.delta.FixCountRecord) WALIterator(org.apache.ignite.internal.pagemem.wal.WALIterator) ScannerHandlers.printToFile(org.apache.ignite.internal.processors.cache.persistence.wal.scanner.ScannerHandlers.printToFile) File(java.io.File) WALPointer(org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer) T2(org.apache.ignite.internal.util.typedef.T2) PageSnapshot(org.apache.ignite.internal.pagemem.wal.record.PageSnapshot) FullPageId(org.apache.ignite.internal.pagemem.FullPageId) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 18 with PageSnapshot

use of org.apache.ignite.internal.pagemem.wal.record.PageSnapshot in project ignite by apache.

the class RecordDataV1Serializer method size.

/**
 * {@inheritDoc}
 */
@Override
public int size(WALRecord record) throws IgniteCheckedException {
    switch(record.type()) {
        case PAGE_RECORD:
            assert record instanceof PageSnapshot;
            PageSnapshot pageRec = (PageSnapshot) record;
            return pageRec.pageData().length + 12;
        case CHECKPOINT_RECORD:
            CheckpointRecord cpRec = (CheckpointRecord) record;
            assert cpRec.checkpointMark() == null || cpRec.checkpointMark() instanceof FileWALPointer : "Invalid WAL record: " + cpRec;
            int cacheStatesSize = cacheStatesSize(cpRec.cacheGroupStates());
            FileWALPointer walPtr = (FileWALPointer) cpRec.checkpointMark();
            return 18 + cacheStatesSize + (walPtr == null ? 0 : 16);
        case META_PAGE_INIT:
            return /*cache ID*/
            4 + /*page ID*/
            8 + /*ioType*/
            2 + /*ioVer*/
            2 + /*tree root*/
            8 + /*reuse root*/
            8;
        case PARTITION_META_PAGE_UPDATE_COUNTERS:
            return /*cache ID*/
            4 + /*page ID*/
            8 + /*upd cntr*/
            8 + /*rmv id*/
            8 + /*part size*/
            4 + /*counters page id*/
            8 + /*state*/
            1 + /*allocatedIdxCandidate*/
            4;
        case MEMORY_RECOVERY:
            return 8;
        case PARTITION_DESTROY:
            return /*cacheId*/
            4 + /*partId*/
            4;
        case DATA_RECORD:
            DataRecord dataRec = (DataRecord) record;
            return 4 + dataSize(dataRec);
        case METASTORE_DATA_RECORD:
            MetastoreDataRecord metastoreDataRec = (MetastoreDataRecord) record;
            return 4 + metastoreDataRec.key().getBytes().length + 4 + (metastoreDataRec.value() != null ? metastoreDataRec.value().length : 0);
        case HEADER_RECORD:
            return HEADER_RECORD_DATA_SIZE;
        case DATA_PAGE_INSERT_RECORD:
            DataPageInsertRecord diRec = (DataPageInsertRecord) record;
            return 4 + 8 + 2 + diRec.payload().length;
        case DATA_PAGE_UPDATE_RECORD:
            DataPageUpdateRecord uRec = (DataPageUpdateRecord) record;
            return 4 + 8 + 2 + 4 + uRec.payload().length;
        case DATA_PAGE_INSERT_FRAGMENT_RECORD:
            final DataPageInsertFragmentRecord difRec = (DataPageInsertFragmentRecord) record;
            return 4 + 8 + 8 + 4 + difRec.payloadSize();
        case DATA_PAGE_REMOVE_RECORD:
            return 4 + 8 + 1;
        case DATA_PAGE_SET_FREE_LIST_PAGE:
            return 4 + 8 + 8;
        case INIT_NEW_PAGE_RECORD:
            return 4 + 8 + 2 + 2 + 8;
        case BTREE_META_PAGE_INIT_ROOT:
            return 4 + 8 + 8;
        case BTREE_META_PAGE_INIT_ROOT2:
            return 4 + 8 + 8 + 2;
        case BTREE_META_PAGE_ADD_ROOT:
            return 4 + 8 + 8;
        case BTREE_META_PAGE_CUT_ROOT:
            return 4 + 8;
        case BTREE_INIT_NEW_ROOT:
            NewRootInitRecord<?> riRec = (NewRootInitRecord<?>) record;
            return 4 + 8 + 8 + 2 + 2 + 8 + 8 + riRec.io().getItemSize();
        case BTREE_PAGE_RECYCLE:
            return 4 + 8 + 8;
        case BTREE_PAGE_INSERT:
            InsertRecord<?> inRec = (InsertRecord<?>) record;
            return 4 + 8 + 2 + 2 + 2 + 8 + inRec.io().getItemSize();
        case BTREE_FIX_LEFTMOST_CHILD:
            return 4 + 8 + 8;
        case BTREE_FIX_COUNT:
            return 4 + 8 + 2;
        case BTREE_PAGE_REPLACE:
            ReplaceRecord<?> rRec = (ReplaceRecord<?>) record;
            return 4 + 8 + 2 + 2 + 2 + rRec.io().getItemSize();
        case BTREE_PAGE_REMOVE:
            return 4 + 8 + 2 + 2;
        case BTREE_PAGE_INNER_REPLACE:
            return 4 + 8 + 2 + 8 + 2 + 8;
        case BTREE_FORWARD_PAGE_SPLIT:
            return 4 + 8 + 8 + 2 + 2 + 8 + 2 + 2;
        case BTREE_EXISTING_PAGE_SPLIT:
            return 4 + 8 + 2 + 8;
        case BTREE_PAGE_MERGE:
            return 4 + 8 + 8 + 2 + 8 + 1;
        case BTREE_FIX_REMOVE_ID:
            return 4 + 8 + 8;
        case PAGES_LIST_SET_NEXT:
            return 4 + 8 + 8;
        case PAGES_LIST_SET_PREVIOUS:
            return 4 + 8 + 8;
        case PAGES_LIST_INIT_NEW_PAGE:
            return 4 + 8 + 4 + 4 + 8 + 8 + 8;
        case PAGES_LIST_ADD_PAGE:
            return 4 + 8 + 8;
        case PAGES_LIST_REMOVE_PAGE:
            return 4 + 8 + 8;
        case TRACKING_PAGE_DELTA:
            return 4 + 8 + 8 + 8 + 8;
        case META_PAGE_UPDATE_LAST_SUCCESSFUL_SNAPSHOT_ID:
            return 4 + 8 + 8 + 8;
        case META_PAGE_UPDATE_LAST_SUCCESSFUL_FULL_SNAPSHOT_ID:
            return 4 + 8 + 8;
        case META_PAGE_UPDATE_NEXT_SNAPSHOT_ID:
            return 4 + 8 + 8;
        case META_PAGE_UPDATE_LAST_ALLOCATED_INDEX:
            return 4 + 8 + 4;
        case PART_META_UPDATE_STATE:
            return /*cacheId*/
            4 + /*partId*/
            4 + /*State*/
            1 + /*Update Counter*/
            8;
        case PAGE_LIST_META_RESET_COUNT_RECORD:
            return /*cacheId*/
            4 + /*pageId*/
            8;
        case SWITCH_SEGMENT_RECORD:
            return 0;
        case TX_RECORD:
            return txRecordSerializer.size((TxRecord) record);
        default:
            throw new UnsupportedOperationException("Type: " + record.type());
    }
}
Also used : DataPageInsertFragmentRecord(org.apache.ignite.internal.pagemem.wal.record.delta.DataPageInsertFragmentRecord) FileWALPointer(org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer) NewRootInitRecord(org.apache.ignite.internal.pagemem.wal.record.delta.NewRootInitRecord) DataPageInsertRecord(org.apache.ignite.internal.pagemem.wal.record.delta.DataPageInsertRecord) InsertRecord(org.apache.ignite.internal.pagemem.wal.record.delta.InsertRecord) DataPageUpdateRecord(org.apache.ignite.internal.pagemem.wal.record.delta.DataPageUpdateRecord) CheckpointRecord(org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord) MetastoreDataRecord(org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord) DataRecord(org.apache.ignite.internal.pagemem.wal.record.DataRecord) MetastoreDataRecord(org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord) MetaPageUpdatePartitionDataRecord(org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdatePartitionDataRecord) DataPageInsertRecord(org.apache.ignite.internal.pagemem.wal.record.delta.DataPageInsertRecord) ReplaceRecord(org.apache.ignite.internal.pagemem.wal.record.delta.ReplaceRecord) InnerReplaceRecord(org.apache.ignite.internal.pagemem.wal.record.delta.InnerReplaceRecord) PageSnapshot(org.apache.ignite.internal.pagemem.wal.record.PageSnapshot)

Example 19 with PageSnapshot

use of org.apache.ignite.internal.pagemem.wal.record.PageSnapshot in project ignite by apache.

the class GridCacheDatabaseSharedManager method restoreMemory.

/**
 * @param status Checkpoint status.
 * @param storeOnly If {@code True} restores Metastorage only.
 */
private WALPointer restoreMemory(CheckpointStatus status, boolean storeOnly, PageMemoryEx storePageMem) throws IgniteCheckedException {
    assert !storeOnly || storePageMem != null;
    if (log.isInfoEnabled())
        log.info("Checking memory state [lastValidPos=" + status.endPtr + ", lastMarked=" + status.startPtr + ", lastCheckpointId=" + status.cpStartId + ']');
    boolean apply = status.needRestoreMemory();
    if (apply) {
        U.quietAndWarn(log, "Ignite node stopped in the middle of checkpoint. Will restore memory state and " + "finish checkpoint on node start.");
        cctx.pageStore().beginRecover();
    } else
        cctx.wal().allowCompressionUntil(status.startPtr);
    long start = U.currentTimeMillis();
    int applied = 0;
    WALPointer lastRead = null;
    Collection<Integer> ignoreGrps = storeOnly ? Collections.emptySet() : initiallyWalDisabledGrps;
    try (WALIterator it = cctx.wal().replay(status.endPtr)) {
        while (it.hasNextX()) {
            IgniteBiTuple<WALPointer, WALRecord> tup = it.nextX();
            WALRecord rec = tup.get2();
            lastRead = tup.get1();
            switch(rec.type()) {
                case CHECKPOINT_RECORD:
                    CheckpointRecord cpRec = (CheckpointRecord) rec;
                    // We roll memory up until we find a checkpoint start record registered in the status.
                    if (F.eq(cpRec.checkpointId(), status.cpStartId)) {
                        log.info("Found last checkpoint marker [cpId=" + cpRec.checkpointId() + ", pos=" + tup.get1() + ']');
                        apply = false;
                    } else if (!F.eq(cpRec.checkpointId(), status.cpEndId))
                        U.warn(log, "Found unexpected checkpoint marker, skipping [cpId=" + cpRec.checkpointId() + ", expCpId=" + status.cpStartId + ", pos=" + tup.get1() + ']');
                    break;
                case PAGE_RECORD:
                    if (apply) {
                        PageSnapshot pageRec = (PageSnapshot) rec;
                        // Here we do not require tag check because we may be applying memory changes after
                        // several repetitive restarts and the same pages may have changed several times.
                        int grpId = pageRec.fullPageId().groupId();
                        if (storeOnly && grpId != METASTORAGE_CACHE_ID)
                            continue;
                        if (!ignoreGrps.contains(grpId)) {
                            long pageId = pageRec.fullPageId().pageId();
                            PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId);
                            long page = pageMem.acquirePage(grpId, pageId, true);
                            try {
                                long pageAddr = pageMem.writeLock(grpId, pageId, page);
                                try {
                                    PageUtils.putBytes(pageAddr, 0, pageRec.pageData());
                                } finally {
                                    pageMem.writeUnlock(grpId, pageId, page, null, true, true);
                                }
                            } finally {
                                pageMem.releasePage(grpId, pageId, page);
                            }
                            applied++;
                        }
                    }
                    break;
                case PARTITION_DESTROY:
                    PartitionDestroyRecord destroyRec = (PartitionDestroyRecord) rec;
                    final int gId = destroyRec.groupId();
                    if (storeOnly && gId != METASTORAGE_CACHE_ID)
                        continue;
                    if (!ignoreGrps.contains(gId)) {
                        final int pId = destroyRec.partitionId();
                        PageMemoryEx pageMem = gId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(gId);
                        pageMem.clearAsync((grpId, pageId) -> grpId == gId && PageIdUtils.partId(pageId) == pId, true).get();
                    }
                    break;
                default:
                    if (apply && rec instanceof PageDeltaRecord) {
                        PageDeltaRecord r = (PageDeltaRecord) rec;
                        int grpId = r.groupId();
                        if (storeOnly && grpId != METASTORAGE_CACHE_ID)
                            continue;
                        if (!ignoreGrps.contains(grpId)) {
                            long pageId = r.pageId();
                            PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId);
                            // Here we do not require tag check because we may be applying memory changes after
                            // several repetitive restarts and the same pages may have changed several times.
                            long page = pageMem.acquirePage(grpId, pageId, true);
                            try {
                                long pageAddr = pageMem.writeLock(grpId, pageId, page);
                                try {
                                    r.applyDelta(pageMem, pageAddr);
                                } finally {
                                    pageMem.writeUnlock(grpId, pageId, page, null, true, true);
                                }
                            } finally {
                                pageMem.releasePage(grpId, pageId, page);
                            }
                            applied++;
                        }
                    }
            }
        }
    }
    if (storeOnly)
        return null;
    if (status.needRestoreMemory()) {
        if (apply)
            throw new IgniteCheckedException("Failed to restore memory state (checkpoint marker is present " + "on disk, but checkpoint record is missed in WAL) " + "[cpStatus=" + status + ", lastRead=" + lastRead + "]");
        log.info("Finished applying memory changes [changesApplied=" + applied + ", time=" + (U.currentTimeMillis() - start) + "ms]");
        if (applied > 0)
            finalizeCheckpointOnRecovery(status.cpStartTs, status.cpStartId, status.startPtr);
    }
    checkpointHist.loadHistory(cpDir);
    return lastRead == null ? null : lastRead.next();
}
Also used : WALRecord(org.apache.ignite.internal.pagemem.wal.record.WALRecord) RandomAccessFile(java.io.RandomAccessFile) OverlappingFileLockException(java.nio.channels.OverlappingFileLockException) Arrays(java.util.Arrays) GridFutureAdapter(org.apache.ignite.internal.util.future.GridFutureAdapter) MetastorageLifecycleListener(org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener) CheckpointWriteOrder(org.apache.ignite.configuration.CheckpointWriteOrder) IGNITE_PDS_SKIP_CRC(org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC) UnsafeMemoryProvider(org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider) FileIO(org.apache.ignite.internal.processors.cache.persistence.file.FileIO) GridPortRecord(org.apache.ignite.internal.processors.port.GridPortRecord) Matcher(java.util.regex.Matcher) PagePartitionMetaIO(org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO) METASTORAGE_CACHE_ID(org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage.METASTORAGE_CACHE_ID) Map(java.util.Map) PageUtils(org.apache.ignite.internal.pagemem.PageUtils) Path(java.nio.file.Path) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) CacheGroupDescriptor(org.apache.ignite.internal.processors.cache.CacheGroupDescriptor) Set(java.util.Set) DataEntry(org.apache.ignite.internal.pagemem.wal.record.DataEntry) IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE(org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE) ByteOrder(java.nio.ByteOrder) CountDownLatch(java.util.concurrent.CountDownLatch) IgniteConfiguration(org.apache.ignite.configuration.IgniteConfiguration) IgnitePageStoreManager(org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager) PageIdUtils(org.apache.ignite.internal.pagemem.PageIdUtils) MappedFileMemoryProvider(org.apache.ignite.internal.mem.file.MappedFileMemoryProvider) GridCacheContext(org.apache.ignite.internal.processors.cache.GridCacheContext) PageSnapshot(org.apache.ignite.internal.pagemem.wal.record.PageSnapshot) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) WALIterator(org.apache.ignite.internal.pagemem.wal.WALIterator) AtomicIntegerFieldUpdater(java.util.concurrent.atomic.AtomicIntegerFieldUpdater) FullPageId(org.apache.ignite.internal.pagemem.FullPageId) IgniteThread(org.apache.ignite.thread.IgniteThread) GridDhtPartitionState(org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState) U(org.apache.ignite.internal.util.typedef.internal.U) IgniteLogger(org.apache.ignite.IgniteLogger) PageMemory(org.apache.ignite.internal.pagemem.PageMemory) WALPointer(org.apache.ignite.internal.pagemem.wal.WALPointer) ReentrantReadWriteLock(java.util.concurrent.locks.ReentrantReadWriteLock) ArrayList(java.util.ArrayList) GridKernalContext(org.apache.ignite.internal.GridKernalContext) READ(java.nio.file.StandardOpenOption.READ) ConcurrentLinkedHashMap(org.jsr166.ConcurrentLinkedHashMap) ClusterNode(org.apache.ignite.cluster.ClusterNode) CI1(org.apache.ignite.internal.util.typedef.CI1) GridDhtPartitionsExchangeFuture(org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture) S(org.apache.ignite.internal.util.typedef.internal.S) IgniteInterruptedCheckedException(org.apache.ignite.internal.IgniteInterruptedCheckedException) SoftReference(java.lang.ref.SoftReference) DataStorageMetrics(org.apache.ignite.DataStorageMetrics) Files(java.nio.file.Files) GridUnsafe(org.apache.ignite.internal.util.GridUnsafe) IOException(java.io.IOException) IGNITE_PDS_WAL_REBALANCE_THRESHOLD(org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD) File(java.io.File) T2(org.apache.ignite.internal.util.typedef.T2) PageMemoryImpl(org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl) FileFilter(java.io.FileFilter) GridCacheSharedContext(org.apache.ignite.internal.processors.cache.GridCacheSharedContext) Paths(java.nio.file.Paths) CacheConfiguration(org.apache.ignite.configuration.CacheConfiguration) IgniteCacheSnapshotManager(org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager) GridInClosure3X(org.apache.ignite.internal.util.lang.GridInClosure3X) PartitionDestroyRecord(org.apache.ignite.internal.pagemem.wal.record.delta.PartitionDestroyRecord) IgniteInternalFuture(org.apache.ignite.internal.IgniteInternalFuture) PageStore(org.apache.ignite.internal.pagemem.store.PageStore) StoredCacheData(org.apache.ignite.internal.processors.cache.StoredCacheData) CheckpointMetricsTracker(org.apache.ignite.internal.processors.cache.persistence.pagemem.CheckpointMetricsTracker) GridDhtLocalPartition(org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition) GridWorker(org.apache.ignite.internal.util.worker.GridWorker) ByteBuffer(java.nio.ByteBuffer) FileLock(java.nio.channels.FileLock) IgniteSystemProperties(org.apache.ignite.IgniteSystemProperties) SB(org.apache.ignite.internal.util.typedef.internal.SB) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) PureJavaCrc32(org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32) DataStorageMetricsMXBean(org.apache.ignite.mxbean.DataStorageMetricsMXBean) FileWALPointer(org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer) FilePageStore(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore) IgniteFuture(org.apache.ignite.lang.IgniteFuture) CacheState(org.apache.ignite.internal.pagemem.wal.record.CacheState) IgniteOutClosure(org.apache.ignite.lang.IgniteOutClosure) EventType(org.apache.ignite.events.EventType) Collection(java.util.Collection) IgniteException(org.apache.ignite.IgniteException) WALRecord(org.apache.ignite.internal.pagemem.wal.record.WALRecord) StandardOpenOption(java.nio.file.StandardOpenOption) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) StorageException(org.apache.ignite.internal.pagemem.wal.StorageException) FilePageStoreManager(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager) MetaStorage(org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage) UUID(java.util.UUID) ObjectName(javax.management.ObjectName) NavigableMap(java.util.NavigableMap) NodeInvalidator(org.apache.ignite.internal.NodeInvalidator) DirectMemoryProvider(org.apache.ignite.internal.mem.DirectMemoryProvider) DataRecord(org.apache.ignite.internal.pagemem.wal.record.DataRecord) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) IgniteBiTuple(org.apache.ignite.lang.IgniteBiTuple) PageDeltaRecord(org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord) ExchangeActions(org.apache.ignite.internal.processors.cache.ExchangeActions) Nullable(org.jetbrains.annotations.Nullable) List(java.util.List) CU(org.apache.ignite.internal.util.typedef.internal.CU) Pattern(java.util.regex.Pattern) NotNull(org.jetbrains.annotations.NotNull) LongAdder(java.util.concurrent.atomic.LongAdder) NodeStoppingException(org.apache.ignite.internal.NodeStoppingException) DiscoveryEvent(org.apache.ignite.events.DiscoveryEvent) GridMultiCollectionWrapper(org.apache.ignite.internal.util.GridMultiCollectionWrapper) HashMap(java.util.HashMap) CacheGroupContext(org.apache.ignite.internal.processors.cache.CacheGroupContext) LT(org.apache.ignite.internal.util.typedef.internal.LT) NearCacheConfiguration(org.apache.ignite.configuration.NearCacheConfiguration) ConcurrentMap(java.util.concurrent.ConcurrentMap) HashSet(java.util.HashSet) RejectedExecutionException(java.util.concurrent.RejectedExecutionException) IgnitePredicate(org.apache.ignite.lang.IgnitePredicate) IgniteUtils(org.apache.ignite.internal.util.IgniteUtils) DataStorageConfiguration(org.apache.ignite.configuration.DataStorageConfiguration) PageIO(org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO) IgniteThreadPoolExecutor(org.apache.ignite.thread.IgniteThreadPoolExecutor) MemoryRecoveryRecord(org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord) ExecutorService(java.util.concurrent.ExecutorService) SnapshotOperation(org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotOperation) GridDiscoveryManager(org.apache.ignite.internal.managers.discovery.GridDiscoveryManager) PartitionAllocationMap(org.apache.ignite.internal.processors.cache.persistence.partstate.PartitionAllocationMap) F(org.apache.ignite.internal.util.typedef.F) MetastoreDataRecord(org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord) Iterator(java.util.Iterator) DynamicCacheDescriptor(org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor) CountDownFuture(org.apache.ignite.internal.util.future.CountDownFuture) DataPageEvictionMode(org.apache.ignite.configuration.DataPageEvictionMode) CheckpointRecord(org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord) GridToStringInclude(org.apache.ignite.internal.util.tostring.GridToStringInclude) TimeUnit(java.util.concurrent.TimeUnit) PartitionMetaStateRecord(org.apache.ignite.internal.pagemem.wal.record.delta.PartitionMetaStateRecord) ConcurrentSkipListMap(java.util.concurrent.ConcurrentSkipListMap) PageMemoryEx(org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx) Comparator(java.util.Comparator) FileChannel(java.nio.channels.FileChannel) Collections(java.util.Collections) DataRegionConfiguration(org.apache.ignite.configuration.DataRegionConfiguration) CheckpointRecord(org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord) PageDeltaRecord(org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) WALIterator(org.apache.ignite.internal.pagemem.wal.WALIterator) PartitionDestroyRecord(org.apache.ignite.internal.pagemem.wal.record.delta.PartitionDestroyRecord) PageMemoryEx(org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx) WALPointer(org.apache.ignite.internal.pagemem.wal.WALPointer) FileWALPointer(org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer) PageSnapshot(org.apache.ignite.internal.pagemem.wal.record.PageSnapshot)

Example 20 with PageSnapshot

use of org.apache.ignite.internal.pagemem.wal.record.PageSnapshot in project ignite by apache.

the class IgnitePdsCheckpointSimulationWithRealCpDisabledTest method verifyReads.

/**
 * @param res Result map to verify.
 * @param mem Memory.
 */
private void verifyReads(Map<FullPageId, Integer> res, PageMemory mem, WALPointer start, IgniteWriteAheadLogManager wal) throws Exception {
    Map<FullPageId, byte[]> replay = new HashMap<>();
    try (PartitionMetaStateRecordExcludeIterator it = new PartitionMetaStateRecordExcludeIterator(wal.replay(start))) {
        IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
        assertTrue("Invalid record: " + tup, tup.get2() instanceof CheckpointRecord);
        CheckpointRecord cpRec = (CheckpointRecord) tup.get2();
        while (it.hasNext()) {
            tup = it.next();
            WALRecord rec = tup.get2();
            if (rec instanceof CheckpointRecord) {
                CheckpointRecord end = (CheckpointRecord) rec;
                // Found the finish mark.
                if (end.checkpointId().equals(cpRec.checkpointId()) && end.end())
                    break;
            } else if (rec instanceof PageSnapshot) {
                PageSnapshot page = (PageSnapshot) rec;
                replay.put(page.fullPageId(), page.pageData());
            }
        }
    }
    // Check read-through from the file store.
    for (Map.Entry<FullPageId, Integer> entry : res.entrySet()) {
        FullPageId fullId = entry.getKey();
        int state = entry.getValue();
        if (state == -1) {
            info("Page was never written: " + fullId);
            continue;
        }
        byte[] walData = replay.get(fullId);
        assertNotNull("Missing WAL record for a written page: " + fullId, walData);
        long page = mem.acquirePage(fullId.groupId(), fullId.pageId());
        try {
            long pageAddr = mem.readLock(fullId.groupId(), fullId.pageId(), page);
            try {
                for (int i = PageIO.COMMON_HEADER_END; i < mem.pageSize(); i++) {
                    int expState = state & 0xFF;
                    int pageState = PageUtils.getByte(pageAddr, i) & 0xFF;
                    int walState = walData[i] & 0xFF;
                    if (expState != pageState)
                        assertEquals("Invalid state [pageId=" + fullId + ", pos=" + i + ']', expState, pageState);
                    if (expState != walState)
                        assertEquals("Invalid WAL state [pageId=" + fullId + ", pos=" + i + ']', expState, walState);
                }
            } finally {
                mem.readUnlock(fullId.groupId(), fullId.pageId(), page);
            }
        } finally {
            mem.releasePage(fullId.groupId(), fullId.pageId(), page);
        }
    }
}
Also used : WALRecord(org.apache.ignite.internal.pagemem.wal.record.WALRecord) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) CheckpointRecord(org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord) WALPointer(org.apache.ignite.internal.pagemem.wal.WALPointer) Map(java.util.Map) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) ConcurrentMap(java.util.concurrent.ConcurrentMap) FullPageId(org.apache.ignite.internal.pagemem.FullPageId) PageSnapshot(org.apache.ignite.internal.pagemem.wal.record.PageSnapshot)

Aggregations

PageSnapshot (org.apache.ignite.internal.pagemem.wal.record.PageSnapshot)31 FullPageId (org.apache.ignite.internal.pagemem.FullPageId)23 WALPointer (org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer)19 CheckpointRecord (org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord)18 WALRecord (org.apache.ignite.internal.pagemem.wal.record.WALRecord)17 Test (org.junit.Test)13 File (java.io.File)12 WALIterator (org.apache.ignite.internal.pagemem.wal.WALIterator)12 ByteBuffer (java.nio.ByteBuffer)11 IgniteEx (org.apache.ignite.internal.IgniteEx)10 T2 (org.apache.ignite.internal.util.typedef.T2)10 ArrayList (java.util.ArrayList)9 MetastoreDataRecord (org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord)9 HashSet (java.util.HashSet)8 DataRecord (org.apache.ignite.internal.pagemem.wal.record.DataRecord)8 MemoryRecoveryRecord (org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord)8 HashMap (java.util.HashMap)7 UUID (java.util.UUID)7 FixCountRecord (org.apache.ignite.internal.pagemem.wal.record.delta.FixCountRecord)6 Map (java.util.Map)5