Search in sources :

Example 21 with PageSnapshot

use of org.apache.ignite.internal.pagemem.wal.record.PageSnapshot in project ignite by apache.

the class WalCompactionTest method testApplyingUpdatesFromCompactedWal.

/**
 * @throws Exception If failed.
 */
public void testApplyingUpdatesFromCompactedWal() throws Exception {
    IgniteEx ig = (IgniteEx) startGrids(3);
    ig.active(true);
    IgniteCache<Integer, byte[]> cache = ig.cache("cache");
    for (int i = 0; i < ENTRIES; i++) {
        // At least 20MB of raw data in total.
        final byte[] val = new byte[20000];
        val[i] = 1;
        cache.put(i, val);
    }
    // Spam WAL to move all data records to compressible WAL zone.
    for (int i = 0; i < WAL_SEGMENT_SIZE / DFLT_PAGE_SIZE * 2; i++) ig.context().cache().context().wal().log(new PageSnapshot(new FullPageId(-1, -1), new byte[DFLT_PAGE_SIZE]));
    // WAL archive segment is allowed to be compressed when it's at least one checkpoint away from current WAL head.
    ig.context().cache().context().database().wakeupForCheckpoint("Forced checkpoint").get();
    ig.context().cache().context().database().wakeupForCheckpoint("Forced checkpoint").get();
    // Allow compressor to archive WAL segments.
    Thread.sleep(15_000);
    String nodeFolderName = ig.context().pdsFolderResolver().resolveFolders().folderName();
    File dbDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false);
    File walDir = new File(dbDir, "wal");
    File archiveDir = new File(walDir, "archive");
    File nodeArchiveDir = new File(archiveDir, nodeFolderName);
    File walSegment = new File(nodeArchiveDir, FileWriteAheadLogManager.FileDescriptor.fileName(0) + ".zip");
    assertTrue(walSegment.exists());
    // Should be compressed at least in half.
    assertTrue(walSegment.length() < WAL_SEGMENT_SIZE / 2);
    stopAllGrids();
    File nodeLfsDir = new File(dbDir, nodeFolderName);
    File cpMarkersDir = new File(nodeLfsDir, "cp");
    File[] cpMarkers = cpMarkersDir.listFiles();
    assertNotNull(cpMarkers);
    assertTrue(cpMarkers.length > 0);
    File cacheDir = new File(nodeLfsDir, "cache-" + CACHE_NAME);
    File[] lfsFiles = cacheDir.listFiles();
    assertNotNull(lfsFiles);
    assertTrue(lfsFiles.length > 0);
    // Enforce reading WAL from the very beginning at the next start.
    for (File f : cpMarkers) f.delete();
    for (File f : lfsFiles) f.delete();
    ig = (IgniteEx) startGrids(3);
    ig.active(true);
    cache = ig.cache(CACHE_NAME);
    boolean fail = false;
    // Check that all data is recovered from compacted WAL.
    for (int i = 0; i < ENTRIES; i++) {
        byte[] arr = cache.get(i);
        if (arr == null) {
            System.out.println(">>> Missing: " + i);
            fail = true;
        } else if (arr[i] != 1) {
            System.out.println(">>> Corrupted: " + i);
            fail = true;
        }
    }
    assertFalse(fail);
}
Also used : IgniteEx(org.apache.ignite.internal.IgniteEx) File(java.io.File) PageSnapshot(org.apache.ignite.internal.pagemem.wal.record.PageSnapshot) FullPageId(org.apache.ignite.internal.pagemem.FullPageId)

Example 22 with PageSnapshot

use of org.apache.ignite.internal.pagemem.wal.record.PageSnapshot in project ignite by apache.

the class WalStat method registerRecord.

/**
 * Handles WAL record.
 *
 * @param record record to handle.
 * @param walPointer pointer, used to extract segment index.
 * @param workDir true for work, false for archive folder.
 */
void registerRecord(WALRecord record, WALPointer walPointer, boolean workDir) {
    WALRecord.RecordType type = record.type();
    if (type == WALRecord.RecordType.PAGE_RECORD)
        registerPageSnapshot((PageSnapshot) record);
    else if (type == WALRecord.RecordType.DATA_RECORD)
        registerDataRecord((DataRecord) record);
    else if (type == WALRecord.RecordType.TX_RECORD)
        registerTxRecord((TxRecord) record);
    incrementStat(type.toString(), record, recTypeSizes);
    if (walPointer instanceof FileWALPointer) {
        final FileWALPointer fPtr = (FileWALPointer) walPointer;
        incrementStat(Long.toString(fPtr.index()), record, segmentsIndexes);
        incrementStat(workDir ? "work" : "archive", record, segmentsFolder);
    }
}
Also used : WALRecord(org.apache.ignite.internal.pagemem.wal.record.WALRecord) FileWALPointer(org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer) TxRecord(org.apache.ignite.internal.pagemem.wal.record.TxRecord) PageSnapshot(org.apache.ignite.internal.pagemem.wal.record.PageSnapshot)

Example 23 with PageSnapshot

use of org.apache.ignite.internal.pagemem.wal.record.PageSnapshot in project ignite by apache.

the class RecordDataV2Serializer method readPlainRecord.

/**
 * {@inheritDoc}
 */
@Override
WALRecord readPlainRecord(RecordType type, ByteBufferBackedDataInput in, boolean encrypted, int recordSize) throws IOException, IgniteCheckedException {
    switch(type) {
        case PAGE_RECORD:
            int cacheId = in.readInt();
            long pageId = in.readLong();
            byte[] arr = new byte[recordSize - 4 - /* cacheId */
            8];
            in.readFully(arr);
            return new PageSnapshot(new FullPageId(pageId, cacheId), arr, encrypted ? realPageSize : pageSize);
        case CHECKPOINT_RECORD:
            long msb = in.readLong();
            long lsb = in.readLong();
            boolean hasPtr = in.readByte() != 0;
            long idx0 = hasPtr ? in.readLong() : 0;
            int off = hasPtr ? in.readInt() : 0;
            int len = hasPtr ? in.readInt() : 0;
            Map<Integer, CacheState> states = readPartitionStates(in);
            boolean end = in.readByte() != 0;
            WALPointer walPtr = hasPtr ? new WALPointer(idx0, off, len) : null;
            CheckpointRecord cpRec = new CheckpointRecord(new UUID(msb, lsb), walPtr, end);
            cpRec.cacheGroupStates(states);
            return cpRec;
        case DATA_RECORD:
        case DATA_RECORD_V2:
            int entryCnt = in.readInt();
            long timeStamp = in.readLong();
            if (entryCnt == 1)
                return new DataRecord(readPlainDataEntry(in, type), timeStamp);
            else {
                List<DataEntry> entries = new ArrayList<>(entryCnt);
                for (int i = 0; i < entryCnt; i++) entries.add(readPlainDataEntry(in, type));
                return new DataRecord(entries, timeStamp);
            }
        case MVCC_DATA_RECORD:
            entryCnt = in.readInt();
            timeStamp = in.readLong();
            List<DataEntry> entries = new ArrayList<>(entryCnt);
            for (int i = 0; i < entryCnt; i++) entries.add(readMvccDataEntry(in));
            return new MvccDataRecord(entries, timeStamp);
        case ENCRYPTED_DATA_RECORD:
        case ENCRYPTED_DATA_RECORD_V2:
        case ENCRYPTED_DATA_RECORD_V3:
            entryCnt = in.readInt();
            timeStamp = in.readLong();
            if (entryCnt == 1)
                return new DataRecord(readEncryptedDataEntry(in, type), timeStamp);
            else {
                entries = new ArrayList<>(entryCnt);
                for (int i = 0; i < entryCnt; i++) entries.add(readEncryptedDataEntry(in, type));
                return new DataRecord(entries, timeStamp);
            }
        case SNAPSHOT:
            long snpId = in.readLong();
            byte full = in.readByte();
            return new SnapshotRecord(snpId, full == 1);
        case EXCHANGE:
            int idx = in.readInt();
            short constId = in.readShort();
            long ts = in.readLong();
            return new ExchangeRecord(constId, ExchangeRecord.Type.values()[idx], ts);
        case TX_RECORD:
            return txRecordSerializer.readTx(in);
        case MVCC_TX_RECORD:
            return txRecordSerializer.readMvccTx(in);
        case ROLLBACK_TX_RECORD:
            int grpId = in.readInt();
            int partId = in.readInt();
            long start = in.readLong();
            long range = in.readLong();
            return new RollbackRecord(grpId, partId, start, range);
        case TRACKING_PAGE_REPAIR_DELTA:
            cacheId = in.readInt();
            pageId = in.readLong();
            return new TrackingPageRepairDeltaRecord(cacheId, pageId);
        default:
            return super.readPlainRecord(type, in, encrypted, recordSize);
    }
}
Also used : ArrayList(java.util.ArrayList) CheckpointRecord(org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord) CacheState(org.apache.ignite.internal.pagemem.wal.record.CacheState) SnapshotRecord(org.apache.ignite.internal.pagemem.wal.record.SnapshotRecord) TrackingPageRepairDeltaRecord(org.apache.ignite.internal.pagemem.wal.record.delta.TrackingPageRepairDeltaRecord) MvccDataEntry(org.apache.ignite.internal.pagemem.wal.record.MvccDataEntry) DataEntry(org.apache.ignite.internal.pagemem.wal.record.DataEntry) LazyMvccDataEntry(org.apache.ignite.internal.pagemem.wal.record.LazyMvccDataEntry) ExchangeRecord(org.apache.ignite.internal.pagemem.wal.record.ExchangeRecord) RollbackRecord(org.apache.ignite.internal.pagemem.wal.record.RollbackRecord) MvccDataRecord(org.apache.ignite.internal.pagemem.wal.record.MvccDataRecord) MvccDataRecord(org.apache.ignite.internal.pagemem.wal.record.MvccDataRecord) DataRecord(org.apache.ignite.internal.pagemem.wal.record.DataRecord) UUID(java.util.UUID) WALPointer(org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer) PageSnapshot(org.apache.ignite.internal.pagemem.wal.record.PageSnapshot) FullPageId(org.apache.ignite.internal.pagemem.FullPageId)

Example 24 with PageSnapshot

use of org.apache.ignite.internal.pagemem.wal.record.PageSnapshot in project ignite by apache.

the class GridCacheDatabaseSharedManager method performBinaryMemoryRestore.

/**
 * @param status Checkpoint status.
 * @param cacheGroupsPredicate Cache groups to restore.
 * @throws IgniteCheckedException If failed.
 * @throws StorageException In case I/O error occurred during operations with storage.
 */
private RestoreBinaryState performBinaryMemoryRestore(CheckpointStatus status, IgnitePredicate<Integer> cacheGroupsPredicate, IgniteBiPredicate<WALRecord.RecordType, WALPointer> recordTypePredicate, boolean finalizeState) throws IgniteCheckedException {
    if (log.isInfoEnabled())
        log.info("Checking memory state [lastValidPos=" + status.endPtr + ", lastMarked=" + status.startPtr + ", lastCheckpointId=" + status.cpStartId + ']');
    WALPointer recPtr = status.endPtr;
    boolean apply = status.needRestoreMemory();
    try {
        WALRecord startRec = !CheckpointStatus.NULL_PTR.equals(status.startPtr) || apply ? cctx.wal().read(status.startPtr) : null;
        if (apply) {
            if (finalizeState)
                U.quietAndWarn(log, "Ignite node stopped in the middle of checkpoint. Will restore memory state and " + "finish checkpoint on node start.");
            cctx.cache().cacheGroupDescriptors().forEach((grpId, desc) -> {
                if (!cacheGroupsPredicate.apply(grpId))
                    return;
                try {
                    DataRegion region = cctx.database().dataRegion(desc.config().getDataRegionName());
                    if (region == null || !cctx.isLazyMemoryAllocation(region))
                        return;
                    region.pageMemory().start();
                } catch (IgniteCheckedException e) {
                    throw new IgniteException(e);
                }
            });
            cctx.pageStore().beginRecover();
            if (!(startRec instanceof CheckpointRecord))
                throw new StorageException("Checkpoint marker doesn't point to checkpoint record " + "[ptr=" + status.startPtr + ", rec=" + startRec + "]");
            WALPointer cpMark = ((CheckpointRecord) startRec).checkpointMark();
            if (cpMark != null) {
                if (log.isInfoEnabled())
                    log.info("Restoring checkpoint after logical recovery, will start physical recovery from " + "back pointer: " + cpMark);
                recPtr = cpMark;
            }
        } else
            cctx.wal().notchLastCheckpointPtr(status.startPtr);
    } catch (NoSuchElementException e) {
        throw new StorageException("Failed to read checkpoint record from WAL, persistence consistency " + "cannot be guaranteed. Make sure configuration points to correct WAL folders and WAL folder is " + "properly mounted [ptr=" + status.startPtr + ", walPath=" + persistenceCfg.getWalPath() + ", walArchive=" + persistenceCfg.getWalArchivePath() + "]");
    }
    AtomicReference<Throwable> applyError = new AtomicReference<>();
    StripedExecutor exec = cctx.kernalContext().pools().getStripedExecutorService();
    Semaphore semaphore = new Semaphore(semaphorePertmits(exec));
    long start = U.currentTimeMillis();
    long lastArchivedSegment = cctx.wal().lastArchivedSegment();
    WALIterator it = cctx.wal().replay(recPtr, recordTypePredicate);
    RestoreBinaryState restoreBinaryState = new RestoreBinaryState(status, it, lastArchivedSegment, cacheGroupsPredicate);
    AtomicLong applied = new AtomicLong();
    try {
        while (restoreBinaryState.hasNext()) {
            if (applyError.get() != null)
                break;
            WALRecord rec = restoreBinaryState.next();
            if (rec == null)
                break;
            switch(rec.type()) {
                case PAGE_RECORD:
                    if (restoreBinaryState.needApplyBinaryUpdate()) {
                        PageSnapshot pageSnapshot = (PageSnapshot) rec;
                        // Here we do not require tag check because we may be applying memory changes after
                        // several repetitive restarts and the same pages may have changed several times.
                        int groupId = pageSnapshot.fullPageId().groupId();
                        int partId = partId(pageSnapshot.fullPageId().pageId());
                        if (skipRemovedIndexUpdates(groupId, partId))
                            break;
                        stripedApplyPage((pageMem) -> {
                            try {
                                applyPageSnapshot(pageMem, pageSnapshot);
                                applied.incrementAndGet();
                            } catch (Throwable t) {
                                U.error(log, "Failed to apply page snapshot. rec=[" + pageSnapshot + ']');
                                applyError.compareAndSet(null, (t instanceof IgniteCheckedException) ? (IgniteCheckedException) t : new IgniteCheckedException("Failed to apply page snapshot", t));
                            }
                        }, groupId, partId, exec, semaphore);
                    }
                    break;
                case PART_META_UPDATE_STATE:
                    PartitionMetaStateRecord metaStateRecord = (PartitionMetaStateRecord) rec;
                    {
                        int groupId = metaStateRecord.groupId();
                        int partId = metaStateRecord.partitionId();
                        stripedApplyPage((pageMem) -> {
                            GridDhtPartitionState state = fromOrdinal(metaStateRecord.state());
                            if (state == null || state == GridDhtPartitionState.EVICTED)
                                schedulePartitionDestroy(groupId, partId);
                            else {
                                try {
                                    cancelOrWaitPartitionDestroy(groupId, partId);
                                } catch (Throwable t) {
                                    U.error(log, "Failed to cancel or wait partition destroy. rec=[" + metaStateRecord + ']');
                                    applyError.compareAndSet(null, (t instanceof IgniteCheckedException) ? (IgniteCheckedException) t : new IgniteCheckedException("Failed to cancel or wait partition destroy", t));
                                }
                            }
                        }, groupId, partId, exec, semaphore);
                    }
                    break;
                case PARTITION_DESTROY:
                    PartitionDestroyRecord destroyRecord = (PartitionDestroyRecord) rec;
                    {
                        int groupId = destroyRecord.groupId();
                        int partId = destroyRecord.partitionId();
                        stripedApplyPage((pageMem) -> {
                            pageMem.invalidate(groupId, partId);
                            schedulePartitionDestroy(groupId, partId);
                        }, groupId, partId, exec, semaphore);
                    }
                    break;
                default:
                    if (restoreBinaryState.needApplyBinaryUpdate() && rec instanceof PageDeltaRecord) {
                        PageDeltaRecord pageDelta = (PageDeltaRecord) rec;
                        int groupId = pageDelta.groupId();
                        int partId = partId(pageDelta.pageId());
                        if (skipRemovedIndexUpdates(groupId, partId))
                            break;
                        stripedApplyPage((pageMem) -> {
                            try {
                                applyPageDelta(pageMem, pageDelta, true);
                                applied.incrementAndGet();
                            } catch (Throwable t) {
                                U.error(log, "Failed to apply page delta. rec=[" + pageDelta + ']');
                                applyError.compareAndSet(null, (t instanceof IgniteCheckedException) ? (IgniteCheckedException) t : new IgniteCheckedException("Failed to apply page delta", t));
                            }
                        }, groupId, partId, exec, semaphore);
                    }
            }
        }
    } finally {
        it.close();
        awaitApplyComplete(exec, applyError);
    }
    if (!finalizeState)
        return null;
    WALPointer lastReadPtr = restoreBinaryState.lastReadRecordPointer();
    if (status.needRestoreMemory()) {
        if (restoreBinaryState.needApplyBinaryUpdate())
            throw new StorageException("Failed to restore memory state (checkpoint marker is present " + "on disk, but checkpoint record is missed in WAL) " + "[cpStatus=" + status + ", lastRead=" + lastReadPtr + "]");
        if (log.isInfoEnabled())
            log.info("Finished applying memory changes [changesApplied=" + applied + ", time=" + (U.currentTimeMillis() - start) + " ms]");
        finalizeCheckpointOnRecovery(status.cpStartTs, status.cpStartId, status.startPtr, exec);
    }
    return restoreBinaryState;
}
Also used : WALRecord(org.apache.ignite.internal.pagemem.wal.record.WALRecord) Arrays(java.util.Arrays) GridFutureAdapter(org.apache.ignite.internal.util.future.GridFutureAdapter) TxLog(org.apache.ignite.internal.processors.cache.mvcc.txlog.TxLog) PartitionClearingStartRecord(org.apache.ignite.internal.pagemem.wal.record.PartitionClearingStartRecord) DistributedConfigurationLifecycleListener(org.apache.ignite.internal.processors.configuration.distributed.DistributedConfigurationLifecycleListener) MetastorageLifecycleListener(org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener) CheckpointStatus(org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointStatus) MASTER_KEY_CHANGE_RECORD(org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.MASTER_KEY_CHANGE_RECORD) METASTORE_DATA_RECORD(org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.METASTORE_DATA_RECORD) FileIO(org.apache.ignite.internal.processors.cache.persistence.file.FileIO) GridPortRecord(org.apache.ignite.internal.processors.port.GridPortRecord) LightweightCheckpointManager(org.apache.ignite.internal.processors.cache.persistence.checkpoint.LightweightCheckpointManager) PagePartitionMetaIO(org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO) GridDhtPartitionState.fromOrdinal(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.fromOrdinal) MaintenanceRegistry(org.apache.ignite.maintenance.MaintenanceRegistry) Map(java.util.Map) PageUtils(org.apache.ignite.internal.pagemem.PageUtils) IGNITE_PREFER_WAL_REBALANCE(org.apache.ignite.IgniteSystemProperties.IGNITE_PREFER_WAL_REBALANCE) Path(java.nio.file.Path) IgniteInClosure(org.apache.ignite.lang.IgniteInClosure) PageIdAllocator(org.apache.ignite.internal.pagemem.PageIdAllocator) IgniteDataIntegrityViolationException(org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) CacheGroupDescriptor(org.apache.ignite.internal.processors.cache.CacheGroupDescriptor) Set(java.util.Set) DataEntry(org.apache.ignite.internal.pagemem.wal.record.DataEntry) Serializable(java.io.Serializable) ByteOrder(java.nio.ByteOrder) IgniteConfiguration(org.apache.ignite.configuration.IgniteConfiguration) IgnitePageStoreManager(org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager) CheckpointHistoryResult(org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointHistoryResult) GB(org.apache.ignite.internal.util.IgniteUtils.GB) GridCountDownCallback(org.apache.ignite.internal.util.GridCountDownCallback) GridCacheContext(org.apache.ignite.internal.processors.cache.GridCacheContext) PageSnapshot(org.apache.ignite.internal.pagemem.wal.record.PageSnapshot) WALIterator(org.apache.ignite.internal.pagemem.wal.WALIterator) IgniteBiPredicate(org.apache.ignite.lang.IgniteBiPredicate) FullPageId(org.apache.ignite.internal.pagemem.FullPageId) U(org.apache.ignite.internal.util.typedef.internal.U) IgniteLogger(org.apache.ignite.IgniteLogger) PageMemory(org.apache.ignite.internal.pagemem.PageMemory) IGNITE_RECOVERY_SEMAPHORE_PERMITS(org.apache.ignite.IgniteSystemProperties.IGNITE_RECOVERY_SEMAPHORE_PERMITS) ArrayList(java.util.ArrayList) GridKernalContext(org.apache.ignite.internal.GridKernalContext) CheckpointManager(org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointManager) ClusterNode(org.apache.ignite.cluster.ClusterNode) GridDhtPartitionsExchangeFuture(org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture) IgniteInterruptedException(org.apache.ignite.IgniteInterruptedException) PageReadWriteManager(org.apache.ignite.internal.processors.cache.persistence.pagemem.PageReadWriteManager) MvccDataEntry(org.apache.ignite.internal.pagemem.wal.record.MvccDataEntry) CheckpointListener(org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointListener) FINISHED(org.apache.ignite.internal.processors.cache.persistence.CheckpointState.FINISHED) IgniteTxManager(org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager) CachePartitionDefragmentationManager(org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager) PageIdUtils.partId(org.apache.ignite.internal.pagemem.PageIdUtils.partId) DataStorageMetrics(org.apache.ignite.DataStorageMetrics) IoStatisticsHolderNoOp(org.apache.ignite.internal.metric.IoStatisticsHolderNoOp) TX_RECORD(org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.TX_RECORD) SystemProperty(org.apache.ignite.SystemProperty) A(org.apache.ignite.internal.util.typedef.internal.A) IOException(java.io.IOException) MaintenanceTask(org.apache.ignite.maintenance.MaintenanceTask) IGNITE_PDS_WAL_REBALANCE_THRESHOLD(org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD) File(java.io.File) T2(org.apache.ignite.internal.util.typedef.T2) PageMemoryImpl(org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl) SimpleDistributedProperty(org.apache.ignite.internal.processors.configuration.distributed.SimpleDistributedProperty) AtomicLong(java.util.concurrent.atomic.AtomicLong) GridDhtLocalPartition(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition) GridCacheSharedContext(org.apache.ignite.internal.processors.cache.GridCacheSharedContext) DefragmentationPageReadWriteManager(org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationPageReadWriteManager) IgniteCacheSnapshotManager(org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager) GridInClosure3X(org.apache.ignite.internal.util.lang.GridInClosure3X) WalRecordCacheGroupAware(org.apache.ignite.internal.pagemem.wal.record.WalRecordCacheGroupAware) CompressionProcessor(org.apache.ignite.internal.processors.compress.CompressionProcessor) DefragmentationParameters.fromStore(org.apache.ignite.internal.processors.cache.persistence.defragmentation.maintenance.DefragmentationParameters.fromStore) PartitionDestroyRecord(org.apache.ignite.internal.pagemem.wal.record.delta.PartitionDestroyRecord) IgniteInternalFuture(org.apache.ignite.internal.IgniteInternalFuture) PageStore(org.apache.ignite.internal.pagemem.store.PageStore) StripedExecutor(org.apache.ignite.internal.util.StripedExecutor) IgniteSystemProperties.getBoolean(org.apache.ignite.IgniteSystemProperties.getBoolean) TimeBag(org.apache.ignite.internal.util.TimeBag) TRANSACTIONAL_SNAPSHOT(org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT) WALPointer(org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer) ByteBuffer(java.nio.ByteBuffer) IgniteSystemProperties(org.apache.ignite.IgniteSystemProperties) RollbackRecord(org.apache.ignite.internal.pagemem.wal.record.RollbackRecord) IgniteUtils.checkpointBufferSize(org.apache.ignite.internal.util.IgniteUtils.checkpointBufferSize) SB(org.apache.ignite.internal.util.typedef.internal.SB) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) DataRegionMetricsProvider(org.apache.ignite.DataRegionMetricsProvider) TxRecord(org.apache.ignite.internal.pagemem.wal.record.TxRecord) X(org.apache.ignite.internal.util.typedef.X) DataStorageMetricsMXBean(org.apache.ignite.mxbean.DataStorageMetricsMXBean) Checkpointer(org.apache.ignite.internal.processors.cache.persistence.checkpoint.Checkpointer) DefragmentationWorkflowCallback(org.apache.ignite.internal.processors.cache.persistence.defragmentation.maintenance.DefragmentationWorkflowCallback) IGNITE_DEFRAGMENTATION_REGION_SIZE_PERCENTAGE(org.apache.ignite.IgniteSystemProperties.IGNITE_DEFRAGMENTATION_REGION_SIZE_PERCENTAGE) ToLongFunction(java.util.function.ToLongFunction) GridDhtPartitionState(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState) MetastorageViewWalker(org.apache.ignite.internal.managers.systemview.walker.MetastorageViewWalker) FilePageStore(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore) Collectors.toSet(java.util.stream.Collectors.toSet) DEFRAGMENTATION_MNTC_TASK_NAME(org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.DEFRAGMENTATION_MNTC_TASK_NAME) FailureType(org.apache.ignite.failure.FailureType) CacheState(org.apache.ignite.internal.pagemem.wal.record.CacheState) IgniteOutClosure(org.apache.ignite.lang.IgniteOutClosure) Predicate(java.util.function.Predicate) Collections.emptyList(java.util.Collections.emptyList) Collection(java.util.Collection) IgniteException(org.apache.ignite.IgniteException) WALRecord(org.apache.ignite.internal.pagemem.wal.record.WALRecord) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) FilePageStoreManager(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager) MetaStorage(org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage) OWNING(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING) UUID(java.util.UUID) DirectMemoryProvider(org.apache.ignite.internal.mem.DirectMemoryProvider) DataRecord(org.apache.ignite.internal.pagemem.wal.record.DataRecord) Collectors(java.util.stream.Collectors) IgniteCacheOffheapManager(org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager) IgniteBiTuple(org.apache.ignite.lang.IgniteBiTuple) PageDeltaRecord(org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord) DataRegionMetrics(org.apache.ignite.DataRegionMetrics) Nullable(org.jetbrains.annotations.Nullable) List(java.util.List) DistributedPropertyDispatcher(org.apache.ignite.internal.processors.configuration.distributed.DistributedPropertyDispatcher) CU(org.apache.ignite.internal.util.typedef.internal.CU) Function.identity(java.util.function.Function.identity) CheckpointEntry(org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointEntry) Pattern(java.util.regex.Pattern) NotNull(org.jetbrains.annotations.NotNull) Objects.nonNull(java.util.Objects.nonNull) IgniteSystemProperties.getInteger(org.apache.ignite.IgniteSystemProperties.getInteger) CHECKPOINT_LOCK_HOLD_COUNT(org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointReadWriteLock.CHECKPOINT_LOCK_HOLD_COUNT) CheckpointProgress(org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointProgress) HashMap(java.util.HashMap) MvccTxRecord(org.apache.ignite.internal.pagemem.wal.record.MvccTxRecord) AtomicReference(java.util.concurrent.atomic.AtomicReference) DirectMemoryRegion(org.apache.ignite.internal.mem.DirectMemoryRegion) CacheGroupContext(org.apache.ignite.internal.processors.cache.CacheGroupContext) HashSet(java.util.HashSet) IndexRenameRootPageRecord(org.apache.ignite.internal.pagemem.wal.record.IndexRenameRootPageRecord) FailureContext(org.apache.ignite.failure.FailureContext) IgnitePredicate(org.apache.ignite.lang.IgnitePredicate) MasterKeyChangeRecordV2(org.apache.ignite.internal.pagemem.wal.record.MasterKeyChangeRecordV2) IgniteUtils(org.apache.ignite.internal.util.IgniteUtils) FileIOFactory(org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory) DataStorageConfiguration(org.apache.ignite.configuration.DataStorageConfiguration) PageIO(org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO) NoSuchElementException(java.util.NoSuchElementException) MemoryRecoveryRecord(org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord) MetastorageView(org.apache.ignite.spi.systemview.view.MetastorageView) GridDiscoveryManager(org.apache.ignite.internal.managers.discovery.GridDiscoveryManager) F(org.apache.ignite.internal.util.typedef.F) MetastoreDataRecord(org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord) ReencryptionStartRecord(org.apache.ignite.internal.pagemem.wal.record.ReencryptionStartRecord) GroupPartitionId(org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId) Iterator(java.util.Iterator) Semaphore(java.util.concurrent.Semaphore) CheckpointHistory(org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointHistory) AffinityTopologyVersion(org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion) DynamicCacheDescriptor(org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor) DataPageEvictionMode(org.apache.ignite.configuration.DataPageEvictionMode) GridConcurrentHashSet(org.apache.ignite.internal.util.GridConcurrentHashSet) CHECKPOINT_RECORD(org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.CHECKPOINT_RECORD) CheckpointRecord(org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord) Consumer(java.util.function.Consumer) DistributedConfigurationUtils.makeUpdateListener(org.apache.ignite.internal.cluster.DistributedConfigurationUtils.makeUpdateListener) PartitionMetaStateRecord(org.apache.ignite.internal.pagemem.wal.record.delta.PartitionMetaStateRecord) DistributedConfigurationUtils.setDefaultValue(org.apache.ignite.internal.cluster.DistributedConfigurationUtils.setDefaultValue) Collectors.toList(java.util.stream.Collectors.toList) TransactionState(org.apache.ignite.transactions.TransactionState) LOCK_RELEASED(org.apache.ignite.internal.processors.cache.persistence.CheckpointState.LOCK_RELEASED) PageMemoryEx(org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx) MASTER_KEY_CHANGE_RECORD_V2(org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.MASTER_KEY_CHANGE_RECORD_V2) Comparator(java.util.Comparator) Collections(java.util.Collections) TxState(org.apache.ignite.internal.processors.cache.mvcc.txlog.TxState) DataRegionConfiguration(org.apache.ignite.configuration.DataRegionConfiguration) ReservationReason(org.apache.ignite.internal.processors.cache.persistence.checkpoint.ReservationReason) GridQueryProcessor(org.apache.ignite.internal.processors.query.GridQueryProcessor) CORRUPTED_DATA_FILES_MNTC_TASK_NAME(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.CORRUPTED_DATA_FILES_MNTC_TASK_NAME) PageDeltaRecord(org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord) Semaphore(java.util.concurrent.Semaphore) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) WALIterator(org.apache.ignite.internal.pagemem.wal.WALIterator) IgniteException(org.apache.ignite.IgniteException) PartitionDestroyRecord(org.apache.ignite.internal.pagemem.wal.record.delta.PartitionDestroyRecord) WALPointer(org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer) PageSnapshot(org.apache.ignite.internal.pagemem.wal.record.PageSnapshot) PartitionMetaStateRecord(org.apache.ignite.internal.pagemem.wal.record.delta.PartitionMetaStateRecord) CheckpointRecord(org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord) AtomicReference(java.util.concurrent.atomic.AtomicReference) AtomicLong(java.util.concurrent.atomic.AtomicLong) StripedExecutor(org.apache.ignite.internal.util.StripedExecutor) GridDhtPartitionState(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState) NoSuchElementException(java.util.NoSuchElementException)

Example 25 with PageSnapshot

use of org.apache.ignite.internal.pagemem.wal.record.PageSnapshot in project ignite by apache.

the class GridCacheOffheapManager method getOrAllocateCacheMetas.

/**
 * @return Meta root pages info.
 * @throws IgniteCheckedException If failed.
 */
private Metas getOrAllocateCacheMetas() throws IgniteCheckedException {
    PageMemoryEx pageMem = (PageMemoryEx) grp.dataRegion().pageMemory();
    IgniteWriteAheadLogManager wal = ctx.wal();
    int grpId = grp.groupId();
    long metaId = PageMemory.META_PAGE_ID;
    long metaPage = pageMem.acquirePage(grpId, metaId);
    try {
        long pageAddr = pageMem.writeLock(grpId, metaId, metaPage);
        boolean allocated = false;
        boolean markDirty = false;
        try {
            long metastoreRoot, reuseListRoot;
            PageMetaIOV2 io = (PageMetaIOV2) PageMetaIO.VERSIONS.latest();
            if (PageIO.getType(pageAddr) != PageIO.T_META) {
                PageMetrics metrics = pageMem.metrics().cacheGrpPageMetrics(grpId);
                io.initNewPage(pageAddr, metaId, pageMem.realPageSize(grpId), metrics);
                metastoreRoot = pageMem.allocatePage(grpId, PageIdAllocator.INDEX_PARTITION, PageMemory.FLAG_IDX);
                reuseListRoot = pageMem.allocatePage(grpId, PageIdAllocator.INDEX_PARTITION, PageMemory.FLAG_IDX);
                io.setTreeRoot(pageAddr, metastoreRoot);
                io.setReuseListRoot(pageAddr, reuseListRoot);
                if (isWalDeltaRecordNeeded(pageMem, grpId, metaId, metaPage, wal, null)) {
                    assert io.getType() == PageIO.T_META;
                    wal.log(new MetaPageInitRecord(grpId, metaId, io.getType(), io.getVersion(), metastoreRoot, reuseListRoot));
                }
                allocated = true;
            } else {
                if (io != PageIO.getPageIO(pageAddr)) {
                    if (log.isDebugEnabled()) {
                        log.debug("Upgrade index partition meta page version: [grpId=" + grpId + ", oldVer=" + PagePartitionMetaIO.getVersion(pageAddr) + ", newVer=" + io.getVersion() + ']');
                    }
                    io.upgradePage(pageAddr);
                    markDirty = true;
                }
                metastoreRoot = io.getTreeRoot(pageAddr);
                reuseListRoot = io.getReuseListRoot(pageAddr);
                int encrPageCnt = io.getEncryptedPageCount(pageAddr);
                if (encrPageCnt > 0) {
                    ctx.kernalContext().encryption().setEncryptionState(grp, PageIdAllocator.INDEX_PARTITION, io.getEncryptedPageIndex(pageAddr), encrPageCnt);
                    markDirty = true;
                }
                assert reuseListRoot != 0L;
                if (markDirty && isWalDeltaRecordNeeded(pageMem, grpId, metaId, metaPage, wal, null)) {
                    wal.log(new PageSnapshot(new FullPageId(PageIdAllocator.INDEX_PARTITION, grpId), pageAddr, pageMem.pageSize(), pageMem.realPageSize(grpId)));
                }
            }
            return new Metas(new RootPage(new FullPageId(metastoreRoot, grpId), allocated), new RootPage(new FullPageId(reuseListRoot, grpId), allocated), null, null);
        } finally {
            pageMem.writeUnlock(grpId, metaId, metaPage, null, allocated || markDirty);
        }
    } finally {
        pageMem.releasePage(grpId, metaId, metaPage);
    }
}
Also used : IgniteWriteAheadLogManager(org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager) PageMetaIOV2(org.apache.ignite.internal.processors.cache.persistence.tree.io.PageMetaIOV2) PageMetrics(org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMetrics) MetaPageInitRecord(org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRecord) PageMemoryEx(org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx) PageSnapshot(org.apache.ignite.internal.pagemem.wal.record.PageSnapshot) FullPageId(org.apache.ignite.internal.pagemem.FullPageId)

Aggregations

PageSnapshot (org.apache.ignite.internal.pagemem.wal.record.PageSnapshot)31 FullPageId (org.apache.ignite.internal.pagemem.FullPageId)23 WALPointer (org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer)19 CheckpointRecord (org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord)18 WALRecord (org.apache.ignite.internal.pagemem.wal.record.WALRecord)17 Test (org.junit.Test)13 File (java.io.File)12 WALIterator (org.apache.ignite.internal.pagemem.wal.WALIterator)12 ByteBuffer (java.nio.ByteBuffer)11 IgniteEx (org.apache.ignite.internal.IgniteEx)10 T2 (org.apache.ignite.internal.util.typedef.T2)10 ArrayList (java.util.ArrayList)9 MetastoreDataRecord (org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord)9 HashSet (java.util.HashSet)8 DataRecord (org.apache.ignite.internal.pagemem.wal.record.DataRecord)8 MemoryRecoveryRecord (org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord)8 HashMap (java.util.HashMap)7 UUID (java.util.UUID)7 FixCountRecord (org.apache.ignite.internal.pagemem.wal.record.delta.FixCountRecord)6 Map (java.util.Map)5