Examples with DataEntry org.apache.ignite.internal.pagemem.wal.record.DataEntry used on opensource projects

Example 21 with DataEntry

use of org.apache.ignite.internal.pagemem.wal.record.DataEntry in project ignite by apache.

the class RecordDataV2Serializer method readRecord.

/**
 * {@inheritDoc}
 */
@Override
public WALRecord readRecord(WALRecord.RecordType type, ByteBufferBackedDataInput in) throws IOException, IgniteCheckedException {
    switch(type) {
        case CHECKPOINT_RECORD:
            long msb = in.readLong();
            long lsb = in.readLong();
            boolean hasPtr = in.readByte() != 0;
            int idx0 = hasPtr ? in.readInt() : 0;
            int off = hasPtr ? in.readInt() : 0;
            int len = hasPtr ? in.readInt() : 0;
            Map<Integer, CacheState> states = readPartitionStates(in);
            boolean end = in.readByte() != 0;
            FileWALPointer walPtr = hasPtr ? new FileWALPointer(idx0, off, len) : null;
            CheckpointRecord cpRec = new CheckpointRecord(new UUID(msb, lsb), walPtr, end);
            cpRec.cacheGroupStates(states);
            return cpRec;
        case DATA_RECORD:
            int entryCnt = in.readInt();
            long timeStamp = in.readLong();
            List<DataEntry> entries = new ArrayList<>(entryCnt);
            for (int i = 0; i < entryCnt; i++) entries.add(delegateSerializer.readDataEntry(in));
            return new DataRecord(entries, timeStamp);
        case SNAPSHOT:
            long snpId = in.readLong();
            byte full = in.readByte();
            return new SnapshotRecord(snpId, full == 1);
        case EXCHANGE:
            int idx = in.readInt();
            short constId = in.readShort();
            long ts = in.readLong();
            return new ExchangeRecord(constId, ExchangeRecord.Type.values()[idx], ts);
        case TX_RECORD:
            return txRecordSerializer.read(in);
        case BASELINE_TOP_RECORD:
            return bltRecSerializer.read(in);
        default:
            return delegateSerializer.readRecord(type, in);
    }
}

Example 22 with DataEntry

use of org.apache.ignite.internal.pagemem.wal.record.DataEntry in project ignite by apache.

the class GridCacheDatabaseSharedManager method applyLastUpdates.

/**
 * @param status Last registered checkpoint status.
 * @throws IgniteCheckedException If failed to apply updates.
 * @throws StorageException If IO exception occurred while reading write-ahead log.
 */
private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) throws IgniteCheckedException {
    if (log.isInfoEnabled())
        log.info("Applying lost cache updates since last checkpoint record [lastMarked=" + status.startPtr + ", lastCheckpointId=" + status.cpStartId + ']');
    if (!metastoreOnly)
        cctx.kernalContext().query().skipFieldLookup(true);
    long start = U.currentTimeMillis();
    int applied = 0;
    Collection<Integer> ignoreGrps = metastoreOnly ? Collections.emptySet() : initiallyWalDisabledGrps;
    try (WALIterator it = cctx.wal().replay(status.startPtr)) {
        Map<T2<Integer, Integer>, T2<Integer, Long>> partStates = new HashMap<>();
        while (it.hasNextX()) {
            IgniteBiTuple<WALPointer, WALRecord> next = it.nextX();
            WALRecord rec = next.get2();
            switch(rec.type()) {
                case DATA_RECORD:
                    if (metastoreOnly)
                        continue;
                    DataRecord dataRec = (DataRecord) rec;
                    for (DataEntry dataEntry : dataRec.writeEntries()) {
                        int cacheId = dataEntry.cacheId();
                        int grpId = cctx.cache().cacheDescriptor(cacheId).groupId();
                        if (!ignoreGrps.contains(grpId)) {
                            GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
                            applyUpdate(cacheCtx, dataEntry);
                            applied++;
                        }
                    }
                    break;
                case PART_META_UPDATE_STATE:
                    if (metastoreOnly)
                        continue;
                    PartitionMetaStateRecord metaStateRecord = (PartitionMetaStateRecord) rec;
                    if (!ignoreGrps.contains(metaStateRecord.groupId())) {
                        partStates.put(new T2<>(metaStateRecord.groupId(), metaStateRecord.partitionId()), new T2<>((int) metaStateRecord.state(), metaStateRecord.updateCounter()));
                    }
                    break;
                case METASTORE_DATA_RECORD:
                    MetastoreDataRecord metastoreDataRecord = (MetastoreDataRecord) rec;
                    metaStorage.applyUpdate(metastoreDataRecord.key(), metastoreDataRecord.value());
                    break;
                case META_PAGE_UPDATE_NEXT_SNAPSHOT_ID:
                case META_PAGE_UPDATE_LAST_SUCCESSFUL_SNAPSHOT_ID:
                case META_PAGE_UPDATE_LAST_SUCCESSFUL_FULL_SNAPSHOT_ID:
                    if (metastoreOnly)
                        continue;
                    PageDeltaRecord rec0 = (PageDeltaRecord) rec;
                    PageMemoryEx pageMem = getPageMemoryForCacheGroup(rec0.groupId());
                    long page = pageMem.acquirePage(rec0.groupId(), rec0.pageId(), true);
                    try {
                        long addr = pageMem.writeLock(rec0.groupId(), rec0.pageId(), page, true);
                        try {
                            rec0.applyDelta(pageMem, addr);
                        } finally {
                            pageMem.writeUnlock(rec0.groupId(), rec0.pageId(), page, null, true, true);
                        }
                    } finally {
                        pageMem.releasePage(rec0.groupId(), rec0.pageId(), page);
                    }
                    break;
                default:
            }
        }
        if (!metastoreOnly)
            restorePartitionState(partStates, ignoreGrps);
    } finally {
        if (!metastoreOnly)
            cctx.kernalContext().query().skipFieldLookup(false);
    }
    if (log.isInfoEnabled())
        log.info("Finished applying WAL changes [updatesApplied=" + applied + ", time=" + (U.currentTimeMillis() - start) + "ms]");
}

Example 23 with DataEntry

use of org.apache.ignite.internal.pagemem.wal.record.DataEntry in project ignite by apache.

the class IgnitePdsCheckpointSimulationWithRealCpDisabledTest method testDataWalEntries.

/**
 * @throws Exception if failed.
 */
public void testDataWalEntries() throws Exception {
    IgniteEx ig = startGrid(0);
    ig.active(true);
    GridCacheSharedContext<Object, Object> sharedCtx = ig.context().cache().context();
    GridCacheContext<Object, Object> cctx = sharedCtx.cache().cache(cacheName).context();
    GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) sharedCtx.database();
    IgniteWriteAheadLogManager wal = sharedCtx.wal();
    assertTrue(wal.isAlwaysWriteFullPages());
    db.enableCheckpoints(false).get();
    final int cnt = 10;
    List<DataEntry> entries = new ArrayList<>(cnt);
    for (int i = 0; i < cnt; i++) {
        GridCacheOperation op = i % 2 == 0 ? GridCacheOperation.UPDATE : GridCacheOperation.DELETE;
        KeyCacheObject key = cctx.toCacheKeyObject(i);
        CacheObject val = null;
        if (op != GridCacheOperation.DELETE)
            val = cctx.toCacheObject("value-" + i);
        entries.add(new DataEntry(cctx.cacheId(), key, val, op, null, cctx.versions().next(), 0L, cctx.affinity().partition(i), i));
    }
    UUID cpId = UUID.randomUUID();
    WALPointer start = wal.log(new CheckpointRecord(cpId, null));
    wal.fsync(start);
    for (DataEntry entry : entries) wal.log(new DataRecord(entry));
    // Data will not be written to the page store.
    stopAllGrids();
    ig = startGrid(0);
    ig.active(true);
    sharedCtx = ig.context().cache().context();
    cctx = sharedCtx.cache().cache(cacheName).context();
    db = (GridCacheDatabaseSharedManager) sharedCtx.database();
    wal = sharedCtx.wal();
    db.enableCheckpoints(false).get();
    try (PartitionMetaStateRecordExcludeIterator it = new PartitionMetaStateRecordExcludeIterator(wal.replay(start))) {
        IgniteBiTuple<WALPointer, WALRecord> cpRecordTup = it.next();
        assert cpRecordTup.get2() instanceof CheckpointRecord;
        assertEquals(start, cpRecordTup.get1());
        CheckpointRecord cpRec = (CheckpointRecord) cpRecordTup.get2();
        assertEquals(cpId, cpRec.checkpointId());
        assertNull(cpRec.checkpointMark());
        assertFalse(cpRec.end());
        int idx = 0;
        CacheObjectContext coctx = cctx.cacheObjectContext();
        while (idx < entries.size()) {
            IgniteBiTuple<WALPointer, WALRecord> dataRecTup = it.next();
            assert dataRecTup.get2() instanceof DataRecord;
            DataRecord dataRec = (DataRecord) dataRecTup.get2();
            DataEntry entry = entries.get(idx);
            assertEquals(1, dataRec.writeEntries().size());
            DataEntry readEntry = dataRec.writeEntries().get(0);
            assertEquals(entry.cacheId(), readEntry.cacheId());
            assertEquals(entry.key().<Integer>value(coctx, true), readEntry.key().<Integer>value(coctx, true));
            assertEquals(entry.op(), readEntry.op());
            if (entry.op() == GridCacheOperation.UPDATE)
                assertEquals(entry.value().value(coctx, true), readEntry.value().value(coctx, true));
            else
                assertNull(entry.value());
            assertEquals(entry.writeVersion(), readEntry.writeVersion());
            assertEquals(entry.nearXidVersion(), readEntry.nearXidVersion());
            assertEquals(entry.partitionCounter(), readEntry.partitionCounter());
            idx++;
        }
    }
}

Example 24 with DataEntry

use of org.apache.ignite.internal.pagemem.wal.record.DataEntry in project ignite by apache.

the class IgniteWalReaderTest method iterateAndCount.

/**
 * Iterates on records and closes iterator.
 *
 * @param walIter iterator to count, will be closed.
 * @param touchEntries access data within entries.
 * @return count of records.
 * @throws IgniteCheckedException if failed to iterate.
 */
private int iterateAndCount(WALIterator walIter, boolean touchEntries) throws IgniteCheckedException {
    int cnt = 0;
    try (WALIterator it = walIter) {
        while (it.hasNextX()) {
            final IgniteBiTuple<WALPointer, WALRecord> next = it.nextX();
            final WALRecord walRecord = next.get2();
            if (touchEntries && walRecord.type() == WALRecord.RecordType.DATA_RECORD) {
                final DataRecord record = (DataRecord) walRecord;
                for (DataEntry entry : record.writeEntries()) {
                    final KeyCacheObject key = entry.key();
                    final CacheObject val = entry.value();
                    if (dumpRecords)
                        log.info("Op: " + entry.op() + ", Key: " + key + ", Value: " + val);
                }
            }
            if (dumpRecords)
                log.info("Record: " + walRecord);
            cnt++;
        }
    }
    return cnt;
}

Example 25 with DataEntry

use of org.apache.ignite.internal.pagemem.wal.record.DataEntry in project ignite by apache.

the class StandaloneWalRecordsIterator method postProcessDataRecord.

/**
 * Performs post processing of lazy data record, converts it to unwrap record.
 *
 * @param dataRec data record to post process records.
 * @param kernalCtx kernal context.
 * @param processor processor to convert binary form from WAL into CacheObject/BinaryObject.
 * @return post-processed record.
 * @throws IgniteCheckedException if failed.
 */
@NotNull
private WALRecord postProcessDataRecord(@NotNull DataRecord dataRec, GridKernalContext kernalCtx, IgniteCacheObjectProcessor processor) throws IgniteCheckedException {
    final CacheObjectContext fakeCacheObjCtx = new CacheObjectContext(kernalCtx, null, null, false, false, false, false, false);
    final int entryCnt = dataRec.entryCount();
    final List<DataEntry> postProcessedEntries = new ArrayList<>(entryCnt);
    for (int i = 0; i < entryCnt; i++) {
        final DataEntry postProcessedEntry = postProcessDataEntry(processor, fakeCacheObjCtx, dataRec.get(i));
        postProcessedEntries.add(postProcessedEntry);
    }
    DataRecord res = dataRec instanceof MvccDataRecord ? new MvccDataRecord(postProcessedEntries, dataRec.timestamp()) : new DataRecord(postProcessedEntries, dataRec.timestamp());
    res.size(dataRec.size());
    res.position(dataRec.position());
    return res;
}