Examples with RecordSerializer org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer used on opensource projects

Example 1 with RecordSerializer

use of org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer in project ignite by apache.

the class FileWriteAheadLogManager method restoreWriteHandle.

/**
 * @param lastReadPtr Last read WAL file pointer.
 * @return Initialized file write handle.
 * @throws IgniteCheckedException If failed to initialize WAL write handle.
 */
private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws IgniteCheckedException {
    long absIdx = lastReadPtr == null ? 0 : lastReadPtr.index();
    @Nullable FileArchiver archiver0 = archiver;
    long segNo = archiver0 == null ? absIdx : absIdx % dsCfg.getWalSegments();
    File curFile = new File(walWorkDir, FileDescriptor.fileName(segNo));
    int off = lastReadPtr == null ? 0 : lastReadPtr.fileOffset();
    int len = lastReadPtr == null ? 0 : lastReadPtr.length();
    try {
        FileIO fileIO = ioFactory.create(curFile);
        IgniteInClosure<FileIO> lsnr = createWalFileListener;
        if (lsnr != null)
            lsnr.apply(fileIO);
        try {
            int serVer = serializerVer;
            // If we have existing segment, try to read version from it.
            if (lastReadPtr != null) {
                try {
                    serVer = readSerializerVersionAndCompactedFlag(fileIO).get1();
                } catch (SegmentEofException | EOFException ignore) {
                    serVer = serializerVer;
                }
            }
            RecordSerializer ser = new RecordSerializerFactoryImpl(cctx).createSerializer(serVer);
            if (log.isInfoEnabled())
                log.info("Resuming logging to WAL segment [file=" + curFile.getAbsolutePath() + ", offset=" + off + ", ver=" + serVer + ']');
            SegmentedRingByteBuffer rbuf;
            if (mmap) {
                try {
                    MappedByteBuffer buf = fileIO.map((int) maxWalSegmentSize);
                    rbuf = new SegmentedRingByteBuffer(buf, metrics);
                } catch (IOException e) {
                    throw new IgniteCheckedException(e);
                }
            } else
                rbuf = new SegmentedRingByteBuffer(dsCfg.getWalBufferSize(), maxWalSegmentSize, DIRECT, metrics);
            if (lastReadPtr != null)
                rbuf.init(lastReadPtr.fileOffset() + lastReadPtr.length());
            FileWriteHandle hnd = new FileWriteHandle(fileIO, absIdx, off + len, true, ser, rbuf);
            if (archiver0 != null)
                archiver0.currentWalIndex(absIdx);
            else
                archivedMonitor.setLastArchivedAbsoluteIndex(absIdx - 1);
            return hnd;
        } catch (IgniteCheckedException | IOException e) {
            fileIO.close();
            throw e;
        }
    } catch (IOException e) {
        throw new IgniteCheckedException("Failed to restore WAL write handle: " + curFile.getAbsolutePath(), e);
    }
}

Example 2 with RecordSerializer

use of org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer in project ignite by apache.

the class FileWriteAheadLogManager method restoreWriteHandle.

/**
 * @param lastReadPtr Last read WAL file pointer.
 * @return Initialized file write handle.
 * @throws StorageException If failed to initialize WAL write handle.
 */
private FileWriteHandle restoreWriteHandle(@Nullable WALPointer lastReadPtr) throws StorageException {
    long absIdx = lastReadPtr == null ? 0 : lastReadPtr.index();
    @Nullable FileArchiver archiver0 = archiver;
    long segNo = archiver0 == null ? absIdx : absIdx % dsCfg.getWalSegments();
    File curFile = new File(walWorkDir, fileName(segNo));
    int off = lastReadPtr == null ? 0 : lastReadPtr.fileOffset();
    int len = lastReadPtr == null ? 0 : lastReadPtr.length();
    try {
        SegmentIO fileIO = new SegmentIO(absIdx, ioFactory.create(curFile));
        IgniteInClosure<FileIO> lsnr = createWalFileListener;
        if (lsnr != null)
            lsnr.apply(fileIO);
        try {
            int serVer = serializerVer;
            // If we have existing segment, try to read version from it.
            if (lastReadPtr != null) {
                try {
                    serVer = readSegmentHeader(fileIO, segmentFileInputFactory).getSerializerVersion();
                } catch (SegmentEofException | EOFException ignore) {
                    serVer = serializerVer;
                }
            }
            RecordSerializer ser = new RecordSerializerFactoryImpl(cctx).createSerializer(serVer);
            if (log.isInfoEnabled()) {
                log.info("Resuming logging to WAL segment [file=" + curFile.getAbsolutePath() + ", offset=" + off + ", ver=" + serVer + ']');
            }
            FileWriteHandle hnd = fileHandleManager.initHandle(fileIO, off + len, ser);
            segmentAware.curAbsWalIdx(absIdx);
            FileDescriptor[] walArchiveFiles = walArchiveFiles();
            segmentAware.minReserveIndex(F.isEmpty(walArchiveFiles) ? -1 : walArchiveFiles[0].idx - 1);
            segmentAware.lastTruncatedArchiveIdx(F.isEmpty(walArchiveFiles) ? -1 : walArchiveFiles[0].idx - 1);
            if (archiver0 == null)
                segmentAware.setLastArchivedAbsoluteIndex(absIdx - 1);
            // Getting segment sizes.
            F.asList(walArchiveDir.listFiles(WAL_SEGMENT_COMPACTED_OR_RAW_FILE_FILTER)).stream().map(FileDescriptor::new).forEach(fd -> {
                if (fd.isCompressed())
                    segmentSize.put(fd.idx(), fd.file().length());
                else
                    segmentSize.putIfAbsent(fd.idx(), fd.file().length());
            });
            // Size of the 8th segment will be set in #resumeLogging.
            if (archiver0 != null) {
                for (long i = absIdx - (absIdx % dsCfg.getWalSegments()); i < absIdx; i++) segmentSize.putIfAbsent(i, maxWalSegmentSize);
            }
            return hnd;
        } catch (IgniteCheckedException | IOException e) {
            try {
                fileIO.close();
            } catch (IOException suppressed) {
                e.addSuppressed(suppressed);
            }
            if (e instanceof StorageException)
                throw (StorageException) e;
            throw e instanceof IOException ? (IOException) e : new IOException(e);
        }
    } catch (IOException e) {
        throw new StorageException("Failed to restore WAL write handle: " + curFile.getAbsolutePath(), e);
    }
}

Example 3 with RecordSerializer

use of org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer in project ignite by apache.

the class IgniteWalIteratorSwitchSegmentTest method checkInvariantSwitchSegmentSize.

/**
 * @param serVer WAL serializer version.
 * @throws Exception If some thing failed.
 */
private void checkInvariantSwitchSegmentSize(int serVer) throws Exception {
    GridKernalContext kctx = new StandaloneGridKernalContext(log, null, null) {

        @Override
        public IgniteCacheObjectProcessor cacheObjects() {
            return new CacheObjectBinaryProcessorImpl(this);
        }
    };
    RecordSerializer serializer = new RecordSerializerFactoryImpl(new GridCacheSharedContext<>(kctx, null, null, null, null, null, null, new IgniteCacheDatabaseSharedManager() {

        @Override
        public int pageSize() {
            return DataStorageConfiguration.DFLT_PAGE_SIZE;
        }
    }, null, null, null, null, null, null, null, null, null, null, null, null, null)).createSerializer(serVer);
    SwitchSegmentRecord switchSegmentRecord = new SwitchSegmentRecord();
    int recordSize = serializer.size(switchSegmentRecord);
    Assert.assertEquals(1, recordSize);
}

Example 4 with RecordSerializer

use of org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer in project ignite by apache.

the class FsyncModeFileWriteAheadLogManager method restoreWriteHandle.

/**
 * @param lastReadPtr Last read WAL file pointer.
 * @return Initialized file write handle.
 * @throws IgniteCheckedException If failed to initialize WAL write handle.
 */
private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws IgniteCheckedException {
    long absIdx = lastReadPtr == null ? 0 : lastReadPtr.index();
    long segNo = absIdx % dsCfg.getWalSegments();
    File curFile = new File(walWorkDir, FileDescriptor.fileName(segNo));
    int offset = lastReadPtr == null ? 0 : lastReadPtr.fileOffset();
    int len = lastReadPtr == null ? 0 : lastReadPtr.length();
    try {
        FileIO fileIO = ioFactory.create(curFile);
        try {
            int serVer = serializerVersion;
            // If we have existing segment, try to read version from it.
            if (lastReadPtr != null) {
                try {
                    serVer = readSerializerVersionAndCompactedFlag(fileIO).get1();
                } catch (SegmentEofException | EOFException ignore) {
                    serVer = serializerVersion;
                }
            }
            RecordSerializer ser = new RecordSerializerFactoryImpl(cctx).createSerializer(serVer);
            if (log.isInfoEnabled())
                log.info("Resuming logging to WAL segment [file=" + curFile.getAbsolutePath() + ", offset=" + offset + ", ver=" + serVer + ']');
            FileWriteHandle hnd = new FileWriteHandle(fileIO, absIdx, offset + len, maxWalSegmentSize, ser);
            // For new handle write serializer version to it.
            if (lastReadPtr == null)
                hnd.writeSerializerVersion();
            archiver.currentWalIndex(absIdx);
            return hnd;
        } catch (IgniteCheckedException | IOException e) {
            fileIO.close();
            throw e;
        }
    } catch (IOException e) {
        throw new IgniteCheckedException("Failed to restore WAL write handle: " + curFile.getAbsolutePath(), e);
    }
}

Example 5 with RecordSerializer

use of org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer in project ignite by apache.

the class FileWriteHandleImpl method close.

/**
 * @return {@code true} If this thread actually closed the segment.
 * @throws IgniteCheckedException If failed.
 * @throws StorageException If failed.
 */
@Override
public boolean close(boolean rollOver) throws IgniteCheckedException, StorageException {
    if (stop.compareAndSet(false, true)) {
        lock.lock();
        try {
            flushOrWait(null);
            try {
                RecordSerializer backwardSerializer = new RecordSerializerFactoryImpl(cctx).createSerializer(serializerVer);
                SwitchSegmentRecord segmentRecord = new SwitchSegmentRecord();
                int switchSegmentRecSize = backwardSerializer.size(segmentRecord);
                if (rollOver && written + switchSegmentRecSize < maxWalSegmentSize) {
                    segmentRecord.size(switchSegmentRecSize);
                    WALPointer segRecPtr = addRecord(segmentRecord);
                    if (segRecPtr != null) {
                        fsync(segRecPtr);
                        switchSegmentRecordOffset = segRecPtr.fileOffset() + switchSegmentRecSize;
                    } else {
                        if (log.isDebugEnabled())
                            log.debug("Not enough space in wal segment to write segment switch");
                    }
                } else {
                    if (log.isDebugEnabled()) {
                        log.debug("Not enough space in wal segment to write segment switch, written=" + written + ", switchSegmentRecSize=" + switchSegmentRecSize);
                    }
                }
                // Unconditional flush (tail of the buffer)
                flushOrWait(null);
                if (mmap) {
                    List<SegmentedRingByteBuffer.ReadSegment> segs = buf.poll(maxWalSegmentSize);
                    if (segs != null) {
                        assert segs.size() == 1;
                        segs.get(0).release();
                    }
                }
                // Do the final fsync.
                if (mode != WALMode.NONE) {
                    if (mmap)
                        ((MappedByteBuffer) buf.buf).force();
                    else
                        fileIO.force();
                    lastFsyncPos = written;
                }
                if (mmap) {
                    try {
                        fileIO.close();
                    } catch (IOException ignore) {
                    // No-op.
                    }
                } else {
                    walWriter.close();
                    if (!rollOver)
                        buf.free();
                }
            } catch (IOException e) {
                throw new StorageException("Failed to close WAL write handle [idx=" + getSegmentId() + "]", e);
            }
            if (log.isDebugEnabled())
                log.debug("Closed WAL write handle [idx=" + getSegmentId() + "]");
            return true;
        } finally {
            if (mmap)
                buf.free();
            lock.unlock();
        }
    } else
        return false;
}