Examples with FileWriteHandle org.apache.ignite.internal.processors.cache.persistence.wal.filehandle.FileWriteHandle used on opensource projects

Example 6 with FileWriteHandle

use of org.apache.ignite.internal.processors.cache.persistence.wal.filehandle.FileWriteHandle in project ignite by apache.

the class FileWriteAheadLogManager method restoreWriteHandle.

/**
 * @param lastReadPtr Last read WAL file pointer.
 * @return Initialized file write handle.
 * @throws StorageException If failed to initialize WAL write handle.
 */
private FileWriteHandle restoreWriteHandle(@Nullable WALPointer lastReadPtr) throws StorageException {
    long absIdx = lastReadPtr == null ? 0 : lastReadPtr.index();
    @Nullable FileArchiver archiver0 = archiver;
    long segNo = archiver0 == null ? absIdx : absIdx % dsCfg.getWalSegments();
    File curFile = new File(walWorkDir, fileName(segNo));
    int off = lastReadPtr == null ? 0 : lastReadPtr.fileOffset();
    int len = lastReadPtr == null ? 0 : lastReadPtr.length();
    try {
        SegmentIO fileIO = new SegmentIO(absIdx, ioFactory.create(curFile));
        IgniteInClosure<FileIO> lsnr = createWalFileListener;
        if (lsnr != null)
            lsnr.apply(fileIO);
        try {
            int serVer = serializerVer;
            // If we have existing segment, try to read version from it.
            if (lastReadPtr != null) {
                try {
                    serVer = readSegmentHeader(fileIO, segmentFileInputFactory).getSerializerVersion();
                } catch (SegmentEofException | EOFException ignore) {
                    serVer = serializerVer;
                }
            }
            RecordSerializer ser = new RecordSerializerFactoryImpl(cctx).createSerializer(serVer);
            if (log.isInfoEnabled()) {
                log.info("Resuming logging to WAL segment [file=" + curFile.getAbsolutePath() + ", offset=" + off + ", ver=" + serVer + ']');
            }
            FileWriteHandle hnd = fileHandleManager.initHandle(fileIO, off + len, ser);
            segmentAware.curAbsWalIdx(absIdx);
            FileDescriptor[] walArchiveFiles = walArchiveFiles();
            segmentAware.minReserveIndex(F.isEmpty(walArchiveFiles) ? -1 : walArchiveFiles[0].idx - 1);
            segmentAware.lastTruncatedArchiveIdx(F.isEmpty(walArchiveFiles) ? -1 : walArchiveFiles[0].idx - 1);
            if (archiver0 == null)
                segmentAware.setLastArchivedAbsoluteIndex(absIdx - 1);
            // Getting segment sizes.
            F.asList(walArchiveDir.listFiles(WAL_SEGMENT_COMPACTED_OR_RAW_FILE_FILTER)).stream().map(FileDescriptor::new).forEach(fd -> {
                if (fd.isCompressed())
                    segmentSize.put(fd.idx(), fd.file().length());
                else
                    segmentSize.putIfAbsent(fd.idx(), fd.file().length());
            });
            // Size of the 8th segment will be set in #resumeLogging.
            if (archiver0 != null) {
                for (long i = absIdx - (absIdx % dsCfg.getWalSegments()); i < absIdx; i++) segmentSize.putIfAbsent(i, maxWalSegmentSize);
            }
            return hnd;
        } catch (IgniteCheckedException | IOException e) {
            try {
                fileIO.close();
            } catch (IOException suppressed) {
                e.addSuppressed(suppressed);
            }
            if (e instanceof StorageException)
                throw (StorageException) e;
            throw e instanceof IOException ? (IOException) e : new IOException(e);
        }
    } catch (IOException e) {
        throw new StorageException("Failed to restore WAL write handle: " + curFile.getAbsolutePath(), e);
    }
}

Example 7 with FileWriteHandle

use of org.apache.ignite.internal.processors.cache.persistence.wal.filehandle.FileWriteHandle in project ignite by apache.

the class IgnitePdsStartWIthEmptyArchive method test.

/**
 * @throws Exception If failed.
 */
@Test
public void test() throws Exception {
    IgniteEx ig = startGrid(0);
    ig.cluster().active(true);
    FileWriteAheadLogManager walMgr = (FileWriteAheadLogManager) ig.context().cache().context().wal();
    // Populate data for generate WAL archive segments.
    try (IgniteDataStreamer<Integer, byte[]> st = ig.dataStreamer(DEFAULT_CACHE_NAME)) {
        int entries = 1000;
        for (int i = 0; i < entries; i++) st.addData(i, new byte[1024 * 1024]);
    }
    File archiveDir = U.field(walMgr, "walArchiveDir");
    stopGrid(0, false);
    SegmentAware beforeSaw = U.field(walMgr, "segmentAware");
    long beforeLastArchivedAbsoluteIdx = beforeSaw.lastArchivedAbsoluteIndex();
    FileWriteHandle fhBefore = U.field(walMgr, "currHnd");
    long idxBefore = fhBefore.getSegmentId();
    File[] files = archiveDir.listFiles(WAL_SEGMENT_COMPACTED_OR_RAW_FILE_FILTER);
    Arrays.sort(files);
    // Cleanup archive directory.
    for (File f : files) {
        if (f.delete())
            log.info("File " + f.getAbsolutePath() + " deleted");
    }
    Assert.assertEquals(0, archiveDir.listFiles().length);
    evts.clear();
    // Restart grid again after archive was removed.
    ig = startGrid(0);
    walMgr = (FileWriteAheadLogManager) ig.context().cache().context().wal();
    SegmentAware afterSaw = U.field(walMgr, "segmentAware");
    long afterLastArchivedAbsoluteIndex = afterSaw.lastArchivedAbsoluteIndex();
    int segments = ig.configuration().getDataStorageConfiguration().getWalSegments();
    Assert.assertTrue("lastArchivedBeforeIdx=" + beforeLastArchivedAbsoluteIdx + ", lastArchivedAfterIdx=" + afterLastArchivedAbsoluteIndex + ",  segments=" + segments, afterLastArchivedAbsoluteIndex >= (beforeLastArchivedAbsoluteIdx - segments));
    ig.cluster().active(true);
    FileWriteHandle fhAfter = U.field(walMgr, "currHnd");
    Assert.assertNotNull(fhAfter);
    long idxAfter = fhAfter.getSegmentId();
    Assert.assertEquals(idxBefore, idxAfter);
    Assert.assertTrue(idxAfter >= beforeLastArchivedAbsoluteIdx);
    log.info("currentIdx=" + idxAfter + ", lastArchivedBeforeIdx=" + beforeLastArchivedAbsoluteIdx + ", lastArchivedAfteridx=" + afterLastArchivedAbsoluteIndex + ",  segments=" + segments);
    // One is a last archived, secod is a current write segment.
    final long awaitAchviedSegments = idxAfter - afterLastArchivedAbsoluteIndex - 2;
    // Await all current available semgment will be archived.
    assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() {

        @Override
        public boolean apply() {
            long cut = evts.keySet().stream().filter(e -> e > afterLastArchivedAbsoluteIndex).count();
            return cut >= awaitAchviedSegments;
        }
    }, 10_000));
}

Example 8 with FileWriteHandle

use of org.apache.ignite.internal.processors.cache.persistence.wal.filehandle.FileWriteHandle in project ignite by apache.

the class IgniteLogicalRecoveryWithParamsTest method getWalPos.

/**
 * @param grid Ignite instance.
 * @return Returns current wal position.
 */
private int getWalPos(Ignite grid) {
    IgniteEx grid0 = (IgniteEx) grid;
    FileWriteAheadLogManager walMgr = (FileWriteAheadLogManager) grid0.context().cache().context().wal();
    FileWriteHandle fhAfter = U.field(walMgr, "currHnd");
    try {
        fhAfter.fsync(null);
    } catch (IgniteCheckedException e) {
        U.warn(log, e);
    }
    return fhAfter.position().fileOffset();
}

Example 9 with FileWriteHandle

use of org.apache.ignite.internal.processors.cache.persistence.wal.filehandle.FileWriteHandle in project ignite by apache.

the class FileWriteAheadLogManager method hasIndex.

/**
 * Checking for the existence of an index.
 *
 * @param absIdx Segment index.
 * @return {@code True} exists.
 */
private boolean hasIndex(long absIdx) {
    String segmentName = fileName(absIdx);
    boolean inArchive = new File(walArchiveDir, segmentName).exists() || new File(walArchiveDir, segmentName + ZIP_SUFFIX).exists();
    if (inArchive)
        return true;
    if (absIdx <= lastArchivedIndex())
        return false;
    FileWriteHandle cur = currHnd;
    return cur != null && cur.getSegmentId() >= absIdx;
}

Example 10 with FileWriteHandle

use of org.apache.ignite.internal.processors.cache.persistence.wal.filehandle.FileWriteHandle in project ignite by apache.

the class IgniteLocalWalSizeTest method checkLocalSegmentSizes.

/**
 * Check that local segment sizes in the memory and actual match.
 *
 * @param n Node.
 * @throws Exception If failed.
 */
private void checkLocalSegmentSizes(IgniteEx n) throws Exception {
    disableWal(n, true);
    if (walMgr(n).getSegmentRouter().hasArchive()) {
        assertTrue(waitForCondition(() -> walMgr(n).lastArchivedSegment() == walMgr(n).currentSegment() - 1, getTestTimeout()));
    }
    if (n.context().config().getDataStorageConfiguration().isWalCompactionEnabled()) {
        assertTrue(waitForCondition(() -> walMgr(n).lastCompactedSegment() == walMgr(n).lastArchivedSegment(), getTestTimeout()));
    }
    File walWorkDir = getFieldValue(walMgr(n), "walWorkDir");
    File walArchiveDir = getFieldValue(walMgr(n), "walArchiveDir");
    Map<Long, Long> expSegmentSize = new HashMap<>();
    F.asList(walArchiveDir.listFiles(WAL_SEGMENT_COMPACTED_OR_RAW_FILE_FILTER)).stream().map(FileDescriptor::new).forEach(fd -> {
        if (fd.isCompressed())
            expSegmentSize.put(fd.idx(), fd.file().length());
        else
            expSegmentSize.putIfAbsent(fd.idx(), fd.file().length());
    });
    FileWriteHandle currHnd = getFieldValue(walMgr(n), "currHnd");
    if (!walArchiveDir.equals(walWorkDir)) {
        long absIdx = currHnd.getSegmentId();
        int segments = n.configuration().getDataStorageConfiguration().getWalSegments();
        for (long i = absIdx - (absIdx % segments); i <= absIdx; i++) expSegmentSize.putIfAbsent(i, new File(walWorkDir, fileName(i % segments)).length());
    }
    assertEquals(currHnd.getSegmentId() + 1, expSegmentSize.size());
    Map<Long, Long> segmentSize = getFieldValue(walMgr(n), "segmentSize");
    assertEquals(expSegmentSize.size(), segmentSize.size());
    expSegmentSize.forEach((idx, size) -> {
        assertEquals(idx.toString(), size, segmentSize.get(idx));
        assertEquals(idx.toString(), size.longValue(), walMgr(n).segmentSize(idx));
    });
    assertEquals(0, walMgr(n).segmentSize(currHnd.getSegmentId() + 1));
}