Examples with FileDescriptor org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor used on opensource projects

Example 1 with FileDescriptor

use of org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor in project ignite by apache.

the class IgniteWalIteratorFactory method hasGaps.

/**
 * @param descriptors File descriptors.
 * @return List of tuples, low and high index segments with gap.
 */
public List<T2<Long, Long>> hasGaps(@NotNull List<FileDescriptor> descriptors) throws IllegalArgumentException {
    List<T2<Long, Long>> gaps = new ArrayList<>();
    Iterator<FileDescriptor> it = descriptors.iterator();
    FileDescriptor prevFd = null;
    while (it.hasNext()) {
        FileDescriptor nextFd = it.next();
        if (prevFd == null) {
            prevFd = nextFd;
            continue;
        }
        if (prevFd.idx() + 1 != nextFd.idx())
            gaps.add(new T2<>(prevFd.idx(), nextFd.idx()));
        prevFd = nextFd;
    }
    return gaps;
}

Example 2 with FileDescriptor

use of org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor in project ignite by apache.

the class IgniteWalIteratorFactory method readFileDescriptor.

/**
 * @param file File to read.
 * @param ioFactory IO factory.
 */
private FileDescriptor readFileDescriptor(File file, FileIOFactory ioFactory) {
    FileDescriptor ds = new FileDescriptor(file);
    try (SegmentIO fileIO = ds.toReadOnlyIO(ioFactory);
        ByteBufferExpander buf = new ByteBufferExpander(HEADER_RECORD_SIZE, ByteOrder.nativeOrder())) {
        final DataInput in = segmentFileInputFactory.createFileInput(fileIO, buf);
        // Header record must be agnostic to the serializer version.
        final int type = in.readUnsignedByte();
        if (type == RecordType.STOP_ITERATION_RECORD_TYPE) {
            if (log.isInfoEnabled())
                log.info("Reached logical end of the segment for file " + file);
            return null;
        }
        WALPointer ptr = readPosition(in);
        return new FileDescriptor(file, ptr.index());
    } catch (IOException e) {
        U.warn(log, "Failed to scan index from file [" + file + "]. Skipping this file during iteration", e);
        return null;
    }
}

Example 3 with FileDescriptor

use of org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor in project ignite by apache.

the class IgniteWalIteratorFactory method resolveWalFiles.

/**
 * This methods checks all provided files to be correct WAL segment.
 * Header record and its position is checked. WAL position is used to determine real index.
 * File index from file name is ignored.
 *
 * @param iteratorParametersBuilder IteratorParametersBuilder.
 * @return list of file descriptors with checked header records, having correct file index is set
 */
public List<FileDescriptor> resolveWalFiles(IteratorParametersBuilder iteratorParametersBuilder) {
    File[] filesOrDirs = iteratorParametersBuilder.filesOrDirs;
    if (filesOrDirs == null || filesOrDirs.length == 0)
        return Collections.emptyList();
    final FileIOFactory ioFactory = iteratorParametersBuilder.ioFactory;
    final TreeSet<FileDescriptor> descriptors = new TreeSet<>();
    for (File file : filesOrDirs) {
        if (file.isDirectory()) {
            try {
                walkFileTree(file.toPath(), new SimpleFileVisitor<Path>() {

                    @Override
                    public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) {
                        addFileDescriptor(path.toFile(), ioFactory, descriptors);
                        return FileVisitResult.CONTINUE;
                    }
                });
            } catch (IOException e) {
                U.error(log, "Failed to walk directories from root [" + file + "]. Skipping this directory.", e);
            }
            continue;
        }
        addFileDescriptor(file, ioFactory, descriptors);
    }
    return new ArrayList<>(descriptors);
}

Example 4 with FileDescriptor

use of org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor in project ignite by apache.

the class StandaloneWalRecordsIteratorTest method testStrictBounds.

/**
 * Check correct check bounds.
 *
 * @throws Exception if test failed.
 */
@Test
public void testStrictBounds() throws Exception {
    String dir = createWalFiles();
    WALPointer lowBound = null, highBound = null;
    for (IgniteBiTuple<WALPointer, WALRecord> p : createWalIterator(dir, null, null, false)) {
        if (lowBound == null)
            lowBound = p.get1();
        highBound = p.get1();
    }
    assertNotNull(lowBound);
    assertNotNull(highBound);
    createWalIterator(dir, lowBound, highBound, true);
    final WALPointer lBound = lowBound;
    final WALPointer hBound = highBound;
    // noinspection ThrowableNotThrown
    GridTestUtils.assertThrows(log, () -> {
        createWalIterator(dir, new WALPointer(lBound.index() - 1, 0, 0), hBound, true);
        return 0;
    }, IgniteCheckedException.class, null);
    // noinspection ThrowableNotThrown
    GridTestUtils.assertThrows(log, () -> {
        createWalIterator(dir, lBound, new WALPointer(hBound.index() + 1, 0, 0), true);
        return 0;
    }, IgniteCheckedException.class, null);
    List<FileDescriptor> walFiles = listWalFiles(dir);
    assertNotNull(walFiles);
    assertTrue(!walFiles.isEmpty());
    assertTrue(walFiles.get(new Random().nextInt(walFiles.size())).file().delete());
    // noinspection ThrowableNotThrown
    GridTestUtils.assertThrows(log, () -> {
        createWalIterator(dir, lBound, hBound, true);
        return 0;
    }, IgniteCheckedException.class, null);
}

Example 5 with FileDescriptor

use of org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor in project ignite by apache.

the class IgniteWithoutArchiverWalIteratorInvalidCrcTest method nodeShouldStartIfLogicalRecordCorruptedAfterCheckpointOrWalStart.

/**
 *  A logical record was corrupted or just doesn't exist because the end of wal is reached, after start checkpoint without end.
 *  -----||------||----X----> OR ----X----->
 *  We recover all before it, and start the node.
 */
@Test
@WithSystemProperty(key = GridCacheDatabaseSharedManager.IGNITE_PDS_SKIP_CHECKPOINT_ON_NODE_STOP, value = "true")
public void nodeShouldStartIfLogicalRecordCorruptedAfterCheckpointOrWalStart() throws Exception {
    startNodeAndPopulate();
    stopGrid(0);
    IgniteWriteAheadLogManager walMgr = ignite.context().cache().context().wal();
    File walDir = U.field(walMgr, "walWorkDir");
    IgniteWalIteratorFactory iterFactory = new IgniteWalIteratorFactory();
    List<FileDescriptor> walFiles = getWalFiles(walDir, iterFactory);
    FileDescriptor lastWalFile = walFiles.get(walFiles.size() - 1);
    List<WALPointer> pointers = WalTestUtils.getPointers(lastWalFile, iterFactory, LOGICAL);
    WalTestUtils.corruptWalSegmentFile(lastWalFile, pointers.get(pointers.size() - 1));
    IgniteEx ex = startGrid(0);
    ex.cluster().active(true);
}