Examples with RandomAccessFileIOFactory org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory used on opensource projects

Example 21 with RandomAccessFileIOFactory

use of org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory in project ignite by apache.

the class StandaloneWalRecordsIteratorTest method createWalIterator.

/**
 * @param walDir Wal directory.
 * @param lowBound Low bound.
 * @param highBound High bound.
 * @param strictCheck Strict check.
 */
private WALIterator createWalIterator(String walDir, WALPointer lowBound, WALPointer highBound, boolean strictCheck, IgniteBiPredicate<WALRecord.RecordType, WALPointer> filter) throws IgniteCheckedException {
    IteratorParametersBuilder params = new IteratorParametersBuilder();
    params.ioFactory(new RandomAccessFileIOFactory()).filesOrDirs(walDir).strictBoundsCheck(strictCheck);
    if (lowBound != null)
        params.from(lowBound);
    if (highBound != null)
        params.to(highBound);
    if (filter != null)
        params.filter(filter);
    return new IgniteWalIteratorFactory(log).iterator(params);
}

Example 22 with RandomAccessFileIOFactory

use of org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory in project ignite by apache.

the class StandaloneWalRecordsIteratorTest method listWalFiles.

/**
 * @param walDir Wal directory.
 */
private List<FileDescriptor> listWalFiles(String walDir) throws IgniteCheckedException {
    IteratorParametersBuilder params = new IteratorParametersBuilder();
    params.ioFactory(new RandomAccessFileIOFactory());
    return new IgniteWalIteratorFactory(log).resolveWalFiles(params.filesOrDirs(walDir));
}

Example 23 with RandomAccessFileIOFactory

use of org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory in project ignite by apache.

the class IgniteWalIteratorSwitchSegmentTest method initiate.

/**
 * Initiate WAL manager.
 *
 * @param serVer WAL serializer version.
 * @param workDir Work directory path.
 * @return Tuple of WAL manager and WAL record serializer.
 * @throws IgniteCheckedException If some think failed.
 */
private T2<IgniteWriteAheadLogManager, RecordSerializer> initiate(int serVer, String workDir) throws IgniteCheckedException {
    GridKernalContext kctx = new StandaloneGridKernalContext(log, null, null) {

        @Override
        protected IgniteConfiguration prepareIgniteConfiguration() {
            IgniteConfiguration cfg = super.prepareIgniteConfiguration();
            cfg.setDataStorageConfiguration(new DataStorageConfiguration().setWalSegmentSize(SEGMENT_SIZE).setWalRecordIteratorBufferSize(SEGMENT_SIZE / 2).setWalMode(WALMode.FSYNC).setWalPath(workDir + WORK_SUB_DIR).setWalArchivePath(workDir + ARCHIVE_SUB_DIR).setFileIOFactory(new RandomAccessFileIOFactory()));
            cfg.setEventStorageSpi(new NoopEventStorageSpi());
            return cfg;
        }

        @Override
        public GridInternalSubscriptionProcessor internalSubscriptionProcessor() {
            return new GridInternalSubscriptionProcessor(this);
        }

        @Override
        public GridEventStorageManager event() {
            return new GridEventStorageManager(this);
        }
    };
    IgniteWriteAheadLogManager walMgr = new FileWriteAheadLogManager(kctx);
    GridTestUtils.setFieldValue(walMgr, "serializerVer", serVer);
    GridCacheSharedContext<?, ?> ctx = new GridCacheSharedContext<>(kctx, null, null, null, null, walMgr, new WalStateManager(kctx), new GridCacheDatabaseSharedManager(kctx), null, null, null, null, null, new GridCacheIoManager(), null, null, null, null, null, null, null);
    walMgr.start(ctx);
    walMgr.onActivate(kctx);
    walMgr.resumeLogging(null);
    RecordSerializer recordSerializer = new RecordSerializerFactoryImpl(ctx).createSerializer(walMgr.serializerVersion());
    return new T2<>(walMgr, recordSerializer);
}

Example 24 with RandomAccessFileIOFactory

use of org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory in project ignite by apache.

the class IgniteWalIteratorSwitchSegmentTest method checkInvariantSwitchSegment.

/**
 * @param serVer WAL serializer version.
 * @throws Exception If some thing failed.
 */
private void checkInvariantSwitchSegment(int serVer) throws Exception {
    String workDir = U.defaultWorkDirectory();
    T2<IgniteWriteAheadLogManager, RecordSerializer> initTup = initiate(serVer, workDir);
    IgniteWriteAheadLogManager walMgr = initTup.get1();
    RecordSerializer recordSerializer = initTup.get2();
    int switchSegmentRecordSize = recordSerializer.size(new SwitchSegmentRecord());
    log.info("switchSegmentRecordSize:" + switchSegmentRecordSize);
    int tailSize = 0;
    /* Initial record payload size. */
    int payloadSize = 1024;
    int recSize = 0;
    MetastoreDataRecord rec = null;
    /* Record size. */
    int recordTypeSize = 1;
    /* Record pointer. */
    int recordPointerSize = 8 + 4 + 4;
    int lowBound = recordTypeSize + recordPointerSize;
    int highBound = lowBound + /*CRC*/
    4;
    int attempt = 1000;
    // Try find how many record need for specific tail size.
    while (true) {
        if (attempt < 0)
            throw new IgniteCheckedException("Can not find any payload size for test, " + "lowBound=" + lowBound + ", highBound=" + highBound);
        if (tailSize >= lowBound && tailSize < highBound)
            break;
        payloadSize++;
        byte[] payload = new byte[payloadSize];
        // Fake record for payload.
        rec = new MetastoreDataRecord("0", payload);
        recSize = recordSerializer.size(rec);
        tailSize = (SEGMENT_SIZE - HEADER_RECORD_SIZE) % recSize;
        attempt--;
    }
    Assert.assertNotNull(rec);
    int recordsToWrite = SEGMENT_SIZE / recSize;
    log.info("records to write " + recordsToWrite + " tail size " + (SEGMENT_SIZE - HEADER_RECORD_SIZE) % recSize);
    // Add more record for rollover to the next segment.
    recordsToWrite += 100;
    for (int i = 0; i < recordsToWrite; i++) walMgr.log(new MetastoreDataRecord(rec.key(), rec.value()));
    walMgr.flush(null, true);
    SegmentAware segmentAware = GridTestUtils.getFieldValue(walMgr, "segmentAware");
    // Await archiver move segment to WAL archive.
    waitForCondition(() -> segmentAware.lastArchivedAbsoluteIndex() == 0, 5_000);
    // Filling tail some garbage. Simulate tail garbage on rotate segment in WAL work directory.
    if (switchSegmentRecordSize > 1) {
        File seg = new File(workDir + ARCHIVE_SUB_DIR + "/0000000000000000.wal");
        FileIOFactory ioFactory = new RandomAccessFileIOFactory();
        FileIO seg0 = ioFactory.create(seg);
        byte[] bytes = new byte[tailSize];
        Random rnd = new Random();
        rnd.nextBytes(bytes);
        // Some record type.
        bytes[0] = (byte) (METASTORE_DATA_RECORD.ordinal() + 1);
        seg0.position((int) (seg0.size() - tailSize));
        seg0.write(bytes, 0, tailSize);
        seg0.force(true);
        seg0.close();
    }
    int expRecords = recordsToWrite;
    int actualRecords = 0;
    // Check that switch segment works as expected and all record is reachable.
    try (WALIterator it = walMgr.replay(null)) {
        while (it.hasNext()) {
            IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
            WALRecord rec0 = tup.get2();
            if (rec0.type() == METASTORE_DATA_RECORD)
                actualRecords++;
        }
    }
    Assert.assertEquals("Not all records read during iteration.", expRecords, actualRecords);
}

Example 25 with RandomAccessFileIOFactory

use of org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory in project ignite by apache.

the class IgniteWALTailIsReachedDuringIterationOverArchiveTest method doTest.

/**
 * @param walMgr WAL manager.
 * @param it WAL iterator.
 * @throws IOException If IO exception.
 * @throws IgniteCheckedException If WAL iterator failed.
 */
private void doTest(IgniteWriteAheadLogManager walMgr, WALIterator it) throws IOException, IgniteCheckedException {
    File walArchiveDir = U.field(walMgr, "walArchiveDir");
    IgniteWalIteratorFactory iteratorFactory = new IgniteWalIteratorFactory();
    List<FileDescriptor> descs = iteratorFactory.resolveWalFiles(new IteratorParametersBuilder().filesOrDirs(walArchiveDir));
    int maxIndex = descs.size() - 1;
    int minIndex = 1;
    int corruptedIdx = current().nextInt(minIndex, maxIndex);
    log.info("Corrupted segment with idx:" + corruptedIdx);
    WALPointer corruptedPtr = corruptedWAlSegmentFile(descs.get(corruptedIdx), new RandomAccessFileIOFactory(), iteratorFactory);
    log.info("Should fail on ptr " + corruptedPtr);
    WALPointer lastReadPtr = null;
    boolean exception = false;
    try (WALIterator it0 = it) {
        while (it0.hasNextX()) {
            IgniteBiTuple<WALPointer, WALRecord> tup = it0.nextX();
            lastReadPtr = tup.get1();
        }
    } catch (IgniteCheckedException e) {
        if (e.getMessage().contains("WAL tail reached in archive directory, WAL segment file is corrupted") || e.getMessage().contains("WAL tail reached not in the last available segment"))
            exception = true;
    }
    Assert.assertNotNull(lastReadPtr);
    if (!exception) {
        fail("Last read ptr=" + lastReadPtr + ", corruptedPtr=" + corruptedPtr);
    }
}