Examples with IteratorParametersBuilder org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder used on opensource projects

Example 11 with IteratorParametersBuilder

use of org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder in project ignite by apache.

the class IgniteWALTailIsReachedDuringIterationOverArchiveTest method doTest.

/**
 * @param walMgr WAL manager.
 * @param it WAL iterator.
 * @throws IOException If IO exception.
 * @throws IgniteCheckedException If WAL iterator failed.
 */
private void doTest(IgniteWriteAheadLogManager walMgr, WALIterator it) throws IOException, IgniteCheckedException {
    File walArchiveDir = U.field(walMgr, "walArchiveDir");
    IgniteWalIteratorFactory iteratorFactory = new IgniteWalIteratorFactory();
    List<FileDescriptor> descs = iteratorFactory.resolveWalFiles(new IteratorParametersBuilder().filesOrDirs(walArchiveDir));
    int maxIndex = descs.size() - 1;
    int minIndex = 1;
    int corruptedIdx = current().nextInt(minIndex, maxIndex);
    log.info("Corrupted segment with idx:" + corruptedIdx);
    WALPointer corruptedPtr = corruptedWAlSegmentFile(descs.get(corruptedIdx), new RandomAccessFileIOFactory(), iteratorFactory);
    log.info("Should fail on ptr " + corruptedPtr);
    WALPointer lastReadPtr = null;
    boolean exception = false;
    try (WALIterator it0 = it) {
        while (it0.hasNextX()) {
            IgniteBiTuple<WALPointer, WALRecord> tup = it0.nextX();
            lastReadPtr = tup.get1();
        }
    } catch (IgniteCheckedException e) {
        if (e.getMessage().contains("WAL tail reached in archive directory, WAL segment file is corrupted") || e.getMessage().contains("WAL tail reached not in the last available segment"))
            exception = true;
    }
    Assert.assertNotNull(lastReadPtr);
    if (!exception) {
        fail("Last read ptr=" + lastReadPtr + ", corruptedPtr=" + corruptedPtr);
    }
}

Example 12 with IteratorParametersBuilder

use of org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder in project ignite by apache.

the class IgniteWalIteratorExceptionDuringReadTest method test.

/**
 * @throws Exception If failed.
 */
@Test
public void test() throws Exception {
    IgniteEx ig = startGrid();
    ig.cluster().state(ClusterState.ACTIVE);
    IgniteCache<Integer, byte[]> cache = ig.cache(DEFAULT_CACHE_NAME);
    for (int i = 0; i < 20 * 4; i++) cache.put(i, new byte[1024 * 1024]);
    ig.cluster().state(ClusterState.INACTIVE);
    IgniteWalIteratorFactory iterFactory = new IgniteWalIteratorFactory(log);
    WALPointer failOnPtr = new WALPointer(3, 1024 * 1024 * 5, 0);
    String failMsg = "test fail message";
    IteratorParametersBuilder builder = new IteratorParametersBuilder().filesOrDirs(U.defaultWorkDirectory()).filter((r, ptr) -> {
        if (ptr.compareTo(failOnPtr) >= 0)
            throw new TestRuntimeException(failMsg);
        return true;
    });
    try (WALIterator it = iterFactory.iterator(builder)) {
        WALPointer ptr = null;
        boolean failed = false;
        try {
            while (it.hasNext()) {
                IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
                ptr = tup.get1();
            }
        } catch (IgniteException e) {
            Assert.assertNotNull(ptr);
            Assert.assertEquals(failOnPtr.index(), ptr.index());
            Assert.assertTrue(ptr.compareTo(failOnPtr) < 0);
            failed = X.hasCause(e, TestRuntimeException.class);
        }
        assertTrue(failed);
    }
}

Example 13 with IteratorParametersBuilder

use of org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder in project ignite by apache.

the class IgniteWalReaderTest method testTxFillWalAndExtractDataRecords.

/**
 * Places records under transaction, checks its value using WAL.
 *
 * @throws Exception if failed.
 */
@Test
public void testTxFillWalAndExtractDataRecords() throws Exception {
    Ignite ignite0 = startGrid();
    ignite0.cluster().active(true);
    int cntEntries = 1000;
    int txCnt = 100;
    IgniteCache<Object, Object> entries = txPutDummyRecords(ignite0, cntEntries, txCnt);
    Map<Object, Object> ctrlMap = new HashMap<>();
    for (Cache.Entry<Object, Object> next : entries) ctrlMap.put(next.getKey(), next.getValue());
    String subfolderName = genDbSubfolderName(ignite0, 0);
    stopGrid();
    String workDir = U.defaultWorkDirectory();
    IteratorParametersBuilder params = createIteratorParametersBuilder(workDir, subfolderName);
    params.filesOrDirs(workDir);
    IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log);
    IgniteBiInClosure<Object, Object> objConsumer = (key, val) -> {
        boolean rmv = remove(ctrlMap, key, val);
        if (!rmv)
            log.error("Unable to remove Key and value from control Map K:[" + key + "] V: [" + val + "]");
        if (val instanceof IndexedObject) {
            IndexedObject indexedObj = (IndexedObject) val;
            assertEquals(indexedObj.iVal, indexedObj.jVal);
            assertEquals(indexedObj.iVal, key);
            for (byte datum : indexedObj.getData()) assertTrue(datum >= 'A' && datum <= 'A' + 10);
        }
    };
    scanIterateAndCount(factory, params, cntEntries, txCnt, objConsumer, null);
    assertTrue(" Control Map is not empty after reading entries " + ctrlMap, ctrlMap.isEmpty());
}

Example 14 with IteratorParametersBuilder

use of org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder in project ignite by apache.

the class IgniteWalReaderTest method createIteratorParametersBuilder.

/**
 * @param workDir Work directory.
 * @param subfolderName Subfolder name.
 * @return WAL iterator factory.
 * @throws IgniteCheckedException If failed.
 */
@NotNull
private IteratorParametersBuilder createIteratorParametersBuilder(String workDir, String subfolderName) throws IgniteCheckedException {
    File binaryMeta = U.resolveWorkDirectory(workDir, DataStorageConfiguration.DFLT_BINARY_METADATA_PATH, false);
    File binaryMetaWithConsId = new File(binaryMeta, subfolderName);
    File marshallerMapping = U.resolveWorkDirectory(workDir, DataStorageConfiguration.DFLT_MARSHALLER_PATH, false);
    return new IteratorParametersBuilder().binaryMetadataFileStoreDir(binaryMetaWithConsId).marshallerMappingFileStoreDir(marshallerMapping);
}

Example 15 with IteratorParametersBuilder

use of org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder in project ignite by apache.

the class IgniteWalReaderTest method testCheckBoundsIterator.

/**
 * @throws Exception If failed.
 */
@Test
public void testCheckBoundsIterator() throws Exception {
    Ignite ignite = startGrid("node0");
    ignite.cluster().active(true);
    try (IgniteDataStreamer<Integer, IndexedObject> st = ignite.dataStreamer(CACHE_NAME)) {
        st.allowOverwrite(true);
        for (int i = 0; i < 10_000; i++) st.addData(i, new IndexedObject(i));
    }
    stopAllGrids();
    List<WALPointer> wal = new ArrayList<>();
    String workDir = U.defaultWorkDirectory();
    IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory();
    try (WALIterator it = factory.iterator(workDir)) {
        while (it.hasNext()) {
            IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
            wal.add(tup.get1());
        }
    }
    Random rnd = new Random();
    int from0 = rnd.nextInt(wal.size() - 2) + 1;
    int to0 = wal.size() - 1;
    // +1 for skip first record.
    WALPointer exp0First = wal.get(from0);
    WALPointer exp0Last = wal.get(to0);
    T2<WALPointer, WALRecord> actl0First = null;
    T2<WALPointer, WALRecord> actl0Last = null;
    int records0 = 0;
    try (WALIterator it = factory.iterator(exp0First, workDir)) {
        while (it.hasNext()) {
            IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
            if (actl0First == null)
                actl0First = new T2<>(tup.get1(), tup.get2());
            actl0Last = new T2<>(tup.get1(), tup.get2());
            records0++;
        }
    }
    log.info("Check REPLAY FROM:" + exp0First + "\n" + "expFirst=" + exp0First + " actlFirst=" + actl0First + ", " + "expLast=" + exp0Last + " actlLast=" + actl0Last);
    // +1 because bound include.
    Assert.assertEquals(to0 - from0 + 1, records0);
    Assert.assertNotNull(actl0First);
    Assert.assertNotNull(actl0Last);
    Assert.assertEquals(exp0First, actl0First.get1());
    Assert.assertEquals(exp0Last, actl0Last.get1());
    int from1 = 0;
    int to1 = rnd.nextInt(wal.size() - 3) + 1;
    // -3 for skip last record.
    WALPointer exp1First = wal.get(from1);
    WALPointer exp1Last = wal.get(to1);
    T2<WALPointer, WALRecord> actl1First = null;
    T2<WALPointer, WALRecord> actl1Last = null;
    int records1 = 0;
    try (WALIterator it = factory.iterator(new IteratorParametersBuilder().filesOrDirs(workDir).to(exp1Last))) {
        while (it.hasNext()) {
            IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
            if (actl1First == null)
                actl1First = new T2<>(tup.get1(), tup.get2());
            actl1Last = new T2<>(tup.get1(), tup.get2());
            records1++;
        }
    }
    log.info("Check REPLAY TO:" + exp1Last + "\n" + "expFirst=" + exp1First + " actlFirst=" + actl1First + ", " + "expLast=" + exp1Last + " actlLast=" + actl1Last);
    // +1 because bound include.
    Assert.assertEquals(to1 - from1 + 1, records1);
    Assert.assertNotNull(actl1First);
    Assert.assertNotNull(actl1Last);
    Assert.assertEquals(exp1First, actl1First.get1());
    Assert.assertEquals(exp1Last, actl1Last.get1());
    int from2 = rnd.nextInt(wal.size() - 2);
    int to2 = rnd.nextInt((wal.size() - 1) - from2) + from2;
    WALPointer exp2First = wal.get(from2);
    WALPointer exp2Last = wal.get(to2);
    T2<WALPointer, WALRecord> actl2First = null;
    T2<WALPointer, WALRecord> actl2Last = null;
    int records2 = 0;
    try (WALIterator it = factory.iterator(new IteratorParametersBuilder().filesOrDirs(workDir).from(exp2First).to(exp2Last))) {
        while (it.hasNext()) {
            IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
            if (actl2First == null)
                actl2First = new T2<>(tup.get1(), tup.get2());
            actl2Last = new T2<>(tup.get1(), tup.get2());
            records2++;
        }
    }
    log.info("Check REPLAY BETWEEN:" + exp2First + " " + exp2Last + "\n" + "expFirst=" + exp2First + " actlFirst=" + actl2First + ", " + "expLast=" + exp2Last + " actlLast=" + actl2Last);
    // +1 because bound include.
    Assert.assertEquals(to2 - from2 + 1, records2);
    Assert.assertNotNull(actl2First);
    Assert.assertNotNull(actl2Last);
    Assert.assertEquals(exp2First, actl2First.get1());
    Assert.assertEquals(exp2Last, actl2Last.get1());
}