Examples with IteratorParametersBuilder org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder used on opensource projects

Example 6 with IteratorParametersBuilder

use of org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder in project ignite by apache.

the class IgniteWalReaderTest method testFillWalWithDifferentTypes.

/**
 * @throws Exception if failed.
 */
@Test
public void testFillWalWithDifferentTypes() throws Exception {
    Ignite ig = startGrid();
    ig.cluster().active(true);
    IgniteCache<Object, Object> addlCache = ig.getOrCreateCache(CACHE_ADDL_NAME);
    addlCache.put("1", "2");
    addlCache.put(1, 2);
    addlCache.put(1L, 2L);
    addlCache.put(TestEnum.A, "Enum_As_Key");
    addlCache.put("Enum_As_Value", TestEnum.B);
    addlCache.put(TestEnum.C, TestEnum.C);
    addlCache.put("Serializable", new TestSerializable(42));
    addlCache.put(new TestSerializable(42), "Serializable_As_Key");
    addlCache.put("Externalizable", new TestExternalizable(42));
    addlCache.put(new TestExternalizable(42), "Externalizable_As_Key");
    addlCache.put(292, new IndexedObject(292));
    String search1 = "SomeUnexpectedStringValueAsKeyToSearch";
    Collection<String> ctrlStringsToSearch = new HashSet<>();
    ctrlStringsToSearch.add(search1);
    Collection<String> ctrlStringsForBinaryObjSearch = new HashSet<>();
    ctrlStringsForBinaryObjSearch.add(search1);
    addlCache.put(search1, "SearchKey");
    String search2 = "SomeTestStringContainerToBePrintedLongLine";
    TestStringContainerToBePrinted val = new TestStringContainerToBePrinted(search2);
    // will validate original toString() was called
    ctrlStringsToSearch.add("v = [ " + val.getClass().getSimpleName() + "{data='" + search2 + "'}]");
    ctrlStringsForBinaryObjSearch.add(search2);
    addlCache.put("SearchValue", val);
    String search3 = "SomeTestStringContainerToBePrintedLongLine2";
    TestStringContainerToBePrinted key = new TestStringContainerToBePrinted(search3);
    // will validate original toString() was called
    ctrlStringsToSearch.add("k = " + key.getClass().getSimpleName() + "{data='" + search3 + "'}");
    // validate only string itself
    ctrlStringsForBinaryObjSearch.add(search3);
    addlCache.put(key, "SearchKey");
    int cntEntries = addlCache.size();
    Map<Object, Object> ctrlMap = new HashMap<>();
    for (Cache.Entry<Object, Object> next : addlCache) ctrlMap.put(next.getKey(), next.getValue());
    Map<Object, Object> ctrlMapForBinaryObjects = new HashMap<>();
    for (Cache.Entry<Object, Object> next : addlCache) ctrlMapForBinaryObjects.put(next.getKey(), next.getValue());
    String subfolderName = genDbSubfolderName(ig, 0);
    // Wait async allocation wal segment file by archiver.
    Thread.sleep(1000);
    stopGrid("node0", false);
    String workDir = U.defaultWorkDirectory();
    IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log);
    IteratorParametersBuilder params0 = createIteratorParametersBuilder(workDir, subfolderName);
    params0.filesOrDirs(workDir);
    IgniteBiInClosure<Object, Object> objConsumer = (key12, val1) -> {
        log.info("K: [" + key12 + ", " + (key12 != null ? key12.getClass().getName() : "?") + "]" + " V: [" + val1 + ", " + (val1 != null ? val1.getClass().getName() : "?") + "]");
        boolean rmv = remove(ctrlMap, key12, val1);
        if (!rmv) {
            String msg = "Unable to remove pair from control map " + "K: [" + key12 + "] V: [" + val1 + "]";
            log.error(msg);
        }
        assertFalse(val1 instanceof BinaryObject);
    };
    IgniteInClosure<DataRecord> toStrChecker = record -> {
        String strRepresentation = record.toString();
        for (Iterator<String> iter = ctrlStringsToSearch.iterator(); iter.hasNext(); ) {
            final String next = iter.next();
            if (strRepresentation.contains(next)) {
                iter.remove();
                break;
            }
        }
    };
    scanIterateAndCount(factory, params0, cntEntries, 0, objConsumer, toStrChecker);
    assertTrue(" Control Map is not empty after reading entries: " + ctrlMap, ctrlMap.isEmpty());
    assertTrue(" Control Map for strings in entries is not empty after" + " reading records: " + ctrlStringsToSearch, ctrlStringsToSearch.isEmpty());
    IgniteBiInClosure<Object, Object> binObjConsumer = (key13, val12) -> {
        log.info("K(KeepBinary): [" + key13 + ", " + (key13 != null ? key13.getClass().getName() : "?") + "]" + " V(KeepBinary): [" + val12 + ", " + (val12 != null ? val12.getClass().getName() : "?") + "]");
        boolean rmv = remove(ctrlMapForBinaryObjects, key13, val12);
        if (!rmv) {
            if (key13 instanceof BinaryObject) {
                BinaryObject keyBinObj = (BinaryObject) key13;
                String binaryObjTypeName = keyBinObj.type().typeName();
                if (Objects.equals(TestStringContainerToBePrinted.class.getName(), binaryObjTypeName)) {
                    String data = keyBinObj.field("data");
                    rmv = ctrlMapForBinaryObjects.remove(new TestStringContainerToBePrinted(data)) != null;
                } else if (Objects.equals(TestSerializable.class.getName(), binaryObjTypeName)) {
                    Integer iVal = keyBinObj.field("iVal");
                    rmv = ctrlMapForBinaryObjects.remove(new TestSerializable(iVal)) != null;
                } else if (Objects.equals(TestEnum.class.getName(), binaryObjTypeName)) {
                    TestEnum key1 = TestEnum.values()[keyBinObj.enumOrdinal()];
                    rmv = ctrlMapForBinaryObjects.remove(key1) != null;
                }
            } else if (val12 instanceof BinaryObject) {
                // don't compare BO values, just remove by key
                rmv = ctrlMapForBinaryObjects.remove(key13) != null;
            }
        }
        if (!rmv)
            log.error("Unable to remove pair from control map " + "K: [" + key13 + "] V: [" + val12 + "]");
        if (val12 instanceof BinaryObject) {
            BinaryObject binaryObj = (BinaryObject) val12;
            String binaryObjTypeName = binaryObj.type().typeName();
            if (Objects.equals(IndexedObject.class.getName(), binaryObjTypeName)) {
                assertEquals(binaryObj.field("iVal").toString(), binaryObj.field("jVal").toString());
                byte[] data = binaryObj.field("data");
                for (byte datum : data) assertTrue(datum >= 'A' && datum <= 'A' + 10);
            }
        }
    };
    IgniteInClosure<DataRecord> binObjToStrChecker = record -> {
        String strRepresentation = record.toString();
        for (Iterator<String> iter = ctrlStringsForBinaryObjSearch.iterator(); iter.hasNext(); ) {
            final String next = iter.next();
            if (strRepresentation.contains(next)) {
                iter.remove();
                break;
            }
        }
    };
    IteratorParametersBuilder params1 = createIteratorParametersBuilder(workDir, subfolderName);
    params1.filesOrDirs(workDir).keepBinary(true);
    // Validate same WAL log with flag binary objects only
    IgniteWalIteratorFactory keepBinFactory = new IgniteWalIteratorFactory(log);
    scanIterateAndCount(keepBinFactory, params1, cntEntries, 0, binObjConsumer, binObjToStrChecker);
    assertTrue(" Control Map is not empty after reading entries: " + ctrlMapForBinaryObjects, ctrlMapForBinaryObjects.isEmpty());
    assertTrue(" Control Map for strings in entries is not empty after" + " reading records: " + ctrlStringsForBinaryObjSearch, ctrlStringsForBinaryObjSearch.isEmpty());
}

Example 7 with IteratorParametersBuilder

use of org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder in project ignite by apache.

the class IgniteWalReplayingAfterRestartTest method testWalRecordsAfterRestart.

/**
 * @throws Exception If failed.
 */
@Test
public void testWalRecordsAfterRestart() throws Exception {
    IgniteEx ignite = startGrid(0);
    ignite.cluster().active(true);
    IgniteCache<Integer, byte[]> cache = ignite.getOrCreateCache(CACHE_NAME);
    int key = 0;
    while (ignite.context().cache().context().wal().lastArchivedSegment() < SEGMENTS_CNT) cache.put(key++ % PART_NUM, new byte[1024]);
    ignite.context().cache().context().database().waitForCheckpoint("test-checkpoint");
    long lastArchived = ignite.context().cache().context().wal().lastArchivedSegment();
    while (ignite.context().cache().context().wal().lastArchivedSegment() < lastArchived + 1) cache.put(key++ % PART_NUM, new byte[1024]);
    stopGrid(0);
    // There are no exceptions should be thrown here.
    ignite = startGrid(0);
    ignite.cluster().active();
    // delta records should always follow PageSnapshot records.
    String workDir = U.defaultWorkDirectory();
    IteratorParametersBuilder builder = new IteratorParametersBuilder().filesOrDirs(workDir).filter((rec, ptr) -> rec.purpose() == PHYSICAL);
    Map<FullPageId, PageSnapshot> snapshots = new HashMap<>();
    try (WALIterator it = new IgniteWalIteratorFactory().iterator(builder)) {
        while (it.hasNext()) {
            IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
            WALRecord rec = tup.get2();
            if (rec.type() == CHECKPOINT_RECORD)
                snapshots.clear();
            // let's check partition meta pages.
            if (rec instanceof PageSnapshot) {
                PageSnapshot snpRec = (PageSnapshot) rec;
                assertFalse(snapshots.containsKey(snpRec.fullPageId()));
                snapshots.put(snpRec.fullPageId(), snpRec);
            } else if (rec instanceof MetaPageUpdatePartitionDataRecord) {
                MetaPageUpdatePartitionDataRecord metaRec = (MetaPageUpdatePartitionDataRecord) rec;
                assertTrue(snapshots.containsKey(metaRec.fullPageId()));
            }
        }
    }
}

Example 8 with IteratorParametersBuilder

use of org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder in project ignite by apache.

the class StandaloneWalRecordsIteratorTest method createWalIterator.

/**
 * Creates WALIterator associated with files inside walDir.
 *
 * @param walDir - path to WAL directory.
 * @return WALIterator associated with files inside walDir.
 * @throws IgniteCheckedException if error occur.
 */
private WALIterator createWalIterator(String walDir) throws IgniteCheckedException {
    IteratorParametersBuilder params = new IteratorParametersBuilder();
    params.ioFactory(new CountedFileIOFactory());
    return new IgniteWalIteratorFactory(log).iterator(params.filesOrDirs(walDir));
}

Example 9 with IteratorParametersBuilder

use of org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder in project ignite by apache.

the class StandaloneWalRecordsIteratorTest method createWalIterator.

/**
 * @param walDir Wal directory.
 * @param lowBound Low bound.
 * @param highBound High bound.
 * @param strictCheck Strict check.
 */
private WALIterator createWalIterator(String walDir, WALPointer lowBound, WALPointer highBound, boolean strictCheck, IgniteBiPredicate<WALRecord.RecordType, WALPointer> filter) throws IgniteCheckedException {
    IteratorParametersBuilder params = new IteratorParametersBuilder();
    params.ioFactory(new RandomAccessFileIOFactory()).filesOrDirs(walDir).strictBoundsCheck(strictCheck);
    if (lowBound != null)
        params.from(lowBound);
    if (highBound != null)
        params.to(highBound);
    if (filter != null)
        params.filter(filter);
    return new IgniteWalIteratorFactory(log).iterator(params);
}

Example 10 with IteratorParametersBuilder

use of org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder in project ignite by apache.

the class StandaloneWalRecordsIteratorTest method listWalFiles.

/**
 * @param walDir Wal directory.
 */
private List<FileDescriptor> listWalFiles(String walDir) throws IgniteCheckedException {
    IteratorParametersBuilder params = new IteratorParametersBuilder();
    params.ioFactory(new RandomAccessFileIOFactory());
    return new IgniteWalIteratorFactory(log).resolveWalFiles(params.filesOrDirs(walDir));
}