Examples with IteratorParametersBuilder org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder used on opensource projects

Example 1 with IteratorParametersBuilder

use of org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder in project ignite by apache.

the class IgnitePdsSporadicDataRecordsOnBackupTest method findSporadicDataRecords.

/**
 * Returns a number of {@link DataRecord} that do not have a xid version,
 * in other words, number of records that were created by GridCacheUtils#createBackupPostProcessingClosure().
 *
 * @throws IgniteCheckedException If failed.
 */
private long findSporadicDataRecords(String nodeFolderName) throws IgniteCheckedException {
    File dbDir = new File(U.defaultWorkDirectory(), "db");
    File commonWalDir = new File(dbDir, "wal");
    File walDir = new File(commonWalDir, nodeFolderName);
    File walArchiveDir = new File(new File(commonWalDir, "archive"), nodeFolderName);
    assertTrue(walDir.exists());
    assertTrue(walArchiveDir.exists());
    IteratorParametersBuilder params = new IteratorParametersBuilder();
    params.bufferSize(1024 * 1024);
    params.filesOrDirs(walDir, walArchiveDir);
    params.filter((type, pointer) -> type == WALRecord.RecordType.DATA_RECORD_V2);
    int cacheId = CU.cacheId(TX_CACHE_NAME);
    long createOpCnt = 0;
    try (WALIterator itr = new IgniteWalIteratorFactory().iterator(params)) {
        while (itr.hasNext()) {
            IgniteBiTuple<WALPointer, WALRecord> walEntry = itr.next();
            assertTrue(walEntry.get2() instanceof DataRecord);
            DataRecord rec = (DataRecord) walEntry.get2();
            Predicate<DataEntry> filter = e -> e.cacheId() == cacheId && GridCacheOperation.CREATE == e.op() && e.nearXidVersion() == null;
            for (int i = 0; i < rec.entryCount(); i++) {
                if (filter.test(rec.get(i)))
                    createOpCnt++;
            }
        }
    }
    return createOpCnt;
}

Example 2 with IteratorParametersBuilder

use of org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder in project ignite by apache.

the class IgniteWalReaderTest method runRemoveOperationTest.

/**
 * Test if DELETE operation can be found after mixed cache operations including remove().
 *
 * @param mode Cache Atomicity Mode.
 * @throws Exception if failed.
 */
private void runRemoveOperationTest(CacheAtomicityMode mode) throws Exception {
    Ignite ignite = startGrid();
    ignite.cluster().active(true);
    createCache2(ignite, mode);
    ignite.cluster().active(false);
    String subfolderName = genDbSubfolderName(ignite, 0);
    stopGrid();
    String workDir = U.defaultWorkDirectory();
    IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log);
    IteratorParametersBuilder params = createIteratorParametersBuilder(workDir, subfolderName);
    params.filesOrDirs(workDir);
    StringBuilder sb = new StringBuilder();
    Map<GridCacheOperation, Integer> operationsFound = new EnumMap<>(GridCacheOperation.class);
    scanIterateAndCount(factory, params, 0, 0, null, dataRecord -> {
        sb.append("{");
        for (int i = 0; i < dataRecord.entryCount(); i++) {
            DataEntry entry = dataRecord.get(i);
            GridCacheOperation op = entry.op();
            Integer cnt = operationsFound.get(op);
            operationsFound.put(op, cnt == null ? 1 : (cnt + 1));
            if (entry instanceof UnwrapDataEntry) {
                UnwrapDataEntry entry1 = (UnwrapDataEntry) entry;
                sb.append(entry1.op()).append(" for ").append(entry1.unwrappedKey());
                GridCacheVersion ver = entry.nearXidVersion();
                sb.append(", ");
                if (ver != null)
                    sb.append("tx=").append(ver).append(", ");
            }
        }
        sb.append("}\n");
    });
    final Integer deletesFound = operationsFound.get(DELETE);
    if (log.isInfoEnabled())
        log.info(sb.toString());
    assertTrue("Delete operations should be found in log: " + operationsFound, deletesFound != null && deletesFound > 0);
}

Example 3 with IteratorParametersBuilder

use of org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder in project ignite by apache.

the class IgniteWalReaderTest method testFillWalAndReadRecords.

/**
 * @throws Exception if failed.
 */
@Test
public void testFillWalAndReadRecords() throws Exception {
    setWalAndArchiveToSameVal = false;
    Ignite ignite0 = startGrid();
    ignite0.cluster().active(true);
    Serializable consistentId = (Serializable) ignite0.cluster().localNode().consistentId();
    String subfolderName = genNewStyleSubfolderName(0, (UUID) consistentId);
    int cacheObjectsToWrite = 10_000;
    putDummyRecords(ignite0, cacheObjectsToWrite);
    stopGrid();
    String workDir = U.defaultWorkDirectory();
    File db = U.resolveWorkDirectory(workDir, DFLT_STORE_DIR, false);
    IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log);
    IteratorParametersBuilder params = createIteratorParametersBuilder(workDir, subfolderName).filesOrDirs(db);
    // Check iteratorArchiveDirectory and iteratorArchiveFiles are same.
    int cntArchiveDir = iterateAndCount(factory.iterator(params));
    log.info("Total records loaded using directory : " + cntArchiveDir);
    assertTrue(cntArchiveDir > 0);
    // Check iteratorArchiveFiles + iteratorWorkFiles iterate over all entries.
    int[] checkKeyIterArr = new int[cacheObjectsToWrite];
    fill(checkKeyIterArr, 0);
    iterateAndCountDataRecord(factory.iterator(params), (o1, o2) -> checkKeyIterArr[(Integer) o1]++, null);
    for (int i = 0; i < cacheObjectsToWrite; i++) assertTrue("Iterator didn't find key=" + i, checkKeyIterArr[i] > 0);
}

Example 4 with IteratorParametersBuilder

use of org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder in project ignite by apache.

the class IgniteWalReaderTest method testIteratorWithCurrentKernelContext.

/**
 * Tests WAL iterator which uses shared cache context of currently started Ignite node.
 */
@Test
public void testIteratorWithCurrentKernelContext() throws Exception {
    IgniteEx ignite = startGrid(0);
    ignite.cluster().active(true);
    int cntEntries = 100;
    putDummyRecords(ignite, cntEntries);
    String workDir = U.defaultWorkDirectory();
    IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log);
    IteratorParametersBuilder iterParametersBuilder = createIteratorParametersBuilder(workDir, genDbSubfolderName(ignite, 0)).filesOrDirs(workDir).binaryMetadataFileStoreDir(null).marshallerMappingFileStoreDir(null).sharedContext(ignite.context().cache().context());
    AtomicInteger cnt = new AtomicInteger();
    IgniteBiInClosure<Object, Object> objConsumer = (key, val) -> {
        if (val instanceof IndexedObject) {
            assertEquals(key, ((IndexedObject) val).iVal);
            assertEquals(key, cnt.getAndIncrement());
        }
    };
    iterateAndCountDataRecord(factory.iterator(iterParametersBuilder.copy()), objConsumer, null);
    assertEquals(cntEntries, cnt.get());
    // Test without converting non primary types.
    iterParametersBuilder.keepBinary(true);
    cnt.set(0);
    IgniteBiInClosure<Object, Object> binObjConsumer = (key, val) -> {
        if (val instanceof BinaryObject) {
            assertEquals(key, ((BinaryObject) val).field("iVal"));
            assertEquals(key, cnt.getAndIncrement());
        }
    };
    iterateAndCountDataRecord(factory.iterator(iterParametersBuilder.copy()), binObjConsumer, null);
    assertEquals(cntEntries, cnt.get());
}

Example 5 with IteratorParametersBuilder

use of org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder in project ignite by apache.

the class IgniteWalReaderTest method testReadEmptyWal.

/**
 * Tests reading of empty WAL from non filled cluster.
 *
 * @throws Exception if failed.
 */
@Test
public void testReadEmptyWal() throws Exception {
    customWalMode = WALMode.FSYNC;
    Ignite ignite = startGrid();
    ignite.cluster().active(true);
    ignite.cluster().active(false);
    final String subfolderName = genDbSubfolderName(ignite, 0);
    stopGrid();
    String workDir = U.defaultWorkDirectory();
    IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log);
    IteratorParametersBuilder iterParametersBuilder = createIteratorParametersBuilder(workDir, subfolderName).filesOrDirs(workDir);
    scanIterateAndCount(factory, iterParametersBuilder, 0, 0, null, null);
}