Examples with IgniteWalIteratorFactory org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory used on opensource projects

Example 6 with IgniteWalIteratorFactory

use of org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory in project ignite by apache.

the class IgniteWalReaderTest method testTxRecordsReadWoBinaryMeta.

/**
 * Tests transaction generation and WAL for putAll cache operation.
 * @throws Exception if failed.
 */
public void testTxRecordsReadWoBinaryMeta() throws Exception {
    clearProperties = true;
    System.setProperty(IgniteSystemProperties.IGNITE_WAL_LOG_TX_RECORDS, "true");
    final Ignite ignite = startGrid("node0");
    ignite.active(true);
    final Map<Object, IndexedObject> map = new TreeMap<>();
    for (int i = 0; i < 1000; i++) map.put(i, new IndexedObject(i));
    ignite.cache(CACHE_NAME).putAll(map);
    ignite.active(false);
    final String workDir = U.defaultWorkDirectory();
    final String subfolderName = genDbSubfolderName(ignite, 0);
    stopAllGrids();
    IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(new NullLogger(), PAGE_SIZE, null, null, false);
    scanIterateAndCount(factory, workDir, subfolderName, 1000, 1, null, null);
}

Example 7 with IgniteWalIteratorFactory

use of org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory in project ignite by apache.

the class IgniteWalReaderTest method testTxFillWalAndExtractDataRecords.

/**
 * Places records under transaction, checks its value using WAL.
 *
 * @throws Exception if failed.
 */
public void testTxFillWalAndExtractDataRecords() throws Exception {
    final int cntEntries = 1000;
    final int txCnt = 100;
    final Ignite ignite0 = startGrid("node0");
    ignite0.active(true);
    final IgniteCache<Object, Object> entries = txPutDummyRecords(ignite0, cntEntries, txCnt);
    final Map<Object, Object> ctrlMap = new HashMap<>();
    for (Cache.Entry<Object, Object> next : entries) ctrlMap.put(next.getKey(), next.getValue());
    final String subfolderName = genDbSubfolderName(ignite0, 0);
    stopGrid("node0");
    final String workDir = U.defaultWorkDirectory();
    final File binaryMeta = U.resolveWorkDirectory(workDir, "binary_meta", false);
    final File binaryMetaWithConsId = new File(binaryMeta, subfolderName);
    final File marshallerMapping = U.resolveWorkDirectory(workDir, "marshaller", false);
    final IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log, PAGE_SIZE, binaryMetaWithConsId, marshallerMapping);
    final IgniteBiInClosure<Object, Object> objConsumer = new IgniteBiInClosure<Object, Object>() {

        @Override
        public void apply(Object key, Object val) {
            boolean rmv = remove(ctrlMap, key, val);
            if (!rmv)
                log.error("Unable to remove Key and value from control Map K:[" + key + "] V: [" + val + "]");
            if (val instanceof IndexedObject) {
                IndexedObject indexedObj = (IndexedObject) val;
                assertEquals(indexedObj.iVal, indexedObj.jVal);
                assertEquals(indexedObj.iVal, key);
                for (byte datum : indexedObj.getData()) assertTrue(datum >= 'A' && datum <= 'A' + 10);
            }
        }
    };
    scanIterateAndCount(factory, workDir, subfolderName, cntEntries, txCnt, objConsumer, null);
    assertTrue(" Control Map is not empty after reading entries " + ctrlMap, ctrlMap.isEmpty());
}

Example 8 with IgniteWalIteratorFactory

use of org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory in project ignite by apache.

the class IgniteWalReaderTest method createWalIteratorFactory.

/**
 * @param workDir Work directory.
 * @param subfolderName Subfolder name.
 * @return WAL iterator factory.
 * @throws IgniteCheckedException If failed.
 */
@NotNull
private IgniteWalIteratorFactory createWalIteratorFactory(final String workDir, final String subfolderName) throws IgniteCheckedException {
    final File binaryMeta = U.resolveWorkDirectory(workDir, "binary_meta", false);
    final File binaryMetaWithConsId = new File(binaryMeta, subfolderName);
    final File marshallerMapping = U.resolveWorkDirectory(workDir, "marshaller", false);
    return new IgniteWalIteratorFactory(log, PAGE_SIZE, binaryMetaWithConsId, marshallerMapping, false);
}

Example 9 with IgniteWalIteratorFactory

use of org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory in project ignite by apache.

the class IgniteWalReaderTest method testFillWalForExactSegmentsCount.

/**
 * Tests archive completed event is fired.
 *
 * @throws Exception if failed.
 */
public void testFillWalForExactSegmentsCount() throws Exception {
    customWalMode = WALMode.FSYNC;
    final CountDownLatch reqSegments = new CountDownLatch(15);
    final Ignite ignite = startGrid("node0");
    ignite.active(true);
    final IgniteEvents evts = ignite.events();
    if (!evts.isEnabled(EVT_WAL_SEGMENT_ARCHIVED))
        assertTrue("nothing to test", false);
    evts.localListen(new IgnitePredicate<Event>() {

        @Override
        public boolean apply(Event e) {
            WalSegmentArchivedEvent archComplEvt = (WalSegmentArchivedEvent) e;
            long idx = archComplEvt.getAbsWalSegmentIdx();
            log.info("Finished archive for segment [" + idx + ", " + archComplEvt.getArchiveFile() + "]: [" + e + "]");
            reqSegments.countDown();
            return true;
        }
    }, EVT_WAL_SEGMENT_ARCHIVED);
    int totalEntries = 0;
    while (reqSegments.getCount() > 0) {
        final int write = 500;
        putAllDummyRecords(ignite, write);
        totalEntries += write;
        Assert.assertTrue("Too much entries generated, but segments was not become available", totalEntries < 10000);
    }
    final String subfolderName = genDbSubfolderName(ignite, 0);
    stopGrid("node0");
    final String workDir = U.defaultWorkDirectory();
    final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
    scanIterateAndCount(factory, workDir, subfolderName, totalEntries, 0, null, null);
}

Example 10 with IgniteWalIteratorFactory

use of org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory in project ignite by apache.

the class IgniteWalConverter method main.

/**
 * @param args Args.
 */
public static void main(String[] args) throws Exception {
    if (args.length < 2)
        throw new IllegalArgumentException("\nYou need to provide:\n" + "\t1. Size of pages, which was selected for file store (1024, 2048, 4096, etc).\n" + "\t2. Path to dir with wal files.\n" + "\t3. (Optional) Path to dir with archive wal files.");
    PageIO.registerH2(H2InnerIO.VERSIONS, H2LeafIO.VERSIONS);
    H2ExtrasInnerIO.register();
    H2ExtrasLeafIO.register();
    boolean printRecords = IgniteSystemProperties.getBoolean("PRINT_RECORDS", false);
    boolean printStat = IgniteSystemProperties.getBoolean("PRINT_STAT", true);
    final IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(new NullLogger(), Integer.parseInt(args[0]), null, null, false);
    final File walWorkDirWithConsistentId = new File(args[1]);
    final File[] workFiles = walWorkDirWithConsistentId.listFiles(FileWriteAheadLogManager.WAL_SEGMENT_FILE_FILTER);
    if (workFiles == null)
        throw new IllegalArgumentException("No .wal files in dir: " + args[1]);
    @Nullable final WalStat stat = printStat ? new WalStat() : null;
    try (WALIterator stIt = factory.iteratorWorkFiles(workFiles)) {
        while (stIt.hasNextX()) {
            IgniteBiTuple<WALPointer, WALRecord> next = stIt.nextX();
            final WALPointer pointer = next.get1();
            final WALRecord record = next.get2();
            if (stat != null)
                stat.registerRecord(record, pointer, true);
            if (printRecords)
                System.out.println("[W] " + record);
        }
    }
    if (args.length >= 3) {
        final File walArchiveDirWithConsistentId = new File(args[2]);
        try (WALIterator stIt = factory.iteratorArchiveDirectory(walArchiveDirWithConsistentId)) {
            while (stIt.hasNextX()) {
                IgniteBiTuple<WALPointer, WALRecord> next = stIt.nextX();
                final WALPointer pointer = next.get1();
                final WALRecord record = next.get2();
                if (stat != null)
                    stat.registerRecord(record, pointer, false);
                if (printRecords)
                    System.out.println("[A] " + record);
            }
        }
    }
    System.err.flush();
    if (stat != null)
        System.out.println("Statistic collected:\n" + stat.toString());
}