Examples with IgniteWalIteratorFactory org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory used on opensource projects

Example 1 with IgniteWalIteratorFactory

use of org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory in project ignite by apache.

the class IgniteWalReaderTest method testReadEmptyWal.

/**
 * Tests reading of empty WAL from non filled cluster.
 *
 * @throws Exception if failed.
 */
public void testReadEmptyWal() throws Exception {
    customWalMode = WALMode.FSYNC;
    final Ignite ignite = startGrid("node0");
    ignite.active(true);
    ignite.active(false);
    final String subfolderName = genDbSubfolderName(ignite, 0);
    stopGrid("node0");
    final String workDir = U.defaultWorkDirectory();
    final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
    scanIterateAndCount(factory, workDir, subfolderName, 0, 0, null, null);
}

Example 2 with IgniteWalIteratorFactory

use of org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory in project ignite by apache.

the class IgniteWalReaderTest method testFillWalAndReadRecords.

/**
 * @throws Exception if failed.
 */
public void testFillWalAndReadRecords() throws Exception {
    setWalAndArchiveToSameValue = false;
    final int cacheObjectsToWrite = 10000;
    final Ignite ignite0 = startGrid("node0");
    ignite0.active(true);
    final Serializable consistentId = (Serializable) ignite0.cluster().localNode().consistentId();
    final String subfolderName = genNewStyleSubfolderName(0, (UUID) consistentId);
    putDummyRecords(ignite0, cacheObjectsToWrite);
    stopGrid("node0");
    final String workDir = U.defaultWorkDirectory();
    final File db = U.resolveWorkDirectory(workDir, DFLT_STORE_DIR, false);
    final File wal = new File(db, "wal");
    final File walArchive = setWalAndArchiveToSameValue ? wal : new File(wal, "archive");
    final MockWalIteratorFactory mockItFactory = new MockWalIteratorFactory(log, PAGE_SIZE, consistentId, subfolderName, WAL_SEGMENTS);
    final WALIterator it = mockItFactory.iterator(wal, walArchive);
    final int cntUsingMockIter = iterateAndCount(it, false);
    log.info("Total records loaded " + cntUsingMockIter);
    assertTrue(cntUsingMockIter > 0);
    assertTrue(cntUsingMockIter > cacheObjectsToWrite);
    final File walArchiveDirWithConsistentId = new File(walArchive, subfolderName);
    final File walWorkDirWithConsistentId = new File(wal, subfolderName);
    final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
    final int cntArchiveDir = iterateAndCount(factory.iteratorArchiveDirectory(walArchiveDirWithConsistentId));
    log.info("Total records loaded using directory : " + cntArchiveDir);
    final int cntArchiveFileByFile = iterateAndCount(factory.iteratorArchiveFiles(walArchiveDirWithConsistentId.listFiles(FileWriteAheadLogManager.WAL_SEGMENT_FILE_FILTER)));
    log.info("Total records loaded using archive directory (file-by-file): " + cntArchiveFileByFile);
    assertTrue(cntArchiveFileByFile > cacheObjectsToWrite);
    assertTrue(cntArchiveDir > cacheObjectsToWrite);
    assertTrue(cntArchiveDir == cntArchiveFileByFile);
    // really count2 may be less because work dir correct loading is not supported yet
    assertTrue("Mock based reader loaded " + cntUsingMockIter + " records " + "but standalone has loaded only " + cntArchiveDir, cntUsingMockIter >= cntArchiveDir);
    final File[] workFiles = walWorkDirWithConsistentId.listFiles(FileWriteAheadLogManager.WAL_SEGMENT_FILE_FILTER);
    final int cntWork = iterateAndCount(factory.iteratorWorkFiles(workFiles));
    log.info("Total records loaded from work: " + cntWork);
    assertTrue("Work iterator loaded [" + cntWork + "] " + "Archive iterator loaded [" + cntArchiveFileByFile + "]; " + "mock iterator [" + cntUsingMockIter + "]", cntWork + cntArchiveFileByFile == cntUsingMockIter);
}

Example 3 with IgniteWalIteratorFactory

use of org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory in project ignite by apache.

the class IgniteWalReaderTest method runRemoveOperationTest.

/**
 * Test if DELETE operation can be found after mixed cache operations including remove().
 *
 * @throws Exception if failed.
 * @param mode Cache Atomicity Mode.
 */
private void runRemoveOperationTest(CacheAtomicityMode mode) throws Exception {
    final Ignite ignite = startGrid("node0");
    ignite.active(true);
    createCache2(ignite, mode);
    ignite.active(false);
    final String subfolderName = genDbSubfolderName(ignite, 0);
    stopGrid("node0");
    final String workDir = U.defaultWorkDirectory();
    final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
    final StringBuilder builder = new StringBuilder();
    final Map<GridCacheOperation, Integer> operationsFound = new EnumMap<>(GridCacheOperation.class);
    scanIterateAndCount(factory, workDir, subfolderName, 0, 0, null, new IgniteInClosure<DataRecord>() {

        @Override
        public void apply(DataRecord dataRecord) {
            final List<DataEntry> entries = dataRecord.writeEntries();
            builder.append("{");
            for (DataEntry entry : entries) {
                final GridCacheOperation op = entry.op();
                final Integer cnt = operationsFound.get(op);
                operationsFound.put(op, cnt == null ? 1 : (cnt + 1));
                if (entry instanceof UnwrapDataEntry) {
                    final UnwrapDataEntry entry1 = (UnwrapDataEntry) entry;
                    builder.append(entry1.op()).append(" for ").append(entry1.unwrappedKey());
                    final GridCacheVersion ver = entry.nearXidVersion();
                    builder.append(", ");
                    if (ver != null)
                        builder.append("tx=").append(ver).append(", ");
                }
            }
            builder.append("}\n");
        }
    });
    final Integer deletesFound = operationsFound.get(DELETE);
    if (log.isInfoEnabled())
        log.info(builder.toString());
    assertTrue("Delete operations should be found in log: " + operationsFound, deletesFound != null && deletesFound > 0);
}

Example 4 with IgniteWalIteratorFactory

use of org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory in project ignite by apache.

the class IgniteWalReaderTest method testFillWalWithDifferentTypes.

/**
 * @throws Exception if failed.
 */
public void testFillWalWithDifferentTypes() throws Exception {
    int cntEntries;
    final Map<Object, Object> ctrlMap = new HashMap<>();
    final Map<Object, Object> ctrlMapForBinaryObjects = new HashMap<>();
    final Collection<String> ctrlStringsToSearch = new HashSet<>();
    final Collection<String> ctrlStringsForBinaryObjSearch = new HashSet<>();
    final Ignite ignite0 = startGrid("node0");
    ignite0.active(true);
    final IgniteCache<Object, Object> addlCache = ignite0.getOrCreateCache(CACHE_ADDL_NAME);
    addlCache.put("1", "2");
    addlCache.put(1, 2);
    addlCache.put(1L, 2L);
    addlCache.put(TestEnum.A, "Enum_As_Key");
    addlCache.put("Enum_As_Value", TestEnum.B);
    addlCache.put(TestEnum.C, TestEnum.C);
    addlCache.put("Serializable", new TestSerializable(42));
    addlCache.put(new TestSerializable(42), "Serializable_As_Key");
    addlCache.put("Externalizable", new TestExternalizable(42));
    addlCache.put(new TestExternalizable(42), "Externalizable_As_Key");
    addlCache.put(292, new IndexedObject(292));
    final String search1 = "SomeUnexpectedStringValueAsKeyToSearch";
    ctrlStringsToSearch.add(search1);
    ctrlStringsForBinaryObjSearch.add(search1);
    addlCache.put(search1, "SearchKey");
    String search2 = "SomeTestStringContainerToBePrintedLongLine";
    final TestStringContainerToBePrinted val = new TestStringContainerToBePrinted(search2);
    // will validate original toString() was called
    ctrlStringsToSearch.add(val.toString());
    ctrlStringsForBinaryObjSearch.add(search2);
    addlCache.put("SearchValue", val);
    String search3 = "SomeTestStringContainerToBePrintedLongLine2";
    final TestStringContainerToBePrinted key = new TestStringContainerToBePrinted(search3);
    // will validate original toString() was called
    ctrlStringsToSearch.add(key.toString());
    // validate only string itself
    ctrlStringsForBinaryObjSearch.add(search3);
    addlCache.put(key, "SearchKey");
    cntEntries = addlCache.size();
    for (Cache.Entry<Object, Object> next : addlCache) ctrlMap.put(next.getKey(), next.getValue());
    for (Cache.Entry<Object, Object> next : addlCache) ctrlMapForBinaryObjects.put(next.getKey(), next.getValue());
    final String subfolderName = genDbSubfolderName(ignite0, 0);
    stopGrid("node0");
    final String workDir = U.defaultWorkDirectory();
    final File binaryMeta = U.resolveWorkDirectory(workDir, "binary_meta", false);
    final File binaryMetaWithNodeSubfolder = new File(binaryMeta, subfolderName);
    final File marshallerMapping = U.resolveWorkDirectory(workDir, "marshaller", false);
    final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
    final IgniteBiInClosure<Object, Object> objConsumer = new IgniteBiInClosure<Object, Object>() {

        @Override
        public void apply(Object key, Object val) {
            log.info("K: [" + key + ", " + (key != null ? key.getClass().getName() : "?") + "]" + " V: [" + val + ", " + (val != null ? val.getClass().getName() : "?") + "]");
            boolean rmv = remove(ctrlMap, key, val);
            if (!rmv) {
                String msg = "Unable to remove pair from control map " + "K: [" + key + "] V: [" + val + "]";
                log.error(msg);
            }
            assertFalse(val instanceof BinaryObject);
        }
    };
    final IgniteInClosure<DataRecord> toStrChecker = new IgniteInClosure<DataRecord>() {

        @Override
        public void apply(DataRecord record) {
            String strRepresentation = record.toString();
            for (Iterator<String> iter = ctrlStringsToSearch.iterator(); iter.hasNext(); ) {
                final String next = iter.next();
                if (strRepresentation.contains(next)) {
                    iter.remove();
                    break;
                }
            }
        }
    };
    scanIterateAndCount(factory, workDir, subfolderName, cntEntries, 0, objConsumer, toStrChecker);
    assertTrue(" Control Map is not empty after reading entries: " + ctrlMap, ctrlMap.isEmpty());
    assertTrue(" Control Map for strings in entries is not empty after" + " reading records: " + ctrlStringsToSearch, ctrlStringsToSearch.isEmpty());
    // Validate same WAL log with flag binary objects only
    final IgniteWalIteratorFactory keepBinFactory = new IgniteWalIteratorFactory(log, PAGE_SIZE, binaryMetaWithNodeSubfolder, marshallerMapping, true);
    final IgniteBiInClosure<Object, Object> binObjConsumer = new IgniteBiInClosure<Object, Object>() {

        @Override
        public void apply(Object key, Object val) {
            log.info("K(KeepBinary): [" + key + ", " + (key != null ? key.getClass().getName() : "?") + "]" + " V(KeepBinary): [" + val + ", " + (val != null ? val.getClass().getName() : "?") + "]");
            boolean rmv = remove(ctrlMapForBinaryObjects, key, val);
            if (!rmv) {
                if (key instanceof BinaryObject) {
                    BinaryObject keyBinObj = (BinaryObject) key;
                    String binaryObjTypeName = keyBinObj.type().typeName();
                    if (Objects.equals(TestStringContainerToBePrinted.class.getName(), binaryObjTypeName)) {
                        String data = keyBinObj.field("data");
                        rmv = ctrlMapForBinaryObjects.remove(new TestStringContainerToBePrinted(data)) != null;
                    } else if (Objects.equals(TestSerializable.class.getName(), binaryObjTypeName)) {
                        Integer iVal = keyBinObj.field("iVal");
                        rmv = ctrlMapForBinaryObjects.remove(new TestSerializable(iVal)) != null;
                    } else if (Objects.equals(TestEnum.class.getName(), binaryObjTypeName)) {
                        TestEnum key1 = TestEnum.values()[keyBinObj.enumOrdinal()];
                        rmv = ctrlMapForBinaryObjects.remove(key1) != null;
                    }
                } else if (val instanceof BinaryObject) {
                    // don't compare BO values, just remove by key
                    rmv = ctrlMapForBinaryObjects.remove(key) != null;
                }
            }
            if (!rmv)
                log.error("Unable to remove pair from control map " + "K: [" + key + "] V: [" + val + "]");
            if (val instanceof BinaryObject) {
                BinaryObject binaryObj = (BinaryObject) val;
                String binaryObjTypeName = binaryObj.type().typeName();
                if (Objects.equals(IndexedObject.class.getName(), binaryObjTypeName)) {
                    assertEquals(binaryObj.field("iVal").toString(), binaryObj.field("jVal").toString());
                    byte[] data = binaryObj.field("data");
                    for (byte datum : data) assertTrue(datum >= 'A' && datum <= 'A' + 10);
                }
            }
        }
    };
    final IgniteInClosure<DataRecord> binObjToStrChecker = new IgniteInClosure<DataRecord>() {

        @Override
        public void apply(DataRecord record) {
            String strRepresentation = record.toString();
            for (Iterator<String> iter = ctrlStringsForBinaryObjSearch.iterator(); iter.hasNext(); ) {
                final String next = iter.next();
                if (strRepresentation.contains(next)) {
                    iter.remove();
                    break;
                }
            }
        }
    };
    scanIterateAndCount(keepBinFactory, workDir, subfolderName, cntEntries, 0, binObjConsumer, binObjToStrChecker);
    assertTrue(" Control Map is not empty after reading entries: " + ctrlMapForBinaryObjects, ctrlMapForBinaryObjects.isEmpty());
    assertTrue(" Control Map for strings in entries is not empty after" + " reading records: " + ctrlStringsForBinaryObjSearch, ctrlStringsForBinaryObjSearch.isEmpty());
}

Example 5 with IgniteWalIteratorFactory

use of org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory in project ignite by apache.

the class IgniteWalReaderTest method testPutAllTxIntoTwoNodes.

/**
 * Tests transaction generation and WAL for putAll cache operation.
 * @throws Exception if failed.
 */
public void testPutAllTxIntoTwoNodes() throws Exception {
    final Ignite ignite = startGrid("node0");
    final Ignite ignite1 = startGrid(1);
    ignite.active(true);
    final Map<Object, IndexedObject> map = new TreeMap<>();
    final int cntEntries = 1000;
    for (int i = 0; i < cntEntries; i++) map.put(i, new IndexedObject(i));
    ignite.cache(CACHE_NAME).putAll(map);
    ignite.active(false);
    final String subfolderName = genDbSubfolderName(ignite, 0);
    final String subfolderName1 = genDbSubfolderName(ignite1, 1);
    stopAllGrids();
    final String workDir = U.defaultWorkDirectory();
    final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
    final StringBuilder builder = new StringBuilder();
    final Map<GridCacheOperation, Integer> operationsFound = new EnumMap<>(GridCacheOperation.class);
    final IgniteInClosure<DataRecord> drHnd = new IgniteInClosure<DataRecord>() {

        @Override
        public void apply(DataRecord dataRecord) {
            final List<DataEntry> entries = dataRecord.writeEntries();
            builder.append("{");
            for (DataEntry entry : entries) {
                final GridCacheOperation op = entry.op();
                final Integer cnt = operationsFound.get(op);
                operationsFound.put(op, cnt == null ? 1 : (cnt + 1));
                if (entry instanceof UnwrapDataEntry) {
                    final UnwrapDataEntry entry1 = (UnwrapDataEntry) entry;
                    builder.append(entry1.op()).append(" for ").append(entry1.unwrappedKey());
                    final GridCacheVersion ver = entry.nearXidVersion();
                    builder.append(", ");
                    if (ver != null)
                        builder.append("tx=").append(ver).append(", ");
                }
            }
            builder.append("}\n");
        }
    };
    scanIterateAndCount(factory, workDir, subfolderName, 1, 1, null, drHnd);
    scanIterateAndCount(factory, workDir, subfolderName1, 1, 1, null, drHnd);
    final Integer createsFound = operationsFound.get(CREATE);
    if (log.isInfoEnabled())
        log.info(builder.toString());
    assertTrue("Create operations should be found in log: " + operationsFound, createsFound != null && createsFound > 0);
    assertTrue("Create operations count should be at least " + cntEntries + " in log: " + operationsFound, createsFound != null && createsFound >= cntEntries);
}