Search in sources :

Example 1 with ProgressPrinter

use of org.apache.ignite.internal.commandline.ProgressPrinter in project ignite by apache.

the class IgniteIndexReader method readIdx.

/**
 * Read index file.
 */
public void readIdx() {
    long partPageStoresNum = Arrays.stream(partStores).filter(Objects::nonNull).count();
    print("Partitions files num: " + partPageStoresNum);
    Map<Class, Long> pageClasses = new HashMap<>();
    long pagesNum = isNull(idxStore) ? 0 : (idxStore.size() - idxStore.headerSize()) / pageSize;
    print("Going to check " + pagesNum + " pages.");
    Set<Long> pageIds = new HashSet<>();
    AtomicReference<Map<String, TreeTraversalInfo>> treeInfo = new AtomicReference<>();
    AtomicReference<Map<String, TreeTraversalInfo>> horizontalScans = new AtomicReference<>();
    AtomicReference<PageListsInfo> pageListsInfo = new AtomicReference<>();
    List<Throwable> errors;
    try {
        IgniteBiTuple<Long, Long> indexPartitionRoots = partitionRoots(partMetaPageId(INDEX_PARTITION, FLAG_IDX));
        long metaTreeRootId = indexPartitionRoots.get1();
        long pageListMetaPageId = indexPartitionRoots.get2();
        // Traversing trees.
        treeInfo.set(traverseAllTrees("Index trees traversal", metaTreeRootId, CountOnlyStorage::new, this::traverseTree));
        treeInfo.get().forEach((name, info) -> {
            pageIds.addAll(info.innerPageIds);
            pageIds.add(info.rootPageId);
        });
        Supplier<ItemStorage> itemStorageFactory = checkParts ? LinkStorage::new : CountOnlyStorage::new;
        horizontalScans.set(traverseAllTrees("Scan index trees horizontally", metaTreeRootId, itemStorageFactory, this::horizontalTreeScan));
        // Scanning page reuse lists.
        if (pageListMetaPageId != 0)
            pageListsInfo.set(getPageListsInfo(pageListMetaPageId));
        ProgressPrinter progressPrinter = new ProgressPrinter(System.out, "Reading pages sequentially", pagesNum);
        // Scan all pages in file.
        errors = scanFileStore(INDEX_PARTITION, FLAG_IDX, idxStore, (pageId, addr, io) -> {
            progressPrinter.printProgress();
            pageClasses.compute(io.getClass(), (k, v) -> v == null ? 1 : v + 1);
            if (!(io instanceof PageMetaIO || io instanceof PagesListMetaIO)) {
                if (idxFilter == null) {
                    if ((io instanceof BPlusMetaIO || io instanceof BPlusInnerIO) && !pageIds.contains(pageId) && pageListsInfo.get() != null && !pageListsInfo.get().allPages.contains(pageId)) {
                        throw new IgniteException("Possibly orphan " + io.getClass().getSimpleName() + " page, pageId=" + pageId);
                    }
                }
            }
            return true;
        });
    } catch (IgniteCheckedException e) {
        throw new IgniteException(INDEX_FILE_NAME + " scan problem", e);
    }
    if (treeInfo.get() == null)
        printErr("No tree meta info found.");
    else {
        printTraversalResults(RECURSIVE_TRAVERSE_NAME, treeInfo.get());
        printTraversalResults(HORIZONTAL_SCAN_NAME, horizontalScans.get());
    }
    compareTraversals(treeInfo.get(), horizontalScans.get());
    if (pageListsInfo.get() == null)
        printErr("No page lists meta info found.");
    else
        printPagesListsInfo(pageListsInfo.get());
    printPageStat("", "\n---These pages types were encountered during sequential scan:", pageClasses);
    if (!errors.isEmpty()) {
        printErr("---");
        printErr("Errors:");
        errors.forEach(this::printStackTrace);
    }
    print("---");
    print("Total pages encountered during sequential scan: " + pageClasses.values().stream().mapToLong(a -> a).sum());
    print("Total errors occurred during sequential scan: " + errors.size());
    if (idxFilter != null)
        print("Orphan pages were not reported due to --indexes filter.");
    print("Note that some pages can be occupied by meta info, tracking info, etc., so total page count can differ " + "from count of pages found in index trees and page lists.");
    if (checkParts) {
        Map<Integer, List<Throwable>> checkPartsErrors = checkParts(horizontalScans.get());
        print("");
        printErrors("", "Partitions check:", "Partitions check detected no errors.", "Errors detected in partition, partId=%s", false, checkPartsErrors);
        print("\nPartition check finished, total errors: " + checkPartsErrors.values().stream().mapToInt(List::size).sum() + ", total problem partitions: " + checkPartsErrors.size());
    }
}
Also used : Arrays(java.util.Arrays) PAGE_SIZE(org.apache.ignite.internal.commandline.indexreader.IgniteIndexReader.Args.PAGE_SIZE) PART_CNT(org.apache.ignite.internal.commandline.indexreader.IgniteIndexReader.Args.PART_CNT) IgnitePair(org.apache.ignite.internal.util.lang.IgnitePair) MvccInnerIO(org.apache.ignite.internal.cache.query.index.sorted.inline.io.MvccInnerIO) Collections.singletonList(java.util.Collections.singletonList) Matcher(java.util.regex.Matcher) PagePartitionMetaIO(org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO) Arrays.asList(java.util.Arrays.asList) PageIdUtils.itemId(org.apache.ignite.internal.pagemem.PageIdUtils.itemId) Map(java.util.Map) PageUtils(org.apache.ignite.internal.pagemem.PageUtils) Objects.isNull(java.util.Objects.isNull) GridStringBuilder(org.apache.ignite.internal.util.GridStringBuilder) IgniteDataIntegrityViolationException(org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) Set(java.util.Set) CLIArgument.optionalArg(org.apache.ignite.internal.commandline.argument.parser.CLIArgument.optionalArg) StringBuilderOutputStream(org.apache.ignite.internal.commandline.StringBuilderOutputStream) Collectors.joining(java.util.stream.Collectors.joining) PageIO.getType(org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO.getType) INDEX_PARTITION(org.apache.ignite.internal.pagemem.PageIdAllocator.INDEX_PARTITION) GridUnsafe.allocateBuffer(org.apache.ignite.internal.util.GridUnsafe.allocateBuffer) CLIArgument.mandatoryArg(org.apache.ignite.internal.commandline.argument.parser.CLIArgument.mandatoryArg) PageIdUtils(org.apache.ignite.internal.pagemem.PageIdUtils) PageIdUtils.pageIndex(org.apache.ignite.internal.pagemem.PageIdUtils.pageIndex) Supplier(java.util.function.Supplier) PagesListNodeIO(org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListNodeIO) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) PageIO.getVersion(org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO.getVersion) PagesListMetaIO(org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListMetaIO) AbstractInlineInnerIO(org.apache.ignite.internal.cache.query.index.sorted.inline.io.AbstractInlineInnerIO) PageIdUtils.partId(org.apache.ignite.internal.pagemem.PageIdUtils.partId) LongStream(java.util.stream.LongStream) FileOutputStream(java.io.FileOutputStream) IOException(java.io.IOException) InnerIO(org.apache.ignite.internal.cache.query.index.sorted.inline.io.InnerIO) File(java.io.File) BPlusMetaIO(org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusMetaIO) AtomicLong(java.util.concurrent.atomic.AtomicLong) PageIdUtils.flag(org.apache.ignite.internal.pagemem.PageIdUtils.flag) PageIdUtils.pageId(org.apache.ignite.internal.pagemem.PageIdUtils.pageId) StorageException(org.apache.ignite.internal.processors.cache.persistence.StorageException) FLAG_DATA(org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA) MvccLeafIO(org.apache.ignite.internal.cache.query.index.sorted.inline.io.MvccLeafIO) ByteBuffer(java.nio.ByteBuffer) GridLongList(org.apache.ignite.internal.util.GridLongList) PAGE_STORE_VER(org.apache.ignite.internal.commandline.indexreader.IgniteIndexReader.Args.PAGE_STORE_VER) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) MvccDataLeafIO(org.apache.ignite.internal.processors.cache.tree.mvcc.data.MvccDataLeafIO) INDEXES(org.apache.ignite.internal.commandline.indexreader.IgniteIndexReader.Args.INDEXES) PendingRowIO(org.apache.ignite.internal.processors.cache.tree.PendingRowIO) FilePageStore(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore) BPlusLeafIO(org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusLeafIO) Predicate(java.util.function.Predicate) IgniteException(org.apache.ignite.IgniteException) FilePageStoreManager(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager) CacheObject(org.apache.ignite.internal.processors.cache.CacheObject) DEST_FILE(org.apache.ignite.internal.commandline.indexreader.IgniteIndexReader.Args.DEST_FILE) String.format(java.lang.String.format) IgniteBiTuple(org.apache.ignite.lang.IgniteBiTuple) Objects(java.util.Objects) GridUnsafe.freeBuffer(org.apache.ignite.internal.util.GridUnsafe.freeBuffer) Nullable(org.jetbrains.annotations.Nullable) List(java.util.List) Pattern(java.util.regex.Pattern) AbstractInlineLeafIO(org.apache.ignite.internal.cache.query.index.sorted.inline.io.AbstractInlineLeafIO) CLIArgumentParser(org.apache.ignite.internal.commandline.argument.parser.CLIArgumentParser) Objects.nonNull(java.util.Objects.nonNull) InlineIO(org.apache.ignite.internal.cache.query.index.sorted.inline.io.InlineIO) FLAG_IDX(org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX) IntStream(java.util.stream.IntStream) ProgressPrinter(org.apache.ignite.internal.commandline.ProgressPrinter) DataPagePayload(org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPagePayload) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) AtomicReference(java.util.concurrent.atomic.AtomicReference) HashSet(java.util.HashSet) PageMetaIO(org.apache.ignite.internal.processors.cache.persistence.tree.io.PageMetaIO) IndexStorageImpl(org.apache.ignite.internal.processors.cache.persistence.IndexStorageImpl) GridUnsafe.bufferAddress(org.apache.ignite.internal.util.GridUnsafe.bufferAddress) PageIO(org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO) DIR(org.apache.ignite.internal.commandline.indexreader.IgniteIndexReader.Args.DIR) LinkedList(java.util.LinkedList) AbstractDataLeafIO(org.apache.ignite.internal.processors.cache.tree.AbstractDataLeafIO) OutputStream(java.io.OutputStream) PrintStream(java.io.PrintStream) RowLinkIO(org.apache.ignite.internal.processors.cache.tree.RowLinkIO) F(org.apache.ignite.internal.util.typedef.F) BPlusIO(org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO) GridClosure3(org.apache.ignite.internal.util.lang.GridClosure3) Integer.parseInt(java.lang.Integer.parseInt) AbstractDataPageIO(org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO) BPlusInnerIO(org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusInnerIO) LeafIO(org.apache.ignite.internal.cache.query.index.sorted.inline.io.LeafIO) CLIArgument(org.apache.ignite.internal.commandline.argument.parser.CLIArgument) Collectors.toList(java.util.stream.Collectors.toList) INDEX_FILE_NAME(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.INDEX_FILE_NAME) FileChannel(java.nio.channels.FileChannel) Collections(java.util.Collections) CHECK_PARTS(org.apache.ignite.internal.commandline.indexreader.IgniteIndexReader.Args.CHECK_PARTS) PageMetaIO(org.apache.ignite.internal.processors.cache.persistence.tree.io.PageMetaIO) LinkedHashMap(java.util.LinkedHashMap) HashMap(java.util.HashMap) PagesListMetaIO(org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListMetaIO) ProgressPrinter(org.apache.ignite.internal.commandline.ProgressPrinter) BPlusMetaIO(org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusMetaIO) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) IgniteException(org.apache.ignite.IgniteException) Collections.singletonList(java.util.Collections.singletonList) Arrays.asList(java.util.Arrays.asList) ArrayList(java.util.ArrayList) GridLongList(org.apache.ignite.internal.util.GridLongList) List(java.util.List) LinkedList(java.util.LinkedList) Collectors.toList(java.util.stream.Collectors.toList) HashSet(java.util.HashSet) BPlusInnerIO(org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusInnerIO) AtomicReference(java.util.concurrent.atomic.AtomicReference) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) AtomicLong(java.util.concurrent.atomic.AtomicLong) Map(java.util.Map) LinkedHashMap(java.util.LinkedHashMap) HashMap(java.util.HashMap)

Example 2 with ProgressPrinter

use of org.apache.ignite.internal.commandline.ProgressPrinter in project ignite by apache.

the class IgniteIndexReader method checkParts.

/**
 * Checks partitions, comparing partition indexes (cache data tree) to indexes given in {@code aTreesInfo}.
 *
 * @param aTreesInfo Index trees info to compare cache data tree with.
 * @return Map of errors, bound to partition id.
 */
private Map<Integer, List<Throwable>> checkParts(Map<String, TreeTraversalInfo> aTreesInfo) {
    System.out.println();
    // Map partId -> errors.
    Map<Integer, List<Throwable>> res = new HashMap<>();
    Map<String, TreeTraversalInfo> treesInfo = new HashMap<>(aTreesInfo);
    treesInfo.remove(META_TREE_NAME);
    ProgressPrinter progressPrinter = new ProgressPrinter(System.out, "Checking partitions", partCnt);
    for (int i = 0; i < partCnt; i++) {
        progressPrinter.printProgress();
        FilePageStore partStore = partStores[i];
        if (partStore == null)
            continue;
        List<Throwable> errors = new LinkedList<>();
        final int partId = i;
        try {
            long partMetaId = partMetaPageId(i, FLAG_DATA);
            doWithBuffer((buf, addr) -> {
                readPage(partStore, partMetaId, buf);
                PagePartitionMetaIO partMetaIO = PageIO.getPageIO(addr);
                long cacheDataTreeRoot = partMetaIO.getTreeRoot(addr);
                TreeTraversalInfo cacheDataTreeInfo = horizontalTreeScan(cacheDataTreeRoot, "dataTree-" + partId, new ItemsListStorage());
                for (Object dataTreeItem : cacheDataTreeInfo.itemStorage) {
                    CacheAwareLink cacheAwareLink = (CacheAwareLink) dataTreeItem;
                    for (Map.Entry<String, TreeTraversalInfo> e : treesInfo.entrySet()) {
                        String name = e.getKey();
                        TreeTraversalInfo tree = e.getValue();
                        int cacheId = getCacheId(name);
                        if (cacheId != cacheAwareLink.cacheId)
                            // It's index for other cache, don't check.
                            continue;
                        // Tombstones are not indexed and shouldn't be tested.
                        if (!tree.itemStorage.contains(cacheAwareLink) && !cacheAwareLink.tombstone)
                            errors.add(new IgniteException(cacheDataTreeEntryMissingError(name, cacheAwareLink)));
                    }
                    if (errors.size() >= CHECK_PARTS_MAX_ERRORS_PER_PARTITION) {
                        errors.add(new IgniteException("Too many errors (" + CHECK_PARTS_MAX_ERRORS_PER_PARTITION + ") found for partId=" + partId + ", stopping analysis for this partition."));
                        break;
                    }
                }
                return null;
            });
        } catch (IgniteCheckedException e) {
            errors.add(new IgniteException("Partition check failed, partId=" + i, e));
        }
        if (!errors.isEmpty())
            res.put(partId, errors);
    }
    return res;
}
Also used : LinkedHashMap(java.util.LinkedHashMap) HashMap(java.util.HashMap) ProgressPrinter(org.apache.ignite.internal.commandline.ProgressPrinter) FilePageStore(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore) LinkedList(java.util.LinkedList) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) IgniteException(org.apache.ignite.IgniteException) Collections.singletonList(java.util.Collections.singletonList) Arrays.asList(java.util.Arrays.asList) ArrayList(java.util.ArrayList) GridLongList(org.apache.ignite.internal.util.GridLongList) List(java.util.List) LinkedList(java.util.LinkedList) Collectors.toList(java.util.stream.Collectors.toList) CacheObject(org.apache.ignite.internal.processors.cache.CacheObject) PagePartitionMetaIO(org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO) Map(java.util.Map) LinkedHashMap(java.util.LinkedHashMap) HashMap(java.util.HashMap)

Example 3 with ProgressPrinter

use of org.apache.ignite.internal.commandline.ProgressPrinter in project ignite by apache.

the class IgniteIndexReader method traverseAllTrees.

/**
 * Traverse all trees in file and return their info.
 *
 * @param metaTreeRoot Meta tree root page id.
 * @return Index trees info.
 */
private Map<String, TreeTraversalInfo> traverseAllTrees(String traverseProcCaption, long metaTreeRoot, Supplier<ItemStorage> itemStorageFactory, TraverseProc traverseProc) {
    Map<String, TreeTraversalInfo> treeInfos = new LinkedHashMap<>();
    TreeTraversalInfo metaTreeTraversalInfo = traverseProc.traverse(metaTreeRoot, META_TREE_NAME, new ItemsListStorage<IndexStorageImpl.IndexItem>());
    treeInfos.put(META_TREE_NAME, metaTreeTraversalInfo);
    ProgressPrinter progressPrinter = new ProgressPrinter(System.out, traverseProcCaption, metaTreeTraversalInfo.itemStorage.size());
    metaTreeTraversalInfo.itemStorage.forEach(item -> {
        progressPrinter.printProgress();
        IndexStorageImpl.IndexItem idxItem = (IndexStorageImpl.IndexItem) item;
        if (nonNull(idxFilter) && !idxFilter.test(idxItem.nameString()))
            return;
        TreeTraversalInfo treeTraversalInfo = traverseProc.traverse(normalizePageId(idxItem.pageId()), idxItem.nameString(), itemStorageFactory.get());
        treeInfos.put(idxItem.toString(), treeTraversalInfo);
    });
    return treeInfos;
}
Also used : IndexStorageImpl(org.apache.ignite.internal.processors.cache.persistence.IndexStorageImpl) ProgressPrinter(org.apache.ignite.internal.commandline.ProgressPrinter) LinkedHashMap(java.util.LinkedHashMap)

Aggregations

LinkedHashMap (java.util.LinkedHashMap)3 ProgressPrinter (org.apache.ignite.internal.commandline.ProgressPrinter)3 ArrayList (java.util.ArrayList)2 Arrays.asList (java.util.Arrays.asList)2 Collections.singletonList (java.util.Collections.singletonList)2 HashMap (java.util.HashMap)2 LinkedList (java.util.LinkedList)2 List (java.util.List)2 Map (java.util.Map)2 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)2 Collectors.toList (java.util.stream.Collectors.toList)2 IgniteCheckedException (org.apache.ignite.IgniteCheckedException)2 IgniteException (org.apache.ignite.IgniteException)2 CacheObject (org.apache.ignite.internal.processors.cache.CacheObject)2 IndexStorageImpl (org.apache.ignite.internal.processors.cache.persistence.IndexStorageImpl)2 FilePageStore (org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore)2 PagePartitionMetaIO (org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO)2 GridLongList (org.apache.ignite.internal.util.GridLongList)2 File (java.io.File)1 FileOutputStream (java.io.FileOutputStream)1