use of org.apache.ignite.internal.commandline.ProgressPrinter in project ignite by apache.
the class IgniteIndexReader method readIdx.
/**
* Read index file.
*/
public void readIdx() {
long partPageStoresNum = Arrays.stream(partStores).filter(Objects::nonNull).count();
print("Partitions files num: " + partPageStoresNum);
Map<Class, Long> pageClasses = new HashMap<>();
long pagesNum = isNull(idxStore) ? 0 : (idxStore.size() - idxStore.headerSize()) / pageSize;
print("Going to check " + pagesNum + " pages.");
Set<Long> pageIds = new HashSet<>();
AtomicReference<Map<String, TreeTraversalInfo>> treeInfo = new AtomicReference<>();
AtomicReference<Map<String, TreeTraversalInfo>> horizontalScans = new AtomicReference<>();
AtomicReference<PageListsInfo> pageListsInfo = new AtomicReference<>();
List<Throwable> errors;
try {
IgniteBiTuple<Long, Long> indexPartitionRoots = partitionRoots(partMetaPageId(INDEX_PARTITION, FLAG_IDX));
long metaTreeRootId = indexPartitionRoots.get1();
long pageListMetaPageId = indexPartitionRoots.get2();
// Traversing trees.
treeInfo.set(traverseAllTrees("Index trees traversal", metaTreeRootId, CountOnlyStorage::new, this::traverseTree));
treeInfo.get().forEach((name, info) -> {
pageIds.addAll(info.innerPageIds);
pageIds.add(info.rootPageId);
});
Supplier<ItemStorage> itemStorageFactory = checkParts ? LinkStorage::new : CountOnlyStorage::new;
horizontalScans.set(traverseAllTrees("Scan index trees horizontally", metaTreeRootId, itemStorageFactory, this::horizontalTreeScan));
// Scanning page reuse lists.
if (pageListMetaPageId != 0)
pageListsInfo.set(getPageListsInfo(pageListMetaPageId));
ProgressPrinter progressPrinter = new ProgressPrinter(System.out, "Reading pages sequentially", pagesNum);
// Scan all pages in file.
errors = scanFileStore(INDEX_PARTITION, FLAG_IDX, idxStore, (pageId, addr, io) -> {
progressPrinter.printProgress();
pageClasses.compute(io.getClass(), (k, v) -> v == null ? 1 : v + 1);
if (!(io instanceof PageMetaIO || io instanceof PagesListMetaIO)) {
if (idxFilter == null) {
if ((io instanceof BPlusMetaIO || io instanceof BPlusInnerIO) && !pageIds.contains(pageId) && pageListsInfo.get() != null && !pageListsInfo.get().allPages.contains(pageId)) {
throw new IgniteException("Possibly orphan " + io.getClass().getSimpleName() + " page, pageId=" + pageId);
}
}
}
return true;
});
} catch (IgniteCheckedException e) {
throw new IgniteException(INDEX_FILE_NAME + " scan problem", e);
}
if (treeInfo.get() == null)
printErr("No tree meta info found.");
else {
printTraversalResults(RECURSIVE_TRAVERSE_NAME, treeInfo.get());
printTraversalResults(HORIZONTAL_SCAN_NAME, horizontalScans.get());
}
compareTraversals(treeInfo.get(), horizontalScans.get());
if (pageListsInfo.get() == null)
printErr("No page lists meta info found.");
else
printPagesListsInfo(pageListsInfo.get());
printPageStat("", "\n---These pages types were encountered during sequential scan:", pageClasses);
if (!errors.isEmpty()) {
printErr("---");
printErr("Errors:");
errors.forEach(this::printStackTrace);
}
print("---");
print("Total pages encountered during sequential scan: " + pageClasses.values().stream().mapToLong(a -> a).sum());
print("Total errors occurred during sequential scan: " + errors.size());
if (idxFilter != null)
print("Orphan pages were not reported due to --indexes filter.");
print("Note that some pages can be occupied by meta info, tracking info, etc., so total page count can differ " + "from count of pages found in index trees and page lists.");
if (checkParts) {
Map<Integer, List<Throwable>> checkPartsErrors = checkParts(horizontalScans.get());
print("");
printErrors("", "Partitions check:", "Partitions check detected no errors.", "Errors detected in partition, partId=%s", false, checkPartsErrors);
print("\nPartition check finished, total errors: " + checkPartsErrors.values().stream().mapToInt(List::size).sum() + ", total problem partitions: " + checkPartsErrors.size());
}
}
use of org.apache.ignite.internal.commandline.ProgressPrinter in project ignite by apache.
the class IgniteIndexReader method checkParts.
/**
* Checks partitions, comparing partition indexes (cache data tree) to indexes given in {@code aTreesInfo}.
*
* @param aTreesInfo Index trees info to compare cache data tree with.
* @return Map of errors, bound to partition id.
*/
private Map<Integer, List<Throwable>> checkParts(Map<String, TreeTraversalInfo> aTreesInfo) {
System.out.println();
// Map partId -> errors.
Map<Integer, List<Throwable>> res = new HashMap<>();
Map<String, TreeTraversalInfo> treesInfo = new HashMap<>(aTreesInfo);
treesInfo.remove(META_TREE_NAME);
ProgressPrinter progressPrinter = new ProgressPrinter(System.out, "Checking partitions", partCnt);
for (int i = 0; i < partCnt; i++) {
progressPrinter.printProgress();
FilePageStore partStore = partStores[i];
if (partStore == null)
continue;
List<Throwable> errors = new LinkedList<>();
final int partId = i;
try {
long partMetaId = partMetaPageId(i, FLAG_DATA);
doWithBuffer((buf, addr) -> {
readPage(partStore, partMetaId, buf);
PagePartitionMetaIO partMetaIO = PageIO.getPageIO(addr);
long cacheDataTreeRoot = partMetaIO.getTreeRoot(addr);
TreeTraversalInfo cacheDataTreeInfo = horizontalTreeScan(cacheDataTreeRoot, "dataTree-" + partId, new ItemsListStorage());
for (Object dataTreeItem : cacheDataTreeInfo.itemStorage) {
CacheAwareLink cacheAwareLink = (CacheAwareLink) dataTreeItem;
for (Map.Entry<String, TreeTraversalInfo> e : treesInfo.entrySet()) {
String name = e.getKey();
TreeTraversalInfo tree = e.getValue();
int cacheId = getCacheId(name);
if (cacheId != cacheAwareLink.cacheId)
// It's index for other cache, don't check.
continue;
// Tombstones are not indexed and shouldn't be tested.
if (!tree.itemStorage.contains(cacheAwareLink) && !cacheAwareLink.tombstone)
errors.add(new IgniteException(cacheDataTreeEntryMissingError(name, cacheAwareLink)));
}
if (errors.size() >= CHECK_PARTS_MAX_ERRORS_PER_PARTITION) {
errors.add(new IgniteException("Too many errors (" + CHECK_PARTS_MAX_ERRORS_PER_PARTITION + ") found for partId=" + partId + ", stopping analysis for this partition."));
break;
}
}
return null;
});
} catch (IgniteCheckedException e) {
errors.add(new IgniteException("Partition check failed, partId=" + i, e));
}
if (!errors.isEmpty())
res.put(partId, errors);
}
return res;
}
use of org.apache.ignite.internal.commandline.ProgressPrinter in project ignite by apache.
the class IgniteIndexReader method traverseAllTrees.
/**
* Traverse all trees in file and return their info.
*
* @param metaTreeRoot Meta tree root page id.
* @return Index trees info.
*/
private Map<String, TreeTraversalInfo> traverseAllTrees(String traverseProcCaption, long metaTreeRoot, Supplier<ItemStorage> itemStorageFactory, TraverseProc traverseProc) {
Map<String, TreeTraversalInfo> treeInfos = new LinkedHashMap<>();
TreeTraversalInfo metaTreeTraversalInfo = traverseProc.traverse(metaTreeRoot, META_TREE_NAME, new ItemsListStorage<IndexStorageImpl.IndexItem>());
treeInfos.put(META_TREE_NAME, metaTreeTraversalInfo);
ProgressPrinter progressPrinter = new ProgressPrinter(System.out, traverseProcCaption, metaTreeTraversalInfo.itemStorage.size());
metaTreeTraversalInfo.itemStorage.forEach(item -> {
progressPrinter.printProgress();
IndexStorageImpl.IndexItem idxItem = (IndexStorageImpl.IndexItem) item;
if (nonNull(idxFilter) && !idxFilter.test(idxItem.nameString()))
return;
TreeTraversalInfo treeTraversalInfo = traverseProc.traverse(normalizePageId(idxItem.pageId()), idxItem.nameString(), itemStorageFactory.get());
treeInfos.put(idxItem.toString(), treeTraversalInfo);
});
return treeInfos;
}
Aggregations