Examples with ConsistencyReporter org.neo4j.consistency.report.ConsistencyReporter used on opensource projects

Example 1 with ConsistencyReporter

use of org.neo4j.consistency.report.ConsistencyReporter in project neo4j by neo4j.

the class ConsistencyCheckTasks method createTasksForFullCheck.

public List<ConsistencyCheckerTask> createTasksForFullCheck(boolean checkLabelScanStore, boolean checkIndexes, boolean checkGraph) {
    List<ConsistencyCheckerTask> tasks = new ArrayList<>();
    if (checkGraph) {
        MandatoryProperties mandatoryProperties = new MandatoryProperties(nativeStores);
        StoreProcessor processor = multiPass.processor(CheckStage.Stage1_NS_PropsLabels, PROPERTIES);
        tasks.add(create(CheckStage.Stage1_NS_PropsLabels.name(), nativeStores.getNodeStore(), processor, ROUND_ROBIN));
        //ReltionshipStore pass - check label counts using cached labels, check properties, skip nodes and relationships
        processor = multiPass.processor(CheckStage.Stage2_RS_Labels, LABELS);
        multiPass.reDecorateRelationship(processor, RelationshipRecordCheck.relationshipRecordCheckForwardPass());
        tasks.add(create(CheckStage.Stage2_RS_Labels.name(), nativeStores.getRelationshipStore(), processor, ROUND_ROBIN));
        //NodeStore pass - just cache nextRel and inUse
        tasks.add(new CacheTask.CacheNextRel(CheckStage.Stage3_NS_NextRel, cacheAccess, Scanner.scan(nativeStores.getNodeStore())));
        //RelationshipStore pass - check nodes inUse, FirstInFirst, FirstInSecond using cached info
        processor = multiPass.processor(CheckStage.Stage4_RS_NextRel, NODES);
        multiPass.reDecorateRelationship(processor, RelationshipRecordCheck.relationshipRecordCheckBackwardPass(new PropertyChain<>(mandatoryProperties.forRelationships(reporter))));
        tasks.add(create(CheckStage.Stage4_RS_NextRel.name(), nativeStores.getRelationshipStore(), processor, ROUND_ROBIN));
        //NodeStore pass - just cache nextRel and inUse
        multiPass.reDecorateNode(processor, NodeRecordCheck.toCheckNextRel(), true);
        multiPass.reDecorateNode(processor, NodeRecordCheck.toCheckNextRelationshipGroup(), false);
        tasks.add(new CacheTask.CheckNextRel(CheckStage.Stage5_Check_NextRel, cacheAccess, nativeStores, processor));
        // source chain
        //RelationshipStore pass - forward scan of source chain using the cache.
        processor = multiPass.processor(CheckStage.Stage6_RS_Forward, RELATIONSHIPS);
        multiPass.reDecorateRelationship(processor, RelationshipRecordCheck.relationshipRecordCheckSourceChain());
        tasks.add(create(CheckStage.Stage6_RS_Forward.name(), nativeStores.getRelationshipStore(), processor, QueueDistribution.RELATIONSHIPS));
        //RelationshipStore pass - reverse scan of source chain using the cache.
        processor = multiPass.processor(CheckStage.Stage7_RS_Backward, RELATIONSHIPS);
        multiPass.reDecorateRelationship(processor, RelationshipRecordCheck.relationshipRecordCheckSourceChain());
        tasks.add(create(CheckStage.Stage7_RS_Backward.name(), nativeStores.getRelationshipStore(), processor, QueueDistribution.RELATIONSHIPS));
        //relationshipGroup
        StoreProcessor relGrpProcessor = multiPass.processor(Stage.PARALLEL_FORWARD, RELATIONSHIP_GROUPS);
        tasks.add(create("RelationshipGroupStore-RelGrp", nativeStores.getRelationshipGroupStore(), relGrpProcessor, ROUND_ROBIN));
        PropertyReader propertyReader = new PropertyReader(nativeStores);
        tasks.add(recordScanner(CheckStage.Stage8_PS_Props.name(), new IterableStore<>(nativeStores.getNodeStore(), true), new PropertyAndNode2LabelIndexProcessor(reporter, checkIndexes ? indexes : null, propertyReader, cacheAccess, mandatoryProperties.forNodes(reporter)), CheckStage.Stage8_PS_Props, ROUND_ROBIN, new IterableStore<>(nativeStores.getPropertyStore(), true)));
        tasks.add(create("StringStore-Str", nativeStores.getStringStore(), multiPass.processor(Stage.SEQUENTIAL_FORWARD, STRINGS), ROUND_ROBIN));
        tasks.add(create("ArrayStore-Arrays", nativeStores.getArrayStore(), multiPass.processor(Stage.SEQUENTIAL_FORWARD, ARRAYS), ROUND_ROBIN));
    }
    // The schema store is verified in multiple passes that share state since it fits into memory
    // and we care about the consistency of back references (cf. SemanticCheck)
    // PASS 1: Dynamic record chains
    tasks.add(create("SchemaStore", nativeStores.getSchemaStore(), ROUND_ROBIN));
    // PASS 2: Rule integrity and obligation build up
    final SchemaRecordCheck schemaCheck = new SchemaRecordCheck(new SchemaStorage(nativeStores.getSchemaStore()));
    tasks.add(new SchemaStoreProcessorTask<>("SchemaStoreProcessor-check_rules", statistics, numberOfThreads, nativeStores.getSchemaStore(), nativeStores, "check_rules", schemaCheck, progress, cacheAccess, defaultProcessor, ROUND_ROBIN));
    // PASS 3: Obligation verification and semantic rule uniqueness
    tasks.add(new SchemaStoreProcessorTask<>("SchemaStoreProcessor-check_obligations", statistics, numberOfThreads, nativeStores.getSchemaStore(), nativeStores, "check_obligations", schemaCheck.forObligationChecking(), progress, cacheAccess, defaultProcessor, ROUND_ROBIN));
    if (checkGraph) {
        tasks.add(create("RelationshipTypeTokenStore", nativeStores.getRelationshipTypeTokenStore(), ROUND_ROBIN));
        tasks.add(create("PropertyKeyTokenStore", nativeStores.getPropertyKeyTokenStore(), ROUND_ROBIN));
        tasks.add(create("LabelTokenStore", nativeStores.getLabelTokenStore(), ROUND_ROBIN));
        tasks.add(create("RelationshipTypeNameStore", nativeStores.getRelationshipTypeNameStore(), ROUND_ROBIN));
        tasks.add(create("PropertyKeyNameStore", nativeStores.getPropertyKeyNameStore(), ROUND_ROBIN));
        tasks.add(create("LabelNameStore", nativeStores.getLabelNameStore(), ROUND_ROBIN));
        tasks.add(create("NodeDynamicLabelStore", nativeStores.getNodeDynamicLabelStore(), ROUND_ROBIN));
    }
    ConsistencyReporter filteredReporter = multiPass.reporter(NODES);
    if (checkLabelScanStore) {
        tasks.add(recordScanner("LabelScanStore", labelScanStore.allNodeLabelRanges(), new LabelScanDocumentProcessor(filteredReporter, new LabelScanCheck()), Stage.SEQUENTIAL_FORWARD, ROUND_ROBIN));
    }
    if (checkIndexes) {
        for (IndexRule indexRule : indexes.rules()) {
            tasks.add(recordScanner(format("Index_%d", indexRule.getId()), new IndexIterator(indexes.accessorFor(indexRule)), new IndexEntryProcessor(filteredReporter, new IndexCheck(indexRule)), Stage.SEQUENTIAL_FORWARD, ROUND_ROBIN));
        }
    }
    return tasks;
}

Example 2 with ConsistencyReporter

use of org.neo4j.consistency.report.ConsistencyReporter in project neo4j by neo4j.

the class FullCheck method execute.

void execute(final DirectStoreAccess directStoreAccess, final CheckDecorator decorator, final RecordAccess recordAccess, final InconsistencyReport report, CacheAccess cacheAccess, Monitor reportMonitor) throws ConsistencyCheckIncompleteException {
    final ConsistencyReporter reporter = new ConsistencyReporter(recordAccess, report, reportMonitor);
    StoreProcessor processEverything = new StoreProcessor(decorator, reporter, Stage.SEQUENTIAL_FORWARD, cacheAccess);
    ProgressMonitorFactory.MultiPartBuilder progress = progressFactory.multipleParts("Full Consistency Check");
    final StoreAccess nativeStores = directStoreAccess.nativeStores();
    try (IndexAccessors indexes = new IndexAccessors(directStoreAccess.indexes(), nativeStores.getSchemaStore(), samplingConfig)) {
        MultiPassStore.Factory multiPass = new MultiPassStore.Factory(decorator, recordAccess, cacheAccess, report, reportMonitor);
        ConsistencyCheckTasks taskCreator = new ConsistencyCheckTasks(progress, processEverything, nativeStores, statistics, cacheAccess, directStoreAccess.labelScanStore(), indexes, multiPass, reporter, threads);
        List<ConsistencyCheckerTask> tasks = taskCreator.createTasksForFullCheck(checkLabelScanStore, checkIndexes, checkGraph);
        TaskExecutor.execute(tasks, decorator::prepare);
    } catch (Exception e) {
        throw new ConsistencyCheckIncompleteException(e);
    }
}

Example 3 with ConsistencyReporter

use of org.neo4j.consistency.report.ConsistencyReporter in project neo4j by neo4j.

the class FullCheck method execute.

ConsistencySummaryStatistics execute(DirectStoreAccess stores, Log log, Monitor reportMonitor) throws ConsistencyCheckIncompleteException {
    ConsistencySummaryStatistics summary = new ConsistencySummaryStatistics();
    InconsistencyReport report = new InconsistencyReport(new InconsistencyMessageLogger(log), summary);
    OwnerCheck ownerCheck = new OwnerCheck(checkPropertyOwners);
    CountsBuilderDecorator countsBuilder = new CountsBuilderDecorator(stores.nativeStores());
    CheckDecorator decorator = new CheckDecorator.ChainCheckDecorator(ownerCheck, countsBuilder);
    CacheAccess cacheAccess = new DefaultCacheAccess(statistics.getCounts(), threads);
    RecordAccess records = recordAccess(stores.nativeStores(), cacheAccess);
    execute(stores, decorator, records, report, cacheAccess, reportMonitor);
    ownerCheck.scanForOrphanChains(progressFactory);
    if (checkGraph) {
        CountsAccessor countsAccessor = stores.nativeStores().getCounts();
        if (countsAccessor instanceof CountsTracker) {
            CountsTracker tracker = (CountsTracker) countsAccessor;
            try {
                tracker.start();
            } catch (Exception e) {
            // let's hope it was already started :)
            }
        }
        countsBuilder.checkCounts(countsAccessor, new ConsistencyReporter(records, report), progressFactory);
    }
    if (!summary.isConsistent()) {
        log.warn("Inconsistencies found: " + summary);
    }
    return summary;
}

Example 4 with ConsistencyReporter

use of org.neo4j.consistency.report.ConsistencyReporter in project neo4j by neo4j.

the class CheckerTestBase method context.

CheckerContext context(int numberOfThreads, ConsistencyFlags consistencyFlags, ConsistencySummaryStatistics inconsistenciesSummary) throws Exception {
    if (context != null) {
        return context;
    }
    // We do this as late as possible because of how it eagerly caches which indexes exist so if the test creates an index
    // this lazy instantiation allows the context to pick it up
    Config config = Config.defaults(neo4j_home, directory.homePath());
    DependencyResolver dependencies = db.getDependencyResolver();
    IndexProviderMap indexProviders = dependencies.resolveDependency(IndexProviderMap.class);
    IndexingService indexingService = dependencies.resolveDependency(IndexingService.class);
    IndexAccessors indexAccessors = new IndexAccessors(indexProviders, neoStores, new IndexSamplingConfig(config), new LookupAccessorsFromRunningDb(indexingService), PageCacheTracer.NULL, tokenHolders, neoStores.getMetaDataStore());
    InconsistencyReport report = new InconsistencyReport(new InconsistencyMessageLogger(NullLog.getInstance()), inconsistenciesSummary);
    monitor = mock(ConsistencyReporter.Monitor.class);
    reporter = new ConsistencyReporter(report, monitor);
    countsState = new CountsState(neoStores, cacheAccess, INSTANCE);
    NodeBasedMemoryLimiter limiter = new NodeBasedMemoryLimiter(pageCache.pageSize() * pageCache.maxCachedPages(), Runtime.getRuntime().maxMemory(), Long.MAX_VALUE, CacheSlots.CACHE_LINE_SIZE_BYTES, nodeStore.getHighId(), 1);
    ProgressMonitorFactory.MultiPartBuilder progress = ProgressMonitorFactory.NONE.multipleParts("Test");
    ParallelExecution execution = new ParallelExecution(numberOfThreads, NOOP_EXCEPTION_HANDLER, IDS_PER_CHUNK);
    context = new CheckerContext(neoStores, indexAccessors, execution, reporter, cacheAccess, tokenHolders, new RecordLoading(neoStores), countsState, limiter, progress, pageCache, PageCacheTracer.NULL, INSTANCE, DebugContext.NO_DEBUG, consistencyFlags);
    context.initialize();
    return context;
}