Examples with IndexEntry org.neo4j.consistency.store.synthetic.IndexEntry used on opensource projects

Example 1 with IndexEntry

use of org.neo4j.consistency.store.synthetic.IndexEntry in project neo4j by neo4j.

the class IndexChecker method checkVsEntities.

private void checkVsEntities(List<IndexContext> indexes, long fromEntityId, long toEntityId) {
    // This is one thread
    CheckerContext noReportingContext = context.withoutReporting();
    try (var cursorContext = new CursorContext(context.pageCacheTracer.createPageCursorTracer(CONSISTENCY_INDEX_ENTITY_CHECK_TAG));
        RecordReader<NodeRecord> nodeReader = new RecordReader<>(context.neoStores.getNodeStore(), true, cursorContext);
        RecordReader<DynamicRecord> labelReader = new RecordReader<>(context.neoStores.getNodeStore().getDynamicLabelStore(), false, cursorContext);
        SafePropertyChainReader propertyReader = new SafePropertyChainReader(noReportingContext, cursorContext)) {
        ProgressListener localScanProgress = scanProgress.threadLocalReporter();
        IntObjectHashMap<Value> allValues = new IntObjectHashMap<>();
        var client = cacheAccess.client();
        int numberOfIndexes = indexes.size();
        for (long entityId = fromEntityId; entityId < toEntityId && !context.isCancelled(); entityId++) {
            NodeRecord nodeRecord = nodeReader.read(entityId);
            if (nodeRecord.inUse()) {
                long[] entityTokens = safeGetNodeLabels(noReportingContext, nodeRecord.getId(), nodeRecord.getLabelField(), labelReader, cursorContext);
                lightClear(allValues);
                boolean propertyChainRead = entityTokens != null && propertyReader.read(allValues, nodeRecord, noReportingContext.reporter::forNode, cursorContext);
                if (propertyChainRead) {
                    for (int i = 0; i < numberOfIndexes; i++) {
                        IndexContext index = indexes.get(i);
                        IndexDescriptor descriptor = index.descriptor;
                        long cachedValue = client.getFromCache(entityId, i);
                        boolean nodeIsInIndex = (cachedValue & IN_USE_MASK) != 0;
                        Value[] values = RecordLoading.entityIntersectionWithSchema(entityTokens, allValues, descriptor.schema());
                        if (index.descriptor.schema().isFulltextSchemaDescriptor()) {
                            // The strategy for fulltext indexes is way simpler. Simply check of the sets of tokens (label tokens and property key tokens)
                            // and if they match the index schema descriptor then the node should be in the index, otherwise not
                            int[] nodePropertyKeys = allValues.keySet().toArray();
                            int[] indexPropertyKeys = index.descriptor.schema().getPropertyIds();
                            boolean nodeShouldBeInIndex = index.descriptor.schema().isAffected(entityTokens) && containsAny(indexPropertyKeys, nodePropertyKeys) && valuesContainTextProperty(values);
                            if (nodeShouldBeInIndex && !nodeIsInIndex) {
                                getReporter(context.recordLoader.node(entityId, cursorContext)).notIndexed(descriptor, new Object[0]);
                            } else if (!nodeShouldBeInIndex && nodeIsInIndex) {
                                // Fulltext indexes created before 4.3.0-drop02 can contain empty documents (added when the schema matched but the values
                                // were not text). The index still works with those empty documents present, so we don't want to report them as
                                // inconsistencies and force rebuilds.
                                // This utilizes the fact that countIndexedEntities in FulltextIndexReader with non-text values will ask
                                // about documents that doesn't contain those property keys - a document found by that query should be an empty
                                // document we just want to ignore.
                                Value[] noValues = new Value[indexPropertyKeys.length];
                                Arrays.fill(noValues, NO_VALUE);
                                long docsWithNoneOfProperties = indexAccessors.readers().reader(descriptor).countIndexedEntities(entityId, cursorContext, indexPropertyKeys, noValues);
                                if (docsWithNoneOfProperties != 1) {
                                    getReporter(context.recordLoader.node(entityId, cursorContext)).notIndexed(descriptor, new Object[0]);
                                }
                            }
                        } else {
                            if (values != null) {
                                // This node should really be in the index, is it?
                                if (!nodeIsInIndex) {
                                    // It wasn't, report it
                                    getReporter(context.recordLoader.node(entityId, cursorContext)).notIndexed(descriptor, Values.asObjects(values));
                                } else if (index.hasValues) {
                                    int cachedChecksum = (int) cachedValue & CHECKSUM_MASK;
                                    int actualChecksum = checksum(values);
                                    if (cachedChecksum != actualChecksum) {
                                        getReporter(context.recordLoader.node(entityId, cursorContext)).notIndexed(descriptor, Values.asObjects(values));
                                    }
                                }
                            } else {
                                if (nodeIsInIndex) {
                                    reporter.forIndexEntry(new IndexEntry(descriptor, context.tokenNameLookup, entityId)).nodeNotInUse(context.recordLoader.node(entityId, cursorContext));
                                }
                            }
                        }
                    }
                }
            // else this would be reported elsewhere
            } else {
                // This node shouldn't be in any index
                for (int i = 0; i < numberOfIndexes; i++) {
                    boolean isInIndex = (client.getFromCache(entityId, i) & IN_USE_MASK) != 0;
                    if (isInIndex) {
                        reporter.forIndexEntry(new IndexEntry(indexes.get(i).descriptor, context.tokenNameLookup, entityId)).nodeNotInUse(context.recordLoader.node(entityId, cursorContext));
                    }
                }
            }
            localScanProgress.add(1);
        }
        localScanProgress.done();
    }
}

Example 2 with IndexEntry

use of org.neo4j.consistency.store.synthetic.IndexEntry in project neo4j by neo4j.

the class NodeChecker method checkIndexVsNodes.

private void checkIndexVsNodes(LongRange range, IndexDescriptor descriptor, boolean lastRange) throws Exception {
    CacheAccess.Client client = context.cacheAccess.client();
    IndexAccessor accessor = context.indexAccessors.accessorFor(descriptor);
    RelationshipCounter.NodeLabelsLookup nodeLabelsLookup = observedCounts.nodeLabelsLookup();
    SchemaDescriptor schema = descriptor.schema();
    PropertySchemaType propertySchemaType = schema.propertySchemaType();
    long[] indexEntityTokenIds = toLongArray(schema.getEntityTokenIds());
    indexEntityTokenIds = sortAndDeduplicate(indexEntityTokenIds);
    try (var cursorContext = new CursorContext(context.pageCacheTracer.createPageCursorTracer(NODE_INDEXES_CHECKER_TAG));
        var allEntriesReader = accessor.newAllEntriesValueReader(range.from(), lastRange ? Long.MAX_VALUE : range.to(), cursorContext)) {
        for (long entityId : allEntriesReader) {
            try {
                boolean entityExists = client.getBooleanFromCache(entityId, CacheSlots.NodeLink.SLOT_IN_USE);
                if (!entityExists) {
                    reporter.forIndexEntry(new IndexEntry(descriptor, context.tokenNameLookup, entityId)).nodeNotInUse(recordLoader.node(entityId, cursorContext));
                } else {
                    long[] entityTokenIds = nodeLabelsLookup.nodeLabels(entityId);
                    compareTwoSortedLongArrays(propertySchemaType, entityTokenIds, indexEntityTokenIds, indexLabel -> reporter.forIndexEntry(new IndexEntry(descriptor, context.tokenNameLookup, entityId)).nodeDoesNotHaveExpectedLabel(recordLoader.node(entityId, cursorContext), indexLabel), storeLabel -> {
                    /*here we're only interested in what the the index has that the store doesn't have*/
                    });
                }
            } catch (ArrayIndexOutOfBoundsException e) {
                // OK so apparently the index has a node way outside node highId
                reporter.forIndexEntry(new IndexEntry(descriptor, context.tokenNameLookup, entityId)).nodeNotInUse(recordLoader.node(entityId, cursorContext));
            }
        }
    }
}

Example 3 with IndexEntry

use of org.neo4j.consistency.store.synthetic.IndexEntry in project neo4j by neo4j.

the class IndexChecker method cacheIndex.

private void cacheIndex(IndexContext index, LongRange nodeIdRange, boolean firstRange, CursorContext cursorContext) throws Exception {
    IndexAccessor accessor = indexAccessors.accessorFor(index.descriptor);
    IndexEntriesReader[] partitions = accessor.newAllEntriesValueReader(context.execution.getNumberOfThreads(), cursorContext);
    try {
        Value[][] firstValues = new Value[partitions.length][];
        Value[][] lastValues = new Value[partitions.length][];
        long[] firstEntityIds = new long[partitions.length];
        long[] lastEntityIds = new long[partitions.length];
        ThrowingRunnable[] workers = new ThrowingRunnable[partitions.length];
        for (int i = 0; i < partitions.length; i++) {
            IndexEntriesReader partition = partitions[i];
            int slot = i;
            workers[i] = () -> {
                int lastChecksum = 0;
                int progressPart = 0;
                ProgressListener localCacheProgress = cacheProgress.threadLocalReporter();
                var client = cacheAccess.client();
                try (var context = new CursorContext(this.context.pageCacheTracer.createPageCursorTracer(CONSISTENCY_INDEX_CACHER_TAG))) {
                    while (partition.hasNext() && !this.context.isCancelled()) {
                        long entityId = partition.next();
                        if (!nodeIdRange.isWithinRangeExclusiveTo(entityId)) {
                            if (firstRange && entityId >= this.context.highNodeId) {
                                reporter.forIndexEntry(new IndexEntry(index.descriptor, this.context.tokenNameLookup, entityId)).nodeNotInUse(this.context.recordLoader.node(entityId, context));
                            } else if (firstRange && index.descriptor.isUnique() && index.hasValues) {
                                // We check all values belonging to unique indexes while we are checking the first range, to not
                                // miss duplicated values belonging to different ranges.
                                Value[] indexedValues = partition.values();
                                int checksum = checksum(indexedValues);
                                assert checksum <= CHECKSUM_MASK;
                                lastChecksum = verifyUniquenessInPartition(index, firstValues, lastValues, firstEntityIds, lastEntityIds, slot, lastChecksum, context, entityId, indexedValues, checksum);
                            }
                            continue;
                        }
                        int data = IN_USE_MASK;
                        if (index.hasValues) {
                            Value[] indexedValues = partition.values();
                            int checksum = checksum(indexedValues);
                            assert checksum <= CHECKSUM_MASK;
                            data |= checksum;
                            // Also take the opportunity to verify uniqueness, if the index is a uniqueness index
                            if (firstRange && index.descriptor.isUnique()) {
                                lastChecksum = verifyUniquenessInPartition(index, firstValues, lastValues, firstEntityIds, lastEntityIds, slot, lastChecksum, context, entityId, indexedValues, checksum);
                            }
                        }
                        client.putToCacheSingle(entityId, index.cacheSlotOffset, data);
                        if (++progressPart == INDEX_CACHING_PROGRESS_FACTOR) {
                            localCacheProgress.add(1);
                            progressPart = 0;
                        }
                    }
                }
                localCacheProgress.done();
            };
        }
        // Run the workers that cache the index contents and that do partition-local uniqueness checking, if index is unique
        context.execution.run("Cache index", workers);
        // Then, also if the index is unique then do uniqueness checking of the seams between the partitions
        if (firstRange && index.descriptor.isUnique() && !context.isCancelled()) {
            for (int i = 0; i < partitions.length - 1; i++) {
                Value[] left = lastValues[i];
                Value[] right = firstValues[i + 1];
                // Skip any empty partition - can be empty if all entries in a partition of the index were for nodes outside of the current range.
                if (left != null && right != null && Arrays.equals(left, right)) {
                    long leftEntityId = lastEntityIds[i];
                    long rightEntityId = firstEntityIds[i + 1];
                    reporter.forNode(context.recordLoader.node(leftEntityId, cursorContext)).uniqueIndexNotUnique(index.descriptor, left, rightEntityId);
                }
            }
        }
    } finally {
        IOUtils.closeAll(partitions);
    }
}