Examples with ProgressListener org.neo4j.internal.helpers.progress.ProgressListener used on opensource projects

Example 1 with ProgressListener

use of org.neo4j.internal.helpers.progress.ProgressListener in project neo4j by neo4j.

the class RelationshipChecker method check.

private void check(LongRange nodeIdRange, boolean firstRound, long fromRelationshipId, long toRelationshipId, boolean checkToEndOfIndex) throws Exception {
    RelationshipCounter counter = observedCounts.instantiateRelationshipCounter();
    long[] typeHolder = new long[1];
    try (var cursorContext = new CursorContext(context.pageCacheTracer.createPageCursorTracer(RELATIONSHIP_RANGE_CHECKER_TAG));
        RecordReader<RelationshipRecord> relationshipReader = new RecordReader<>(context.neoStores.getRelationshipStore(), true, cursorContext);
        BoundedIterable<EntityTokenRange> relationshipTypeReader = getRelationshipTypeIndexReader(fromRelationshipId, toRelationshipId, checkToEndOfIndex, cursorContext);
        SafePropertyChainReader property = new SafePropertyChainReader(context, cursorContext);
        SchemaComplianceChecker schemaComplianceChecker = new SchemaComplianceChecker(context, mandatoryProperties, indexes, cursorContext, context.memoryTracker)) {
        ProgressListener localProgress = progress.threadLocalReporter();
        CacheAccess.Client client = cacheAccess.client();
        MutableIntObjectMap<Value> propertyValues = new IntObjectHashMap<>();
        Iterator<EntityTokenRange> relationshipTypeRangeIterator = relationshipTypeReader.iterator();
        EntityTokenIndexCheckState typeIndexState = new EntityTokenIndexCheckState(null, fromRelationshipId - 1);
        for (long relationshipId = fromRelationshipId; relationshipId < toRelationshipId && !context.isCancelled(); relationshipId++) {
            localProgress.add(1);
            RelationshipRecord relationshipRecord = relationshipReader.read(relationshipId);
            if (!relationshipRecord.inUse()) {
                continue;
            }
            // Start/end nodes
            long startNode = relationshipRecord.getFirstNode();
            boolean startNodeIsWithinRange = nodeIdRange.isWithinRangeExclusiveTo(startNode);
            boolean startNodeIsNegativeOnFirstRound = startNode < 0 && firstRound;
            if (startNodeIsWithinRange || startNodeIsNegativeOnFirstRound) {
                checkRelationshipVsNode(client, relationshipRecord, startNode, relationshipRecord.isFirstInFirstChain(), (relationship, node) -> reporter.forRelationship(relationship).sourceNodeNotInUse(node), (relationship, node) -> reporter.forRelationship(relationship).sourceNodeDoesNotReferenceBack(node), (relationship, node) -> reporter.forNode(node).relationshipNotFirstInSourceChain(relationship), (relationship, node) -> reporter.forRelationship(relationship).sourceNodeHasNoRelationships(node), relationship -> reporter.forRelationship(relationship).illegalSourceNode(), cursorContext);
            }
            long endNode = relationshipRecord.getSecondNode();
            boolean endNodeIsWithinRange = nodeIdRange.isWithinRangeExclusiveTo(endNode);
            boolean endNodeIsNegativeOnFirstRound = endNode < 0 && firstRound;
            if (endNodeIsWithinRange || endNodeIsNegativeOnFirstRound) {
                checkRelationshipVsNode(client, relationshipRecord, endNode, relationshipRecord.isFirstInSecondChain(), (relationship, node) -> reporter.forRelationship(relationship).targetNodeNotInUse(node), (relationship, node) -> reporter.forRelationship(relationship).targetNodeDoesNotReferenceBack(node), (relationship, node) -> reporter.forNode(node).relationshipNotFirstInTargetChain(relationship), (relationship, node) -> reporter.forRelationship(relationship).targetNodeHasNoRelationships(node), relationship -> reporter.forRelationship(relationship).illegalTargetNode(), cursorContext);
            }
            if (firstRound) {
                if (startNode >= context.highNodeId) {
                    reporter.forRelationship(relationshipRecord).sourceNodeNotInUse(context.recordLoader.node(startNode, cursorContext));
                }
                if (endNode >= context.highNodeId) {
                    reporter.forRelationship(relationshipRecord).targetNodeNotInUse(context.recordLoader.node(endNode, cursorContext));
                }
                // Properties
                typeHolder[0] = relationshipRecord.getType();
                lightClear(propertyValues);
                boolean propertyChainIsOk = property.read(propertyValues, relationshipRecord, reporter::forRelationship, cursorContext);
                if (propertyChainIsOk) {
                    schemaComplianceChecker.checkContainsMandatoryProperties(relationshipRecord, typeHolder, propertyValues, reporter::forRelationship);
                    // gets checked this way, larger indexes will be checked in IndexChecker
                    if (context.consistencyFlags.isCheckIndexes()) {
                        schemaComplianceChecker.checkCorrectlyIndexed((RelationshipRecord) relationshipRecord, typeHolder, propertyValues, reporter::forRelationship);
                    }
                }
                // Type and count
                checkValidToken(relationshipRecord, relationshipRecord.getType(), tokenHolders.relationshipTypeTokens(), neoStores.getRelationshipTypeTokenStore(), (rel, token) -> reporter.forRelationship(rel).illegalRelationshipType(), (rel, token) -> reporter.forRelationship(rel).relationshipTypeNotInUse(token), cursorContext);
                observedCounts.incrementRelationshipTypeCounts(counter, relationshipRecord);
                // Relationship type index
                if (relationshipTypeReader.maxCount() != 0) {
                    checkRelationshipVsRelationshipTypeIndex(relationshipRecord, relationshipTypeRangeIterator, typeIndexState, relationshipId, relationshipRecord.getType(), fromRelationshipId, cursorContext);
                }
            }
            observedCounts.incrementRelationshipNodeCounts(counter, relationshipRecord, startNodeIsWithinRange, endNodeIsWithinRange);
        }
        if (firstRound && !context.isCancelled() && relationshipTypeReader.maxCount() != 0) {
            reportRemainingRelationshipTypeIndexEntries(relationshipTypeRangeIterator, typeIndexState, checkToEndOfIndex ? Long.MAX_VALUE : toRelationshipId, cursorContext);
        }
        localProgress.done();
    }
}

Example 2 with ProgressListener

use of org.neo4j.internal.helpers.progress.ProgressListener in project neo4j by neo4j.

the class IndexChecker method checkVsEntities.

private void checkVsEntities(List<IndexContext> indexes, long fromEntityId, long toEntityId) {
    // This is one thread
    CheckerContext noReportingContext = context.withoutReporting();
    try (var cursorContext = new CursorContext(context.pageCacheTracer.createPageCursorTracer(CONSISTENCY_INDEX_ENTITY_CHECK_TAG));
        RecordReader<NodeRecord> nodeReader = new RecordReader<>(context.neoStores.getNodeStore(), true, cursorContext);
        RecordReader<DynamicRecord> labelReader = new RecordReader<>(context.neoStores.getNodeStore().getDynamicLabelStore(), false, cursorContext);
        SafePropertyChainReader propertyReader = new SafePropertyChainReader(noReportingContext, cursorContext)) {
        ProgressListener localScanProgress = scanProgress.threadLocalReporter();
        IntObjectHashMap<Value> allValues = new IntObjectHashMap<>();
        var client = cacheAccess.client();
        int numberOfIndexes = indexes.size();
        for (long entityId = fromEntityId; entityId < toEntityId && !context.isCancelled(); entityId++) {
            NodeRecord nodeRecord = nodeReader.read(entityId);
            if (nodeRecord.inUse()) {
                long[] entityTokens = safeGetNodeLabels(noReportingContext, nodeRecord.getId(), nodeRecord.getLabelField(), labelReader, cursorContext);
                lightClear(allValues);
                boolean propertyChainRead = entityTokens != null && propertyReader.read(allValues, nodeRecord, noReportingContext.reporter::forNode, cursorContext);
                if (propertyChainRead) {
                    for (int i = 0; i < numberOfIndexes; i++) {
                        IndexContext index = indexes.get(i);
                        IndexDescriptor descriptor = index.descriptor;
                        long cachedValue = client.getFromCache(entityId, i);
                        boolean nodeIsInIndex = (cachedValue & IN_USE_MASK) != 0;
                        Value[] values = RecordLoading.entityIntersectionWithSchema(entityTokens, allValues, descriptor.schema());
                        if (index.descriptor.schema().isFulltextSchemaDescriptor()) {
                            // The strategy for fulltext indexes is way simpler. Simply check of the sets of tokens (label tokens and property key tokens)
                            // and if they match the index schema descriptor then the node should be in the index, otherwise not
                            int[] nodePropertyKeys = allValues.keySet().toArray();
                            int[] indexPropertyKeys = index.descriptor.schema().getPropertyIds();
                            boolean nodeShouldBeInIndex = index.descriptor.schema().isAffected(entityTokens) && containsAny(indexPropertyKeys, nodePropertyKeys) && valuesContainTextProperty(values);
                            if (nodeShouldBeInIndex && !nodeIsInIndex) {
                                getReporter(context.recordLoader.node(entityId, cursorContext)).notIndexed(descriptor, new Object[0]);
                            } else if (!nodeShouldBeInIndex && nodeIsInIndex) {
                                // Fulltext indexes created before 4.3.0-drop02 can contain empty documents (added when the schema matched but the values
                                // were not text). The index still works with those empty documents present, so we don't want to report them as
                                // inconsistencies and force rebuilds.
                                // This utilizes the fact that countIndexedEntities in FulltextIndexReader with non-text values will ask
                                // about documents that doesn't contain those property keys - a document found by that query should be an empty
                                // document we just want to ignore.
                                Value[] noValues = new Value[indexPropertyKeys.length];
                                Arrays.fill(noValues, NO_VALUE);
                                long docsWithNoneOfProperties = indexAccessors.readers().reader(descriptor).countIndexedEntities(entityId, cursorContext, indexPropertyKeys, noValues);
                                if (docsWithNoneOfProperties != 1) {
                                    getReporter(context.recordLoader.node(entityId, cursorContext)).notIndexed(descriptor, new Object[0]);
                                }
                            }
                        } else {
                            if (values != null) {
                                // This node should really be in the index, is it?
                                if (!nodeIsInIndex) {
                                    // It wasn't, report it
                                    getReporter(context.recordLoader.node(entityId, cursorContext)).notIndexed(descriptor, Values.asObjects(values));
                                } else if (index.hasValues) {
                                    int cachedChecksum = (int) cachedValue & CHECKSUM_MASK;
                                    int actualChecksum = checksum(values);
                                    if (cachedChecksum != actualChecksum) {
                                        getReporter(context.recordLoader.node(entityId, cursorContext)).notIndexed(descriptor, Values.asObjects(values));
                                    }
                                }
                            } else {
                                if (nodeIsInIndex) {
                                    reporter.forIndexEntry(new IndexEntry(descriptor, context.tokenNameLookup, entityId)).nodeNotInUse(context.recordLoader.node(entityId, cursorContext));
                                }
                            }
                        }
                    }
                }
            // else this would be reported elsewhere
            } else {
                // This node shouldn't be in any index
                for (int i = 0; i < numberOfIndexes; i++) {
                    boolean isInIndex = (client.getFromCache(entityId, i) & IN_USE_MASK) != 0;
                    if (isInIndex) {
                        reporter.forIndexEntry(new IndexEntry(indexes.get(i).descriptor, context.tokenNameLookup, entityId)).nodeNotInUse(context.recordLoader.node(entityId, cursorContext));
                    }
                }
            }
            localScanProgress.add(1);
        }
        localScanProgress.done();
    }
}

Example 3 with ProgressListener

use of org.neo4j.internal.helpers.progress.ProgressListener in project neo4j by neo4j.

the class EncodingIdMapperTest method shouldReportyProgressForSortAndDetect.

@Test
public void shouldReportyProgressForSortAndDetect() {
    // GIVEN
    IdMapper idMapper = mapper(new StringEncoder(), Radix.STRING, EncodingIdMapper.NO_MONITOR);
    ProgressListener progress = mock(ProgressListener.class);
    idMapper.prepare(null, mock(Collector.class), progress);
    // WHEN
    long id = idMapper.get("123", Group.GLOBAL);
    // THEN
    assertEquals(IdMapper.ID_NOT_FOUND, id);
    verify(progress, times(3)).started(anyString());
    verify(progress, times(3)).done();
}

Example 4 with ProgressListener

use of org.neo4j.internal.helpers.progress.ProgressListener in project neo4j by neo4j.

the class NodeChecker method check.

private void check(long fromNodeId, long toNodeId, boolean last) throws Exception {
    long usedNodes = 0;
    try (var cursorContext = new CursorContext(context.pageCacheTracer.createPageCursorTracer(NODE_RANGE_CHECKER_TAG));
        RecordReader<NodeRecord> nodeReader = new RecordReader<>(context.neoStores.getNodeStore(), true, cursorContext);
        RecordReader<DynamicRecord> labelReader = new RecordReader<>(context.neoStores.getNodeStore().getDynamicLabelStore(), false, cursorContext);
        BoundedIterable<EntityTokenRange> labelIndexReader = getLabelIndexReader(fromNodeId, toNodeId, last, cursorContext);
        SafePropertyChainReader property = new SafePropertyChainReader(context, cursorContext);
        SchemaComplianceChecker schemaComplianceChecker = new SchemaComplianceChecker(context, mandatoryProperties, smallIndexes, cursorContext, context.memoryTracker)) {
        ProgressListener localProgress = nodeProgress.threadLocalReporter();
        MutableIntObjectMap<Value> propertyValues = new IntObjectHashMap<>();
        CacheAccess.Client client = context.cacheAccess.client();
        long[] nextRelCacheFields = new long[] { -1, -1, 1, /*inUse*/
        0, 0, 1, /*note that this needs to be checked*/
        0, 0 };
        Iterator<EntityTokenRange> nodeLabelRangeIterator = labelIndexReader.iterator();
        EntityTokenIndexCheckState labelIndexState = new EntityTokenIndexCheckState(null, fromNodeId - 1);
        for (long nodeId = fromNodeId; nodeId < toNodeId && !context.isCancelled(); nodeId++) {
            localProgress.add(1);
            NodeRecord nodeRecord = nodeReader.read(nodeId);
            if (!nodeRecord.inUse()) {
                continue;
            }
            // Cache nextRel
            long nextRel = nodeRecord.getNextRel();
            if (nextRel < NULL_REFERENCE.longValue()) {
                reporter.forNode(nodeRecord).relationshipNotInUse(new RelationshipRecord(nextRel));
                nextRel = NULL_REFERENCE.longValue();
            }
            nextRelCacheFields[CacheSlots.NodeLink.SLOT_RELATIONSHIP_ID] = nextRel;
            nextRelCacheFields[CacheSlots.NodeLink.SLOT_IS_DENSE] = longOf(nodeRecord.isDense());
            usedNodes++;
            // Labels
            long[] unverifiedLabels = RecordLoading.safeGetNodeLabels(context, nodeRecord.getId(), nodeRecord.getLabelField(), labelReader, cursorContext);
            long[] labels = checkNodeLabels(nodeRecord, unverifiedLabels, cursorContext);
            // Cache the label field, so that if it contains inlined labels then it's free.
            // Otherwise cache the dynamic labels in another data structure and point into it.
            long labelField = nodeRecord.getLabelField();
            boolean hasInlinedLabels = !NodeLabelsField.fieldPointsToDynamicRecordOfLabels(nodeRecord.getLabelField());
            if (labels == null) {
                // There was some inconsistency in the label field or dynamic label chain. Let's continue but w/o labels for this node
                hasInlinedLabels = true;
                labelField = NO_LABELS_FIELD.longValue();
            }
            boolean hasSingleLabel = labels != null && labels.length == 1;
            nextRelCacheFields[CacheSlots.NodeLink.SLOT_HAS_INLINED_LABELS] = longOf(hasInlinedLabels);
            nextRelCacheFields[CacheSlots.NodeLink.SLOT_LABELS] = hasSingleLabel ? // this makes RelationshipChecker "parse" the cached node labels more efficiently for single-label nodes
            labels[0] : // Otherwise put the encoded label field if inlined, otherwise a ref to the cached dynamic labels
            hasInlinedLabels ? labelField : observedCounts.cacheDynamicNodeLabels(labels);
            nextRelCacheFields[CacheSlots.NodeLink.SLOT_HAS_SINGLE_LABEL] = longOf(hasSingleLabel);
            // Properties
            lightClear(propertyValues);
            boolean propertyChainIsOk = property.read(propertyValues, nodeRecord, reporter::forNode, cursorContext);
            // Label index
            if (labelIndexReader.maxCount() != 0) {
                checkNodeVsLabelIndex(nodeRecord, nodeLabelRangeIterator, labelIndexState, nodeId, labels, fromNodeId, cursorContext);
            }
            client.putToCache(nodeId, nextRelCacheFields);
            // Mandatory properties and (some) indexing
            if (labels != null && propertyChainIsOk) {
                schemaComplianceChecker.checkContainsMandatoryProperties(nodeRecord, labels, propertyValues, reporter::forNode);
                // gets checked this way, larger indexes will be checked in IndexChecker
                if (context.consistencyFlags.isCheckIndexes()) {
                    schemaComplianceChecker.checkCorrectlyIndexed(nodeRecord, labels, propertyValues, reporter::forNode);
                }
            }
        // Large indexes are checked elsewhere, more efficiently than per-entity
        }
        if (!context.isCancelled() && labelIndexReader.maxCount() != 0) {
            reportRemainingLabelIndexEntries(nodeLabelRangeIterator, labelIndexState, last ? Long.MAX_VALUE : toNodeId, cursorContext);
        }
        localProgress.done();
    }
    observedCounts.incrementNodeLabel(ANY_LABEL, usedNodes);
}

Example 5 with ProgressListener

use of org.neo4j.internal.helpers.progress.ProgressListener in project neo4j by neo4j.

the class ProgressTrackingOutputStreamTest method shouldNoteInitialPosition.

@Test
void shouldNoteInitialPosition() {
    // given
    ProgressListener progressListener = mock(ProgressListener.class);
    // when
    new ProgressTrackingOutputStream.Progress(progressListener, 10);
    // then
    verify(progressListener).add(10);
    verifyNoMoreInteractions(progressListener);
}