Examples with ProgressListener org.neo4j.internal.helpers.progress.ProgressListener used on opensource projects

Example 11 with ProgressListener

use of org.neo4j.internal.helpers.progress.ProgressListener in project neo4j by neo4j.

the class BatchTransactionTest method shouldUseProgressListener.

@Test
void shouldUseProgressListener() {
    // GIVEN
    Transaction transaction = mock(Transaction.class);
    GraphDatabaseService db = mock(GraphDatabaseService.class);
    when(db.beginTx()).thenReturn(transaction);
    ProgressListener progress = mock(ProgressListener.class);
    BatchTransaction tx = beginBatchTx(db).withIntermediarySize(10).withProgress(progress);
    // WHEN
    tx.increment();
    tx.increment(9);
    // THEN
    verify(db, times(2)).beginTx();
    verify(transaction).commit();
    verify(progress).add(1);
    verify(progress).add(9);
}

Example 12 with ProgressListener

use of org.neo4j.internal.helpers.progress.ProgressListener in project neo4j by neo4j.

the class ProgressTrackingOutputStreamTest method shouldTrackByteArrayWrites.

@Test
void shouldTrackByteArrayWrites() throws IOException {
    // given
    OutputStream actual = mock(OutputStream.class);
    ProgressListener progressListener = mock(ProgressListener.class);
    ProgressTrackingOutputStream.Progress progress = new ProgressTrackingOutputStream.Progress(progressListener, 0);
    int length = 14;
    try (ProgressTrackingOutputStream out = new ProgressTrackingOutputStream(actual, progress)) {
        // when
        out.write(new byte[length]);
    }
    progress.done();
    // then
    verify(progressListener).add(length);
    verify(progressListener).done();
    verifyNoMoreInteractions(progressListener);
}

Example 13 with ProgressListener

use of org.neo4j.internal.helpers.progress.ProgressListener in project neo4j by neo4j.

the class IndexChecker method cacheIndex.

private void cacheIndex(IndexContext index, LongRange nodeIdRange, boolean firstRange, CursorContext cursorContext) throws Exception {
    IndexAccessor accessor = indexAccessors.accessorFor(index.descriptor);
    IndexEntriesReader[] partitions = accessor.newAllEntriesValueReader(context.execution.getNumberOfThreads(), cursorContext);
    try {
        Value[][] firstValues = new Value[partitions.length][];
        Value[][] lastValues = new Value[partitions.length][];
        long[] firstEntityIds = new long[partitions.length];
        long[] lastEntityIds = new long[partitions.length];
        ThrowingRunnable[] workers = new ThrowingRunnable[partitions.length];
        for (int i = 0; i < partitions.length; i++) {
            IndexEntriesReader partition = partitions[i];
            int slot = i;
            workers[i] = () -> {
                int lastChecksum = 0;
                int progressPart = 0;
                ProgressListener localCacheProgress = cacheProgress.threadLocalReporter();
                var client = cacheAccess.client();
                try (var context = new CursorContext(this.context.pageCacheTracer.createPageCursorTracer(CONSISTENCY_INDEX_CACHER_TAG))) {
                    while (partition.hasNext() && !this.context.isCancelled()) {
                        long entityId = partition.next();
                        if (!nodeIdRange.isWithinRangeExclusiveTo(entityId)) {
                            if (firstRange && entityId >= this.context.highNodeId) {
                                reporter.forIndexEntry(new IndexEntry(index.descriptor, this.context.tokenNameLookup, entityId)).nodeNotInUse(this.context.recordLoader.node(entityId, context));
                            } else if (firstRange && index.descriptor.isUnique() && index.hasValues) {
                                // We check all values belonging to unique indexes while we are checking the first range, to not
                                // miss duplicated values belonging to different ranges.
                                Value[] indexedValues = partition.values();
                                int checksum = checksum(indexedValues);
                                assert checksum <= CHECKSUM_MASK;
                                lastChecksum = verifyUniquenessInPartition(index, firstValues, lastValues, firstEntityIds, lastEntityIds, slot, lastChecksum, context, entityId, indexedValues, checksum);
                            }
                            continue;
                        }
                        int data = IN_USE_MASK;
                        if (index.hasValues) {
                            Value[] indexedValues = partition.values();
                            int checksum = checksum(indexedValues);
                            assert checksum <= CHECKSUM_MASK;
                            data |= checksum;
                            // Also take the opportunity to verify uniqueness, if the index is a uniqueness index
                            if (firstRange && index.descriptor.isUnique()) {
                                lastChecksum = verifyUniquenessInPartition(index, firstValues, lastValues, firstEntityIds, lastEntityIds, slot, lastChecksum, context, entityId, indexedValues, checksum);
                            }
                        }
                        client.putToCacheSingle(entityId, index.cacheSlotOffset, data);
                        if (++progressPart == INDEX_CACHING_PROGRESS_FACTOR) {
                            localCacheProgress.add(1);
                            progressPart = 0;
                        }
                    }
                }
                localCacheProgress.done();
            };
        }
        // Run the workers that cache the index contents and that do partition-local uniqueness checking, if index is unique
        context.execution.run("Cache index", workers);
        // Then, also if the index is unique then do uniqueness checking of the seams between the partitions
        if (firstRange && index.descriptor.isUnique() && !context.isCancelled()) {
            for (int i = 0; i < partitions.length - 1; i++) {
                Value[] left = lastValues[i];
                Value[] right = firstValues[i + 1];
                // Skip any empty partition - can be empty if all entries in a partition of the index were for nodes outside of the current range.
                if (left != null && right != null && Arrays.equals(left, right)) {
                    long leftEntityId = lastEntityIds[i];
                    long rightEntityId = firstEntityIds[i + 1];
                    reporter.forNode(context.recordLoader.node(leftEntityId, cursorContext)).uniqueIndexNotUnique(index.descriptor, left, rightEntityId);
                }
            }
        }
    } finally {
        IOUtils.closeAll(partitions);
    }
}

Example 14 with ProgressListener

use of org.neo4j.internal.helpers.progress.ProgressListener in project neo4j by neo4j.

the class RelationshipChainChecker method checkDirection.

private void checkDirection(LongRange nodeIdRange, ScanDirection direction) throws Exception {
    RelationshipStore relationshipStore = context.neoStores.getRelationshipStore();
    long highId = relationshipStore.getHighId();
    AtomicBoolean end = new AtomicBoolean();
    int numberOfThreads = numberOfChainCheckers + 1;
    ThrowingRunnable[] workers = new ThrowingRunnable[numberOfThreads];
    ProgressListener localProgress = progress.threadLocalReporter();
    ArrayBlockingQueue<BatchedRelationshipRecords>[] threadQueues = new ArrayBlockingQueue[numberOfChainCheckers];
    BatchedRelationshipRecords[] threadBatches = new BatchedRelationshipRecords[numberOfChainCheckers];
    for (int i = 0; i < numberOfChainCheckers; i++) {
        threadQueues[i] = new ArrayBlockingQueue<>(20);
        threadBatches[i] = new BatchedRelationshipRecords();
        workers[i] = relationshipVsRelationshipChecker(nodeIdRange, direction, relationshipStore, threadQueues[i], end, i);
    }
    // Record reader
    workers[workers.length - 1] = () -> {
        RelationshipRecord relationship = relationshipStore.newRecord();
        try (var cursorContext = new CursorContext(context.pageCacheTracer.createPageCursorTracer(RELATIONSHIP_CONSISTENCY_CHECKER_TAG));
            var cursor = relationshipStore.openPageCursorForReadingWithPrefetching(0, cursorContext)) {
            int recordsPerPage = relationshipStore.getRecordsPerPage();
            long id = direction.startingId(highId);
            while (id >= 0 && id < highId && !context.isCancelled()) {
                for (int i = 0; i < recordsPerPage && id >= 0 && id < highId; i++, id = direction.nextId(id)) {
                    relationshipStore.getRecordByCursor(id, relationship, FORCE, cursor);
                    localProgress.add(1);
                    if (relationship.inUse()) {
                        queueRelationshipCheck(threadQueues, threadBatches, relationship);
                    }
                }
            }
            processLastRelationshipChecks(threadQueues, threadBatches, end);
            localProgress.done();
        }
    };
    Stopwatch stopwatch = Stopwatch.start();
    cacheAccess.clearCache();
    context.execution.runAll(getClass().getSimpleName() + "-" + direction.name(), workers);
    detectSingleRelationshipChainInconsistencies(nodeIdRange);
    context.paddedDebug("%s %s took %s", this, direction, duration(stopwatch.elapsed(TimeUnit.MILLISECONDS)));
}

Example 15 with ProgressListener

use of org.neo4j.internal.helpers.progress.ProgressListener in project neo4j by neo4j.

the class RelationshipGroupChecker method checkToOwner.

/**
 * Check relationship group to owner node
 */
private void checkToOwner(LongRange nodeIdRange, PageCacheTracer pageCacheTracer) {
    ProgressListener localProgress = progress.threadLocalReporter();
    RelationshipGroupStore groupStore = neoStores.getRelationshipGroupStore();
    CacheAccess.Client client = context.cacheAccess.client();
    final long highId = groupStore.getHighId();
    try (var cursorContext = new CursorContext(pageCacheTracer.createPageCursorTracer(RELATIONSHIP_GROUPS_CHECKER_TAG));
        RecordReader<RelationshipGroupRecord> groupReader = new RecordReader<>(neoStores.getRelationshipGroupStore(), true, cursorContext)) {
        for (long id = 0; id < highId && !context.isCancelled(); id++) {
            localProgress.add(1);
            RelationshipGroupRecord record = groupReader.read(id);
            if (!record.inUse()) {
                continue;
            }
            long owningNode = record.getOwningNode();
            if (nodeIdRange.isWithinRangeExclusiveTo(owningNode)) {
                long cachedOwnerNextRel = client.getFromCache(owningNode, CacheSlots.NodeLink.SLOT_RELATIONSHIP_ID);
                boolean nodeIsInUse = client.getBooleanFromCache(owningNode, CacheSlots.NodeLink.SLOT_IN_USE);
                if (!nodeIsInUse) {
                    reporter.forRelationshipGroup(record).ownerNotInUse();
                } else if (cachedOwnerNextRel == id) {
                    // The old checker only verified that the relationship group that node.nextGroup pointed to had this node as its owner
                    client.putToCacheSingle(owningNode, CacheSlots.NodeLink.SLOT_CHECK_MARK, 0);
                }
                if (NULL_REFERENCE.is(record.getNext())) {
                    // This is the last group in the chain for this node. Verify that there's only one such last group.
                    boolean hasAlreadySeenLastGroup = client.getBooleanFromCache(owningNode, CacheSlots.NodeLink.SLOT_HAS_LAST_GROUP);
                    if (hasAlreadySeenLastGroup) {
                        reporter.forRelationshipGroup(record).multipleLastGroups(context.recordLoader.node(owningNode, cursorContext));
                    }
                    client.putToCacheSingle(owningNode, CacheSlots.NodeLink.SLOT_HAS_LAST_GROUP, 1);
                }
            }
        }
    }
    localProgress.done();
}