Examples with IndexEntriesReader org.neo4j.kernel.api.index.IndexEntriesReader used on opensource projects

Example 1 with IndexEntriesReader

use of org.neo4j.kernel.api.index.IndexEntriesReader in project neo4j by neo4j.

the class LuceneIndexAccessorIT method shouldPartitionAndReadAllDocuments.

@Test
void shouldPartitionAndReadAllDocuments() throws Exception {
    // given
    IndexDescriptor descriptor = IndexPrototype.forSchema(SchemaDescriptor.forLabel(0, 1)).withName("test").materialise(1);
    MutableLongSet expectedNodes = new LongHashSet();
    populateWithInitialNodes(descriptor, random.nextInt(1_000, 10_000), expectedNodes);
    try (IndexAccessor accessor = indexProvider.getOnlineAccessor(descriptor, samplingConfig, mock(TokenNameLookup.class))) {
        // when
        MutableLongSet readNodes = new LongHashSet();
        IndexEntriesReader[] partitionReaders = accessor.newAllEntriesValueReader(random.nextInt(2, 16), NULL);
        for (IndexEntriesReader partitionReader : partitionReaders) {
            while (partitionReader.hasNext()) {
                boolean added = readNodes.add(partitionReader.next());
                assertThat(added).isTrue();
            }
            partitionReader.close();
        }
        assertThat(readNodes).isEqualTo(expectedNodes);
    }
}

Example 2 with IndexEntriesReader

use of org.neo4j.kernel.api.index.IndexEntriesReader in project neo4j by neo4j.

the class GenericNativeIndexAccessor method newAllEntriesValueReader.

@Override
public IndexEntriesReader[] newAllEntriesValueReader(int partitions, CursorContext cursorContext) {
    GenericKey lowest = layout.newKey();
    lowest.initialize(Long.MIN_VALUE);
    lowest.initValuesAsLowest();
    GenericKey highest = layout.newKey();
    highest.initialize(Long.MAX_VALUE);
    highest.initValuesAsHighest();
    try {
        Collection<Seeker<GenericKey, NativeIndexValue>> seekers = tree.partitionedSeek(lowest, highest, partitions, cursorContext);
        Collection<IndexEntriesReader> readers = new ArrayList<>();
        for (Seeker<GenericKey, NativeIndexValue> seeker : seekers) {
            readers.add(new IndexEntriesReader() {

                @Override
                public long next() {
                    return seeker.key().getEntityId();
                }

                @Override
                public boolean hasNext() {
                    try {
                        return seeker.next();
                    } catch (IOException e) {
                        throw new UncheckedIOException(e);
                    }
                }

                @Override
                public Value[] values() {
                    return seeker.key().asValues();
                }

                @Override
                public void close() {
                    try {
                        seeker.close();
                    } catch (IOException e) {
                        throw new UncheckedIOException(e);
                    }
                }
            });
        }
        return readers.toArray(IndexEntriesReader[]::new);
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    }
}

Example 3 with IndexEntriesReader

use of org.neo4j.kernel.api.index.IndexEntriesReader in project neo4j by neo4j.

the class IndexChecker method cacheIndex.

private void cacheIndex(IndexContext index, LongRange nodeIdRange, boolean firstRange, CursorContext cursorContext) throws Exception {
    IndexAccessor accessor = indexAccessors.accessorFor(index.descriptor);
    IndexEntriesReader[] partitions = accessor.newAllEntriesValueReader(context.execution.getNumberOfThreads(), cursorContext);
    try {
        Value[][] firstValues = new Value[partitions.length][];
        Value[][] lastValues = new Value[partitions.length][];
        long[] firstEntityIds = new long[partitions.length];
        long[] lastEntityIds = new long[partitions.length];
        ThrowingRunnable[] workers = new ThrowingRunnable[partitions.length];
        for (int i = 0; i < partitions.length; i++) {
            IndexEntriesReader partition = partitions[i];
            int slot = i;
            workers[i] = () -> {
                int lastChecksum = 0;
                int progressPart = 0;
                ProgressListener localCacheProgress = cacheProgress.threadLocalReporter();
                var client = cacheAccess.client();
                try (var context = new CursorContext(this.context.pageCacheTracer.createPageCursorTracer(CONSISTENCY_INDEX_CACHER_TAG))) {
                    while (partition.hasNext() && !this.context.isCancelled()) {
                        long entityId = partition.next();
                        if (!nodeIdRange.isWithinRangeExclusiveTo(entityId)) {
                            if (firstRange && entityId >= this.context.highNodeId) {
                                reporter.forIndexEntry(new IndexEntry(index.descriptor, this.context.tokenNameLookup, entityId)).nodeNotInUse(this.context.recordLoader.node(entityId, context));
                            } else if (firstRange && index.descriptor.isUnique() && index.hasValues) {
                                // We check all values belonging to unique indexes while we are checking the first range, to not
                                // miss duplicated values belonging to different ranges.
                                Value[] indexedValues = partition.values();
                                int checksum = checksum(indexedValues);
                                assert checksum <= CHECKSUM_MASK;
                                lastChecksum = verifyUniquenessInPartition(index, firstValues, lastValues, firstEntityIds, lastEntityIds, slot, lastChecksum, context, entityId, indexedValues, checksum);
                            }
                            continue;
                        }
                        int data = IN_USE_MASK;
                        if (index.hasValues) {
                            Value[] indexedValues = partition.values();
                            int checksum = checksum(indexedValues);
                            assert checksum <= CHECKSUM_MASK;
                            data |= checksum;
                            // Also take the opportunity to verify uniqueness, if the index is a uniqueness index
                            if (firstRange && index.descriptor.isUnique()) {
                                lastChecksum = verifyUniquenessInPartition(index, firstValues, lastValues, firstEntityIds, lastEntityIds, slot, lastChecksum, context, entityId, indexedValues, checksum);
                            }
                        }
                        client.putToCacheSingle(entityId, index.cacheSlotOffset, data);
                        if (++progressPart == INDEX_CACHING_PROGRESS_FACTOR) {
                            localCacheProgress.add(1);
                            progressPart = 0;
                        }
                    }
                }
                localCacheProgress.done();
            };
        }
        // Run the workers that cache the index contents and that do partition-local uniqueness checking, if index is unique
        context.execution.run("Cache index", workers);
        // Then, also if the index is unique then do uniqueness checking of the seams between the partitions
        if (firstRange && index.descriptor.isUnique() && !context.isCancelled()) {
            for (int i = 0; i < partitions.length - 1; i++) {
                Value[] left = lastValues[i];
                Value[] right = firstValues[i + 1];
                // Skip any empty partition - can be empty if all entries in a partition of the index were for nodes outside of the current range.
                if (left != null && right != null && Arrays.equals(left, right)) {
                    long leftEntityId = lastEntityIds[i];
                    long rightEntityId = firstEntityIds[i + 1];
                    reporter.forNode(context.recordLoader.node(leftEntityId, cursorContext)).uniqueIndexNotUnique(index.descriptor, left, rightEntityId);
                }
            }
        }
    } finally {
        IOUtils.closeAll(partitions);
    }
}

Example 4 with IndexEntriesReader

use of org.neo4j.kernel.api.index.IndexEntriesReader in project neo4j by neo4j.

the class AbstractLuceneIndexAccessor method newAllEntriesValueReader.

public IndexEntriesReader[] newAllEntriesValueReader(ToLongFunction<Document> entityIdReader, int numPartitions) {
    LuceneAllDocumentsReader allDocumentsReader = luceneIndex.allDocumentsReader();
    List<Iterator<Document>> partitions = allDocumentsReader.partition(numPartitions);
    AtomicInteger closeCount = new AtomicInteger(partitions.size());
    List<IndexEntriesReader> readers = partitions.stream().map(partitionDocuments -> new PartitionIndexEntriesReader(closeCount, allDocumentsReader, entityIdReader, partitionDocuments)).collect(Collectors.toList());
    return readers.toArray(IndexEntriesReader[]::new);
}