Search in sources :

Example 1 with NODE

use of org.neo4j.common.EntityType.NODE in project neo4j by neo4j.

the class IndexingService method init.

/**
 * Called while the database starts up, before recovery.
 */
@Override
public void init() throws IOException {
    validateDefaultProviderExisting();
    try (var cursorContext = new CursorContext(pageCacheTracer.createPageCursorTracer(INIT_TAG))) {
        indexMapRef.modify(indexMap -> {
            Map<InternalIndexState, List<IndexLogRecord>> indexStates = new EnumMap<>(InternalIndexState.class);
            for (IndexDescriptor descriptor : indexDescriptors) {
                // No index (except NLI) is allowed to have the name generated for NLI.
                if (descriptor.getName().equals(IndexDescriptor.NLI_GENERATED_NAME) && !(descriptor.schema().isAnyTokenSchemaDescriptor() && descriptor.schema().entityType() == NODE)) {
                    throw new IllegalStateException("Index '" + descriptor.userDescription(tokenNameLookup) + "' is using a reserved name: '" + IndexDescriptor.NLI_GENERATED_NAME + "'. This index must be removed on an earlier version " + "to be able to use binaries for version 4.3 or newer.");
                }
                IndexProxy indexProxy;
                IndexProviderDescriptor providerDescriptor = descriptor.getIndexProvider();
                IndexProvider provider = providerMap.lookup(providerDescriptor);
                InternalIndexState initialState = provider.getInitialState(descriptor, cursorContext);
                indexStates.computeIfAbsent(initialState, internalIndexState -> new ArrayList<>()).add(new IndexLogRecord(descriptor));
                internalLog.debug(indexStateInfo("init", initialState, descriptor));
                switch(initialState) {
                    case ONLINE:
                        monitor.initialState(databaseName, descriptor, ONLINE);
                        indexProxy = indexProxyCreator.createOnlineIndexProxy(descriptor);
                        break;
                    case POPULATING:
                        // The database was shut down during population, or a crash has occurred, or some other sad thing.
                        monitor.initialState(databaseName, descriptor, POPULATING);
                        indexProxy = indexProxyCreator.createRecoveringIndexProxy(descriptor);
                        break;
                    case FAILED:
                        monitor.initialState(databaseName, descriptor, FAILED);
                        IndexPopulationFailure failure = failure(provider.getPopulationFailure(descriptor, cursorContext));
                        indexProxy = indexProxyCreator.createFailedIndexProxy(descriptor, failure);
                        break;
                    default:
                        throw new IllegalArgumentException("" + initialState);
                }
                indexMap.putIndexProxy(indexProxy);
            }
            logIndexStateSummary("init", indexStates);
            return indexMap;
        });
    }
    indexStatisticsStore.init();
}
Also used : LifecycleAdapter(org.neo4j.kernel.lifecycle.LifecycleAdapter) Arrays(java.util.Arrays) ResourceIterator(org.neo4j.graphdb.ResourceIterator) Log(org.neo4j.logging.Log) CursorContext(org.neo4j.io.pagecache.context.CursorContext) TokenNameLookup(org.neo4j.common.TokenNameLookup) IndexPopulator(org.neo4j.kernel.api.index.IndexPopulator) LongObjectProcedure(org.eclipse.collections.api.block.procedure.primitive.LongObjectProcedure) UnaryOperator(java.util.function.UnaryOperator) Config(org.neo4j.configuration.Config) Value(org.neo4j.values.storable.Value) Preconditions(org.neo4j.util.Preconditions) IndexStatisticsStore(org.neo4j.kernel.impl.api.index.stats.IndexStatisticsStore) UnderlyingStorageException(org.neo4j.exceptions.UnderlyingStorageException) IndexUpdater(org.neo4j.kernel.api.index.IndexUpdater) IndexNotFoundKernelException(org.neo4j.internal.kernel.api.exceptions.schema.IndexNotFoundKernelException) IndexPopulationFailedKernelException(org.neo4j.kernel.api.exceptions.index.IndexPopulationFailedKernelException) LongHashSet(org.eclipse.collections.impl.set.mutable.primitive.LongHashSet) Map(java.util.Map) LongIterable(org.eclipse.collections.api.LongIterable) PageCacheTracer(org.neo4j.io.pagecache.tracing.PageCacheTracer) IndexProviderDescriptor(org.neo4j.internal.schema.IndexProviderDescriptor) Path(java.nio.file.Path) EnumMap(java.util.EnumMap) SYSTEM(org.neo4j.common.Subject.SYSTEM) Collection(java.util.Collection) Set(java.util.Set) IndexEntryUpdate(org.neo4j.storageengine.api.IndexEntryUpdate) Collectors(java.util.stream.Collectors) String.format(java.lang.String.format) UncheckedIOException(java.io.UncheckedIOException) List(java.util.List) EntityType(org.neo4j.common.EntityType) IndexPrototype(org.neo4j.internal.schema.IndexPrototype) FAILED(org.neo4j.internal.kernel.api.InternalIndexState.FAILED) IndexDescriptor(org.neo4j.internal.schema.IndexDescriptor) MutableBoolean(org.apache.commons.lang3.mutable.MutableBoolean) DatabaseReadOnlyChecker(org.neo4j.configuration.helpers.DatabaseReadOnlyChecker) NODE(org.neo4j.common.EntityType.NODE) Iterators.asResourceIterator(org.neo4j.internal.helpers.collection.Iterators.asResourceIterator) GraphDatabaseSettings(org.neo4j.configuration.GraphDatabaseSettings) InternalIndexState(org.neo4j.internal.kernel.api.InternalIndexState) IndexSamplingController(org.neo4j.kernel.impl.api.index.sampling.IndexSamplingController) LogProvider(org.neo4j.logging.LogProvider) HashMap(java.util.HashMap) MutableLongObjectMap(org.eclipse.collections.api.map.primitive.MutableLongObjectMap) Iterators.iterator(org.neo4j.internal.helpers.collection.Iterators.iterator) IndexProvider(org.neo4j.kernel.api.index.IndexProvider) ArrayList(java.util.ArrayList) IndexEntryConflictException(org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException) LongObjectHashMap(org.eclipse.collections.impl.map.mutable.primitive.LongObjectHashMap) ThrowingConsumer(org.neo4j.function.ThrowingConsumer) POPULATING(org.neo4j.internal.kernel.api.InternalIndexState.POPULATING) IndexPopulationFailure.failure(org.neo4j.kernel.impl.api.index.IndexPopulationFailure.failure) ONLINE(org.neo4j.internal.kernel.api.InternalIndexState.ONLINE) IndexStoreViewFactory(org.neo4j.kernel.impl.transaction.state.storeview.IndexStoreViewFactory) JobScheduler(org.neo4j.scheduler.JobScheduler) MemoryTracker(org.neo4j.memory.MemoryTracker) NodePropertyAccessor(org.neo4j.storageengine.api.NodePropertyAccessor) IndexUpdateListener(org.neo4j.storageengine.api.IndexUpdateListener) Subject(org.neo4j.common.Subject) Iterators(org.neo4j.internal.helpers.collection.Iterators) IndexSamplingMode(org.neo4j.kernel.impl.api.index.sampling.IndexSamplingMode) Iterables.asList(org.neo4j.internal.helpers.collection.Iterables.asList) IOException(java.io.IOException) IndexActivationFailedKernelException(org.neo4j.kernel.api.exceptions.index.IndexActivationFailedKernelException) RELATIONSHIP(org.neo4j.common.EntityType.RELATIONSHIP) MutableLongSet(org.eclipse.collections.api.set.primitive.MutableLongSet) TimeUnit(java.util.concurrent.TimeUnit) KernelException(org.neo4j.exceptions.KernelException) StringJoiner(java.util.StringJoiner) UniquePropertyValueValidationException(org.neo4j.kernel.api.exceptions.schema.UniquePropertyValueValidationException) SchemaState(org.neo4j.internal.schema.SchemaState) LongSet(org.eclipse.collections.api.set.primitive.LongSet) IndexProviderDescriptor(org.neo4j.internal.schema.IndexProviderDescriptor) ArrayList(java.util.ArrayList) CursorContext(org.neo4j.io.pagecache.context.CursorContext) IndexDescriptor(org.neo4j.internal.schema.IndexDescriptor) IndexProvider(org.neo4j.kernel.api.index.IndexProvider) InternalIndexState(org.neo4j.internal.kernel.api.InternalIndexState) List(java.util.List) ArrayList(java.util.ArrayList) Iterables.asList(org.neo4j.internal.helpers.collection.Iterables.asList) EnumMap(java.util.EnumMap)

Example 2 with NODE

use of org.neo4j.common.EntityType.NODE in project neo4j by neo4j.

the class TokenScanWriteMonitorTest method shouldPruneAtConfiguredThreshold.

@Test
void shouldPruneAtConfiguredThreshold() {
    // given
    long pruneThreshold = 200;
    RecordingMonitor monitor = new RecordingMonitor();
    FakeClock clock = Clocks.fakeClock();
    TokenScanWriteMonitor writeMonitor = new TokenScanWriteMonitor(fs, databaseLayout, 500, ByteUnit.Byte, pruneThreshold, TimeUnit.MILLISECONDS, NODE, monitor, clock);
    // when
    long startTime = clock.millis();
    long endTime = startTime + TimeUnit.SECONDS.toMillis(1);
    for (int i = 0; clock.millis() < endTime; i++) {
        long timeLeft = endTime - clock.millis();
        clock.forward(min(timeLeft, random.nextInt(1, 10)), TimeUnit.MILLISECONDS);
        writeMonitor.range(i, 1);
        writeMonitor.prepareAdd(i, 5);
        writeMonitor.mergeAdd(new TokenScanValue(), new TokenScanValue().set(5));
        writeMonitor.writeSessionEnded();
    }
    // then
    writeMonitor.close();
    List<Entry> remainingFiles = monitor.rotations.stream().filter(r -> monitor.prunes.stream().noneMatch(p -> r.timestamp == p.timestamp)).collect(Collectors.toList());
    assertThat(remainingFiles).isNotEmpty();
    assertThat(monitor.rotations.size()).isGreaterThan(remainingFiles.size());
    assertThat(monitor.prunes).isNotEmpty();
    remainingFiles.forEach(e -> assertThat(endTime - e.timestamp).isLessThan(pruneThreshold * 2));
}
Also used : BeforeEach(org.junit.jupiter.api.BeforeEach) Arrays(java.util.Arrays) RandomExtension(org.neo4j.test.extension.RandomExtension) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) Config(org.neo4j.configuration.Config) DatabaseLayout(org.neo4j.io.layout.DatabaseLayout) ArrayList(java.util.ArrayList) DirectoryStream(java.nio.file.DirectoryStream) FakeClock(org.neo4j.time.FakeClock) ExtendWith(org.junit.jupiter.api.extension.ExtendWith) Inject(org.neo4j.test.extension.Inject) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) RandomRule(org.neo4j.test.rule.RandomRule) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Path(java.nio.file.Path) ByteUnit(org.neo4j.io.ByteUnit) Long.min(java.lang.Long.min) InOrder(org.mockito.InOrder) Files(java.nio.file.Files) IOException(java.io.IOException) Collectors(java.util.stream.Collectors) Neo4jLayoutExtension(org.neo4j.test.extension.Neo4jLayoutExtension) RELATIONSHIP(org.neo4j.common.EntityType.RELATIONSHIP) Test(org.junit.jupiter.api.Test) TimeUnit(java.util.concurrent.TimeUnit) Mockito(org.mockito.Mockito) List(java.util.List) DefaultFileSystemAbstraction(org.neo4j.io.fs.DefaultFileSystemAbstraction) Stream(java.util.stream.Stream) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) Clocks(org.neo4j.time.Clocks) NODE(org.neo4j.common.EntityType.NODE) Mockito.mock(org.mockito.Mockito.mock) FakeClock(org.neo4j.time.FakeClock) Test(org.junit.jupiter.api.Test)

Aggregations

IOException (java.io.IOException)2 Path (java.nio.file.Path)2 ArrayList (java.util.ArrayList)2 Arrays (java.util.Arrays)2 List (java.util.List)2 TimeUnit (java.util.concurrent.TimeUnit)2 Collectors (java.util.stream.Collectors)2 NODE (org.neo4j.common.EntityType.NODE)2 RELATIONSHIP (org.neo4j.common.EntityType.RELATIONSHIP)2 Config (org.neo4j.configuration.Config)2 UncheckedIOException (java.io.UncheckedIOException)1 Long.min (java.lang.Long.min)1 String.format (java.lang.String.format)1 DirectoryStream (java.nio.file.DirectoryStream)1 Files (java.nio.file.Files)1 Collection (java.util.Collection)1 EnumMap (java.util.EnumMap)1 HashMap (java.util.HashMap)1 Map (java.util.Map)1 Set (java.util.Set)1