Search in sources :

Example 1 with RecordStorageEngine

use of org.neo4j.internal.recordstorage.RecordStorageEngine in project neo4j by neo4j.

the class IndexWorkSyncTransactionApplicationStressIT method shouldApplyIndexUpdatesInWorkSyncedBatches.

@Test
public void shouldApplyIndexUpdatesInWorkSyncedBatches() throws Exception {
    // GIVEN
    long duration = parseTimeMillis.apply(System.getProperty(getClass().getName() + ".duration", "2s"));
    int numThreads = Integer.getInteger(getClass().getName() + ".numThreads", Runtime.getRuntime().availableProcessors());
    DefaultFileSystemAbstraction fs = fileSystemRule.get();
    PageCache pageCache = pageCacheRule.getPageCache(fs);
    CollectingIndexUpdateListener index = new CollectingIndexUpdateListener();
    RecordStorageEngine storageEngine = storageEngineRule.getWith(fs, pageCache, DatabaseLayout.ofFlat(directory.directory(DEFAULT_DATABASE_NAME))).indexUpdateListener(index).build();
    storageEngine.apply(tx(singletonList(Commands.createIndexRule(DESCRIPTOR, 1, descriptor))), EXTERNAL);
    // WHEN
    Workers<Worker> workers = new Workers<>(getClass().getSimpleName());
    final AtomicBoolean end = new AtomicBoolean();
    for (int i = 0; i < numThreads; i++) {
        workers.start(new Worker(i, end, storageEngine, 10, index));
    }
    // let the threads hammer the storage engine for some time
    Thread.sleep(duration);
    end.set(true);
    // THEN (assertions as part of the workers applying transactions)
    workers.awaitAndThrowOnError();
}
Also used : AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) DefaultFileSystemAbstraction(org.neo4j.io.fs.DefaultFileSystemAbstraction) Workers(org.neo4j.internal.batchimport.cache.idmapping.string.Workers) RecordStorageEngine(org.neo4j.internal.recordstorage.RecordStorageEngine) PageCache(org.neo4j.io.pagecache.PageCache) Test(org.junit.Test)

Example 2 with RecordStorageEngine

use of org.neo4j.internal.recordstorage.RecordStorageEngine in project neo4j by neo4j.

the class BatchingNeoStoresTest method someDataInTheDatabase.

private void someDataInTheDatabase(Config config) throws Exception {
    NullLog nullLog = NullLog.getInstance();
    try (JobScheduler scheduler = JobSchedulerFactory.createInitialisedScheduler();
        PageCache pageCache = new ConfiguringPageCacheFactory(fileSystem, Config.defaults(), PageCacheTracer.NULL, nullLog, scheduler, Clocks.nanoClock(), new MemoryPools()).getOrCreatePageCache();
        Lifespan life = new Lifespan()) {
        // TODO this little dance with TokenHolders is really annoying and must be solved with a better abstraction
        DeferredInitializedTokenCreator propertyKeyTokenCreator = new DeferredInitializedTokenCreator() {

            @Override
            void create(String name, boolean internal, int id) {
                txState.propertyKeyDoCreateForName(name, internal, id);
            }
        };
        DeferredInitializedTokenCreator labelTokenCreator = new DeferredInitializedTokenCreator() {

            @Override
            void create(String name, boolean internal, int id) {
                txState.labelDoCreateForName(name, internal, id);
            }
        };
        DeferredInitializedTokenCreator relationshipTypeTokenCreator = new DeferredInitializedTokenCreator() {

            @Override
            void create(String name, boolean internal, int id) {
                txState.relationshipTypeDoCreateForName(name, internal, id);
            }
        };
        TokenHolders tokenHolders = new TokenHolders(new DelegatingTokenHolder(propertyKeyTokenCreator, TokenHolder.TYPE_PROPERTY_KEY), new DelegatingTokenHolder(labelTokenCreator, TokenHolder.TYPE_LABEL), new DelegatingTokenHolder(relationshipTypeTokenCreator, TokenHolder.TYPE_RELATIONSHIP_TYPE));
        IndexConfigCompleter indexConfigCompleter = index -> index;
        RecoveryCleanupWorkCollector recoveryCleanupWorkCollector = immediate();
        RecordStorageEngine storageEngine = life.add(new RecordStorageEngine(databaseLayout, Config.defaults(), pageCache, fileSystem, NullLogProvider.getInstance(), tokenHolders, new DatabaseSchemaState(NullLogProvider.getInstance()), new StandardConstraintSemantics(), indexConfigCompleter, LockService.NO_LOCK_SERVICE, new DatabaseHealth(PanicEventGenerator.NO_OP, nullLog), new DefaultIdGeneratorFactory(fileSystem, immediate(), DEFAULT_DATABASE_NAME), new DefaultIdController(), recoveryCleanupWorkCollector, PageCacheTracer.NULL, true, INSTANCE, writable(), CommandLockVerification.Factory.IGNORE, LockVerificationMonitor.Factory.IGNORE));
        // Create the relationship type token
        TxState txState = new TxState();
        NeoStores neoStores = storageEngine.testAccessNeoStores();
        CommandCreationContext commandCreationContext = storageEngine.newCommandCreationContext(INSTANCE);
        commandCreationContext.initialize(NULL);
        propertyKeyTokenCreator.initialize(neoStores.getPropertyKeyTokenStore(), txState);
        labelTokenCreator.initialize(neoStores.getLabelTokenStore(), txState);
        relationshipTypeTokenCreator.initialize(neoStores.getRelationshipTypeTokenStore(), txState);
        int relTypeId = tokenHolders.relationshipTypeTokens().getOrCreateId(RELTYPE.name());
        apply(txState, commandCreationContext, storageEngine);
        // Finally, we're initialized and ready to create two nodes and a relationship
        txState = new TxState();
        long node1 = commandCreationContext.reserveNode();
        long node2 = commandCreationContext.reserveNode();
        txState.nodeDoCreate(node1);
        txState.nodeDoCreate(node2);
        txState.relationshipDoCreate(commandCreationContext.reserveRelationship(), relTypeId, node1, node2);
        apply(txState, commandCreationContext, storageEngine);
        neoStores.flush(NULL);
    }
}
Also used : JobScheduler(org.neo4j.scheduler.JobScheduler) ThreadPoolJobScheduler(org.neo4j.test.scheduler.ThreadPoolJobScheduler) DefaultIdController(org.neo4j.internal.id.DefaultIdController) NullLogService(org.neo4j.logging.internal.NullLogService) Predicates(org.neo4j.function.Predicates) CursorContext(org.neo4j.io.pagecache.context.CursorContext) TokenCreator(org.neo4j.token.TokenCreator) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) Config(org.neo4j.configuration.Config) LockVerificationMonitor(org.neo4j.internal.recordstorage.LockVerificationMonitor) NullLogProvider(org.neo4j.logging.NullLogProvider) ANY_LABEL(org.neo4j.token.api.TokenConstants.ANY_LABEL) TransactionToApply(org.neo4j.kernel.impl.api.TransactionToApply) DatabaseLayout(org.neo4j.io.layout.DatabaseLayout) DEFAULT_DATABASE_NAME(org.neo4j.configuration.GraphDatabaseSettings.DEFAULT_DATABASE_NAME) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) PageCacheTracer(org.neo4j.io.pagecache.tracing.PageCacheTracer) AbstractBaseRecord(org.neo4j.kernel.impl.store.record.AbstractBaseRecord) NULL(org.neo4j.io.pagecache.context.CursorContext.NULL) RecoveryCleanupWorkCollector(org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector) DatabaseSchemaState(org.neo4j.kernel.impl.api.DatabaseSchemaState) JobSchedulerFactory(org.neo4j.kernel.impl.scheduler.JobSchedulerFactory) NullLog(org.neo4j.logging.NullLog) Input(org.neo4j.internal.batchimport.input.Input) PageCache(org.neo4j.io.pagecache.PageCache) Lifespan(org.neo4j.kernel.lifecycle.Lifespan) DatabaseReadOnlyChecker.writable(org.neo4j.configuration.helpers.DatabaseReadOnlyChecker.writable) TestDirectory(org.neo4j.test.rule.TestDirectory) RecordStorageEngine(org.neo4j.internal.recordstorage.RecordStorageEngine) LockService(org.neo4j.lock.LockService) Neo4jLayoutExtension(org.neo4j.test.extension.Neo4jLayoutExtension) Test(org.junit.jupiter.api.Test) RecordStore(org.neo4j.kernel.impl.store.RecordStore) List(java.util.List) CommandCreationContext(org.neo4j.storageengine.api.CommandCreationContext) INSTANCE(org.neo4j.memory.EmptyMemoryTracker.INSTANCE) Stream(java.util.stream.Stream) DefaultIdGeneratorFactory(org.neo4j.internal.id.DefaultIdGeneratorFactory) StoreType(org.neo4j.kernel.impl.store.StoreType) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) CommandsToApply(org.neo4j.storageengine.api.CommandsToApply) RelationshipType(org.neo4j.graphdb.RelationshipType) ANONYMOUS(org.neo4j.internal.kernel.api.security.AuthSubject.ANONYMOUS) RecordFormatSelector.selectForConfig(org.neo4j.kernel.impl.store.format.RecordFormatSelector.selectForConfig) PhysicalTransactionRepresentation(org.neo4j.kernel.impl.transaction.log.PhysicalTransactionRepresentation) EMPTY(org.neo4j.internal.batchimport.AdditionalInitialIds.EMPTY) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) BASE_TX_ID(org.neo4j.storageengine.api.TransactionIdStore.BASE_TX_ID) ConfiguringPageCacheFactory(org.neo4j.kernel.impl.pagecache.ConfiguringPageCacheFactory) ArrayList(java.util.ArrayList) TxState(org.neo4j.kernel.impl.api.state.TxState) RecordFormats(org.neo4j.kernel.impl.store.format.RecordFormats) StorageCommand(org.neo4j.storageengine.api.StorageCommand) Values(org.neo4j.values.storable.Values) TokenHolder(org.neo4j.token.api.TokenHolder) IndexConfigCompleter(org.neo4j.internal.schema.IndexConfigCompleter) NeoStores(org.neo4j.kernel.impl.store.NeoStores) Inject(org.neo4j.test.extension.Inject) DelegatingTokenHolder(org.neo4j.token.DelegatingTokenHolder) PropertyBlock(org.neo4j.kernel.impl.store.record.PropertyBlock) TransactionApplicationMode(org.neo4j.storageengine.api.TransactionApplicationMode) GraphDatabaseInternalSettings(org.neo4j.configuration.GraphDatabaseInternalSettings) GBPTreeCountsStore(org.neo4j.internal.counts.GBPTreeCountsStore) RecordStorageReader(org.neo4j.internal.recordstorage.RecordStorageReader) MemoryPools(org.neo4j.memory.MemoryPools) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) JobScheduler(org.neo4j.scheduler.JobScheduler) MemoryTracker(org.neo4j.memory.MemoryTracker) DOUBLE_RELATIONSHIP_RECORD_UNIT_THRESHOLD(org.neo4j.internal.batchimport.store.BatchingNeoStores.DOUBLE_RELATIONSHIP_RECORD_UNIT_THRESHOLD) PropertyStore(org.neo4j.kernel.impl.store.PropertyStore) BatchingNeoStores.batchingNeoStores(org.neo4j.internal.batchimport.store.BatchingNeoStores.batchingNeoStores) LockTracer(org.neo4j.lock.LockTracer) StandardConstraintSemantics(org.neo4j.kernel.impl.constraints.StandardConstraintSemantics) PageCacheExtension(org.neo4j.test.extension.pagecache.PageCacheExtension) Configuration(org.neo4j.internal.batchimport.Configuration) PropertyRecord(org.neo4j.kernel.impl.store.record.PropertyRecord) PanicEventGenerator(org.neo4j.monitoring.PanicEventGenerator) DirectoryNotEmptyException(java.nio.file.DirectoryNotEmptyException) IOException(java.io.IOException) TransactionState(org.neo4j.kernel.api.txstate.TransactionState) CommandLockVerification(org.neo4j.internal.recordstorage.CommandLockVerification) TokenStore(org.neo4j.kernel.impl.store.TokenStore) DatabaseHealth(org.neo4j.monitoring.DatabaseHealth) LATEST_RECORD_FORMATS(org.neo4j.kernel.impl.store.format.standard.Standard.LATEST_RECORD_FORMATS) ResourceLocker(org.neo4j.lock.ResourceLocker) CountsAccessor(org.neo4j.counts.CountsAccessor) RecoveryCleanupWorkCollector.immediate(org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector.immediate) ForcedSecondaryUnitRecordFormats(org.neo4j.kernel.impl.store.format.ForcedSecondaryUnitRecordFormats) TokenHolders(org.neo4j.token.TokenHolders) CountsBuilder(org.neo4j.internal.counts.CountsBuilder) ThreadPoolJobScheduler(org.neo4j.test.scheduler.ThreadPoolJobScheduler) Clocks(org.neo4j.time.Clocks) FileSystemAbstraction(org.neo4j.io.fs.FileSystemAbstraction) DatabaseHealth(org.neo4j.monitoring.DatabaseHealth) DefaultIdGeneratorFactory(org.neo4j.internal.id.DefaultIdGeneratorFactory) NullLog(org.neo4j.logging.NullLog) ConfiguringPageCacheFactory(org.neo4j.kernel.impl.pagecache.ConfiguringPageCacheFactory) RecoveryCleanupWorkCollector(org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector) IndexConfigCompleter(org.neo4j.internal.schema.IndexConfigCompleter) DefaultIdController(org.neo4j.internal.id.DefaultIdController) CommandCreationContext(org.neo4j.storageengine.api.CommandCreationContext) MemoryPools(org.neo4j.memory.MemoryPools) DatabaseSchemaState(org.neo4j.kernel.impl.api.DatabaseSchemaState) TxState(org.neo4j.kernel.impl.api.state.TxState) RecordStorageEngine(org.neo4j.internal.recordstorage.RecordStorageEngine) NeoStores(org.neo4j.kernel.impl.store.NeoStores) BatchingNeoStores.batchingNeoStores(org.neo4j.internal.batchimport.store.BatchingNeoStores.batchingNeoStores) TokenHolders(org.neo4j.token.TokenHolders) Lifespan(org.neo4j.kernel.lifecycle.Lifespan) DelegatingTokenHolder(org.neo4j.token.DelegatingTokenHolder) StandardConstraintSemantics(org.neo4j.kernel.impl.constraints.StandardConstraintSemantics) PageCache(org.neo4j.io.pagecache.PageCache)

Example 3 with RecordStorageEngine

use of org.neo4j.internal.recordstorage.RecordStorageEngine in project neo4j by neo4j.

the class DegreesRebuildFromStoreTest method shouldRebuildDegreesStore.

@Test
void shouldRebuildDegreesStore() throws Exception {
    // given a dataset containing mixed sparse and dense nodes with relationships in random directions,
    // where some chains have been marked as having external degrees
    int denseThreshold = dense_node_threshold.defaultValue();
    DatabaseLayout layout = DatabaseLayout.ofFlat(directory.homePath());
    int[] relationshipTypes;
    MutableLongLongMap expectedDegrees = LongLongMaps.mutable.empty();
    try (Lifespan life = new Lifespan()) {
        RecordStorageEngine storageEngine = openStorageEngine(layout, denseThreshold);
        relationshipTypes = createRelationshipTypes(storageEngine);
        life.add(storageEngine);
        generateData(storageEngine, denseThreshold, relationshipTypes);
        storageEngine.relationshipGroupDegreesStore().accept((groupId, direction, degree) -> expectedDegrees.put(combinedKeyOnGroupAndDirection(groupId, direction), degree), NULL);
        assertThat(expectedDegrees.isEmpty()).isFalse();
        storageEngine.flushAndForce(NULL);
    }
    // when
    directory.getFileSystem().deleteFile(layout.relationshipGroupDegreesStore());
    try (Lifespan life = new Lifespan()) {
        RecordStorageEngine storageEngine = life.add(openStorageEngine(layout, denseThreshold));
        // then
        storageEngine.relationshipGroupDegreesStore().accept((groupId, direction, degree) -> {
            long key = combinedKeyOnGroupAndDirection(groupId, direction);
            assertThat(expectedDegrees.containsKey(key)).isTrue();
            long expectedDegree = expectedDegrees.get(key);
            expectedDegrees.remove(key);
            assertThat(degree).isEqualTo(expectedDegree);
        }, NULL);
    }
    assertThat(expectedDegrees.size()).isEqualTo(0);
}
Also used : RecordStorageEngine(org.neo4j.internal.recordstorage.RecordStorageEngine) DatabaseLayout(org.neo4j.io.layout.DatabaseLayout) MutableLongLongMap(org.eclipse.collections.api.map.primitive.MutableLongLongMap) Lifespan(org.neo4j.kernel.lifecycle.Lifespan) Test(org.junit.jupiter.api.Test)

Example 4 with RecordStorageEngine

use of org.neo4j.internal.recordstorage.RecordStorageEngine in project neo4j by neo4j.

the class MultiIndexPopulationConcurrentUpdatesIT method launchCustomIndexPopulation.

private void launchCustomIndexPopulation(GraphDatabaseSettings.SchemaIndex schemaIndex, Map<String, Integer> labelNameIdMap, int propertyId, Runnable customAction) throws Throwable {
    RecordStorageEngine storageEngine = getStorageEngine();
    try (Transaction transaction = db.beginTx()) {
        Config config = Config.defaults();
        KernelTransaction ktx = ((InternalTransaction) transaction).kernelTransaction();
        JobScheduler scheduler = getJobScheduler();
        NullLogProvider nullLogProvider = NullLogProvider.getInstance();
        IndexStoreViewFactory indexStoreViewFactory = mock(IndexStoreViewFactory.class);
        when(indexStoreViewFactory.createTokenIndexStoreView(any())).thenAnswer(invocation -> dynamicIndexStoreViewWrapper(customAction, storageEngine::newReader, invocation.getArgument(0), config, scheduler));
        IndexProviderMap providerMap = getIndexProviderMap();
        indexService = IndexingServiceFactory.createIndexingService(config, scheduler, providerMap, indexStoreViewFactory, ktx.tokenRead(), initialSchemaRulesLoader(storageEngine), nullLogProvider, nullLogProvider, IndexingService.NO_MONITOR, getSchemaState(), mock(IndexStatisticsStore.class), PageCacheTracer.NULL, INSTANCE, "", writable());
        indexService.start();
        rules = createIndexRules(schemaIndex, labelNameIdMap, propertyId);
        schemaCache = new SchemaCache(new StandardConstraintSemantics(), providerMap);
        schemaCache.load(iterable(rules));
        indexService.createIndexes(AUTH_DISABLED, rules);
        transaction.commit();
    }
}
Also used : JobScheduler(org.neo4j.scheduler.JobScheduler) KernelTransaction(org.neo4j.kernel.api.KernelTransaction) InternalTransaction(org.neo4j.kernel.impl.coreapi.InternalTransaction) Transaction(org.neo4j.graphdb.Transaction) KernelTransaction(org.neo4j.kernel.api.KernelTransaction) RecordStorageEngine(org.neo4j.internal.recordstorage.RecordStorageEngine) Config(org.neo4j.configuration.Config) NullLogProvider(org.neo4j.logging.NullLogProvider) SchemaCache(org.neo4j.internal.recordstorage.SchemaCache) InternalTransaction(org.neo4j.kernel.impl.coreapi.InternalTransaction) IndexProviderMap(org.neo4j.kernel.impl.api.index.IndexProviderMap) StandardConstraintSemantics(org.neo4j.kernel.impl.constraints.StandardConstraintSemantics) IndexStoreViewFactory(org.neo4j.kernel.impl.transaction.state.storeview.IndexStoreViewFactory)

Example 5 with RecordStorageEngine

use of org.neo4j.internal.recordstorage.RecordStorageEngine in project neo4j by neo4j.

the class RecordStorageEngineRule method get.

private RecordStorageEngine get(FileSystemAbstraction fs, PageCache pageCache, Health databaseHealth, DatabaseLayout databaseLayout, Function<TransactionApplierFactoryChain, TransactionApplierFactoryChain> transactionApplierTransformer, IndexUpdateListener indexUpdateListener, LockService lockService, TokenHolders tokenHolders, Config config, ConstraintRuleAccessor constraintSemantics, IndexConfigCompleter indexConfigCompleter) {
    IdGeneratorFactory idGeneratorFactory = new DefaultIdGeneratorFactory(fs, immediate(), databaseLayout.getDatabaseName());
    NullLogProvider nullLogProvider = NullLogProvider.getInstance();
    RecordStorageEngine engine = new ExtendedRecordStorageEngine(databaseLayout, config, pageCache, fs, nullLogProvider, tokenHolders, mock(SchemaState.class), constraintSemantics, indexConfigCompleter, lockService, databaseHealth, idGeneratorFactory, new DefaultIdController(), transactionApplierTransformer);
    engine.addIndexUpdateListener(indexUpdateListener);
    life.add(engine);
    return engine;
}
Also used : DefaultIdController(org.neo4j.internal.id.DefaultIdController) SchemaState(org.neo4j.internal.schema.SchemaState) RecordStorageEngine(org.neo4j.internal.recordstorage.RecordStorageEngine) DefaultIdGeneratorFactory(org.neo4j.internal.id.DefaultIdGeneratorFactory) IdGeneratorFactory(org.neo4j.internal.id.IdGeneratorFactory) DefaultIdGeneratorFactory(org.neo4j.internal.id.DefaultIdGeneratorFactory) NullLogProvider(org.neo4j.logging.NullLogProvider)

Aggregations

RecordStorageEngine (org.neo4j.internal.recordstorage.RecordStorageEngine)13 Test (org.junit.jupiter.api.Test)8 Transaction (org.neo4j.graphdb.Transaction)5 PageCache (org.neo4j.io.pagecache.PageCache)5 DatabaseManagementService (org.neo4j.dbms.api.DatabaseManagementService)4 Node (org.neo4j.graphdb.Node)4 DatabaseLayout (org.neo4j.io.layout.DatabaseLayout)4 NeoStores (org.neo4j.kernel.impl.store.NeoStores)4 CountsAccessor (org.neo4j.counts.CountsAccessor)3 Relationship (org.neo4j.graphdb.Relationship)3 IdGeneratorFactory (org.neo4j.internal.id.IdGeneratorFactory)3 NullLogProvider (org.neo4j.logging.NullLogProvider)3 TestDatabaseManagementServiceBuilder (org.neo4j.test.TestDatabaseManagementServiceBuilder)3 TokenHolders (org.neo4j.token.TokenHolders)3 IOException (java.io.IOException)2 ArrayList (java.util.ArrayList)2 List (java.util.List)2 MutableLongLongMap (org.eclipse.collections.api.map.primitive.MutableLongLongMap)2 DependencyResolver (org.neo4j.common.DependencyResolver)2 Config (org.neo4j.configuration.Config)2