use of org.neo4j.internal.recordstorage.RecordStorageEngine in project neo4j by neo4j.
the class IndexWorkSyncTransactionApplicationStressIT method shouldApplyIndexUpdatesInWorkSyncedBatches.
@Test
public void shouldApplyIndexUpdatesInWorkSyncedBatches() throws Exception {
// GIVEN
long duration = parseTimeMillis.apply(System.getProperty(getClass().getName() + ".duration", "2s"));
int numThreads = Integer.getInteger(getClass().getName() + ".numThreads", Runtime.getRuntime().availableProcessors());
DefaultFileSystemAbstraction fs = fileSystemRule.get();
PageCache pageCache = pageCacheRule.getPageCache(fs);
CollectingIndexUpdateListener index = new CollectingIndexUpdateListener();
RecordStorageEngine storageEngine = storageEngineRule.getWith(fs, pageCache, DatabaseLayout.ofFlat(directory.directory(DEFAULT_DATABASE_NAME))).indexUpdateListener(index).build();
storageEngine.apply(tx(singletonList(Commands.createIndexRule(DESCRIPTOR, 1, descriptor))), EXTERNAL);
// WHEN
Workers<Worker> workers = new Workers<>(getClass().getSimpleName());
final AtomicBoolean end = new AtomicBoolean();
for (int i = 0; i < numThreads; i++) {
workers.start(new Worker(i, end, storageEngine, 10, index));
}
// let the threads hammer the storage engine for some time
Thread.sleep(duration);
end.set(true);
// THEN (assertions as part of the workers applying transactions)
workers.awaitAndThrowOnError();
}
use of org.neo4j.internal.recordstorage.RecordStorageEngine in project neo4j by neo4j.
the class BatchingNeoStoresTest method someDataInTheDatabase.
private void someDataInTheDatabase(Config config) throws Exception {
NullLog nullLog = NullLog.getInstance();
try (JobScheduler scheduler = JobSchedulerFactory.createInitialisedScheduler();
PageCache pageCache = new ConfiguringPageCacheFactory(fileSystem, Config.defaults(), PageCacheTracer.NULL, nullLog, scheduler, Clocks.nanoClock(), new MemoryPools()).getOrCreatePageCache();
Lifespan life = new Lifespan()) {
// TODO this little dance with TokenHolders is really annoying and must be solved with a better abstraction
DeferredInitializedTokenCreator propertyKeyTokenCreator = new DeferredInitializedTokenCreator() {
@Override
void create(String name, boolean internal, int id) {
txState.propertyKeyDoCreateForName(name, internal, id);
}
};
DeferredInitializedTokenCreator labelTokenCreator = new DeferredInitializedTokenCreator() {
@Override
void create(String name, boolean internal, int id) {
txState.labelDoCreateForName(name, internal, id);
}
};
DeferredInitializedTokenCreator relationshipTypeTokenCreator = new DeferredInitializedTokenCreator() {
@Override
void create(String name, boolean internal, int id) {
txState.relationshipTypeDoCreateForName(name, internal, id);
}
};
TokenHolders tokenHolders = new TokenHolders(new DelegatingTokenHolder(propertyKeyTokenCreator, TokenHolder.TYPE_PROPERTY_KEY), new DelegatingTokenHolder(labelTokenCreator, TokenHolder.TYPE_LABEL), new DelegatingTokenHolder(relationshipTypeTokenCreator, TokenHolder.TYPE_RELATIONSHIP_TYPE));
IndexConfigCompleter indexConfigCompleter = index -> index;
RecoveryCleanupWorkCollector recoveryCleanupWorkCollector = immediate();
RecordStorageEngine storageEngine = life.add(new RecordStorageEngine(databaseLayout, Config.defaults(), pageCache, fileSystem, NullLogProvider.getInstance(), tokenHolders, new DatabaseSchemaState(NullLogProvider.getInstance()), new StandardConstraintSemantics(), indexConfigCompleter, LockService.NO_LOCK_SERVICE, new DatabaseHealth(PanicEventGenerator.NO_OP, nullLog), new DefaultIdGeneratorFactory(fileSystem, immediate(), DEFAULT_DATABASE_NAME), new DefaultIdController(), recoveryCleanupWorkCollector, PageCacheTracer.NULL, true, INSTANCE, writable(), CommandLockVerification.Factory.IGNORE, LockVerificationMonitor.Factory.IGNORE));
// Create the relationship type token
TxState txState = new TxState();
NeoStores neoStores = storageEngine.testAccessNeoStores();
CommandCreationContext commandCreationContext = storageEngine.newCommandCreationContext(INSTANCE);
commandCreationContext.initialize(NULL);
propertyKeyTokenCreator.initialize(neoStores.getPropertyKeyTokenStore(), txState);
labelTokenCreator.initialize(neoStores.getLabelTokenStore(), txState);
relationshipTypeTokenCreator.initialize(neoStores.getRelationshipTypeTokenStore(), txState);
int relTypeId = tokenHolders.relationshipTypeTokens().getOrCreateId(RELTYPE.name());
apply(txState, commandCreationContext, storageEngine);
// Finally, we're initialized and ready to create two nodes and a relationship
txState = new TxState();
long node1 = commandCreationContext.reserveNode();
long node2 = commandCreationContext.reserveNode();
txState.nodeDoCreate(node1);
txState.nodeDoCreate(node2);
txState.relationshipDoCreate(commandCreationContext.reserveRelationship(), relTypeId, node1, node2);
apply(txState, commandCreationContext, storageEngine);
neoStores.flush(NULL);
}
}
use of org.neo4j.internal.recordstorage.RecordStorageEngine in project neo4j by neo4j.
the class DegreesRebuildFromStoreTest method shouldRebuildDegreesStore.
@Test
void shouldRebuildDegreesStore() throws Exception {
// given a dataset containing mixed sparse and dense nodes with relationships in random directions,
// where some chains have been marked as having external degrees
int denseThreshold = dense_node_threshold.defaultValue();
DatabaseLayout layout = DatabaseLayout.ofFlat(directory.homePath());
int[] relationshipTypes;
MutableLongLongMap expectedDegrees = LongLongMaps.mutable.empty();
try (Lifespan life = new Lifespan()) {
RecordStorageEngine storageEngine = openStorageEngine(layout, denseThreshold);
relationshipTypes = createRelationshipTypes(storageEngine);
life.add(storageEngine);
generateData(storageEngine, denseThreshold, relationshipTypes);
storageEngine.relationshipGroupDegreesStore().accept((groupId, direction, degree) -> expectedDegrees.put(combinedKeyOnGroupAndDirection(groupId, direction), degree), NULL);
assertThat(expectedDegrees.isEmpty()).isFalse();
storageEngine.flushAndForce(NULL);
}
// when
directory.getFileSystem().deleteFile(layout.relationshipGroupDegreesStore());
try (Lifespan life = new Lifespan()) {
RecordStorageEngine storageEngine = life.add(openStorageEngine(layout, denseThreshold));
// then
storageEngine.relationshipGroupDegreesStore().accept((groupId, direction, degree) -> {
long key = combinedKeyOnGroupAndDirection(groupId, direction);
assertThat(expectedDegrees.containsKey(key)).isTrue();
long expectedDegree = expectedDegrees.get(key);
expectedDegrees.remove(key);
assertThat(degree).isEqualTo(expectedDegree);
}, NULL);
}
assertThat(expectedDegrees.size()).isEqualTo(0);
}
use of org.neo4j.internal.recordstorage.RecordStorageEngine in project neo4j by neo4j.
the class MultiIndexPopulationConcurrentUpdatesIT method launchCustomIndexPopulation.
private void launchCustomIndexPopulation(GraphDatabaseSettings.SchemaIndex schemaIndex, Map<String, Integer> labelNameIdMap, int propertyId, Runnable customAction) throws Throwable {
RecordStorageEngine storageEngine = getStorageEngine();
try (Transaction transaction = db.beginTx()) {
Config config = Config.defaults();
KernelTransaction ktx = ((InternalTransaction) transaction).kernelTransaction();
JobScheduler scheduler = getJobScheduler();
NullLogProvider nullLogProvider = NullLogProvider.getInstance();
IndexStoreViewFactory indexStoreViewFactory = mock(IndexStoreViewFactory.class);
when(indexStoreViewFactory.createTokenIndexStoreView(any())).thenAnswer(invocation -> dynamicIndexStoreViewWrapper(customAction, storageEngine::newReader, invocation.getArgument(0), config, scheduler));
IndexProviderMap providerMap = getIndexProviderMap();
indexService = IndexingServiceFactory.createIndexingService(config, scheduler, providerMap, indexStoreViewFactory, ktx.tokenRead(), initialSchemaRulesLoader(storageEngine), nullLogProvider, nullLogProvider, IndexingService.NO_MONITOR, getSchemaState(), mock(IndexStatisticsStore.class), PageCacheTracer.NULL, INSTANCE, "", writable());
indexService.start();
rules = createIndexRules(schemaIndex, labelNameIdMap, propertyId);
schemaCache = new SchemaCache(new StandardConstraintSemantics(), providerMap);
schemaCache.load(iterable(rules));
indexService.createIndexes(AUTH_DISABLED, rules);
transaction.commit();
}
}
use of org.neo4j.internal.recordstorage.RecordStorageEngine in project neo4j by neo4j.
the class RecordStorageEngineRule method get.
private RecordStorageEngine get(FileSystemAbstraction fs, PageCache pageCache, Health databaseHealth, DatabaseLayout databaseLayout, Function<TransactionApplierFactoryChain, TransactionApplierFactoryChain> transactionApplierTransformer, IndexUpdateListener indexUpdateListener, LockService lockService, TokenHolders tokenHolders, Config config, ConstraintRuleAccessor constraintSemantics, IndexConfigCompleter indexConfigCompleter) {
IdGeneratorFactory idGeneratorFactory = new DefaultIdGeneratorFactory(fs, immediate(), databaseLayout.getDatabaseName());
NullLogProvider nullLogProvider = NullLogProvider.getInstance();
RecordStorageEngine engine = new ExtendedRecordStorageEngine(databaseLayout, config, pageCache, fs, nullLogProvider, tokenHolders, mock(SchemaState.class), constraintSemantics, indexConfigCompleter, lockService, databaseHealth, idGeneratorFactory, new DefaultIdController(), transactionApplierTransformer);
engine.addIndexUpdateListener(indexUpdateListener);
life.add(engine);
return engine;
}
Aggregations