Examples with StorageEngine org.neo4j.storageengine.api.StorageEngine used on opensource projects

Example 21 with StorageEngine

use of org.neo4j.storageengine.api.StorageEngine in project neo4j by neo4j.

the class DatabaseFileListingTest method shouldCloseIndexSnapshots.

@Test
void shouldCloseIndexSnapshots() throws Exception {
    // Given
    IndexingService indexingService = mock(IndexingService.class);
    DatabaseLayout databaseLayout = mock(DatabaseLayout.class);
    when(databaseLayout.metadataStore()).thenReturn(mock(Path.class));
    LogFiles logFiles = mock(LogFiles.class);
    filesInStoreDirAre(databaseLayout, STANDARD_STORE_DIR_FILES, STANDARD_STORE_DIR_DIRECTORIES);
    StorageEngine storageEngine = mock(StorageEngine.class);
    IdGeneratorFactory idGeneratorFactory = mock(IdGeneratorFactory.class);
    DatabaseFileListing fileListing = new DatabaseFileListing(databaseLayout, logFiles, indexingService, storageEngine, idGeneratorFactory);
    ResourceIterator<Path> indexSnapshot = indexFilesAre(indexingService, new String[] { "schema/index/my.index" });
    ResourceIterator<StoreFileMetadata> result = fileListing.builder().excludeLogFiles().build();
    // When
    result.close();
    // Then
    verify(indexSnapshot).close();
}

Example 22 with StorageEngine

use of org.neo4j.storageengine.api.StorageEngine in project neo4j by neo4j.

the class DbmsDiagnosticsManager method dumpDatabaseDiagnostics.

private static void dumpDatabaseDiagnostics(Database database, Log log, boolean checkStatus) {
    dumpAsSingleMessageWithDbPrefix(log, stringJoiner -> {
        dumpDatabaseSectionName(database, stringJoiner::add);
        if (checkStatus) {
            logDatabaseStatus(database, stringJoiner::add);
            if (!database.isStarted()) {
                return;
            }
        }
        Dependencies databaseResolver = database.getDependencyResolver();
        DbmsInfo dbmsInfo = databaseResolver.resolveDependency(DbmsInfo.class);
        FileSystemAbstraction fs = databaseResolver.resolveDependency(FileSystemAbstraction.class);
        StorageEngineFactory storageEngineFactory = databaseResolver.resolveDependency(StorageEngineFactory.class);
        StorageEngine storageEngine = databaseResolver.resolveDependency(StorageEngine.class);
        DiagnosticsManager.dump(new VersionDiagnostics(dbmsInfo, database.getStoreId()), log, stringJoiner::add);
        DiagnosticsManager.dump(new StoreFilesDiagnostics(storageEngineFactory, fs, database.getDatabaseLayout()), log, stringJoiner::add);
        DiagnosticsManager.dump(new TransactionRangeDiagnostics(database), log, stringJoiner::add);
        storageEngine.dumpDiagnostics(log, stringJoiner::add);
    }, database.getNamedDatabaseId());
}

Example 23 with StorageEngine

use of org.neo4j.storageengine.api.StorageEngine in project neo4j by neo4j.

the class KernelTransactionsTest method newKernelTransactions.

private KernelTransactions newKernelTransactions(boolean testKernelTransactions, TransactionCommitProcess commitProcess, StorageReader firstReader, Config config, StorageReader... otherReaders) throws Throwable {
    Locks locks = mock(Locks.class);
    Locks.Client client = mock(Locks.Client.class);
    when(locks.newClient()).thenReturn(client);
    StorageEngine storageEngine = mock(StorageEngine.class);
    when(storageEngine.newReader()).thenReturn(firstReader, otherReaders);
    when(storageEngine.newCommandCreationContext(any())).thenReturn(mock(CommandCreationContext.class));
    doAnswer(invocation -> {
        Collection<StorageCommand> argument = invocation.getArgument(0);
        argument.add(mock(StorageCommand.class));
        return null;
    }).when(storageEngine).createCommands(anyCollection(), any(ReadableTransactionState.class), any(StorageReader.class), any(CommandCreationContext.class), any(ResourceLocker.class), any(LockTracer.class), anyLong(), any(TxStateVisitor.Decorator.class), any(CursorContext.class), any(MemoryTracker.class));
    return newKernelTransactions(locks, storageEngine, commitProcess, testKernelTransactions, config);
}

Example 24 with StorageEngine

use of org.neo4j.storageengine.api.StorageEngine in project neo4j by neo4j.

the class Recovery method performRecovery.

/**
 * Performs recovery of database described by provided layout.
 *
 * @param fs database filesystem
 * @param pageCache page cache used to perform database recovery.
 * @param tracers underlying operation tracers
 * @param config custom configuration
 * @param databaseLayout database to recover layout.
 * @param storageEngineFactory {@link StorageEngineFactory} for the storage to recover.
 * @param logProvider log provider
 * @param globalMonitors global server monitors
 * @param extensionFactories extension factories for extensions that should participate in recovery
 * @param providedLogFiles log files from database
 * @param forceRunRecovery to force recovery to run even if the usual checks indicates that it's not required.
 * In specific cases, like after store copy there's always a need for doing a recovery or at least to start the db, checkpoint and shut down,
 * even if the normal "is recovery required" checks says that recovery isn't required.
 * @throws IOException on any unexpected I/O exception encountered during recovery.
 */
public static void performRecovery(FileSystemAbstraction fs, PageCache pageCache, DatabaseTracers tracers, Config config, DatabaseLayout databaseLayout, StorageEngineFactory storageEngineFactory, boolean forceRunRecovery, LogProvider logProvider, Monitors globalMonitors, Iterable<ExtensionFactory<?>> extensionFactories, Optional<LogFiles> providedLogFiles, RecoveryStartupChecker startupChecker, MemoryTracker memoryTracker, Clock clock) throws IOException {
    Log recoveryLog = logProvider.getLog(Recovery.class);
    if (!forceRunRecovery && !isRecoveryRequired(fs, pageCache, databaseLayout, storageEngineFactory, config, providedLogFiles, memoryTracker)) {
        return;
    }
    checkAllFilesPresence(databaseLayout, fs, pageCache, storageEngineFactory);
    LifeSupport recoveryLife = new LifeSupport();
    Monitors monitors = new Monitors(globalMonitors, logProvider);
    DatabasePageCache databasePageCache = new DatabasePageCache(pageCache, IOController.DISABLED);
    SimpleLogService logService = new SimpleLogService(logProvider);
    VersionAwareLogEntryReader logEntryReader = new VersionAwareLogEntryReader(storageEngineFactory.commandReaderFactory());
    DatabaseReadOnlyChecker readOnlyChecker = writable();
    DatabaseSchemaState schemaState = new DatabaseSchemaState(logProvider);
    JobScheduler scheduler = JobSchedulerFactory.createInitialisedScheduler();
    VersionContextSupplier versionContextSupplier = EmptyVersionContextSupplier.EMPTY;
    DatabaseHealth databaseHealth = new DatabaseHealth(PanicEventGenerator.NO_OP, recoveryLog);
    TokenHolders tokenHolders = new TokenHolders(new DelegatingTokenHolder(new ReadOnlyTokenCreator(), TYPE_PROPERTY_KEY), new DelegatingTokenHolder(new ReadOnlyTokenCreator(), TYPE_LABEL), new DelegatingTokenHolder(new ReadOnlyTokenCreator(), TYPE_RELATIONSHIP_TYPE));
    RecoveryCleanupWorkCollector recoveryCleanupCollector = new GroupingRecoveryCleanupWorkCollector(scheduler, INDEX_CLEANUP, INDEX_CLEANUP_WORK, databaseLayout.getDatabaseName());
    DatabaseExtensions extensions = instantiateRecoveryExtensions(databaseLayout, fs, config, logService, databasePageCache, scheduler, DbmsInfo.TOOL, monitors, tokenHolders, recoveryCleanupCollector, readOnlyChecker, extensionFactories, tracers.getPageCacheTracer());
    DefaultIndexProviderMap indexProviderMap = new DefaultIndexProviderMap(extensions, config);
    StorageEngine storageEngine = storageEngineFactory.instantiate(fs, databaseLayout, config, databasePageCache, tokenHolders, schemaState, getConstraintSemantics(), indexProviderMap, NO_LOCK_SERVICE, new DefaultIdGeneratorFactory(fs, recoveryCleanupCollector, databaseLayout.getDatabaseName()), new DefaultIdController(), databaseHealth, logService.getInternalLogProvider(), recoveryCleanupCollector, tracers.getPageCacheTracer(), true, readOnlyChecker, memoryTracker);
    // Schema indexes
    FullScanStoreView fullScanStoreView = new FullScanStoreView(NO_LOCK_SERVICE, storageEngine::newReader, config, scheduler);
    IndexStoreViewFactory indexStoreViewFactory = new IndexStoreViewFactory(config, storageEngine::newReader, NO_LOCKS, fullScanStoreView, NO_LOCK_SERVICE, logProvider);
    IndexStatisticsStore indexStatisticsStore = new IndexStatisticsStore(databasePageCache, databaseLayout, recoveryCleanupCollector, readOnlyChecker, tracers.getPageCacheTracer());
    IndexingService indexingService = Database.buildIndexingService(storageEngine, schemaState, indexStoreViewFactory, indexStatisticsStore, config, scheduler, indexProviderMap, tokenHolders, logProvider, logProvider, monitors.newMonitor(IndexingService.Monitor.class), tracers.getPageCacheTracer(), memoryTracker, databaseLayout.getDatabaseName(), readOnlyChecker);
    MetadataProvider metadataProvider = storageEngine.metadataProvider();
    Dependencies dependencies = new Dependencies();
    dependencies.satisfyDependencies(databaseLayout, config, databasePageCache, fs, logProvider, tokenHolders, schemaState, getConstraintSemantics(), NO_LOCK_SERVICE, databaseHealth, new DefaultIdGeneratorFactory(fs, recoveryCleanupCollector, databaseLayout.getDatabaseName()), new DefaultIdController(), readOnlyChecker, versionContextSupplier, logService, metadataProvider);
    LogFiles logFiles = LogFilesBuilder.builder(databaseLayout, fs).withLogEntryReader(logEntryReader).withConfig(config).withDependencies(dependencies).withMemoryTracker(memoryTracker).build();
    boolean failOnCorruptedLogFiles = config.get(GraphDatabaseInternalSettings.fail_on_corrupted_log_files);
    validateStoreId(logFiles, storageEngine.getStoreId(), config);
    TransactionMetadataCache metadataCache = new TransactionMetadataCache();
    PhysicalLogicalTransactionStore transactionStore = new PhysicalLogicalTransactionStore(logFiles, metadataCache, logEntryReader, monitors, failOnCorruptedLogFiles);
    BatchingTransactionAppender transactionAppender = new BatchingTransactionAppender(logFiles, LogRotation.NO_ROTATION, metadataCache, metadataProvider, databaseHealth);
    LifeSupport schemaLife = new LifeSupport();
    schemaLife.add(storageEngine.schemaAndTokensLifecycle());
    schemaLife.add(indexingService);
    var doParallelRecovery = config.get(GraphDatabaseInternalSettings.do_parallel_recovery);
    TransactionLogsRecovery transactionLogsRecovery = transactionLogRecovery(fs, metadataProvider, monitors.newMonitor(RecoveryMonitor.class), monitors.newMonitor(RecoveryStartInformationProvider.Monitor.class), logFiles, storageEngine, transactionStore, metadataProvider, schemaLife, databaseLayout, failOnCorruptedLogFiles, recoveryLog, startupChecker, tracers.getPageCacheTracer(), memoryTracker, doParallelRecovery);
    CheckPointerImpl.ForceOperation forceOperation = new DefaultForceOperation(indexingService, storageEngine);
    var checkpointAppender = logFiles.getCheckpointFile().getCheckpointAppender();
    CheckPointerImpl checkPointer = new CheckPointerImpl(metadataProvider, RecoveryThreshold.INSTANCE, forceOperation, LogPruning.NO_PRUNING, checkpointAppender, databaseHealth, logProvider, tracers, IOController.DISABLED, new StoreCopyCheckPointMutex(), versionContextSupplier, clock);
    recoveryLife.add(scheduler);
    recoveryLife.add(recoveryCleanupCollector);
    recoveryLife.add(extensions);
    recoveryLife.add(indexProviderMap);
    recoveryLife.add(storageEngine);
    recoveryLife.add(new MissingTransactionLogsCheck(databaseLayout, config, fs, logFiles, recoveryLog));
    recoveryLife.add(logFiles);
    recoveryLife.add(transactionLogsRecovery);
    recoveryLife.add(transactionAppender);
    recoveryLife.add(checkPointer);
    try {
        recoveryLife.start();
        if (databaseHealth.isHealthy()) {
            checkPointer.forceCheckPoint(new SimpleTriggerInfo("Recovery completed."));
        }
    } finally {
        recoveryLife.shutdown();
    }
}

Example 25 with StorageEngine

use of org.neo4j.storageengine.api.StorageEngine in project neo4j by neo4j.

the class KernelTransactionFactory method kernelTransactionWithInternals.

private static Instances kernelTransactionWithInternals(LoginContext loginContext) {
    StorageEngine storageEngine = mock(StorageEngine.class);
    StorageReader storageReader = mock(StorageReader.class);
    when(storageEngine.newReader()).thenReturn(storageReader);
    when(storageEngine.newCommandCreationContext(any())).thenReturn(mock(CommandCreationContext.class));
    Dependencies dependencies = new Dependencies();
    dependencies.satisfyDependency(mock(GraphDatabaseFacade.class));
    KernelTransactionImplementation transaction = new KernelTransactionImplementation(Config.defaults(), mock(DatabaseTransactionEventListeners.class), mock(ConstraintIndexCreator.class), mock(GlobalProcedures.class), mock(InternalTransactionCommitProcess.class), mock(TransactionMonitor.class), mock(Pool.class), Clocks.nanoClock(), new AtomicReference<>(CpuClock.NOT_AVAILABLE), mock(DatabaseTracers.class, RETURNS_MOCKS), storageEngine, any -> CanWrite.INSTANCE, EmptyVersionContextSupplier.EMPTY, ON_HEAP, new StandardConstraintSemantics(), mock(SchemaState.class), mockedTokenHolders(), mock(IndexingService.class), mock(IndexStatisticsStore.class), dependencies, new TestDatabaseIdRepository().defaultDatabase(), LeaseService.NO_LEASES, MemoryPools.NO_TRACKING, DatabaseReadOnlyChecker.writable(), TransactionExecutionMonitor.NO_OP, CommunitySecurityLog.NULL_LOG, () -> KernelVersion.LATEST, mock(DbmsRuntimeRepository.class));
    transaction.initialize(0, 0, new NoOpClient(), KernelTransaction.Type.IMPLICIT, loginContext.authorize(LoginContext.IdLookup.EMPTY, DEFAULT_DATABASE_NAME, CommunitySecurityLog.NULL_LOG), 0L, 1L, EMBEDDED_CONNECTION);
    return new Instances(transaction);
}