Examples with BatchingTransactionAppender org.neo4j.kernel.impl.transaction.log.BatchingTransactionAppender used on opensource projects

Example 1 with BatchingTransactionAppender

use of org.neo4j.kernel.impl.transaction.log.BatchingTransactionAppender in project neo4j by neo4j.

the class NeoStoreDataSource method buildTransactionLogs.

private NeoStoreTransactionLogModule buildTransactionLogs(File storeDir, Config config, LogProvider logProvider, JobScheduler scheduler, FileSystemAbstraction fileSystemAbstraction, StorageEngine storageEngine, LogEntryReader<ReadableClosablePositionAwareChannel> logEntryReader, SynchronizedArrayIdOrderingQueue legacyIndexTransactionOrdering, TransactionIdStore transactionIdStore, LogVersionRepository logVersionRepository) {
    TransactionMetadataCache transactionMetadataCache = new TransactionMetadataCache(100_000);
    LogHeaderCache logHeaderCache = new LogHeaderCache(1000);
    final PhysicalLogFiles logFiles = new PhysicalLogFiles(storeDir, PhysicalLogFile.DEFAULT_NAME, fileSystemAbstraction);
    final PhysicalLogFile logFile = life.add(new PhysicalLogFile(fileSystemAbstraction, logFiles, config.get(GraphDatabaseSettings.logical_log_rotation_threshold), transactionIdStore::getLastCommittedTransactionId, logVersionRepository, physicalLogMonitor, logHeaderCache));
    final PhysicalLogFileInformation.LogVersionToTimestamp logInformation = version -> {
        LogPosition position = LogPosition.start(version);
        try (ReadableLogChannel channel = logFile.getReader(position)) {
            LogEntry entry;
            while ((entry = logEntryReader.readLogEntry(channel)) != null) {
                if (entry instanceof LogEntryStart) {
                    return entry.<LogEntryStart>as().getTimeWritten();
                }
            }
        }
        return -1;
    };
    final LogFileInformation logFileInformation = new PhysicalLogFileInformation(logFiles, logHeaderCache, transactionIdStore::getLastCommittedTransactionId, logInformation);
    if (config.get(GraphDatabaseFacadeFactory.Configuration.ephemeral)) {
        config = config.withDefaults(stringMap(GraphDatabaseSettings.keep_logical_logs.name(), "1 files"));
    }
    String pruningConf = config.get(GraphDatabaseSettings.keep_logical_logs);
    LogPruneStrategy logPruneStrategy = fromConfigValue(fs, logFileInformation, logFiles, pruningConf);
    final LogPruning logPruning = new LogPruningImpl(logPruneStrategy, logProvider);
    final LogRotation logRotation = new LogRotationImpl(monitors.newMonitor(LogRotation.Monitor.class), logFile, databaseHealth);
    final TransactionAppender appender = life.add(new BatchingTransactionAppender(logFile, logRotation, transactionMetadataCache, transactionIdStore, legacyIndexTransactionOrdering, databaseHealth));
    final LogicalTransactionStore logicalTransactionStore = new PhysicalLogicalTransactionStore(logFile, transactionMetadataCache, logEntryReader);
    int txThreshold = config.get(GraphDatabaseSettings.check_point_interval_tx);
    final CountCommittedTransactionThreshold countCommittedTransactionThreshold = new CountCommittedTransactionThreshold(txThreshold);
    long timeMillisThreshold = config.get(GraphDatabaseSettings.check_point_interval_time);
    TimeCheckPointThreshold timeCheckPointThreshold = new TimeCheckPointThreshold(timeMillisThreshold, clock);
    CheckPointThreshold threshold = CheckPointThresholds.or(countCommittedTransactionThreshold, timeCheckPointThreshold);
    final CheckPointerImpl checkPointer = new CheckPointerImpl(transactionIdStore, threshold, storageEngine, logPruning, appender, databaseHealth, logProvider, tracers.checkPointTracer, ioLimiter, storeCopyCheckPointMutex);
    long recurringPeriod = Math.min(timeMillisThreshold, TimeUnit.SECONDS.toMillis(10));
    CheckPointScheduler checkPointScheduler = new CheckPointScheduler(checkPointer, scheduler, recurringPeriod, databaseHealth);
    life.add(checkPointer);
    life.add(checkPointScheduler);
    return new NeoStoreTransactionLogModule(logicalTransactionStore, logFileInformation, logFiles, logFile, logRotation, checkPointer, appender, legacyIndexTransactionOrdering);
}

Example 2 with BatchingTransactionAppender

use of org.neo4j.kernel.impl.transaction.log.BatchingTransactionAppender in project neo4j by neo4j.

the class Runner method createBatchingTransactionAppender.

private BatchingTransactionAppender createBatchingTransactionAppender(TransactionIdStore transactionIdStore, TransactionMetadataCache transactionMetadataCache, LogFile logFile) {
    Log log = NullLog.getInstance();
    KernelEventHandlers kernelEventHandlers = new KernelEventHandlers(log);
    DatabasePanicEventGenerator panicEventGenerator = new DatabasePanicEventGenerator(kernelEventHandlers);
    DatabaseHealth databaseHealth = new DatabaseHealth(panicEventGenerator, log);
    LogRotationImpl logRotation = new LogRotationImpl(NOOP_LOGROTATION_MONITOR, logFile, databaseHealth);
    return new BatchingTransactionAppender(logFile, logRotation, transactionMetadataCache, transactionIdStore, IdOrderingQueue.BYPASS, databaseHealth);
}

Example 3 with BatchingTransactionAppender

use of org.neo4j.kernel.impl.transaction.log.BatchingTransactionAppender in project neo4j by neo4j.

the class Database method buildTransactionLogs.

private DatabaseTransactionLogModule buildTransactionLogs(LogFiles logFiles, Config config, LogProvider logProvider, JobScheduler scheduler, CheckPointerImpl.ForceOperation forceOperation, LogEntryReader logEntryReader, MetadataProvider metadataProvider, Monitors monitors, Dependencies databaseDependencies) {
    TransactionMetadataCache transactionMetadataCache = new TransactionMetadataCache();
    final LogPruning logPruning = new LogPruningImpl(fs, logFiles, logProvider, new LogPruneStrategyFactory(), clock, config);
    final LogRotation logRotation = transactionLogRotation(logFiles, clock, databaseHealth, monitors.newMonitor(LogRotationMonitor.class));
    final BatchingTransactionAppender appender = life.add(new BatchingTransactionAppender(logFiles, logRotation, transactionMetadataCache, metadataProvider, databaseHealth));
    final LogicalTransactionStore logicalTransactionStore = new PhysicalLogicalTransactionStore(logFiles, transactionMetadataCache, logEntryReader, monitors, true);
    CheckPointThreshold threshold = CheckPointThreshold.createThreshold(config, clock, logPruning, logProvider);
    var checkpointAppender = logFiles.getCheckpointFile().getCheckpointAppender();
    final CheckPointerImpl checkPointer = new CheckPointerImpl(metadataProvider, threshold, forceOperation, logPruning, checkpointAppender, databaseHealth, logProvider, tracers, ioController, storeCopyCheckPointMutex, versionContextSupplier, clock);
    long recurringPeriod = threshold.checkFrequencyMillis();
    CheckPointScheduler checkPointScheduler = new CheckPointScheduler(checkPointer, ioController, scheduler, recurringPeriod, databaseHealth, namedDatabaseId.name());
    life.add(checkPointer);
    life.add(checkPointScheduler);
    databaseDependencies.satisfyDependencies(checkPointer, logFiles, logicalTransactionStore, logRotation, appender);
    return new DatabaseTransactionLogModule(checkPointer, appender);
}

Example 4 with BatchingTransactionAppender

use of org.neo4j.kernel.impl.transaction.log.BatchingTransactionAppender in project neo4j by neo4j.

the class Recovery method performRecovery.

/**
 * Performs recovery of database described by provided layout.
 *
 * @param fs database filesystem
 * @param pageCache page cache used to perform database recovery.
 * @param tracers underlying operation tracers
 * @param config custom configuration
 * @param databaseLayout database to recover layout.
 * @param storageEngineFactory {@link StorageEngineFactory} for the storage to recover.
 * @param logProvider log provider
 * @param globalMonitors global server monitors
 * @param extensionFactories extension factories for extensions that should participate in recovery
 * @param providedLogFiles log files from database
 * @param forceRunRecovery to force recovery to run even if the usual checks indicates that it's not required.
 * In specific cases, like after store copy there's always a need for doing a recovery or at least to start the db, checkpoint and shut down,
 * even if the normal "is recovery required" checks says that recovery isn't required.
 * @throws IOException on any unexpected I/O exception encountered during recovery.
 */
public static void performRecovery(FileSystemAbstraction fs, PageCache pageCache, DatabaseTracers tracers, Config config, DatabaseLayout databaseLayout, StorageEngineFactory storageEngineFactory, boolean forceRunRecovery, LogProvider logProvider, Monitors globalMonitors, Iterable<ExtensionFactory<?>> extensionFactories, Optional<LogFiles> providedLogFiles, RecoveryStartupChecker startupChecker, MemoryTracker memoryTracker, Clock clock) throws IOException {
    Log recoveryLog = logProvider.getLog(Recovery.class);
    if (!forceRunRecovery && !isRecoveryRequired(fs, pageCache, databaseLayout, storageEngineFactory, config, providedLogFiles, memoryTracker)) {
        return;
    }
    checkAllFilesPresence(databaseLayout, fs, pageCache, storageEngineFactory);
    LifeSupport recoveryLife = new LifeSupport();
    Monitors monitors = new Monitors(globalMonitors, logProvider);
    DatabasePageCache databasePageCache = new DatabasePageCache(pageCache, IOController.DISABLED);
    SimpleLogService logService = new SimpleLogService(logProvider);
    VersionAwareLogEntryReader logEntryReader = new VersionAwareLogEntryReader(storageEngineFactory.commandReaderFactory());
    DatabaseReadOnlyChecker readOnlyChecker = writable();
    DatabaseSchemaState schemaState = new DatabaseSchemaState(logProvider);
    JobScheduler scheduler = JobSchedulerFactory.createInitialisedScheduler();
    VersionContextSupplier versionContextSupplier = EmptyVersionContextSupplier.EMPTY;
    DatabaseHealth databaseHealth = new DatabaseHealth(PanicEventGenerator.NO_OP, recoveryLog);
    TokenHolders tokenHolders = new TokenHolders(new DelegatingTokenHolder(new ReadOnlyTokenCreator(), TYPE_PROPERTY_KEY), new DelegatingTokenHolder(new ReadOnlyTokenCreator(), TYPE_LABEL), new DelegatingTokenHolder(new ReadOnlyTokenCreator(), TYPE_RELATIONSHIP_TYPE));
    RecoveryCleanupWorkCollector recoveryCleanupCollector = new GroupingRecoveryCleanupWorkCollector(scheduler, INDEX_CLEANUP, INDEX_CLEANUP_WORK, databaseLayout.getDatabaseName());
    DatabaseExtensions extensions = instantiateRecoveryExtensions(databaseLayout, fs, config, logService, databasePageCache, scheduler, DbmsInfo.TOOL, monitors, tokenHolders, recoveryCleanupCollector, readOnlyChecker, extensionFactories, tracers.getPageCacheTracer());
    DefaultIndexProviderMap indexProviderMap = new DefaultIndexProviderMap(extensions, config);
    StorageEngine storageEngine = storageEngineFactory.instantiate(fs, databaseLayout, config, databasePageCache, tokenHolders, schemaState, getConstraintSemantics(), indexProviderMap, NO_LOCK_SERVICE, new DefaultIdGeneratorFactory(fs, recoveryCleanupCollector, databaseLayout.getDatabaseName()), new DefaultIdController(), databaseHealth, logService.getInternalLogProvider(), recoveryCleanupCollector, tracers.getPageCacheTracer(), true, readOnlyChecker, memoryTracker);
    // Schema indexes
    FullScanStoreView fullScanStoreView = new FullScanStoreView(NO_LOCK_SERVICE, storageEngine::newReader, config, scheduler);
    IndexStoreViewFactory indexStoreViewFactory = new IndexStoreViewFactory(config, storageEngine::newReader, NO_LOCKS, fullScanStoreView, NO_LOCK_SERVICE, logProvider);
    IndexStatisticsStore indexStatisticsStore = new IndexStatisticsStore(databasePageCache, databaseLayout, recoveryCleanupCollector, readOnlyChecker, tracers.getPageCacheTracer());
    IndexingService indexingService = Database.buildIndexingService(storageEngine, schemaState, indexStoreViewFactory, indexStatisticsStore, config, scheduler, indexProviderMap, tokenHolders, logProvider, logProvider, monitors.newMonitor(IndexingService.Monitor.class), tracers.getPageCacheTracer(), memoryTracker, databaseLayout.getDatabaseName(), readOnlyChecker);
    MetadataProvider metadataProvider = storageEngine.metadataProvider();
    Dependencies dependencies = new Dependencies();
    dependencies.satisfyDependencies(databaseLayout, config, databasePageCache, fs, logProvider, tokenHolders, schemaState, getConstraintSemantics(), NO_LOCK_SERVICE, databaseHealth, new DefaultIdGeneratorFactory(fs, recoveryCleanupCollector, databaseLayout.getDatabaseName()), new DefaultIdController(), readOnlyChecker, versionContextSupplier, logService, metadataProvider);
    LogFiles logFiles = LogFilesBuilder.builder(databaseLayout, fs).withLogEntryReader(logEntryReader).withConfig(config).withDependencies(dependencies).withMemoryTracker(memoryTracker).build();
    boolean failOnCorruptedLogFiles = config.get(GraphDatabaseInternalSettings.fail_on_corrupted_log_files);
    validateStoreId(logFiles, storageEngine.getStoreId(), config);
    TransactionMetadataCache metadataCache = new TransactionMetadataCache();
    PhysicalLogicalTransactionStore transactionStore = new PhysicalLogicalTransactionStore(logFiles, metadataCache, logEntryReader, monitors, failOnCorruptedLogFiles);
    BatchingTransactionAppender transactionAppender = new BatchingTransactionAppender(logFiles, LogRotation.NO_ROTATION, metadataCache, metadataProvider, databaseHealth);
    LifeSupport schemaLife = new LifeSupport();
    schemaLife.add(storageEngine.schemaAndTokensLifecycle());
    schemaLife.add(indexingService);
    var doParallelRecovery = config.get(GraphDatabaseInternalSettings.do_parallel_recovery);
    TransactionLogsRecovery transactionLogsRecovery = transactionLogRecovery(fs, metadataProvider, monitors.newMonitor(RecoveryMonitor.class), monitors.newMonitor(RecoveryStartInformationProvider.Monitor.class), logFiles, storageEngine, transactionStore, metadataProvider, schemaLife, databaseLayout, failOnCorruptedLogFiles, recoveryLog, startupChecker, tracers.getPageCacheTracer(), memoryTracker, doParallelRecovery);
    CheckPointerImpl.ForceOperation forceOperation = new DefaultForceOperation(indexingService, storageEngine);
    var checkpointAppender = logFiles.getCheckpointFile().getCheckpointAppender();
    CheckPointerImpl checkPointer = new CheckPointerImpl(metadataProvider, RecoveryThreshold.INSTANCE, forceOperation, LogPruning.NO_PRUNING, checkpointAppender, databaseHealth, logProvider, tracers, IOController.DISABLED, new StoreCopyCheckPointMutex(), versionContextSupplier, clock);
    recoveryLife.add(scheduler);
    recoveryLife.add(recoveryCleanupCollector);
    recoveryLife.add(extensions);
    recoveryLife.add(indexProviderMap);
    recoveryLife.add(storageEngine);
    recoveryLife.add(new MissingTransactionLogsCheck(databaseLayout, config, fs, logFiles, recoveryLog));
    recoveryLife.add(logFiles);
    recoveryLife.add(transactionLogsRecovery);
    recoveryLife.add(transactionAppender);
    recoveryLife.add(checkPointer);
    try {
        recoveryLife.start();
        if (databaseHealth.isHealthy()) {
            checkPointer.forceCheckPoint(new SimpleTriggerInfo("Recovery completed."));
        }
    } finally {
        recoveryLife.shutdown();
    }
}

Example 5 with BatchingTransactionAppender

use of org.neo4j.kernel.impl.transaction.log.BatchingTransactionAppender in project neo4j by neo4j.

the class Runner method createBatchingTransactionAppender.

private static BatchingTransactionAppender createBatchingTransactionAppender(TransactionIdStore transactionIdStore, TransactionMetadataCache transactionMetadataCache, LogFiles logFiles) {
    Log log = NullLog.getInstance();
    Health databaseHealth = new DatabaseHealth(PanicEventGenerator.NO_OP, log);
    LogRotation logRotation = transactionLogRotation(logFiles, Clock.systemUTC(), databaseHealth, EMPTY);
    return new BatchingTransactionAppender(logFiles, logRotation, transactionMetadataCache, transactionIdStore, databaseHealth);
}