Search in sources :

Example 1 with LogPruneStrategy

use of org.neo4j.kernel.impl.transaction.log.pruning.LogPruneStrategy in project neo4j by neo4j.

the class NeoStoreDataSource method buildTransactionLogs.

private NeoStoreTransactionLogModule buildTransactionLogs(File storeDir, Config config, LogProvider logProvider, JobScheduler scheduler, FileSystemAbstraction fileSystemAbstraction, StorageEngine storageEngine, LogEntryReader<ReadableClosablePositionAwareChannel> logEntryReader, SynchronizedArrayIdOrderingQueue legacyIndexTransactionOrdering, TransactionIdStore transactionIdStore, LogVersionRepository logVersionRepository) {
    TransactionMetadataCache transactionMetadataCache = new TransactionMetadataCache(100_000);
    LogHeaderCache logHeaderCache = new LogHeaderCache(1000);
    final PhysicalLogFiles logFiles = new PhysicalLogFiles(storeDir, PhysicalLogFile.DEFAULT_NAME, fileSystemAbstraction);
    final PhysicalLogFile logFile = life.add(new PhysicalLogFile(fileSystemAbstraction, logFiles, config.get(GraphDatabaseSettings.logical_log_rotation_threshold), transactionIdStore::getLastCommittedTransactionId, logVersionRepository, physicalLogMonitor, logHeaderCache));
    final PhysicalLogFileInformation.LogVersionToTimestamp logInformation = version -> {
        LogPosition position = LogPosition.start(version);
        try (ReadableLogChannel channel = logFile.getReader(position)) {
            LogEntry entry;
            while ((entry = logEntryReader.readLogEntry(channel)) != null) {
                if (entry instanceof LogEntryStart) {
                    return entry.<LogEntryStart>as().getTimeWritten();
                }
            }
        }
        return -1;
    };
    final LogFileInformation logFileInformation = new PhysicalLogFileInformation(logFiles, logHeaderCache, transactionIdStore::getLastCommittedTransactionId, logInformation);
    if (config.get(GraphDatabaseFacadeFactory.Configuration.ephemeral)) {
        config = config.withDefaults(stringMap(GraphDatabaseSettings.keep_logical_logs.name(), "1 files"));
    }
    String pruningConf = config.get(GraphDatabaseSettings.keep_logical_logs);
    LogPruneStrategy logPruneStrategy = fromConfigValue(fs, logFileInformation, logFiles, pruningConf);
    final LogPruning logPruning = new LogPruningImpl(logPruneStrategy, logProvider);
    final LogRotation logRotation = new LogRotationImpl(monitors.newMonitor(LogRotation.Monitor.class), logFile, databaseHealth);
    final TransactionAppender appender = life.add(new BatchingTransactionAppender(logFile, logRotation, transactionMetadataCache, transactionIdStore, legacyIndexTransactionOrdering, databaseHealth));
    final LogicalTransactionStore logicalTransactionStore = new PhysicalLogicalTransactionStore(logFile, transactionMetadataCache, logEntryReader);
    int txThreshold = config.get(GraphDatabaseSettings.check_point_interval_tx);
    final CountCommittedTransactionThreshold countCommittedTransactionThreshold = new CountCommittedTransactionThreshold(txThreshold);
    long timeMillisThreshold = config.get(GraphDatabaseSettings.check_point_interval_time);
    TimeCheckPointThreshold timeCheckPointThreshold = new TimeCheckPointThreshold(timeMillisThreshold, clock);
    CheckPointThreshold threshold = CheckPointThresholds.or(countCommittedTransactionThreshold, timeCheckPointThreshold);
    final CheckPointerImpl checkPointer = new CheckPointerImpl(transactionIdStore, threshold, storageEngine, logPruning, appender, databaseHealth, logProvider, tracers.checkPointTracer, ioLimiter, storeCopyCheckPointMutex);
    long recurringPeriod = Math.min(timeMillisThreshold, TimeUnit.SECONDS.toMillis(10));
    CheckPointScheduler checkPointScheduler = new CheckPointScheduler(checkPointer, scheduler, recurringPeriod, databaseHealth);
    life.add(checkPointer);
    life.add(checkPointScheduler);
    return new NeoStoreTransactionLogModule(logicalTransactionStore, logFileInformation, logFiles, logFile, logRotation, checkPointer, appender, legacyIndexTransactionOrdering);
}
Also used : LegacyIndexStore(org.neo4j.kernel.impl.index.LegacyIndexStore) StoreId(org.neo4j.kernel.impl.store.StoreId) ResourceIterator(org.neo4j.graphdb.ResourceIterator) Lifecycles(org.neo4j.kernel.lifecycle.Lifecycles) RelationshipTypeTokenHolder(org.neo4j.kernel.impl.core.RelationshipTypeTokenHolder) SystemNanoClock(org.neo4j.time.SystemNanoClock) AccessCapability(org.neo4j.kernel.impl.factory.AccessCapability) SchemaWriteGuard(org.neo4j.kernel.impl.api.SchemaWriteGuard) CheckPointThresholds(org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointThresholds) VersionAwareLogEntryReader(org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader) RecordStorageEngine(org.neo4j.kernel.impl.storageengine.impl.recordstorage.RecordStorageEngine) LogHeader(org.neo4j.kernel.impl.transaction.log.entry.LogHeader) SchemaIndexProvider(org.neo4j.kernel.api.index.SchemaIndexProvider) Map(java.util.Map) IndexProviders(org.neo4j.kernel.spi.legacyindex.IndexProviders) TokenNameLookup(org.neo4j.kernel.api.TokenNameLookup) KernelSchemaStateStore(org.neo4j.kernel.impl.api.KernelSchemaStateStore) RecordFormatPropertyConfigurator(org.neo4j.kernel.impl.store.format.RecordFormatPropertyConfigurator) TransactionMonitor(org.neo4j.kernel.impl.transaction.TransactionMonitor) DefaultRecoverySPI(org.neo4j.kernel.recovery.DefaultRecoverySPI) HighestSelectionStrategy(org.neo4j.kernel.extension.dependency.HighestSelectionStrategy) TransactionEventHandlers(org.neo4j.kernel.internal.TransactionEventHandlers) IdReuseEligibility(org.neo4j.kernel.impl.store.id.IdReuseEligibility) LogEntryStart(org.neo4j.kernel.impl.transaction.log.entry.LogEntryStart) StartupStatisticsProvider(org.neo4j.kernel.impl.core.StartupStatisticsProvider) StateHandlingStatementOperations(org.neo4j.kernel.impl.api.StateHandlingStatementOperations) PropertyAccessor(org.neo4j.kernel.api.index.PropertyAccessor) PositionToRecoverFrom(org.neo4j.kernel.recovery.PositionToRecoverFrom) SimpleTriggerInfo(org.neo4j.kernel.impl.transaction.log.checkpoint.SimpleTriggerInfo) LockSupport(java.util.concurrent.locks.LockSupport) SchemaStateConcern(org.neo4j.kernel.impl.api.SchemaStateConcern) UpdateableSchemaState(org.neo4j.kernel.impl.api.UpdateableSchemaState) Kernel(org.neo4j.kernel.impl.api.Kernel) TransactionCommitProcess(org.neo4j.kernel.impl.api.TransactionCommitProcess) Logger(org.neo4j.logging.Logger) LabelScanStoreProvider(org.neo4j.kernel.impl.api.scan.LabelScanStoreProvider) LogEntry(org.neo4j.kernel.impl.transaction.log.entry.LogEntry) GraphDatabaseSettings(org.neo4j.graphdb.factory.GraphDatabaseSettings) Tracers(org.neo4j.kernel.monitoring.tracing.Tracers) SynchronizedArrayIdOrderingQueue(org.neo4j.kernel.impl.util.SynchronizedArrayIdOrderingQueue) Guard(org.neo4j.kernel.guard.Guard) IndexingService(org.neo4j.kernel.impl.api.index.IndexingService) IndexImplementation(org.neo4j.kernel.spi.legacyindex.IndexImplementation) Exceptions(org.neo4j.helpers.Exceptions) ConstraintSemantics(org.neo4j.kernel.impl.constraints.ConstraintSemantics) RecordFormatSelector(org.neo4j.kernel.impl.store.format.RecordFormatSelector) TimeCheckPointThreshold(org.neo4j.kernel.impl.transaction.log.checkpoint.TimeCheckPointThreshold) LabelScanStore(org.neo4j.kernel.api.labelscan.LabelScanStore) JobScheduler(org.neo4j.kernel.impl.util.JobScheduler) Supplier(java.util.function.Supplier) ReadableClosablePositionAwareChannel(org.neo4j.kernel.impl.transaction.log.ReadableClosablePositionAwareChannel) LogPosition(org.neo4j.kernel.impl.transaction.log.LogPosition) PropertyKeyTokenHolder(org.neo4j.kernel.impl.core.PropertyKeyTokenHolder) CountCommittedTransactionThreshold(org.neo4j.kernel.impl.transaction.log.checkpoint.CountCommittedTransactionThreshold) GuardingStatementOperations(org.neo4j.kernel.impl.api.GuardingStatementOperations) DeleteStoresFromOtherLabelScanStoreProviders(org.neo4j.kernel.extension.dependency.DeleteStoresFromOtherLabelScanStoreProviders) Lifecycle(org.neo4j.kernel.lifecycle.Lifecycle) LogEntryReader(org.neo4j.kernel.impl.transaction.log.entry.LogEntryReader) TransactionMetadataCache(org.neo4j.kernel.impl.transaction.log.TransactionMetadataCache) IOException(java.io.IOException) LatestCheckPointFinder(org.neo4j.kernel.recovery.LatestCheckPointFinder) File(java.io.File) StackingQueryRegistrationOperations(org.neo4j.kernel.impl.api.StackingQueryRegistrationOperations) CheckPointScheduler(org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointScheduler) LockingStatementOperations(org.neo4j.kernel.impl.api.LockingStatementOperations) Procedures(org.neo4j.kernel.impl.proc.Procedures) IndexConfigStore(org.neo4j.kernel.impl.index.IndexConfigStore) KernelTransactionsSnapshot(org.neo4j.kernel.impl.api.KernelTransactionsSnapshot) LogPruneStrategyFactory.fromConfigValue(org.neo4j.kernel.impl.transaction.log.pruning.LogPruneStrategyFactory.fromConfigValue) StoreCopyCheckPointMutex(org.neo4j.kernel.impl.transaction.log.checkpoint.StoreCopyCheckPointMutex) ReentrantLockService(org.neo4j.kernel.impl.locking.ReentrantLockService) DatabaseMigrator(org.neo4j.kernel.impl.storemigration.DatabaseMigrator) LifecycleAdapter(org.neo4j.kernel.lifecycle.LifecycleAdapter) LogPruneStrategy(org.neo4j.kernel.impl.transaction.log.pruning.LogPruneStrategy) Log(org.neo4j.logging.Log) StoreFileMetadata(org.neo4j.storageengine.api.StoreFileMetadata) IdGeneratorFactory(org.neo4j.kernel.impl.store.id.IdGeneratorFactory) GraphDatabaseFacadeFactory(org.neo4j.kernel.impl.factory.GraphDatabaseFacadeFactory) CheckPointThreshold(org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointThreshold) IdTypeConfigurationProvider(org.neo4j.kernel.impl.store.id.configuration.IdTypeConfigurationProvider) BatchingTransactionAppender(org.neo4j.kernel.impl.transaction.log.BatchingTransactionAppender) Dependencies(org.neo4j.kernel.impl.util.Dependencies) LifeSupport(org.neo4j.kernel.lifecycle.LifeSupport) KernelAPI(org.neo4j.kernel.api.KernelAPI) PhysicalLogFileInformation(org.neo4j.kernel.impl.transaction.log.PhysicalLogFileInformation) DiagnosticsExtractor(org.neo4j.kernel.info.DiagnosticsExtractor) ConstraintEnforcingEntityOperations(org.neo4j.kernel.impl.api.ConstraintEnforcingEntityOperations) NeoStoreFileListing(org.neo4j.kernel.impl.transaction.state.NeoStoreFileListing) LogRotation(org.neo4j.kernel.impl.transaction.log.rotation.LogRotation) StatementOperationContainer(org.neo4j.kernel.impl.api.StatementOperationContainer) DataIntegrityValidatingStatementOperations(org.neo4j.kernel.impl.api.DataIntegrityValidatingStatementOperations) QueryRegistrationOperations(org.neo4j.kernel.impl.api.operations.QueryRegistrationOperations) CheckPointerImpl(org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointerImpl) ReadableLogChannel(org.neo4j.kernel.impl.transaction.log.ReadableLogChannel) StatementLocksFactory(org.neo4j.kernel.impl.locking.StatementLocksFactory) PageCache(org.neo4j.io.pagecache.PageCache) Recovery(org.neo4j.kernel.recovery.Recovery) StorageEngine(org.neo4j.storageengine.api.StorageEngine) LogVersionRepository(org.neo4j.kernel.impl.transaction.log.LogVersionRepository) LogService(org.neo4j.kernel.impl.logging.LogService) KernelException(org.neo4j.kernel.api.exceptions.KernelException) PhysicalLogFile(org.neo4j.kernel.impl.transaction.log.PhysicalLogFile) StoreReadLayer(org.neo4j.storageengine.api.StoreReadLayer) LoggingLogFileMonitor(org.neo4j.kernel.impl.transaction.log.LoggingLogFileMonitor) NamedLabelScanStoreSelectionStrategy(org.neo4j.kernel.extension.dependency.NamedLabelScanStoreSelectionStrategy) MetaDataStore(org.neo4j.kernel.impl.store.MetaDataStore) StatementOperationParts(org.neo4j.kernel.impl.api.StatementOperationParts) LegacyIndexProviderLookup(org.neo4j.kernel.impl.api.LegacyIndexProviderLookup) DependencyResolver(org.neo4j.graphdb.DependencyResolver) LogHeaderCache(org.neo4j.kernel.impl.transaction.log.LogHeaderCache) KernelTransactions(org.neo4j.kernel.impl.api.KernelTransactions) TransactionHeaderInformationFactory(org.neo4j.kernel.impl.transaction.TransactionHeaderInformationFactory) PhysicalLogicalTransactionStore(org.neo4j.kernel.impl.transaction.log.PhysicalLogicalTransactionStore) TransactionAppender(org.neo4j.kernel.impl.transaction.log.TransactionAppender) TransactionIdStore(org.neo4j.kernel.impl.transaction.log.TransactionIdStore) LogicalTransactionStore(org.neo4j.kernel.impl.transaction.log.LogicalTransactionStore) Monitors(org.neo4j.kernel.monitoring.Monitors) LogProvider(org.neo4j.logging.LogProvider) HashMap(java.util.HashMap) DiagnosticsManager(org.neo4j.kernel.info.DiagnosticsManager) IOLimiter(org.neo4j.io.pagecache.IOLimiter) RecordFormats(org.neo4j.kernel.impl.store.format.RecordFormats) ConstraintIndexCreator(org.neo4j.kernel.impl.api.state.ConstraintIndexCreator) StoreMigrator(org.neo4j.kernel.impl.storemigration.participant.StoreMigrator) LogFileInformation(org.neo4j.kernel.impl.transaction.log.LogFileInformation) LogRotationImpl(org.neo4j.kernel.impl.transaction.log.rotation.LogRotationImpl) DiagnosticsPhase(org.neo4j.kernel.info.DiagnosticsPhase) LogPruningImpl(org.neo4j.kernel.impl.transaction.log.pruning.LogPruningImpl) LogPruning(org.neo4j.kernel.impl.transaction.log.pruning.LogPruning) MapUtil.stringMap(org.neo4j.helpers.collection.MapUtil.stringMap) PhysicalLogFiles(org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles) AutoIndexing(org.neo4j.kernel.api.legacyindex.AutoIndexing) Config(org.neo4j.kernel.configuration.Config) LockService(org.neo4j.kernel.impl.locking.LockService) Setting(org.neo4j.graphdb.config.Setting) LabelTokenHolder(org.neo4j.kernel.impl.core.LabelTokenHolder) VisibleMigrationProgressMonitor(org.neo4j.kernel.impl.storemigration.monitoring.VisibleMigrationProgressMonitor) TimeUnit(java.util.concurrent.TimeUnit) DatabaseHealth(org.neo4j.kernel.internal.DatabaseHealth) CommitProcessFactory(org.neo4j.kernel.impl.api.CommitProcessFactory) Clock(java.time.Clock) FileSystemAbstraction(org.neo4j.io.fs.FileSystemAbstraction) TransactionHooks(org.neo4j.kernel.impl.api.TransactionHooks) LogPruneStrategy(org.neo4j.kernel.impl.transaction.log.pruning.LogPruneStrategy) CheckPointerImpl(org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointerImpl) TimeCheckPointThreshold(org.neo4j.kernel.impl.transaction.log.checkpoint.TimeCheckPointThreshold) BatchingTransactionAppender(org.neo4j.kernel.impl.transaction.log.BatchingTransactionAppender) PhysicalLogicalTransactionStore(org.neo4j.kernel.impl.transaction.log.PhysicalLogicalTransactionStore) LogicalTransactionStore(org.neo4j.kernel.impl.transaction.log.LogicalTransactionStore) PhysicalLogFiles(org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles) LogPruning(org.neo4j.kernel.impl.transaction.log.pruning.LogPruning) TransactionMonitor(org.neo4j.kernel.impl.transaction.TransactionMonitor) LoggingLogFileMonitor(org.neo4j.kernel.impl.transaction.log.LoggingLogFileMonitor) VisibleMigrationProgressMonitor(org.neo4j.kernel.impl.storemigration.monitoring.VisibleMigrationProgressMonitor) TimeCheckPointThreshold(org.neo4j.kernel.impl.transaction.log.checkpoint.TimeCheckPointThreshold) CheckPointThreshold(org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointThreshold) LogEntry(org.neo4j.kernel.impl.transaction.log.entry.LogEntry) LogRotationImpl(org.neo4j.kernel.impl.transaction.log.rotation.LogRotationImpl) LogPosition(org.neo4j.kernel.impl.transaction.log.LogPosition) LogEntryStart(org.neo4j.kernel.impl.transaction.log.entry.LogEntryStart) PhysicalLogicalTransactionStore(org.neo4j.kernel.impl.transaction.log.PhysicalLogicalTransactionStore) ReadableLogChannel(org.neo4j.kernel.impl.transaction.log.ReadableLogChannel) CountCommittedTransactionThreshold(org.neo4j.kernel.impl.transaction.log.checkpoint.CountCommittedTransactionThreshold) LogPruningImpl(org.neo4j.kernel.impl.transaction.log.pruning.LogPruningImpl) BatchingTransactionAppender(org.neo4j.kernel.impl.transaction.log.BatchingTransactionAppender) TransactionAppender(org.neo4j.kernel.impl.transaction.log.TransactionAppender) TransactionMetadataCache(org.neo4j.kernel.impl.transaction.log.TransactionMetadataCache) CheckPointScheduler(org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointScheduler) PhysicalLogFileInformation(org.neo4j.kernel.impl.transaction.log.PhysicalLogFileInformation) LogFileInformation(org.neo4j.kernel.impl.transaction.log.LogFileInformation) LogRotation(org.neo4j.kernel.impl.transaction.log.rotation.LogRotation) LogHeaderCache(org.neo4j.kernel.impl.transaction.log.LogHeaderCache) PhysicalLogFile(org.neo4j.kernel.impl.transaction.log.PhysicalLogFile) PhysicalLogFileInformation(org.neo4j.kernel.impl.transaction.log.PhysicalLogFileInformation)

Aggregations

File (java.io.File)1 IOException (java.io.IOException)1 Clock (java.time.Clock)1 HashMap (java.util.HashMap)1 Map (java.util.Map)1 TimeUnit (java.util.concurrent.TimeUnit)1 LockSupport (java.util.concurrent.locks.LockSupport)1 Supplier (java.util.function.Supplier)1 DependencyResolver (org.neo4j.graphdb.DependencyResolver)1 ResourceIterator (org.neo4j.graphdb.ResourceIterator)1 Setting (org.neo4j.graphdb.config.Setting)1 GraphDatabaseSettings (org.neo4j.graphdb.factory.GraphDatabaseSettings)1 Exceptions (org.neo4j.helpers.Exceptions)1 MapUtil.stringMap (org.neo4j.helpers.collection.MapUtil.stringMap)1 FileSystemAbstraction (org.neo4j.io.fs.FileSystemAbstraction)1 IOLimiter (org.neo4j.io.pagecache.IOLimiter)1 PageCache (org.neo4j.io.pagecache.PageCache)1 KernelAPI (org.neo4j.kernel.api.KernelAPI)1 TokenNameLookup (org.neo4j.kernel.api.TokenNameLookup)1 KernelException (org.neo4j.kernel.api.exceptions.KernelException)1