Search in sources :

Example 1 with LogFileInformation

use of org.neo4j.kernel.impl.transaction.log.LogFileInformation in project neo4j by neo4j.

the class NeoStoreDataSource method buildTransactionLogs.

private NeoStoreTransactionLogModule buildTransactionLogs(File storeDir, Config config, LogProvider logProvider, JobScheduler scheduler, FileSystemAbstraction fileSystemAbstraction, StorageEngine storageEngine, LogEntryReader<ReadableClosablePositionAwareChannel> logEntryReader, SynchronizedArrayIdOrderingQueue legacyIndexTransactionOrdering, TransactionIdStore transactionIdStore, LogVersionRepository logVersionRepository) {
    TransactionMetadataCache transactionMetadataCache = new TransactionMetadataCache(100_000);
    LogHeaderCache logHeaderCache = new LogHeaderCache(1000);
    final PhysicalLogFiles logFiles = new PhysicalLogFiles(storeDir, PhysicalLogFile.DEFAULT_NAME, fileSystemAbstraction);
    final PhysicalLogFile logFile = life.add(new PhysicalLogFile(fileSystemAbstraction, logFiles, config.get(GraphDatabaseSettings.logical_log_rotation_threshold), transactionIdStore::getLastCommittedTransactionId, logVersionRepository, physicalLogMonitor, logHeaderCache));
    final PhysicalLogFileInformation.LogVersionToTimestamp logInformation = version -> {
        LogPosition position = LogPosition.start(version);
        try (ReadableLogChannel channel = logFile.getReader(position)) {
            LogEntry entry;
            while ((entry = logEntryReader.readLogEntry(channel)) != null) {
                if (entry instanceof LogEntryStart) {
                    return entry.<LogEntryStart>as().getTimeWritten();
                }
            }
        }
        return -1;
    };
    final LogFileInformation logFileInformation = new PhysicalLogFileInformation(logFiles, logHeaderCache, transactionIdStore::getLastCommittedTransactionId, logInformation);
    if (config.get(GraphDatabaseFacadeFactory.Configuration.ephemeral)) {
        config = config.withDefaults(stringMap(GraphDatabaseSettings.keep_logical_logs.name(), "1 files"));
    }
    String pruningConf = config.get(GraphDatabaseSettings.keep_logical_logs);
    LogPruneStrategy logPruneStrategy = fromConfigValue(fs, logFileInformation, logFiles, pruningConf);
    final LogPruning logPruning = new LogPruningImpl(logPruneStrategy, logProvider);
    final LogRotation logRotation = new LogRotationImpl(monitors.newMonitor(LogRotation.Monitor.class), logFile, databaseHealth);
    final TransactionAppender appender = life.add(new BatchingTransactionAppender(logFile, logRotation, transactionMetadataCache, transactionIdStore, legacyIndexTransactionOrdering, databaseHealth));
    final LogicalTransactionStore logicalTransactionStore = new PhysicalLogicalTransactionStore(logFile, transactionMetadataCache, logEntryReader);
    int txThreshold = config.get(GraphDatabaseSettings.check_point_interval_tx);
    final CountCommittedTransactionThreshold countCommittedTransactionThreshold = new CountCommittedTransactionThreshold(txThreshold);
    long timeMillisThreshold = config.get(GraphDatabaseSettings.check_point_interval_time);
    TimeCheckPointThreshold timeCheckPointThreshold = new TimeCheckPointThreshold(timeMillisThreshold, clock);
    CheckPointThreshold threshold = CheckPointThresholds.or(countCommittedTransactionThreshold, timeCheckPointThreshold);
    final CheckPointerImpl checkPointer = new CheckPointerImpl(transactionIdStore, threshold, storageEngine, logPruning, appender, databaseHealth, logProvider, tracers.checkPointTracer, ioLimiter, storeCopyCheckPointMutex);
    long recurringPeriod = Math.min(timeMillisThreshold, TimeUnit.SECONDS.toMillis(10));
    CheckPointScheduler checkPointScheduler = new CheckPointScheduler(checkPointer, scheduler, recurringPeriod, databaseHealth);
    life.add(checkPointer);
    life.add(checkPointScheduler);
    return new NeoStoreTransactionLogModule(logicalTransactionStore, logFileInformation, logFiles, logFile, logRotation, checkPointer, appender, legacyIndexTransactionOrdering);
}
Also used : LegacyIndexStore(org.neo4j.kernel.impl.index.LegacyIndexStore) StoreId(org.neo4j.kernel.impl.store.StoreId) ResourceIterator(org.neo4j.graphdb.ResourceIterator) Lifecycles(org.neo4j.kernel.lifecycle.Lifecycles) RelationshipTypeTokenHolder(org.neo4j.kernel.impl.core.RelationshipTypeTokenHolder) SystemNanoClock(org.neo4j.time.SystemNanoClock) AccessCapability(org.neo4j.kernel.impl.factory.AccessCapability) SchemaWriteGuard(org.neo4j.kernel.impl.api.SchemaWriteGuard) CheckPointThresholds(org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointThresholds) VersionAwareLogEntryReader(org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader) RecordStorageEngine(org.neo4j.kernel.impl.storageengine.impl.recordstorage.RecordStorageEngine) LogHeader(org.neo4j.kernel.impl.transaction.log.entry.LogHeader) SchemaIndexProvider(org.neo4j.kernel.api.index.SchemaIndexProvider) Map(java.util.Map) IndexProviders(org.neo4j.kernel.spi.legacyindex.IndexProviders) TokenNameLookup(org.neo4j.kernel.api.TokenNameLookup) KernelSchemaStateStore(org.neo4j.kernel.impl.api.KernelSchemaStateStore) RecordFormatPropertyConfigurator(org.neo4j.kernel.impl.store.format.RecordFormatPropertyConfigurator) TransactionMonitor(org.neo4j.kernel.impl.transaction.TransactionMonitor) DefaultRecoverySPI(org.neo4j.kernel.recovery.DefaultRecoverySPI) HighestSelectionStrategy(org.neo4j.kernel.extension.dependency.HighestSelectionStrategy) TransactionEventHandlers(org.neo4j.kernel.internal.TransactionEventHandlers) IdReuseEligibility(org.neo4j.kernel.impl.store.id.IdReuseEligibility) LogEntryStart(org.neo4j.kernel.impl.transaction.log.entry.LogEntryStart) StartupStatisticsProvider(org.neo4j.kernel.impl.core.StartupStatisticsProvider) StateHandlingStatementOperations(org.neo4j.kernel.impl.api.StateHandlingStatementOperations) PropertyAccessor(org.neo4j.kernel.api.index.PropertyAccessor) PositionToRecoverFrom(org.neo4j.kernel.recovery.PositionToRecoverFrom) SimpleTriggerInfo(org.neo4j.kernel.impl.transaction.log.checkpoint.SimpleTriggerInfo) LockSupport(java.util.concurrent.locks.LockSupport) SchemaStateConcern(org.neo4j.kernel.impl.api.SchemaStateConcern) UpdateableSchemaState(org.neo4j.kernel.impl.api.UpdateableSchemaState) Kernel(org.neo4j.kernel.impl.api.Kernel) TransactionCommitProcess(org.neo4j.kernel.impl.api.TransactionCommitProcess) Logger(org.neo4j.logging.Logger) LabelScanStoreProvider(org.neo4j.kernel.impl.api.scan.LabelScanStoreProvider) LogEntry(org.neo4j.kernel.impl.transaction.log.entry.LogEntry) GraphDatabaseSettings(org.neo4j.graphdb.factory.GraphDatabaseSettings) Tracers(org.neo4j.kernel.monitoring.tracing.Tracers) SynchronizedArrayIdOrderingQueue(org.neo4j.kernel.impl.util.SynchronizedArrayIdOrderingQueue) Guard(org.neo4j.kernel.guard.Guard) IndexingService(org.neo4j.kernel.impl.api.index.IndexingService) IndexImplementation(org.neo4j.kernel.spi.legacyindex.IndexImplementation) Exceptions(org.neo4j.helpers.Exceptions) ConstraintSemantics(org.neo4j.kernel.impl.constraints.ConstraintSemantics) RecordFormatSelector(org.neo4j.kernel.impl.store.format.RecordFormatSelector) TimeCheckPointThreshold(org.neo4j.kernel.impl.transaction.log.checkpoint.TimeCheckPointThreshold) LabelScanStore(org.neo4j.kernel.api.labelscan.LabelScanStore) JobScheduler(org.neo4j.kernel.impl.util.JobScheduler) Supplier(java.util.function.Supplier) ReadableClosablePositionAwareChannel(org.neo4j.kernel.impl.transaction.log.ReadableClosablePositionAwareChannel) LogPosition(org.neo4j.kernel.impl.transaction.log.LogPosition) PropertyKeyTokenHolder(org.neo4j.kernel.impl.core.PropertyKeyTokenHolder) CountCommittedTransactionThreshold(org.neo4j.kernel.impl.transaction.log.checkpoint.CountCommittedTransactionThreshold) GuardingStatementOperations(org.neo4j.kernel.impl.api.GuardingStatementOperations) DeleteStoresFromOtherLabelScanStoreProviders(org.neo4j.kernel.extension.dependency.DeleteStoresFromOtherLabelScanStoreProviders) Lifecycle(org.neo4j.kernel.lifecycle.Lifecycle) LogEntryReader(org.neo4j.kernel.impl.transaction.log.entry.LogEntryReader) TransactionMetadataCache(org.neo4j.kernel.impl.transaction.log.TransactionMetadataCache) IOException(java.io.IOException) LatestCheckPointFinder(org.neo4j.kernel.recovery.LatestCheckPointFinder) File(java.io.File) StackingQueryRegistrationOperations(org.neo4j.kernel.impl.api.StackingQueryRegistrationOperations) CheckPointScheduler(org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointScheduler) LockingStatementOperations(org.neo4j.kernel.impl.api.LockingStatementOperations) Procedures(org.neo4j.kernel.impl.proc.Procedures) IndexConfigStore(org.neo4j.kernel.impl.index.IndexConfigStore) KernelTransactionsSnapshot(org.neo4j.kernel.impl.api.KernelTransactionsSnapshot) LogPruneStrategyFactory.fromConfigValue(org.neo4j.kernel.impl.transaction.log.pruning.LogPruneStrategyFactory.fromConfigValue) StoreCopyCheckPointMutex(org.neo4j.kernel.impl.transaction.log.checkpoint.StoreCopyCheckPointMutex) ReentrantLockService(org.neo4j.kernel.impl.locking.ReentrantLockService) DatabaseMigrator(org.neo4j.kernel.impl.storemigration.DatabaseMigrator) LifecycleAdapter(org.neo4j.kernel.lifecycle.LifecycleAdapter) LogPruneStrategy(org.neo4j.kernel.impl.transaction.log.pruning.LogPruneStrategy) Log(org.neo4j.logging.Log) StoreFileMetadata(org.neo4j.storageengine.api.StoreFileMetadata) IdGeneratorFactory(org.neo4j.kernel.impl.store.id.IdGeneratorFactory) GraphDatabaseFacadeFactory(org.neo4j.kernel.impl.factory.GraphDatabaseFacadeFactory) CheckPointThreshold(org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointThreshold) IdTypeConfigurationProvider(org.neo4j.kernel.impl.store.id.configuration.IdTypeConfigurationProvider) BatchingTransactionAppender(org.neo4j.kernel.impl.transaction.log.BatchingTransactionAppender) Dependencies(org.neo4j.kernel.impl.util.Dependencies) LifeSupport(org.neo4j.kernel.lifecycle.LifeSupport) KernelAPI(org.neo4j.kernel.api.KernelAPI) PhysicalLogFileInformation(org.neo4j.kernel.impl.transaction.log.PhysicalLogFileInformation) DiagnosticsExtractor(org.neo4j.kernel.info.DiagnosticsExtractor) ConstraintEnforcingEntityOperations(org.neo4j.kernel.impl.api.ConstraintEnforcingEntityOperations) NeoStoreFileListing(org.neo4j.kernel.impl.transaction.state.NeoStoreFileListing) LogRotation(org.neo4j.kernel.impl.transaction.log.rotation.LogRotation) StatementOperationContainer(org.neo4j.kernel.impl.api.StatementOperationContainer) DataIntegrityValidatingStatementOperations(org.neo4j.kernel.impl.api.DataIntegrityValidatingStatementOperations) QueryRegistrationOperations(org.neo4j.kernel.impl.api.operations.QueryRegistrationOperations) CheckPointerImpl(org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointerImpl) ReadableLogChannel(org.neo4j.kernel.impl.transaction.log.ReadableLogChannel) StatementLocksFactory(org.neo4j.kernel.impl.locking.StatementLocksFactory) PageCache(org.neo4j.io.pagecache.PageCache) Recovery(org.neo4j.kernel.recovery.Recovery) StorageEngine(org.neo4j.storageengine.api.StorageEngine) LogVersionRepository(org.neo4j.kernel.impl.transaction.log.LogVersionRepository) LogService(org.neo4j.kernel.impl.logging.LogService) KernelException(org.neo4j.kernel.api.exceptions.KernelException) PhysicalLogFile(org.neo4j.kernel.impl.transaction.log.PhysicalLogFile) StoreReadLayer(org.neo4j.storageengine.api.StoreReadLayer) LoggingLogFileMonitor(org.neo4j.kernel.impl.transaction.log.LoggingLogFileMonitor) NamedLabelScanStoreSelectionStrategy(org.neo4j.kernel.extension.dependency.NamedLabelScanStoreSelectionStrategy) MetaDataStore(org.neo4j.kernel.impl.store.MetaDataStore) StatementOperationParts(org.neo4j.kernel.impl.api.StatementOperationParts) LegacyIndexProviderLookup(org.neo4j.kernel.impl.api.LegacyIndexProviderLookup) DependencyResolver(org.neo4j.graphdb.DependencyResolver) LogHeaderCache(org.neo4j.kernel.impl.transaction.log.LogHeaderCache) KernelTransactions(org.neo4j.kernel.impl.api.KernelTransactions) TransactionHeaderInformationFactory(org.neo4j.kernel.impl.transaction.TransactionHeaderInformationFactory) PhysicalLogicalTransactionStore(org.neo4j.kernel.impl.transaction.log.PhysicalLogicalTransactionStore) TransactionAppender(org.neo4j.kernel.impl.transaction.log.TransactionAppender) TransactionIdStore(org.neo4j.kernel.impl.transaction.log.TransactionIdStore) LogicalTransactionStore(org.neo4j.kernel.impl.transaction.log.LogicalTransactionStore) Monitors(org.neo4j.kernel.monitoring.Monitors) LogProvider(org.neo4j.logging.LogProvider) HashMap(java.util.HashMap) DiagnosticsManager(org.neo4j.kernel.info.DiagnosticsManager) IOLimiter(org.neo4j.io.pagecache.IOLimiter) RecordFormats(org.neo4j.kernel.impl.store.format.RecordFormats) ConstraintIndexCreator(org.neo4j.kernel.impl.api.state.ConstraintIndexCreator) StoreMigrator(org.neo4j.kernel.impl.storemigration.participant.StoreMigrator) LogFileInformation(org.neo4j.kernel.impl.transaction.log.LogFileInformation) LogRotationImpl(org.neo4j.kernel.impl.transaction.log.rotation.LogRotationImpl) DiagnosticsPhase(org.neo4j.kernel.info.DiagnosticsPhase) LogPruningImpl(org.neo4j.kernel.impl.transaction.log.pruning.LogPruningImpl) LogPruning(org.neo4j.kernel.impl.transaction.log.pruning.LogPruning) MapUtil.stringMap(org.neo4j.helpers.collection.MapUtil.stringMap) PhysicalLogFiles(org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles) AutoIndexing(org.neo4j.kernel.api.legacyindex.AutoIndexing) Config(org.neo4j.kernel.configuration.Config) LockService(org.neo4j.kernel.impl.locking.LockService) Setting(org.neo4j.graphdb.config.Setting) LabelTokenHolder(org.neo4j.kernel.impl.core.LabelTokenHolder) VisibleMigrationProgressMonitor(org.neo4j.kernel.impl.storemigration.monitoring.VisibleMigrationProgressMonitor) TimeUnit(java.util.concurrent.TimeUnit) DatabaseHealth(org.neo4j.kernel.internal.DatabaseHealth) CommitProcessFactory(org.neo4j.kernel.impl.api.CommitProcessFactory) Clock(java.time.Clock) FileSystemAbstraction(org.neo4j.io.fs.FileSystemAbstraction) TransactionHooks(org.neo4j.kernel.impl.api.TransactionHooks) LogPruneStrategy(org.neo4j.kernel.impl.transaction.log.pruning.LogPruneStrategy) CheckPointerImpl(org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointerImpl) TimeCheckPointThreshold(org.neo4j.kernel.impl.transaction.log.checkpoint.TimeCheckPointThreshold) BatchingTransactionAppender(org.neo4j.kernel.impl.transaction.log.BatchingTransactionAppender) PhysicalLogicalTransactionStore(org.neo4j.kernel.impl.transaction.log.PhysicalLogicalTransactionStore) LogicalTransactionStore(org.neo4j.kernel.impl.transaction.log.LogicalTransactionStore) PhysicalLogFiles(org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles) LogPruning(org.neo4j.kernel.impl.transaction.log.pruning.LogPruning) TransactionMonitor(org.neo4j.kernel.impl.transaction.TransactionMonitor) LoggingLogFileMonitor(org.neo4j.kernel.impl.transaction.log.LoggingLogFileMonitor) VisibleMigrationProgressMonitor(org.neo4j.kernel.impl.storemigration.monitoring.VisibleMigrationProgressMonitor) TimeCheckPointThreshold(org.neo4j.kernel.impl.transaction.log.checkpoint.TimeCheckPointThreshold) CheckPointThreshold(org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointThreshold) LogEntry(org.neo4j.kernel.impl.transaction.log.entry.LogEntry) LogRotationImpl(org.neo4j.kernel.impl.transaction.log.rotation.LogRotationImpl) LogPosition(org.neo4j.kernel.impl.transaction.log.LogPosition) LogEntryStart(org.neo4j.kernel.impl.transaction.log.entry.LogEntryStart) PhysicalLogicalTransactionStore(org.neo4j.kernel.impl.transaction.log.PhysicalLogicalTransactionStore) ReadableLogChannel(org.neo4j.kernel.impl.transaction.log.ReadableLogChannel) CountCommittedTransactionThreshold(org.neo4j.kernel.impl.transaction.log.checkpoint.CountCommittedTransactionThreshold) LogPruningImpl(org.neo4j.kernel.impl.transaction.log.pruning.LogPruningImpl) BatchingTransactionAppender(org.neo4j.kernel.impl.transaction.log.BatchingTransactionAppender) TransactionAppender(org.neo4j.kernel.impl.transaction.log.TransactionAppender) TransactionMetadataCache(org.neo4j.kernel.impl.transaction.log.TransactionMetadataCache) CheckPointScheduler(org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointScheduler) PhysicalLogFileInformation(org.neo4j.kernel.impl.transaction.log.PhysicalLogFileInformation) LogFileInformation(org.neo4j.kernel.impl.transaction.log.LogFileInformation) LogRotation(org.neo4j.kernel.impl.transaction.log.rotation.LogRotation) LogHeaderCache(org.neo4j.kernel.impl.transaction.log.LogHeaderCache) PhysicalLogFile(org.neo4j.kernel.impl.transaction.log.PhysicalLogFile) PhysicalLogFileInformation(org.neo4j.kernel.impl.transaction.log.PhysicalLogFileInformation)

Aggregations

File (java.io.File)1 IOException (java.io.IOException)1 Clock (java.time.Clock)1 HashMap (java.util.HashMap)1 Map (java.util.Map)1 TimeUnit (java.util.concurrent.TimeUnit)1 LockSupport (java.util.concurrent.locks.LockSupport)1 Supplier (java.util.function.Supplier)1 DependencyResolver (org.neo4j.graphdb.DependencyResolver)1 ResourceIterator (org.neo4j.graphdb.ResourceIterator)1 Setting (org.neo4j.graphdb.config.Setting)1 GraphDatabaseSettings (org.neo4j.graphdb.factory.GraphDatabaseSettings)1 Exceptions (org.neo4j.helpers.Exceptions)1 MapUtil.stringMap (org.neo4j.helpers.collection.MapUtil.stringMap)1 FileSystemAbstraction (org.neo4j.io.fs.FileSystemAbstraction)1 IOLimiter (org.neo4j.io.pagecache.IOLimiter)1 PageCache (org.neo4j.io.pagecache.PageCache)1 KernelAPI (org.neo4j.kernel.api.KernelAPI)1 TokenNameLookup (org.neo4j.kernel.api.TokenNameLookup)1 KernelException (org.neo4j.kernel.api.exceptions.KernelException)1