Search in sources :

Example 1 with LogRotation

use of org.neo4j.kernel.impl.transaction.log.rotation.LogRotation in project neo4j by neo4j.

the class NeoStoreDataSource method buildTransactionLogs.

private NeoStoreTransactionLogModule buildTransactionLogs(File storeDir, Config config, LogProvider logProvider, JobScheduler scheduler, FileSystemAbstraction fileSystemAbstraction, StorageEngine storageEngine, LogEntryReader<ReadableClosablePositionAwareChannel> logEntryReader, SynchronizedArrayIdOrderingQueue legacyIndexTransactionOrdering, TransactionIdStore transactionIdStore, LogVersionRepository logVersionRepository) {
    TransactionMetadataCache transactionMetadataCache = new TransactionMetadataCache(100_000);
    LogHeaderCache logHeaderCache = new LogHeaderCache(1000);
    final PhysicalLogFiles logFiles = new PhysicalLogFiles(storeDir, PhysicalLogFile.DEFAULT_NAME, fileSystemAbstraction);
    final PhysicalLogFile logFile = life.add(new PhysicalLogFile(fileSystemAbstraction, logFiles, config.get(GraphDatabaseSettings.logical_log_rotation_threshold), transactionIdStore::getLastCommittedTransactionId, logVersionRepository, physicalLogMonitor, logHeaderCache));
    final PhysicalLogFileInformation.LogVersionToTimestamp logInformation = version -> {
        LogPosition position = LogPosition.start(version);
        try (ReadableLogChannel channel = logFile.getReader(position)) {
            LogEntry entry;
            while ((entry = logEntryReader.readLogEntry(channel)) != null) {
                if (entry instanceof LogEntryStart) {
                    return entry.<LogEntryStart>as().getTimeWritten();
                }
            }
        }
        return -1;
    };
    final LogFileInformation logFileInformation = new PhysicalLogFileInformation(logFiles, logHeaderCache, transactionIdStore::getLastCommittedTransactionId, logInformation);
    if (config.get(GraphDatabaseFacadeFactory.Configuration.ephemeral)) {
        config = config.withDefaults(stringMap(GraphDatabaseSettings.keep_logical_logs.name(), "1 files"));
    }
    String pruningConf = config.get(GraphDatabaseSettings.keep_logical_logs);
    LogPruneStrategy logPruneStrategy = fromConfigValue(fs, logFileInformation, logFiles, pruningConf);
    final LogPruning logPruning = new LogPruningImpl(logPruneStrategy, logProvider);
    final LogRotation logRotation = new LogRotationImpl(monitors.newMonitor(LogRotation.Monitor.class), logFile, databaseHealth);
    final TransactionAppender appender = life.add(new BatchingTransactionAppender(logFile, logRotation, transactionMetadataCache, transactionIdStore, legacyIndexTransactionOrdering, databaseHealth));
    final LogicalTransactionStore logicalTransactionStore = new PhysicalLogicalTransactionStore(logFile, transactionMetadataCache, logEntryReader);
    int txThreshold = config.get(GraphDatabaseSettings.check_point_interval_tx);
    final CountCommittedTransactionThreshold countCommittedTransactionThreshold = new CountCommittedTransactionThreshold(txThreshold);
    long timeMillisThreshold = config.get(GraphDatabaseSettings.check_point_interval_time);
    TimeCheckPointThreshold timeCheckPointThreshold = new TimeCheckPointThreshold(timeMillisThreshold, clock);
    CheckPointThreshold threshold = CheckPointThresholds.or(countCommittedTransactionThreshold, timeCheckPointThreshold);
    final CheckPointerImpl checkPointer = new CheckPointerImpl(transactionIdStore, threshold, storageEngine, logPruning, appender, databaseHealth, logProvider, tracers.checkPointTracer, ioLimiter, storeCopyCheckPointMutex);
    long recurringPeriod = Math.min(timeMillisThreshold, TimeUnit.SECONDS.toMillis(10));
    CheckPointScheduler checkPointScheduler = new CheckPointScheduler(checkPointer, scheduler, recurringPeriod, databaseHealth);
    life.add(checkPointer);
    life.add(checkPointScheduler);
    return new NeoStoreTransactionLogModule(logicalTransactionStore, logFileInformation, logFiles, logFile, logRotation, checkPointer, appender, legacyIndexTransactionOrdering);
}
Also used : LegacyIndexStore(org.neo4j.kernel.impl.index.LegacyIndexStore) StoreId(org.neo4j.kernel.impl.store.StoreId) ResourceIterator(org.neo4j.graphdb.ResourceIterator) Lifecycles(org.neo4j.kernel.lifecycle.Lifecycles) RelationshipTypeTokenHolder(org.neo4j.kernel.impl.core.RelationshipTypeTokenHolder) SystemNanoClock(org.neo4j.time.SystemNanoClock) AccessCapability(org.neo4j.kernel.impl.factory.AccessCapability) SchemaWriteGuard(org.neo4j.kernel.impl.api.SchemaWriteGuard) CheckPointThresholds(org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointThresholds) VersionAwareLogEntryReader(org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader) RecordStorageEngine(org.neo4j.kernel.impl.storageengine.impl.recordstorage.RecordStorageEngine) LogHeader(org.neo4j.kernel.impl.transaction.log.entry.LogHeader) SchemaIndexProvider(org.neo4j.kernel.api.index.SchemaIndexProvider) Map(java.util.Map) IndexProviders(org.neo4j.kernel.spi.legacyindex.IndexProviders) TokenNameLookup(org.neo4j.kernel.api.TokenNameLookup) KernelSchemaStateStore(org.neo4j.kernel.impl.api.KernelSchemaStateStore) RecordFormatPropertyConfigurator(org.neo4j.kernel.impl.store.format.RecordFormatPropertyConfigurator) TransactionMonitor(org.neo4j.kernel.impl.transaction.TransactionMonitor) DefaultRecoverySPI(org.neo4j.kernel.recovery.DefaultRecoverySPI) HighestSelectionStrategy(org.neo4j.kernel.extension.dependency.HighestSelectionStrategy) TransactionEventHandlers(org.neo4j.kernel.internal.TransactionEventHandlers) IdReuseEligibility(org.neo4j.kernel.impl.store.id.IdReuseEligibility) LogEntryStart(org.neo4j.kernel.impl.transaction.log.entry.LogEntryStart) StartupStatisticsProvider(org.neo4j.kernel.impl.core.StartupStatisticsProvider) StateHandlingStatementOperations(org.neo4j.kernel.impl.api.StateHandlingStatementOperations) PropertyAccessor(org.neo4j.kernel.api.index.PropertyAccessor) PositionToRecoverFrom(org.neo4j.kernel.recovery.PositionToRecoverFrom) SimpleTriggerInfo(org.neo4j.kernel.impl.transaction.log.checkpoint.SimpleTriggerInfo) LockSupport(java.util.concurrent.locks.LockSupport) SchemaStateConcern(org.neo4j.kernel.impl.api.SchemaStateConcern) UpdateableSchemaState(org.neo4j.kernel.impl.api.UpdateableSchemaState) Kernel(org.neo4j.kernel.impl.api.Kernel) TransactionCommitProcess(org.neo4j.kernel.impl.api.TransactionCommitProcess) Logger(org.neo4j.logging.Logger) LabelScanStoreProvider(org.neo4j.kernel.impl.api.scan.LabelScanStoreProvider) LogEntry(org.neo4j.kernel.impl.transaction.log.entry.LogEntry) GraphDatabaseSettings(org.neo4j.graphdb.factory.GraphDatabaseSettings) Tracers(org.neo4j.kernel.monitoring.tracing.Tracers) SynchronizedArrayIdOrderingQueue(org.neo4j.kernel.impl.util.SynchronizedArrayIdOrderingQueue) Guard(org.neo4j.kernel.guard.Guard) IndexingService(org.neo4j.kernel.impl.api.index.IndexingService) IndexImplementation(org.neo4j.kernel.spi.legacyindex.IndexImplementation) Exceptions(org.neo4j.helpers.Exceptions) ConstraintSemantics(org.neo4j.kernel.impl.constraints.ConstraintSemantics) RecordFormatSelector(org.neo4j.kernel.impl.store.format.RecordFormatSelector) TimeCheckPointThreshold(org.neo4j.kernel.impl.transaction.log.checkpoint.TimeCheckPointThreshold) LabelScanStore(org.neo4j.kernel.api.labelscan.LabelScanStore) JobScheduler(org.neo4j.kernel.impl.util.JobScheduler) Supplier(java.util.function.Supplier) ReadableClosablePositionAwareChannel(org.neo4j.kernel.impl.transaction.log.ReadableClosablePositionAwareChannel) LogPosition(org.neo4j.kernel.impl.transaction.log.LogPosition) PropertyKeyTokenHolder(org.neo4j.kernel.impl.core.PropertyKeyTokenHolder) CountCommittedTransactionThreshold(org.neo4j.kernel.impl.transaction.log.checkpoint.CountCommittedTransactionThreshold) GuardingStatementOperations(org.neo4j.kernel.impl.api.GuardingStatementOperations) DeleteStoresFromOtherLabelScanStoreProviders(org.neo4j.kernel.extension.dependency.DeleteStoresFromOtherLabelScanStoreProviders) Lifecycle(org.neo4j.kernel.lifecycle.Lifecycle) LogEntryReader(org.neo4j.kernel.impl.transaction.log.entry.LogEntryReader) TransactionMetadataCache(org.neo4j.kernel.impl.transaction.log.TransactionMetadataCache) IOException(java.io.IOException) LatestCheckPointFinder(org.neo4j.kernel.recovery.LatestCheckPointFinder) File(java.io.File) StackingQueryRegistrationOperations(org.neo4j.kernel.impl.api.StackingQueryRegistrationOperations) CheckPointScheduler(org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointScheduler) LockingStatementOperations(org.neo4j.kernel.impl.api.LockingStatementOperations) Procedures(org.neo4j.kernel.impl.proc.Procedures) IndexConfigStore(org.neo4j.kernel.impl.index.IndexConfigStore) KernelTransactionsSnapshot(org.neo4j.kernel.impl.api.KernelTransactionsSnapshot) LogPruneStrategyFactory.fromConfigValue(org.neo4j.kernel.impl.transaction.log.pruning.LogPruneStrategyFactory.fromConfigValue) StoreCopyCheckPointMutex(org.neo4j.kernel.impl.transaction.log.checkpoint.StoreCopyCheckPointMutex) ReentrantLockService(org.neo4j.kernel.impl.locking.ReentrantLockService) DatabaseMigrator(org.neo4j.kernel.impl.storemigration.DatabaseMigrator) LifecycleAdapter(org.neo4j.kernel.lifecycle.LifecycleAdapter) LogPruneStrategy(org.neo4j.kernel.impl.transaction.log.pruning.LogPruneStrategy) Log(org.neo4j.logging.Log) StoreFileMetadata(org.neo4j.storageengine.api.StoreFileMetadata) IdGeneratorFactory(org.neo4j.kernel.impl.store.id.IdGeneratorFactory) GraphDatabaseFacadeFactory(org.neo4j.kernel.impl.factory.GraphDatabaseFacadeFactory) CheckPointThreshold(org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointThreshold) IdTypeConfigurationProvider(org.neo4j.kernel.impl.store.id.configuration.IdTypeConfigurationProvider) BatchingTransactionAppender(org.neo4j.kernel.impl.transaction.log.BatchingTransactionAppender) Dependencies(org.neo4j.kernel.impl.util.Dependencies) LifeSupport(org.neo4j.kernel.lifecycle.LifeSupport) KernelAPI(org.neo4j.kernel.api.KernelAPI) PhysicalLogFileInformation(org.neo4j.kernel.impl.transaction.log.PhysicalLogFileInformation) DiagnosticsExtractor(org.neo4j.kernel.info.DiagnosticsExtractor) ConstraintEnforcingEntityOperations(org.neo4j.kernel.impl.api.ConstraintEnforcingEntityOperations) NeoStoreFileListing(org.neo4j.kernel.impl.transaction.state.NeoStoreFileListing) LogRotation(org.neo4j.kernel.impl.transaction.log.rotation.LogRotation) StatementOperationContainer(org.neo4j.kernel.impl.api.StatementOperationContainer) DataIntegrityValidatingStatementOperations(org.neo4j.kernel.impl.api.DataIntegrityValidatingStatementOperations) QueryRegistrationOperations(org.neo4j.kernel.impl.api.operations.QueryRegistrationOperations) CheckPointerImpl(org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointerImpl) ReadableLogChannel(org.neo4j.kernel.impl.transaction.log.ReadableLogChannel) StatementLocksFactory(org.neo4j.kernel.impl.locking.StatementLocksFactory) PageCache(org.neo4j.io.pagecache.PageCache) Recovery(org.neo4j.kernel.recovery.Recovery) StorageEngine(org.neo4j.storageengine.api.StorageEngine) LogVersionRepository(org.neo4j.kernel.impl.transaction.log.LogVersionRepository) LogService(org.neo4j.kernel.impl.logging.LogService) KernelException(org.neo4j.kernel.api.exceptions.KernelException) PhysicalLogFile(org.neo4j.kernel.impl.transaction.log.PhysicalLogFile) StoreReadLayer(org.neo4j.storageengine.api.StoreReadLayer) LoggingLogFileMonitor(org.neo4j.kernel.impl.transaction.log.LoggingLogFileMonitor) NamedLabelScanStoreSelectionStrategy(org.neo4j.kernel.extension.dependency.NamedLabelScanStoreSelectionStrategy) MetaDataStore(org.neo4j.kernel.impl.store.MetaDataStore) StatementOperationParts(org.neo4j.kernel.impl.api.StatementOperationParts) LegacyIndexProviderLookup(org.neo4j.kernel.impl.api.LegacyIndexProviderLookup) DependencyResolver(org.neo4j.graphdb.DependencyResolver) LogHeaderCache(org.neo4j.kernel.impl.transaction.log.LogHeaderCache) KernelTransactions(org.neo4j.kernel.impl.api.KernelTransactions) TransactionHeaderInformationFactory(org.neo4j.kernel.impl.transaction.TransactionHeaderInformationFactory) PhysicalLogicalTransactionStore(org.neo4j.kernel.impl.transaction.log.PhysicalLogicalTransactionStore) TransactionAppender(org.neo4j.kernel.impl.transaction.log.TransactionAppender) TransactionIdStore(org.neo4j.kernel.impl.transaction.log.TransactionIdStore) LogicalTransactionStore(org.neo4j.kernel.impl.transaction.log.LogicalTransactionStore) Monitors(org.neo4j.kernel.monitoring.Monitors) LogProvider(org.neo4j.logging.LogProvider) HashMap(java.util.HashMap) DiagnosticsManager(org.neo4j.kernel.info.DiagnosticsManager) IOLimiter(org.neo4j.io.pagecache.IOLimiter) RecordFormats(org.neo4j.kernel.impl.store.format.RecordFormats) ConstraintIndexCreator(org.neo4j.kernel.impl.api.state.ConstraintIndexCreator) StoreMigrator(org.neo4j.kernel.impl.storemigration.participant.StoreMigrator) LogFileInformation(org.neo4j.kernel.impl.transaction.log.LogFileInformation) LogRotationImpl(org.neo4j.kernel.impl.transaction.log.rotation.LogRotationImpl) DiagnosticsPhase(org.neo4j.kernel.info.DiagnosticsPhase) LogPruningImpl(org.neo4j.kernel.impl.transaction.log.pruning.LogPruningImpl) LogPruning(org.neo4j.kernel.impl.transaction.log.pruning.LogPruning) MapUtil.stringMap(org.neo4j.helpers.collection.MapUtil.stringMap) PhysicalLogFiles(org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles) AutoIndexing(org.neo4j.kernel.api.legacyindex.AutoIndexing) Config(org.neo4j.kernel.configuration.Config) LockService(org.neo4j.kernel.impl.locking.LockService) Setting(org.neo4j.graphdb.config.Setting) LabelTokenHolder(org.neo4j.kernel.impl.core.LabelTokenHolder) VisibleMigrationProgressMonitor(org.neo4j.kernel.impl.storemigration.monitoring.VisibleMigrationProgressMonitor) TimeUnit(java.util.concurrent.TimeUnit) DatabaseHealth(org.neo4j.kernel.internal.DatabaseHealth) CommitProcessFactory(org.neo4j.kernel.impl.api.CommitProcessFactory) Clock(java.time.Clock) FileSystemAbstraction(org.neo4j.io.fs.FileSystemAbstraction) TransactionHooks(org.neo4j.kernel.impl.api.TransactionHooks) LogPruneStrategy(org.neo4j.kernel.impl.transaction.log.pruning.LogPruneStrategy) CheckPointerImpl(org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointerImpl) TimeCheckPointThreshold(org.neo4j.kernel.impl.transaction.log.checkpoint.TimeCheckPointThreshold) BatchingTransactionAppender(org.neo4j.kernel.impl.transaction.log.BatchingTransactionAppender) PhysicalLogicalTransactionStore(org.neo4j.kernel.impl.transaction.log.PhysicalLogicalTransactionStore) LogicalTransactionStore(org.neo4j.kernel.impl.transaction.log.LogicalTransactionStore) PhysicalLogFiles(org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles) LogPruning(org.neo4j.kernel.impl.transaction.log.pruning.LogPruning) TransactionMonitor(org.neo4j.kernel.impl.transaction.TransactionMonitor) LoggingLogFileMonitor(org.neo4j.kernel.impl.transaction.log.LoggingLogFileMonitor) VisibleMigrationProgressMonitor(org.neo4j.kernel.impl.storemigration.monitoring.VisibleMigrationProgressMonitor) TimeCheckPointThreshold(org.neo4j.kernel.impl.transaction.log.checkpoint.TimeCheckPointThreshold) CheckPointThreshold(org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointThreshold) LogEntry(org.neo4j.kernel.impl.transaction.log.entry.LogEntry) LogRotationImpl(org.neo4j.kernel.impl.transaction.log.rotation.LogRotationImpl) LogPosition(org.neo4j.kernel.impl.transaction.log.LogPosition) LogEntryStart(org.neo4j.kernel.impl.transaction.log.entry.LogEntryStart) PhysicalLogicalTransactionStore(org.neo4j.kernel.impl.transaction.log.PhysicalLogicalTransactionStore) ReadableLogChannel(org.neo4j.kernel.impl.transaction.log.ReadableLogChannel) CountCommittedTransactionThreshold(org.neo4j.kernel.impl.transaction.log.checkpoint.CountCommittedTransactionThreshold) LogPruningImpl(org.neo4j.kernel.impl.transaction.log.pruning.LogPruningImpl) BatchingTransactionAppender(org.neo4j.kernel.impl.transaction.log.BatchingTransactionAppender) TransactionAppender(org.neo4j.kernel.impl.transaction.log.TransactionAppender) TransactionMetadataCache(org.neo4j.kernel.impl.transaction.log.TransactionMetadataCache) CheckPointScheduler(org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointScheduler) PhysicalLogFileInformation(org.neo4j.kernel.impl.transaction.log.PhysicalLogFileInformation) LogFileInformation(org.neo4j.kernel.impl.transaction.log.LogFileInformation) LogRotation(org.neo4j.kernel.impl.transaction.log.rotation.LogRotation) LogHeaderCache(org.neo4j.kernel.impl.transaction.log.LogHeaderCache) PhysicalLogFile(org.neo4j.kernel.impl.transaction.log.PhysicalLogFile) PhysicalLogFileInformation(org.neo4j.kernel.impl.transaction.log.PhysicalLogFileInformation)

Example 2 with LogRotation

use of org.neo4j.kernel.impl.transaction.log.rotation.LogRotation in project neo4j by neo4j.

the class TransactionLogAppendAndRotateIT method shouldKeepTransactionsIntactWhenConcurrentlyRotationAndAppending.

@Test
public void shouldKeepTransactionsIntactWhenConcurrentlyRotationAndAppending() throws Throwable {
    // GIVEN
    PhysicalLogFiles logFiles = new PhysicalLogFiles(directory.directory().getAbsoluteFile(), fileSystemRule.get());
    long rotationThreshold = mebiBytes(1);
    LogVersionRepository logVersionRepository = new DeadSimpleLogVersionRepository(0);
    final AtomicBoolean end = new AtomicBoolean();
    AllTheMonitoring monitoring = new AllTheMonitoring(end, 100);
    TransactionIdStore txIdStore = new DeadSimpleTransactionIdStore();
    TransactionMetadataCache metadataCache = new TransactionMetadataCache(100);
    LogHeaderCache logHeaderCache = new LogHeaderCache(10);
    LogFile logFile = life.add(new PhysicalLogFile(fileSystemRule.get(), logFiles, rotationThreshold, () -> txIdStore.getLastCommittedTransactionId(), logVersionRepository, monitoring, logHeaderCache));
    monitoring.setLogFile(logFile);
    DatabaseHealth health = new DatabaseHealth(mock(DatabasePanicEventGenerator.class), NullLog.getInstance());
    LogRotation rotation = new LogRotationImpl(monitoring, logFile, health);
    final TransactionAppender appender = life.add(new BatchingTransactionAppender(logFile, rotation, metadataCache, txIdStore, BYPASS, health));
    // WHEN
    Race race = new Race();
    for (int i = 0; i < 10; i++) {
        race.addContestant(new Runnable() {

            @Override
            public void run() {
                while (!end.get()) {
                    try {
                        appender.append(new TransactionToApply(sillyTransaction(1_000)), NULL);
                    } catch (Exception e) {
                        e.printStackTrace(System.out);
                        end.set(true);
                        fail(e.getMessage());
                    }
                }
            }
        });
    }
    race.addContestant(endAfterMax(10, SECONDS, end));
    race.go();
    // THEN
    assertTrue(monitoring.numberOfRotations() > 0);
}
Also used : DatabaseHealth(org.neo4j.kernel.internal.DatabaseHealth) TransactionToApply(org.neo4j.kernel.impl.api.TransactionToApply) DeadSimpleTransactionIdStore(org.neo4j.kernel.impl.transaction.DeadSimpleTransactionIdStore) DeadSimpleTransactionIdStore(org.neo4j.kernel.impl.transaction.DeadSimpleTransactionIdStore) DatabasePanicEventGenerator(org.neo4j.kernel.impl.core.DatabasePanicEventGenerator) DeadSimpleLogVersionRepository(org.neo4j.kernel.impl.transaction.DeadSimpleLogVersionRepository) DeadSimpleLogVersionRepository(org.neo4j.kernel.impl.transaction.DeadSimpleLogVersionRepository) IOException(java.io.IOException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Race(org.neo4j.test.Race) LogRotation(org.neo4j.kernel.impl.transaction.log.rotation.LogRotation) LogRotationImpl(org.neo4j.kernel.impl.transaction.log.rotation.LogRotationImpl) Test(org.junit.Test)

Aggregations

IOException (java.io.IOException)2 File (java.io.File)1 Clock (java.time.Clock)1 HashMap (java.util.HashMap)1 Map (java.util.Map)1 TimeUnit (java.util.concurrent.TimeUnit)1 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)1 LockSupport (java.util.concurrent.locks.LockSupport)1 Supplier (java.util.function.Supplier)1 Test (org.junit.Test)1 DependencyResolver (org.neo4j.graphdb.DependencyResolver)1 ResourceIterator (org.neo4j.graphdb.ResourceIterator)1 Setting (org.neo4j.graphdb.config.Setting)1 GraphDatabaseSettings (org.neo4j.graphdb.factory.GraphDatabaseSettings)1 Exceptions (org.neo4j.helpers.Exceptions)1 MapUtil.stringMap (org.neo4j.helpers.collection.MapUtil.stringMap)1 FileSystemAbstraction (org.neo4j.io.fs.FileSystemAbstraction)1 IOLimiter (org.neo4j.io.pagecache.IOLimiter)1 PageCache (org.neo4j.io.pagecache.PageCache)1 KernelAPI (org.neo4j.kernel.api.KernelAPI)1