Examples with StoreCopyCheckPointMutex org.neo4j.kernel.impl.transaction.log.checkpoint.StoreCopyCheckPointMutex used on opensource projects

Example 1 with StoreCopyCheckPointMutex

use of org.neo4j.kernel.impl.transaction.log.checkpoint.StoreCopyCheckPointMutex in project neo4j by neo4j.

the class ConnectionInfoIT method catchupServerMessage.

@Test
public void catchupServerMessage() throws Throwable {
    // given
    testSocket = bindPort("localhost", 4242);
    // when
    AssertableLogProvider logProvider = new AssertableLogProvider();
    AssertableLogProvider userLogProvider = new AssertableLogProvider();
    CoreState coreState = mock(CoreState.class);
    Config config = Config.defaults().with(singletonMap(transaction_listen_address.name(), ":" + testSocket.getLocalPort()));
    CatchupServer catchupServer = new CatchupServer(logProvider, userLogProvider, mockSupplier(), mockSupplier(), mockSupplier(), mockSupplier(), mock(BooleanSupplier.class), coreState, config, new Monitors(), mockSupplier(), mock(FileSystemAbstraction.class), mock(PageCache.class), new StoreCopyCheckPointMutex());
    //then
    try {
        catchupServer.start();
    } catch (Throwable throwable) {
    //expected.
    }
    logProvider.assertContainsMessageContaining("Address is already bound for setting");
    userLogProvider.assertContainsMessageContaining("Address is already bound for setting");
}

Example 2 with StoreCopyCheckPointMutex

use of org.neo4j.kernel.impl.transaction.log.checkpoint.StoreCopyCheckPointMutex in project neo4j by neo4j.

the class NeoStoreDataSourceRule method getDataSource.

public NeoStoreDataSource getDataSource(File storeDir, FileSystemAbstraction fs, IdGeneratorFactory idGeneratorFactory, IdTypeConfigurationProvider idConfigurationProvider, PageCache pageCache, Config config, DatabaseHealth databaseHealth, LogService logService) {
    if (dataSource != null) {
        dataSource.stop();
        dataSource.shutdown();
    }
    StatementLocksFactory locksFactory = mock(StatementLocksFactory.class);
    StatementLocks statementLocks = mock(StatementLocks.class);
    Locks.Client locks = mock(Locks.Client.class);
    when(statementLocks.optimistic()).thenReturn(locks);
    when(statementLocks.pessimistic()).thenReturn(locks);
    when(locksFactory.newInstance()).thenReturn(statementLocks);
    JobScheduler jobScheduler = mock(JobScheduler.class, RETURNS_MOCKS);
    Monitors monitors = new Monitors();
    LabelScanStoreProvider labelScanStoreProvider = nativeLabelScanStoreProvider(storeDir, fs, pageCache, config, logService);
    SystemNanoClock clock = Clocks.nanoClock();
    dataSource = new NeoStoreDataSource(storeDir, config, idGeneratorFactory, IdReuseEligibility.ALWAYS, idConfigurationProvider, logService, mock(JobScheduler.class, RETURNS_MOCKS), mock(TokenNameLookup.class), dependencyResolverForNoIndexProvider(labelScanStoreProvider), mock(PropertyKeyTokenHolder.class), mock(LabelTokenHolder.class), mock(RelationshipTypeTokenHolder.class), locksFactory, mock(SchemaWriteGuard.class), mock(TransactionEventHandlers.class), IndexingService.NO_MONITOR, fs, mock(TransactionMonitor.class), databaseHealth, mock(PhysicalLogFile.Monitor.class), TransactionHeaderInformationFactory.DEFAULT, new StartupStatisticsProvider(), null, new CommunityCommitProcessFactory(), mock(InternalAutoIndexing.class), pageCache, new StandardConstraintSemantics(), monitors, new Tracers("null", NullLog.getInstance(), monitors, jobScheduler), mock(Procedures.class), IOLimiter.unlimited(), new AvailabilityGuard(clock, NullLog.getInstance()), clock, new CanWrite(), new StoreCopyCheckPointMutex());
    return dataSource;
}

Example 3 with StoreCopyCheckPointMutex

use of org.neo4j.kernel.impl.transaction.log.checkpoint.StoreCopyCheckPointMutex in project neo4j by neo4j.

the class NeoStoreDataSource method buildTransactionLogs.

private NeoStoreTransactionLogModule buildTransactionLogs(File storeDir, Config config, LogProvider logProvider, JobScheduler scheduler, FileSystemAbstraction fileSystemAbstraction, StorageEngine storageEngine, LogEntryReader<ReadableClosablePositionAwareChannel> logEntryReader, SynchronizedArrayIdOrderingQueue legacyIndexTransactionOrdering, TransactionIdStore transactionIdStore, LogVersionRepository logVersionRepository) {
    TransactionMetadataCache transactionMetadataCache = new TransactionMetadataCache(100_000);
    LogHeaderCache logHeaderCache = new LogHeaderCache(1000);
    final PhysicalLogFiles logFiles = new PhysicalLogFiles(storeDir, PhysicalLogFile.DEFAULT_NAME, fileSystemAbstraction);
    final PhysicalLogFile logFile = life.add(new PhysicalLogFile(fileSystemAbstraction, logFiles, config.get(GraphDatabaseSettings.logical_log_rotation_threshold), transactionIdStore::getLastCommittedTransactionId, logVersionRepository, physicalLogMonitor, logHeaderCache));
    final PhysicalLogFileInformation.LogVersionToTimestamp logInformation = version -> {
        LogPosition position = LogPosition.start(version);
        try (ReadableLogChannel channel = logFile.getReader(position)) {
            LogEntry entry;
            while ((entry = logEntryReader.readLogEntry(channel)) != null) {
                if (entry instanceof LogEntryStart) {
                    return entry.<LogEntryStart>as().getTimeWritten();
                }
            }
        }
        return -1;
    };
    final LogFileInformation logFileInformation = new PhysicalLogFileInformation(logFiles, logHeaderCache, transactionIdStore::getLastCommittedTransactionId, logInformation);
    if (config.get(GraphDatabaseFacadeFactory.Configuration.ephemeral)) {
        config = config.withDefaults(stringMap(GraphDatabaseSettings.keep_logical_logs.name(), "1 files"));
    }
    String pruningConf = config.get(GraphDatabaseSettings.keep_logical_logs);
    LogPruneStrategy logPruneStrategy = fromConfigValue(fs, logFileInformation, logFiles, pruningConf);
    final LogPruning logPruning = new LogPruningImpl(logPruneStrategy, logProvider);
    final LogRotation logRotation = new LogRotationImpl(monitors.newMonitor(LogRotation.Monitor.class), logFile, databaseHealth);
    final TransactionAppender appender = life.add(new BatchingTransactionAppender(logFile, logRotation, transactionMetadataCache, transactionIdStore, legacyIndexTransactionOrdering, databaseHealth));
    final LogicalTransactionStore logicalTransactionStore = new PhysicalLogicalTransactionStore(logFile, transactionMetadataCache, logEntryReader);
    int txThreshold = config.get(GraphDatabaseSettings.check_point_interval_tx);
    final CountCommittedTransactionThreshold countCommittedTransactionThreshold = new CountCommittedTransactionThreshold(txThreshold);
    long timeMillisThreshold = config.get(GraphDatabaseSettings.check_point_interval_time);
    TimeCheckPointThreshold timeCheckPointThreshold = new TimeCheckPointThreshold(timeMillisThreshold, clock);
    CheckPointThreshold threshold = CheckPointThresholds.or(countCommittedTransactionThreshold, timeCheckPointThreshold);
    final CheckPointerImpl checkPointer = new CheckPointerImpl(transactionIdStore, threshold, storageEngine, logPruning, appender, databaseHealth, logProvider, tracers.checkPointTracer, ioLimiter, storeCopyCheckPointMutex);
    long recurringPeriod = Math.min(timeMillisThreshold, TimeUnit.SECONDS.toMillis(10));
    CheckPointScheduler checkPointScheduler = new CheckPointScheduler(checkPointer, scheduler, recurringPeriod, databaseHealth);
    life.add(checkPointer);
    life.add(checkPointScheduler);
    return new NeoStoreTransactionLogModule(logicalTransactionStore, logFileInformation, logFiles, logFile, logRotation, checkPointer, appender, legacyIndexTransactionOrdering);
}

Example 4 with StoreCopyCheckPointMutex

use of org.neo4j.kernel.impl.transaction.log.checkpoint.StoreCopyCheckPointMutex in project neo4j by neo4j.

the class DefaultMasterImplSPITest method flushStoreFilesWithCorrectCheckpointTriggerName.

@Test
public void flushStoreFilesWithCorrectCheckpointTriggerName() throws IOException {
    CheckPointer checkPointer = mock(CheckPointer.class);
    NeoStoreDataSource dataSource = mock(NeoStoreDataSource.class);
    when(dataSource.listStoreFiles(anyBoolean())).thenReturn(Iterators.emptyIterator());
    DefaultMasterImplSPI master = new DefaultMasterImplSPI(mock(GraphDatabaseAPI.class, RETURNS_MOCKS), mock(FileSystemAbstraction.class), new Monitors(), mock(LabelTokenHolder.class), mock(PropertyKeyTokenHolder.class), mock(RelationshipTypeTokenHolder.class), mock(IdGeneratorFactory.class), mock(TransactionCommitProcess.class), checkPointer, mock(TransactionIdStore.class), mock(LogicalTransactionStore.class), dataSource, mock(PageCache.class), new StoreCopyCheckPointMutex());
    master.flushStoresAndStreamStoreFiles(mock(StoreWriter.class));
    TriggerInfo expectedTriggerInfo = new SimpleTriggerInfo(DefaultMasterImplSPI.STORE_COPY_CHECKPOINT_TRIGGER);
    verify(checkPointer).tryCheckPoint(expectedTriggerInfo);
}

Example 5 with StoreCopyCheckPointMutex

use of org.neo4j.kernel.impl.transaction.log.checkpoint.StoreCopyCheckPointMutex in project neo4j by neo4j.

the class Recovery method performRecovery.

/**
 * Performs recovery of database described by provided layout.
 *
 * @param fs database filesystem
 * @param pageCache page cache used to perform database recovery.
 * @param tracers underlying operation tracers
 * @param config custom configuration
 * @param databaseLayout database to recover layout.
 * @param storageEngineFactory {@link StorageEngineFactory} for the storage to recover.
 * @param logProvider log provider
 * @param globalMonitors global server monitors
 * @param extensionFactories extension factories for extensions that should participate in recovery
 * @param providedLogFiles log files from database
 * @param forceRunRecovery to force recovery to run even if the usual checks indicates that it's not required.
 * In specific cases, like after store copy there's always a need for doing a recovery or at least to start the db, checkpoint and shut down,
 * even if the normal "is recovery required" checks says that recovery isn't required.
 * @throws IOException on any unexpected I/O exception encountered during recovery.
 */
public static void performRecovery(FileSystemAbstraction fs, PageCache pageCache, DatabaseTracers tracers, Config config, DatabaseLayout databaseLayout, StorageEngineFactory storageEngineFactory, boolean forceRunRecovery, LogProvider logProvider, Monitors globalMonitors, Iterable<ExtensionFactory<?>> extensionFactories, Optional<LogFiles> providedLogFiles, RecoveryStartupChecker startupChecker, MemoryTracker memoryTracker, Clock clock) throws IOException {
    Log recoveryLog = logProvider.getLog(Recovery.class);
    if (!forceRunRecovery && !isRecoveryRequired(fs, pageCache, databaseLayout, storageEngineFactory, config, providedLogFiles, memoryTracker)) {
        return;
    }
    checkAllFilesPresence(databaseLayout, fs, pageCache, storageEngineFactory);
    LifeSupport recoveryLife = new LifeSupport();
    Monitors monitors = new Monitors(globalMonitors, logProvider);
    DatabasePageCache databasePageCache = new DatabasePageCache(pageCache, IOController.DISABLED);
    SimpleLogService logService = new SimpleLogService(logProvider);
    VersionAwareLogEntryReader logEntryReader = new VersionAwareLogEntryReader(storageEngineFactory.commandReaderFactory());
    DatabaseReadOnlyChecker readOnlyChecker = writable();
    DatabaseSchemaState schemaState = new DatabaseSchemaState(logProvider);
    JobScheduler scheduler = JobSchedulerFactory.createInitialisedScheduler();
    VersionContextSupplier versionContextSupplier = EmptyVersionContextSupplier.EMPTY;
    DatabaseHealth databaseHealth = new DatabaseHealth(PanicEventGenerator.NO_OP, recoveryLog);
    TokenHolders tokenHolders = new TokenHolders(new DelegatingTokenHolder(new ReadOnlyTokenCreator(), TYPE_PROPERTY_KEY), new DelegatingTokenHolder(new ReadOnlyTokenCreator(), TYPE_LABEL), new DelegatingTokenHolder(new ReadOnlyTokenCreator(), TYPE_RELATIONSHIP_TYPE));
    RecoveryCleanupWorkCollector recoveryCleanupCollector = new GroupingRecoveryCleanupWorkCollector(scheduler, INDEX_CLEANUP, INDEX_CLEANUP_WORK, databaseLayout.getDatabaseName());
    DatabaseExtensions extensions = instantiateRecoveryExtensions(databaseLayout, fs, config, logService, databasePageCache, scheduler, DbmsInfo.TOOL, monitors, tokenHolders, recoveryCleanupCollector, readOnlyChecker, extensionFactories, tracers.getPageCacheTracer());
    DefaultIndexProviderMap indexProviderMap = new DefaultIndexProviderMap(extensions, config);
    StorageEngine storageEngine = storageEngineFactory.instantiate(fs, databaseLayout, config, databasePageCache, tokenHolders, schemaState, getConstraintSemantics(), indexProviderMap, NO_LOCK_SERVICE, new DefaultIdGeneratorFactory(fs, recoveryCleanupCollector, databaseLayout.getDatabaseName()), new DefaultIdController(), databaseHealth, logService.getInternalLogProvider(), recoveryCleanupCollector, tracers.getPageCacheTracer(), true, readOnlyChecker, memoryTracker);
    // Schema indexes
    FullScanStoreView fullScanStoreView = new FullScanStoreView(NO_LOCK_SERVICE, storageEngine::newReader, config, scheduler);
    IndexStoreViewFactory indexStoreViewFactory = new IndexStoreViewFactory(config, storageEngine::newReader, NO_LOCKS, fullScanStoreView, NO_LOCK_SERVICE, logProvider);
    IndexStatisticsStore indexStatisticsStore = new IndexStatisticsStore(databasePageCache, databaseLayout, recoveryCleanupCollector, readOnlyChecker, tracers.getPageCacheTracer());
    IndexingService indexingService = Database.buildIndexingService(storageEngine, schemaState, indexStoreViewFactory, indexStatisticsStore, config, scheduler, indexProviderMap, tokenHolders, logProvider, logProvider, monitors.newMonitor(IndexingService.Monitor.class), tracers.getPageCacheTracer(), memoryTracker, databaseLayout.getDatabaseName(), readOnlyChecker);
    MetadataProvider metadataProvider = storageEngine.metadataProvider();
    Dependencies dependencies = new Dependencies();
    dependencies.satisfyDependencies(databaseLayout, config, databasePageCache, fs, logProvider, tokenHolders, schemaState, getConstraintSemantics(), NO_LOCK_SERVICE, databaseHealth, new DefaultIdGeneratorFactory(fs, recoveryCleanupCollector, databaseLayout.getDatabaseName()), new DefaultIdController(), readOnlyChecker, versionContextSupplier, logService, metadataProvider);
    LogFiles logFiles = LogFilesBuilder.builder(databaseLayout, fs).withLogEntryReader(logEntryReader).withConfig(config).withDependencies(dependencies).withMemoryTracker(memoryTracker).build();
    boolean failOnCorruptedLogFiles = config.get(GraphDatabaseInternalSettings.fail_on_corrupted_log_files);
    validateStoreId(logFiles, storageEngine.getStoreId(), config);
    TransactionMetadataCache metadataCache = new TransactionMetadataCache();
    PhysicalLogicalTransactionStore transactionStore = new PhysicalLogicalTransactionStore(logFiles, metadataCache, logEntryReader, monitors, failOnCorruptedLogFiles);
    BatchingTransactionAppender transactionAppender = new BatchingTransactionAppender(logFiles, LogRotation.NO_ROTATION, metadataCache, metadataProvider, databaseHealth);
    LifeSupport schemaLife = new LifeSupport();
    schemaLife.add(storageEngine.schemaAndTokensLifecycle());
    schemaLife.add(indexingService);
    var doParallelRecovery = config.get(GraphDatabaseInternalSettings.do_parallel_recovery);
    TransactionLogsRecovery transactionLogsRecovery = transactionLogRecovery(fs, metadataProvider, monitors.newMonitor(RecoveryMonitor.class), monitors.newMonitor(RecoveryStartInformationProvider.Monitor.class), logFiles, storageEngine, transactionStore, metadataProvider, schemaLife, databaseLayout, failOnCorruptedLogFiles, recoveryLog, startupChecker, tracers.getPageCacheTracer(), memoryTracker, doParallelRecovery);
    CheckPointerImpl.ForceOperation forceOperation = new DefaultForceOperation(indexingService, storageEngine);
    var checkpointAppender = logFiles.getCheckpointFile().getCheckpointAppender();
    CheckPointerImpl checkPointer = new CheckPointerImpl(metadataProvider, RecoveryThreshold.INSTANCE, forceOperation, LogPruning.NO_PRUNING, checkpointAppender, databaseHealth, logProvider, tracers, IOController.DISABLED, new StoreCopyCheckPointMutex(), versionContextSupplier, clock);
    recoveryLife.add(scheduler);
    recoveryLife.add(recoveryCleanupCollector);
    recoveryLife.add(extensions);
    recoveryLife.add(indexProviderMap);
    recoveryLife.add(storageEngine);
    recoveryLife.add(new MissingTransactionLogsCheck(databaseLayout, config, fs, logFiles, recoveryLog));
    recoveryLife.add(logFiles);
    recoveryLife.add(transactionLogsRecovery);
    recoveryLife.add(transactionAppender);
    recoveryLife.add(checkPointer);
    try {
        recoveryLife.start();
        if (databaseHealth.isHealthy()) {
            checkPointer.forceCheckPoint(new SimpleTriggerInfo("Recovery completed."));
        }
    } finally {
        recoveryLife.shutdown();
    }
}