Examples with JobScheduler org.neo4j.scheduler.JobScheduler used on opensource projects

Example 26 with JobScheduler

use of org.neo4j.scheduler.JobScheduler in project neo4j by neo4j.

the class AbstractNeoWebServer method createHttpTransactionManager.

private HttpTransactionManager createHttpTransactionManager() {
    JobScheduler jobScheduler = globalDependencies.resolveDependency(JobScheduler.class);
    Clock clock = Clocks.systemClock();
    Duration transactionTimeout = getTransactionTimeout();
    return new HttpTransactionManager(databaseManagementService, transactionMemoryPool, jobScheduler, clock, transactionTimeout, userLogProvider);
}

Example 27 with JobScheduler

use of org.neo4j.scheduler.JobScheduler in project neo4j by neo4j.

the class PageCacheStressTest method run.

public void run() throws Exception {
    try (FileSystemAbstraction fs = new DefaultFileSystemAbstraction();
        JobScheduler jobScheduler = new ThreadPoolJobScheduler()) {
        PageSwapperFactory swapperFactory = new SingleFilePageSwapperFactory(fs);
        try (PageCache pageCacheUnderTest = new MuninnPageCache(swapperFactory, jobScheduler, config(numberOfCachePages).pageCacheTracer(tracer))) {
            PageCacheStresser pageCacheStresser = new PageCacheStresser(numberOfPages, numberOfThreads, workingDirectory);
            pageCacheStresser.stress(pageCacheUnderTest, tracer, condition);
        }
    }
}

Example 28 with JobScheduler

use of org.neo4j.scheduler.JobScheduler in project neo4j by neo4j.

the class RandomPageCacheTestHarness method runIteration.

@SuppressWarnings("unchecked")
private void runIteration(long timeout, TimeUnit unit) throws Exception {
    assert filePageSize % recordFormat.getRecordSize() == 0 : "File page size must be a multiple of the record size";
    if (!fixedRandomSeed) {
        randomSeed = ThreadLocalRandom.current().nextLong();
    }
    FileSystemAbstraction fs = this.fs;
    Path[] files = buildFileNames();
    RandomAdversary adversary = new RandomAdversary(mischiefRate, failureRate, errorRate);
    adversary.setProbabilityFactor(0.0);
    if (useAdversarialIO) {
        adversary.setSeed(randomSeed);
        fs = new AdversarialFileSystemAbstraction(adversary, fs);
    }
    PageSwapperFactory swapperFactory = new SingleFilePageSwapperFactory(fs);
    JobScheduler jobScheduler = new ThreadPoolJobScheduler();
    MuninnPageCache cache = new MuninnPageCache(swapperFactory, jobScheduler, MuninnPageCache.config(cachePageCount).pageCacheTracer(tracer));
    if (filePageSize == 0) {
        filePageSize = cache.pageSize();
    }
    cache.setPrintExceptionsOnClose(false);
    Map<Path, PagedFile> fileMap = new HashMap<>(files.length);
    for (int i = 0; i < Math.min(files.length, initialMappedFiles); i++) {
        Path file = files[i];
        fileMap.put(file, cache.map(file, filePageSize, DEFAULT_DATABASE_NAME));
    }
    plan = plan(cache, files, fileMap);
    AtomicBoolean stopSignal = new AtomicBoolean();
    Callable<Void> planRunner = new PlanRunner(plan, stopSignal, profiler);
    Future<Void>[] futures = new Future[concurrencyLevel];
    for (int i = 0; i < concurrencyLevel; i++) {
        futures[i] = executorService.submit(planRunner);
    }
    if (preparation != null) {
        preparation.run(cache, this.fs, plan.getFilesTouched());
    }
    adversary.setProbabilityFactor(1.0);
    plan.start();
    long deadlineMillis = System.currentTimeMillis() + unit.toMillis(timeout);
    long now;
    try {
        for (Future<Void> future : futures) {
            now = System.currentTimeMillis();
            if (deadlineMillis < now) {
                throw new TimeoutException();
            }
            future.get(deadlineMillis - now, TimeUnit.MILLISECONDS);
        }
        adversary.setProbabilityFactor(0.0);
        runVerificationPhase(cache);
    } finally {
        stopSignal.set(true);
        adversary.setProbabilityFactor(0.0);
        try {
            for (Future<Void> future : futures) {
                future.get(10, TimeUnit.SECONDS);
            }
        } catch (InterruptedException | TimeoutException e) {
            for (Future<Void> future : futures) {
                future.cancel(true);
            }
            throw new RuntimeException(e);
        }
        try {
            plan.close();
            cache.close();
            jobScheduler.close();
            if (this.fs instanceof EphemeralFileSystemAbstraction) {
                this.fs.close();
                this.fs = new EphemeralFileSystemAbstraction();
            } else {
                for (Path file : files) {
                    Files.delete(file);
                }
            }
        } catch (IOException e) {
            throw new UncheckedIOException(e);
        }
    }
}

Example 29 with JobScheduler

use of org.neo4j.scheduler.JobScheduler in project neo4j by neo4j.

the class Recovery method performRecovery.

/**
 * Performs recovery of database described by provided layout.
 * Transaction logs should be located in their default location.
 * If recovery is not required nothing will be done to the the database or logs.
 * Note: used by external tools.
 * @param databaseLayout database to recover layout.
 * @param tracers underlying events tracers.
 * @throws IOException on any unexpected I/O exception encountered during recovery.
 */
public static void performRecovery(DatabaseLayout databaseLayout, DatabaseTracers tracers, MemoryTracker memoryTracker) throws Exception {
    requireNonNull(databaseLayout);
    Config config = defaults();
    try (DefaultFileSystemAbstraction fs = new DefaultFileSystemAbstraction();
        JobScheduler jobScheduler = JobSchedulerFactory.createInitialisedScheduler();
        PageCache pageCache = getPageCache(config, fs, jobScheduler)) {
        performRecovery(fs, pageCache, tracers, config, databaseLayout, memoryTracker);
    }
}

Example 30 with JobScheduler

use of org.neo4j.scheduler.JobScheduler in project neo4j by neo4j.

the class Recovery method performRecovery.

/**
 * Performs recovery of database described by provided layout.
 *
 * @param fs database filesystem
 * @param pageCache page cache used to perform database recovery.
 * @param tracers underlying operation tracers
 * @param config custom configuration
 * @param databaseLayout database to recover layout.
 * @param storageEngineFactory {@link StorageEngineFactory} for the storage to recover.
 * @param logProvider log provider
 * @param globalMonitors global server monitors
 * @param extensionFactories extension factories for extensions that should participate in recovery
 * @param providedLogFiles log files from database
 * @param forceRunRecovery to force recovery to run even if the usual checks indicates that it's not required.
 * In specific cases, like after store copy there's always a need for doing a recovery or at least to start the db, checkpoint and shut down,
 * even if the normal "is recovery required" checks says that recovery isn't required.
 * @throws IOException on any unexpected I/O exception encountered during recovery.
 */
public static void performRecovery(FileSystemAbstraction fs, PageCache pageCache, DatabaseTracers tracers, Config config, DatabaseLayout databaseLayout, StorageEngineFactory storageEngineFactory, boolean forceRunRecovery, LogProvider logProvider, Monitors globalMonitors, Iterable<ExtensionFactory<?>> extensionFactories, Optional<LogFiles> providedLogFiles, RecoveryStartupChecker startupChecker, MemoryTracker memoryTracker, Clock clock) throws IOException {
    Log recoveryLog = logProvider.getLog(Recovery.class);
    if (!forceRunRecovery && !isRecoveryRequired(fs, pageCache, databaseLayout, storageEngineFactory, config, providedLogFiles, memoryTracker)) {
        return;
    }
    checkAllFilesPresence(databaseLayout, fs, pageCache, storageEngineFactory);
    LifeSupport recoveryLife = new LifeSupport();
    Monitors monitors = new Monitors(globalMonitors, logProvider);
    DatabasePageCache databasePageCache = new DatabasePageCache(pageCache, IOController.DISABLED);
    SimpleLogService logService = new SimpleLogService(logProvider);
    VersionAwareLogEntryReader logEntryReader = new VersionAwareLogEntryReader(storageEngineFactory.commandReaderFactory());
    DatabaseReadOnlyChecker readOnlyChecker = writable();
    DatabaseSchemaState schemaState = new DatabaseSchemaState(logProvider);
    JobScheduler scheduler = JobSchedulerFactory.createInitialisedScheduler();
    VersionContextSupplier versionContextSupplier = EmptyVersionContextSupplier.EMPTY;
    DatabaseHealth databaseHealth = new DatabaseHealth(PanicEventGenerator.NO_OP, recoveryLog);
    TokenHolders tokenHolders = new TokenHolders(new DelegatingTokenHolder(new ReadOnlyTokenCreator(), TYPE_PROPERTY_KEY), new DelegatingTokenHolder(new ReadOnlyTokenCreator(), TYPE_LABEL), new DelegatingTokenHolder(new ReadOnlyTokenCreator(), TYPE_RELATIONSHIP_TYPE));
    RecoveryCleanupWorkCollector recoveryCleanupCollector = new GroupingRecoveryCleanupWorkCollector(scheduler, INDEX_CLEANUP, INDEX_CLEANUP_WORK, databaseLayout.getDatabaseName());
    DatabaseExtensions extensions = instantiateRecoveryExtensions(databaseLayout, fs, config, logService, databasePageCache, scheduler, DbmsInfo.TOOL, monitors, tokenHolders, recoveryCleanupCollector, readOnlyChecker, extensionFactories, tracers.getPageCacheTracer());
    DefaultIndexProviderMap indexProviderMap = new DefaultIndexProviderMap(extensions, config);
    StorageEngine storageEngine = storageEngineFactory.instantiate(fs, databaseLayout, config, databasePageCache, tokenHolders, schemaState, getConstraintSemantics(), indexProviderMap, NO_LOCK_SERVICE, new DefaultIdGeneratorFactory(fs, recoveryCleanupCollector, databaseLayout.getDatabaseName()), new DefaultIdController(), databaseHealth, logService.getInternalLogProvider(), recoveryCleanupCollector, tracers.getPageCacheTracer(), true, readOnlyChecker, memoryTracker);
    // Schema indexes
    FullScanStoreView fullScanStoreView = new FullScanStoreView(NO_LOCK_SERVICE, storageEngine::newReader, config, scheduler);
    IndexStoreViewFactory indexStoreViewFactory = new IndexStoreViewFactory(config, storageEngine::newReader, NO_LOCKS, fullScanStoreView, NO_LOCK_SERVICE, logProvider);
    IndexStatisticsStore indexStatisticsStore = new IndexStatisticsStore(databasePageCache, databaseLayout, recoveryCleanupCollector, readOnlyChecker, tracers.getPageCacheTracer());
    IndexingService indexingService = Database.buildIndexingService(storageEngine, schemaState, indexStoreViewFactory, indexStatisticsStore, config, scheduler, indexProviderMap, tokenHolders, logProvider, logProvider, monitors.newMonitor(IndexingService.Monitor.class), tracers.getPageCacheTracer(), memoryTracker, databaseLayout.getDatabaseName(), readOnlyChecker);
    MetadataProvider metadataProvider = storageEngine.metadataProvider();
    Dependencies dependencies = new Dependencies();
    dependencies.satisfyDependencies(databaseLayout, config, databasePageCache, fs, logProvider, tokenHolders, schemaState, getConstraintSemantics(), NO_LOCK_SERVICE, databaseHealth, new DefaultIdGeneratorFactory(fs, recoveryCleanupCollector, databaseLayout.getDatabaseName()), new DefaultIdController(), readOnlyChecker, versionContextSupplier, logService, metadataProvider);
    LogFiles logFiles = LogFilesBuilder.builder(databaseLayout, fs).withLogEntryReader(logEntryReader).withConfig(config).withDependencies(dependencies).withMemoryTracker(memoryTracker).build();
    boolean failOnCorruptedLogFiles = config.get(GraphDatabaseInternalSettings.fail_on_corrupted_log_files);
    validateStoreId(logFiles, storageEngine.getStoreId(), config);
    TransactionMetadataCache metadataCache = new TransactionMetadataCache();
    PhysicalLogicalTransactionStore transactionStore = new PhysicalLogicalTransactionStore(logFiles, metadataCache, logEntryReader, monitors, failOnCorruptedLogFiles);
    BatchingTransactionAppender transactionAppender = new BatchingTransactionAppender(logFiles, LogRotation.NO_ROTATION, metadataCache, metadataProvider, databaseHealth);
    LifeSupport schemaLife = new LifeSupport();
    schemaLife.add(storageEngine.schemaAndTokensLifecycle());
    schemaLife.add(indexingService);
    var doParallelRecovery = config.get(GraphDatabaseInternalSettings.do_parallel_recovery);
    TransactionLogsRecovery transactionLogsRecovery = transactionLogRecovery(fs, metadataProvider, monitors.newMonitor(RecoveryMonitor.class), monitors.newMonitor(RecoveryStartInformationProvider.Monitor.class), logFiles, storageEngine, transactionStore, metadataProvider, schemaLife, databaseLayout, failOnCorruptedLogFiles, recoveryLog, startupChecker, tracers.getPageCacheTracer(), memoryTracker, doParallelRecovery);
    CheckPointerImpl.ForceOperation forceOperation = new DefaultForceOperation(indexingService, storageEngine);
    var checkpointAppender = logFiles.getCheckpointFile().getCheckpointAppender();
    CheckPointerImpl checkPointer = new CheckPointerImpl(metadataProvider, RecoveryThreshold.INSTANCE, forceOperation, LogPruning.NO_PRUNING, checkpointAppender, databaseHealth, logProvider, tracers, IOController.DISABLED, new StoreCopyCheckPointMutex(), versionContextSupplier, clock);
    recoveryLife.add(scheduler);
    recoveryLife.add(recoveryCleanupCollector);
    recoveryLife.add(extensions);
    recoveryLife.add(indexProviderMap);
    recoveryLife.add(storageEngine);
    recoveryLife.add(new MissingTransactionLogsCheck(databaseLayout, config, fs, logFiles, recoveryLog));
    recoveryLife.add(logFiles);
    recoveryLife.add(transactionLogsRecovery);
    recoveryLife.add(transactionAppender);
    recoveryLife.add(checkPointer);
    try {
        recoveryLife.start();
        if (databaseHealth.isHealthy()) {
            checkPointer.forceCheckPoint(new SimpleTriggerInfo("Recovery completed."));
        }
    } finally {
        recoveryLife.shutdown();
    }
}