Examples with JobScheduler org.neo4j.kernel.impl.util.JobScheduler used on opensource projects

Example 1 with JobScheduler

use of org.neo4j.kernel.impl.util.JobScheduler in project neo4j by neo4j.

the class NeoStoreDataSource method buildTransactionLogs.

private NeoStoreTransactionLogModule buildTransactionLogs(File storeDir, Config config, LogProvider logProvider, JobScheduler scheduler, FileSystemAbstraction fileSystemAbstraction, StorageEngine storageEngine, LogEntryReader<ReadableClosablePositionAwareChannel> logEntryReader, SynchronizedArrayIdOrderingQueue legacyIndexTransactionOrdering, TransactionIdStore transactionIdStore, LogVersionRepository logVersionRepository) {
    TransactionMetadataCache transactionMetadataCache = new TransactionMetadataCache(100_000);
    LogHeaderCache logHeaderCache = new LogHeaderCache(1000);
    final PhysicalLogFiles logFiles = new PhysicalLogFiles(storeDir, PhysicalLogFile.DEFAULT_NAME, fileSystemAbstraction);
    final PhysicalLogFile logFile = life.add(new PhysicalLogFile(fileSystemAbstraction, logFiles, config.get(GraphDatabaseSettings.logical_log_rotation_threshold), transactionIdStore::getLastCommittedTransactionId, logVersionRepository, physicalLogMonitor, logHeaderCache));
    final PhysicalLogFileInformation.LogVersionToTimestamp logInformation = version -> {
        LogPosition position = LogPosition.start(version);
        try (ReadableLogChannel channel = logFile.getReader(position)) {
            LogEntry entry;
            while ((entry = logEntryReader.readLogEntry(channel)) != null) {
                if (entry instanceof LogEntryStart) {
                    return entry.<LogEntryStart>as().getTimeWritten();
                }
            }
        }
        return -1;
    };
    final LogFileInformation logFileInformation = new PhysicalLogFileInformation(logFiles, logHeaderCache, transactionIdStore::getLastCommittedTransactionId, logInformation);
    if (config.get(GraphDatabaseFacadeFactory.Configuration.ephemeral)) {
        config = config.withDefaults(stringMap(GraphDatabaseSettings.keep_logical_logs.name(), "1 files"));
    }
    String pruningConf = config.get(GraphDatabaseSettings.keep_logical_logs);
    LogPruneStrategy logPruneStrategy = fromConfigValue(fs, logFileInformation, logFiles, pruningConf);
    final LogPruning logPruning = new LogPruningImpl(logPruneStrategy, logProvider);
    final LogRotation logRotation = new LogRotationImpl(monitors.newMonitor(LogRotation.Monitor.class), logFile, databaseHealth);
    final TransactionAppender appender = life.add(new BatchingTransactionAppender(logFile, logRotation, transactionMetadataCache, transactionIdStore, legacyIndexTransactionOrdering, databaseHealth));
    final LogicalTransactionStore logicalTransactionStore = new PhysicalLogicalTransactionStore(logFile, transactionMetadataCache, logEntryReader);
    int txThreshold = config.get(GraphDatabaseSettings.check_point_interval_tx);
    final CountCommittedTransactionThreshold countCommittedTransactionThreshold = new CountCommittedTransactionThreshold(txThreshold);
    long timeMillisThreshold = config.get(GraphDatabaseSettings.check_point_interval_time);
    TimeCheckPointThreshold timeCheckPointThreshold = new TimeCheckPointThreshold(timeMillisThreshold, clock);
    CheckPointThreshold threshold = CheckPointThresholds.or(countCommittedTransactionThreshold, timeCheckPointThreshold);
    final CheckPointerImpl checkPointer = new CheckPointerImpl(transactionIdStore, threshold, storageEngine, logPruning, appender, databaseHealth, logProvider, tracers.checkPointTracer, ioLimiter, storeCopyCheckPointMutex);
    long recurringPeriod = Math.min(timeMillisThreshold, TimeUnit.SECONDS.toMillis(10));
    CheckPointScheduler checkPointScheduler = new CheckPointScheduler(checkPointer, scheduler, recurringPeriod, databaseHealth);
    life.add(checkPointer);
    life.add(checkPointScheduler);
    return new NeoStoreTransactionLogModule(logicalTransactionStore, logFileInformation, logFiles, logFile, logRotation, checkPointer, appender, legacyIndexTransactionOrdering);
}

Example 2 with JobScheduler

use of org.neo4j.kernel.impl.util.JobScheduler in project neo4j by neo4j.

the class EditionModule method setupSecurityModule.

protected static void setupSecurityModule(PlatformModule platformModule, Log log, Procedures procedures, String key) {
    for (SecurityModule candidate : Service.load(SecurityModule.class)) {
        if (candidate.matches(key)) {
            try {
                candidate.setup(new SecurityModule.Dependencies() {

                    @Override
                    public LogService logService() {
                        return platformModule.logging;
                    }

                    @Override
                    public Config config() {
                        return platformModule.config;
                    }

                    @Override
                    public Procedures procedures() {
                        return procedures;
                    }

                    @Override
                    public JobScheduler scheduler() {
                        return platformModule.jobScheduler;
                    }

                    @Override
                    public FileSystemAbstraction fileSystem() {
                        return platformModule.fileSystem;
                    }

                    @Override
                    public LifeSupport lifeSupport() {
                        return platformModule.life;
                    }

                    @Override
                    public DependencySatisfier dependencySatisfier() {
                        return platformModule.dependencies;
                    }
                });
                return;
            } catch (Exception e) {
                String errorMessage = "Failed to load security module.";
                log.error(errorMessage);
                throw new RuntimeException(errorMessage, e);
            }
        }
    }
    String errorMessage = "Failed to load security module with key '" + key + "'.";
    log.error(errorMessage);
    throw new IllegalArgumentException(errorMessage);
}

Example 3 with JobScheduler

use of org.neo4j.kernel.impl.util.JobScheduler in project neo4j by neo4j.

the class NeoStoreDataSourceRule method getDataSource.

public NeoStoreDataSource getDataSource(File storeDir, FileSystemAbstraction fs, IdGeneratorFactory idGeneratorFactory, IdTypeConfigurationProvider idConfigurationProvider, PageCache pageCache, Config config, DatabaseHealth databaseHealth, LogService logService) {
    if (dataSource != null) {
        dataSource.stop();
        dataSource.shutdown();
    }
    StatementLocksFactory locksFactory = mock(StatementLocksFactory.class);
    StatementLocks statementLocks = mock(StatementLocks.class);
    Locks.Client locks = mock(Locks.Client.class);
    when(statementLocks.optimistic()).thenReturn(locks);
    when(statementLocks.pessimistic()).thenReturn(locks);
    when(locksFactory.newInstance()).thenReturn(statementLocks);
    JobScheduler jobScheduler = mock(JobScheduler.class, RETURNS_MOCKS);
    Monitors monitors = new Monitors();
    LabelScanStoreProvider labelScanStoreProvider = nativeLabelScanStoreProvider(storeDir, fs, pageCache, config, logService);
    SystemNanoClock clock = Clocks.nanoClock();
    dataSource = new NeoStoreDataSource(storeDir, config, idGeneratorFactory, IdReuseEligibility.ALWAYS, idConfigurationProvider, logService, mock(JobScheduler.class, RETURNS_MOCKS), mock(TokenNameLookup.class), dependencyResolverForNoIndexProvider(labelScanStoreProvider), mock(PropertyKeyTokenHolder.class), mock(LabelTokenHolder.class), mock(RelationshipTypeTokenHolder.class), locksFactory, mock(SchemaWriteGuard.class), mock(TransactionEventHandlers.class), IndexingService.NO_MONITOR, fs, mock(TransactionMonitor.class), databaseHealth, mock(PhysicalLogFile.Monitor.class), TransactionHeaderInformationFactory.DEFAULT, new StartupStatisticsProvider(), null, new CommunityCommitProcessFactory(), mock(InternalAutoIndexing.class), pageCache, new StandardConstraintSemantics(), monitors, new Tracers("null", NullLog.getInstance(), monitors, jobScheduler), mock(Procedures.class), IOLimiter.unlimited(), new AvailabilityGuard(clock, NullLog.getInstance()), clock, new CanWrite(), new StoreCopyCheckPointMutex());
    return dataSource;
}

Example 4 with JobScheduler

use of org.neo4j.kernel.impl.util.JobScheduler in project neo4j by neo4j.

the class BoltKernelExtension method newInstance.

@Override
public Lifecycle newInstance(KernelContext context, Dependencies dependencies) throws Throwable {
    Config config = dependencies.config();
    GraphDatabaseService gdb = dependencies.db();
    GraphDatabaseAPI api = (GraphDatabaseAPI) gdb;
    LogService logService = dependencies.logService();
    Clock clock = dependencies.clock();
    Log log = logService.getInternalLog(WorkerFactory.class);
    LifeSupport life = new LifeSupport();
    JobScheduler scheduler = dependencies.scheduler();
    InternalLoggerFactory.setDefaultFactory(new Netty4LoggerFactory(logService.getInternalLogProvider()));
    Authentication authentication = authentication(dependencies.authManager(), dependencies.userManagerSupplier());
    BoltFactory boltFactory = life.add(new BoltFactoryImpl(api, dependencies.usageData(), logService, dependencies.txBridge(), authentication, dependencies.sessionTracker(), config));
    WorkerFactory workerFactory = createWorkerFactory(boltFactory, scheduler, dependencies, logService, clock);
    List<ProtocolInitializer> connectors = config.enabledBoltConnectors().stream().map((connConfig) -> {
        ListenSocketAddress listenAddress = config.get(connConfig.listen_address);
        AdvertisedSocketAddress advertisedAddress = config.get(connConfig.advertised_address);
        SslContext sslCtx;
        boolean requireEncryption;
        final BoltConnector.EncryptionLevel encryptionLevel = config.get(connConfig.encryption_level);
        switch(encryptionLevel) {
            case REQUIRED:
                // Encrypted connections are mandatory, a self-signed certificate may be generated.
                requireEncryption = true;
                sslCtx = createSslContext(config, log, advertisedAddress);
                break;
            case OPTIONAL:
                // Encrypted connections are optional, a self-signed certificate may be generated.
                requireEncryption = false;
                sslCtx = createSslContext(config, log, advertisedAddress);
                break;
            case DISABLED:
                // Encryption is turned off, no self-signed certificate will be generated.
                requireEncryption = false;
                sslCtx = null;
                break;
            default:
                // In the unlikely event that we happen to fall through to the default option here,
                // there is a mismatch between the BoltConnector.EncryptionLevel enum and the options
                // handled in this switch statement. In this case, we'll log a warning and default to
                // disabling encryption, since this mirrors the functionality introduced in 3.0.
                log.warn(format("Unhandled encryption level %s - assuming DISABLED.", encryptionLevel.name()));
                requireEncryption = false;
                sslCtx = null;
                break;
        }
        final Map<Long, BiFunction<Channel, Boolean, BoltProtocol>> versions = newVersions(logService, workerFactory);
        return new SocketTransport(listenAddress, sslCtx, requireEncryption, logService.getInternalLogProvider(), versions);
    }).collect(toList());
    if (connectors.size() > 0 && !config.get(GraphDatabaseSettings.disconnected)) {
        life.add(new NettyServer(scheduler.threadFactory(boltNetworkIO), connectors));
        log.info("Bolt Server extension loaded.");
        for (ProtocolInitializer connector : connectors) {
            logService.getUserLog(WorkerFactory.class).info("Bolt enabled on %s.", connector.address());
        }
    }
    return life;
}

Example 5 with JobScheduler

use of org.neo4j.kernel.impl.util.JobScheduler in project neo4j by neo4j.

the class IndexSamplingJobTrackerTest method shouldNotRunASampleJobWhichIsAlreadyRunning.

@Test
public void shouldNotRunASampleJobWhichIsAlreadyRunning() throws Throwable {
    // given
    when(config.jobLimit()).thenReturn(2);
    JobScheduler jobScheduler = new Neo4jJobScheduler();
    jobScheduler.init();
    IndexSamplingJobTracker jobTracker = new IndexSamplingJobTracker(config, jobScheduler);
    final DoubleLatch latch = new DoubleLatch();
    // when
    final AtomicInteger count = new AtomicInteger(0);
    assertTrue(jobTracker.canExecuteMoreSamplingJobs());
    IndexSamplingJob job = new IndexSamplingJob() {

        @Override
        public void run() {
            count.incrementAndGet();
            latch.waitForAllToStart();
            latch.finish();
        }

        @Override
        public long indexId() {
            return indexId12;
        }
    };
    jobTracker.scheduleSamplingJob(job);
    jobTracker.scheduleSamplingJob(job);
    latch.startAndWaitForAllToStart();
    latch.waitForAllToFinish();
    assertEquals(1, count.get());
}