Examples with Scheduler org.exist.scheduler.Scheduler used on opensource projects

Example 1 with Scheduler

use of org.exist.scheduler.Scheduler in project exist by eXist-db.

the class BrokerPool method _initialize.

private void _initialize() throws EXistException, DatabaseConfigurationException {
    this.lockManager = new LockManager(conf, concurrencyLevel);
    // Flag to indicate that we are initializing
    status.process(Event.INITIALIZE);
    if (LOG.isDebugEnabled()) {
        LOG.debug("initializing database instance '{}'...", instanceName);
    }
    // register core broker pool services
    this.scheduler = servicesManager.register(new QuartzSchedulerImpl(this));
    // NOTE: this must occur after the scheduler, and before any other service which requires access to the data directory
    this.dataLock = servicesManager.register(new FileLockService("dbx_dir.lck", BrokerPool.PROPERTY_DATA_DIR, NativeBroker.DEFAULT_DATA_DIR));
    this.securityManager = servicesManager.register(new SecurityManagerImpl(this));
    this.cacheManager = servicesManager.register(new DefaultCacheManager(this));
    this.xQueryPool = servicesManager.register(new XQueryPool());
    this.processMonitor = servicesManager.register(new ProcessMonitor());
    this.xqueryStats = servicesManager.register(new PerformanceStats(this));
    final XMLReaderObjectFactory xmlReaderObjectFactory = servicesManager.register(new XMLReaderObjectFactory());
    this.xmlReaderPool = servicesManager.register(new XMLReaderPool(xmlReaderObjectFactory, maxBrokers, 0));
    final int bufferSize = Optional.of(conf.getInteger(PROPERTY_COLLECTION_CACHE_SIZE)).filter(size -> size != -1).orElse(DEFAULT_COLLECTION_BUFFER_SIZE);
    this.collectionCache = servicesManager.register(new CollectionCache());
    this.notificationService = servicesManager.register(new NotificationService());
    this.journalManager = recoveryEnabled ? Optional.of(new JournalManager()) : Optional.empty();
    journalManager.ifPresent(servicesManager::register);
    final SystemTaskManager systemTaskManager = servicesManager.register(new SystemTaskManager(this));
    this.transactionManager = servicesManager.register(new TransactionManager(this, journalManager, systemTaskManager));
    this.blobStoreService = servicesManager.register(new BlobStoreImplService());
    this.symbols = servicesManager.register(new SymbolTable());
    this.expathRepo = Optional.ofNullable(new ExistRepository());
    expathRepo.ifPresent(servicesManager::register);
    servicesManager.register(new ClasspathHelper());
    this.indexManager = servicesManager.register(new IndexManager(this));
    // prepare those services that require system (single-user) mode
    this.pluginManager = servicesManager.register(new PluginsManagerImpl());
    // Get a manager to handle further collections configuration
    this.collectionConfigurationManager = servicesManager.register(new CollectionConfigurationManager(this));
    this.startupTriggersManager = servicesManager.register(new StartupTriggersManager());
    // this is just used for unit tests
    final BrokerPoolService testBrokerPoolService = (BrokerPoolService) conf.getProperty("exist.testBrokerPoolService");
    if (testBrokerPoolService != null) {
        servicesManager.register(testBrokerPoolService);
    }
    // configure the registered services
    try {
        servicesManager.configureServices(conf);
    } catch (final BrokerPoolServiceException e) {
        throw new EXistException(e);
    }
    // calculate how much memory is reserved for caches to grow
    final Runtime rt = Runtime.getRuntime();
    final long maxMem = rt.maxMemory();
    final long minFree = maxMem / 5;
    reservedMem = cacheManager.getTotalMem() + collectionCache.getMaxCacheSize() + minFree;
    LOG.debug("Reserved memory: {}; max: {}; min: {}", reservedMem, maxMem, minFree);
    // prepare the registered services, before entering system (single-user) mode
    try {
        servicesManager.prepareServices(this);
    } catch (final BrokerPoolServiceException e) {
        throw new EXistException(e);
    }
    // setup database synchronization job
    if (majorSyncPeriod > 0) {
        final SyncTask syncTask = new SyncTask();
        syncTask.configure(conf, null);
        scheduler.createPeriodicJob(2500, new SystemTaskJobImpl(SyncTask.getJobName(), syncTask), 2500);
    }
    try {
        statusReporter = new StatusReporter(SIGNAL_STARTUP);
        statusObservers.forEach(statusReporter::addObserver);
        final Thread statusThread = newInstanceThread(this, "startup-status-reporter", statusReporter);
        statusThread.start();
        // statusReporter may have to be terminated or the thread can/will hang.
        try {
            final boolean exportOnly = conf.getProperty(PROPERTY_EXPORT_ONLY, false);
            // or the FileSyncThread for the journal can/will hang.
            try {
                // Enter System Mode
                try (final DBBroker systemBroker = get(Optional.of(securityManager.getSystemSubject()))) {
                    status.process(Event.INITIALIZE_SYSTEM_MODE);
                    if (isReadOnly()) {
                        journalManager.ifPresent(JournalManager::disableJournalling);
                    }
                    try (final Txn transaction = transactionManager.beginTransaction()) {
                        servicesManager.startPreSystemServices(systemBroker, transaction);
                        transaction.commit();
                    } catch (final BrokerPoolServiceException e) {
                        throw new EXistException(e);
                    }
                    // Run the recovery process
                    boolean recovered = false;
                    if (isRecoveryEnabled()) {
                        recovered = runRecovery(systemBroker);
                        // TODO : extract the following from this block ? What if we are not transactional ? -pb
                        if (!recovered) {
                            try {
                                if (systemBroker.getCollection(XmldbURI.ROOT_COLLECTION_URI) == null) {
                                    final Txn txn = transactionManager.beginTransaction();
                                    try {
                                        systemBroker.getOrCreateCollection(txn, XmldbURI.ROOT_COLLECTION_URI);
                                        transactionManager.commit(txn);
                                    } catch (final IOException | TriggerException | PermissionDeniedException e) {
                                        transactionManager.abort(txn);
                                    } finally {
                                        transactionManager.close(txn);
                                    }
                                }
                            } catch (final PermissionDeniedException pde) {
                                LOG.fatal(pde.getMessage(), pde);
                            }
                        }
                    }
                    /* initialise required collections if they don't exist yet */
                    if (!exportOnly) {
                        try {
                            initialiseSystemCollections(systemBroker);
                        } catch (final PermissionDeniedException pde) {
                            LOG.error(pde.getMessage(), pde);
                            throw new EXistException(pde.getMessage(), pde);
                        }
                    }
                    statusReporter.setStatus(SIGNAL_READINESS);
                    try (final Txn transaction = transactionManager.beginTransaction()) {
                        servicesManager.startSystemServices(systemBroker, transaction);
                        transaction.commit();
                    } catch (final BrokerPoolServiceException e) {
                        throw new EXistException(e);
                    }
                    // TODO : merge this with the recovery process ?
                    if (isRecoveryEnabled() && recovered) {
                        if (!exportOnly) {
                            reportStatus("Reindexing database files...");
                            try {
                                systemBroker.repair();
                            } catch (final PermissionDeniedException e) {
                                LOG.warn("Error during recovery: {}", e.getMessage(), e);
                            }
                        }
                        if ((Boolean) conf.getProperty(PROPERTY_RECOVERY_CHECK)) {
                            final ConsistencyCheckTask task = new ConsistencyCheckTask();
                            final Properties props = new Properties();
                            props.setProperty("backup", "no");
                            props.setProperty("output", "sanity");
                            task.configure(conf, props);
                            try (final Txn transaction = transactionManager.beginTransaction()) {
                                task.execute(systemBroker, transaction);
                                transaction.commit();
                            }
                        }
                    }
                    // OK : the DB is repaired; let's make a few RW operations
                    statusReporter.setStatus(SIGNAL_WRITABLE);
                    // initialize configurations watcher trigger
                    if (!exportOnly) {
                        try {
                            initialiseTriggersForCollections(systemBroker, XmldbURI.SYSTEM_COLLECTION_URI);
                        } catch (final PermissionDeniedException pde) {
                            // XXX: do not catch exception!
                            LOG.error(pde.getMessage(), pde);
                        }
                    }
                    // remove temporary docs
                    try {
                        systemBroker.cleanUpTempResources(true);
                    } catch (final PermissionDeniedException pde) {
                        LOG.error(pde.getMessage(), pde);
                    }
                    sync(systemBroker, Sync.MAJOR);
                    // system mode before entering multi-user mode
                    try (final Txn transaction = transactionManager.beginTransaction()) {
                        servicesManager.startPreMultiUserSystemServices(systemBroker, transaction);
                        transaction.commit();
                    } catch (final BrokerPoolServiceException e) {
                        throw new EXistException(e);
                    }
                }
                // Create the minimal number of brokers required by the configuration
                for (int i = 1; i < minBrokers; i++) {
                    createBroker();
                }
                status.process(Event.INITIALIZE_MULTI_USER_MODE);
                // register some MBeans to provide access to this instance
                AgentFactory.getInstance().initDBInstance(this);
                if (LOG.isDebugEnabled()) {
                    LOG.debug("database instance '{}' initialized", instanceName);
                }
                servicesManager.startMultiUserServices(this);
                status.process(Event.READY);
                statusReporter.setStatus(SIGNAL_STARTED);
            } catch (final Throwable t) {
                transactionManager.shutdown();
                throw t;
            }
        } catch (final EXistException e) {
            throw e;
        } catch (final Throwable t) {
            throw new EXistException(t.getMessage(), t);
        }
    } finally {
        if (statusReporter != null) {
            statusReporter.terminate();
            statusReporter = null;
        }
    }
}

Example 2 with Scheduler

use of org.exist.scheduler.Scheduler in project exist by eXist-db.

the class JournalTest method switchFilesBacksUpOldFileFirst.

@Test
public void switchFilesBacksUpOldFileFirst() throws EXistException, IOException, ReadOnlyException {
    final BrokerPool mockBrokerPool = mock(BrokerPool.class);
    final Configuration mockConfiguration = mock(Configuration.class);
    final Scheduler mockScheduler = createNiceMock(Scheduler.class);
    expect(mockBrokerPool.getConfiguration()).andReturn(mockConfiguration);
    expect(mockConfiguration.getProperty(Journal.PROPERTY_RECOVERY_SYNC_ON_COMMIT, Journal.DEFAULT_SYNC_ON_COMMIT)).andReturn(Journal.DEFAULT_SYNC_ON_COMMIT);
    expect(mockConfiguration.getProperty(Journal.PROPERTY_RECOVERY_JOURNAL_DIR)).andReturn(null);
    expect(mockConfiguration.getProperty(Journal.PROPERTY_RECOVERY_SIZE_MIN, Journal.DEFAULT_MIN_SIZE)).andReturn(Journal.DEFAULT_MIN_SIZE);
    expect(mockConfiguration.getProperty(Journal.PROPERTY_RECOVERY_SIZE_LIMIT, Journal.DEFAULT_MAX_SIZE)).andReturn(Journal.DEFAULT_MAX_SIZE);
    expect(mockBrokerPool.getScheduler()).andReturn(mockScheduler);
    replay(mockBrokerPool, mockConfiguration);
    final Path tempJournalDir = TEMPORARY_FOLDER.newFolder().toPath();
    Files.createDirectories(tempJournalDir);
    assertTrue(Files.exists(tempJournalDir));
    final Journal journal = new Journal(mockBrokerPool, tempJournalDir);
    journal.initialize();
    // journal is not yet operating!
    assertEquals(-1, journal.getCurrentJournalFileNumber());
    // create an existing (old) journal file first
    final Path existingJournalFile = Files.createFile(tempJournalDir.resolve(Journal.getFileName((short) 0)));
    assertTrue(Files.exists(existingJournalFile));
    // start the journal by calling switch files
    journal.switchFiles();
    short currentJournalFileNumber = journal.getCurrentJournalFileNumber();
    assertEquals(0, currentJournalFileNumber);
    final Path journalFile0 = journal.getFile(currentJournalFileNumber);
    assertNotNull(journalFile0);
    assertTrue(Files.exists(journalFile0));
    // check that the existingJournalFile was moved to a backup
    assertTrue(Files.exists(existingJournalFile.resolveSibling(FileUtils.fileName(existingJournalFile) + Journal.BAK_FILE_SUFFIX)));
    verify(mockBrokerPool, mockConfiguration);
}

Example 3 with Scheduler

use of org.exist.scheduler.Scheduler in project exist by eXist-db.

the class BlobStoreRecoveryTest method newBlobDb.

private BlobDb newBlobDb(final Path journalDir, final Path blobDbx, final Path blobDir) throws BrokerPoolServiceException, EXistException {
    final Configuration mockConfiguration = createNiceMock(Configuration.class);
    expect(mockConfiguration.getProperty(Journal.PROPERTY_RECOVERY_JOURNAL_DIR)).andReturn(journalDir);
    expect(mockConfiguration.getProperty(BrokerPool.PROPERTY_RECOVERY_GROUP_COMMIT, false)).andReturn(false);
    expect(mockConfiguration.getProperty(PROPERTY_RECOVERY_SYNC_ON_COMMIT, true)).andReturn(true);
    expect(mockConfiguration.getProperty(PROPERTY_RECOVERY_SIZE_MIN, 1)).andReturn(1);
    expect(mockConfiguration.getProperty(PROPERTY_RECOVERY_SIZE_LIMIT, 100)).andReturn(100);
    replay(mockConfiguration);
    final BrokerPool mockBrokerPool = createNiceMock(BrokerPool.class);
    if (!cleanShutdown) {
        // NOTE: needed so we don't checkpoint at clean shutdown and can simulate a crash!
        mockBrokerPool.FORCE_CORRUPTION = true;
    }
    final SecurityManager mockSecurityManager = createNiceMock(SecurityManager.class);
    final Subject mockSystemSubject = createNiceMock(Subject.class);
    expect(mockBrokerPool.getSecurityManager()).andReturn(mockSecurityManager).anyTimes();
    expect(mockSecurityManager.getSystemSubject()).andReturn(mockSystemSubject).anyTimes();
    replay(mockSecurityManager);
    final JournalManager journalManager = new JournalManager();
    journalManager.configure(mockConfiguration);
    final DBBroker mockSystemBroker = createNiceMock(DBBroker.class);
    final Txn mockSystemTransaction = createNiceMock(Txn.class);
    final SystemTaskManager mockSystemTaskManager = createNiceMock(SystemTaskManager.class);
    mockSystemTaskManager.processTasks(mockSystemBroker, mockSystemTransaction);
    expectLastCall().anyTimes();
    replay(mockSystemTaskManager);
    final DBBroker mockBroker = createNiceMock(DBBroker.class);
    expect(mockBroker.getBrokerPool()).andReturn(mockBrokerPool).anyTimes();
    expect(mockBrokerPool.getBroker()).andReturn(mockBroker).anyTimes();
    replay(mockBroker);
    final TransactionManager transactionManager = new TransactionManager(mockBrokerPool, Optional.of(journalManager), mockSystemTaskManager);
    final Scheduler mockScheduler = createNiceMock(Scheduler.class);
    final BlobStore blobStore = new BlobStoreImpl(mockBrokerPool, blobDbx, blobDir, DIGEST_TYPE);
    expect(mockBrokerPool.getConfiguration()).andReturn(mockConfiguration).anyTimes();
    expect(mockBrokerPool.getScheduler()).andReturn(mockScheduler);
    expect(mockScheduler.createPeriodicJob(anyLong(), anyObject(FileLockHeartBeat.class), anyLong(), anyObject(Properties.class))).andReturn(true);
    expect(mockBrokerPool.getTransactionManager()).andReturn(transactionManager).anyTimes();
    expect(mockBrokerPool.getThreadGroup()).andReturn(Thread.currentThread().getThreadGroup());
    expect(mockBrokerPool.getId()).andReturn("BlobStoreRecoveryTest").times(2);
    expect(mockBrokerPool.getJournalManager()).andReturn(Optional.of(journalManager)).anyTimes();
    expect(mockBrokerPool.getBlobStore()).andReturn(blobStore).anyTimes();
    replay(mockBrokerPool);
    journalManager.prepare(mockBrokerPool);
    final RecoveryManager recoveryManager = new RecoveryManager(mockBroker, journalManager, false);
    recoveryManager.recover();
    return new BlobDb(transactionManager, blobStore);
}

Example 4 with Scheduler

use of org.exist.scheduler.Scheduler in project exist by eXist-db.

the class QuartzSchedulerImpl method prepare.

@Override
public void prepare(final BrokerPool brokerPool) throws BrokerPoolServiceException {
    // NOTE: we create the scheduler in a separate thread with its own thread-group so that the thread group is used by Quartz
    final ThreadGroup instanceQuartzThreadGroup = newInstanceSubThreadGroup(brokerPool, "scheduler.quartz-simple-thread-pool");
    final QuartzSchedulerCreator creator = new QuartzSchedulerCreator(getQuartzProperties());
    final Thread schedulerCreatorThread = new Thread(instanceQuartzThreadGroup, creator, nameInstanceThread(brokerPool, "prepare-quartz-scheduler"));
    schedulerCreatorThread.start();
    try {
        schedulerCreatorThread.join();
        this.scheduler = creator.getScheduler().valueOrThrow(e -> new BrokerPoolServiceException("Unable to create Scheduler: " + e.getMessage(), e));
    } catch (final InterruptedException e) {
        // restore interrupted state
        Thread.currentThread().interrupt();
        throw new BrokerPoolServiceException("Unable to create Scheduler: " + e.getMessage(), e);
    }
}

Example 5 with Scheduler

use of org.exist.scheduler.Scheduler in project exist by eXist-db.

the class JournalTest method switchFiles.

@Test
public void switchFiles() throws EXistException, IOException, ReadOnlyException, InterruptedException {
    final BrokerPool mockBrokerPool = mock(BrokerPool.class);
    final Configuration mockConfiguration = mock(Configuration.class);
    final Scheduler mockScheduler = createNiceMock(Scheduler.class);
    expect(mockBrokerPool.getConfiguration()).andReturn(mockConfiguration);
    expect(mockConfiguration.getProperty(Journal.PROPERTY_RECOVERY_SYNC_ON_COMMIT, Journal.DEFAULT_SYNC_ON_COMMIT)).andReturn(Journal.DEFAULT_SYNC_ON_COMMIT);
    expect(mockConfiguration.getProperty(Journal.PROPERTY_RECOVERY_JOURNAL_DIR)).andReturn(null);
    expect(mockConfiguration.getProperty(Journal.PROPERTY_RECOVERY_SIZE_MIN, Journal.DEFAULT_MIN_SIZE)).andReturn(Journal.DEFAULT_MIN_SIZE);
    expect(mockConfiguration.getProperty(Journal.PROPERTY_RECOVERY_SIZE_LIMIT, Journal.DEFAULT_MAX_SIZE)).andReturn(Journal.DEFAULT_MAX_SIZE);
    expect(mockBrokerPool.getScheduler()).andReturn(mockScheduler);
    replay(mockBrokerPool, mockConfiguration);
    final Path tempJournalDir = TEMPORARY_FOLDER.newFolder().toPath();
    Files.createDirectories(tempJournalDir);
    assertTrue(Files.exists(tempJournalDir));
    final Journal journal = new Journal(mockBrokerPool, tempJournalDir);
    journal.initialize();
    // journal is not yet operating!
    assertEquals(-1, journal.getCurrentJournalFileNumber());
    // start the journal by calling switch files
    journal.switchFiles();
    short currentJournalFileNumber = journal.getCurrentJournalFileNumber();
    assertEquals(0, currentJournalFileNumber);
    final Path journalFile0 = journal.getFile(currentJournalFileNumber);
    assertNotNull(journalFile0);
    assertTrue(Files.exists(journalFile0));
    // advance to a new journal by calling switch files
    journal.switchFiles();
    currentJournalFileNumber = journal.getCurrentJournalFileNumber();
    assertEquals(1, currentJournalFileNumber);
    final Path journalFile1 = journal.getFile(currentJournalFileNumber);
    assertNotNull(journalFile1);
    assertNotEquals(journalFile0.toAbsolutePath().toString(), journalFile1.toAbsolutePath().toString());
    assertTrue(Files.exists(journalFile1));
    verify(mockBrokerPool, mockConfiguration);
}