use of org.exist.storage.journal.JournalManager in project exist by eXist-db.
the class BlobStoreImpl method add.
@Override
public Tuple2<BlobId, Long> add(final Txn transaction, final InputStream is) throws IOException {
if (state.get() != State.OPEN) {
throw new IOException("Blob Store is not open!");
}
// stage the BLOB file
final Tuple3<Path, Long, MessageDigest> staged = stage(is);
final BlobVacuum.RequestDeleteStagedBlobFile requestDeleteStagedBlobFile = new BlobVacuum.RequestDeleteStagedBlobFile(stagingDir, staged._1.getFileName().toString());
// register a callback to cleanup the staged BLOB file ONLY after commit+checkpoint
final JournalManager journalManager = database.getJournalManager().orElse(null);
if (journalManager != null) {
final DeleteStagedBlobFile cleanupStagedBlob = new DeleteStagedBlobFile(vacuumQueue, requestDeleteStagedBlobFile);
journalManager.listen(cleanupStagedBlob);
transaction.registerListener(cleanupStagedBlob);
}
final BlobId blobId = new BlobId(staged._3.getValue());
// if the blob entry does not exist, we exclusively compute it as STAGED.
BlobReference blobReference = references.computeIfAbsent(blobId, k -> new BlobReference(STAGED));
try {
while (true) {
if (blobReference.count.compareAndSet(STAGED, PROMOTING)) {
// write journal entries to the WAL
if (journalManager != null) {
try {
journalManager.journal(new StoreBlobFileLoggable(transaction.getId(), blobId, staged._1.getFileName().toString()));
journalManager.journal(new UpdateBlobRefCountLoggable(transaction.getId(), blobId, 0, 1));
// force WAL entries to disk!
journalManager.flush(true, true);
} catch (final JournalException e) {
references.remove(blobId);
throw new IOException(e);
}
}
// promote the staged blob
promote(staged);
if (journalManager == null) {
// no journal (or recovery)... so go ahead and schedule cleanup of the staged blob file
enqueueVacuum(vacuumQueue, requestDeleteStagedBlobFile);
}
// schedule disk persist of the new value
persistQueue.put(Tuple(blobId, blobReference, 1));
// update memory with the new value
blobReference.count.set(1);
// done!
return Tuple(blobId, staged._2);
}
final int count = blobReference.count.get();
// guard against a concurrent #add or #remove
if (count == PROMOTING || count == UPDATING_COUNT) {
// spin whilst another thread promotes the blob, or updates the reference count
// sleep a small time to save CPU
Thread.sleep(10);
continue;
}
// i.e. wait for the deletion of the blob to complete, and then we can add the blob again
if (count == DELETING) {
blobReference = references.computeIfAbsent(blobId, k -> new BlobReference(STAGED));
// loop again
continue;
}
// only increment the blob reference if the blob is active!
if (count >= 0 && blobReference.count.compareAndSet(count, UPDATING_COUNT)) {
// NOTE: we are the only thread that can be in this branch for the blobId
final int newCount = count + 1;
// write journal entries to the WAL
if (journalManager != null) {
try {
journalManager.journal(new UpdateBlobRefCountLoggable(transaction.getId(), blobId, count, newCount));
// force WAL entries to disk!
journalManager.flush(true, true);
} catch (final JournalException e) {
// restore the state of the blobReference first!
blobReference.count.set(count);
throw new IOException(e);
}
}
// persist the new value
persistQueue.put(Tuple(blobId, blobReference, newCount));
// update memory with the new value, and release other spinning threads
blobReference.count.set(newCount);
// done!
return Tuple(blobId, staged._2);
}
}
} catch (final InterruptedException e) {
// thrown by persistQueue.put or Thread.sleep
Thread.currentThread().interrupt();
throw new IOException(e);
}
}
use of org.exist.storage.journal.JournalManager in project exist by eXist-db.
the class BrokerPool method _initialize.
private void _initialize() throws EXistException, DatabaseConfigurationException {
this.lockManager = new LockManager(conf, concurrencyLevel);
// Flag to indicate that we are initializing
status.process(Event.INITIALIZE);
if (LOG.isDebugEnabled()) {
LOG.debug("initializing database instance '{}'...", instanceName);
}
// register core broker pool services
this.scheduler = servicesManager.register(new QuartzSchedulerImpl(this));
// NOTE: this must occur after the scheduler, and before any other service which requires access to the data directory
this.dataLock = servicesManager.register(new FileLockService("dbx_dir.lck", BrokerPool.PROPERTY_DATA_DIR, NativeBroker.DEFAULT_DATA_DIR));
this.securityManager = servicesManager.register(new SecurityManagerImpl(this));
this.cacheManager = servicesManager.register(new DefaultCacheManager(this));
this.xQueryPool = servicesManager.register(new XQueryPool());
this.processMonitor = servicesManager.register(new ProcessMonitor());
this.xqueryStats = servicesManager.register(new PerformanceStats(this));
final XMLReaderObjectFactory xmlReaderObjectFactory = servicesManager.register(new XMLReaderObjectFactory());
this.xmlReaderPool = servicesManager.register(new XMLReaderPool(xmlReaderObjectFactory, maxBrokers, 0));
final int bufferSize = Optional.of(conf.getInteger(PROPERTY_COLLECTION_CACHE_SIZE)).filter(size -> size != -1).orElse(DEFAULT_COLLECTION_BUFFER_SIZE);
this.collectionCache = servicesManager.register(new CollectionCache());
this.notificationService = servicesManager.register(new NotificationService());
this.journalManager = recoveryEnabled ? Optional.of(new JournalManager()) : Optional.empty();
journalManager.ifPresent(servicesManager::register);
final SystemTaskManager systemTaskManager = servicesManager.register(new SystemTaskManager(this));
this.transactionManager = servicesManager.register(new TransactionManager(this, journalManager, systemTaskManager));
this.blobStoreService = servicesManager.register(new BlobStoreImplService());
this.symbols = servicesManager.register(new SymbolTable());
this.expathRepo = Optional.ofNullable(new ExistRepository());
expathRepo.ifPresent(servicesManager::register);
servicesManager.register(new ClasspathHelper());
this.indexManager = servicesManager.register(new IndexManager(this));
// prepare those services that require system (single-user) mode
this.pluginManager = servicesManager.register(new PluginsManagerImpl());
// Get a manager to handle further collections configuration
this.collectionConfigurationManager = servicesManager.register(new CollectionConfigurationManager(this));
this.startupTriggersManager = servicesManager.register(new StartupTriggersManager());
// this is just used for unit tests
final BrokerPoolService testBrokerPoolService = (BrokerPoolService) conf.getProperty("exist.testBrokerPoolService");
if (testBrokerPoolService != null) {
servicesManager.register(testBrokerPoolService);
}
// configure the registered services
try {
servicesManager.configureServices(conf);
} catch (final BrokerPoolServiceException e) {
throw new EXistException(e);
}
// calculate how much memory is reserved for caches to grow
final Runtime rt = Runtime.getRuntime();
final long maxMem = rt.maxMemory();
final long minFree = maxMem / 5;
reservedMem = cacheManager.getTotalMem() + collectionCache.getMaxCacheSize() + minFree;
LOG.debug("Reserved memory: {}; max: {}; min: {}", reservedMem, maxMem, minFree);
// prepare the registered services, before entering system (single-user) mode
try {
servicesManager.prepareServices(this);
} catch (final BrokerPoolServiceException e) {
throw new EXistException(e);
}
// setup database synchronization job
if (majorSyncPeriod > 0) {
final SyncTask syncTask = new SyncTask();
syncTask.configure(conf, null);
scheduler.createPeriodicJob(2500, new SystemTaskJobImpl(SyncTask.getJobName(), syncTask), 2500);
}
try {
statusReporter = new StatusReporter(SIGNAL_STARTUP);
statusObservers.forEach(statusReporter::addObserver);
final Thread statusThread = newInstanceThread(this, "startup-status-reporter", statusReporter);
statusThread.start();
// statusReporter may have to be terminated or the thread can/will hang.
try {
final boolean exportOnly = conf.getProperty(PROPERTY_EXPORT_ONLY, false);
// or the FileSyncThread for the journal can/will hang.
try {
// Enter System Mode
try (final DBBroker systemBroker = get(Optional.of(securityManager.getSystemSubject()))) {
status.process(Event.INITIALIZE_SYSTEM_MODE);
if (isReadOnly()) {
journalManager.ifPresent(JournalManager::disableJournalling);
}
try (final Txn transaction = transactionManager.beginTransaction()) {
servicesManager.startPreSystemServices(systemBroker, transaction);
transaction.commit();
} catch (final BrokerPoolServiceException e) {
throw new EXistException(e);
}
// Run the recovery process
boolean recovered = false;
if (isRecoveryEnabled()) {
recovered = runRecovery(systemBroker);
// TODO : extract the following from this block ? What if we are not transactional ? -pb
if (!recovered) {
try {
if (systemBroker.getCollection(XmldbURI.ROOT_COLLECTION_URI) == null) {
final Txn txn = transactionManager.beginTransaction();
try {
systemBroker.getOrCreateCollection(txn, XmldbURI.ROOT_COLLECTION_URI);
transactionManager.commit(txn);
} catch (final IOException | TriggerException | PermissionDeniedException e) {
transactionManager.abort(txn);
} finally {
transactionManager.close(txn);
}
}
} catch (final PermissionDeniedException pde) {
LOG.fatal(pde.getMessage(), pde);
}
}
}
/* initialise required collections if they don't exist yet */
if (!exportOnly) {
try {
initialiseSystemCollections(systemBroker);
} catch (final PermissionDeniedException pde) {
LOG.error(pde.getMessage(), pde);
throw new EXistException(pde.getMessage(), pde);
}
}
statusReporter.setStatus(SIGNAL_READINESS);
try (final Txn transaction = transactionManager.beginTransaction()) {
servicesManager.startSystemServices(systemBroker, transaction);
transaction.commit();
} catch (final BrokerPoolServiceException e) {
throw new EXistException(e);
}
// TODO : merge this with the recovery process ?
if (isRecoveryEnabled() && recovered) {
if (!exportOnly) {
reportStatus("Reindexing database files...");
try {
systemBroker.repair();
} catch (final PermissionDeniedException e) {
LOG.warn("Error during recovery: {}", e.getMessage(), e);
}
}
if ((Boolean) conf.getProperty(PROPERTY_RECOVERY_CHECK)) {
final ConsistencyCheckTask task = new ConsistencyCheckTask();
final Properties props = new Properties();
props.setProperty("backup", "no");
props.setProperty("output", "sanity");
task.configure(conf, props);
try (final Txn transaction = transactionManager.beginTransaction()) {
task.execute(systemBroker, transaction);
transaction.commit();
}
}
}
// OK : the DB is repaired; let's make a few RW operations
statusReporter.setStatus(SIGNAL_WRITABLE);
// initialize configurations watcher trigger
if (!exportOnly) {
try {
initialiseTriggersForCollections(systemBroker, XmldbURI.SYSTEM_COLLECTION_URI);
} catch (final PermissionDeniedException pde) {
// XXX: do not catch exception!
LOG.error(pde.getMessage(), pde);
}
}
// remove temporary docs
try {
systemBroker.cleanUpTempResources(true);
} catch (final PermissionDeniedException pde) {
LOG.error(pde.getMessage(), pde);
}
sync(systemBroker, Sync.MAJOR);
// system mode before entering multi-user mode
try (final Txn transaction = transactionManager.beginTransaction()) {
servicesManager.startPreMultiUserSystemServices(systemBroker, transaction);
transaction.commit();
} catch (final BrokerPoolServiceException e) {
throw new EXistException(e);
}
}
// Create the minimal number of brokers required by the configuration
for (int i = 1; i < minBrokers; i++) {
createBroker();
}
status.process(Event.INITIALIZE_MULTI_USER_MODE);
// register some MBeans to provide access to this instance
AgentFactory.getInstance().initDBInstance(this);
if (LOG.isDebugEnabled()) {
LOG.debug("database instance '{}' initialized", instanceName);
}
servicesManager.startMultiUserServices(this);
status.process(Event.READY);
statusReporter.setStatus(SIGNAL_STARTED);
} catch (final Throwable t) {
transactionManager.shutdown();
throw t;
}
} catch (final EXistException e) {
throw e;
} catch (final Throwable t) {
throw new EXistException(t.getMessage(), t);
}
} finally {
if (statusReporter != null) {
statusReporter.terminate();
statusReporter = null;
}
}
}
use of org.exist.storage.journal.JournalManager in project exist by eXist-db.
the class GetThumbnailsFunction method eval.
/*
* (non-Javadoc)
*
* @see org.exist.xquery.BasicFunction#eval(org.exist.xquery.value.Sequence[],
* org.exist.xquery.value.Sequence)
*/
public Sequence eval(Sequence[] args, Sequence contextSequence) throws XPathException {
ValueSequence result = new ValueSequence();
// boolean isDatabasePath = false;
boolean isSaveToDataBase = false;
if (args[0].isEmpty()) {
return Sequence.EMPTY_SEQUENCE;
}
AnyURIValue picturePath = (AnyURIValue) args[0].itemAt(0);
if (picturePath.getStringValue().startsWith("xmldb:exist://")) {
picturePath = new AnyURIValue(picturePath.getStringValue().substring(14));
}
AnyURIValue thumbPath = null;
if (args[1].isEmpty()) {
thumbPath = new AnyURIValue(picturePath.toXmldbURI().append(THUMBPATH));
isSaveToDataBase = true;
} else {
thumbPath = (AnyURIValue) args[1].itemAt(0);
if (thumbPath.getStringValue().startsWith("file:")) {
isSaveToDataBase = false;
thumbPath = new AnyURIValue(thumbPath.getStringValue().substring(5));
} else {
isSaveToDataBase = true;
try {
XmldbURI thumbsURI = XmldbURI.xmldbUriFor(thumbPath.getStringValue());
if (!thumbsURI.isAbsolute())
thumbsURI = picturePath.toXmldbURI().append(thumbPath.toString());
thumbPath = new AnyURIValue(thumbsURI.toString());
} catch (URISyntaxException e) {
throw new XPathException(this, e.getMessage());
}
}
}
// result.add(new StringValue(picturePath.getStringValue()));
// result.add(new StringValue(thumbPath.getStringValue() + " isDB?= "
// + isSaveToDataBase));
int maxThumbHeight = MAXTHUMBHEIGHT;
int maxThumbWidth = MAXTHUMBWIDTH;
if (!args[2].isEmpty()) {
maxThumbHeight = ((IntegerValue) args[2].itemAt(0)).getInt();
if (args[2].hasMany())
maxThumbWidth = ((IntegerValue) args[2].itemAt(1)).getInt();
}
String prefix = THUMBPREFIX;
if (!args[3].isEmpty()) {
prefix = args[3].itemAt(0).getStringValue();
}
// result.add(new StringValue("maxThumbHeight = " + maxThumbHeight
// + ", maxThumbWidth = " + maxThumbWidth));
BrokerPool pool = null;
try {
pool = BrokerPool.getInstance();
} catch (Exception e) {
result.add(new StringValue(e.getMessage()));
return result;
}
final DBBroker dbbroker = context.getBroker();
// Start transaction
final TransactionManager transact = pool.getTransactionManager();
try (final Txn transaction = transact.beginTransaction()) {
Collection thumbCollection = null;
Path thumbDir = null;
if (isSaveToDataBase) {
try {
thumbCollection = dbbroker.getOrCreateCollection(transaction, thumbPath.toXmldbURI());
dbbroker.saveCollection(transaction, thumbCollection);
} catch (Exception e) {
throw new XPathException(this, e.getMessage());
}
} else {
thumbDir = Paths.get(thumbPath.toString());
if (!Files.isDirectory(thumbDir))
try {
Files.createDirectories(thumbDir);
} catch (IOException e) {
throw new XPathException(this, e.getMessage());
}
}
Collection allPictures = null;
Collection existingThumbsCol = null;
List<Path> existingThumbsArray = null;
try {
allPictures = dbbroker.getCollection(picturePath.toXmldbURI());
if (allPictures == null) {
return Sequence.EMPTY_SEQUENCE;
}
if (isSaveToDataBase) {
existingThumbsCol = dbbroker.getCollection(thumbPath.toXmldbURI());
} else {
existingThumbsArray = FileUtils.list(thumbDir, path -> {
final String fileName = FileUtils.fileName(path);
return fileName.endsWith(".jpeg") || fileName.endsWith(".jpg");
});
}
} catch (PermissionDeniedException | IOException e) {
throw new XPathException(this, e.getMessage(), e);
}
DocumentImpl docImage = null;
BinaryDocument binImage = null;
@SuppressWarnings("unused") BufferedImage bImage = null;
@SuppressWarnings("unused") byte[] imgData = null;
Image image = null;
UnsynchronizedByteArrayOutputStream os = null;
try {
Iterator<DocumentImpl> i = allPictures.iterator(dbbroker);
while (i.hasNext()) {
docImage = i.next();
// is not already existing??
if (!((fileExist(context.getBroker(), existingThumbsCol, docImage, prefix)) || (fileExist(existingThumbsArray, docImage, prefix)))) {
if (docImage.getResourceType() == DocumentImpl.BINARY_FILE)
// TODO maybe extends for gifs too.
if (docImage.getMimeType().startsWith("image/jpeg")) {
binImage = (BinaryDocument) docImage;
try (final InputStream is = dbbroker.getBinaryResource(transaction, binImage)) {
image = ImageIO.read(is);
} catch (IOException ioe) {
throw new XPathException(this, ioe.getMessage());
}
try {
bImage = ImageModule.createThumb(image, maxThumbHeight, maxThumbWidth);
} catch (Exception e) {
throw new XPathException(this, e.getMessage());
}
if (isSaveToDataBase) {
os = new UnsynchronizedByteArrayOutputStream();
try {
ImageIO.write(bImage, "jpg", os);
} catch (Exception e) {
throw new XPathException(this, e.getMessage());
}
try (final StringInputSource sis = new StringInputSource(os.toByteArray())) {
thumbCollection.storeDocument(transaction, dbbroker, XmldbURI.create(prefix + docImage.getFileURI()), sis, new MimeType("image/jpeg", MimeType.BINARY));
} catch (final Exception e) {
throw new XPathException(this, e.getMessage());
}
// result.add(new
// StringValue(""+docImage.getFileURI()+"|"+thumbCollection.getURI()+THUMBPREFIX
// + docImage.getFileURI()));
} else {
try {
ImageIO.write(bImage, "jpg", Paths.get(thumbPath.toString() + "/" + prefix + docImage.getFileURI()).toFile());
} catch (Exception e) {
throw new XPathException(this, e.getMessage());
}
// result.add(new StringValue(
// thumbPath.toString() + "/"
// + THUMBPREFIX
// + docImage.getFileURI()));
}
}
} else {
// result.add(new StringValue(""+docImage.getURI()+"|"
// + ((existingThumbsCol != null) ? ""
// + existingThumbsCol.getURI() : thumbDir
// .toString()) + "/" + prefix
// + docImage.getFileURI()));
result.add(new StringValue(docImage.getFileURI().toString()));
}
}
} catch (final PermissionDeniedException | LockException e) {
throw new XPathException(this, e.getMessage(), e);
}
try {
transact.commit(transaction);
} catch (Exception e) {
throw new XPathException(this, e.getMessage());
}
}
final Optional<JournalManager> journalManager = pool.getJournalManager();
journalManager.ifPresent(j -> j.flush(true, false));
dbbroker.closeDocument();
return result;
}
use of org.exist.storage.journal.JournalManager in project exist by eXist-db.
the class BlobStoreRecoveryTest method newBlobDb.
private BlobDb newBlobDb(final Path journalDir, final Path blobDbx, final Path blobDir) throws BrokerPoolServiceException, EXistException {
final Configuration mockConfiguration = createNiceMock(Configuration.class);
expect(mockConfiguration.getProperty(Journal.PROPERTY_RECOVERY_JOURNAL_DIR)).andReturn(journalDir);
expect(mockConfiguration.getProperty(BrokerPool.PROPERTY_RECOVERY_GROUP_COMMIT, false)).andReturn(false);
expect(mockConfiguration.getProperty(PROPERTY_RECOVERY_SYNC_ON_COMMIT, true)).andReturn(true);
expect(mockConfiguration.getProperty(PROPERTY_RECOVERY_SIZE_MIN, 1)).andReturn(1);
expect(mockConfiguration.getProperty(PROPERTY_RECOVERY_SIZE_LIMIT, 100)).andReturn(100);
replay(mockConfiguration);
final BrokerPool mockBrokerPool = createNiceMock(BrokerPool.class);
if (!cleanShutdown) {
// NOTE: needed so we don't checkpoint at clean shutdown and can simulate a crash!
mockBrokerPool.FORCE_CORRUPTION = true;
}
final SecurityManager mockSecurityManager = createNiceMock(SecurityManager.class);
final Subject mockSystemSubject = createNiceMock(Subject.class);
expect(mockBrokerPool.getSecurityManager()).andReturn(mockSecurityManager).anyTimes();
expect(mockSecurityManager.getSystemSubject()).andReturn(mockSystemSubject).anyTimes();
replay(mockSecurityManager);
final JournalManager journalManager = new JournalManager();
journalManager.configure(mockConfiguration);
final DBBroker mockSystemBroker = createNiceMock(DBBroker.class);
final Txn mockSystemTransaction = createNiceMock(Txn.class);
final SystemTaskManager mockSystemTaskManager = createNiceMock(SystemTaskManager.class);
mockSystemTaskManager.processTasks(mockSystemBroker, mockSystemTransaction);
expectLastCall().anyTimes();
replay(mockSystemTaskManager);
final DBBroker mockBroker = createNiceMock(DBBroker.class);
expect(mockBroker.getBrokerPool()).andReturn(mockBrokerPool).anyTimes();
expect(mockBrokerPool.getBroker()).andReturn(mockBroker).anyTimes();
replay(mockBroker);
final TransactionManager transactionManager = new TransactionManager(mockBrokerPool, Optional.of(journalManager), mockSystemTaskManager);
final Scheduler mockScheduler = createNiceMock(Scheduler.class);
final BlobStore blobStore = new BlobStoreImpl(mockBrokerPool, blobDbx, blobDir, DIGEST_TYPE);
expect(mockBrokerPool.getConfiguration()).andReturn(mockConfiguration).anyTimes();
expect(mockBrokerPool.getScheduler()).andReturn(mockScheduler);
expect(mockScheduler.createPeriodicJob(anyLong(), anyObject(FileLockHeartBeat.class), anyLong(), anyObject(Properties.class))).andReturn(true);
expect(mockBrokerPool.getTransactionManager()).andReturn(transactionManager).anyTimes();
expect(mockBrokerPool.getThreadGroup()).andReturn(Thread.currentThread().getThreadGroup());
expect(mockBrokerPool.getId()).andReturn("BlobStoreRecoveryTest").times(2);
expect(mockBrokerPool.getJournalManager()).andReturn(Optional.of(journalManager)).anyTimes();
expect(mockBrokerPool.getBlobStore()).andReturn(blobStore).anyTimes();
replay(mockBrokerPool);
journalManager.prepare(mockBrokerPool);
final RecoveryManager recoveryManager = new RecoveryManager(mockBroker, journalManager, false);
recoveryManager.recover();
return new BlobDb(transactionManager, blobStore);
}
use of org.exist.storage.journal.JournalManager in project exist by eXist-db.
the class TransactionManagerTestHelper method createTestableTransactionManager.
protected TransactionManager createTestableTransactionManager(final boolean expectTxnClose) throws NoSuchFieldException, IllegalAccessException, EXistException {
mockBrokerPool = createMock(BrokerPool.class);
mockBroker = createMock(NativeBroker.class);
expect(mockBrokerPool.getBroker()).andReturn(mockBroker).atLeastOnce();
mockBroker.addCurrentTransaction(anyObject());
expectLastCall().atLeastOnce();
if (expectTxnClose) {
mockBroker.removeCurrentTransaction(anyObject());
expectLastCall().atLeastOnce();
}
mockBroker.close();
expectLastCall().atLeastOnce();
final SecurityManager mockSecurityManager = createMock(SecurityManager.class);
final Subject mockSystemSubject = createMock(Subject.class);
expect(mockBrokerPool.get(Optional.of(mockSystemSubject))).andReturn(mockBroker).anyTimes();
expect(mockBrokerPool.getSecurityManager()).andReturn(mockSecurityManager).anyTimes();
expect(mockSecurityManager.getSystemSubject()).andReturn(mockSystemSubject).anyTimes();
final JournalManager mockJournalManager = createMock(JournalManager.class);
final SystemTaskManager mockTaskManager = createMock(SystemTaskManager.class);
replay(mockBrokerPool, mockBroker, mockSecurityManager);
return new TransactionManager(mockBrokerPool, Optional.of(mockJournalManager), mockTaskManager);
}
Aggregations