Search in sources :

Example 1 with JournalManager

use of org.exist.storage.journal.JournalManager in project exist by eXist-db.

the class BlobStoreImpl method add.

@Override
public Tuple2<BlobId, Long> add(final Txn transaction, final InputStream is) throws IOException {
    if (state.get() != State.OPEN) {
        throw new IOException("Blob Store is not open!");
    }
    // stage the BLOB file
    final Tuple3<Path, Long, MessageDigest> staged = stage(is);
    final BlobVacuum.RequestDeleteStagedBlobFile requestDeleteStagedBlobFile = new BlobVacuum.RequestDeleteStagedBlobFile(stagingDir, staged._1.getFileName().toString());
    // register a callback to cleanup the staged BLOB file ONLY after commit+checkpoint
    final JournalManager journalManager = database.getJournalManager().orElse(null);
    if (journalManager != null) {
        final DeleteStagedBlobFile cleanupStagedBlob = new DeleteStagedBlobFile(vacuumQueue, requestDeleteStagedBlobFile);
        journalManager.listen(cleanupStagedBlob);
        transaction.registerListener(cleanupStagedBlob);
    }
    final BlobId blobId = new BlobId(staged._3.getValue());
    // if the blob entry does not exist, we exclusively compute it as STAGED.
    BlobReference blobReference = references.computeIfAbsent(blobId, k -> new BlobReference(STAGED));
    try {
        while (true) {
            if (blobReference.count.compareAndSet(STAGED, PROMOTING)) {
                // write journal entries to the WAL
                if (journalManager != null) {
                    try {
                        journalManager.journal(new StoreBlobFileLoggable(transaction.getId(), blobId, staged._1.getFileName().toString()));
                        journalManager.journal(new UpdateBlobRefCountLoggable(transaction.getId(), blobId, 0, 1));
                        // force WAL entries to disk!
                        journalManager.flush(true, true);
                    } catch (final JournalException e) {
                        references.remove(blobId);
                        throw new IOException(e);
                    }
                }
                // promote the staged blob
                promote(staged);
                if (journalManager == null) {
                    // no journal (or recovery)... so go ahead and schedule cleanup of the staged blob file
                    enqueueVacuum(vacuumQueue, requestDeleteStagedBlobFile);
                }
                // schedule disk persist of the new value
                persistQueue.put(Tuple(blobId, blobReference, 1));
                // update memory with the new value
                blobReference.count.set(1);
                // done!
                return Tuple(blobId, staged._2);
            }
            final int count = blobReference.count.get();
            // guard against a concurrent #add or #remove
            if (count == PROMOTING || count == UPDATING_COUNT) {
                // spin whilst another thread promotes the blob, or updates the reference count
                // sleep a small time to save CPU
                Thread.sleep(10);
                continue;
            }
            // i.e. wait for the deletion of the blob to complete, and then we can add the blob again
            if (count == DELETING) {
                blobReference = references.computeIfAbsent(blobId, k -> new BlobReference(STAGED));
                // loop again
                continue;
            }
            // only increment the blob reference if the blob is active!
            if (count >= 0 && blobReference.count.compareAndSet(count, UPDATING_COUNT)) {
                // NOTE: we are the only thread that can be in this branch for the blobId
                final int newCount = count + 1;
                // write journal entries to the WAL
                if (journalManager != null) {
                    try {
                        journalManager.journal(new UpdateBlobRefCountLoggable(transaction.getId(), blobId, count, newCount));
                        // force WAL entries to disk!
                        journalManager.flush(true, true);
                    } catch (final JournalException e) {
                        // restore the state of the blobReference first!
                        blobReference.count.set(count);
                        throw new IOException(e);
                    }
                }
                // persist the new value
                persistQueue.put(Tuple(blobId, blobReference, newCount));
                // update memory with the new value, and release other spinning threads
                blobReference.count.set(newCount);
                // done!
                return Tuple(blobId, staged._2);
            }
        }
    } catch (final InterruptedException e) {
        // thrown by persistQueue.put or Thread.sleep
        Thread.currentThread().interrupt();
        throw new IOException(e);
    }
}
Also used : Path(java.nio.file.Path) Tuple3(com.evolvedbinary.j8fu.tuple.Tuple3) Tuple2(com.evolvedbinary.j8fu.tuple.Tuple2) Txn(org.exist.storage.txn.Txn) JournalException(org.exist.storage.journal.JournalException) RawDataBackup(org.exist.backup.RawDataBackup) ByteBuffer(java.nio.ByteBuffer) FileUtils(org.exist.util.FileUtils) Tuple(com.evolvedbinary.j8fu.tuple.Tuple.Tuple) ThreadUtils.nameInstanceThread(org.exist.util.ThreadUtils.nameInstanceThread) DigestInputStream(org.exist.util.crypto.digest.DigestInputStream) StreamableDigest(org.exist.util.crypto.digest.StreamableDigest) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) DigestType(org.exist.util.crypto.digest.DigestType) LOG_UPDATE_BLOB_REF_COUNT(org.exist.storage.blob.BlobLoggable.LOG_UPDATE_BLOB_REF_COUNT) Path(java.nio.file.Path) java.util.concurrent(java.util.concurrent) StandardOpenOption(java.nio.file.StandardOpenOption) FileNotFoundException(java.io.FileNotFoundException) SeekableByteChannel(java.nio.channels.SeekableByteChannel) Logger(org.apache.logging.log4j.Logger) TxnListener(org.exist.storage.txn.TxnListener) LogEntryTypes(org.exist.storage.journal.LogEntryTypes) java.util(java.util) Try(com.evolvedbinary.j8fu.Try) ThreadSafe(net.jcip.annotations.ThreadSafe) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) FilterInputStream(java.io.FilterInputStream) LOG_STORE_BLOB_FILE(org.exist.storage.blob.BlobLoggable.LOG_STORE_BLOB_FILE) ThreadUtils.newInstanceSubThreadGroup(org.exist.util.ThreadUtils.newInstanceSubThreadGroup) LogException(org.exist.storage.journal.LogException) REPLACE_EXISTING(java.nio.file.StandardCopyOption.REPLACE_EXISTING) Nullable(javax.annotation.Nullable) OutputStream(java.io.OutputStream) Database(org.exist.Database) ATOMIC_MOVE(java.nio.file.StandardCopyOption.ATOMIC_MOVE) CountingInputStream(org.apache.commons.io.input.CountingInputStream) Files(java.nio.file.Files) HexEncoder.bytesToHex(org.exist.util.HexEncoder.bytesToHex) JournalManager(org.exist.storage.journal.JournalManager) IOException(java.io.IOException) BlobReference(org.exist.storage.blob.BlobStoreImpl.BlobReference) UUIDGenerator(org.exist.util.UUIDGenerator) MessageDigest(org.exist.util.crypto.digest.MessageDigest) FileUtils.fileName(org.exist.util.FileUtils.fileName) TaggedTryUnchecked(com.evolvedbinary.j8fu.Try.TaggedTryUnchecked) LogManager(org.apache.logging.log4j.LogManager) InputStream(java.io.InputStream) BlobReference(org.exist.storage.blob.BlobStoreImpl.BlobReference) JournalException(org.exist.storage.journal.JournalException) JournalManager(org.exist.storage.journal.JournalManager) IOException(java.io.IOException) MessageDigest(org.exist.util.crypto.digest.MessageDigest)

Example 2 with JournalManager

use of org.exist.storage.journal.JournalManager in project exist by eXist-db.

the class BrokerPool method _initialize.

private void _initialize() throws EXistException, DatabaseConfigurationException {
    this.lockManager = new LockManager(conf, concurrencyLevel);
    // Flag to indicate that we are initializing
    status.process(Event.INITIALIZE);
    if (LOG.isDebugEnabled()) {
        LOG.debug("initializing database instance '{}'...", instanceName);
    }
    // register core broker pool services
    this.scheduler = servicesManager.register(new QuartzSchedulerImpl(this));
    // NOTE: this must occur after the scheduler, and before any other service which requires access to the data directory
    this.dataLock = servicesManager.register(new FileLockService("dbx_dir.lck", BrokerPool.PROPERTY_DATA_DIR, NativeBroker.DEFAULT_DATA_DIR));
    this.securityManager = servicesManager.register(new SecurityManagerImpl(this));
    this.cacheManager = servicesManager.register(new DefaultCacheManager(this));
    this.xQueryPool = servicesManager.register(new XQueryPool());
    this.processMonitor = servicesManager.register(new ProcessMonitor());
    this.xqueryStats = servicesManager.register(new PerformanceStats(this));
    final XMLReaderObjectFactory xmlReaderObjectFactory = servicesManager.register(new XMLReaderObjectFactory());
    this.xmlReaderPool = servicesManager.register(new XMLReaderPool(xmlReaderObjectFactory, maxBrokers, 0));
    final int bufferSize = Optional.of(conf.getInteger(PROPERTY_COLLECTION_CACHE_SIZE)).filter(size -> size != -1).orElse(DEFAULT_COLLECTION_BUFFER_SIZE);
    this.collectionCache = servicesManager.register(new CollectionCache());
    this.notificationService = servicesManager.register(new NotificationService());
    this.journalManager = recoveryEnabled ? Optional.of(new JournalManager()) : Optional.empty();
    journalManager.ifPresent(servicesManager::register);
    final SystemTaskManager systemTaskManager = servicesManager.register(new SystemTaskManager(this));
    this.transactionManager = servicesManager.register(new TransactionManager(this, journalManager, systemTaskManager));
    this.blobStoreService = servicesManager.register(new BlobStoreImplService());
    this.symbols = servicesManager.register(new SymbolTable());
    this.expathRepo = Optional.ofNullable(new ExistRepository());
    expathRepo.ifPresent(servicesManager::register);
    servicesManager.register(new ClasspathHelper());
    this.indexManager = servicesManager.register(new IndexManager(this));
    // prepare those services that require system (single-user) mode
    this.pluginManager = servicesManager.register(new PluginsManagerImpl());
    // Get a manager to handle further collections configuration
    this.collectionConfigurationManager = servicesManager.register(new CollectionConfigurationManager(this));
    this.startupTriggersManager = servicesManager.register(new StartupTriggersManager());
    // this is just used for unit tests
    final BrokerPoolService testBrokerPoolService = (BrokerPoolService) conf.getProperty("exist.testBrokerPoolService");
    if (testBrokerPoolService != null) {
        servicesManager.register(testBrokerPoolService);
    }
    // configure the registered services
    try {
        servicesManager.configureServices(conf);
    } catch (final BrokerPoolServiceException e) {
        throw new EXistException(e);
    }
    // calculate how much memory is reserved for caches to grow
    final Runtime rt = Runtime.getRuntime();
    final long maxMem = rt.maxMemory();
    final long minFree = maxMem / 5;
    reservedMem = cacheManager.getTotalMem() + collectionCache.getMaxCacheSize() + minFree;
    LOG.debug("Reserved memory: {}; max: {}; min: {}", reservedMem, maxMem, minFree);
    // prepare the registered services, before entering system (single-user) mode
    try {
        servicesManager.prepareServices(this);
    } catch (final BrokerPoolServiceException e) {
        throw new EXistException(e);
    }
    // setup database synchronization job
    if (majorSyncPeriod > 0) {
        final SyncTask syncTask = new SyncTask();
        syncTask.configure(conf, null);
        scheduler.createPeriodicJob(2500, new SystemTaskJobImpl(SyncTask.getJobName(), syncTask), 2500);
    }
    try {
        statusReporter = new StatusReporter(SIGNAL_STARTUP);
        statusObservers.forEach(statusReporter::addObserver);
        final Thread statusThread = newInstanceThread(this, "startup-status-reporter", statusReporter);
        statusThread.start();
        // statusReporter may have to be terminated or the thread can/will hang.
        try {
            final boolean exportOnly = conf.getProperty(PROPERTY_EXPORT_ONLY, false);
            // or the FileSyncThread for the journal can/will hang.
            try {
                // Enter System Mode
                try (final DBBroker systemBroker = get(Optional.of(securityManager.getSystemSubject()))) {
                    status.process(Event.INITIALIZE_SYSTEM_MODE);
                    if (isReadOnly()) {
                        journalManager.ifPresent(JournalManager::disableJournalling);
                    }
                    try (final Txn transaction = transactionManager.beginTransaction()) {
                        servicesManager.startPreSystemServices(systemBroker, transaction);
                        transaction.commit();
                    } catch (final BrokerPoolServiceException e) {
                        throw new EXistException(e);
                    }
                    // Run the recovery process
                    boolean recovered = false;
                    if (isRecoveryEnabled()) {
                        recovered = runRecovery(systemBroker);
                        // TODO : extract the following from this block ? What if we are not transactional ? -pb
                        if (!recovered) {
                            try {
                                if (systemBroker.getCollection(XmldbURI.ROOT_COLLECTION_URI) == null) {
                                    final Txn txn = transactionManager.beginTransaction();
                                    try {
                                        systemBroker.getOrCreateCollection(txn, XmldbURI.ROOT_COLLECTION_URI);
                                        transactionManager.commit(txn);
                                    } catch (final IOException | TriggerException | PermissionDeniedException e) {
                                        transactionManager.abort(txn);
                                    } finally {
                                        transactionManager.close(txn);
                                    }
                                }
                            } catch (final PermissionDeniedException pde) {
                                LOG.fatal(pde.getMessage(), pde);
                            }
                        }
                    }
                    /* initialise required collections if they don't exist yet */
                    if (!exportOnly) {
                        try {
                            initialiseSystemCollections(systemBroker);
                        } catch (final PermissionDeniedException pde) {
                            LOG.error(pde.getMessage(), pde);
                            throw new EXistException(pde.getMessage(), pde);
                        }
                    }
                    statusReporter.setStatus(SIGNAL_READINESS);
                    try (final Txn transaction = transactionManager.beginTransaction()) {
                        servicesManager.startSystemServices(systemBroker, transaction);
                        transaction.commit();
                    } catch (final BrokerPoolServiceException e) {
                        throw new EXistException(e);
                    }
                    // TODO : merge this with the recovery process ?
                    if (isRecoveryEnabled() && recovered) {
                        if (!exportOnly) {
                            reportStatus("Reindexing database files...");
                            try {
                                systemBroker.repair();
                            } catch (final PermissionDeniedException e) {
                                LOG.warn("Error during recovery: {}", e.getMessage(), e);
                            }
                        }
                        if ((Boolean) conf.getProperty(PROPERTY_RECOVERY_CHECK)) {
                            final ConsistencyCheckTask task = new ConsistencyCheckTask();
                            final Properties props = new Properties();
                            props.setProperty("backup", "no");
                            props.setProperty("output", "sanity");
                            task.configure(conf, props);
                            try (final Txn transaction = transactionManager.beginTransaction()) {
                                task.execute(systemBroker, transaction);
                                transaction.commit();
                            }
                        }
                    }
                    // OK : the DB is repaired; let's make a few RW operations
                    statusReporter.setStatus(SIGNAL_WRITABLE);
                    // initialize configurations watcher trigger
                    if (!exportOnly) {
                        try {
                            initialiseTriggersForCollections(systemBroker, XmldbURI.SYSTEM_COLLECTION_URI);
                        } catch (final PermissionDeniedException pde) {
                            // XXX: do not catch exception!
                            LOG.error(pde.getMessage(), pde);
                        }
                    }
                    // remove temporary docs
                    try {
                        systemBroker.cleanUpTempResources(true);
                    } catch (final PermissionDeniedException pde) {
                        LOG.error(pde.getMessage(), pde);
                    }
                    sync(systemBroker, Sync.MAJOR);
                    // system mode before entering multi-user mode
                    try (final Txn transaction = transactionManager.beginTransaction()) {
                        servicesManager.startPreMultiUserSystemServices(systemBroker, transaction);
                        transaction.commit();
                    } catch (final BrokerPoolServiceException e) {
                        throw new EXistException(e);
                    }
                }
                // Create the minimal number of brokers required by the configuration
                for (int i = 1; i < minBrokers; i++) {
                    createBroker();
                }
                status.process(Event.INITIALIZE_MULTI_USER_MODE);
                // register some MBeans to provide access to this instance
                AgentFactory.getInstance().initDBInstance(this);
                if (LOG.isDebugEnabled()) {
                    LOG.debug("database instance '{}' initialized", instanceName);
                }
                servicesManager.startMultiUserServices(this);
                status.process(Event.READY);
                statusReporter.setStatus(SIGNAL_STARTED);
            } catch (final Throwable t) {
                transactionManager.shutdown();
                throw t;
            }
        } catch (final EXistException e) {
            throw e;
        } catch (final Throwable t) {
            throw new EXistException(t.getMessage(), t);
        }
    } finally {
        if (statusReporter != null) {
            statusReporter.terminate();
            statusReporter = null;
        }
    }
}
Also used : FSM(com.evolvedbinary.j8fu.fsm.FSM) Txn(org.exist.storage.txn.Txn) Array(java.lang.reflect.Array) BlobStoreService(org.exist.storage.blob.BlobStoreService) ConfigurationDocumentTrigger(org.exist.config.ConfigurationDocumentTrigger) RecoveryManager(org.exist.storage.recovery.RecoveryManager) org.exist.security(org.exist.security) PluginsManager(org.exist.plugin.PluginsManager) ThreadUtils.nameInstanceThreadGroup(org.exist.util.ThreadUtils.nameInstanceThreadGroup) Configurator(org.exist.config.Configurator) ClasspathHelper(org.exist.repo.ClasspathHelper) BlobStoreImplService(org.exist.storage.blob.BlobStoreImplService) Collection(org.exist.collections.Collection) Path(java.nio.file.Path) SystemTaskJobImpl(org.exist.scheduler.impl.SystemTaskJobImpl) PrintWriter(java.io.PrintWriter) SecurityManagerImpl(org.exist.security.internal.SecurityManagerImpl) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) AtomicLazyVal(com.evolvedbinary.j8fu.lazy.AtomicLazyVal) org.exist.collections.triggers(org.exist.collections.triggers) Reference(java.lang.ref.Reference) SecurityManager(org.exist.security.SecurityManager) Logger(org.apache.logging.log4j.Logger) CollectionCache(org.exist.collections.CollectionCache) LockManager(org.exist.storage.lock.LockManager) GuardedBy(net.jcip.annotations.GuardedBy) Debuggee(org.exist.debuggee.Debuggee) Entry(java.util.Map.Entry) AgentFactory(org.exist.management.AgentFactory) ExistRepository(org.exist.repo.ExistRepository) java.util(java.util) ConfigurationClass(org.exist.config.annotation.ConfigurationClass) ThreadSafe(net.jcip.annotations.ThreadSafe) NumberFormat(java.text.NumberFormat) IndexManager(org.exist.indexing.IndexManager) BlobStore(org.exist.storage.blob.BlobStore) ShutdownListener(org.exist.xmldb.ShutdownListener) XmldbURI(org.exist.xmldb.XmldbURI) QuartzSchedulerImpl(org.exist.scheduler.impl.QuartzSchedulerImpl) EXistException(org.exist.EXistException) ThreadUtils.newInstanceThread(org.exist.util.ThreadUtils.newInstanceThread) PluginsManagerImpl(org.exist.plugin.PluginsManagerImpl) FileLockService(org.exist.storage.lock.FileLockService) XQuery(org.exist.xquery.XQuery) CollectionConfiguration(org.exist.collections.CollectionConfiguration) DebuggeeFactory(org.exist.debuggee.DebuggeeFactory) Database(org.exist.Database) ReentrantLock(java.util.concurrent.locks.ReentrantLock) FileStore(java.nio.file.FileStore) JournalManager(org.exist.storage.journal.JournalManager) StringWriter(java.io.StringWriter) Sync(org.exist.storage.sync.Sync) IOException(java.io.IOException) DLNFactory(org.exist.numbering.DLNFactory) TransactionException(org.exist.storage.txn.TransactionException) Field(java.lang.reflect.Field) CollectionConfigurationManager(org.exist.collections.CollectionConfigurationManager) AtomicFSM(com.evolvedbinary.j8fu.fsm.AtomicFSM) SymbolTable(org.exist.dom.persistent.SymbolTable) Consumer(java.util.function.Consumer) Scheduler(org.exist.scheduler.Scheduler) SyncTask(org.exist.storage.sync.SyncTask) TransactionManager(org.exist.storage.txn.TransactionManager) Lock(java.util.concurrent.locks.Lock) PerformanceStats(org.exist.xquery.PerformanceStats) ConcurrentSkipListSet(java.util.concurrent.ConcurrentSkipListSet) org.exist.util(org.exist.util) NodeIdFactory(org.exist.numbering.NodeIdFactory) LogManager(org.apache.logging.log4j.LogManager) TransitionTable.transitionTable(com.evolvedbinary.j8fu.fsm.TransitionTable.transitionTable) ConfigurationFieldAsAttribute(org.exist.config.annotation.ConfigurationFieldAsAttribute) SyncTask(org.exist.storage.sync.SyncTask) SecurityManagerImpl(org.exist.security.internal.SecurityManagerImpl) SystemTaskJobImpl(org.exist.scheduler.impl.SystemTaskJobImpl) Txn(org.exist.storage.txn.Txn) PluginsManagerImpl(org.exist.plugin.PluginsManagerImpl) CollectionCache(org.exist.collections.CollectionCache) QuartzSchedulerImpl(org.exist.scheduler.impl.QuartzSchedulerImpl) ExistRepository(org.exist.repo.ExistRepository) ClasspathHelper(org.exist.repo.ClasspathHelper) JournalManager(org.exist.storage.journal.JournalManager) SymbolTable(org.exist.dom.persistent.SymbolTable) BlobStoreImplService(org.exist.storage.blob.BlobStoreImplService) EXistException(org.exist.EXistException) IOException(java.io.IOException) FileLockService(org.exist.storage.lock.FileLockService) ThreadUtils.newInstanceThread(org.exist.util.ThreadUtils.newInstanceThread) IndexManager(org.exist.indexing.IndexManager) LockManager(org.exist.storage.lock.LockManager) PerformanceStats(org.exist.xquery.PerformanceStats) TransactionManager(org.exist.storage.txn.TransactionManager) CollectionConfigurationManager(org.exist.collections.CollectionConfigurationManager)

Example 3 with JournalManager

use of org.exist.storage.journal.JournalManager in project exist by eXist-db.

the class GetThumbnailsFunction method eval.

/*
	 * (non-Javadoc)
	 * 
	 * @see org.exist.xquery.BasicFunction#eval(org.exist.xquery.value.Sequence[],
	 *      org.exist.xquery.value.Sequence)
	 */
public Sequence eval(Sequence[] args, Sequence contextSequence) throws XPathException {
    ValueSequence result = new ValueSequence();
    // boolean isDatabasePath = false;
    boolean isSaveToDataBase = false;
    if (args[0].isEmpty()) {
        return Sequence.EMPTY_SEQUENCE;
    }
    AnyURIValue picturePath = (AnyURIValue) args[0].itemAt(0);
    if (picturePath.getStringValue().startsWith("xmldb:exist://")) {
        picturePath = new AnyURIValue(picturePath.getStringValue().substring(14));
    }
    AnyURIValue thumbPath = null;
    if (args[1].isEmpty()) {
        thumbPath = new AnyURIValue(picturePath.toXmldbURI().append(THUMBPATH));
        isSaveToDataBase = true;
    } else {
        thumbPath = (AnyURIValue) args[1].itemAt(0);
        if (thumbPath.getStringValue().startsWith("file:")) {
            isSaveToDataBase = false;
            thumbPath = new AnyURIValue(thumbPath.getStringValue().substring(5));
        } else {
            isSaveToDataBase = true;
            try {
                XmldbURI thumbsURI = XmldbURI.xmldbUriFor(thumbPath.getStringValue());
                if (!thumbsURI.isAbsolute())
                    thumbsURI = picturePath.toXmldbURI().append(thumbPath.toString());
                thumbPath = new AnyURIValue(thumbsURI.toString());
            } catch (URISyntaxException e) {
                throw new XPathException(this, e.getMessage());
            }
        }
    }
    // result.add(new StringValue(picturePath.getStringValue()));
    // result.add(new StringValue(thumbPath.getStringValue() + " isDB?= "
    // + isSaveToDataBase));
    int maxThumbHeight = MAXTHUMBHEIGHT;
    int maxThumbWidth = MAXTHUMBWIDTH;
    if (!args[2].isEmpty()) {
        maxThumbHeight = ((IntegerValue) args[2].itemAt(0)).getInt();
        if (args[2].hasMany())
            maxThumbWidth = ((IntegerValue) args[2].itemAt(1)).getInt();
    }
    String prefix = THUMBPREFIX;
    if (!args[3].isEmpty()) {
        prefix = args[3].itemAt(0).getStringValue();
    }
    // result.add(new StringValue("maxThumbHeight = " + maxThumbHeight
    // + ", maxThumbWidth = " + maxThumbWidth));
    BrokerPool pool = null;
    try {
        pool = BrokerPool.getInstance();
    } catch (Exception e) {
        result.add(new StringValue(e.getMessage()));
        return result;
    }
    final DBBroker dbbroker = context.getBroker();
    // Start transaction
    final TransactionManager transact = pool.getTransactionManager();
    try (final Txn transaction = transact.beginTransaction()) {
        Collection thumbCollection = null;
        Path thumbDir = null;
        if (isSaveToDataBase) {
            try {
                thumbCollection = dbbroker.getOrCreateCollection(transaction, thumbPath.toXmldbURI());
                dbbroker.saveCollection(transaction, thumbCollection);
            } catch (Exception e) {
                throw new XPathException(this, e.getMessage());
            }
        } else {
            thumbDir = Paths.get(thumbPath.toString());
            if (!Files.isDirectory(thumbDir))
                try {
                    Files.createDirectories(thumbDir);
                } catch (IOException e) {
                    throw new XPathException(this, e.getMessage());
                }
        }
        Collection allPictures = null;
        Collection existingThumbsCol = null;
        List<Path> existingThumbsArray = null;
        try {
            allPictures = dbbroker.getCollection(picturePath.toXmldbURI());
            if (allPictures == null) {
                return Sequence.EMPTY_SEQUENCE;
            }
            if (isSaveToDataBase) {
                existingThumbsCol = dbbroker.getCollection(thumbPath.toXmldbURI());
            } else {
                existingThumbsArray = FileUtils.list(thumbDir, path -> {
                    final String fileName = FileUtils.fileName(path);
                    return fileName.endsWith(".jpeg") || fileName.endsWith(".jpg");
                });
            }
        } catch (PermissionDeniedException | IOException e) {
            throw new XPathException(this, e.getMessage(), e);
        }
        DocumentImpl docImage = null;
        BinaryDocument binImage = null;
        @SuppressWarnings("unused") BufferedImage bImage = null;
        @SuppressWarnings("unused") byte[] imgData = null;
        Image image = null;
        UnsynchronizedByteArrayOutputStream os = null;
        try {
            Iterator<DocumentImpl> i = allPictures.iterator(dbbroker);
            while (i.hasNext()) {
                docImage = i.next();
                // is not already existing??
                if (!((fileExist(context.getBroker(), existingThumbsCol, docImage, prefix)) || (fileExist(existingThumbsArray, docImage, prefix)))) {
                    if (docImage.getResourceType() == DocumentImpl.BINARY_FILE)
                        // TODO maybe extends for gifs too.
                        if (docImage.getMimeType().startsWith("image/jpeg")) {
                            binImage = (BinaryDocument) docImage;
                            try (final InputStream is = dbbroker.getBinaryResource(transaction, binImage)) {
                                image = ImageIO.read(is);
                            } catch (IOException ioe) {
                                throw new XPathException(this, ioe.getMessage());
                            }
                            try {
                                bImage = ImageModule.createThumb(image, maxThumbHeight, maxThumbWidth);
                            } catch (Exception e) {
                                throw new XPathException(this, e.getMessage());
                            }
                            if (isSaveToDataBase) {
                                os = new UnsynchronizedByteArrayOutputStream();
                                try {
                                    ImageIO.write(bImage, "jpg", os);
                                } catch (Exception e) {
                                    throw new XPathException(this, e.getMessage());
                                }
                                try (final StringInputSource sis = new StringInputSource(os.toByteArray())) {
                                    thumbCollection.storeDocument(transaction, dbbroker, XmldbURI.create(prefix + docImage.getFileURI()), sis, new MimeType("image/jpeg", MimeType.BINARY));
                                } catch (final Exception e) {
                                    throw new XPathException(this, e.getMessage());
                                }
                            // result.add(new
                            // StringValue(""+docImage.getFileURI()+"|"+thumbCollection.getURI()+THUMBPREFIX
                            // + docImage.getFileURI()));
                            } else {
                                try {
                                    ImageIO.write(bImage, "jpg", Paths.get(thumbPath.toString() + "/" + prefix + docImage.getFileURI()).toFile());
                                } catch (Exception e) {
                                    throw new XPathException(this, e.getMessage());
                                }
                            // result.add(new StringValue(
                            // thumbPath.toString() + "/"
                            // + THUMBPREFIX
                            // + docImage.getFileURI()));
                            }
                        }
                } else {
                    // result.add(new StringValue(""+docImage.getURI()+"|"
                    // + ((existingThumbsCol != null) ? ""
                    // + existingThumbsCol.getURI() : thumbDir
                    // .toString()) + "/" + prefix
                    // + docImage.getFileURI()));
                    result.add(new StringValue(docImage.getFileURI().toString()));
                }
            }
        } catch (final PermissionDeniedException | LockException e) {
            throw new XPathException(this, e.getMessage(), e);
        }
        try {
            transact.commit(transaction);
        } catch (Exception e) {
            throw new XPathException(this, e.getMessage());
        }
    }
    final Optional<JournalManager> journalManager = pool.getJournalManager();
    journalManager.ifPresent(j -> j.flush(true, false));
    dbbroker.closeDocument();
    return result;
}
Also used : Txn(org.exist.storage.txn.Txn) BrokerPool(org.exist.storage.BrokerPool) SequenceType(org.exist.xquery.value.SequenceType) FunctionParameterSequenceType(org.exist.xquery.value.FunctionParameterSequenceType) QName(org.exist.dom.QName) URISyntaxException(java.net.URISyntaxException) ValueSequence(org.exist.xquery.value.ValueSequence) StringInputSource(org.exist.util.StringInputSource) FunctionSignature(org.exist.xquery.FunctionSignature) PermissionDeniedException(org.exist.security.PermissionDeniedException) Cardinality(org.exist.xquery.Cardinality) MimeType(org.exist.util.MimeType) FileUtils(org.exist.util.FileUtils) BasicFunction(org.exist.xquery.BasicFunction) StringValue(org.exist.xquery.value.StringValue) UnsynchronizedByteArrayOutputStream(org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream) LockException(org.exist.util.LockException) FunctionReturnSequenceType(org.exist.xquery.value.FunctionReturnSequenceType) ImageIO(javax.imageio.ImageIO) Collection(org.exist.collections.Collection) XmldbURI(org.exist.xmldb.XmldbURI) DocumentImpl(org.exist.dom.persistent.DocumentImpl) Path(java.nio.file.Path) XQueryContext(org.exist.xquery.XQueryContext) Iterator(java.util.Iterator) Image(java.awt.Image) BufferedImage(java.awt.image.BufferedImage) Files(java.nio.file.Files) JournalManager(org.exist.storage.journal.JournalManager) AnyURIValue(org.exist.xquery.value.AnyURIValue) Type(org.exist.xquery.value.Type) IOException(java.io.IOException) TransactionManager(org.exist.storage.txn.TransactionManager) List(java.util.List) Logger(org.apache.logging.log4j.Logger) Paths(java.nio.file.Paths) DBBroker(org.exist.storage.DBBroker) IntegerValue(org.exist.xquery.value.IntegerValue) Optional(java.util.Optional) Sequence(org.exist.xquery.value.Sequence) LogManager(org.apache.logging.log4j.LogManager) BinaryDocument(org.exist.dom.persistent.BinaryDocument) XPathException(org.exist.xquery.XPathException) InputStream(java.io.InputStream) XPathException(org.exist.xquery.XPathException) URISyntaxException(java.net.URISyntaxException) Txn(org.exist.storage.txn.Txn) Image(java.awt.Image) BufferedImage(java.awt.image.BufferedImage) DocumentImpl(org.exist.dom.persistent.DocumentImpl) BufferedImage(java.awt.image.BufferedImage) MimeType(org.exist.util.MimeType) StringInputSource(org.exist.util.StringInputSource) LockException(org.exist.util.LockException) ValueSequence(org.exist.xquery.value.ValueSequence) StringValue(org.exist.xquery.value.StringValue) XmldbURI(org.exist.xmldb.XmldbURI) Path(java.nio.file.Path) InputStream(java.io.InputStream) AnyURIValue(org.exist.xquery.value.AnyURIValue) IntegerValue(org.exist.xquery.value.IntegerValue) JournalManager(org.exist.storage.journal.JournalManager) IOException(java.io.IOException) UnsynchronizedByteArrayOutputStream(org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream) URISyntaxException(java.net.URISyntaxException) PermissionDeniedException(org.exist.security.PermissionDeniedException) LockException(org.exist.util.LockException) IOException(java.io.IOException) XPathException(org.exist.xquery.XPathException) BinaryDocument(org.exist.dom.persistent.BinaryDocument) DBBroker(org.exist.storage.DBBroker) TransactionManager(org.exist.storage.txn.TransactionManager) Collection(org.exist.collections.Collection) PermissionDeniedException(org.exist.security.PermissionDeniedException) BrokerPool(org.exist.storage.BrokerPool)

Example 4 with JournalManager

use of org.exist.storage.journal.JournalManager in project exist by eXist-db.

the class BlobStoreRecoveryTest method newBlobDb.

private BlobDb newBlobDb(final Path journalDir, final Path blobDbx, final Path blobDir) throws BrokerPoolServiceException, EXistException {
    final Configuration mockConfiguration = createNiceMock(Configuration.class);
    expect(mockConfiguration.getProperty(Journal.PROPERTY_RECOVERY_JOURNAL_DIR)).andReturn(journalDir);
    expect(mockConfiguration.getProperty(BrokerPool.PROPERTY_RECOVERY_GROUP_COMMIT, false)).andReturn(false);
    expect(mockConfiguration.getProperty(PROPERTY_RECOVERY_SYNC_ON_COMMIT, true)).andReturn(true);
    expect(mockConfiguration.getProperty(PROPERTY_RECOVERY_SIZE_MIN, 1)).andReturn(1);
    expect(mockConfiguration.getProperty(PROPERTY_RECOVERY_SIZE_LIMIT, 100)).andReturn(100);
    replay(mockConfiguration);
    final BrokerPool mockBrokerPool = createNiceMock(BrokerPool.class);
    if (!cleanShutdown) {
        // NOTE: needed so we don't checkpoint at clean shutdown and can simulate a crash!
        mockBrokerPool.FORCE_CORRUPTION = true;
    }
    final SecurityManager mockSecurityManager = createNiceMock(SecurityManager.class);
    final Subject mockSystemSubject = createNiceMock(Subject.class);
    expect(mockBrokerPool.getSecurityManager()).andReturn(mockSecurityManager).anyTimes();
    expect(mockSecurityManager.getSystemSubject()).andReturn(mockSystemSubject).anyTimes();
    replay(mockSecurityManager);
    final JournalManager journalManager = new JournalManager();
    journalManager.configure(mockConfiguration);
    final DBBroker mockSystemBroker = createNiceMock(DBBroker.class);
    final Txn mockSystemTransaction = createNiceMock(Txn.class);
    final SystemTaskManager mockSystemTaskManager = createNiceMock(SystemTaskManager.class);
    mockSystemTaskManager.processTasks(mockSystemBroker, mockSystemTransaction);
    expectLastCall().anyTimes();
    replay(mockSystemTaskManager);
    final DBBroker mockBroker = createNiceMock(DBBroker.class);
    expect(mockBroker.getBrokerPool()).andReturn(mockBrokerPool).anyTimes();
    expect(mockBrokerPool.getBroker()).andReturn(mockBroker).anyTimes();
    replay(mockBroker);
    final TransactionManager transactionManager = new TransactionManager(mockBrokerPool, Optional.of(journalManager), mockSystemTaskManager);
    final Scheduler mockScheduler = createNiceMock(Scheduler.class);
    final BlobStore blobStore = new BlobStoreImpl(mockBrokerPool, blobDbx, blobDir, DIGEST_TYPE);
    expect(mockBrokerPool.getConfiguration()).andReturn(mockConfiguration).anyTimes();
    expect(mockBrokerPool.getScheduler()).andReturn(mockScheduler);
    expect(mockScheduler.createPeriodicJob(anyLong(), anyObject(FileLockHeartBeat.class), anyLong(), anyObject(Properties.class))).andReturn(true);
    expect(mockBrokerPool.getTransactionManager()).andReturn(transactionManager).anyTimes();
    expect(mockBrokerPool.getThreadGroup()).andReturn(Thread.currentThread().getThreadGroup());
    expect(mockBrokerPool.getId()).andReturn("BlobStoreRecoveryTest").times(2);
    expect(mockBrokerPool.getJournalManager()).andReturn(Optional.of(journalManager)).anyTimes();
    expect(mockBrokerPool.getBlobStore()).andReturn(blobStore).anyTimes();
    replay(mockBrokerPool);
    journalManager.prepare(mockBrokerPool);
    final RecoveryManager recoveryManager = new RecoveryManager(mockBroker, journalManager, false);
    recoveryManager.recover();
    return new BlobDb(transactionManager, blobStore);
}
Also used : Configuration(org.exist.util.Configuration) SecurityManager(org.exist.security.SecurityManager) Scheduler(org.exist.scheduler.Scheduler) JournalManager(org.exist.storage.journal.JournalManager) Txn(org.exist.storage.txn.Txn) Properties(java.util.Properties) Subject(org.exist.security.Subject) FileLockHeartBeat(org.exist.storage.lock.FileLockHeartBeat) RecoveryManager(org.exist.storage.recovery.RecoveryManager) DBBroker(org.exist.storage.DBBroker) TransactionManager(org.exist.storage.txn.TransactionManager) SystemTaskManager(org.exist.storage.SystemTaskManager) BrokerPool(org.exist.storage.BrokerPool)

Example 5 with JournalManager

use of org.exist.storage.journal.JournalManager in project exist by eXist-db.

the class TransactionManagerTestHelper method createTestableTransactionManager.

protected TransactionManager createTestableTransactionManager(final boolean expectTxnClose) throws NoSuchFieldException, IllegalAccessException, EXistException {
    mockBrokerPool = createMock(BrokerPool.class);
    mockBroker = createMock(NativeBroker.class);
    expect(mockBrokerPool.getBroker()).andReturn(mockBroker).atLeastOnce();
    mockBroker.addCurrentTransaction(anyObject());
    expectLastCall().atLeastOnce();
    if (expectTxnClose) {
        mockBroker.removeCurrentTransaction(anyObject());
        expectLastCall().atLeastOnce();
    }
    mockBroker.close();
    expectLastCall().atLeastOnce();
    final SecurityManager mockSecurityManager = createMock(SecurityManager.class);
    final Subject mockSystemSubject = createMock(Subject.class);
    expect(mockBrokerPool.get(Optional.of(mockSystemSubject))).andReturn(mockBroker).anyTimes();
    expect(mockBrokerPool.getSecurityManager()).andReturn(mockSecurityManager).anyTimes();
    expect(mockSecurityManager.getSystemSubject()).andReturn(mockSystemSubject).anyTimes();
    final JournalManager mockJournalManager = createMock(JournalManager.class);
    final SystemTaskManager mockTaskManager = createMock(SystemTaskManager.class);
    replay(mockBrokerPool, mockBroker, mockSecurityManager);
    return new TransactionManager(mockBrokerPool, Optional.of(mockJournalManager), mockTaskManager);
}
Also used : SecurityManager(org.exist.security.SecurityManager) JournalManager(org.exist.storage.journal.JournalManager) NativeBroker(org.exist.storage.NativeBroker) SystemTaskManager(org.exist.storage.SystemTaskManager) BrokerPool(org.exist.storage.BrokerPool) Subject(org.exist.security.Subject)

Aggregations

JournalManager (org.exist.storage.journal.JournalManager)7 IOException (java.io.IOException)5 Txn (org.exist.storage.txn.Txn)4 Path (java.nio.file.Path)3 LogManager (org.apache.logging.log4j.LogManager)3 Logger (org.apache.logging.log4j.Logger)3 SecurityManager (org.exist.security.SecurityManager)3 BrokerPool (org.exist.storage.BrokerPool)3 TransactionManager (org.exist.storage.txn.TransactionManager)3 InputStream (java.io.InputStream)2 Files (java.nio.file.Files)2 java.util (java.util)2 ThreadSafe (net.jcip.annotations.ThreadSafe)2 Database (org.exist.Database)2 Collection (org.exist.collections.Collection)2 Scheduler (org.exist.scheduler.Scheduler)2 DBBroker (org.exist.storage.DBBroker)2 BlobReference (org.exist.storage.blob.BlobStoreImpl.BlobReference)2 JournalException (org.exist.storage.journal.JournalException)2 RecoveryManager (org.exist.storage.recovery.RecoveryManager)2