Examples with MCRProcessableDefaultCollection org.mycore.common.processing.MCRProcessableDefaultCollection used on opensource projects

Example 1 with MCRProcessableDefaultCollection

use of org.mycore.common.processing.MCRProcessableDefaultCollection in project mycore by MyCoRe-Org.

the class MCRImageTiler method run.

/**
 * Starts local tiler threads ( {@link MCRTilingAction}) and gives {@link MCRTileJob} instances to them. Use
 * property <code>MCR.Module-iview2.TilingThreads</code> to specify how many concurrent threads should be running.
 */
public void run() {
    waiter = Thread.currentThread();
    Thread.currentThread().setName("TileMaster");
    // get this MCRSession a speaking name
    MCRSession mcrSession = MCRSessionMgr.getCurrentSession();
    mcrSession.setUserInformation(MCRSystemUserInformation.getSystemUserInstance());
    boolean activated = MCRConfiguration.instance().getBoolean(MCRIView2Tools.CONFIG_PREFIX + "LocalTiler.activated", true) && MCRHIBConnection.isEnabled();
    LOGGER.info("Local Tiling is {}", activated ? "activated" : "deactivated");
    ImageIO.scanForPlugins();
    LOGGER.info("Supported image file types for reading: {}", Arrays.toString(ImageIO.getReaderFormatNames()));
    MCRProcessableDefaultCollection imageTilerCollection = new MCRProcessableDefaultCollection("Image Tiler");
    MCRProcessableRegistry registry = MCRInjectorConfig.injector().getInstance(MCRProcessableRegistry.class);
    registry.register(imageTilerCollection);
    if (activated) {
        int tilingThreadCount = Integer.parseInt(MCRIView2Tools.getIView2Property("TilingThreads"));
        ThreadFactory slaveFactory = new ThreadFactory() {

            AtomicInteger tNum = new AtomicInteger();

            ThreadGroup tg = new ThreadGroup("MCR slave tiling thread group");

            public Thread newThread(Runnable r) {
                return new Thread(tg, r, "TileSlave#" + tNum.incrementAndGet());
            }
        };
        final AtomicInteger activeThreads = new AtomicInteger();
        final LinkedBlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>();
        ThreadPoolExecutor baseExecutor = new ThreadPoolExecutor(tilingThreadCount, tilingThreadCount, 1, TimeUnit.DAYS, workQueue, slaveFactory) {

            @Override
            protected void afterExecute(Runnable r, Throwable t) {
                super.afterExecute(r, t);
                activeThreads.decrementAndGet();
            }

            @Override
            protected void beforeExecute(Thread t, Runnable r) {
                super.beforeExecute(t, r);
                activeThreads.incrementAndGet();
            }
        };
        this.tilingServe = MCRProcessableFactory.newPool(baseExecutor, imageTilerCollection);
        imageTilerCollection.setProperty("running", running);
        LOGGER.info("TilingMaster is started");
        while (running) {
            try {
                while (activeThreads.get() < tilingThreadCount) {
                    runLock.lock();
                    try {
                        if (!running) {
                            break;
                        }
                        Transaction transaction = null;
                        MCRTileJob job = null;
                        try (Session session = MCRHIBConnection.instance().getSession()) {
                            transaction = session.beginTransaction();
                            job = tq.poll();
                            imageTilerCollection.setProperty("queue", tq.stream().map(MCRTileJob::getPath).collect(Collectors.toList()));
                            transaction.commit();
                        } catch (HibernateException e) {
                            LOGGER.error("Error while getting next tiling job.", e);
                            if (transaction != null) {
                                try {
                                    transaction.rollback();
                                } catch (RuntimeException re) {
                                    LOGGER.warn("Could not rollback transaction.", re);
                                }
                            }
                        }
                        if (job != null && !tilingServe.getExecutor().isShutdown()) {
                            LOGGER.info("Creating:{}", job.getPath());
                            tilingServe.submit(getTilingAction(job));
                        } else {
                            try {
                                synchronized (tq) {
                                    if (running) {
                                        LOGGER.debug("No Picture in TilingQueue going to sleep");
                                        // fixes a race conditioned deadlock situation
                                        // do not wait longer than 60 sec. for a new MCRTileJob
                                        tq.wait(60000);
                                    }
                                }
                            } catch (InterruptedException e) {
                                LOGGER.error("Image Tiling thread was interrupted.", e);
                            }
                        }
                    } finally {
                        runLock.unlock();
                    }
                }
                // while(tilingServe.getActiveCount() < tilingServe.getCorePoolSize())
                if (activeThreads.get() < tilingThreadCount)
                    try {
                        LOGGER.info("Waiting for a tiling job to finish");
                        Thread.sleep(1000);
                    } catch (InterruptedException e) {
                        if (running) {
                            LOGGER.error("Image Tiling thread was interrupted.", e);
                        }
                    }
            } catch (Throwable e) {
                LOGGER.error("Keep running while catching exceptions.", e);
            }
        }
        // while(running)
        imageTilerCollection.setProperty("running", false);
    }
    LOGGER.info("Tiling thread finished");
    MCRSessionMgr.releaseCurrentSession();
    waiter = null;
}

Example 2 with MCRProcessableDefaultCollection

use of org.mycore.common.processing.MCRProcessableDefaultCollection in project mycore by MyCoRe-Org.

the class MCRProcessableFactoryTest method newPool.

@Test
public void newPool() throws Exception {
    MCRProcessableRegistry registry = new MCRCentralProcessableRegistry();
    MCRProcessableCollection collection = new MCRProcessableDefaultCollection("test");
    registry.register(collection);
    int nThreads = 3;
    ExecutorService es = Executors.newFixedThreadPool(nThreads);
    MCRProcessableExecutor pes = MCRProcessableFactory.newPool(es, collection);
    assertEquals("No runnables should be queued right now.", 0, collection.stream().count());
    assertEquals("Only the 'test' collection should be registered.", 1, registry.stream().count());
    Semaphore semaphore = new Semaphore(nThreads);
    // lock threads until ready
    semaphore.acquire(nThreads);
    MCRProcessableSupplier<?> sup1 = pes.submit(sleepyThread(semaphore));
    MCRProcessableSupplier<?> sup2 = pes.submit(sleepyThread(semaphore));
    MCRProcessableSupplier<?> sup3 = pes.submit(sleepyThread(semaphore));
    MCRProcessableStatus s1 = sup1.getStatus();
    MCRProcessableStatus s2 = sup2.getStatus();
    MCRProcessableStatus s3 = sup3.getStatus();
    String msgPrefix = "Job should be created or in processing: ";
    assertTrue(msgPrefix + s1, MCRProcessableStatus.processing == s1 || MCRProcessableStatus.created == s1);
    assertTrue(msgPrefix + s2, MCRProcessableStatus.processing == s2 || MCRProcessableStatus.created == s2);
    assertTrue(msgPrefix + s3, MCRProcessableStatus.processing == s3 || MCRProcessableStatus.created == s3);
    assertEquals(3, collection.stream().count());
    // go
    semaphore.release(nThreads);
    CompletableFuture.allOf(sup1.getFuture(), sup2.getFuture(), sup3.getFuture()).get();
    assertEquals(MCRProcessableStatus.successful, sup1.getStatus());
    assertEquals(MCRProcessableStatus.successful, sup2.getStatus());
    assertEquals(MCRProcessableStatus.successful, sup3.getStatus());
    es.shutdown();
    es.awaitTermination(10, TimeUnit.SECONDS);
}