Examples with ContainerNotAvailableException org.codelibs.fess.exception.ContainerNotAvailableException used on opensource projects

Example 1 with ContainerNotAvailableException

use of org.codelibs.fess.exception.ContainerNotAvailableException in project fess by codelibs.

the class ThumbnailGenerator method main.

public static void main(final String[] args) {
    final Options options = new Options();
    final CmdLineParser parser = new CmdLineParser(options);
    try {
        parser.parseArgument(args);
    } catch (final CmdLineException e) {
        System.err.println(e.getMessage());
        System.err.println("java " + ThumbnailGenerator.class.getCanonicalName() + " [options...] arguments...");
        parser.printUsage(System.err);
        return;
    }
    if (logger.isDebugEnabled()) {
        try {
            ManagementFactory.getRuntimeMXBean().getInputArguments().stream().forEach(s -> logger.debug("Parameter: {}", s));
            System.getProperties().entrySet().stream().forEach(e -> logger.debug("Property: {}={}", e.getKey(), e.getValue()));
            System.getenv().entrySet().forEach(e -> logger.debug("Env: {}={}", e.getKey(), e.getValue()));
            logger.debug("Option: {}", options);
        } catch (final Exception e) {
        // ignore
        }
    }
    final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
    if (StringUtil.isNotBlank(httpAddress)) {
        System.setProperty(FesenClient.HTTP_ADDRESS, httpAddress);
    }
    TimeoutTask systemMonitorTask = null;
    int exitCode;
    try {
        SingletonLaContainerFactory.setConfigPath("app.xml");
        SingletonLaContainerFactory.setExternalContext(new GenericExternalContext());
        SingletonLaContainerFactory.setExternalContextComponentDefRegister(new GenericExternalContextComponentDefRegister());
        SingletonLaContainerFactory.init();
        final Thread shutdownCallback = new Thread("ShutdownHook") {

            @Override
            public void run() {
                if (logger.isDebugEnabled()) {
                    logger.debug("Destroying LaContainer..");
                }
                destroyContainer();
            }
        };
        Runtime.getRuntime().addShutdownHook(shutdownCallback);
        systemMonitorTask = TimeoutManager.getInstance().addTimeoutTarget(new SystemMonitorTarget(), ComponentUtil.getFessConfig().getThumbnailSystemMonitorIntervalAsInteger(), true);
        final int totalCount = process(options);
        if (totalCount != 0) {
            logger.info("Created {} thumbnail files.", totalCount);
        } else {
            logger.info("No new thumbnails found.");
        }
        exitCode = 0;
    } catch (final ContainerNotAvailableException e) {
        if (logger.isDebugEnabled()) {
            logger.debug("ThumbnailGenerator is stopped.", e);
        } else if (logger.isInfoEnabled()) {
            logger.info("ThumbnailGenerator is stopped.");
        }
        exitCode = Constants.EXIT_FAIL;
    } catch (final Throwable t) {
        logger.error("ThumbnailGenerator does not work correctly.", t);
        exitCode = Constants.EXIT_FAIL;
    } finally {
        if (systemMonitorTask != null) {
            systemMonitorTask.cancel();
        }
        destroyContainer();
    }
    System.exit(exitCode);
}

Example 2 with ContainerNotAvailableException

use of org.codelibs.fess.exception.ContainerNotAvailableException in project fess by codelibs.

the class SuggestCreator method main.

public static void main(final String[] args) {
    final Options options = new Options();
    final CmdLineParser parser = new CmdLineParser(options);
    try {
        parser.parseArgument(args);
    } catch (final CmdLineException e) {
        System.err.println(e.getMessage());
        System.err.println("java " + SuggestCreator.class.getCanonicalName() + " [options...] arguments...");
        parser.printUsage(System.err);
        return;
    }
    if (logger.isDebugEnabled()) {
        try {
            ManagementFactory.getRuntimeMXBean().getInputArguments().stream().forEach(s -> logger.debug("Parameter: {}", s));
            System.getProperties().entrySet().stream().forEach(e -> logger.debug("Property: {}={}", e.getKey(), e.getValue()));
            System.getenv().entrySet().forEach(e -> logger.debug("Env: {}={}", e.getKey(), e.getValue()));
            logger.debug("Option: {}", options);
        } catch (final Exception e) {
        // ignore
        }
    }
    final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
    if (StringUtil.isNotBlank(httpAddress)) {
        System.setProperty(FesenClient.HTTP_ADDRESS, httpAddress);
    }
    TimeoutTask systemMonitorTask = null;
    int exitCode;
    try {
        SingletonLaContainerFactory.setConfigPath("app.xml");
        SingletonLaContainerFactory.setExternalContext(new GenericExternalContext());
        SingletonLaContainerFactory.setExternalContextComponentDefRegister(new GenericExternalContextComponentDefRegister());
        SingletonLaContainerFactory.init();
        final Thread shutdownCallback = new Thread("ShutdownHook") {

            @Override
            public void run() {
                if (logger.isDebugEnabled()) {
                    logger.debug("Destroying LaContainer..");
                }
                destroyContainer();
            }
        };
        Runtime.getRuntime().addShutdownHook(shutdownCallback);
        systemMonitorTask = TimeoutManager.getInstance().addTimeoutTarget(new SystemMonitorTarget(), ComponentUtil.getFessConfig().getSuggestSystemMonitorIntervalAsInteger(), true);
        exitCode = process(options);
    } catch (final ContainerNotAvailableException e) {
        if (logger.isDebugEnabled()) {
            logger.debug("SuggestCreator is stopped.", e);
        } else if (logger.isInfoEnabled()) {
            logger.info("SuggestCreator is stopped.");
        }
        exitCode = Constants.EXIT_FAIL;
    } catch (final Throwable t) {
        logger.error("Suggest creator does not work correctly.", t);
        exitCode = Constants.EXIT_FAIL;
    } finally {
        if (systemMonitorTask != null) {
            systemMonitorTask.cancel();
        }
        destroyContainer();
    }
    logger.info("Finished SuggestCreator.");
    System.exit(exitCode);
}

Example 3 with ContainerNotAvailableException

use of org.codelibs.fess.exception.ContainerNotAvailableException in project fess by codelibs.

the class IndexUpdater method run.

@Override
public void run() {
    if (dataService == null) {
        throw new FessSystemException("DataService is null.");
    }
    if (logger.isDebugEnabled()) {
        logger.debug("Starting indexUpdater.");
    }
    executeTime = 0;
    documentSize = 0;
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    final long updateInterval = fessConfig.getIndexerWebfsUpdateIntervalAsInteger().longValue();
    final int maxEmptyListCount = fessConfig.getIndexerWebfsMaxEmptyListCountAsInteger();
    final IntervalControlHelper intervalControlHelper = ComponentUtil.getIntervalControlHelper();
    try {
        final Consumer<SearchRequestBuilder> cb = builder -> {
            final QueryBuilder queryBuilder = QueryBuilders.boolQuery().filter(QueryBuilders.termsQuery(EsAccessResult.SESSION_ID, sessionIdList)).filter(QueryBuilders.termQuery(EsAccessResult.STATUS, org.codelibs.fess.crawler.Constants.OK_STATUS));
            builder.setQuery(queryBuilder);
            builder.setFrom(0);
            final int maxDocumentCacheSize = fessConfig.getIndexerWebfsMaxDocumentCacheSizeAsInteger();
            builder.setSize(maxDocumentCacheSize <= 0 ? 1 : maxDocumentCacheSize);
            builder.addSort(EsAccessResult.CREATE_TIME, SortOrder.ASC);
        };
        final DocList docList = new DocList();
        final List<EsAccessResult> accessResultList = new ArrayList<>();
        long updateTime = System.currentTimeMillis();
        int errorCount = 0;
        int emptyListCount = 0;
        long cleanupTime = -1;
        while (!finishCrawling || !accessResultList.isEmpty()) {
            try {
                final int sessionIdListSize = finishedSessionIdList.size();
                intervalControlHelper.setCrawlerRunning(true);
                updateTime = System.currentTimeMillis() - updateTime;
                final long interval = updateInterval - updateTime;
                if (interval > 0) {
                    // sleep
                    // 10 sec (default)
                    ThreadUtil.sleep(interval);
                }
                systemHelper.calibrateCpuLoad();
                docList.clear();
                accessResultList.clear();
                intervalControlHelper.delayByRules();
                if (logger.isDebugEnabled()) {
                    logger.debug("Processing documents in IndexUpdater queue.");
                }
                updateTime = System.currentTimeMillis();
                List<EsAccessResult> arList = getAccessResultList(cb, cleanupTime);
                if (arList.isEmpty()) {
                    emptyListCount++;
                } else {
                    // reset
                    emptyListCount = 0;
                }
                long hitCount = ((EsResultList<EsAccessResult>) arList).getTotalHits();
                while (hitCount > 0) {
                    if (arList.isEmpty()) {
                        ThreadUtil.sleep(fessConfig.getIndexerWebfsCommitMarginTimeAsInteger().longValue());
                        cleanupTime = -1;
                    } else {
                        processAccessResults(docList, accessResultList, arList);
                        cleanupTime = cleanupAccessResults(accessResultList);
                    }
                    arList = getAccessResultList(cb, cleanupTime);
                    hitCount = ((EsResultList<EsAccessResult>) arList).getTotalHits();
                }
                if (!docList.isEmpty()) {
                    indexingHelper.sendDocuments(searchEngineClient, docList);
                }
                synchronized (finishedSessionIdList) {
                    if (sessionIdListSize != 0 && sessionIdListSize == finishedSessionIdList.size()) {
                        cleanupFinishedSessionData();
                    }
                }
                executeTime += System.currentTimeMillis() - updateTime;
                if (logger.isDebugEnabled()) {
                    logger.debug("Processed documents in IndexUpdater queue.");
                }
                // reset count
                errorCount = 0;
            } catch (final Exception e) {
                if (errorCount > maxErrorCount) {
                    throw e;
                }
                errorCount++;
                logger.warn("Failed to access data. Retry to access it {} times.", errorCount, e);
            } finally {
                if (systemHelper.isForceStop()) {
                    finishCrawling = true;
                    if (logger.isDebugEnabled()) {
                        logger.debug("Stopped indexUpdater.");
                    }
                }
            }
            if (emptyListCount >= maxEmptyListCount) {
                if (logger.isInfoEnabled()) {
                    logger.info("Terminating indexUpdater. emptyListCount is over {}.", maxEmptyListCount);
                }
                // terminate crawling
                finishCrawling = true;
                forceStop();
                if (fessConfig.getIndexerThreadDumpEnabledAsBoolean()) {
                    ThreadDumpUtil.printThreadDump();
                }
                org.codelibs.fess.exec.Crawler.addError("QueueTimeout");
            }
            if (!ComponentUtil.available()) {
                logger.info("IndexUpdater is terminated.");
                forceStop();
                break;
            }
        }
        if (logger.isDebugEnabled()) {
            logger.debug("Finished indexUpdater.");
        }
    } catch (final ContainerNotAvailableException e) {
        if (logger.isDebugEnabled()) {
            logger.error("IndexUpdater is terminated.", e);
        } else if (logger.isInfoEnabled()) {
            logger.info("IndexUpdater is terminated.");
        }
        forceStop();
    } catch (final Throwable t) {
        if (ComponentUtil.available()) {
            logger.error("IndexUpdater is terminated.", t);
        } else if (logger.isDebugEnabled()) {
            logger.error("IndexUpdater is terminated.", t);
            org.codelibs.fess.exec.Crawler.addError(t.getClass().getSimpleName());
        } else if (logger.isInfoEnabled()) {
            logger.info("IndexUpdater is terminated.");
            org.codelibs.fess.exec.Crawler.addError(t.getClass().getSimpleName());
        }
        forceStop();
    } finally {
        intervalControlHelper.setCrawlerRunning(true);
    }
    if (logger.isInfoEnabled()) {
        logger.info("[EXEC TIME] index update time: {}ms", executeTime);
    }
}

Example 4 with ContainerNotAvailableException

use of org.codelibs.fess.exception.ContainerNotAvailableException in project fess by codelibs.

the class FailureUrlService method store.

public void store(final CrawlingConfig crawlingConfig, final String errorName, final String url, final Throwable e) {
    if (e instanceof ContainerNotAvailableException) {
        return;
    }
    final FailureUrlBhv bhv = ComponentUtil.getComponent(FailureUrlBhv.class);
    final FailureUrl failureUrl = bhv.selectEntity(cb -> {
        cb.query().setUrl_Equal(url);
        if (crawlingConfig != null) {
            cb.query().setConfigId_Equal(crawlingConfig.getConfigId());
        }
    }).map(entity -> {
        entity.setErrorCount(entity.getErrorCount() + 1);
        return entity;
    }).orElseGet(() -> {
        final FailureUrl entity = new FailureUrl();
        entity.setErrorCount(1);
        entity.setUrl(url);
        if (crawlingConfig != null) {
            entity.setConfigId(crawlingConfig.getConfigId());
        }
        return entity;
    });
    failureUrl.setErrorName(errorName);
    failureUrl.setErrorLog(StringUtils.abbreviate(getStackTrace(e), 4000));
    failureUrl.setLastAccessTime(ComponentUtil.getSystemHelper().getCurrentTimeAsLong());
    failureUrl.setThreadName(Thread.currentThread().getName());
    bhv.insertOrUpdate(failureUrl, op -> {
        op.setRefreshPolicy(Constants.TRUE);
    });
}

Example 5 with ContainerNotAvailableException

use of org.codelibs.fess.exception.ContainerNotAvailableException in project fess by codelibs.

the class Crawler method main.

public static void main(final String[] args) {
    final Options options = new Options();
    final CmdLineParser parser = new CmdLineParser(options);
    try {
        parser.parseArgument(args);
    } catch (final CmdLineException e) {
        System.err.println(e.getMessage());
        System.err.println("java " + Crawler.class.getCanonicalName() + " [options...] arguments...");
        parser.printUsage(System.err);
        return;
    }
    if (logger.isDebugEnabled()) {
        try {
            ManagementFactory.getRuntimeMXBean().getInputArguments().stream().forEach(s -> logger.debug("Parameter: {}", s));
            System.getProperties().entrySet().stream().forEach(e -> logger.debug("Property: {}={}", e.getKey(), e.getValue()));
            System.getenv().entrySet().forEach(e -> logger.debug("Env: {}={}", e.getKey(), e.getValue()));
            logger.debug("Option: {}", options);
        } catch (final Exception e) {
        // ignore
        }
    }
    initializeProbes();
    final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
    if (StringUtil.isNotBlank(httpAddress)) {
        System.setProperty(FesenClient.HTTP_ADDRESS, httpAddress);
    }
    TimeoutTask systemMonitorTask = null;
    Thread commandThread = null;
    int exitCode;
    try {
        running.set(true);
        SingletonLaContainerFactory.setConfigPath("app.xml");
        SingletonLaContainerFactory.setExternalContext(new GenericExternalContext());
        SingletonLaContainerFactory.setExternalContextComponentDefRegister(new GenericExternalContextComponentDefRegister());
        SingletonLaContainerFactory.init();
        final Thread shutdownCallback = new Thread("ShutdownHook") {

            @Override
            public void run() {
                destroyContainer();
            }
        };
        Runtime.getRuntime().addShutdownHook(shutdownCallback);
        commandThread = new Thread(() -> {
            try (BufferedReader reader = new BufferedReader(new InputStreamReader(System.in))) {
                String command;
                while (true) {
                    try {
                        while (!reader.ready()) {
                            ThreadUtil.sleep(1000L);
                        }
                        command = reader.readLine().trim();
                        if (logger.isDebugEnabled()) {
                            logger.debug("Process command: {}", command);
                        }
                        if (Constants.CRAWLER_PROCESS_COMMAND_THREAD_DUMP.equals(command)) {
                            ThreadDumpUtil.printThreadDump();
                        } else {
                            logger.warn("Unknown process command: {}", command);
                        }
                        if (Thread.interrupted()) {
                            return;
                        }
                    } catch (final InterruptedRuntimeException e) {
                        return;
                    }
                }
            } catch (final IOException e) {
                logger.debug("I/O exception.", e);
            }
        }, "ProcessCommand");
        commandThread.start();
        systemMonitorTask = TimeoutManager.getInstance().addTimeoutTarget(new SystemMonitorTarget(), ComponentUtil.getFessConfig().getCrawlerSystemMonitorIntervalAsInteger(), true);
        exitCode = process(options);
    } catch (final ContainerNotAvailableException e) {
        if (logger.isDebugEnabled()) {
            logger.debug("Crawler is stopped.", e);
        } else if (logger.isInfoEnabled()) {
            logger.info("Crawler is stopped.");
        }
        exitCode = Constants.EXIT_FAIL;
    } catch (final Throwable t) {
        logger.error("Crawler does not work correctly.", t);
        exitCode = Constants.EXIT_FAIL;
    } finally {
        if (commandThread != null && commandThread.isAlive()) {
            commandThread.interrupt();
        }
        if (systemMonitorTask != null) {
            systemMonitorTask.cancel();
        }
        destroyContainer();
    }
    if (exitCode != Constants.EXIT_OK) {
        System.exit(exitCode);
    }
}