Examples with StringUtil org.codelibs.core.lang.StringUtil used on opensource projects

Example 16 with StringUtil

use of org.codelibs.core.lang.StringUtil in project fess by codelibs.

the class LdapManager method addUserAttributes.

protected void addUserAttributes(final BasicAttributes entry, final User user, final FessConfig fessConfig) {
    entry.put(new BasicAttribute("cn", user.getName()));
    entry.put(new BasicAttribute("userPassword", user.getOriginalPassword()));
    OptionalUtil.ofNullable(user.getSurname()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrSurname(), s)));
    OptionalUtil.ofNullable(user.getGivenName()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrGivenName(), s)));
    OptionalUtil.ofNullable(user.getMail()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrMail(), s)));
    OptionalUtil.ofNullable(user.getEmployeeNumber()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrEmployeeNumber(), s)));
    OptionalUtil.ofNullable(user.getTelephoneNumber()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrTelephoneNumber(), s)));
    OptionalUtil.ofNullable(user.getHomePhone()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrHomePhone(), s)));
    OptionalUtil.ofNullable(user.getHomePostalAddress()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrHomePostalAddress(), s)));
    OptionalUtil.ofNullable(user.getLabeledURI()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrLabeleduri(), s)));
    OptionalUtil.ofNullable(user.getRoomNumber()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrRoomNumber(), s)));
    OptionalUtil.ofNullable(user.getDescription()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrDescription(), s)));
    OptionalUtil.ofNullable(user.getTitle()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrTitle(), s)));
    OptionalUtil.ofNullable(user.getPager()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrPager(), s)));
    OptionalUtil.ofNullable(user.getStreet()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrStreet(), s)));
    OptionalUtil.ofNullable(user.getPostalCode()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrPostalCode(), s)));
    OptionalUtil.ofNullable(user.getPhysicalDeliveryOfficeName()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrPhysicalDeliveryOfficeName(), s)));
    OptionalUtil.ofNullable(user.getDestinationIndicator()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrDestinationIndicator(), s)));
    OptionalUtil.ofNullable(user.getInternationaliSDNNumber()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrInternationalisdnNumber(), s)));
    OptionalUtil.ofNullable(user.getState()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrState(), s)));
    OptionalUtil.ofNullable(user.getEmployeeType()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrEmployeeType(), s)));
    OptionalUtil.ofNullable(user.getFacsimileTelephoneNumber()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrFacsimileTelephoneNumber(), s)));
    OptionalUtil.ofNullable(user.getPostOfficeBox()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrPostOfficeBox(), s)));
    OptionalUtil.ofNullable(user.getInitials()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrInitials(), s)));
    OptionalUtil.ofNullable(user.getCarLicense()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrCarLicense(), s)));
    OptionalUtil.ofNullable(user.getMobile()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrMobile(), s)));
    OptionalUtil.ofNullable(user.getPostalAddress()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrPostalAddress(), s)));
    OptionalUtil.ofNullable(user.getCity()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrCity(), s)));
    OptionalUtil.ofNullable(user.getTeletexTerminalIdentifier()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrTeletexTerminalIdentifier(), s)));
    OptionalUtil.ofNullable(user.getX121Address()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrX121Address(), s)));
    OptionalUtil.ofNullable(user.getBusinessCategory()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrBusinessCategory(), s)));
    OptionalUtil.ofNullable(user.getRegisteredAddress()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrRegisteredAddress(), s)));
    OptionalUtil.ofNullable(user.getDisplayName()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrDisplayName(), s)));
    OptionalUtil.ofNullable(user.getPreferredLanguage()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrPreferredLanguage(), s)));
    OptionalUtil.ofNullable(user.getDepartmentNumber()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrDepartmentNumber(), s)));
    OptionalUtil.ofNullable(user.getUidNumber()).filter(s -> StringUtil.isNotBlank(s.toString())).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrUidNumber(), s)));
    OptionalUtil.ofNullable(user.getGidNumber()).filter(s -> StringUtil.isNotBlank(s.toString())).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrGidNumber(), s)));
    OptionalUtil.ofNullable(user.getHomeDirectory()).filter(StringUtil::isNotBlank).ifPresent(s -> entry.put(new BasicAttribute(fessConfig.getLdapAttrHomeDirectory(), s)));
}

Example 17 with StringUtil

use of org.codelibs.core.lang.StringUtil in project fess by codelibs.

the class FessProp method getCrawlerDocumentHtmlPrunedTagsAsArray.

public default default PrunedTag[] getCrawlerDocumentHtmlPrunedTagsAsArray() {
    PrunedTag[] tags = (PrunedTag[]) propMap.get("crawlerDocumentHtmlPrunedTags");
    if (tags == null) {
        tags = split(getCrawlerDocumentHtmlPrunedTags(), ",").get(stream -> stream.filter(StringUtil::isNotBlank).map(v -> {
            final String[] cssValues = v.split("\\.", 2);
            final String css;
            if (cssValues.length == 2) {
                css = cssValues[1];
            } else {
                css = null;
            }
            final String[] idValues = cssValues[0].split("#", 2);
            final String id;
            if (idValues.length == 2) {
                id = idValues[1];
            } else {
                id = null;
            }
            return new PrunedTag(idValues[0], id, css);
        }).toArray(n -> new PrunedTag[n]));
        propMap.put("crawlerDocumentHtmlPrunedTags", tags);
    }
    return tags;
}

Example 18 with StringUtil

use of org.codelibs.core.lang.StringUtil in project fess by codelibs.

the class ViewHelper method init.

@PostConstruct
public void init() {
    if (logger.isDebugEnabled()) {
        logger.debug("Initialize {}", this.getClass().getSimpleName());
    }
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    escapedHighlightPre = LaFunctions.h(originalHighlightTagPre);
    escapedHighlightPost = LaFunctions.h(originalHighlightTagPost);
    highlightTagPre = fessConfig.getQueryHighlightTagPre();
    highlightTagPost = fessConfig.getQueryHighlightTagPost();
    highlightedFields = fessConfig.getQueryHighlightContentDescriptionFieldsAsArray();
    for (final int v : fessConfig.getQueryHighlightTerminalCharsAsArray()) {
        highlightTerminalCharSet.add(v);
    }
    try {
        final ServletContext servletContext = ComponentUtil.getComponent(ServletContext.class);
        servletContext.setSessionTrackingModes(fessConfig.getSessionTrackingModesAsSet().stream().map(SessionTrackingMode::valueOf).collect(Collectors.toSet()));
    } catch (final Throwable t) {
        logger.warn("Failed to set SessionTrackingMode.", t);
    }
    split(fessConfig.getQueryFacetQueries(), "\n").of(stream -> stream.map(String::trim).filter(StringUtil::isNotEmpty).forEach(s -> {
        final String[] values = StringUtils.split(s, ":", 2);
        if (values.length != 2) {
            return;
        }
        final FacetQueryView facetQueryView = new FacetQueryView();
        facetQueryView.setTitle(values[0]);
        split(values[1], "\t").of(subStream -> subStream.map(String::trim).filter(StringUtil::isNotEmpty).forEach(v -> {
            final String[] facet = StringUtils.split(v, "=", 2);
            if (facet.length == 2) {
                facetQueryView.addQuery(facet[0], facet[1]);
            }
        }));
        facetQueryView.init();
        facetQueryViewList.add(facetQueryView);
        if (logger.isDebugEnabled()) {
            logger.debug("loaded {}", facetQueryView);
        }
    }));
    facetCache = CacheBuilder.newBuilder().maximumSize(1000).expireAfterWrite(facetCacheDuration, TimeUnit.SECONDS).build();
    textFragmentPrefixLength = fessConfig.getQueryHighlightTextFragmentPrefixLengthAsInteger();
    textFragmentSuffixLength = fessConfig.getQueryHighlightTextFragmentSuffixLengthAsInteger();
    textFragmentSize = fessConfig.getQueryHighlightTextFragmentSizeAsInteger();
}

Example 19 with StringUtil

use of org.codelibs.core.lang.StringUtil in project fess by codelibs.

the class WebFsIndexHelper method doCrawl.

protected void doCrawl(final String sessionId, final List<WebConfig> webConfigList, final List<FileConfig> fileConfigList) {
    final int multiprocessCrawlingCount = ComponentUtil.getFessConfig().getCrawlingThreadCount();
    final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    final long startTime = System.currentTimeMillis();
    final List<String> sessionIdList = new ArrayList<>();
    crawlerList.clear();
    final List<String> crawlerStatusList = new ArrayList<>();
    // Web
    for (final WebConfig webConfig : webConfigList) {
        final String sid = ComponentUtil.getCrawlingConfigHelper().store(sessionId, webConfig);
        // create crawler
        final Crawler crawler = ComponentUtil.getComponent(Crawler.class);
        crawler.setSessionId(sid);
        sessionIdList.add(sid);
        final String urlsStr = webConfig.getUrls();
        if (StringUtil.isBlank(urlsStr)) {
            logger.warn("No target urls. Skipped");
            break;
        }
        // interval time
        final int intervalTime = webConfig.getIntervalTime() != null ? webConfig.getIntervalTime() : Constants.DEFAULT_INTERVAL_TIME_FOR_WEB;
        ((FessIntervalController) crawler.getIntervalController()).setDelayMillisForWaitingNewUrl(intervalTime);
        final String includedUrlsStr = webConfig.getIncludedUrls() != null ? webConfig.getIncludedUrls() : StringUtil.EMPTY;
        final String excludedUrlsStr = webConfig.getExcludedUrls() != null ? webConfig.getExcludedUrls() : StringUtil.EMPTY;
        // num of threads
        final CrawlerContext crawlerContext = crawler.getCrawlerContext();
        final int numOfThread = webConfig.getNumOfThread() != null ? webConfig.getNumOfThread() : Constants.DEFAULT_NUM_OF_THREAD_FOR_WEB;
        crawlerContext.setNumOfThread(numOfThread);
        // depth
        final int depth = webConfig.getDepth() != null ? webConfig.getDepth() : -1;
        crawlerContext.setMaxDepth(depth);
        // max count
        final long maxCount = webConfig.getMaxAccessCount() != null ? webConfig.getMaxAccessCount() : maxAccessCount;
        crawlerContext.setMaxAccessCount(maxCount);
        webConfig.initializeClientFactory(() -> crawler.getClientFactory());
        final Map<String, String> configParamMap = webConfig.getConfigParameterMap(ConfigName.CONFIG);
        if (Constants.TRUE.equalsIgnoreCase(configParamMap.get(Config.CLEANUP_ALL))) {
            deleteCrawlData(sid);
        } else if (Constants.TRUE.equalsIgnoreCase(configParamMap.get(Config.CLEANUP_URL_FILTERS))) {
            final EsUrlFilterService urlFilterService = ComponentUtil.getComponent(EsUrlFilterService.class);
            try {
                urlFilterService.delete(sid);
            } catch (final Exception e) {
                logger.warn("Failed to delete url filters for {}", sid);
            }
        }
        final DuplicateHostHelper duplicateHostHelper = ComponentUtil.getDuplicateHostHelper();
        // set urls
        split(urlsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).distinct().forEach(urlValue -> {
            if (!urlValue.startsWith("#") && fessConfig.isValidCrawlerWebProtocol(urlValue)) {
                final String u = duplicateHostHelper.convert(urlValue);
                crawler.addUrl(u);
                if (logger.isInfoEnabled()) {
                    logger.info("Target URL: {}", u);
                }
            }
        }));
        // set included urls
        split(includedUrlsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).forEach(urlValue -> {
            if (!urlValue.startsWith("#")) {
                crawler.addIncludeFilter(urlValue);
                if (logger.isInfoEnabled()) {
                    logger.info("Included URL: {}", urlValue);
                }
            }
        }));
        // set excluded urls
        split(excludedUrlsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).forEach(urlValue -> {
            if (!urlValue.startsWith("#")) {
                crawler.addExcludeFilter(urlValue);
                if (logger.isInfoEnabled()) {
                    logger.info("Excluded URL: {}", urlValue);
                }
            }
        }));
        // failure url
        final List<String> excludedUrlList = ComponentUtil.getCrawlingConfigHelper().getExcludedUrlList(webConfig.getConfigId());
        if (excludedUrlList != null) {
            excludedUrlList.stream().filter(StringUtil::isNotBlank).map(String::trim).distinct().forEach(u -> {
                final String urlValue = Pattern.quote(u);
                crawler.addExcludeFilter(urlValue);
                if (logger.isInfoEnabled()) {
                    logger.info("Excluded URL from failures: {}", urlValue);
                }
            });
        }
        if (logger.isDebugEnabled()) {
            logger.debug("Crawling {}", urlsStr);
        }
        crawler.setBackground(true);
        crawler.setThreadPriority(crawlerPriority);
        crawlerList.add(crawler);
        crawlerStatusList.add(Constants.READY);
    }
    // File
    for (final FileConfig fileConfig : fileConfigList) {
        final String sid = ComponentUtil.getCrawlingConfigHelper().store(sessionId, fileConfig);
        // create crawler
        final Crawler crawler = ComponentUtil.getComponent(Crawler.class);
        crawler.setSessionId(sid);
        sessionIdList.add(sid);
        final String pathsStr = fileConfig.getPaths();
        if (StringUtil.isBlank(pathsStr)) {
            logger.warn("No target uris. Skipped");
            break;
        }
        final int intervalTime = fileConfig.getIntervalTime() != null ? fileConfig.getIntervalTime() : Constants.DEFAULT_INTERVAL_TIME_FOR_FS;
        ((FessIntervalController) crawler.getIntervalController()).setDelayMillisForWaitingNewUrl(intervalTime);
        final String includedPathsStr = fileConfig.getIncludedPaths() != null ? fileConfig.getIncludedPaths() : StringUtil.EMPTY;
        final String excludedPathsStr = fileConfig.getExcludedPaths() != null ? fileConfig.getExcludedPaths() : StringUtil.EMPTY;
        // num of threads
        final CrawlerContext crawlerContext = crawler.getCrawlerContext();
        final int numOfThread = fileConfig.getNumOfThread() != null ? fileConfig.getNumOfThread() : Constants.DEFAULT_NUM_OF_THREAD_FOR_FS;
        crawlerContext.setNumOfThread(numOfThread);
        // depth
        final int depth = fileConfig.getDepth() != null ? fileConfig.getDepth() : -1;
        crawlerContext.setMaxDepth(depth);
        // max count
        final long maxCount = fileConfig.getMaxAccessCount() != null ? fileConfig.getMaxAccessCount() : maxAccessCount;
        crawlerContext.setMaxAccessCount(maxCount);
        fileConfig.initializeClientFactory(() -> crawler.getClientFactory());
        final Map<String, String> configParamMap = fileConfig.getConfigParameterMap(ConfigName.CONFIG);
        if (Constants.TRUE.equalsIgnoreCase(configParamMap.get(Config.CLEANUP_ALL))) {
            deleteCrawlData(sid);
        } else if (Constants.TRUE.equalsIgnoreCase(configParamMap.get(Config.CLEANUP_URL_FILTERS))) {
            final EsUrlFilterService urlFilterService = ComponentUtil.getComponent(EsUrlFilterService.class);
            try {
                urlFilterService.delete(sid);
            } catch (final Exception e) {
                logger.warn("Failed to delete url filters for {}", sid);
            }
        }
        // set paths
        split(pathsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).distinct().forEach(urlValue -> {
            if (!urlValue.startsWith("#")) {
                final String u;
                if (!fessConfig.isValidCrawlerFileProtocol(urlValue)) {
                    if (urlValue.startsWith("/")) {
                        u = "file:" + urlValue;
                    } else {
                        u = "file:/" + urlValue;
                    }
                } else {
                    u = urlValue;
                }
                crawler.addUrl(u);
                if (logger.isInfoEnabled()) {
                    logger.info("Target Path: {}", u);
                }
            }
        }));
        // set included paths
        final AtomicBoolean urlEncodeDisabled = new AtomicBoolean(false);
        split(includedPathsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).forEach(line -> {
            if (!line.startsWith("#")) {
                final String urlValue;
                if (urlEncodeDisabled.get()) {
                    urlValue = line;
                    urlEncodeDisabled.set(false);
                } else {
                    urlValue = systemHelper.encodeUrlFilter(line);
                }
                crawler.addIncludeFilter(urlValue);
                if (logger.isInfoEnabled()) {
                    logger.info("Included Path: {}", urlValue);
                }
            } else if (line.startsWith("#DISABLE_URL_ENCODE")) {
                urlEncodeDisabled.set(true);
            }
        }));
        // set excluded paths
        urlEncodeDisabled.set(false);
        split(excludedPathsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).forEach(line -> {
            if (!line.startsWith("#")) {
                final String urlValue;
                if (urlEncodeDisabled.get()) {
                    urlValue = line;
                    urlEncodeDisabled.set(false);
                } else {
                    urlValue = systemHelper.encodeUrlFilter(line);
                }
                crawler.addExcludeFilter(urlValue);
                if (logger.isInfoEnabled()) {
                    logger.info("Excluded Path: {}", urlValue);
                }
            } else if (line.startsWith("#DISABLE_URL_ENCODE")) {
                urlEncodeDisabled.set(true);
            }
        }));
        // failure url
        final List<String> excludedUrlList = ComponentUtil.getCrawlingConfigHelper().getExcludedUrlList(fileConfig.getConfigId());
        if (excludedUrlList != null) {
            excludedUrlList.stream().filter(StringUtil::isNotBlank).map(String::trim).distinct().forEach(u -> {
                final String urlValue = Pattern.quote(u);
                crawler.addExcludeFilter(urlValue);
                if (logger.isInfoEnabled()) {
                    logger.info("Excluded Path from failures: {}", urlValue);
                }
            });
        }
        if (logger.isDebugEnabled()) {
            logger.debug("Crawling {}", pathsStr);
        }
        crawler.setBackground(true);
        crawler.setThreadPriority(crawlerPriority);
        crawlerList.add(crawler);
        crawlerStatusList.add(Constants.READY);
    }
    // run index update
    final IndexUpdater indexUpdater = ComponentUtil.getIndexUpdater();
    indexUpdater.setName("IndexUpdater");
    indexUpdater.setPriority(indexUpdaterPriority);
    indexUpdater.setSessionIdList(sessionIdList);
    indexUpdater.setDaemon(true);
    indexUpdater.setCrawlerList(crawlerList);
    getAvailableBoostDocumentRuleList().forEach(rule -> {
        indexUpdater.addDocBoostMatcher(new org.codelibs.fess.indexer.DocBoostMatcher(rule));
    });
    indexUpdater.start();
    int startedCrawlerNum = 0;
    int activeCrawlerNum = 0;
    while (startedCrawlerNum < crawlerList.size()) {
        // Force to stop crawl
        if (systemHelper.isForceStop()) {
            for (final Crawler crawler : crawlerList) {
                crawler.stop();
            }
            break;
        }
        if (activeCrawlerNum < multiprocessCrawlingCount) {
            // start crawling
            crawlerList.get(startedCrawlerNum).execute();
            crawlerStatusList.set(startedCrawlerNum, Constants.RUNNING);
            startedCrawlerNum++;
            activeCrawlerNum++;
            ThreadUtil.sleep(crawlingExecutionInterval);
            continue;
        }
        // check status
        for (int i = 0; i < startedCrawlerNum; i++) {
            if (crawlerList.get(i).getCrawlerContext().getStatus() == CrawlerStatus.DONE && Constants.RUNNING.equals(crawlerStatusList.get(i))) {
                crawlerList.get(i).awaitTermination();
                crawlerStatusList.set(i, Constants.DONE);
                final String sid = crawlerList.get(i).getCrawlerContext().getSessionId();
                indexUpdater.addFinishedSessionId(sid);
                activeCrawlerNum--;
            }
        }
        ThreadUtil.sleep(crawlingExecutionInterval);
    }
    boolean finishedAll = false;
    while (!finishedAll) {
        finishedAll = true;
        for (int i = 0; i < crawlerList.size(); i++) {
            crawlerList.get(i).awaitTermination(crawlingExecutionInterval);
            if (crawlerList.get(i).getCrawlerContext().getStatus() == CrawlerStatus.DONE && !Constants.DONE.equals(crawlerStatusList.get(i))) {
                crawlerStatusList.set(i, Constants.DONE);
                final String sid = crawlerList.get(i).getCrawlerContext().getSessionId();
                indexUpdater.addFinishedSessionId(sid);
            }
            if (!Constants.DONE.equals(crawlerStatusList.get(i))) {
                finishedAll = false;
            }
        }
    }
    crawlerList.clear();
    crawlerStatusList.clear();
    // put cralwing info
    final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
    final long execTime = System.currentTimeMillis() - startTime;
    crawlingInfoHelper.putToInfoMap(Constants.WEB_FS_CRAWLING_EXEC_TIME, Long.toString(execTime));
    if (logger.isInfoEnabled()) {
        logger.info("[EXEC TIME] crawling time: {}ms", execTime);
    }
    indexUpdater.setFinishCrawling(true);
    try {
        indexUpdater.join();
    } catch (final InterruptedException e) {
        logger.warn("Interrupted index update.", e);
    }
    crawlingInfoHelper.putToInfoMap(Constants.WEB_FS_INDEX_EXEC_TIME, Long.toString(indexUpdater.getExecuteTime()));
    crawlingInfoHelper.putToInfoMap(Constants.WEB_FS_INDEX_SIZE, Long.toString(indexUpdater.getDocumentSize()));
    if (systemHelper.isForceStop()) {
        return;
    }
    for (final String sid : sessionIdList) {
        // remove config
        ComponentUtil.getCrawlingConfigHelper().remove(sid);
        deleteCrawlData(sid);
    }
}

Example 20 with StringUtil

use of org.codelibs.core.lang.StringUtil in project fess by codelibs.

the class NotificationHelper method sendToGoogleChat.

protected void sendToGoogleChat(final CardView cardView, final SMailPostingDiscloser discloser) {
    // https://developers.google.com/hangouts/chat/how-tos/webhooks
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    final String googleChatWebhookUrls = fessConfig.getGoogleChatWebhookUrls();
    if (StringUtil.isBlank(googleChatWebhookUrls)) {
        return;
    }
    final String body = toGoogleChatMessage(discloser);
    StreamUtil.split(googleChatWebhookUrls, "[,\\s]").of(stream -> stream.filter(StringUtil::isNotBlank).forEach(url -> {
        try (CurlResponse response = Curl.post(url).header("Content-Type", "application/json").body(body).execute()) {
            if (response.getHttpStatusCode() == 200) {
                if (logger.isDebugEnabled()) {
                    logger.debug("Sent {} to {}.", body, url);
                }
            } else {
                logger.warn("Failed to send {} to {}. HTTP Status is {}. {}", body, url, response.getHttpStatusCode(), response.getContentAsString());
            }
        } catch (final IOException e) {
            logger.warn("Failed to send {} to {}.", body, url, e);
        }
    }));
}