Examples with SearchEngineClient org.codelibs.fess.es.client.SearchEngineClient used on opensource projects

Example 11 with SearchEngineClient

use of org.codelibs.fess.es.client.SearchEngineClient in project fess by codelibs.

the class BaseThumbnailGenerator method process.

protected boolean process(final String id, final BiPredicate<String, String> consumer) {
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
    final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
    try {
        final Map<String, Object> doc = indexingHelper.getDocument(searchEngineClient, id, new String[] { fessConfig.getIndexFieldThumbnail(), fessConfig.getIndexFieldConfigId() });
        if (doc == null) {
            throw new ThumbnailGenerationException("Document is not found: " + id);
        }
        final String url = DocumentUtil.getValue(doc, fessConfig.getIndexFieldThumbnail(), String.class);
        if (StringUtil.isBlank(url)) {
            throw new ThumbnailGenerationException("Invalid thumbnail: " + url);
        }
        final String configId = DocumentUtil.getValue(doc, fessConfig.getIndexFieldConfigId(), String.class);
        if (configId == null || configId.length() < 2) {
            throw new ThumbnailGenerationException("Invalid configId: " + configId);
        }
        return consumer.test(configId, url);
    } catch (final ThumbnailGenerationException e) {
        if (e.getCause() == null) {
            logger.debug(e.getMessage());
        } else {
            logger.warn("Failed to process {}", id, e);
        }
    } catch (final Exception e) {
        logger.warn("Failed to process {}", id, e);
    }
    return false;
}

Example 12 with SearchEngineClient

use of org.codelibs.fess.es.client.SearchEngineClient in project fess by codelibs.

the class SystemMonitorTarget method appendFesenStats.

private void appendFesenStats(final StringBuilder buf) {
    String stats = null;
    try {
        final SearchEngineClient esClient = ComponentUtil.getSearchEngineClient();
        final NodesStatsResponse response = esClient.admin().cluster().prepareNodesStats().all().execute().actionGet(10000L);
        final XContentBuilder builder = XContentFactory.jsonBuilder();
        builder.startObject();
        response.toXContent(builder, ToXContent.EMPTY_PARAMS);
        builder.endObject();
        builder.flush();
        try (OutputStream out = builder.getOutputStream()) {
            stats = ((ByteArrayOutputStream) out).toString(Constants.UTF_8);
        }
    } catch (final Exception e) {
        logger.debug("Failed to access Fesen stats.", e);
    }
    buf.append("\"elasticsearch\":").append(stats).append(',');
}

Example 13 with SearchEngineClient

use of org.codelibs.fess.es.client.SearchEngineClient in project fess by codelibs.

the class FileListIndexUpdateCallbackImpl method deleteDocuments.

protected void deleteDocuments() {
    final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
    final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
    for (final String url : deleteUrlList) {
        indexingHelper.deleteDocumentByUrl(searchEngineClient, url);
    }
    if (logger.isDebugEnabled()) {
        logger.debug("Deleted {}", deleteUrlList);
    }
    deleteUrlList.clear();
}

Example 14 with SearchEngineClient

use of org.codelibs.fess.es.client.SearchEngineClient in project fess by codelibs.

the class IndexUpdateCallbackImpl method store.

/* (non-Javadoc)
     * @see org.codelibs.fess.ds.callback.IndexUpdateCallback#store(java.util.Map)
     */
@Override
public void store(final Map<String, String> paramMap, final Map<String, Object> dataMap) {
    final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
    systemHelper.calibrateCpuLoad();
    final long startTime = System.currentTimeMillis();
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
    if (logger.isDebugEnabled()) {
        logger.debug("Adding {}", dataMap);
    }
    // required check
    final Object urlObj = dataMap.get(fessConfig.getIndexFieldUrl());
    if (urlObj == null) {
        throw new DataStoreException("url is null. dataMap=" + dataMap);
    }
    final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
    final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
    dataMap.put(fessConfig.getIndexFieldId(), crawlingInfoHelper.generateId(dataMap));
    final String url = dataMap.get(fessConfig.getIndexFieldUrl()).toString();
    if (fessConfig.getIndexerClickCountEnabledAsBoolean()) {
        addClickCountField(dataMap, url, fessConfig.getIndexFieldClickCount());
    }
    if (fessConfig.getIndexerFavoriteCountEnabledAsBoolean()) {
        addFavoriteCountField(dataMap, url, fessConfig.getIndexFieldFavoriteCount());
    }
    final Set<String> matchedLabelSet = ComponentUtil.getLabelTypeHelper().getMatchedLabelValueSet(url);
    if (!matchedLabelSet.isEmpty()) {
        final Set<String> newLabelSet = new HashSet<>();
        final String[] oldLabels = DocumentUtil.getValue(dataMap, fessConfig.getIndexFieldLabel(), String[].class);
        StreamUtil.stream(oldLabels).of(stream -> stream.forEach(newLabelSet::add));
        matchedLabelSet.stream().forEach(newLabelSet::add);
        dataMap.put(fessConfig.getIndexFieldLabel(), newLabelSet.toArray(new String[newLabelSet.size()]));
    }
    if (!dataMap.containsKey(fessConfig.getIndexFieldDocId())) {
        dataMap.put(fessConfig.getIndexFieldDocId(), systemHelper.generateDocId(dataMap));
    }
    ComponentUtil.getLanguageHelper().updateDocument(dataMap);
    synchronized (docList) {
        docList.add(ingest(paramMap, dataMap));
        final long contentSize = indexingHelper.calculateDocumentSize(dataMap);
        docList.addContentSize(contentSize);
        final long processingTime = System.currentTimeMillis() - startTime;
        docList.addProcessingTime(processingTime);
        if (logger.isDebugEnabled()) {
            logger.debug("Added the document({}, {}ms). The number of a document cache is {}.", MemoryUtil.byteCountToDisplaySize(contentSize), processingTime, docList.size());
        }
        if (docList.getContentSize() >= maxDocumentRequestSize || docList.size() >= maxDocumentCacheSize) {
            indexingHelper.sendDocuments(searchEngineClient, docList);
        }
        executeTime += processingTime;
    }
    documentSize.getAndIncrement();
    if (logger.isDebugEnabled()) {
        logger.debug("The number of an added document is {}.", documentSize.get());
    }
}

Example 15 with SearchEngineClient

use of org.codelibs.fess.es.client.SearchEngineClient in project fess by codelibs.

the class PingSearchEngineJob method execute.

public String execute() {
    final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
    final StringBuilder resultBuf = new StringBuilder();
    final PingResponse ping = searchEngineClient.ping();
    final int status = ping.getStatus();
    if (systemHelper.isChangedClusterState(status)) {
        if (fessConfig.hasNotification()) {
            final String toStrs = fessConfig.getNotificationTo();
            final String[] toAddresses;
            if (StringUtil.isNotBlank(toStrs)) {
                toAddresses = toStrs.split(",");
            } else {
                toAddresses = StringUtil.EMPTY_STRINGS;
            }
            final Postbox postbox = ComponentUtil.getComponent(Postbox.class);
            try {
                final NotificationHelper notificationHelper = ComponentUtil.getNotificationHelper();
                SMailCallbackContext.setPreparedMessageHookOnThread(notificationHelper::send);
                EsStatusPostcard.droppedInto(postbox, postcard -> {
                    postcard.setFrom(fessConfig.getMailFromAddress(), fessConfig.getMailFromName());
                    postcard.addReplyTo(fessConfig.getMailReturnPath());
                    if (toAddresses.length > 0) {
                        stream(toAddresses).of(stream -> stream.map(String::trim).forEach(address -> {
                            postcard.addTo(address);
                        }));
                    } else {
                        postcard.addTo(fessConfig.getMailFromAddress());
                        postcard.dryrun();
                    }
                    postcard.setHostname(systemHelper.getHostname());
                    postcard.setClustername(ping.getClusterName());
                    postcard.setClusterstatus(ping.getClusterStatus());
                });
            } catch (final Exception e) {
                logger.warn("Failed to send a test mail.", e);
            } finally {
                SMailCallbackContext.clearPreparedMessageHookOnThread();
            }
        }
        resultBuf.append("Status of ").append(ping.getClusterName()).append(" is changed to ").append(ping.getClusterStatus()).append('.');
    } else if (status == 0) {
        resultBuf.append(ping.getClusterName()).append(" is alive.");
    } else {
        resultBuf.append(ping.getClusterName()).append(" is not available.");
    }
    return resultBuf.toString();
}