Examples with FessConfig org.codelibs.fess.mylasta.direction.FessConfig used on opensource projects

Example 11 with FessConfig

use of org.codelibs.fess.mylasta.direction.FessConfig in project fess by codelibs.

the class FessCrawlerThread method processResponse.

@Override
protected void processResponse(final UrlQueue<?> urlQueue, final ResponseData responseData) {
    super.processResponse(urlQueue, responseData);
    FessConfig fessConfig = ComponentUtil.getFessConfig();
    if (fessConfig.isCrawlerFailureUrlStatusCodes(responseData.getHttpStatusCode())) {
        String sessionId = crawlerContext.getSessionId();
        final CrawlingConfig crawlingConfig = ComponentUtil.getCrawlingConfigHelper().get(sessionId);
        final String url = urlQueue.getUrl();
        final FailureUrlService failureUrlService = ComponentUtil.getComponent(FailureUrlService.class);
        failureUrlService.store(crawlingConfig, ContentNotFoundException.class.getCanonicalName(), url, new ContentNotFoundException(url));
    }
}

Example 12 with FessConfig

use of org.codelibs.fess.mylasta.direction.FessConfig in project fess by codelibs.

the class FessEsClient method getDocumentList.

public List<Map<String, Object>> getDocumentList(final String index, final String type, final SearchCondition<SearchRequestBuilder> condition) {
    return getDocumentList(index, type, condition, (response, hit) -> {
        final FessConfig fessConfig = ComponentUtil.getFessConfig();
        final Map<String, Object> source = hit.getSource();
        if (source != null) {
            final Map<String, Object> docMap = new HashMap<>(source);
            docMap.put(fessConfig.getIndexFieldId(), hit.getId());
            return docMap;
        }
        final Map<String, SearchHitField> fields = hit.getFields();
        if (fields != null) {
            final Map<String, Object> docMap = fields.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> (Object) e.getValue().getValues()));
            docMap.put(fessConfig.getIndexFieldId(), hit.getId());
            return docMap;
        }
        return null;
    });
}

Example 13 with FessConfig

use of org.codelibs.fess.mylasta.direction.FessConfig in project fess by codelibs.

the class FessEsClient method getDocument.

public OptionalEntity<Map<String, Object>> getDocument(final String index, final String type, final SearchCondition<SearchRequestBuilder> condition) {
    return getDocument(index, type, condition, (response, hit) -> {
        final FessConfig fessConfig = ComponentUtil.getFessConfig();
        final Map<String, Object> source = hit.getSource();
        if (source != null) {
            final Map<String, Object> docMap = new HashMap<>(source);
            docMap.put(fessConfig.getIndexFieldId(), hit.getId());
            docMap.put(fessConfig.getIndexFieldVersion(), hit.getVersion());
            return docMap;
        }
        final Map<String, SearchHitField> fields = hit.getFields();
        if (fields != null) {
            final Map<String, Object> docMap = fields.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> (Object) e.getValue().getValues()));
            docMap.put(fessConfig.getIndexFieldId(), hit.getId());
            docMap.put(fessConfig.getIndexFieldVersion(), hit.getVersion());
            return docMap;
        }
        return null;
    });
}

Example 14 with FessConfig

use of org.codelibs.fess.mylasta.direction.FessConfig in project fess by codelibs.

the class FessEsClient method store.

public boolean store(final String index, final String type, final Object obj) {
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    @SuppressWarnings("unchecked") final Map<String, Object> source = obj instanceof Map ? (Map<String, Object>) obj : BeanUtil.copyBeanToNewMap(obj);
    final String id = (String) source.remove(fessConfig.getIndexFieldId());
    final Long version = (Long) source.remove(fessConfig.getIndexFieldVersion());
    IndexResponse response;
    try {
        if (id == null) {
            // create
            response = client.prepareIndex(index, type).setSource(new DocMap(source)).setRefreshPolicy(RefreshPolicy.IMMEDIATE).setOpType(OpType.CREATE).execute().actionGet(fessConfig.getIndexIndexTimeout());
        } else {
            // create or update
            final IndexRequestBuilder builder = client.prepareIndex(index, type, id).setSource(new DocMap(source)).setRefreshPolicy(RefreshPolicy.IMMEDIATE).setOpType(OpType.INDEX);
            if (version != null && version.longValue() > 0) {
                builder.setVersion(version);
            }
            response = builder.execute().actionGet(fessConfig.getIndexIndexTimeout());
        }
        final Result result = response.getResult();
        return result == Result.CREATED || result == Result.UPDATED;
    } catch (final ElasticsearchException e) {
        throw new FessEsClientException("Failed to store: " + obj, e);
    }
}

Example 15 with FessConfig

use of org.codelibs.fess.mylasta.direction.FessConfig in project fess by codelibs.

the class FileListIndexUpdateCallbackImpl method addDocument.

protected void addDocument(final Map<String, String> paramMap, final Map<String, Object> dataMap) {
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    synchronized (indexUpdateCallback) {
        // required check
        if (!dataMap.containsKey(fessConfig.getIndexFieldUrl()) || dataMap.get(fessConfig.getIndexFieldUrl()) == null) {
            logger.warn("Could not add a doc. Invalid data: " + dataMap);
            return;
        }
        final String url = dataMap.get(fessConfig.getIndexFieldUrl()).toString();
        final CrawlerClient client = crawlerClientFactory.getClient(url);
        if (client == null) {
            logger.warn("CrawlerClient is null. Data: " + dataMap);
            return;
        }
        String processingUrl = url;
        for (int i = 0; i < maxRedirectCount; i++) {
            processingUrl = processRequest(paramMap, dataMap, processingUrl, client);
            if (processingUrl == null) {
                break;
            }
            dataMap.put(fessConfig.getIndexFieldUrl(), processingUrl);
        }
    }
}