Examples with EsAccessException org.codelibs.fess.crawler.exception.EsAccessException used on opensource projects

Example 6 with EsAccessException

use of org.codelibs.fess.crawler.exception.EsAccessException in project fess-crawler by codelibs.

the class AbstractCrawlerService method getList.

protected <T> List<T> getList(final Class<T> clazz, final Consumer<SearchRequestBuilder> callback) {
    final SearchResponse response = getClient().get(c -> {
        final SearchRequestBuilder builder = c.prepareSearch(index).setTypes(type);
        callback.accept(builder);
        return builder.execute();
    });
    final EsResultList<T> targetList = new EsResultList<>();
    final SearchHits hits = response.getHits();
    targetList.setTotalHits(hits.getTotalHits());
    targetList.setTookInMillis(response.getTook().getMillis());
    if (hits.getTotalHits() != 0) {
        try {
            for (final SearchHit searchHit : hits.getHits()) {
                final Map<String, Object> source = searchHit.getSourceAsMap();
                final T target = BeanUtil.copyMapToNewBean(source, clazz, option -> {
                    option.converter(new EsTimestampConverter(), timestampFields).excludeWhitespace();
                    option.exclude(EsAccessResult.ACCESS_RESULT_DATA);
                });
                @SuppressWarnings("unchecked") final Map<String, Object> data = (Map<String, Object>) source.get(EsAccessResult.ACCESS_RESULT_DATA);
                if (data != null) {
                    ((EsAccessResult) target).setAccessResultData(new EsAccessResultData(data));
                }
                setId(target, searchHit.getId());
                targetList.add(target);
            }
        } catch (final Exception e) {
            throw new EsAccessException("response: " + response, e);
        }
    }
    return targetList;
}

Example 7 with EsAccessException

use of org.codelibs.fess.crawler.exception.EsAccessException in project fess-crawler by codelibs.

the class AbstractCrawlerService method insertAll.

protected <T> void insertAll(final List<T> list, final OpType opType, final boolean ignoreAlreadyExists) {
    final List<T> bufferedList = new ArrayList<>(bulkBufferSize);
    final StringBuilder failureBuf = new StringBuilder(100);
    list.stream().forEach(target -> {
        bufferedList.add(target);
        if (bufferedList.size() >= bulkBufferSize) {
            final BulkResponse response = doInsertAll(bufferedList, opType);
            if (response.hasFailures()) {
                final String failureMessage = buildFailureMessage(response, ignoreAlreadyExists);
                if (failureMessage.length() > 0) {
                    failureBuf.append(response.buildFailureMessage()).append('\n');
                }
            }
            bufferedList.clear();
        }
    });
    if (!bufferedList.isEmpty()) {
        final BulkResponse response = doInsertAll(bufferedList, opType);
        if (response.hasFailures()) {
            final String failureMessage = buildFailureMessage(response, ignoreAlreadyExists);
            if (failureMessage.length() > 0) {
                failureBuf.append(response.buildFailureMessage()).append('\n');
            }
        }
    }
    if (failureBuf.length() > 0) {
        throw new EsAccessException(failureBuf.toString());
    }
}

Example 8 with EsAccessException

use of org.codelibs.fess.crawler.exception.EsAccessException in project fess-crawler by codelibs.

the class EsDataService method getAccessResultList.

public List<EsAccessResult> getAccessResultList(final Consumer<SearchRequestBuilder> callback) {
    final SearchResponse response = getClient().get(c -> {
        final SearchRequestBuilder builder = c.prepareSearch(index).setTypes(type);
        callback.accept(builder);
        builder.setFetchSource(new String[] { "parentUrl", "method", "mimeType", "sessionId", "url", "executionTime", "createTime", "contentLength", "lastModified", "ruleId", "httpStatusCode", "status" }, null);
        return builder.execute();
    });
    final EsResultList<EsAccessResult> targetList = new EsResultList<>();
    final SearchHits hits = response.getHits();
    targetList.setTotalHits(hits.getTotalHits());
    targetList.setTookInMillis(response.getTook().getMillis());
    if (hits.getTotalHits() != 0) {
        try {
            for (final SearchHit searchHit : hits.getHits()) {
                final EsAccessResult target = new EsAccessResult();
                final Map<String, Object> fields = searchHit.getSourceAsMap();
                target.setParentUrl(getFieldValue(fields.get("parentUrl"), String.class));
                target.setMethod(getFieldValue(fields.get("method"), String.class));
                target.setMimeType(getFieldValue(fields.get("mimeType"), String.class));
                target.setSessionId(getFieldValue(fields.get("sessionId"), String.class));
                target.setUrl(getFieldValue(fields.get("url"), String.class));
                target.setExecutionTime(getFieldValue(fields.get("executionTime"), Integer.class));
                target.setContentLength(getFieldValue(fields.get("contentLength"), Long.class));
                target.setRuleId(getFieldValue(fields.get("ruleId"), String.class));
                target.setHttpStatusCode(getFieldValue(fields.get("httpStatusCode"), Integer.class));
                target.setStatus(getFieldValue(fields.get("status"), Integer.class));
                target.setCreateTime(getFieldValue(fields.get("createTime"), Long.class));
                target.setLastModified(getFieldValue(fields.get("lastModified"), Long.class));
                setId(target, searchHit.getId());
                targetList.add(target);
            }
        } catch (final Exception e) {
            throw new EsAccessException("response: " + response, e);
        }
    }
    return targetList;
}

Example 9 with EsAccessException

use of org.codelibs.fess.crawler.exception.EsAccessException in project fess-crawler by codelibs.

the class EsClient method deleteByQuery.

public int deleteByQuery(final String index, final String type, final QueryBuilder queryBuilder) {
    int count = 0;
    String scrollId = null;
    while (true) {
        final SearchResponse scrollResponse;
        if (scrollId == null) {
            scrollResponse = get(c -> c.prepareSearch(index).setTypes(type).setScroll(scrollForDelete).setSize(sizeForDelete).setQuery(queryBuilder).execute());
        } else {
            final String sid = scrollId;
            scrollResponse = get(c -> c.prepareSearchScroll(sid).setScroll(scrollForDelete).execute());
        }
        final SearchHit[] hits = scrollResponse.getHits().getHits();
        if (hits.length == 0) {
            break;
        }
        scrollId = scrollResponse.getScrollId();
        count += hits.length;
        final BulkResponse bulkResponse = get(c -> {
            final BulkRequestBuilder bulkRequest = client.prepareBulk();
            for (final SearchHit hit : hits) {
                bulkRequest.add(client.prepareDelete(hit.getIndex(), hit.getType(), hit.getId()));
            }
            return bulkRequest.execute();
        });
        if (bulkResponse.hasFailures()) {
            throw new EsAccessException(bulkResponse.buildFailureMessage());
        }
    }
    return count;
}