Examples with FessConfig org.codelibs.fess.mylasta.direction.FessConfig used on opensource projects

Example 16 with FessConfig

use of org.codelibs.fess.mylasta.direction.FessConfig in project fess by codelibs.

the class FessEsClient method createIndex.

public boolean createIndex(final String index, final String docType, final String indexName) {
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    waitForConfigSyncStatus();
    sendConfigFiles(index);
    final String indexConfigFile = indexConfigPath + "/" + index + ".json";
    try {
        String source = FileUtil.readUTF8(indexConfigFile);
        final String dictionaryPath = System.getProperty("fess.dictionary.path", StringUtil.EMPTY);
        source = source.replaceAll(Pattern.quote("${fess.dictionary.path}"), dictionaryPath);
        final CreateIndexResponse indexResponse = client.admin().indices().prepareCreate(indexName).setSource(source, XContentFactory.xContentType(source)).execute().actionGet(fessConfig.getIndexIndicesTimeout());
        if (indexResponse.isAcknowledged()) {
            logger.info("Created " + indexName + " index.");
            return true;
        } else if (logger.isDebugEnabled()) {
            logger.debug("Failed to create " + indexName + " index.");
        }
    } catch (final Exception e) {
        logger.warn(indexConfigFile + " is not found.", e);
    }
    return false;
}

Example 17 with FessConfig

use of org.codelibs.fess.mylasta.direction.FessConfig in project fess by codelibs.

the class FessEsClient method deleteByQuery.

public int deleteByQuery(final String index, final String type, final QueryBuilder queryBuilder) {
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    SearchResponse response = client.prepareSearch(index).setTypes(type).setScroll(scrollForDelete).setSize(sizeForDelete).setFetchSource(new String[] { fessConfig.getIndexFieldId() }, null).setQuery(queryBuilder).setPreference(Constants.SEARCH_PREFERENCE_PRIMARY).execute().actionGet(fessConfig.getIndexScrollSearchTimeoutTimeout());
    int count = 0;
    String scrollId = response.getScrollId();
    while (scrollId != null) {
        final SearchHits searchHits = response.getHits();
        final SearchHit[] hits = searchHits.getHits();
        if (hits.length == 0) {
            scrollId = null;
            break;
        }
        final BulkRequestBuilder bulkRequest = client.prepareBulk();
        for (final SearchHit hit : hits) {
            bulkRequest.add(client.prepareDelete(index, type, hit.getId()));
        }
        count += hits.length;
        final BulkResponse bulkResponse = bulkRequest.execute().actionGet(fessConfig.getIndexBulkTimeout());
        if (bulkResponse.hasFailures()) {
            throw new IllegalBehaviorStateException(bulkResponse.buildFailureMessage());
        }
        response = client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute().actionGet(fessConfig.getIndexBulkTimeout());
        scrollId = response.getScrollId();
    }
    return count;
}

Example 18 with FessConfig

use of org.codelibs.fess.mylasta.direction.FessConfig in project fess by codelibs.

the class AbstractDataStoreImpl method store.

@Override
public void store(final DataConfig config, final IndexUpdateCallback callback, final Map<String, String> initParamMap) {
    final Map<String, String> configParamMap = config.getHandlerParameterMap();
    final Map<String, String> configScriptMap = config.getHandlerScriptMap();
    final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
    final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
    final Date documentExpires = crawlingInfoHelper.getDocumentExpires(config);
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    initParamMap.putAll(configParamMap);
    final Map<String, String> paramMap = initParamMap;
    // default values
    final Map<String, Object> defaultDataMap = new HashMap<>();
    // cid
    final String configId = config.getConfigId();
    if (configId != null) {
        defaultDataMap.put(fessConfig.getIndexFieldConfigId(), configId);
    }
    //  expires
    if (documentExpires != null) {
        defaultDataMap.put(fessConfig.getIndexFieldExpires(), documentExpires);
    }
    // segment
    defaultDataMap.put(fessConfig.getIndexFieldSegment(), initParamMap.get(Constants.SESSION_ID));
    // created
    defaultDataMap.put(fessConfig.getIndexFieldCreated(), systemHelper.getCurrentTime());
    // boost
    defaultDataMap.put(fessConfig.getIndexFieldBoost(), config.getBoost().toString());
    // label: labelType
    final List<String> labelTypeList = new ArrayList<>();
    for (final String labelType : config.getLabelTypeValues()) {
        labelTypeList.add(labelType);
    }
    defaultDataMap.put(fessConfig.getIndexFieldLabel(), labelTypeList);
    // role: roleType
    final List<String> roleTypeList = new ArrayList<>();
    stream(config.getPermissions()).of(stream -> stream.forEach(p -> roleTypeList.add(p)));
    defaultDataMap.put(fessConfig.getIndexFieldRole(), roleTypeList);
    // mimetype
    defaultDataMap.put(fessConfig.getIndexFieldMimetype(), mimeType);
    // title
    // content
    // cache
    // digest
    // host
    // site
    // url
    // anchor
    // content_length
    // last_modified
    // id
    storeData(config, callback, paramMap, configScriptMap, defaultDataMap);
}

Example 19 with FessConfig

use of org.codelibs.fess.mylasta.direction.FessConfig in project fess by codelibs.

the class WebConfig method getLabelTypeList.

public List<LabelType> getLabelTypeList() {
    if (labelTypeList == null) {
        synchronized (this) {
            if (labelTypeList == null) {
                final FessConfig fessConfig = ComponentUtil.getFessConfig();
                final WebConfigToLabelBhv webConfigToLabelBhv = ComponentUtil.getComponent(WebConfigToLabelBhv.class);
                final ListResultBean<WebConfigToLabel> mappingList = webConfigToLabelBhv.selectList(cb -> {
                    cb.query().setWebConfigId_Equal(getId());
                    cb.specify().columnLabelTypeId();
                    cb.paging(fessConfig.getPageLabeltypeMaxFetchSizeAsInteger().intValue(), 1);
                });
                final List<String> labelIdList = new ArrayList<>();
                for (final WebConfigToLabel mapping : mappingList) {
                    labelIdList.add(mapping.getLabelTypeId());
                }
                final LabelTypeBhv labelTypeBhv = ComponentUtil.getComponent(LabelTypeBhv.class);
                labelTypeList = labelIdList.isEmpty() ? Collections.emptyList() : labelTypeBhv.selectList(cb -> {
                    cb.query().setId_InScope(labelIdList);
                    cb.query().addOrderBy_SortOrder_Asc();
                    cb.fetchFirst(fessConfig.getPageLabeltypeMaxFetchSizeAsInteger());
                });
            }
        }
    }
    return labelTypeList;
}

Example 20 with FessConfig

use of org.codelibs.fess.mylasta.direction.FessConfig in project fess by codelibs.

the class WebConfig method initializeClientFactory.

@Override
public Map<String, Object> initializeClientFactory(final CrawlerClientFactory clientFactory) {
    final WebAuthenticationService webAuthenticationService = ComponentUtil.getComponent(WebAuthenticationService.class);
    final RequestHeaderService requestHeaderService = ComponentUtil.getComponent(RequestHeaderService.class);
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    // HttpClient Parameters
    final Map<String, Object> paramMap = new HashMap<>();
    clientFactory.setInitParameterMap(paramMap);
    final Map<String, String> clientConfigMap = getConfigParameterMap(ConfigName.CLIENT);
    if (clientConfigMap != null) {
        paramMap.putAll(clientConfigMap);
    }
    // robots txt enabled
    if (paramMap.get(HcHttpClient.ROBOTS_TXT_ENABLED_PROPERTY) == null) {
        paramMap.put(HcHttpClient.ROBOTS_TXT_ENABLED_PROPERTY, !fessConfig.isCrawlerIgnoreRobotsTxt());
    }
    final String userAgent = getUserAgent();
    if (StringUtil.isNotBlank(userAgent)) {
        paramMap.put(HcHttpClient.USER_AGENT_PROPERTY, userAgent);
    }
    final List<WebAuthentication> webAuthList = webAuthenticationService.getWebAuthenticationList(getId());
    final List<Authentication> basicAuthList = new ArrayList<>();
    for (final WebAuthentication webAuth : webAuthList) {
        basicAuthList.add(webAuth.getAuthentication());
    }
    paramMap.put(HcHttpClient.BASIC_AUTHENTICATIONS_PROPERTY, basicAuthList.toArray(new Authentication[basicAuthList.size()]));
    // request header
    final List<RequestHeader> requestHeaderList = requestHeaderService.getRequestHeaderList(getId());
    final List<org.codelibs.fess.crawler.client.http.RequestHeader> rhList = new ArrayList<>();
    for (final RequestHeader requestHeader : requestHeaderList) {
        rhList.add(requestHeader.getCrawlerRequestHeader());
    }
    paramMap.put(HcHttpClient.REQUERT_HEADERS_PROPERTY, rhList.toArray(new org.codelibs.fess.crawler.client.http.RequestHeader[rhList.size()]));
    // proxy credentials
    if (paramMap.get("proxyUsername") != null && paramMap.get("proxyPassword") != null) {
        paramMap.put(HcHttpClient.PROXY_CREDENTIALS_PROPERTY, new UsernamePasswordCredentials(paramMap.remove("proxyUsername").toString(), paramMap.remove("proxyPassword").toString()));
    }
    return paramMap;
}