Examples with DataConfig org.codelibs.fess.es.config.exentity.DataConfig used on opensource projects

Example 1 with DataConfig

use of org.codelibs.fess.es.config.exentity.DataConfig in project fess by codelibs.

the class ApiAdminDataconfigAction method settings.

// ===================================================================================
//                                                                      Search Execute
//                                                                      ==============
// GET /api/admin/dataconfig/settings
// POST /api/admin/dataconfig/settings
@Execute
public JsonResponse<ApiResult> settings(final SearchBody body) {
    validateApi(body, messages -> {
    });
    final DataConfigPager pager = copyBeanToNewBean(body, DataConfigPager.class);
    final List<DataConfig> list = dataConfigService.getDataConfigList(pager);
    return asJson(new ApiResult.ApiConfigsResponse<EditBody>().settings(list.stream().map(entity -> createEditBody(entity)).collect(Collectors.toList())).total(pager.getAllRecordCount()).status(ApiResult.Status.OK).result());
}

Example 2 with DataConfig

use of org.codelibs.fess.es.config.exentity.DataConfig in project fess by codelibs.

the class ApiAdminDataconfigAction method post$setting.

// POST /api/admin/dataconfig/setting
@Execute
public JsonResponse<ApiResult> post$setting(final EditBody body) {
    validateApi(body, messages -> {
    });
    body.crudMode = CrudMode.EDIT;
    final DataConfig dataConfig = getDataConfig(body).map(entity -> {
        try {
            dataConfigService.store(entity);
        } catch (final Exception e) {
            throwValidationErrorApi(messages -> messages.addErrorsCrudFailedToUpdateCrudTable(GLOBAL, buildThrowableMessage(e)));
        }
        return entity;
    }).orElseGet(() -> {
        throwValidationErrorApi(messages -> messages.addErrorsCrudCouldNotFindCrudTable(GLOBAL, body.id));
        return null;
    });
    return asJson(new ApiUpdateResponse().id(dataConfig.getId()).created(false).status(Status.OK).result());
}

Example 3 with DataConfig

use of org.codelibs.fess.es.config.exentity.DataConfig in project fess by codelibs.

the class AbstractDataStoreImpl method store.

@Override
public void store(final DataConfig config, final IndexUpdateCallback callback, final Map<String, String> initParamMap) {
    final Map<String, String> configParamMap = config.getHandlerParameterMap();
    final Map<String, String> configScriptMap = config.getHandlerScriptMap();
    final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
    final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
    final Date documentExpires = crawlingInfoHelper.getDocumentExpires(config);
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    initParamMap.putAll(configParamMap);
    final Map<String, String> paramMap = initParamMap;
    // default values
    final Map<String, Object> defaultDataMap = new HashMap<>();
    // cid
    final String configId = config.getConfigId();
    if (configId != null) {
        defaultDataMap.put(fessConfig.getIndexFieldConfigId(), configId);
    }
    //  expires
    if (documentExpires != null) {
        defaultDataMap.put(fessConfig.getIndexFieldExpires(), documentExpires);
    }
    // segment
    defaultDataMap.put(fessConfig.getIndexFieldSegment(), initParamMap.get(Constants.SESSION_ID));
    // created
    defaultDataMap.put(fessConfig.getIndexFieldCreated(), systemHelper.getCurrentTime());
    // boost
    defaultDataMap.put(fessConfig.getIndexFieldBoost(), config.getBoost().toString());
    // label: labelType
    final List<String> labelTypeList = new ArrayList<>();
    for (final String labelType : config.getLabelTypeValues()) {
        labelTypeList.add(labelType);
    }
    defaultDataMap.put(fessConfig.getIndexFieldLabel(), labelTypeList);
    // role: roleType
    final List<String> roleTypeList = new ArrayList<>();
    stream(config.getPermissions()).of(stream -> stream.forEach(p -> roleTypeList.add(p)));
    defaultDataMap.put(fessConfig.getIndexFieldRole(), roleTypeList);
    // mimetype
    defaultDataMap.put(fessConfig.getIndexFieldMimetype(), mimeType);
    // title
    // content
    // cache
    // digest
    // host
    // site
    // url
    // anchor
    // content_length
    // last_modified
    // id
    storeData(config, callback, paramMap, configScriptMap, defaultDataMap);
}

Example 4 with DataConfig

use of org.codelibs.fess.es.config.exentity.DataConfig in project fess by codelibs.

the class EsDataStoreImpl method storeData.

@Override
protected void storeData(final DataConfig dataConfig, final IndexUpdateCallback callback, final Map<String, String> paramMap, final Map<String, String> scriptMap, final Map<String, Object> defaultDataMap) {
    final String hostsStr = paramMap.get(HOSTS);
    if (StringUtil.isBlank(hostsStr)) {
        logger.info("hosts is empty.");
        return;
    }
    final long readInterval = getReadInterval(paramMap);
    final Settings settings = Settings.builder().put(paramMap.entrySet().stream().filter(e -> e.getKey().startsWith(SETTINGS_PREFIX)).collect(Collectors.toMap(e -> e.getKey().replaceFirst("^settings\\.", StringUtil.EMPTY), e -> e.getValue()))).build();
    logger.info("Connecting to " + hostsStr + " with [" + settings.toDelimitedString(',') + "]");
    final InetSocketTransportAddress[] addresses = split(hostsStr, ",").get(stream -> stream.map(h -> {
        final String[] values = h.trim().split(":");
        try {
            if (values.length == 1) {
                return new InetSocketTransportAddress(InetAddress.getByName(values[0]), 9300);
            } else if (values.length == 2) {
                return new InetSocketTransportAddress(InetAddress.getByName(values[0]), Integer.parseInt(values[1]));
            }
        } catch (final Exception e) {
            logger.warn("Failed to parse address: " + h, e);
        }
        return null;
    }).filter(v -> v != null).toArray(n -> new InetSocketTransportAddress[n]));
    try (PreBuiltTransportClient client = new PreBuiltTransportClient(settings)) {
        client.addTransportAddresses(addresses);
        processData(dataConfig, callback, paramMap, scriptMap, defaultDataMap, readInterval, client);
    }
}

Example 5 with DataConfig

use of org.codelibs.fess.es.config.exentity.DataConfig in project fess by codelibs.

the class DataIndexHelper method doCrawl.

protected void doCrawl(final String sessionId, final List<DataConfig> configList) {
    final int multiprocessCrawlingCount = ComponentUtil.getFessConfig().getCrawlingThreadCount();
    final long startTime = System.currentTimeMillis();
    final IndexUpdateCallback indexUpdateCallback = ComponentUtil.getComponent(IndexUpdateCallback.class);
    final List<String> sessionIdList = new ArrayList<>();
    final Map<String, String> initParamMap = new HashMap<>();
    dataCrawlingThreadList.clear();
    final List<String> dataCrawlingThreadStatusList = new ArrayList<>();
    for (final DataConfig dataConfig : configList) {
        final String sid = crawlingConfigHelper.store(sessionId, dataConfig);
        sessionIdList.add(sid);
        initParamMap.put(Constants.SESSION_ID, sessionId);
        initParamMap.put(Constants.CRAWLING_INFO_ID, sid);
        final DataCrawlingThread dataCrawlingThread = new DataCrawlingThread(dataConfig, indexUpdateCallback, initParamMap);
        dataCrawlingThread.setPriority(crawlerPriority);
        dataCrawlingThread.setName(sid);
        dataCrawlingThread.setDaemon(true);
        dataCrawlingThreadList.add(dataCrawlingThread);
        dataCrawlingThreadStatusList.add(Constants.READY);
    }
    final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
    int startedCrawlerNum = 0;
    int activeCrawlerNum = 0;
    while (startedCrawlerNum < dataCrawlingThreadList.size()) {
        // Force to stop crawl
        if (systemHelper.isForceStop()) {
            for (final DataCrawlingThread crawlerThread : dataCrawlingThreadList) {
                crawlerThread.stopCrawling();
            }
            break;
        }
        if (activeCrawlerNum < multiprocessCrawlingCount) {
            // start crawling
            dataCrawlingThreadList.get(startedCrawlerNum).start();
            dataCrawlingThreadStatusList.set(startedCrawlerNum, Constants.RUNNING);
            startedCrawlerNum++;
            activeCrawlerNum++;
            try {
                Thread.sleep(crawlingExecutionInterval);
            } catch (final InterruptedException e) {
                if (logger.isDebugEnabled()) {
                    logger.debug("Interrupted.", e);
                }
            }
            continue;
        }
        // check status
        for (int i = 0; i < startedCrawlerNum; i++) {
            if (!dataCrawlingThreadList.get(i).isRunning() && dataCrawlingThreadStatusList.get(i).equals(Constants.RUNNING)) {
                dataCrawlingThreadList.get(i).awaitTermination();
                dataCrawlingThreadStatusList.set(i, Constants.DONE);
                activeCrawlerNum--;
            }
        }
        try {
            Thread.sleep(crawlingExecutionInterval);
        } catch (final InterruptedException e) {
            if (logger.isDebugEnabled()) {
                logger.debug("Interrupted.", e);
            }
        }
    }
    boolean finishedAll = false;
    while (!finishedAll) {
        finishedAll = true;
        for (int i = 0; i < dataCrawlingThreadList.size(); i++) {
            dataCrawlingThreadList.get(i).awaitTermination(crawlingExecutionInterval);
            if (!dataCrawlingThreadList.get(i).isRunning() && dataCrawlingThreadStatusList.get(i).equals(Constants.RUNNING)) {
                dataCrawlingThreadStatusList.set(i, Constants.DONE);
            }
            if (!dataCrawlingThreadStatusList.get(i).equals(Constants.DONE)) {
                finishedAll = false;
            }
        }
    }
    dataCrawlingThreadList.clear();
    dataCrawlingThreadStatusList.clear();
    // put cralwing info
    final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
    final long execTime = System.currentTimeMillis() - startTime;
    crawlingInfoHelper.putToInfoMap(Constants.DATA_CRAWLING_EXEC_TIME, Long.toString(execTime));
    if (logger.isInfoEnabled()) {
        logger.info("[EXEC TIME] crawling time: " + execTime + "ms");
    }
    crawlingInfoHelper.putToInfoMap(Constants.DATA_INDEX_EXEC_TIME, Long.toString(indexUpdateCallback.getExecuteTime()));
    crawlingInfoHelper.putToInfoMap(Constants.DATA_INDEX_SIZE, Long.toString(indexUpdateCallback.getDocumentSize()));
    for (final String sid : sessionIdList) {
        // remove config
        crawlingConfigHelper.remove(sid);
    }
}