Examples with EcReptileKeyValueDTO com.easy.cloud.core.reptile.common.pojo.dto.EcReptileKeyValueDTO used on opensource projects

Example 1 with EcReptileKeyValueDTO

use of com.easy.cloud.core.reptile.common.pojo.dto.EcReptileKeyValueDTO in project dq-easy-cloud by dq-open-cloud.

the class EcReptileUtils method intoScheduler.

/**
 * <p>
 * 将要爬取数据的url放到Scheduler中
 * </p>
 * <p>
 * <pre>
 *     所需参数示例及其说明
 *     参数名称 : 示例值 : 说明 : 是否必须
 * </pre>
 *
 * @param geccoEngine    : GeccoEngine : 爬虫引擎
 * @param dynamicBeanDTO : EcReptileDynamicBeanDTO : 爬虫动态bean数据传输对象
 * @param dataDTO        : EcReptileDataDTO : 爬虫数据的数据传输对象由业务系统传入
 * @author daiqi
 * @创建时间 2018年6月8日 下午3:07:08
 */
public static void intoScheduler(GeccoEngine geccoEngine, EcReptileDynamicBeanDTO dynamicBeanDTO, EcReptileDataDTO dataDTO) {
    List<String> matchUrls = dynamicBeanDTO.getMatchUrlList();
    if (EcCollectionsUtils.isEmpty(matchUrls)) {
        throw new EcBaseBusinessException("设置异常", "待设置异常");
    }
    for (String matchUrl : matchUrls) {
        String intoUrl = matchUrl;
        if (EcCollectionsUtils.isEmpty(dataDTO.getUrlKeyValueDTOs())) {
            geccoEngine.getScheduler().into(new HttpGetRequest(intoUrl));
            continue;
        }
        for (EcReptileKeyValueDTO keyValueDTO : dataDTO.getUrlKeyValueDTOs()) {
            intoUrl = UrlMatcher.replaceParams(intoUrl, keyValueDTO.getKey(), keyValueDTO.getValue());
        }
        geccoEngine.getScheduler().into(new HttpGetRequest(intoUrl));
    }
}

Example 2 with EcReptileKeyValueDTO

use of com.easy.cloud.core.reptile.common.pojo.dto.EcReptileKeyValueDTO in project dq-easy-cloud by dq-open-cloud.

the class EcReptileController method addReptileUrl.

@RequestMapping(value = "addReptileUrl")
public EcBaseServiceResult addReptileUrl(@RequestParam(name = "serialNum") String serialNum) {
    // try {
    // EcPipelineTest.queue.put((new EcReptileKeyValueDTO [] {new EcReptileKeyValueDTO("code", serialNum)}));
    // } catch (InterruptedException e) {
    // e.printStackTrace();
    // }
    EcReptileKeyValueDTO reptileKeyValueDTO = new EcReptileKeyValueDTO("code", serialNum);
    String tempUrl = EcReptileConstant.MATCH_URL_DETAIL;
    tempUrl = tempUrl.replace(reptileKeyValueDTO.getKey(), reptileKeyValueDTO.getValue());
    String fullUrl = tempUrl.replace("{", "").replace("}", "");
    geccoEngine.getScheduler().into(new HttpGetRequest(fullUrl));
    return EcBaseServiceResult.newInstanceOfSuccess();
}