Examples with HttpClientDownloader us.codecraft.webmagic.downloader.HttpClientDownloader used on opensource projects

Example 1 with HttpClientDownloader

use of us.codecraft.webmagic.downloader.HttpClientDownloader in project webmagic by code4craft.

the class Spider method initComponent.

protected void initComponent() {
    if (downloader == null) {
        this.downloader = new HttpClientDownloader();
    }
    if (pipelines.isEmpty()) {
        pipelines.add(new ConsolePipeline());
    }
    downloader.setThread(threadNum);
    if (threadPool == null || threadPool.isShutdown()) {
        if (executorService != null && !executorService.isShutdown()) {
            threadPool = new CountableThreadPool(threadNum, executorService);
        } else {
            threadPool = new CountableThreadPool(threadNum);
        }
    }
    if (startRequests != null) {
        for (Request request : startRequests) {
            addRequest(request);
        }
        startRequests.clear();
    }
    startTime = new Date();
}