Examples with CollectorPipeline us.codecraft.webmagic.pipeline.CollectorPipeline used on opensource projects

Example 1 with CollectorPipeline

use of us.codecraft.webmagic.pipeline.CollectorPipeline in project webmagic by code4craft.

the class Spider method getAll.

/**
     * Download urls synchronizing.
     *
     * @param urls urls
     * @return list downloaded
     */
public <T> List<T> getAll(Collection<String> urls) {
    destroyWhenExit = false;
    spawnUrl = false;
    startRequests.clear();
    for (Request request : UrlUtils.convertToRequests(urls)) {
        addRequest(request);
    }
    CollectorPipeline collectorPipeline = getCollectorPipeline();
    pipelines.add(collectorPipeline);
    run();
    spawnUrl = true;
    destroyWhenExit = true;
    return collectorPipeline.getCollected();
}