Examples with Downloader us.codecraft.webmagic.downloader.Downloader used on opensource projects

Example 1 with Downloader

use of us.codecraft.webmagic.downloader.Downloader in project webmagic by code4craft.

the class SpiderTest method testRound.

private void testRound() {
    Spider spider = Spider.create(new PageProcessor() {

        private AtomicInteger count = new AtomicInteger();

        @Override
        public void process(Page page) {
            page.setSkip(true);
        }

        @Override
        public Site getSite() {
            return Site.me().setSleepTime(0);
        }
    }).setDownloader(new Downloader() {

        @Override
        public Page download(Request request, Task task) {
            return new Page().setRawText("");
        }

        @Override
        public void setThread(int threadNum) {
        }
    }).setScheduler(new Scheduler() {

        private AtomicInteger count = new AtomicInteger();

        private Random random = new Random();

        @Override
        public void push(Request request, Task task) {
        }

        @Override
        public synchronized Request poll(Task task) {
            if (count.incrementAndGet() > 1000) {
                return null;
            }
            if (random.nextInt(100) > 90) {
                return null;
            }
            return new Request("test");
        }
    }).thread(10);
    spider.run();
}